mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 12:09:12 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3f2d6f38df | |||
| 0de98f8139 | |||
| 28f677d723 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
|
||||
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
|
||||
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
{
|
||||
"lane": "ci/test-and-lint",
|
||||
"tests": [
|
||||
{
|
||||
"invariant": "write-quorum",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-rollback",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
|
||||
},
|
||||
{
|
||||
"invariant": "stale-writer",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
|
||||
},
|
||||
{
|
||||
"invariant": "range-body",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
|
||||
},
|
||||
{
|
||||
"invariant": "multipart-cancellation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
|
||||
},
|
||||
{
|
||||
"invariant": "list-uncommitted-version",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
|
||||
},
|
||||
{
|
||||
"invariant": "minio-object-fixture",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::parses_real_minio_object_xlmeta"
|
||||
},
|
||||
{
|
||||
"invariant": "corrupt-part-arrays",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
|
||||
"sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
|
||||
"sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
|
||||
"sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
|
||||
"sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
|
||||
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -269,7 +269,6 @@ jobs:
|
||||
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
set +e
|
||||
@@ -278,12 +277,6 @@ jobs:
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
if [[ "${status}" -eq 0 ]]; then
|
||||
cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json \
|
||||
&& python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
|
||||
&& test -s target/nextest/ci/junit.xml || status=$?
|
||||
fi
|
||||
{
|
||||
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
|
||||
echo "exit_status=${status}"
|
||||
|
||||
@@ -18,7 +18,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
from_version:
|
||||
description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
|
||||
description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)'
|
||||
required: false
|
||||
default: '1.0.0-rc.3'
|
||||
from_url:
|
||||
@@ -26,7 +26,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
to_version:
|
||||
description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
|
||||
description: 'NEW RustFS release tag (leave empty for latest nightly)'
|
||||
required: false
|
||||
to_url:
|
||||
description: 'NEW .deb URL. Overrides to_version / nightly default.'
|
||||
@@ -145,7 +145,6 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-upgrade-test.sh
|
||||
@@ -176,29 +175,6 @@ jobs:
|
||||
else
|
||||
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
# Fail fast with a clear message when a requested release tag has
|
||||
# no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
|
||||
# letting the suite die mid-run on a 404.
|
||||
check_release_asset() {
|
||||
local version="$1" tag asset url
|
||||
[ -n "${version}" ] && [ "${version}" != "null" ] || return 0
|
||||
tag="${version#v}"
|
||||
asset="rustfs_${tag//-/.}_amd64.deb"
|
||||
url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
|
||||
if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
|
||||
echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
|
||||
echo " ${url}" >&2
|
||||
echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "resolved ${tag} -> ${url}"
|
||||
}
|
||||
if [ -z "${FROM_URL}" ]; then
|
||||
check_release_asset "${FROM_VERSION}"
|
||||
fi
|
||||
if [ -z "${TO_URL}" ]; then
|
||||
check_release_asset "${TO_VERSION}"
|
||||
fi
|
||||
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
|
||||
@@ -33,7 +33,6 @@ profile.json
|
||||
*.zst
|
||||
.secrets
|
||||
*.go
|
||||
!crates/zip/tests/fixtures/snowball/**/generate/*.go
|
||||
*.pb
|
||||
*.svg
|
||||
deploy/logs/*.log.*
|
||||
|
||||
Generated
+112
-324
File diff suppressed because it is too large
Load Diff
+6
-10
@@ -168,7 +168,7 @@ reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.5" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.4" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.5"
|
||||
tokio-util = { version = "0.7.19" }
|
||||
@@ -234,19 +234,15 @@ tokio-postgres-rustls = "0.14.0"
|
||||
# Utilities and Tools
|
||||
anyhow = "1.0.104"
|
||||
arc-swap = "1.9.2"
|
||||
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
|
||||
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
|
||||
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
|
||||
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
|
||||
tar-codec = "0.0.14"
|
||||
tar-framing = "0.0.14"
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-config = { version = "1.11.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.145.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.117.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.144.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.113.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.16.0" }
|
||||
aws-smithy-types = { version = "1.6.3" }
|
||||
|
||||
@@ -6743,99 +6743,6 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
// Keep compilation outside the scenario timeout. Recovery itself waits
|
||||
// for the production 30-second lightweight retry tick.
|
||||
let _rustfs_binary = rustfs_binary_path();
|
||||
|
||||
match timeout(Duration::from_secs(150), async {
|
||||
let mut site_env = replication_fast_env();
|
||||
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
|
||||
let mut site_a_env = RustFSTestEnvironment::new().await?;
|
||||
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
|
||||
|
||||
let mut site_b_env = RustFSTestEnvironment::new().await?;
|
||||
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
|
||||
|
||||
let site_a_client = site_a_env.create_s3_client();
|
||||
let site_b_client = site_b_env.create_s3_client();
|
||||
let bucket = "site-repl-peer-outage";
|
||||
let key = "after-recovery.txt";
|
||||
let payload = b"site replication recovered the missed bucket".to_vec();
|
||||
|
||||
let add_status = site_replication_add(
|
||||
&site_a_env,
|
||||
&[
|
||||
PeerSite {
|
||||
name: "outage-site-a".to_string(),
|
||||
endpoint: site_a_env.url.clone(),
|
||||
access_key: site_a_env.access_key.clone(),
|
||||
secret_key: site_a_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "outage-site-b".to_string(),
|
||||
endpoint: site_b_env.url.clone(),
|
||||
access_key: site_b_env.access_key.clone(),
|
||||
secret_key: site_b_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert!(add_status.success, "unexpected site add result: {add_status:?}");
|
||||
wait_for_site_replication_enabled(&site_a_env, 2).await?;
|
||||
wait_for_site_replication_enabled(&site_b_env, 2).await?;
|
||||
|
||||
site_b_env.stop_server();
|
||||
site_a_client.create_bucket().bucket(bucket).send().await?;
|
||||
site_a_client.head_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let queued = site_replication_info(&site_a_env)
|
||||
.await?
|
||||
.retry_stats
|
||||
.ok_or("peer outage did not persist a site replication retry event")?;
|
||||
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
|
||||
|
||||
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
|
||||
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
|
||||
loop {
|
||||
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
|
||||
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
|
||||
if bucket_recovered && queue_empty {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= recovery_deadline {
|
||||
return Err(format!(
|
||||
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
site_a_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-applying a site's own replication config must not disable the peer's reverse direction.
|
||||
///
|
||||
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication
|
||||
|
||||
@@ -244,7 +244,6 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-smithy-async.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -479,11 +479,9 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
|
||||
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ use rustfs_utils::http::{
|
||||
insert_header,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::str::FromStr as _;
|
||||
@@ -376,11 +376,6 @@ pub struct BucketTargetSys {
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
|
||||
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
|
||||
/// alongside `targets_map`, and read before it so an unreadable
|
||||
/// configuration surfaces as a typed error instead of an empty target set.
|
||||
unreadable_targets: Arc<RwLock<HashSet<String>>>,
|
||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
pub hc_client: Arc<HttpClient>,
|
||||
@@ -424,7 +419,6 @@ impl BucketTargetSys {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
hc_client: Arc::new(build_health_check_client()),
|
||||
@@ -634,40 +628,30 @@ impl BucketTargetSys {
|
||||
health_map.clone()
|
||||
}
|
||||
|
||||
/// Targets of one bucket, or of every bucket when `bucket` is empty.
|
||||
///
|
||||
/// A bucket that simply has no targets yields an empty list; a bucket
|
||||
/// whose persisted configuration cannot be decoded is an error, so an
|
||||
/// admin listing reports the fault instead of an empty list that reads as
|
||||
/// "replication is not configured" (rustfs/backlog#2282).
|
||||
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Result<Vec<BucketTarget>, BucketTargetError> {
|
||||
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec<BucketTarget> {
|
||||
let health_stats = self.target_health_stats().await;
|
||||
let mut targets = Vec::new();
|
||||
|
||||
if !bucket.is_empty() {
|
||||
match self.list_bucket_targets(bucket).await {
|
||||
Ok(bucket_targets) => {
|
||||
for mut target in bucket_targets.targets {
|
||||
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
|
||||
if let Some(health) = health_stats.get(&target.arn) {
|
||||
target.total_downtime = health.offline_duration;
|
||||
target.online = health.online;
|
||||
target.last_online = health.last_online;
|
||||
target.latency = target::LatencyStat {
|
||||
curr: health.latency.curr,
|
||||
avg: health.latency.avg,
|
||||
max: health.latency.peak,
|
||||
};
|
||||
target.offline_count = health.offline_count;
|
||||
}
|
||||
targets.push(target);
|
||||
if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await {
|
||||
for mut target in bucket_targets.targets {
|
||||
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
|
||||
if let Some(health) = health_stats.get(&target.arn) {
|
||||
target.total_downtime = health.offline_duration;
|
||||
target.online = health.online;
|
||||
target.last_online = health.last_online;
|
||||
target.latency = target::LatencyStat {
|
||||
curr: health.latency.curr,
|
||||
avg: health.latency.avg,
|
||||
max: health.latency.peak,
|
||||
};
|
||||
target.offline_count = health.offline_count;
|
||||
}
|
||||
targets.push(target);
|
||||
}
|
||||
}
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
return Ok(targets);
|
||||
return targets;
|
||||
}
|
||||
|
||||
let targets_map = self.targets_map.read().await;
|
||||
@@ -690,16 +674,10 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
Ok(targets)
|
||||
targets
|
||||
}
|
||||
|
||||
pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
|
||||
if self.unreadable_targets.read().await.contains(bucket) {
|
||||
return Err(BucketTargetError::BucketRemoteTargetsUnreadable {
|
||||
bucket: bucket.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let targets_map = self.targets_map.read().await;
|
||||
if let Some(targets) = targets_map.get(bucket) {
|
||||
Ok(BucketTargets {
|
||||
@@ -712,30 +690,13 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
/// Record that this bucket's persisted targets configuration exists but
|
||||
/// cannot be decoded (rustfs/backlog#2282).
|
||||
///
|
||||
/// Any snapshot published from an earlier readable load is deliberately
|
||||
/// left in place: withdrawing it would produce exactly the silent "no
|
||||
/// targets configured" state this marker exists to prevent. The marker is
|
||||
/// cleared by the next successful publish, which is what makes a repaired
|
||||
/// configuration take effect without a restart.
|
||||
pub async fn mark_targets_unreadable(&self, bucket: &str) {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
self.unreadable_targets.write().await.insert(bucket.to_string());
|
||||
}
|
||||
|
||||
pub async fn delete(&self, bucket: &str) {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
// Lock order: unreadable_targets, then targets_map, then
|
||||
// arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
|
||||
// (always last; also taken standalone by the capability accessors).
|
||||
self.unreadable_targets.write().await.remove(bucket);
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
@@ -1132,11 +1093,6 @@ impl BucketTargetSys {
|
||||
/// Keeping persisted-config reads under the same mutex prevents a stale
|
||||
/// reload from overwriting a concurrent credential rotation.
|
||||
async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
|
||||
// Reaching here means the persisted configuration decoded, so the
|
||||
// unreadable marker (if any) is stale. Cleared before the maps below
|
||||
// so `unreadable_targets` stays the outermost of this module's locks.
|
||||
self.unreadable_targets.write().await.remove(bucket);
|
||||
|
||||
let mut clients = Vec::new();
|
||||
if let Some(new_targets) = targets {
|
||||
for target in &new_targets.targets {
|
||||
@@ -1144,9 +1100,9 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order: unreadable_targets (above), then targets_map, then
|
||||
// arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
|
||||
// (always last; also taken standalone by the capability accessors).
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
@@ -1205,11 +1161,6 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
|
||||
if meta.bucket_targets_unreadable() {
|
||||
self.mark_targets_unreadable(bucket).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(config) = &meta.bucket_target_config else {
|
||||
return;
|
||||
};
|
||||
@@ -2325,13 +2276,6 @@ pub enum BucketTargetError {
|
||||
BucketRemoteTargetNotFound {
|
||||
bucket: String,
|
||||
},
|
||||
/// The bucket's persisted targets configuration exists but cannot be
|
||||
/// decoded. Distinct from `BucketRemoteTargetNotFound`, which means the
|
||||
/// bucket genuinely has no targets: callers must not degrade this one to
|
||||
/// an empty target set (rustfs/backlog#2282).
|
||||
BucketRemoteTargetsUnreadable {
|
||||
bucket: String,
|
||||
},
|
||||
BucketRemoteArnTypeInvalid {
|
||||
bucket: String,
|
||||
},
|
||||
@@ -2365,9 +2309,6 @@ impl fmt::Display for BucketTargetError {
|
||||
BucketTargetError::BucketRemoteTargetNotFound { bucket } => {
|
||||
write!(f, "Remote target not found for bucket: {bucket}")
|
||||
}
|
||||
BucketTargetError::BucketRemoteTargetsUnreadable { bucket } => {
|
||||
write!(f, "Persisted replication target configuration is unreadable for bucket: {bucket}")
|
||||
}
|
||||
BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => {
|
||||
write!(f, "Invalid ARN type for bucket: {bucket}")
|
||||
}
|
||||
@@ -3315,7 +3256,7 @@ mod tests {
|
||||
}],
|
||||
);
|
||||
|
||||
let targets = sys.list_targets("", "").await.expect("listing every bucket's targets");
|
||||
let targets = sys.list_targets("", "").await;
|
||||
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert!(!targets[0].online);
|
||||
|
||||
@@ -584,173 +584,33 @@ impl ExpiryOp for FreeVersionTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TransitionDeleteVersionPlan {
|
||||
Direct { version_id_exact: bool },
|
||||
ProbeLegacyUnknown,
|
||||
}
|
||||
|
||||
fn legacy_transition_version_state_missing(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_str, get_consistent_str,
|
||||
};
|
||||
|
||||
if !contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
let version_key_present = contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
if version_key_present {
|
||||
if oi.transitioned_object.version_id.is_empty() {
|
||||
let has_non_empty_version = oi.user_defined.iter().any(|(key, value)| {
|
||||
rustfs_utils::http::metadata_compat::strip_internal_prefix_preserving_case(key)
|
||||
.is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
|
||||
&& !value.is_empty()
|
||||
});
|
||||
if !has_non_empty_version {
|
||||
// MinIO writes the transitioned-versionID key with an empty value
|
||||
// for unversioned tier objects. The backend probe remains the proof.
|
||||
return Ok(true);
|
||||
}
|
||||
} else if get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID)
|
||||
== Some(oi.transitioned_object.version_id.as_str())
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"legacy remote tier version metadata is conflicting or malformed",
|
||||
));
|
||||
}
|
||||
if !oi.transitioned_object.version_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"legacy remote tier version metadata is missing or inconsistent",
|
||||
));
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
let persisted = get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object has conflicting transition version state metadata",
|
||||
)
|
||||
})?;
|
||||
if persisted != oi.transition_version_state.as_str() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object transition version state metadata changed during decoding",
|
||||
));
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn transition_remote_version_delete_plan(oi: &ObjectInfo) -> Result<TransitionDeleteVersionPlan, std::io::Error> {
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => {
|
||||
if legacy_transition_version_state_missing(oi)? {
|
||||
Ok(TransitionDeleteVersionPlan::ProbeLegacyUnknown)
|
||||
} else {
|
||||
validate_transition_remote_version(oi)
|
||||
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact })
|
||||
}
|
||||
}
|
||||
_ => validate_transition_remote_version(oi)
|
||||
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct ResolvedTransitionDeleteVersion {
|
||||
version_id_exact: bool,
|
||||
remote_already_missing: bool,
|
||||
}
|
||||
|
||||
async fn acquire_free_version_tier_lease(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(TierOperationLease, TransitionDeleteVersionPlan), std::io::Error> {
|
||||
let delete_plan = transition_remote_version_delete_plan(oi)?;
|
||||
) -> Result<(TierOperationLease, bool), std::io::Error> {
|
||||
let version_id_exact = validate_transition_remote_version(oi)?;
|
||||
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
|
||||
let lease =
|
||||
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
Ok((lease, delete_plan))
|
||||
}
|
||||
|
||||
async fn resolve_transition_delete_version_plan(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
delete_plan: TransitionDeleteVersionPlan,
|
||||
) -> Result<ResolvedTransitionDeleteVersion, std::io::Error> {
|
||||
match delete_plan {
|
||||
TransitionDeleteVersionPlan::Direct { version_id_exact } => Ok(ResolvedTransitionDeleteVersion {
|
||||
version_id_exact,
|
||||
remote_already_missing: false,
|
||||
}),
|
||||
TransitionDeleteVersionPlan::ProbeLegacyUnknown => {
|
||||
let expected_version = oi.transitioned_object.version_id.as_str();
|
||||
if expected_version.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"remote tier cannot safely delete a legacy object without an exact version ID",
|
||||
));
|
||||
}
|
||||
let probe = lease
|
||||
.probe_transition_version(&oi.transitioned_object.name, expected_version)
|
||||
.await?;
|
||||
match (expected_version, probe) {
|
||||
(expected, crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(actual))
|
||||
if expected == actual =>
|
||||
{
|
||||
lease.validate_remote_version_id(expected)?;
|
||||
Ok(ResolvedTransitionDeleteVersion {
|
||||
version_id_exact: true,
|
||||
remote_already_missing: false,
|
||||
})
|
||||
}
|
||||
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Missing) => {
|
||||
Ok(ResolvedTransitionDeleteVersion {
|
||||
version_id_exact: false,
|
||||
remote_already_missing: true,
|
||||
})
|
||||
}
|
||||
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"remote tier cannot prove legacy transition delete state",
|
||||
)),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"remote tier object version state is unknown",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_resolved_transition_delete(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
resolved: ResolvedTransitionDeleteVersion,
|
||||
) -> Result<(), std::io::Error> {
|
||||
if !resolved.remote_already_missing {
|
||||
delete_object_from_remote_tier_with_lease_idempotent(
|
||||
&oi.transitioned_object.name,
|
||||
&oi.transitioned_object.version_id,
|
||||
lease,
|
||||
resolved.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
Ok((lease, version_id_exact))
|
||||
}
|
||||
|
||||
async fn delete_free_version_remote_object_with_lease(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
delete_plan: TransitionDeleteVersionPlan,
|
||||
version_id_exact: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let resolved = resolve_transition_delete_version_plan(oi, lease, delete_plan).await?;
|
||||
execute_resolved_transition_delete(oi, lease, resolved).await
|
||||
delete_object_from_remote_tier_with_lease_idempotent(
|
||||
&oi.transitioned_object.name,
|
||||
&oi.transitioned_object.version_id,
|
||||
lease,
|
||||
version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn free_version_physical_topology_generation(api: &ECStore) -> String {
|
||||
@@ -781,16 +641,6 @@ fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectIn
|
||||
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
let candidate_legacy_missing = legacy_transition_version_state_missing(candidate)?;
|
||||
let expected_legacy_missing = legacy_transition_version_state_missing(expected)?;
|
||||
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
&& expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
&& candidate_legacy_missing
|
||||
&& expected_legacy_missing
|
||||
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version remote version state is unknown",
|
||||
@@ -866,7 +716,7 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
let object_guards = api
|
||||
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
|
||||
@@ -884,30 +734,16 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
"tier free-version cleanup fence is invalid before remote delete",
|
||||
));
|
||||
}
|
||||
let resolved = tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
|
||||
}
|
||||
result = tokio::time::timeout_at(deadline, resolve_transition_delete_version_plan(oi, &lease, delete_plan)) => {
|
||||
result.map_err(|_| {
|
||||
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote probe timed out")
|
||||
})??
|
||||
}
|
||||
};
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version cleanup fence changed after remote probe",
|
||||
));
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
|
||||
}
|
||||
result = tokio::time::timeout_at(deadline, execute_resolved_transition_delete(oi, &lease, resolved)) => {
|
||||
result.map_err(|_| {
|
||||
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out")
|
||||
})??;
|
||||
result = tokio::time::timeout_at(
|
||||
deadline,
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
|
||||
) => {
|
||||
result
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
|
||||
}
|
||||
}
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
@@ -955,8 +791,8 @@ async fn delete_free_version_remote_object(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
@@ -972,8 +808,8 @@ where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = T>,
|
||||
{
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await?;
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
|
||||
let result = delete_local().await;
|
||||
drop(lease);
|
||||
Ok(result)
|
||||
@@ -4852,39 +4688,6 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TransitionReadVersionPlan {
|
||||
Direct,
|
||||
ProbeLegacyUnversioned,
|
||||
}
|
||||
|
||||
const LEGACY_TRANSITION_READ_PROBE_TIMEOUT: StdDuration = StdDuration::from_secs(30);
|
||||
|
||||
fn transition_remote_version_read_plan(oi: &ObjectInfo) -> Result<TransitionReadVersionPlan, std::io::Error> {
|
||||
let version = oi.transitioned_object.version_id.as_str();
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => {
|
||||
if !legacy_transition_version_state_missing(oi)? {
|
||||
return validate_transition_remote_version(oi).map(|_| TransitionReadVersionPlan::Direct);
|
||||
}
|
||||
if version.is_empty() {
|
||||
Ok(TransitionReadVersionPlan::ProbeLegacyUnversioned)
|
||||
} else {
|
||||
Ok(TransitionReadVersionPlan::Direct)
|
||||
}
|
||||
}
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(TransitionReadVersionPlan::Direct),
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(TransitionReadVersionPlan::Direct),
|
||||
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => {
|
||||
Ok(TransitionReadVersionPlan::Direct)
|
||||
}
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state conflicts with its version ID",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// The resolver joins the tier manager as the second injected port this read
|
||||
// needs; grouping the request half into a struct would churn every call site of
|
||||
// a bug fix.
|
||||
@@ -4899,12 +4702,7 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
resolver: Option<&dyn ObjectEncryptionResolver>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
let read_plan = transition_remote_version_read_plan(oi)?;
|
||||
// Reject invalid ranges and encryption requests before a compatibility
|
||||
// probe can amplify them into remote listing work.
|
||||
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
|
||||
.await
|
||||
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
|
||||
validate_transition_remote_version(oi)?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
|
||||
let lease = match expected_identity {
|
||||
Some(identity) => {
|
||||
@@ -4918,36 +4716,7 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
|
||||
match read_plan {
|
||||
TransitionReadVersionPlan::Direct => {
|
||||
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
|
||||
}
|
||||
TransitionReadVersionPlan::ProbeLegacyUnversioned => {
|
||||
// RUSTFS_COMPAT_TODO(backlog#2203): remove operation-time probing
|
||||
// after an admin reconcile can persist every proven legacy state.
|
||||
let probe = tokio::time::timeout(
|
||||
LEGACY_TRANSITION_READ_PROBE_TIMEOUT,
|
||||
tgt_client.probe_transition_candidate(&oi.transitioned_object.name),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "legacy remote tier version probe timed out"))??;
|
||||
match probe {
|
||||
crate::services::tier::warm_backend::TransitionCandidateProbe::UnversionedPresent => {}
|
||||
crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"remote tier cannot prove legacy unversioned transition state",
|
||||
));
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state is unknown",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
|
||||
|
||||
// The same read plan the local path uses, so the tier fetch is positioned in
|
||||
// the object's *stored* coordinate system and the stream is handed the same
|
||||
@@ -4955,6 +4724,9 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
// through a plaintext-coordinate range and skipping the transform is how a
|
||||
// transitioned SSE object used to come back as silently corrupt bytes of the
|
||||
// right length (rustfs/rustfs#6025).
|
||||
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
|
||||
.await
|
||||
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
|
||||
let (off, length) = (plan.storage_offset() as i64, plan.storage_length());
|
||||
let mut gopts = WarmBackendGetOpts::default();
|
||||
|
||||
@@ -5827,13 +5599,11 @@ mod tests {
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::MockWarmOp;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
|
||||
use crate::services::tier::warm_backend::WarmBackend as _;
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -6529,75 +6299,7 @@ mod tests {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_allows_legacy_unknown_exact_version_for_non_destructive_read() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy transitioned object body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body.clone()),
|
||||
i64::try_from(body.len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: i64::try_from(body.len()).expect("body length should fit"),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object,
|
||||
version_id: remote_version,
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let range = Some(crate::storage_api_contracts::range::HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 7,
|
||||
end: 18,
|
||||
});
|
||||
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&range,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("legacy unknown state should still allow a non-destructive read");
|
||||
let mut got = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut got)
|
||||
.await
|
||||
.expect("transitioned reader should drain");
|
||||
|
||||
assert_eq!(got, &body.as_ref()[7..=18]);
|
||||
assert_eq!(backend.get_count().await, 1);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_explicit_unknown_version_state_before_backend_io() {
|
||||
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
@@ -6613,7 +6315,6 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined_with_transition_version_state(rustfs_filemeta::TransitionVersionState::Unknown).into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -6629,202 +6330,19 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("explicit unknown remote version state must fail before backend IO"),
|
||||
Ok(_) => panic!("unknown remote version state must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_present_but_invalid_legacy_version_metadata() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
|
||||
for persisted_version in [
|
||||
Uuid::nil().to_string(),
|
||||
"\u{fffd}".to_string(),
|
||||
"bad\u{0001}version".to_string(),
|
||||
] {
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_legacy_transition_version_id(&mut user_defined, &persisted_version);
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("present but invalid legacy version metadata must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_probes_legacy_empty_unknown_state_before_unversioned_read() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy unversioned transitioned object body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body.clone()),
|
||||
i64::try_from(body.len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
assert!(remote_version.is_empty());
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: i64::try_from(body.len()).expect("body length should fit"),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: HashMap::from([("x-minio-internal-transitioned-versionID".to_string(), String::new())]).into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("probe-proven legacy unversioned state should allow a non-destructive read");
|
||||
let mut got = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut got)
|
||||
.await
|
||||
.expect("transitioned reader should drain");
|
||||
|
||||
assert_eq!(got, body.as_ref());
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(
|
||||
backend.op_log().await,
|
||||
vec![
|
||||
MockWarmOp::Put {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Probe {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Get { object: remote_object },
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_ambiguous_empty_unknown_state_without_backend_get() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
|
||||
"versioned-candidate".to_string(),
|
||||
)))
|
||||
.await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("versioned legacy unknown state without stored version must fail before backend GET"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.op_log().await, vec![MockWarmOp::Probe { object: remote_object }]);
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_rejects_explicit_unknown_before_backend_io() {
|
||||
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let identity = test_tier_destination_identity(&manager, "WARM").await;
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
|
||||
);
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
@@ -6833,251 +6351,17 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("explicit unknown cleanup must fail before backend IO");
|
||||
.expect_err("unknown remote version state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("version state is unknown"));
|
||||
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn test_tier_destination_identity(
|
||||
manager: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
tier: &str,
|
||||
) -> crate::services::tier::tier::TierDestinationId {
|
||||
TierConfigMgr::acquire_operation_lease(manager, tier)
|
||||
.await
|
||||
.expect("test tier lease should be available")
|
||||
.backend_identity()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn user_defined_with_tier_destination_identity(
|
||||
identity: crate::services::tier::tier::TierDestinationId,
|
||||
) -> HashMap<String, String> {
|
||||
let mut user_defined = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity),
|
||||
);
|
||||
user_defined
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn user_defined_with_transition_version_state(state: rustfs_filemeta::TransitionVersionState) -> HashMap<String, String> {
|
||||
let mut user_defined = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
state.as_str().to_string(),
|
||||
);
|
||||
user_defined
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn insert_legacy_transition_version_id(user_defined: &mut HashMap<String, String>, version_id: &str) {
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
version_id.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_tuple_rejects_mixed_legacy_missing_and_explicit_unknown() {
|
||||
let manager = TierConfigMgr::new();
|
||||
register_mock_tier(&manager, "WARM").await;
|
||||
let identity = test_tier_destination_identity(&manager, "WARM").await;
|
||||
let mut legacy_metadata = user_defined_with_tier_destination_identity(identity);
|
||||
insert_legacy_transition_version_id(&mut legacy_metadata, "legacy-version");
|
||||
let mut explicit_metadata = legacy_metadata.clone();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut explicit_metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
|
||||
);
|
||||
let make_info = |user_defined: HashMap<String, String>| ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::free_version_remote_tuple_matches(&make_info(legacy_metadata), &make_info(explicit_metadata))
|
||||
.expect_err("mixed legacy-missing and explicit unknown provenance must fail closed");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_probes_exact_version_hidden_by_current_delete_marker() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = "WARM";
|
||||
let backend = register_mock_tier(&manager, tier).await;
|
||||
let identity = test_tier_destination_identity(&manager, tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy exact cleanup body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body),
|
||||
i64::try_from(b"legacy exact cleanup body".len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Missing))
|
||||
.await;
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state(&remote_object)
|
||||
.await
|
||||
.expect("current remote view should be readable"),
|
||||
TransitionCandidateProbe::Missing,
|
||||
"a current delete marker must hide the historical data version from an unversioned probe"
|
||||
);
|
||||
backend.clear_op_log().await;
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: remote_version,
|
||||
tier: tier.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect("probe-proven legacy exact cleanup should delete the remote version");
|
||||
super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect("a retry after the exact remote version is already missing should be idempotent");
|
||||
|
||||
assert_eq!(
|
||||
backend.op_log().await,
|
||||
vec![
|
||||
MockWarmOp::Get {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Remove {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Get {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![(remote_object, object_info.transitioned_object.version_id)]
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_retains_legacy_unknown_unversioned_object() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = "WARM";
|
||||
let backend = register_mock_tier(&manager, tier).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let identity = test_tier_destination_identity(&manager, tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy unversioned cleanup body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body),
|
||||
i64::try_from(b"legacy unversioned cleanup body".len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
assert!(remote_version.is_empty());
|
||||
backend.clear_op_log().await;
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
user_defined.insert("x-minio-internal-transitioned-versionID".to_string(), String::new());
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: String::new(),
|
||||
tier: tier.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("legacy unversioned cleanup cannot exclude a versioning-state race");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
assert!(backend.op_log().await.is_empty());
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_does_not_remove_a_different_remote_version() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = "WARM";
|
||||
let backend = register_mock_tier(&manager, tier).await;
|
||||
let identity = test_tier_destination_identity(&manager, tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
backend.set_put_remote_version(Some("different-version".to_string())).await;
|
||||
backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(Bytes::from_static(b"different remote version")),
|
||||
i64::try_from(b"different remote version".len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("different remote version should be stored");
|
||||
backend.clear_op_log().await;
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
insert_legacy_transition_version_id(&mut user_defined, "legacy-version");
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: tier.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect("a missing exact legacy version should be an idempotent cleanup success");
|
||||
|
||||
assert_eq!(backend.op_log().await, vec![MockWarmOp::Get { object: remote_object }]);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_remote_delete_requires_persisted_destination_identity() {
|
||||
|
||||
@@ -25,7 +25,6 @@ use super::{
|
||||
manual_transition_job, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
|
||||
pub(crate) const ILM_META_PREFIX: &str = "ilm";
|
||||
const ILM_META_OBJECT_PREFIX: &str = "ilm/";
|
||||
@@ -36,7 +35,6 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
TierDeleteJournal,
|
||||
TierDeleteDispatchManifest,
|
||||
TransitionTransaction,
|
||||
TierProbeIntent,
|
||||
ManualTransitionJob,
|
||||
ManualTransitionScope,
|
||||
ManualTransitionTask,
|
||||
@@ -75,12 +73,6 @@ pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = Durable
|
||||
max_record_size: transition_transaction::MAX_TRANSITION_TRANSACTION_SIZE,
|
||||
kind: DurableIlmRecordKind::TransitionTransaction,
|
||||
};
|
||||
pub(crate) const TIER_PROBE_INTENT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "tier-probe-intent",
|
||||
prefix: tier_probe_intent::TIER_PROBE_INTENT_RECORD_PREFIX,
|
||||
max_record_size: tier_probe_intent::MAX_TIER_PROBE_INTENT_SIZE,
|
||||
kind: DurableIlmRecordKind::TierProbeIntent,
|
||||
};
|
||||
pub(crate) const MANUAL_TRANSITION_JOB_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "manual-transition-job",
|
||||
prefix: "ilm/manual-transition/jobs",
|
||||
@@ -106,12 +98,11 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
|
||||
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
TRANSITION_TRANSACTION_NAMESPACE,
|
||||
TIER_PROBE_INTENT_NAMESPACE,
|
||||
MANUAL_TRANSITION_JOB_NAMESPACE,
|
||||
MANUAL_TRANSITION_SCOPE_NAMESPACE,
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
@@ -209,15 +200,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
revision: u64,
|
||||
state: transition_transaction::TransitionTransactionState,
|
||||
},
|
||||
TierProbeIntent {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
remote_version_sha256: String,
|
||||
remote_version_known: bool,
|
||||
owner_fence_sha256: String,
|
||||
revision: u64,
|
||||
state: tier_probe_intent::TierProbeIntentState,
|
||||
},
|
||||
ManualTransitionJob {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
@@ -250,7 +232,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::TierDeleteDispatchManifest { content_sha256, .. }
|
||||
| Self::TierDeleteDispatchParent { content_sha256, .. }
|
||||
| Self::TransitionTransaction { content_sha256, .. }
|
||||
| Self::TierProbeIntent { content_sha256, .. }
|
||||
| Self::ManualTransitionJob { content_sha256, .. }
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
@@ -440,32 +421,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
.is_some_and(|expected_revision| *next_revision == expected_revision)
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: previous_identity,
|
||||
remote_version_sha256: previous_remote_version,
|
||||
remote_version_known: previous_remote_version_known,
|
||||
owner_fence_sha256: previous_owner_fence,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
..
|
||||
},
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: next_identity,
|
||||
remote_version_sha256: next_remote_version,
|
||||
owner_fence_sha256: next_owner_fence,
|
||||
revision: next_revision,
|
||||
state: next_state,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == next_identity
|
||||
&& previous_owner_fence == next_owner_fence
|
||||
&& next_revision
|
||||
.checked_sub(*previous_revision)
|
||||
.is_some_and(|distance| distance == 1 && tier_probe_state_reaches(*previous_state, *next_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
Self::ManualTransitionJob {
|
||||
content_sha256: previous_content,
|
||||
@@ -545,14 +500,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
/// after the exact terminal ETag and terminal receipt were committed, to
|
||||
/// purge older object versions exposed by that deletion.
|
||||
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
|
||||
if let Self::TierProbeIntent { state, .. } = terminal
|
||||
&& !matches!(
|
||||
state,
|
||||
tier_probe_intent::TierProbeIntentState::AbortedNoRemote | tier_probe_intent::TierProbeIntentState::Completed
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -621,37 +568,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
}
|
||||
})
|
||||
}
|
||||
(
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: previous_identity,
|
||||
remote_version_sha256: previous_remote_version,
|
||||
remote_version_known: previous_remote_version_known,
|
||||
owner_fence_sha256: previous_owner_fence,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
..
|
||||
},
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: terminal_identity,
|
||||
remote_version_sha256: terminal_remote_version,
|
||||
owner_fence_sha256: terminal_owner_fence,
|
||||
revision: terminal_revision,
|
||||
state: terminal_state,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == terminal_identity
|
||||
&& previous_owner_fence == terminal_owner_fence
|
||||
&& matches!(
|
||||
terminal_state,
|
||||
tier_probe_intent::TierProbeIntentState::AbortedNoRemote
|
||||
| tier_probe_intent::TierProbeIntentState::Completed
|
||||
)
|
||||
&& terminal_revision
|
||||
.checked_sub(*previous_revision)
|
||||
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -690,23 +606,6 @@ fn transition_state_distance(
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_probe_state_reaches(
|
||||
from: tier_probe_intent::TierProbeIntentState,
|
||||
to: tier_probe_intent::TierProbeIntentState,
|
||||
revision_distance: u64,
|
||||
) -> bool {
|
||||
use tier_probe_intent::TierProbeIntentState::{AbortedNoRemote, CleanupPending, Completed, UploadOutcomeUnknown, Uploaded};
|
||||
|
||||
match (from, to) {
|
||||
(UploadOutcomeUnknown, Uploaded | CleanupPending | AbortedNoRemote) => revision_distance == 1,
|
||||
(UploadOutcomeUnknown, Completed) => matches!(revision_distance, 2 | 3),
|
||||
(Uploaded, CleanupPending) => revision_distance == 1,
|
||||
(Uploaded, Completed) => revision_distance == 2,
|
||||
(CleanupPending, Completed) => revision_distance == 1,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn manual_job_state_reaches(
|
||||
from: manual_transition_job::ManualTransitionJobState,
|
||||
to: manual_transition_job::ManualTransitionJobState,
|
||||
@@ -1183,42 +1082,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::TierProbeIntent => {
|
||||
let probe_id = tier_probe_intent::tier_probe_intent_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let intent =
|
||||
tier_probe_intent::TierProbeIntent::decode(probe_id, data).map_err(|err| Error::other(err.to_string()))?;
|
||||
let canonical =
|
||||
tier_probe_intent::tier_probe_intent_record_object_name(probe_id).map_err(|err| Error::other(err.to_string()))?;
|
||||
if canonical != path {
|
||||
return Err(Error::other("tier probe intent path is not canonical"));
|
||||
}
|
||||
let identity_sha256 = checkpoint_hash(&(
|
||||
intent.probe_id,
|
||||
&intent.operation,
|
||||
&intent.tier_name,
|
||||
intent.destination_id,
|
||||
&intent.probe_object,
|
||||
&intent.creator_id,
|
||||
intent.creator_epoch,
|
||||
intent.created_at_unix_nanos,
|
||||
))?;
|
||||
let remote_version_sha256 = checkpoint_hash(&intent.remote_version)?;
|
||||
let owner_fence_sha256 = checkpoint_hash(&intent.owner)?;
|
||||
(
|
||||
"probe_id",
|
||||
probe_id.to_string(),
|
||||
DurableIlmRecordCheckpoint::TierProbeIntent {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
remote_version_sha256,
|
||||
remote_version_known: !intent.remote_version.is_unknown(),
|
||||
owner_fence_sha256,
|
||||
revision: intent.revision,
|
||||
state: intent.state,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1374,102 +1237,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
|
||||
let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
|
||||
tier_probe_intent::TierProbeIntent {
|
||||
probe_id,
|
||||
revision: 1,
|
||||
state: tier_probe_intent::TierProbeIntentState::UploadOutcomeUnknown,
|
||||
operation: tier_probe_intent::TierProbeOperationIdentity::Verify {
|
||||
config_etag: "config-etag".to_string(),
|
||||
backend_identity: [1; 32],
|
||||
},
|
||||
tier_name: "COLD-A".to_string(),
|
||||
destination_id: [1; 32],
|
||||
probe_object: tier_probe_intent::tier_probe_object_name(probe_id),
|
||||
creator_id: "node-a".to_string(),
|
||||
creator_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture creator epoch should parse"),
|
||||
created_at_unix_nanos: 1_780_000_000_000_000_000,
|
||||
owner: tier_probe_intent::TierProbeOwnerFence {
|
||||
owner_id: "node-a".to_string(),
|
||||
owner_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture owner epoch should parse"),
|
||||
not_after_unix_nanos: 1_780_000_900_000_000_000,
|
||||
},
|
||||
remote_version: tier_probe_intent::TierProbeRemoteVersion::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_probe_checkpoint(intent: &tier_probe_intent::TierProbeIntent) -> DurableIlmRecordCheckpoint {
|
||||
let path =
|
||||
tier_probe_intent::tier_probe_intent_record_object_name(intent.probe_id).expect("tier probe path should build");
|
||||
let encoded = intent.encode().expect("tier probe intent should encode");
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("tier probe namespace should classify")
|
||||
.expect("tier probe intent should be durable");
|
||||
assert_eq!(namespace, &TIER_PROBE_INTENT_NAMESPACE);
|
||||
validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("tier probe intent should validate")
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
|
||||
let initial_intent = tier_probe_intent_fixture();
|
||||
let initial = tier_probe_checkpoint(&initial_intent);
|
||||
|
||||
let mut uploaded_intent = initial_intent;
|
||||
uploaded_intent
|
||||
.advance(
|
||||
tier_probe_intent::TierProbeIntentState::Uploaded,
|
||||
tier_probe_intent::TierProbeRemoteVersion::versioned("opaque-v1"),
|
||||
)
|
||||
.expect("uploaded state should advance");
|
||||
let uploaded = tier_probe_checkpoint(&uploaded_intent);
|
||||
initial
|
||||
.validate_successor(&uploaded)
|
||||
.expect("durable receipt may adopt the exact uploaded generation");
|
||||
|
||||
let mut cleanup_intent = uploaded_intent.clone();
|
||||
cleanup_intent
|
||||
.advance(
|
||||
tier_probe_intent::TierProbeIntentState::CleanupPending,
|
||||
uploaded_intent.remote_version.clone(),
|
||||
)
|
||||
.expect("cleanup state should advance");
|
||||
let cleanup = tier_probe_checkpoint(&cleanup_intent);
|
||||
uploaded
|
||||
.validate_successor(&cleanup)
|
||||
.expect("durable receipt may adopt the exact cleanup generation");
|
||||
|
||||
let mut completed_intent = cleanup_intent.clone();
|
||||
completed_intent
|
||||
.advance(tier_probe_intent::TierProbeIntentState::Completed, cleanup_intent.remote_version.clone())
|
||||
.expect("completed state should advance");
|
||||
let completed = tier_probe_checkpoint(&completed_intent);
|
||||
cleanup
|
||||
.validate_successor(&completed)
|
||||
.expect("durable receipt may adopt the exact terminal generation");
|
||||
assert!(
|
||||
initial.is_predecessor_of_terminal(&completed),
|
||||
"terminal cleanup must recognize the full acknowledged-PUT path"
|
||||
);
|
||||
assert!(
|
||||
initial.validate_successor(&completed).is_err(),
|
||||
"ordinary receipt advancement must not skip intermediate generations"
|
||||
);
|
||||
assert!(
|
||||
!initial.is_predecessor_of_terminal(&uploaded),
|
||||
"a nonterminal generation must not be accepted as terminal proof"
|
||||
);
|
||||
|
||||
let mut rebound = uploaded_intent;
|
||||
rebound.owner.owner_epoch = Uuid::new_v4();
|
||||
assert!(
|
||||
rebound.encode().is_err(),
|
||||
"dormant v1 must reject owner takeover before producing a checkpoint"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
|
||||
|
||||
@@ -1170,7 +1170,6 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
@@ -1243,7 +1242,6 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1272,7 +1270,6 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1624,7 +1621,6 @@ pub async fn save_manual_transition_scope_admission_if_absent(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1676,7 +1672,6 @@ pub async fn save_manual_transition_scope_admission_if_current(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
|
||||
@@ -1733,7 +1733,6 @@ async fn save_config_if_none_fenced(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1833,7 +1832,6 @@ async fn save_decommission_manifest_checkpoint_if_match(
|
||||
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(observed_etag),
|
||||
@@ -1962,7 +1960,6 @@ async fn save_config_if_match_fenced(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
@@ -3783,7 +3780,6 @@ where
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -3873,7 +3869,6 @@ where
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
@@ -3898,7 +3893,6 @@ where
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
|
||||
@@ -70,11 +72,9 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
|
||||
|
||||
#[cfg(test)]
|
||||
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
|
||||
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<
|
||||
std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
|
||||
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RemoteDeleteBreaker {
|
||||
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
|
||||
fn prune(&mut self, now: Instant) {
|
||||
while let Some(ts) = self.failures.front().copied() {
|
||||
if now.duration_since(ts) > self.window {
|
||||
let _ = self.failures.pop_front();
|
||||
self.failures.pop_front();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(source) = err.get_ref()
|
||||
&& error_chain_contains_signer_header_marker(source)
|
||||
{
|
||||
return true;
|
||||
if let Some(source) = err.get_ref() {
|
||||
if error_chain_contains_signer_header_marker(source) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
let message = err.to_string().to_ascii_lowercase();
|
||||
@@ -205,7 +205,7 @@ impl ObjSweeper {
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||
self.version_id = vid;
|
||||
self.version_id = vid.clone();
|
||||
self
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ impl ObjSweeper {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id,
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.suspended,
|
||||
..Default::default()
|
||||
@@ -388,8 +388,8 @@ impl Jentry {
|
||||
impl ExpiryOp for Jentry {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(self.tier_name.as_bytes());
|
||||
hasher.update(self.obj_name.as_bytes());
|
||||
hasher.update(format!("{}", self.tier_name).as_bytes());
|
||||
hasher.update(format!("{}", self.obj_name).as_bytes());
|
||||
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
tier_name: &str,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
|
||||
|
||||
@@ -612,7 +612,6 @@ pub(crate) async fn save_transition_transaction_record(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -659,7 +658,6 @@ pub(crate) async fn save_transition_transaction_record_if_current(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
|
||||
@@ -477,18 +477,6 @@ impl BucketMetadata {
|
||||
!self.table_bucket_config_json.is_empty()
|
||||
}
|
||||
|
||||
/// `bucket-targets.json` is stored for this bucket but this build cannot
|
||||
/// decode it.
|
||||
///
|
||||
/// Keeps "no replication targets configured" and "the target
|
||||
/// configuration cannot be read" apart, the same distinction the
|
||||
/// `fabricated` marker draws for the bucket metadata as a whole. Only
|
||||
/// meaningful after [`Self::parse_all_configs`] has run; readers must fail
|
||||
/// closed on `true` instead of serving an empty target set.
|
||||
pub fn bucket_targets_unreadable(&self) -> bool {
|
||||
!self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
///
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
@@ -976,32 +964,7 @@ impl BucketMetadata {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decode every stored sub-configuration into its typed field.
|
||||
///
|
||||
/// A decode failure never fails the whole load: this runs on every bucket
|
||||
/// metadata read, including startup and peer reload, so one bucket's
|
||||
/// corrupt sub-configuration must not make the bucket — or the node —
|
||||
/// unloadable. Instead the failure is *retained*: the raw bytes stay
|
||||
/// untouched and the typed field stays `None`, so `!raw.is_empty() &&
|
||||
/// typed.is_none()` is the durable "exists but cannot be read" signal that
|
||||
/// each accessor keys off. Which accessors must fail closed on it:
|
||||
///
|
||||
/// | Config | Verdict |
|
||||
/// |---|---|
|
||||
/// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. |
|
||||
/// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. |
|
||||
/// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. |
|
||||
/// | replication | Fails closed in `get_replication_config`. |
|
||||
/// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). |
|
||||
/// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. |
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. |
|
||||
/// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. |
|
||||
/// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. |
|
||||
/// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. |
|
||||
/// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. |
|
||||
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
|
||||
fn parse_all_configs(&mut self) -> Result<()> {
|
||||
if let Err(e) = self.parse_policy_config() {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1125,26 +1088,20 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
// A stored targets blob that cannot be decoded must not collapse into
|
||||
// the empty target set: that is indistinguishable from "no replication
|
||||
// configured", so replication stops and no caller ever sees an error
|
||||
// (rustfs/backlog#2282). Leaving the typed field `None` while the raw
|
||||
// bytes stay non-empty is the retained parse failure every targets
|
||||
// reader keys off; the bytes are preserved so the configuration is
|
||||
// still recoverable.
|
||||
self.bucket_target_config = None;
|
||||
if !self.bucket_targets_config_json.is_empty() {
|
||||
match serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json) {
|
||||
Ok(targets) => self.bucket_target_config = Some(targets),
|
||||
Err(e) => tracing::error!(
|
||||
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
|
||||
.map(|t| self.bucket_target_config = Some(t))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "bucket_targets",
|
||||
error = %e,
|
||||
"Bucket replication targets are unreadable; replication for this bucket fails closed"
|
||||
),
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
self.bucket_target_config = Some(BucketTargets::default());
|
||||
}
|
||||
} else {
|
||||
self.bucket_target_config = Some(BucketTargets::default());
|
||||
@@ -1578,117 +1535,6 @@ mod test {
|
||||
assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket");
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2282: a stored targets blob this build cannot decode
|
||||
/// must not become the empty target set, and must stay distinguishable
|
||||
/// from a bucket that never configured a target.
|
||||
#[test]
|
||||
fn unreadable_bucket_targets_never_degrade_to_an_empty_target_set() {
|
||||
let truncated = br#"{"targets":[{"endpoint":"s3.example.com","#.to_vec();
|
||||
let mut corrupt = BucketMetadata::new("corrupt-targets");
|
||||
corrupt.bucket_targets_config_json = truncated.clone();
|
||||
|
||||
corrupt
|
||||
.parse_all_configs()
|
||||
.expect("one unreadable sub-config must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
corrupt.bucket_target_config.is_none(),
|
||||
"an undecodable targets blob must not produce a target set at all"
|
||||
);
|
||||
assert!(corrupt.bucket_targets_unreadable());
|
||||
assert_eq!(
|
||||
corrupt.bucket_targets_config_json, truncated,
|
||||
"the raw bytes must survive so the configuration stays recoverable"
|
||||
);
|
||||
|
||||
// The genuinely-absent case is unchanged, and the two now diverge.
|
||||
let mut absent = BucketMetadata::new("no-targets");
|
||||
absent.parse_all_configs().expect("absent targets parse");
|
||||
assert!(
|
||||
absent.bucket_target_config.as_ref().is_some_and(BucketTargets::is_empty),
|
||||
"a bucket that configured no target still reads as an empty target set"
|
||||
);
|
||||
assert!(!absent.bucket_targets_unreadable());
|
||||
}
|
||||
|
||||
/// `Credentials` carries no struct-level `serde(default)`, so one target
|
||||
/// missing `secretKey` is a hard parse error for the whole document. That
|
||||
/// must surface as "unreadable", never as "no targets configured".
|
||||
#[test]
|
||||
fn bucket_targets_missing_secret_key_are_unreadable_not_empty() {
|
||||
let mut bm = BucketMetadata::new("missing-secret-key");
|
||||
bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.example.com","targetbucket":"remote","arn":"arn:rustfs:replication:us-east-1:src:1","credentials":{"accessKey":"AKIAEXAMPLE"}}]}"#.to_vec();
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a rejected targets document must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
bm.bucket_targets_unreadable(),
|
||||
"a targets document rejected for a missing secretKey is unreadable, not empty"
|
||||
);
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
}
|
||||
|
||||
/// The invariant every branch of `parse_all_configs` shares: a stored but
|
||||
/// undecodable payload keeps its raw bytes and leaves the typed field
|
||||
/// `None`, so no branch fabricates a value. What a reader may then do with
|
||||
/// that state is decided per config; see the table on `parse_all_configs`.
|
||||
#[test]
|
||||
fn every_config_branch_retains_its_parse_failure_instead_of_defaulting() {
|
||||
let malformed_xml = b"<not-a-valid-document".to_vec();
|
||||
let malformed_json = b"{not-json".to_vec();
|
||||
|
||||
let mut bm = BucketMetadata::new("all-configs-malformed");
|
||||
bm.policy_config_json = malformed_json.clone();
|
||||
bm.quota_config_json = malformed_json.clone();
|
||||
bm.bucket_targets_config_json = malformed_json.clone();
|
||||
bm.notification_config_xml = malformed_xml.clone();
|
||||
bm.lifecycle_config_xml = malformed_xml.clone();
|
||||
bm.object_lock_config_xml = malformed_xml.clone();
|
||||
bm.versioning_config_xml = malformed_xml.clone();
|
||||
bm.encryption_config_xml = malformed_xml.clone();
|
||||
bm.tagging_config_xml = malformed_xml.clone();
|
||||
bm.replication_config_xml = malformed_xml.clone();
|
||||
bm.cors_config_xml = malformed_xml.clone();
|
||||
bm.logging_config_xml = malformed_xml.clone();
|
||||
bm.website_config_xml = malformed_xml.clone();
|
||||
bm.accelerate_config_xml = malformed_xml.clone();
|
||||
bm.request_payment_config_xml = malformed_xml.clone();
|
||||
bm.public_access_block_config_xml = malformed_xml.clone();
|
||||
// `bucket_acl_config_json` is only checked for UTF-8, so only invalid
|
||||
// UTF-8 exercises its failure branch.
|
||||
bm.bucket_acl_config_json = vec![0xff, 0xfe];
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a bucket whose every config is corrupt must still load its metadata");
|
||||
|
||||
let cleared: [(&str, bool); 17] = [
|
||||
("policy", bm.policy_config.is_none()),
|
||||
("quota", bm.quota_config.is_none()),
|
||||
("bucket_targets", bm.bucket_target_config.is_none()),
|
||||
("notification", bm.notification_config.is_none()),
|
||||
("lifecycle", bm.lifecycle_config.is_none()),
|
||||
("object_lock", bm.object_lock_config.is_none()),
|
||||
("versioning", bm.versioning_config.is_none()),
|
||||
("encryption", bm.sse_config.is_none()),
|
||||
("tagging", bm.tagging_config.is_none()),
|
||||
("replication", bm.replication_config.is_none()),
|
||||
("cors", bm.cors_config.is_none()),
|
||||
("logging", bm.logging_config.is_none()),
|
||||
("website", bm.website_config.is_none()),
|
||||
("accelerate", bm.accelerate_config.is_none()),
|
||||
("request_payment", bm.request_payment_config.is_none()),
|
||||
("public_access_block", bm.public_access_block_config.is_none()),
|
||||
("bucket_acl", bm.bucket_acl_config.is_none()),
|
||||
];
|
||||
for (config, is_cleared) in cleared {
|
||||
assert!(is_cleared, "{config}: a corrupt payload must not be replaced by a default");
|
||||
}
|
||||
|
||||
assert_eq!(bm.bucket_targets_config_json, malformed_json, "raw bytes are retained");
|
||||
assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_update_config_clears_parsed_config_on_delete() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
|
||||
@@ -360,16 +360,6 @@ async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
|
||||
}
|
||||
|
||||
async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
|
||||
if bm.bucket_targets_unreadable() {
|
||||
// "The configuration cannot be read" is not "no targets configured".
|
||||
// Publishing an empty snapshot here is what silently stopped
|
||||
// replication (rustfs/backlog#2282): mark the bucket instead, so every
|
||||
// targets reader gets a typed error, and leave any snapshot from an
|
||||
// earlier readable load in place rather than withdrawing it.
|
||||
BucketTargetSys::get().mark_targets_unreadable(bucket).await;
|
||||
return;
|
||||
}
|
||||
|
||||
BucketTargetSys::get()
|
||||
.update_all_targets(bucket, bm.bucket_target_config.as_ref())
|
||||
.await;
|
||||
@@ -2128,9 +2118,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() {
|
||||
Err(Error::other("persisted bucket public access block configuration is invalid"))
|
||||
} else if let Some(config) = &bm.public_access_block_config {
|
||||
if let Some(config) = &bm.public_access_block_config {
|
||||
Ok((config.clone(), bm.public_access_block_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2441,9 +2429,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() {
|
||||
Err(Error::other("persisted bucket encryption configuration is invalid"))
|
||||
} else if let Some(config) = &bm.sse_config {
|
||||
if let Some(config) = &bm.sse_config {
|
||||
Ok((config.clone(), bm.encryption_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2514,9 +2500,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.quota_config_json.is_empty() && bm.quota_config.is_none() {
|
||||
Err(Error::other("persisted bucket quota configuration is invalid"))
|
||||
} else if let Some(config) = &bm.quota_config {
|
||||
if let Some(config) = &bm.quota_config {
|
||||
Ok((config.clone(), bm.quota_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2538,9 +2522,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if bm.bucket_targets_unreadable() {
|
||||
Err(Error::other("persisted bucket replication target configuration is invalid"))
|
||||
} else if let Some(config) = &bm.bucket_target_config {
|
||||
if let Some(config) = &bm.bucket_target_config {
|
||||
Ok(config.clone())
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2611,7 +2593,6 @@ pub(crate) mod test_support {
|
||||
mod tests {
|
||||
use super::test_support::isolated_store_over_temp_disks;
|
||||
use super::*;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetError;
|
||||
use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
|
||||
BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
@@ -2807,36 +2788,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The `parse_all_configs` audit (rustfs/backlog#2282): every accessor
|
||||
/// whose configuration grants something — plaintext storage, anonymous
|
||||
/// access, capacity, replication targets — reports a corrupt payload as
|
||||
/// invalid rather than as absent, because "absent" is what grants it.
|
||||
#[tokio::test]
|
||||
async fn malformed_permissive_configs_are_not_reported_as_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "malformed-permissive-config";
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration".to_vec();
|
||||
metadata.public_access_block_config_xml = b"<PublicAccessBlockConfiguration".to_vec();
|
||||
metadata.quota_config_json = b"{not-json".to_vec();
|
||||
metadata.bucket_targets_config_json = b"{not-json".to_vec();
|
||||
metadata
|
||||
.parse_all_configs()
|
||||
.expect("a corrupt sub-config must not fail the load");
|
||||
sys.set(bucket.to_string(), Arc::new(metadata)).await;
|
||||
|
||||
for (config, result) in [
|
||||
("encryption", sys.get_sse_config(bucket).await.err()),
|
||||
("public access block", sys.get_public_access_block_config(bucket).await.err()),
|
||||
("quota", sys.get_quota_config(bucket).await.err()),
|
||||
("bucket targets", sys.get_bucket_targets_config(bucket).await.err()),
|
||||
] {
|
||||
let err = result.unwrap_or_else(|| panic!("malformed {config} metadata must not read as a value"));
|
||||
assert_ne!(err, Error::ConfigNotFound, "malformed {config} metadata must not be reported as absent");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() {
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -4115,114 +4066,6 @@ mod tests {
|
||||
target_sys.delete(bucket).await;
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2282: an unreadable `bucket-targets.json` reaches every
|
||||
/// targets reader as a typed error; it neither withdraws a snapshot a
|
||||
/// previous readable load published, nor collapses into the "no targets
|
||||
/// configured" state that a bucket with an absent configuration reports.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unreadable_bucket_targets_fail_closed_and_stay_distinct_from_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let target_sys = BucketTargetSys::get();
|
||||
let unreadable = "targets-unreadable";
|
||||
let absent = "targets-absent";
|
||||
target_sys.delete(unreadable).await;
|
||||
target_sys.delete(absent).await;
|
||||
|
||||
// A readable load publishes this bucket's targets.
|
||||
let mut readable = BucketMetadata::new(unreadable);
|
||||
readable.bucket_target_config = Some(BucketTargets {
|
||||
targets: vec![target(unreadable, "live")],
|
||||
});
|
||||
sync_bucket_target_sys(unreadable, &readable).await;
|
||||
assert_eq!(
|
||||
target_sys
|
||||
.list_bucket_targets(unreadable)
|
||||
.await
|
||||
.expect("readable targets publish")
|
||||
.targets
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
// The same bucket reloaded with a blob that cannot be decoded.
|
||||
let mut corrupt = BucketMetadata::new(unreadable);
|
||||
corrupt.bucket_targets_config_json = br#"{"targets":[{"endpoint":"#.to_vec();
|
||||
corrupt
|
||||
.parse_all_configs()
|
||||
.expect("an unreadable targets blob must not fail the metadata load");
|
||||
sys.set(unreadable.to_string(), Arc::new(corrupt)).await;
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
target_sys.list_bucket_targets(unreadable).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
),
|
||||
"an unreadable configuration must not read as an empty or a missing target set"
|
||||
);
|
||||
assert!(
|
||||
target_sys.list_targets(unreadable, "").await.is_err(),
|
||||
"the admin listing must surface the fault instead of an empty list"
|
||||
);
|
||||
let err = sys
|
||||
.get_bucket_targets_config(unreadable)
|
||||
.await
|
||||
.expect_err("an unreadable targets configuration must not read as a value");
|
||||
assert_ne!(err, Error::ConfigNotFound, "unreadable must not be reported as absent");
|
||||
|
||||
// A bucket that never configured a target keeps its previous behavior.
|
||||
let mut no_targets = BucketMetadata::new(absent);
|
||||
no_targets.parse_all_configs().expect("absent targets parse");
|
||||
sys.set(absent.to_string(), Arc::new(no_targets)).await;
|
||||
assert!(
|
||||
matches!(
|
||||
target_sys.list_bucket_targets(absent).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. })
|
||||
),
|
||||
"an absent configuration must still report as a missing target set"
|
||||
);
|
||||
assert!(
|
||||
target_sys
|
||||
.list_targets(absent, "")
|
||||
.await
|
||||
.expect("an absent configuration lists no targets")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
sys.get_bucket_targets_config(absent)
|
||||
.await
|
||||
.expect("an absent targets configuration still reads as an empty set")
|
||||
.is_empty(),
|
||||
"the absent path must keep returning an empty target set, exactly as before"
|
||||
);
|
||||
|
||||
// One bucket's unreadable configuration does not reach another bucket.
|
||||
assert!(!matches!(
|
||||
target_sys.list_bucket_targets(absent).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
));
|
||||
|
||||
// A repaired configuration takes effect on the next load, no restart.
|
||||
let mut repaired = BucketMetadata::new(unreadable);
|
||||
repaired.bucket_target_config = Some(BucketTargets {
|
||||
targets: vec![target(unreadable, "repaired")],
|
||||
});
|
||||
sync_bucket_target_sys(unreadable, &repaired).await;
|
||||
assert_eq!(
|
||||
target_sys
|
||||
.list_bucket_targets(unreadable)
|
||||
.await
|
||||
.expect("a repaired configuration clears the unreadable marker")
|
||||
.targets
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
target_sys.delete(unreadable).await;
|
||||
target_sys.delete(absent).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() {
|
||||
|
||||
@@ -684,7 +684,6 @@ async fn write_checkpoint(
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(preconditions),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -86,12 +86,7 @@ impl BreakerVerdict {
|
||||
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
|
||||
BreakerVerdict::Failure
|
||||
}
|
||||
Some(
|
||||
SourceError::AccessDenied
|
||||
| SourceError::Unsupported(_)
|
||||
| SourceError::InvalidPagination(_)
|
||||
| SourceError::Other(_),
|
||||
) => BreakerVerdict::Neutral,
|
||||
Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,8 +189,8 @@ pub enum SourceListPlan {
|
||||
/// delimiter — the source's own roll-up boundary matches the request's.
|
||||
Page { prefix: String },
|
||||
/// `filter.prefix` reaches past a delimiter, so every key the source could
|
||||
/// contribute rolls into this one common prefix. Bounded probes follow
|
||||
/// empty progressing pages until a key proves existence or the source ends.
|
||||
/// contribute rolls into this one common prefix. One bounded probe listing
|
||||
/// decides whether it exists; there is nothing to paginate.
|
||||
Folded { probe_prefix: String, common_prefix: String },
|
||||
}
|
||||
|
||||
@@ -279,29 +279,6 @@ pub struct FetchRequest {
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
/// Invalid pagination metadata. Opaque cursor values are never included in errors.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ListPageError {
|
||||
#[error("truncated listing has no continuation token")]
|
||||
Missing,
|
||||
#[error("truncated listing has an empty continuation token")]
|
||||
Empty,
|
||||
#[error("truncated listing repeats a continuation token")]
|
||||
Repeated,
|
||||
}
|
||||
|
||||
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
|
||||
if is_truncated {
|
||||
match next_token {
|
||||
None => return Err(ListPageError::Missing),
|
||||
Some("") => return Err(ListPageError::Empty),
|
||||
Some(next) if Some(next) == token => return Err(ListPageError::Repeated),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SideState {
|
||||
start: SideCursor,
|
||||
@@ -387,11 +364,6 @@ impl ListThroughMerger {
|
||||
/// or `filter.prefix` excludes it.
|
||||
pub fn disable_source(&mut self) {
|
||||
self.source.disabled = true;
|
||||
// A refill can fail after a valid first page. A local-only response
|
||||
// must discard both that source payload and its ordering horizon.
|
||||
self.source.entries.clear();
|
||||
self.source.pages.clear();
|
||||
self.source.more = false;
|
||||
}
|
||||
|
||||
pub fn next_fetch(&self) -> Option<FetchRequest> {
|
||||
@@ -406,13 +378,7 @@ impl ListThroughMerger {
|
||||
/// Records one fetched page. `entries` must be sorted by `name` and already
|
||||
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads
|
||||
/// in the same order.
|
||||
pub fn push_page(
|
||||
&mut self,
|
||||
side: MergeSide,
|
||||
entries: Vec<ListEntryKey>,
|
||||
is_truncated: bool,
|
||||
next_token: Option<String>,
|
||||
) -> Result<(), ListPageError> {
|
||||
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
|
||||
let state = match side {
|
||||
MergeSide::Local => &mut self.local,
|
||||
MergeSide::Source => &mut self.source,
|
||||
@@ -421,19 +387,15 @@ impl ListThroughMerger {
|
||||
Some(last) => last.next_token.clone(),
|
||||
None => state.start.token.clone(),
|
||||
};
|
||||
validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?;
|
||||
// Also reject a cycle through an earlier page in this bounded fetch.
|
||||
if is_truncated && state.pages.iter().any(|page| page.token == next_token) {
|
||||
return Err(ListPageError::Repeated);
|
||||
}
|
||||
state.more = is_truncated;
|
||||
// A truncated page without a cursor cannot be continued; treating the
|
||||
// side as finished is the only alternative to looping on it forever.
|
||||
state.more = is_truncated && next_token.is_some();
|
||||
state.pages.push(FetchedPage {
|
||||
token,
|
||||
count: entries.len(),
|
||||
next_token: is_truncated.then_some(next_token).flatten(),
|
||||
});
|
||||
state.entries.extend(entries);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn finish(self) -> MergeOutcome {
|
||||
@@ -637,15 +599,9 @@ mod tests {
|
||||
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
|
||||
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
|
||||
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
|
||||
merger
|
||||
.push_page(fetch.side, kept, truncated, next)
|
||||
.expect("reference provider pages must advance");
|
||||
merger.push_page(fetch.side, kept, truncated, next);
|
||||
}
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
|
||||
if outcome.is_truncated {
|
||||
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
|
||||
}
|
||||
page_sizes.push(outcome.picks.len());
|
||||
for pick in &outcome.picks {
|
||||
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
|
||||
@@ -660,25 +616,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
|
||||
// This oracle builds the complete namespace independently of the
|
||||
// provider's page/marker helper and the production merger.
|
||||
let mut namespace = std::collections::BTreeMap::new();
|
||||
for key in local.iter().chain(source) {
|
||||
let Some(suffix) = key.strip_prefix(prefix) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty())
|
||||
&& let Some((directory, _)) = suffix.split_once(delimiter)
|
||||
{
|
||||
namespace.insert(format!("{prefix}{directory}{delimiter}"), true);
|
||||
continue;
|
||||
}
|
||||
namespace.insert(key.clone(), false);
|
||||
}
|
||||
namespace
|
||||
.into_iter()
|
||||
.map(|(name, is_prefix)| ListEntryKey { name, is_prefix })
|
||||
.collect()
|
||||
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
|
||||
all.sort();
|
||||
all.dedup();
|
||||
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
|
||||
entries
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -720,9 +662,7 @@ mod tests {
|
||||
token: None
|
||||
})
|
||||
);
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
|
||||
.expect("local EOF is valid");
|
||||
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
|
||||
assert_eq!(merger.next_fetch(), None);
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.picks.len(), 1);
|
||||
@@ -743,14 +683,12 @@ mod tests {
|
||||
};
|
||||
let mut merger = ListThroughMerger::new(1, Some(&resume));
|
||||
merger.disable_source();
|
||||
merger
|
||||
.push_page(
|
||||
MergeSide::Local,
|
||||
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
|
||||
true,
|
||||
Some("local-2".to_string()),
|
||||
)
|
||||
.expect("local cursor advances");
|
||||
merger.push_page(
|
||||
MergeSide::Local,
|
||||
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
|
||||
true,
|
||||
Some("local-2".to_string()),
|
||||
);
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("truncated page carries a token");
|
||||
@@ -760,212 +698,6 @@ mod tests {
|
||||
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_pages_require_a_nonempty_advancing_cursor() {
|
||||
for side in [MergeSide::Local, MergeSide::Source] {
|
||||
for entries in [vec![], vec![ListEntryKey::object("a")]] {
|
||||
for (next, expected) in [
|
||||
(None, Err(ListPageError::Missing)),
|
||||
(Some(""), Err(ListPageError::Empty)),
|
||||
(Some("stuck"), Err(ListPageError::Repeated)),
|
||||
(Some("advances"), Ok(())),
|
||||
] {
|
||||
let resume = ListThroughToken::new(
|
||||
SideCursor {
|
||||
token: Some("stuck".into()),
|
||||
done: false,
|
||||
},
|
||||
SideCursor {
|
||||
token: Some("stuck".into()),
|
||||
done: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string));
|
||||
assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}");
|
||||
let state = if side == MergeSide::Local {
|
||||
&merger.local
|
||||
} else {
|
||||
&merger.source
|
||||
};
|
||||
assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() {
|
||||
let resume = ListThroughToken::new(
|
||||
SideCursor { token: None, done: true },
|
||||
SideCursor {
|
||||
token: Some("stuck".into()),
|
||||
done: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
assert_eq!(
|
||||
merger.next_fetch(),
|
||||
Some(FetchRequest {
|
||||
side: MergeSide::Source,
|
||||
token: Some("stuck".into())
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_pages_may_advance_within_the_fetch_budget_until_eof() {
|
||||
let mut merger = ListThroughMerger::new(2, None);
|
||||
merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF");
|
||||
for next in ["opaque-z", "opaque-a"] {
|
||||
assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source);
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![], true, Some(next.into()))
|
||||
.expect("opaque cursor advances regardless of sort order");
|
||||
}
|
||||
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.picks.is_empty());
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("empty progressing page has a cursor");
|
||||
assert_eq!(token.source.as_deref(), Some("opaque-a"));
|
||||
let mut merger = ListThroughMerger::new(2, Some(&token));
|
||||
assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a"));
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
|
||||
.expect("source EOF");
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
side: MergeSide::Source,
|
||||
index: 0
|
||||
}]
|
||||
);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() {
|
||||
let resume = ListThroughToken::new(
|
||||
SideCursor { token: None, done: true },
|
||||
SideCursor {
|
||||
token: Some("first".into()),
|
||||
done: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![], true, Some("second".into()))
|
||||
.expect("first page advances");
|
||||
assert_eq!(
|
||||
merger.push_page(MergeSide::Source, vec![], true, Some("first".into())),
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_refill_failure_discards_buffered_source_entries_and_horizon() {
|
||||
let mut merger = ListThroughMerger::new(2, None);
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None)
|
||||
.expect("local EOF");
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into()))
|
||||
.expect("first source page advances");
|
||||
assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck"));
|
||||
assert_eq!(
|
||||
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
merger.disable_source();
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
side: MergeSide::Local,
|
||||
index: 0
|
||||
}]
|
||||
);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_through_static_namespace_boundary_matrix() {
|
||||
let corpus = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/child",
|
||||
"a0",
|
||||
"b",
|
||||
"b/leaf",
|
||||
"quote\"&<",
|
||||
"space key",
|
||||
"z",
|
||||
"é",
|
||||
"中/文",
|
||||
];
|
||||
for count in [0, 1, 3, 4, corpus.len()] {
|
||||
let keys: Vec<String> = corpus[..count].iter().map(|key| (*key).to_string()).collect();
|
||||
for placement in 0..3 {
|
||||
let (local, source): (Vec<_>, Vec<_>) =
|
||||
keys.iter()
|
||||
.enumerate()
|
||||
.fold((vec![], vec![]), |(mut local, mut source), (index, key)| {
|
||||
if placement != 1 || index % 2 == 0 {
|
||||
local.push(key.clone());
|
||||
}
|
||||
if placement != 0 || index % 2 == 0 {
|
||||
source.push(key.clone());
|
||||
}
|
||||
(local, source)
|
||||
});
|
||||
for prefix in ["", "a", "a/", "中/"] {
|
||||
for delimiter in [None, Some("/")] {
|
||||
for max_keys in [1, 3, 4] {
|
||||
let oracle = expected(&local, &source, prefix, delimiter);
|
||||
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
|
||||
assert_eq!(
|
||||
emitted.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>(),
|
||||
oracle,
|
||||
"count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}"
|
||||
);
|
||||
let expected_sizes: Vec<_> = if oracle.is_empty() {
|
||||
vec![0]
|
||||
} else {
|
||||
oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect()
|
||||
};
|
||||
assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_through_large_overlap_walk_keeps_all_5300_keys() {
|
||||
let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect();
|
||||
let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect();
|
||||
let (emitted, sizes) = walk(&local, &source, "", None, 333);
|
||||
assert_eq!(emitted.len(), 5300);
|
||||
for (index, (entry, side)) in emitted.iter().enumerate() {
|
||||
assert_eq!(entry.name, format!("k{index:05}"));
|
||||
assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source });
|
||||
}
|
||||
assert_eq!(sizes, [vec![333; 15], vec![305]].concat());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_round_trips_and_rejects_tampering() {
|
||||
let token = ListThroughToken::new(
|
||||
@@ -1064,10 +796,7 @@ mod tests {
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig {
|
||||
rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706),
|
||||
..ProptestConfig::with_cases(256)
|
||||
})]
|
||||
#![proptest_config(ProptestConfig::with_cases(256))]
|
||||
|
||||
/// Full pagination of a merged listing equals the sorted, deduplicated
|
||||
/// union of both sides, with every shared key served by local, and no
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
|
||||
//! forwarded: v1 rejects SSE-C source objects outright.
|
||||
|
||||
use super::list_through::{ListPageError, validate_list_page};
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
|
||||
};
|
||||
@@ -224,8 +223,6 @@ pub enum SourceError {
|
||||
ServerError(u16),
|
||||
#[error("unsupported source object: {0}")]
|
||||
Unsupported(String),
|
||||
#[error("invalid source listing: {0}")]
|
||||
InvalidPagination(#[from] ListPageError),
|
||||
#[error("source request failed: {0}")]
|
||||
Other(String),
|
||||
}
|
||||
@@ -248,7 +245,6 @@ impl SourceError {
|
||||
SourceError::Connect(_) => "connect",
|
||||
SourceError::ServerError(_) => "server_error",
|
||||
SourceError::Unsupported(_) => "unsupported",
|
||||
SourceError::InvalidPagination(_) => "invalid_pagination",
|
||||
SourceError::Other(_) => "other",
|
||||
}
|
||||
}
|
||||
@@ -718,7 +714,6 @@ impl SourceClient {
|
||||
..*request
|
||||
})
|
||||
.await?;
|
||||
validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?;
|
||||
page.objects = page
|
||||
.objects
|
||||
.into_iter()
|
||||
@@ -805,6 +800,11 @@ impl SourceBackend for S3SourceBackend {
|
||||
|
||||
let is_truncated = output.is_truncated.unwrap_or(false);
|
||||
let next_continuation_token = output.next_continuation_token;
|
||||
if is_truncated && next_continuation_token.is_none() {
|
||||
return Err(SourceError::Other(
|
||||
"source reported a truncated listing without a continuation token".to_string(),
|
||||
));
|
||||
}
|
||||
let objects = output
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
@@ -1274,9 +1274,7 @@ mod tests {
|
||||
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
|
||||
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
|
||||
</ListBucketResult>"#;
|
||||
let next_body = body.replace("data/opaque", "data/next");
|
||||
let (client, requests) =
|
||||
scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await;
|
||||
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
|
||||
let first = client
|
||||
.list_page(&SourceListRequest {
|
||||
prefix: Some("photos/"),
|
||||
@@ -1338,104 +1336,7 @@ mod tests {
|
||||
.list_objects_v2(None, None, 10)
|
||||
.await
|
||||
.expect_err("truncated page without token is corrupt");
|
||||
assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_page_validates_s3_cursor_progress_before_mapping_entries() {
|
||||
for contents in ["", "<Contents><Key>data/a</Key><Size>1</Size></Contents>"] {
|
||||
for (truncated, next, expected) in [
|
||||
(true, None, Some(ListPageError::Missing)),
|
||||
(true, Some(""), Some(ListPageError::Empty)),
|
||||
(true, Some("stuck"), Some(ListPageError::Repeated)),
|
||||
(true, Some("opaque-next"), None),
|
||||
(false, None, None),
|
||||
(false, Some("stuck"), None),
|
||||
] {
|
||||
let next_xml = next
|
||||
.map(|next| format!("<NextContinuationToken>{next}</NextContinuationToken>"))
|
||||
.unwrap_or_default();
|
||||
let body = format!(
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next_xml}{contents}</ListBucketResult>"
|
||||
);
|
||||
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await;
|
||||
let result = client
|
||||
.list_page(&SourceListRequest {
|
||||
continuation_token: Some("stuck"),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
match expected {
|
||||
Some(expected) => {
|
||||
let error = result.expect_err("malformed pagination must fail at the provider boundary");
|
||||
assert!(
|
||||
matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected),
|
||||
"{error:?}"
|
||||
);
|
||||
assert_eq!(error.class_label(), "invalid_pagination");
|
||||
assert!(!error.is_retryable());
|
||||
assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens");
|
||||
}
|
||||
None => {
|
||||
let page = result.expect("progressing empty/nonempty pages and EOF are valid");
|
||||
assert_eq!(page.is_truncated, truncated);
|
||||
assert_eq!(page.next_continuation_token.as_deref(), next);
|
||||
assert_eq!(page.objects.len(), usize::from(!contents.is_empty()));
|
||||
if let Some(object) = page.objects.first() {
|
||||
assert_eq!(object.key, "a");
|
||||
}
|
||||
}
|
||||
}
|
||||
let requests = recorded(&requests);
|
||||
assert_eq!(requests.len(), 1, "invalid pagination must not be retried");
|
||||
assert!(requests[0].uri.contains("continuation-token=stuck"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ListOnlyBackend(SourcePage);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SourceBackend for ListOnlyBackend {
|
||||
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
|
||||
assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged");
|
||||
Ok(self.0.clone())
|
||||
}
|
||||
|
||||
async fn head(&self, _key: &str) -> Result<SourceHead, SourceError> {
|
||||
panic!("unexpected HEAD in list test")
|
||||
}
|
||||
async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
|
||||
panic!("unexpected GET in list test")
|
||||
}
|
||||
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
|
||||
panic!("unexpected tagging in list test")
|
||||
}
|
||||
async fn probe(&self) -> Result<(), SourceError> {
|
||||
panic!("unexpected probe in list test")
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() {
|
||||
for (next, expected) in [
|
||||
(None, ListPageError::Missing),
|
||||
(Some(""), ListPageError::Empty),
|
||||
(Some("stuck"), ListPageError::Repeated),
|
||||
] {
|
||||
let mut client = prefix_client(Some("data/".into()));
|
||||
client.backend = Box::new(ListOnlyBackend(SourcePage {
|
||||
is_truncated: true,
|
||||
next_continuation_token: next.map(str::to_string),
|
||||
..Default::default()
|
||||
}));
|
||||
let error = client
|
||||
.list_objects_v2(None, Some("stuck"), 2)
|
||||
.await
|
||||
.expect_err("all providers must advance pagination");
|
||||
assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected));
|
||||
}
|
||||
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
|
||||
}
|
||||
|
||||
const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
@@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason {
|
||||
SourceError::Connect(_) => PullFailureReason::SourceConnect,
|
||||
SourceError::ServerError(_) => PullFailureReason::SourceServerError,
|
||||
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
|
||||
SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
|
||||
SourceError::Other(_) => PullFailureReason::SourceOther,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -652,10 +652,9 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_async::time::TimeSource;
|
||||
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
|
||||
RemoteS3EndpointSpec {
|
||||
@@ -825,174 +824,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct ClockSkewTimeSource(Arc<AtomicU64>);
|
||||
|
||||
impl TimeSource for ClockSkewTimeSource {
|
||||
fn now(&self) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct ClockSkewConnector {
|
||||
request_headers: RecordedHeaders,
|
||||
error_code: &'static str,
|
||||
skew_seconds: i64,
|
||||
clock: ClockSkewTimeSource,
|
||||
}
|
||||
|
||||
fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for ClockSkewConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
|
||||
assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
|
||||
headers.push(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
|
||||
)
|
||||
} else {
|
||||
(200, String::new())
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
.expect("clock skew fixture response");
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn clock_skew_client(
|
||||
error_code: &'static str,
|
||||
skew_seconds: i64,
|
||||
retry: RemoteS3RetryPolicy,
|
||||
) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
|
||||
let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
|
||||
let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
|
||||
let connector = SharedHttpConnector::new(ClockSkewConnector {
|
||||
request_headers: Arc::clone(&headers),
|
||||
error_code,
|
||||
skew_seconds,
|
||||
clock: clock.clone(),
|
||||
});
|
||||
let mut spec = spec("s3.example.com", true);
|
||||
spec.retry = retry;
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("clock skew fixture uses the production outbound configuration")
|
||||
.http_client(http_client_fn(move |_settings, _components| connector.clone()))
|
||||
.time_source(clock.clone())
|
||||
.build();
|
||||
(S3Client::from_conf(config), headers, clock)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect("clock skew GET must retry successfully");
|
||||
assert_eq!(
|
||||
headers.lock().expect("captured requests").len(),
|
||||
2,
|
||||
"{error_code}: GET needs exactly one retry"
|
||||
);
|
||||
clock.0.fetch_add(17, Ordering::SeqCst);
|
||||
// SDK signing time is independent of Tokio's retry/scheduler clock.
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
client
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
.send()
|
||||
.await
|
||||
.expect("subsequent HEAD must use the client's cached skew");
|
||||
let headers = headers.lock().expect("captured signed requests");
|
||||
assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
recorded_header(&headers[index], "authorization")
|
||||
.rsplit_once("Signature=")
|
||||
.expect("SigV4 authorization contains a signature")
|
||||
.1
|
||||
};
|
||||
assert_ne!(
|
||||
signature(0),
|
||||
signature(1),
|
||||
"{error_code}: retry must be signed again after adjusting its date"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn remote_s3_clock_skew_respects_one_attempt_policy() {
|
||||
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
|
||||
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for retry in [
|
||||
RemoteS3RetryPolicy::Disabled,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts: 1 },
|
||||
] {
|
||||
let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("clock skew must not override the caller's one-attempt budget");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
|
||||
assert_eq!(
|
||||
headers.lock().expect("captured requests").len(),
|
||||
1,
|
||||
"{error_code}: {retry:?} must send exactly one request"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_style_auto_and_path_force_path_style() {
|
||||
assert!(PathStyle::Auto.force_path_style());
|
||||
|
||||
@@ -46,7 +46,7 @@ use super::replication_storage_boundary::{
|
||||
HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO,
|
||||
ReplicationStorage,
|
||||
};
|
||||
use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted};
|
||||
use super::replication_target_boundary::{ReplicationTargetStore, replication_object_is_ssec_encrypted};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use futures_util::stream::{self, StreamExt};
|
||||
@@ -3084,23 +3084,6 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
|
||||
let tgts = match ReplicationTargetStore::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => Some(targets),
|
||||
// A bucket whose persisted target configuration cannot be decoded has
|
||||
// an unknown target set, not an empty one: scheduling against `None`
|
||||
// here would drop every heal for it without a trace
|
||||
// (rustfs/backlog#2282). Report it missed so the object is retried
|
||||
// once the configuration is readable again.
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) => {
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket,
|
||||
reason = "target_config_unreadable",
|
||||
"Bucket replication targets are unreadable; replication heal queue fails closed"
|
||||
);
|
||||
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::BucketTargetError;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
|
||||
use http::HeaderMap;
|
||||
|
||||
@@ -203,7 +203,7 @@ mod tests {
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn encode_context(context: &BTreeMap<String, String>) -> String {
|
||||
fn encode_context(context: &HashMap<String, String>) -> String {
|
||||
let ordered = context.iter().collect::<BTreeMap<_, _>>();
|
||||
serde_json::to_string(&ordered).expect("context serializes")
|
||||
}
|
||||
|
||||
@@ -5493,7 +5493,6 @@ where
|
||||
fence.ensure_held()?;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
|
||||
..Default::default()
|
||||
@@ -14413,7 +14412,6 @@ impl ECStore {
|
||||
encoded.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -14568,7 +14566,6 @@ impl ECStore {
|
||||
encoded,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -14960,7 +14957,6 @@ impl ECStore {
|
||||
encoded,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
|
||||
@@ -317,22 +317,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
|
||||
.await
|
||||
.result
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn rename_data_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::RenameDataObservation {
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -349,35 +333,22 @@ impl LocalDiskWrapper {
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let result = operation
|
||||
run_owned_mutation(external_guard, move || async move {
|
||||
operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed =
|
||||
Box::pin(
|
||||
operation
|
||||
.disk
|
||||
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
|
||||
)
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
operation
|
||||
.disk
|
||||
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
|
||||
.await
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
preflight_rejection,
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2617,46 +2588,6 @@ mod tests {
|
||||
assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
|
||||
for source_exists in [false, true] {
|
||||
for guarded in [false, true] {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
|
||||
.expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
if source_exists {
|
||||
disk.make_volume("source").await.expect("source volume should exist");
|
||||
}
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
|
||||
let mut file_info = FileInfo::new("object", 1, 0);
|
||||
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
|
||||
file_info.erasure.index = 1;
|
||||
let observed = wrapper
|
||||
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
|
||||
.await;
|
||||
assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
|
||||
assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
|
||||
let snapshot = wrapper.metrics_snapshot();
|
||||
assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
|
||||
assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
|
||||
assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
|
||||
|
||||
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
let observed = wrapper
|
||||
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
|
||||
.await;
|
||||
assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
|
||||
assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
|
||||
let snapshot = wrapper.metrics_snapshot();
|
||||
assert_eq!(snapshot.total_errors_availability, 1);
|
||||
assert_eq!(snapshot.total_writes, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_health_wrapper_counts_returned_availability_errors() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
+1143
-202
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -75,25 +75,6 @@ use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
pub(crate) struct RenameDataObservation {
|
||||
pub(crate) result: Result<RenameDataResp>,
|
||||
preflight_rejection: Option<local::LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
impl RenameDataObservation {
|
||||
fn unknown(result: Result<RenameDataResp>) -> Self {
|
||||
Self {
|
||||
result,
|
||||
preflight_rejection: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
|
||||
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
|
||||
|
||||
@@ -730,36 +711,6 @@ impl Disk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
|
||||
@@ -870,18 +870,6 @@ impl TierFreeVersionReceiptSink {
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal PUT completion boundary; this does not change fsync or write quorum.
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum WriteCompletion {
|
||||
/// Return at write quorum when the commit owner can retain its guards.
|
||||
#[default]
|
||||
Quorum,
|
||||
/// Drain the rename fan-out before returning. Minority failures still heal
|
||||
/// after a successful quorum commit; this does not require every disk to succeed.
|
||||
TailDrained,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -908,10 +896,6 @@ pub struct ObjectOptions {
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
pub expected_bucket_incarnation_id: Option<Uuid>,
|
||||
pub no_lock: bool,
|
||||
/// Control-plane writers that immediately read or CAS the same namespace
|
||||
/// key use TailDrained without changing namespace lock ownership.
|
||||
#[doc(hidden)]
|
||||
pub write_completion: WriteCompletion,
|
||||
/// True when an upper layer already holds the object read lock before
|
||||
/// forwarding a no_lock read to the set layer.
|
||||
pub metadata_cache_safe: bool,
|
||||
|
||||
@@ -62,27 +62,12 @@ const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
|
||||
const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
// Keep this synchronized with the version served by node_service. Including
|
||||
// the local member in the minimum prevents an older coordinator from
|
||||
// self-authorizing a policy implemented only by newer remote peers.
|
||||
const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
/// Version 5 is reserved for a fleet whose every metadata writer preserves
|
||||
/// explicit transition version state and destination identity, and implements
|
||||
/// conditional per-generation `xl.meta` writes with strong readback. The node
|
||||
/// service must not advertise this version until the conditional writer from
|
||||
/// rustfs/backlog#684 is available.
|
||||
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
|
||||
|
||||
fn cross_pool_fence_policy_results(
|
||||
peer_epochs: BTreeMap<String, Uuid>,
|
||||
minimum_version: u32,
|
||||
) -> (
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
) {
|
||||
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
|
||||
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
@@ -93,18 +78,7 @@ fn cross_pool_fence_policy_results(
|
||||
} else {
|
||||
Err(Error::other("decommission target fence policy capability version is unsupported"))
|
||||
};
|
||||
let legacy_transition_state_reconcile_result =
|
||||
if minimum_version >= LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
Err(Error::other("legacy transition state reconcile policy capability version is unsupported"))
|
||||
};
|
||||
(
|
||||
Ok(peer_epochs),
|
||||
journal_result,
|
||||
decommission_target_fence_result,
|
||||
legacy_transition_state_reconcile_result,
|
||||
)
|
||||
(Ok(peer_epochs), journal_result, decommission_target_fence_result)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -278,21 +252,10 @@ pub(crate) struct TierDeleteJournalFleetProofToken {
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for one legacy transition-state reconciliation.
|
||||
///
|
||||
/// The token intentionally cannot be cloned. Its permit keeps the admitted
|
||||
/// fleet generation alive until the caller finishes the final strong
|
||||
/// readback, while revocation makes every later validation fail immediately.
|
||||
pub struct LegacyTransitionStateReconcileFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
@@ -311,10 +274,6 @@ fn decommission_target_fence_fleet_proof_slot() -> &'static std::sync::RwLock<Fl
|
||||
DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -485,125 +444,6 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
|
||||
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
|
||||
}
|
||||
|
||||
/// Acquire one non-cloneable authority that must span the complete reconcile
|
||||
/// effect window, including its final strong readback.
|
||||
pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, expected_topology, Instant::now())?
|
||||
};
|
||||
let observed_peer_epochs = observe_legacy_transition_state_reconcile_fleet(expected_topology).await?;
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
&proof,
|
||||
expected_topology,
|
||||
&observed_peer_epochs,
|
||||
Instant::now(),
|
||||
)
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(LegacyTransitionStateReconcileFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
let (peer_epochs, minimum_version) = timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()?;
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peer_epochs, minimum_version);
|
||||
reconcile_result.ok()
|
||||
}
|
||||
|
||||
/// Revalidate the exact fleet generation captured by a reconcile token with a
|
||||
/// fresh synchronous observation. Callers must await this before each
|
||||
/// conditional metadata write and after the final strong readback.
|
||||
pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_with_observer(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
proof,
|
||||
expected_topology,
|
||||
|| observe_legacy_transition_state_reconcile_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
|
||||
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observe: F,
|
||||
) -> bool
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Option<BTreeMap<String, Uuid>>>,
|
||||
{
|
||||
{
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let Some(observed_peer_epochs) = observe().await else {
|
||||
return false;
|
||||
};
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
proof,
|
||||
expected_topology,
|
||||
&observed_peer_epochs,
|
||||
Instant::now(),
|
||||
)
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observed_peer_epochs: &BTreeMap<String, Uuid>,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(state, proof, expected_topology, now)
|
||||
&& proof.token.peer_epochs.as_ref() == observed_peer_epochs
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
|
||||
let state = tier_delete_journal_fleet_proof_slot()
|
||||
@@ -926,7 +766,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
@@ -959,12 +798,11 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
};
|
||||
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
|
||||
let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
(
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message)),
|
||||
@@ -980,7 +818,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1043,24 +880,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
reconcile_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "legacy_transition_state_reconcile_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
@@ -1140,7 +959,7 @@ impl NotificationSys {
|
||||
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
let mut minimum_version = LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION;
|
||||
let mut minimum_version = u32::MAX;
|
||||
for result in join_all(probes).await {
|
||||
let (peer, version, epoch) = result?;
|
||||
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
|
||||
@@ -1149,6 +968,11 @@ impl NotificationSys {
|
||||
minimum_version = minimum_version.min(version);
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
// A single-node deployment has no remote member to lower the local
|
||||
// policy version advertised by this binary.
|
||||
if minimum_version == u32::MAX {
|
||||
minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION;
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
}
|
||||
@@ -3366,36 +3190,20 @@ mod tests {
|
||||
#[test]
|
||||
fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {
|
||||
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
|
||||
let (generic_v2, journal_v2, decommission_v2, reconcile_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
|
||||
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
|
||||
assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence");
|
||||
assert!(reconcile_v2.is_err(), "v2 cannot authorize legacy transition-state reconciliation");
|
||||
|
||||
let (generic_v3, journal_v3, decommission_v3, reconcile_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
|
||||
let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
|
||||
assert!(generic_v3.is_ok());
|
||||
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
|
||||
assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
|
||||
assert!(reconcile_v3.is_err());
|
||||
|
||||
let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4);
|
||||
assert!(generic_v4.is_ok());
|
||||
assert!(journal_v4.is_ok());
|
||||
assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
|
||||
assert!(
|
||||
reconcile_v4.is_err(),
|
||||
"the current local policy lacks the conditional xl.meta writer required by reconcile"
|
||||
);
|
||||
|
||||
let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
|
||||
assert!(generic_v5.is_ok());
|
||||
assert!(journal_v5.is_ok());
|
||||
assert!(decommission_v5.is_ok());
|
||||
assert!(
|
||||
reconcile_v5.is_ok(),
|
||||
"only an all-v5 fleet preserves destination identity and conditional reconcile writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3650,234 +3458,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_admits_only_compatible_single_and_multi_node_fleets() {
|
||||
let now = Instant::now();
|
||||
for peers in [
|
||||
BTreeMap::new(),
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
|
||||
] {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let (_, _, _, result) =
|
||||
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", result, now).is_none());
|
||||
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("an all-compatible fleet should admit reconciliation")
|
||||
};
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_restart_drains_concurrent_effect_windows() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, original_result) =
|
||||
cross_pool_fence_policy_results(original_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
|
||||
|
||||
let (first, second) = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
(
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the first reconcile writer should be admitted"),
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the second reconcile writer should be admitted"),
|
||||
)
|
||||
};
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, restarted_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
let blocked =
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", restarted_result, now + Duration::from_millis(1))
|
||||
.expect("a restarted member must revoke the old generation and wait for both writers");
|
||||
assert!(blocked.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.proof.is_none());
|
||||
assert!(state.draining_generation.is_some());
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&first,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&second,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
}
|
||||
|
||||
drop(first);
|
||||
let (_, _, _, still_blocked_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", still_blocked_result, now + Duration::from_millis(2),)
|
||||
.is_some(),
|
||||
"one remaining writer must keep the successor generation closed"
|
||||
);
|
||||
|
||||
drop(second);
|
||||
let (_, _, _, admitted_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", admitted_result, now + Duration::from_millis(3),)
|
||||
.is_none(),
|
||||
"the restarted generation may publish only after every old writer drains"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_fresh_observation_closes_the_polling_window() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, original_result) =
|
||||
cross_pool_fence_policy_results(original_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
|
||||
"the periodic cache has not observed the restart yet"
|
||||
);
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
&restarted_peers,
|
||||
now,
|
||||
));
|
||||
|
||||
let (_, _, _, downgraded) = cross_pool_fence_policy_results(original_peers, 4);
|
||||
assert!(
|
||||
downgraded.is_err(),
|
||||
"a synchronous observation of a downgraded peer must fail before any cached proof can authorize a write"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_invalid_token_skips_fleet_observation() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
revoke_fleet_capability_proof(&slot);
|
||||
|
||||
assert!(
|
||||
!legacy_transition_state_reconcile_fleet_proof_matches_with_observer(&slot, &admitted, "topology-a", || async {
|
||||
panic!("an invalid local generation must not trigger a fleet observation");
|
||||
},)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_membership_and_topology_changes_revoke_authority() {
|
||||
let now = Instant::now();
|
||||
for replacement in [
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]),
|
||||
] {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let original = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(replacement), now + Duration::from_millis(1),)
|
||||
.is_some(),
|
||||
"membership or process-epoch replacement must wait for the admitted writer"
|
||||
);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
}
|
||||
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(BTreeMap::new()), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original topology should admit reconciliation")
|
||||
};
|
||||
mark_fleet_capability_topology_conflict(&slot);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.topology_conflict);
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_capability_downgrade_fails_closed() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, compatible_result) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", compatible_result, now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("v5 should admit reconciliation")
|
||||
};
|
||||
|
||||
let (_, _, _, downgraded_result) =
|
||||
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION - 1);
|
||||
let err = publish_fleet_capability_probe_result(&slot, "topology-a", downgraded_result, now + Duration::from_millis(1))
|
||||
.expect("a v4 member must revoke reconcile authority");
|
||||
assert!(err.to_string().contains("reconcile policy capability version is unsupported"));
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.proof.is_none());
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
assert!(
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now + Duration::from_millis(1),)
|
||||
.is_none(),
|
||||
"a downgraded fleet must remain inspect-only"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
@@ -3959,57 +3539,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_probe_rejects_missing_or_unreachable_members() {
|
||||
let missing = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let missing_err = missing
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing member slot must prevent reconcile capability proof");
|
||||
assert!(missing_err.to_string().contains("incomplete"));
|
||||
|
||||
let unreachable = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let unreachable_err = unreachable
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable member must prevent reconcile capability proof");
|
||||
assert!(unreachable_err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let (peers, minimum_version) = notification_sys
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect("a single-node capability probe should complete");
|
||||
assert!(peers.is_empty());
|
||||
assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
|
||||
assert!(
|
||||
reconcile_result.is_err(),
|
||||
"the current node must not self-authorize reconcile before the conditional writer lands"
|
||||
);
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -21,7 +21,6 @@ pub mod tier_gen;
|
||||
pub mod tier_handlers;
|
||||
pub(crate) mod tier_mutation_intent;
|
||||
pub mod tier_mutation_peer;
|
||||
pub(crate) mod tier_probe_intent;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
|
||||
@@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.precondition().await?;
|
||||
let barrier = self.inner.get_barrier.lock().await.take();
|
||||
if let Some(barrier) = barrier {
|
||||
@@ -719,9 +719,6 @@ impl WarmBackend for MockWarmBackend {
|
||||
let Some(stored) = objects.get(object) else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
|
||||
};
|
||||
if !rv.is_empty() && stored.remote_version_id != rv {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
|
||||
}
|
||||
let bytes = &stored.bytes;
|
||||
|
||||
let start = opts.start_offset.max(0) as usize;
|
||||
|
||||
@@ -2346,10 +2346,6 @@ impl WarmBackend for SharedWarmBackendProxy {
|
||||
self.0.probe_transition_candidate(object).await
|
||||
}
|
||||
|
||||
async fn probe_transition_version(&self, object: &str, remote_version_id: &str) -> io::Result<TransitionCandidateProbe> {
|
||||
self.0.probe_transition_version(object, remote_version_id).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> io::Result<bool> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
@@ -2462,15 +2458,6 @@ impl TierOperationLease {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_transition_version(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version_id: &str,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
self.validate_remote_version_id(remote_version_id)?;
|
||||
self.inner.driver.probe_transition_version(object, remote_version_id).await
|
||||
}
|
||||
|
||||
pub(crate) fn is_current_generation(&self) -> bool {
|
||||
lock_unpoisoned(&self.runtime)
|
||||
.generations
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
|
||||
|
||||
@@ -143,7 +145,7 @@ mod tests {
|
||||
|
||||
assert_eq!(creds.access_key, "access");
|
||||
assert_eq!(creds.secret_key, "secret");
|
||||
assert_eq!(creds.creds_json.as_slice(), service_account);
|
||||
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
|
||||
|
||||
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
|
||||
assert_eq!(wire["access"], "access");
|
||||
@@ -160,7 +162,7 @@ mod tests {
|
||||
.expect("the former RustFS field names and byte-array encoding should remain readable");
|
||||
assert_eq!(legacy.access_key, "legacy-access");
|
||||
assert_eq!(legacy.secret_key, "legacy-secret");
|
||||
assert_eq!(legacy.creds_json.as_slice(), service_account);
|
||||
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -460,7 +460,6 @@ where
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -557,7 +556,6 @@ where
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -40,7 +40,6 @@ use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_error_response::to_error_response,
|
||||
api_put_object::{AdvancedPutOptions, PutObjectOptions},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
@@ -49,14 +48,11 @@ use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
|
||||
use s3s::header::{
|
||||
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
|
||||
X_AMZ_STORAGE_CLASS,
|
||||
};
|
||||
use s3s::{
|
||||
S3ErrorCode,
|
||||
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -145,42 +141,6 @@ pub trait WarmBackend {
|
||||
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
Ok(TransitionCandidateProbe::Unsupported)
|
||||
}
|
||||
async fn probe_transition_version(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version_id: &str,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"an exact tier probe requires a remote version ID",
|
||||
));
|
||||
}
|
||||
self.validate_remote_version_id(remote_version_id)?;
|
||||
match self
|
||||
.get(
|
||||
object,
|
||||
remote_version_id,
|
||||
WarmBackendGetOpts {
|
||||
start_offset: 0,
|
||||
length: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
|
||||
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
|
||||
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
|
||||
}
|
||||
Err(err)
|
||||
if err.kind() == std::io::ErrorKind::NotFound
|
||||
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
|
||||
{
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error>;
|
||||
}
|
||||
|
||||
@@ -477,17 +437,6 @@ impl WarmBackend for MeteredWarmBackend {
|
||||
Self::record(TierRequestOperation::Probe, result)
|
||||
}
|
||||
|
||||
async fn probe_transition_version(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version_id: &str,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
Self::record(
|
||||
TierRequestOperation::Probe,
|
||||
self.inner.probe_transition_version(object, remote_version_id).await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
Self::record(TierRequestOperation::InUse, self.inner.in_use().await)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
@@ -144,11 +146,11 @@ pub struct WarmBackendGCS {
|
||||
|
||||
impl WarmBackendGCS {
|
||||
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.creds.is_empty() {
|
||||
if conf.creds == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket.is_empty() {
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
@@ -193,11 +195,11 @@ impl WarmBackendGCS {
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
if self.prefix.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{}/{}", self.prefix, object)
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,7 +223,7 @@ impl WarmBackend for WarmBackendGCS {
|
||||
let bucket = gcs_bucket_resource_name(&self.bucket);
|
||||
let Ok(res) = Box::pin(
|
||||
self.client
|
||||
.write_object(&bucket, self.get_dest(object), Bytes::from(d))
|
||||
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
|
||||
.send_buffered(),
|
||||
)
|
||||
.await
|
||||
@@ -238,7 +240,7 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let bucket = gcs_bucket_resource_name(&self.bucket);
|
||||
let mut req = self.client.read_object(&bucket, self.get_dest(object));
|
||||
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
|
||||
let mut max_response_bytes = None;
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -529,10 +529,6 @@ mod tests {
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
];
|
||||
let mut requests = Vec::new();
|
||||
for response in responses {
|
||||
@@ -626,52 +622,15 @@ mod tests {
|
||||
.await
|
||||
.expect_err("an authorization failure must not be mistaken for a missing key");
|
||||
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate("delete-marker-hidden")
|
||||
.await
|
||||
.expect("a current delete marker should hide the data version"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_version("delete-marker-hidden", "historical-version")
|
||||
.await
|
||||
.expect("the stored historical version should be probed exactly"),
|
||||
TransitionCandidateProbe::VersionedPresent("historical-version".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_version("delete-marker-hidden", "missing-version")
|
||||
.await
|
||||
.expect("a missing exact version should be classified"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_version("missing-object", "historical-version")
|
||||
.await
|
||||
.expect("a missing key for an exact version probe should be classified"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
|
||||
let requests = fixture.await.expect("candidate fixture should join");
|
||||
for request in &requests[..6] {
|
||||
for request in requests {
|
||||
let request = request.to_ascii_lowercase();
|
||||
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
|
||||
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
|
||||
assert!(!request.contains("?versioning"));
|
||||
assert!(!request.contains("?versions"));
|
||||
}
|
||||
for request in &requests[6..] {
|
||||
let request = request.to_ascii_lowercase();
|
||||
assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET");
|
||||
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
|
||||
}
|
||||
assert!(!requests[5].to_ascii_lowercase().contains("versionid="));
|
||||
assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version"));
|
||||
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
}
|
||||
|
||||
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,385 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
|
||||
//!
|
||||
//! Disk scheduling, coalescing, cancellation, and late shard materialization
|
||||
//! remain with their existing owners; this module only classifies observations.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||
};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
|
||||
use crate::set_disk::file_info_is_valid_for_metadata;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
|
||||
pub(in crate::set_disk) reason: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) total_disks: usize,
|
||||
pub(in crate::set_disk) default_parity_count: usize,
|
||||
pub(in crate::set_disk) allow_early_stop: bool,
|
||||
pub(in crate::set_disk) valid_responses: usize,
|
||||
pub(in crate::set_disk) not_found_responses: usize,
|
||||
pub(in crate::set_disk) version_not_found_responses: usize,
|
||||
pub(in crate::set_disk) ignored_errors: usize,
|
||||
pub(in crate::set_disk) hard_errors: usize,
|
||||
pub(in crate::set_disk) candidate: Option<FileInfo>,
|
||||
pub(in crate::set_disk) candidate_votes: usize,
|
||||
// Bitset of shard indexes whose metadata matches the candidate. Erasure
|
||||
// layouts are capped at 16 shards, so this stays allocation-free on the
|
||||
// GET metadata hot path.
|
||||
candidate_shard_mask: u16,
|
||||
pub(in crate::set_disk) conflicting_metadata: bool,
|
||||
pub(in crate::set_disk) delete_marker_seen: bool,
|
||||
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
|
||||
pub(in crate::set_disk) delete_marker_votes: usize,
|
||||
pub(in crate::set_disk) requested_version_id: String,
|
||||
pub(in crate::set_disk) matching_version_votes: usize,
|
||||
}
|
||||
|
||||
impl MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
|
||||
Self {
|
||||
total_disks,
|
||||
default_parity_count,
|
||||
allow_early_stop,
|
||||
valid_responses: 0,
|
||||
not_found_responses: 0,
|
||||
version_not_found_responses: 0,
|
||||
ignored_errors: 0,
|
||||
hard_errors: 0,
|
||||
candidate: None,
|
||||
candidate_votes: 0,
|
||||
candidate_shard_mask: 0,
|
||||
conflicting_metadata: false,
|
||||
delete_marker_seen: false,
|
||||
delete_marker_candidates: Vec::new(),
|
||||
delete_marker_votes: 0,
|
||||
requested_version_id: String::new(),
|
||||
matching_version_votes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
|
||||
self.requested_version_id = version_id.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(None, file_info);
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(Some(disk_index), file_info);
|
||||
}
|
||||
|
||||
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
|
||||
if !file_info_is_valid_for_metadata(file_info) {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
self.valid_responses = self.valid_responses.saturating_add(1);
|
||||
|
||||
// Track version match for versioned requests
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& let Some(ref vid) = file_info.version_id
|
||||
&& vid.to_string() == self.requested_version_id
|
||||
{
|
||||
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
|
||||
}
|
||||
|
||||
if file_info.is_canonical_delete_marker() {
|
||||
self.delete_marker_seen = true;
|
||||
if let Some((_, votes)) = self
|
||||
.delete_marker_candidates
|
||||
.iter_mut()
|
||||
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
|
||||
{
|
||||
*votes = votes.saturating_add(1);
|
||||
} else {
|
||||
self.delete_marker_candidates.push((file_info.clone(), 1));
|
||||
}
|
||||
self.delete_marker_votes = self
|
||||
.delete_marker_candidates
|
||||
.iter()
|
||||
.map(|(_, votes)| *votes)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
|
||||
return;
|
||||
}
|
||||
|
||||
match &self.candidate {
|
||||
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
|
||||
self.candidate_votes = self.candidate_votes.saturating_add(1);
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
Some(_) => {
|
||||
self.conflicting_metadata = true;
|
||||
}
|
||||
None => {
|
||||
self.candidate = Some(file_info.clone());
|
||||
self.candidate_votes = 1;
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
|
||||
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
|
||||
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
|
||||
return None;
|
||||
}
|
||||
Some(1u16 << (erasure_index - 1))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
|
||||
self.candidate_read_reserve_target()
|
||||
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
|
||||
let candidate = self.candidate.as_ref()?;
|
||||
Some(
|
||||
candidate
|
||||
.erasure
|
||||
.data_blocks
|
||||
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
|
||||
match err {
|
||||
DiskError::FileNotFound | DiskError::VolumeNotFound => {
|
||||
self.not_found_responses = self.not_found_responses.saturating_add(1);
|
||||
}
|
||||
DiskError::FileVersionNotFound => {
|
||||
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
|
||||
}
|
||||
_ if is_metadata_fanout_ignored_error(err) => {
|
||||
self.ignored_errors = self.ignored_errors.saturating_add(1);
|
||||
}
|
||||
_ => {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.delete_marker_votes >= self.default_write_quorum() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
});
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.candidate
|
||||
.as_ref()
|
||||
.and_then(|candidate| self.candidate_latest_quorum(candidate))
|
||||
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
|
||||
{
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if a versioned request can early-stop because the requested
|
||||
/// version_id has reached quorum across disks.
|
||||
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.requested_version_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self.matching_version_votes >= self.read_quorum_for_version() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
|
||||
if !self.allow_early_stop {
|
||||
return false;
|
||||
}
|
||||
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
|
||||
return true;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
match &self.candidate {
|
||||
Some(candidate) => self
|
||||
.candidate_latest_quorum(candidate)
|
||||
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
|
||||
None => pending >= self.default_write_quorum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the read quorum threshold for version-aware early-stop.
|
||||
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
|
||||
/// `default_parity_count` is set, otherwise requires all disks.
|
||||
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
|
||||
self.missing_response_quorum()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
|
||||
if !self.allow_early_stop {
|
||||
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
|
||||
}
|
||||
if self.conflicting_metadata {
|
||||
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
|
||||
}
|
||||
if self.delete_marker_seen {
|
||||
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
|
||||
}
|
||||
let missing_response_quorum = self.missing_response_quorum();
|
||||
if self.version_not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
|
||||
}
|
||||
if self.not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
|
||||
}
|
||||
if self.hard_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_ERROR;
|
||||
}
|
||||
if self.ignored_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
|
||||
}
|
||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
|
||||
if self.default_parity_count == 0 {
|
||||
return Some(self.total_disks);
|
||||
}
|
||||
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
|
||||
return None;
|
||||
}
|
||||
let data_blocks = candidate.erasure.data_blocks;
|
||||
Some(if data_blocks == candidate.erasure.parity_blocks {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
return self.total_disks;
|
||||
}
|
||||
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
|
||||
if data_blocks == self.default_parity_count {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
self.total_disks
|
||||
} else {
|
||||
self.total_disks / 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
|
||||
left.volume == right.volume
|
||||
&& left.name == right.name
|
||||
&& left.version_id == right.version_id
|
||||
&& left.is_latest == right.is_latest
|
||||
&& left.deleted == right.deleted
|
||||
&& left.mark_deleted == right.mark_deleted
|
||||
&& left.transition_status == right.transition_status
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
&& left.mode == right.mode
|
||||
&& left.written_by_version == right.written_by_version
|
||||
&& left.metadata == right.metadata
|
||||
&& left.replication_state_internal == right.replication_state_internal
|
||||
&& left.parts == right.parts
|
||||
&& left.checksum == right.checksum
|
||||
&& left.versioned == right.versioned
|
||||
&& left.num_versions == right.num_versions
|
||||
&& left.successor_mod_time == right.successor_mod_time
|
||||
&& left.data_dir == right.data_dir
|
||||
&& left.erasure.algorithm == right.erasure.algorithm
|
||||
&& left.erasure.data_blocks == right.erasure.data_blocks
|
||||
&& left.erasure.parity_blocks == right.erasure.parity_blocks
|
||||
&& left.erasure.block_size == right.erasure.block_size
|
||||
&& left.erasure.distribution == right.erasure.distribution
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
|
||||
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
|
||||
}
|
||||
@@ -18,4 +18,3 @@
|
||||
//! duplicating read/write/erasure logic.
|
||||
|
||||
pub(crate) mod io_primitives;
|
||||
mod metadata_quorum;
|
||||
|
||||
@@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
|
||||
@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
use crate::object_api::{
|
||||
NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
|
||||
};
|
||||
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
|
||||
use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
|
||||
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
|
||||
#[cfg(test)]
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
#[cfg(test)]
|
||||
@@ -3548,7 +3548,6 @@ impl SetDisks {
|
||||
(None, None, None)
|
||||
};
|
||||
let mut tmp_cleanup_owned = false;
|
||||
let rollback_receipt = RenameRollbackReceipt::default();
|
||||
let operation = async {
|
||||
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
|
||||
|
||||
@@ -4257,7 +4256,6 @@ impl SetDisks {
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
let commit_tmp_dir = tmp_dir.clone();
|
||||
let commit_rollback_receipt = rollback_receipt.clone();
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_publication_guard = publication_commit_guard.take();
|
||||
@@ -4268,17 +4266,13 @@ impl SetDisks {
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
// remote leases.
|
||||
let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
|
||||
|| commit_decommission_object_lock_guard.is_some()
|
||||
|| commit_publication_guard.is_some();
|
||||
let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& (commit_object_lock_guard.is_some()
|
||||
|| commit_decommission_object_lock_guard.is_some()
|
||||
|| commit_publication_guard.is_some())
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
// Full-tail callers also transfer owned guards to the coordinator:
|
||||
// cancelling their ACK waiter must not cancel an in-flight rename.
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
|| commit_owns_namespace_guard
|
||||
|| commit_allows_early_ack
|
||||
|| commit_bucket_lifecycle_guard.is_some()
|
||||
|| quota_mutation_fence;
|
||||
let commit_write_path_label = write_path.metric_label();
|
||||
@@ -4458,8 +4452,7 @@ impl SetDisks {
|
||||
write_quorum,
|
||||
commit_scanner_publication_lease_tokens.as_ref(),
|
||||
)
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone()),
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone()),
|
||||
)
|
||||
.await;
|
||||
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
|
||||
@@ -4592,11 +4585,6 @@ impl SetDisks {
|
||||
let rename_commit = match rename_result {
|
||||
Ok(commit) => commit,
|
||||
Err(err) => {
|
||||
if commit_rollback_receipt.is_incomplete() {
|
||||
// Incomplete undo retains the staging source and
|
||||
// rollback backup for recovery; cleanup is unsafe.
|
||||
return Err(err.into());
|
||||
}
|
||||
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
|
||||
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
|
||||
} else if issue3031_diag_enabled() {
|
||||
@@ -4629,8 +4617,9 @@ impl SetDisks {
|
||||
request.object_version_id = committed_version_id
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
let heal_set = commit_set.clone();
|
||||
tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
|
||||
let rename_stage_elapsed = rename_stage_start.elapsed();
|
||||
@@ -4896,7 +4885,7 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
});
|
||||
} else if !rollback_receipt.is_incomplete() {
|
||||
} else {
|
||||
// Failure path (quorum loss / rollback): keep the cleanup inline so
|
||||
// a failed PUT never returns while its tmp shards are still on disk
|
||||
// (state-residue hardening tracked by backlog#864 / backlog#898).
|
||||
@@ -17505,69 +17494,27 @@ mod put_object_tmp_cleanup_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn put_object_failure_cleans_tmp_workspace_inline() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
|
||||
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "tmp-clean-missing-bucket";
|
||||
let object = "orphan-object";
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let writer = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("missing-bucket PUT must stage before rename");
|
||||
let staged = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
|
||||
for workspace in staged {
|
||||
let mut entries = tokio::fs::read_dir(&workspace)
|
||||
.await
|
||||
.expect("staged workspace should be readable");
|
||||
let mut shards = 0;
|
||||
while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
|
||||
if entry.file_type().await.expect("staged entry type").is_dir() {
|
||||
let part = tokio::fs::metadata(entry.path().join("part.1"))
|
||||
.await
|
||||
.expect("staging must contain an actual erasure shard");
|
||||
assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
|
||||
shards += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(shards, 1);
|
||||
}
|
||||
assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
|
||||
barrier.release();
|
||||
let err = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("missing-bucket PUT must finish")
|
||||
.expect("PUT task should join")
|
||||
.expect_err("put_object into a missing bucket volume must fail");
|
||||
assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
|
||||
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
// No polling: known pre-publication rejection must clean staging
|
||||
// inline, before PUT returns (backlog#864 / backlog#898).
|
||||
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
leftovers.is_empty(),
|
||||
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
// The bucket volume is never created, so the shards are written into
|
||||
// the tmp workspace and the commit fails at rename_data with a quorum
|
||||
// error — exercising the failure-path cleanup.
|
||||
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
|
||||
let err = set_disks
|
||||
.put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("put_object into a missing bucket volume must fail");
|
||||
|
||||
// No polling: the failure path must clean the tmp workspace inline,
|
||||
// before put_object returns (backlog#864 / backlog#898 hardening).
|
||||
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
leftovers.is_empty(),
|
||||
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
|
||||
);
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -18210,354 +18157,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
|
||||
for disk in disks {
|
||||
disk.make_volume(bucket)
|
||||
.await
|
||||
.expect("completion test bucket should be created");
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe the actual metadata quorum while the remaining rename is parked.
|
||||
/// A completed task count alone can race tasks that have not started yet.
|
||||
async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let mut committed = 0;
|
||||
for disk in disks {
|
||||
match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
|
||||
Ok(_) => committed += 1,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
|
||||
Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
|
||||
}
|
||||
}
|
||||
if committed == 3 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("three disks must publish metadata while the fourth rename remains paused");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for size in [4096, 1024 * 1024] {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-full-tail-cas";
|
||||
let object = "full-tail-cas-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("full-tail PUT must reach the rename barrier");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
|
||||
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(lock_probe.as_mut()).is_pending(),
|
||||
"the owned namespace guard must remain held"
|
||||
);
|
||||
barrier.release();
|
||||
let written = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("full-tail PUT should finish after release")
|
||||
.expect("full-tail PUT task should join")
|
||||
.expect("full-tail PUT must commit");
|
||||
assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(5), lock_probe)
|
||||
.await
|
||||
.expect("same-key lock should be available on return")
|
||||
.expect("same-key lock probe should succeed"),
|
||||
);
|
||||
for disk in &disks {
|
||||
disk.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("successful full-tail PUT must publish on every healthy disk");
|
||||
}
|
||||
drop(barrier);
|
||||
let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut replacement,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: written.etag,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("immediate same-key CAS must acquire the namespace guard");
|
||||
let mut read = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("CAS successor must be immediately readable");
|
||||
let mut body = Vec::new();
|
||||
read.stream.read_to_end(&mut body).await.expect("successor body must drain");
|
||||
assert_eq!(body, b"cas successor");
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-full-tail-heal";
|
||||
let object = "full-tail-heal-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let mut heals = set.capture_test_rename_tail_heals();
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("failed tail must first reach the rename barrier");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("failed tail should drain")
|
||||
.expect("PUT task should join")
|
||||
.expect("a minority tail error must not negate committed quorum");
|
||||
assert_eq!(tasks.running(), 0);
|
||||
let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
|
||||
.await
|
||||
.expect("failed tail must schedule heal")
|
||||
.expect("heal capture must remain connected");
|
||||
assert_eq!(heal.bucket, bucket);
|
||||
assert_eq!(heal.object_prefix.as_deref(), Some(object));
|
||||
let info = set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object must remain readable despite the failed tail");
|
||||
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_rejects_quorum_minus_one() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-full-tail-no-quorum";
|
||||
let object = "full-tail-no-quorum-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
let err = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("draining two successful disks cannot satisfy write quorum three");
|
||||
assert!(
|
||||
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
|
||||
"original quorum error expected: {err}"
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
|
||||
assert!(
|
||||
set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
|
||||
"failed fresh write must not become visible"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
|
||||
use crate::set_disk::core::io_primitives::rollback_fault_injection;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
|
||||
for fault in [
|
||||
rollback_fault_injection::Fault::Io,
|
||||
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
|
||||
] {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-incomplete-undo";
|
||||
let object = "incomplete-undo-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old generation should be completely committed");
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
|
||||
let old = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("old metadata must be readable");
|
||||
let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
|
||||
let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("overwrite must enter the actual rename fan-out before failure injection");
|
||||
barrier.release();
|
||||
let err = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("incomplete undo must return without hanging")
|
||||
.expect("PUT task should join")
|
||||
.expect_err("two renamed disks cannot satisfy write quorum three");
|
||||
assert!(
|
||||
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
|
||||
"original quorum error expected: {err}"
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
|
||||
let leftovers = non_trash_tmp_entries(&dirs).await;
|
||||
assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
|
||||
let backups = dirs
|
||||
.iter()
|
||||
.filter(|dir| {
|
||||
dir.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(old_data_dir.to_string())
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
|
||||
.exists()
|
||||
})
|
||||
.count();
|
||||
assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
|
||||
// The remaining three disks still serve the old generation;
|
||||
// the failed minority must never become an acknowledged write.
|
||||
let mut read = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("old generation must remain readable after incomplete rollback");
|
||||
let mut body = Vec::new();
|
||||
read.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("old generation should stream");
|
||||
assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = RUSTFS_META_BUCKET;
|
||||
let object = "full-tail-cancelled-receipt";
|
||||
// Internal config writes do not own a bucket lifecycle guard. The object
|
||||
// guard alone must keep the full-tail coordinator alive after cancellation.
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("cancelled receipt must first reach the rename barrier");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
|
||||
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(lock_probe.as_mut()).is_pending(),
|
||||
"owned coordinator must retain the namespace guard after waiter cancellation"
|
||||
);
|
||||
barrier.release();
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(30), lock_probe)
|
||||
.await
|
||||
.expect("cancelled coordinator must eventually release its guard")
|
||||
.expect("post-commit lock probe should succeed"),
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
|
||||
for disk in &disks {
|
||||
disk.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("caller cancellation must not interrupt committed receipt materialization");
|
||||
}
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
|
||||
@@ -18585,7 +18184,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -18611,18 +18209,7 @@ mod put_object_tmp_cleanup_tests {
|
||||
put.await
|
||||
.expect("no-lock PUT task should join")
|
||||
.expect("no-lock PUT should commit after the rename tail releases");
|
||||
let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(lock_probe.as_mut()).is_pending(),
|
||||
"full-tail PUT must not release the caller's outer guard"
|
||||
);
|
||||
drop(outer_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(5), lock_probe)
|
||||
.await
|
||||
.expect("outer owner releasing its guard should unblock the probe")
|
||||
.expect("post-outer-guard probe should succeed"),
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ use super::{
|
||||
};
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::object_api::WriteCompletion;
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
@@ -26,24 +25,19 @@ use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
set_disks
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("object metadata should resolve");
|
||||
let generation = set_disks
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata generation should be active");
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
|
||||
return key;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("metadata read should publish the generation under test")
|
||||
set_disks
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("object metadata should resolve");
|
||||
let generation = set_disks
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata generation should be active");
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
assert!(
|
||||
set_disks.get_object_metadata_cache.get(&key).await.is_some(),
|
||||
"metadata read should publish the generation under test"
|
||||
);
|
||||
key
|
||||
}
|
||||
|
||||
async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) {
|
||||
@@ -66,17 +60,8 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
// Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
|
||||
let original = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let source_generation = prime_metadata_generation(&set_disks, bucket, object).await;
|
||||
@@ -179,17 +164,8 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
// Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
|
||||
let original = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
|
||||
|
||||
@@ -864,11 +864,6 @@ mod tests {
|
||||
save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
|
||||
},
|
||||
tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
|
||||
tier_probe_intent::{
|
||||
TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
|
||||
delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
|
||||
save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend},
|
||||
},
|
||||
set_disk::SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
|
||||
@@ -2979,33 +2974,6 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
|
||||
|
||||
fn decommission_retry_fault_hook(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
faults: Arc<AtomicUsize>,
|
||||
) -> crate::core::pools::DecommissionTestFaultDecision {
|
||||
let target_bucket = bucket.to_string();
|
||||
let target_object = object.to_string();
|
||||
Arc::new(move |stage, bucket, object, _attempt, succeeded| {
|
||||
if !succeeded
|
||||
|| stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|
||||
|| bucket != target_bucket
|
||||
|| object != target_object
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Entry retries reset the local attempt; real copy errors can skip
|
||||
// successful attempts. Only injected faults spend this global budget.
|
||||
faults
|
||||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
|
||||
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
|
||||
.then_some(faults.saturating_add(1))
|
||||
})
|
||||
.is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
async fn seed_decommission_source(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
@@ -5147,33 +5115,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
|
||||
for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] {
|
||||
let faults = Arc::new(AtomicUsize::new(0));
|
||||
let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
|
||||
|
||||
for (stage, bucket, object, succeeded) in [
|
||||
("other-stage", "bucket", "object", true),
|
||||
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
|
||||
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
|
||||
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
|
||||
] {
|
||||
assert!(!hook(stage, bucket, object, 1, succeeded));
|
||||
}
|
||||
assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
|
||||
|
||||
for (index, attempt) in attempts.into_iter().enumerate() {
|
||||
assert_eq!(
|
||||
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true),
|
||||
index < 2,
|
||||
"attempts={attempts:?}, index={index}"
|
||||
);
|
||||
}
|
||||
assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
|
||||
@@ -5268,8 +5209,31 @@ mod tests {
|
||||
));
|
||||
|
||||
let ordinary_faults = Arc::new(AtomicUsize::new(0));
|
||||
let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
|
||||
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
|
||||
let ordinary_faults_for_hook = Arc::clone(&ordinary_faults);
|
||||
let fault_bucket = other_bucket.clone();
|
||||
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
|
||||
move |stage, bucket, object, attempt, succeeded| {
|
||||
let candidate = succeeded
|
||||
&& stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|
||||
&& bucket == fault_bucket.as_str()
|
||||
&& object == other_object;
|
||||
if !candidate {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Keep the fault budget global across any
|
||||
// entry-level re-list; its inner attempt counter
|
||||
// restarts after SourceChanged.
|
||||
ordinary_faults_for_hook
|
||||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
|
||||
let next_fault = faults.saturating_add(1);
|
||||
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)
|
||||
&& attempt == next_fault)
|
||||
.then_some(next_fault)
|
||||
})
|
||||
.is_ok()
|
||||
},
|
||||
));
|
||||
|
||||
let rx = CancellationToken::new();
|
||||
let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
|
||||
@@ -8076,15 +8040,10 @@ mod tests {
|
||||
);
|
||||
assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
|
||||
|
||||
let full_tail = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
|
||||
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone())
|
||||
.await
|
||||
.expect("second page receipt should restore");
|
||||
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
|
||||
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("second page receipt should corrupt deterministically");
|
||||
let corrupt = store
|
||||
@@ -11611,7 +11570,6 @@ mod tests {
|
||||
pool_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
minio_unversioned: bool,
|
||||
) {
|
||||
for disk_index in 0..4 {
|
||||
let metadata_path =
|
||||
@@ -11645,11 +11603,6 @@ mod tests {
|
||||
] {
|
||||
rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
|
||||
}
|
||||
if minio_unversioned {
|
||||
object_meta
|
||||
.meta_sys
|
||||
.insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
|
||||
}
|
||||
*shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
|
||||
.expect("legacy transitioned version should re-encode");
|
||||
}
|
||||
@@ -11660,152 +11613,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn read_store_body(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Vec<u8> {
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
|
||||
.await
|
||||
.expect("object reader should open");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("object body should drain");
|
||||
body
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let bucket = "legacy-unknown-unversioned-read-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("legacy source bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let source = store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy source should be written");
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("legacy source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("legacy source should transition");
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
|
||||
backend.clear_op_log().await;
|
||||
|
||||
let opts = ObjectOptions {
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
};
|
||||
let head = store
|
||||
.get_object_info(bucket, object, &opts)
|
||||
.await
|
||||
.expect("legacy transitioned HEAD should use local metadata");
|
||||
assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(head.transitioned_object.version_id.is_empty());
|
||||
assert_eq!(
|
||||
head.user_defined
|
||||
.get("x-minio-internal-transitioned-versionID")
|
||||
.map(String::as_str),
|
||||
Some(""),
|
||||
"the MinIO empty version-key provenance must survive xl.meta decoding"
|
||||
);
|
||||
assert!(
|
||||
!rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&head.user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
),
|
||||
"the compatibility read must not synthesize version-state metadata"
|
||||
);
|
||||
|
||||
let full_body = read_store_body(&store, bucket, object, None, &opts).await;
|
||||
assert_eq!(full_body, payload);
|
||||
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 7,
|
||||
end: 38,
|
||||
};
|
||||
let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
|
||||
assert_eq!(ranged_body, &payload[7..=38]);
|
||||
|
||||
let after_read = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("legacy metadata should remain readable after GET")
|
||||
.expect("legacy object metadata should remain on disk")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
|
||||
.expect("legacy transitioned source should remain visible after GET");
|
||||
assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(after_read.transition_version.is_none());
|
||||
assert!(after_read.transition_version_id.is_none());
|
||||
assert_eq!(
|
||||
after_read
|
||||
.metadata
|
||||
.get("x-minio-internal-transitioned-versionID")
|
||||
.map(String::as_str),
|
||||
Some(""),
|
||||
"the MinIO empty version-key provenance must remain after GET and Range GET"
|
||||
);
|
||||
assert!(
|
||||
!rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&after_read.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
),
|
||||
"the compatibility read must remain side-effect free"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
backend.op_log().await,
|
||||
vec![
|
||||
MockWarmOp::Probe {
|
||||
object: after_read.transitioned_objname.clone(),
|
||||
},
|
||||
MockWarmOp::Get {
|
||||
object: after_read.transitioned_objname.clone(),
|
||||
},
|
||||
MockWarmOp::Probe {
|
||||
object: after_read.transitioned_objname.clone(),
|
||||
},
|
||||
MockWarmOp::Get {
|
||||
object: after_read.transitioned_objname,
|
||||
},
|
||||
],
|
||||
"legacy reads should probe before each unversioned GET and never mutate local metadata"
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -11846,7 +11653,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("legacy source should transition");
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await;
|
||||
let legacy = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
@@ -12987,7 +12794,7 @@ mod tests {
|
||||
.expect("merge-loser source should transition");
|
||||
copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
|
||||
}
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await;
|
||||
backend.set_remove_failure(true);
|
||||
store.pools[1]
|
||||
.delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
|
||||
@@ -17054,10 +16861,6 @@ mod tests {
|
||||
.find(|version| version.version_id == history.version_id)
|
||||
.expect("transitioned history should exist");
|
||||
transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut transitioned.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
);
|
||||
metadata
|
||||
.add_version(transitioned)
|
||||
.expect("unknown state should replace the transitioned version");
|
||||
@@ -17393,147 +17196,6 @@ mod tests {
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
|
||||
let probe_id = uuid::Uuid::new_v4();
|
||||
let creator_epoch = uuid::Uuid::new_v4();
|
||||
let initial = TierProbeIntent {
|
||||
probe_id,
|
||||
revision: 1,
|
||||
state: TierProbeIntentState::UploadOutcomeUnknown,
|
||||
operation: TierProbeOperationIdentity::Verify {
|
||||
config_etag: "config-etag".to_string(),
|
||||
backend_identity: [1; 32],
|
||||
},
|
||||
tier_name: "COLD-A".to_string(),
|
||||
destination_id: [1; 32],
|
||||
probe_object: format!("rustfs-tier-probe-{probe_id}"),
|
||||
creator_id: "node-a".to_string(),
|
||||
creator_epoch,
|
||||
created_at_unix_nanos: 1_780_000_000_000_000_000,
|
||||
owner: TierProbeOwnerFence {
|
||||
owner_id: "node-a".to_string(),
|
||||
owner_epoch: creator_epoch,
|
||||
not_after_unix_nanos: 1_780_000_900_000_000_000,
|
||||
},
|
||||
remote_version: TierProbeRemoteVersion::default(),
|
||||
};
|
||||
|
||||
save_tier_probe_intent_record_if_absent(store.clone(), &initial)
|
||||
.await
|
||||
.expect("initial probe intent should persist with create-only semantics");
|
||||
let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
|
||||
.await
|
||||
.expect_err("duplicate create must fail closed");
|
||||
assert!(matches!(duplicate, Error::PreconditionFailed));
|
||||
|
||||
let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("initial probe intent should load with an ETag");
|
||||
assert_eq!(observed_initial.intent(), &initial);
|
||||
|
||||
let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
|
||||
.await
|
||||
.expect_err("nonterminal evidence must not be deleted");
|
||||
assert!(nonterminal_delete.to_string().contains("must be terminal"));
|
||||
|
||||
let mut fabricated_current_intent = initial.clone();
|
||||
fabricated_current_intent.tier_name = "COLD-B".to_string();
|
||||
let mut fabricated_successor = fabricated_current_intent.clone();
|
||||
fabricated_successor
|
||||
.advance(
|
||||
TierProbeIntentState::Uploaded,
|
||||
TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
|
||||
)
|
||||
.expect("fabricated successor should be internally valid");
|
||||
let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
|
||||
let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
|
||||
.await
|
||||
.expect_err("a live ETag must not authorize a different caller record");
|
||||
assert!(matches!(crossed_cas, Error::PreconditionFailed));
|
||||
assert_eq!(
|
||||
load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("crossed CAS must retain the authoritative record")
|
||||
.intent(),
|
||||
&initial
|
||||
);
|
||||
|
||||
let mut fabricated_terminal_intent = fabricated_current_intent;
|
||||
fabricated_terminal_intent
|
||||
.advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
|
||||
.expect("fabricated terminal should be internally valid");
|
||||
let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
|
||||
let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
|
||||
.await
|
||||
.expect_err("a live ETag must not delete for a different caller record");
|
||||
assert!(matches!(crossed_delete, Error::PreconditionFailed));
|
||||
assert_eq!(
|
||||
load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("crossed delete must retain the authoritative record")
|
||||
.intent(),
|
||||
&initial
|
||||
);
|
||||
|
||||
let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
|
||||
let mut uploaded = observed_initial.intent().clone();
|
||||
uploaded
|
||||
.advance(TierProbeIntentState::Uploaded, remote_version.clone())
|
||||
.expect("known PUT result should advance");
|
||||
save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
|
||||
.await
|
||||
.expect("the matching initial ETag should admit one successor");
|
||||
|
||||
let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
|
||||
.await
|
||||
.expect_err("a consumed ETag must not overwrite the current generation");
|
||||
assert!(matches!(stale_cas, Error::PreconditionFailed));
|
||||
|
||||
let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("uploaded generation should load");
|
||||
assert_eq!(observed_uploaded.intent(), &uploaded);
|
||||
let mut cleanup = observed_uploaded.intent().clone();
|
||||
cleanup
|
||||
.advance(TierProbeIntentState::CleanupPending, remote_version.clone())
|
||||
.expect("known candidate should become cleanup-pending");
|
||||
save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
|
||||
.await
|
||||
.expect("cleanup generation should persist by exact ETag");
|
||||
|
||||
let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("cleanup generation should load");
|
||||
let mut completed = observed_cleanup.intent().clone();
|
||||
completed
|
||||
.advance(TierProbeIntentState::Completed, remote_version)
|
||||
.expect("exact cleanup should become terminal");
|
||||
save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
|
||||
.await
|
||||
.expect("terminal generation should persist by exact ETag");
|
||||
|
||||
let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
|
||||
let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
|
||||
.await
|
||||
.expect_err("a stale ETag must not delete terminal evidence");
|
||||
assert!(matches!(stale_delete, Error::PreconditionFailed));
|
||||
|
||||
let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("terminal generation should remain after stale delete");
|
||||
assert_eq!(observed_completed.intent(), &completed);
|
||||
delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
|
||||
.await
|
||||
.expect("the exact terminal ETag should delete the record");
|
||||
assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
|
||||
@@ -425,7 +425,7 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use object::DeleteAfterObjectLockSnapshotBarrier;
|
||||
pub(crate) use object::{
|
||||
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
|
||||
|
||||
@@ -297,20 +297,6 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String {
|
||||
decoded.map(str::to_owned).unwrap_or_else(|| {
|
||||
if raw.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
String::from_utf8_lossy(raw).into_owned()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_transition_version_metadata_key(key: &str) -> bool {
|
||||
strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
|
||||
}
|
||||
|
||||
fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
|
||||
let valid = match state {
|
||||
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
||||
@@ -380,26 +366,14 @@ impl<'a> DerivedInternalMetadata<'a> {
|
||||
}
|
||||
*slot = Some(value.as_slice());
|
||||
}
|
||||
fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result<Option<&'a [u8]>> {
|
||||
if let (Some(canonical), Some(legacy)) = (canonical, legacy)
|
||||
&& canonical != legacy
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(canonical.or(legacy))
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
checksum: canonical.checksum.or(legacy.checksum),
|
||||
part_checksums: canonical.part_checksums.or(legacy.part_checksums),
|
||||
transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?,
|
||||
transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?,
|
||||
transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?,
|
||||
transitioned_version_state: merge_consistent(
|
||||
canonical.transitioned_version_state,
|
||||
legacy.transitioned_version_state,
|
||||
)?,
|
||||
transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?,
|
||||
transition_status: canonical.transition_status.or(legacy.transition_status),
|
||||
transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object),
|
||||
transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version),
|
||||
transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state),
|
||||
transition_tier: canonical.transition_tier.or(legacy.transition_tier),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -464,14 +438,8 @@ impl FileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
fn set_transition_version_state(
|
||||
meta_sys: &mut HashMap<String, Vec<u8>>,
|
||||
state: TransitionVersionState,
|
||||
source_metadata: &HashMap<String, String>,
|
||||
) {
|
||||
if state == TransitionVersionState::Unknown
|
||||
&& !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE)
|
||||
{
|
||||
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
|
||||
if state == TransitionVersionState::Unknown {
|
||||
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
||||
} else {
|
||||
insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
|
||||
@@ -2675,11 +2643,6 @@ impl MetaObject {
|
||||
if derived_metadata.transitioned_version_state.is_some() {
|
||||
validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
|
||||
}
|
||||
for (key, value) in &self.meta_sys {
|
||||
if is_transition_version_metadata_key(key) {
|
||||
metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref()));
|
||||
}
|
||||
}
|
||||
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
let transition_tier = derived_metadata
|
||||
.transition_tier
|
||||
@@ -2726,7 +2689,7 @@ impl MetaObject {
|
||||
} else {
|
||||
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
}
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata);
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
|
||||
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
|
||||
@@ -2867,7 +2830,7 @@ impl From<FileInfo> for MetaObject {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
@@ -3022,12 +2985,6 @@ impl MetaDeleteMarker {
|
||||
fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
|
||||
fi.transition_version =
|
||||
transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
|
||||
for (key, value) in &self.meta_sys {
|
||||
if is_transition_version_metadata_key(key) {
|
||||
fi.metadata
|
||||
.insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref()));
|
||||
}
|
||||
}
|
||||
fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
if derived_metadata.transitioned_version_state.is_some() {
|
||||
validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
|
||||
@@ -3195,7 +3152,7 @@ impl From<FileInfo> for MetaDeleteMarker {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() || value.tier_free_version() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
}
|
||||
if !value.transition_tier.is_empty() {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
||||
@@ -4617,7 +4574,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4629,10 +4585,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert!(
|
||||
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
|
||||
"nil UUID bytes must remain distinguishable from an empty MinIO version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4646,7 +4598,6 @@ mod tests {
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4686,36 +4637,6 @@ mod tests {
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
TransitionVersionState::Unknown.as_str().to_string(),
|
||||
);
|
||||
let fi = FileInfo {
|
||||
transition_status: "complete".to_string(),
|
||||
transition_version_state: TransitionVersionState::Unknown,
|
||||
metadata,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let object = MetaObject::from(fi);
|
||||
assert_eq!(
|
||||
get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE),
|
||||
Some(b"unknown".as_slice())
|
||||
);
|
||||
let decoded = object
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("explicit unknown state should decode");
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,),
|
||||
Some("unknown")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
|
||||
let id = sample_version_id();
|
||||
@@ -4832,10 +4753,6 @@ mod tests {
|
||||
.expect("invalid transition version bytes must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
assert!(
|
||||
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
|
||||
"invalid raw bytes must remain distinguishable from an empty MinIO version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4878,10 +4795,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("nil tier version should remain an absent remote version");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert!(
|
||||
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
|
||||
"nil UUID bytes must remain distinguishable from an empty MinIO version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4899,7 +4812,6 @@ mod tests {
|
||||
.expect("legacy binary UUID tier version should decode");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4998,23 +4910,6 @@ mod tests {
|
||||
assert_eq!(err, Error::FileCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() {
|
||||
let sys = HashMap::from([
|
||||
(
|
||||
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"),
|
||||
b"unknown".to_vec(),
|
||||
),
|
||||
("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()),
|
||||
]);
|
||||
|
||||
let err = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect_err("mixed-case transition state aliases must agree");
|
||||
|
||||
assert_eq!(err, Error::FileCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
|
||||
let object = FileMetaVersionHeader {
|
||||
|
||||
@@ -52,9 +52,6 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock<AsyncMutex<()>> = LazyLock::new(||
|
||||
|
||||
mod stream;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use stream::checkpoint_fixture_partial_return;
|
||||
|
||||
pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request};
|
||||
pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket};
|
||||
use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle};
|
||||
|
||||
@@ -1017,48 +1017,6 @@ fn finish_remote_scanner_stream(
|
||||
#[cfg(test)]
|
||||
const TEST_NEXT_CYCLE: u64 = 11;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) {
|
||||
let request_id = Uuid::new_v4();
|
||||
let writer_auth = FrameAuthenticator::for_test(request_id);
|
||||
let reader_auth = FrameAuthenticator::for_test(request_id);
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&writer_auth,
|
||||
&mut 0,
|
||||
&RemoteScannerFrame::terminal(
|
||||
RemoteScannerProgress {
|
||||
objects_scanned: progress.0,
|
||||
directories_started: progress.1,
|
||||
entries_visited,
|
||||
},
|
||||
RemoteScannerFrameResult::Partial,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint partial frame must encode");
|
||||
let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0)
|
||||
.await
|
||||
.expect("checkpoint progress frame must authenticate");
|
||||
assert_eq!(frame.progress.entries_visited, entries_visited);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
|
||||
let result = consume_remote_scanner_stream(
|
||||
std::io::Cursor::new(bytes),
|
||||
parent,
|
||||
budget.clone(),
|
||||
"bucket",
|
||||
DataUsageCacheSource::new(0, 0),
|
||||
DataUsageScanPlanDigest([17; 32]),
|
||||
reader_auth,
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint partial frame must decode");
|
||||
assert!(matches!(result, RemoteScannerOutcome::Partial));
|
||||
assert_eq!(budget.progress(), progress);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn consume_remote_scanner_stream<R>(
|
||||
reader: R,
|
||||
|
||||
@@ -24,8 +24,6 @@ use std::io::Write;
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::Mutex;
|
||||
|
||||
mod checkpoint_fixture;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
*SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
|
||||
@@ -1,410 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
|
||||
use crate::storage_api::scanner_io::ObjectIO;
|
||||
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
|
||||
const STATIC_OBJECTS: u64 = 24;
|
||||
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
|
||||
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
|
||||
|
||||
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
|
||||
#[derive(Debug)]
|
||||
struct FixtureStore {
|
||||
root: tempfile::TempDir,
|
||||
reject_save: AtomicBool,
|
||||
}
|
||||
|
||||
impl FixtureStore {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
root: tempfile::tempdir().expect("checkpoint fixture storage directory"),
|
||||
reject_save: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn path(&self, object: &str) -> std::path::PathBuf {
|
||||
assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp")));
|
||||
self.root
|
||||
.path()
|
||||
.join(if object.ends_with(".bkp") { "backup" } else { "main" })
|
||||
}
|
||||
|
||||
async fn strict_load(&self) -> DataUsageCache {
|
||||
let bytes = tokio::fs::read(self.root.path().join("main"))
|
||||
.await
|
||||
.expect("saved checkpoint fixture must exist");
|
||||
decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root")
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectIO for FixtureStore {
|
||||
type Error = crate::EcstoreError;
|
||||
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
|
||||
type HeaderMap = http::HeaderMap;
|
||||
type ObjectOptions = crate::ScannerObjectOptions;
|
||||
type ObjectInfo = crate::ScannerObjectInfo;
|
||||
type GetObjectReader = crate::ScannerGetObjectReader;
|
||||
type PutObjectReader = crate::ScannerPutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
_range: Option<Self::RangeSpec>,
|
||||
_headers: Self::HeaderMap,
|
||||
_options: &Self::ObjectOptions,
|
||||
) -> crate::EcstoreResult<Self::GetObjectReader> {
|
||||
let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
crate::EcstoreError::FileNotFound
|
||||
} else {
|
||||
crate::EcstoreError::from(error)
|
||||
}
|
||||
})?;
|
||||
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
|
||||
Ok(crate::ScannerGetObjectReader {
|
||||
stream: Box::new(Cursor::new(bytes)),
|
||||
object_info: crate::ScannerObjectInfo {
|
||||
etag: Some("fixture".into()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
data: &mut Self::PutObjectReader,
|
||||
options: &Self::ObjectOptions,
|
||||
) -> crate::EcstoreResult<Self::ObjectInfo> {
|
||||
if self.reject_save.load(Ordering::SeqCst) {
|
||||
return Err(crate::EcstoreError::PreconditionFailed);
|
||||
}
|
||||
let path = self.path(object);
|
||||
let exists = tokio::fs::try_exists(&path).await?;
|
||||
let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS");
|
||||
if (exists && preconditions.if_none_match_value() == Some("*"))
|
||||
|| (!exists && preconditions.if_match_value().is_some())
|
||||
|| (exists && preconditions.if_match_value() != Some("fixture"))
|
||||
{
|
||||
return Err(crate::EcstoreError::PreconditionFailed);
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
(&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?;
|
||||
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
|
||||
tokio::fs::write(path, bytes).await?;
|
||||
Ok(crate::ScannerObjectInfo {
|
||||
etag: Some("fixture".into()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for FixtureStore {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_options: crate::ScannerObjectOptions,
|
||||
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
|
||||
Err(crate::EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_fixture(bytes: &[u8]) -> Result<DataUsageCache, &'static str> {
|
||||
if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") {
|
||||
return Err("missing or oversized checkpoint fixture");
|
||||
}
|
||||
let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?;
|
||||
if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() {
|
||||
return Err("checkpoint fixture has no valid bucket root");
|
||||
}
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
fn retained(cache: &DataUsageCache) -> u64 {
|
||||
assert!(
|
||||
!cache.root().is_some_and(|root| root.compacted),
|
||||
"a compacted bucket root cannot prove static-prefix coverage"
|
||||
);
|
||||
cache
|
||||
.checked_flatten("bucket/static")
|
||||
.map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64"))
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum CoverageDiagnosis {
|
||||
Progress,
|
||||
NoNewWork,
|
||||
LostAtPrepare,
|
||||
LostAtReload,
|
||||
WalkWithoutRetention,
|
||||
}
|
||||
|
||||
fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis {
|
||||
if reloaded < scanned {
|
||||
CoverageDiagnosis::LostAtReload
|
||||
} else if prepared < previous {
|
||||
CoverageDiagnosis::LostAtPrepare
|
||||
} else if walked > 0 && reloaded <= previous {
|
||||
CoverageDiagnosis::WalkWithoutRetention
|
||||
} else if reloaded > previous {
|
||||
CoverageDiagnosis::Progress
|
||||
} else {
|
||||
CoverageDiagnosis::NoNewWork
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() {
|
||||
assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress);
|
||||
assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention);
|
||||
assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare);
|
||||
assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload);
|
||||
assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_missing_and_corrupt_inputs_fail() {
|
||||
for bytes in [
|
||||
vec![],
|
||||
vec![0xc1],
|
||||
DataUsageCache::default().marshal_msg().expect("empty cache encoding"),
|
||||
vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")],
|
||||
] {
|
||||
assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".to_string();
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache.replace("bucket/static", "bucket", DataUsageEntry::default());
|
||||
for index in 0..4 {
|
||||
cache.replace(
|
||||
&format!("bucket/static/{index}"),
|
||||
"bucket/static",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
cache.reduce_children_of(&hash_path("bucket/static"), 1, true);
|
||||
let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture");
|
||||
let entry = decoded
|
||||
.find("bucket/static")
|
||||
.expect("compaction must retain the static subtree root");
|
||||
assert!(entry.compacted);
|
||||
assert!(entry.children.is_empty());
|
||||
assert_eq!(
|
||||
retained(&decoded),
|
||||
4,
|
||||
"compaction retains aggregate coverage even when leaf keys are absent"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_save_reload_resume() {
|
||||
run_checkpoint_fixture(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_hot_digest_diagnostic() {
|
||||
run_checkpoint_fixture(true).await;
|
||||
}
|
||||
|
||||
async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for index in 0..STATIC_OBJECTS {
|
||||
write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
|
||||
}
|
||||
let store = FixtureStore::new();
|
||||
let mut previous = 0;
|
||||
let mut visited = 0;
|
||||
for round in 0..3_u8 {
|
||||
write_test_object_metadata(&root, "bucket", "hot/current").await;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let revisions = cache
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("load checkpoint revisions");
|
||||
if round > 0 {
|
||||
assert_eq!(retained(&store.strict_load().await), previous);
|
||||
}
|
||||
let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round)));
|
||||
crate::scanner_io::current_cache_root_or_prepare_with_generation(
|
||||
&mut cache,
|
||||
"bucket",
|
||||
SOURCE,
|
||||
11,
|
||||
7,
|
||||
plan,
|
||||
crate::scanner_io::DataUsageCacheReuseOptions {
|
||||
require_source: true,
|
||||
tier_registry_generation: None,
|
||||
},
|
||||
);
|
||||
let prepared = retained(&cache);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(4),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let outcome = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await
|
||||
.expect("budgeted local disk scan returns partial cache");
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("budgeted fixture must remain partial")
|
||||
};
|
||||
assert!(!cache.info.snapshot_complete, "partial must never publish a complete root");
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
let scanned = retained(&cache);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist partial checkpoint");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
loaded
|
||||
.load(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("reload persisted partial checkpoint");
|
||||
let reloaded = retained(&loaded);
|
||||
assert_eq!(reloaded, retained(&store.strict_load().await));
|
||||
assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage");
|
||||
assert!(!loaded.info.snapshot_complete);
|
||||
visited += budget.entries_visited();
|
||||
let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded);
|
||||
eprintln!(
|
||||
"checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
|
||||
);
|
||||
if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
|
||||
assert_eq!(
|
||||
diagnosis,
|
||||
CoverageDiagnosis::Progress,
|
||||
"visited growth must produce durable static coverage"
|
||||
);
|
||||
}
|
||||
crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
|
||||
previous = reloaded;
|
||||
}
|
||||
assert!(visited > 0, "fixture must exercise the directory walk");
|
||||
assert!(previous > 0, "fixture must retain and enumerate static subtree entries");
|
||||
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let revisions = loaded
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("load final checkpoint");
|
||||
let before = tokio::fs::read(store.root.path().join("main"))
|
||||
.await
|
||||
.expect("read durable checkpoint bytes");
|
||||
let epoch_error = loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
|
||||
.await
|
||||
.expect_err("stale publication epoch must reject persistence");
|
||||
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
store.reject_save.store(true, Ordering::SeqCst);
|
||||
loaded.info.next_cycle += 1;
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect_err("injected save failure must not report durable progress");
|
||||
assert_eq!(
|
||||
tokio::fs::read(store.root.path().join("main"))
|
||||
.await
|
||||
.expect("read unchanged checkpoint bytes"),
|
||||
before
|
||||
);
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
parent.cancel();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
|
||||
assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await
|
||||
.expect("unbounded scan must complete after durable partial progress");
|
||||
let ScannerDiskScanOutcome::Complete(cache) = result else {
|
||||
panic!("unbounded fixture must produce a complete disk cache");
|
||||
};
|
||||
assert!(cache.info.snapshot_complete);
|
||||
assert!(cache.info.scan_checkpoint.is_none());
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").expect("complete bucket root").objects,
|
||||
usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
|
||||
);
|
||||
}
|
||||
@@ -209,14 +209,6 @@ fn scanner_bucket_cache_digest(
|
||||
DataUsageScanPlanDigest(hasher.finalize().into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn checkpoint_fixture_bucket_digest(
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
dirty_generation: Option<u64>,
|
||||
) -> DataUsageScanPlanDigest {
|
||||
scanner_bucket_cache_digest(scan_plan_digest, dirty_generation)
|
||||
}
|
||||
|
||||
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
|
||||
if results.iter().any(|result| result.info.last_update.is_some()) {
|
||||
return Ok(());
|
||||
|
||||
@@ -1048,27 +1048,6 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() {
|
||||
for (budget, cancelled, bucket, expected) in [
|
||||
(false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded),
|
||||
(true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
|
||||
(false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
|
||||
] {
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
true,
|
||||
budget,
|
||||
cancelled,
|
||||
bucket,
|
||||
DirtyUsageSnapshotStatus::Changed,
|
||||
ScannerCycleActivityStatus::Unchanged
|
||||
),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unverified_activity_defers_partial_and_floor_cycles() {
|
||||
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
|
||||
@@ -49,14 +49,5 @@ rustfs-rio.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
astral-tokio-tar = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
tar-codec = { workspace = true }
|
||||
tar-framing = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# minio-go Snowball fixtures
|
||||
|
||||
These request bodies are generated by
|
||||
`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned
|
||||
in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by
|
||||
RustFS Snowball extraction.
|
||||
|
||||
The decoded TAR intentionally ends immediately after the final padded member
|
||||
body because minio-go flushes, rather than closes, its TAR writer. The
|
||||
compatibility test permits that shape only when the authenticated request body
|
||||
is complete at the exact member boundary; it does not make incomplete TAR
|
||||
terminators generally valid.
|
||||
|
||||
Regenerate them from this directory with Go 1.25:
|
||||
|
||||
```console
|
||||
cd generate
|
||||
go mod download
|
||||
go run . -out ..
|
||||
```
|
||||
|
||||
`manifest.json` records the input objects and SHA-256 digest of each captured
|
||||
request body. Review changes to the manifest and binary fixtures together when
|
||||
updating minio-go.
|
||||
@@ -1,26 +0,0 @@
|
||||
module rustfs.local/snowball-fixture
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require github.com/minio/minio-go/v7 v7.3.0
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
@@ -1,59 +0,0 @@
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU=
|
||||
github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -1,193 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
const minioGoVersion = "v7.3.0"
|
||||
|
||||
type fixtureManifest struct {
|
||||
Generator string `json:"generator"`
|
||||
MinioGo string `json:"minio_go"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Objects []fixtureObject `json:"objects"`
|
||||
Archives []fixtureArchive `json:"archives"`
|
||||
}
|
||||
|
||||
type fixtureObject struct {
|
||||
Key string `json:"key"`
|
||||
Body string `json:"body"`
|
||||
ModTime string `json:"mod_time"`
|
||||
VersionID string `json:"version_id,omitempty"`
|
||||
Headers map[string][]string `json:"headers,omitempty"`
|
||||
}
|
||||
|
||||
type fixtureArchive struct {
|
||||
File string `json:"file"`
|
||||
Compressed bool `json:"compressed"`
|
||||
Length int `json:"length"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
func objects() []fixtureObject {
|
||||
return []fixtureObject{
|
||||
{
|
||||
Key: "alpha.txt",
|
||||
Body: "alpha-body",
|
||||
ModTime: "2024-01-02T03:04:05Z",
|
||||
VersionID: "018cc251-f400-7c22-9e8d-8b1800000001",
|
||||
Headers: map[string][]string{
|
||||
"Content-Type": {"text/plain"},
|
||||
"X-Amz-Meta-Owner": {"snowball-fixture"},
|
||||
"X-Amz-Tagging": {"project=rustfs&source=minio-go"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "nested/世界.txt",
|
||||
Body: "bravo-body",
|
||||
ModTime: "2024-01-02T03:05:05Z",
|
||||
Headers: map[string][]string{
|
||||
"Content-Language": {"zh-CN"},
|
||||
"X-Amz-Meta-Note": {"unicode-path"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) {
|
||||
body := make(chan []byte, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
payload, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
http.Error(writer, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
body <- payload
|
||||
writer.Header().Set("ETag", `"snowball-fixture"`)
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{
|
||||
// The S3 authentication layer removes AWS streaming-signature framing
|
||||
// before Snowball extraction sees the request body. Anonymous signing
|
||||
// captures those decoded archive bytes directly.
|
||||
Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous),
|
||||
Secure: false,
|
||||
Region: "us-east-1",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct minio client: %w", err)
|
||||
}
|
||||
|
||||
input := make(chan minio.SnowballObject, len(specs))
|
||||
for _, spec := range specs {
|
||||
modTime, err := time.Parse(time.RFC3339, spec.ModTime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err)
|
||||
}
|
||||
headers := make(http.Header, len(spec.Headers))
|
||||
for name, values := range spec.Headers {
|
||||
headers[name] = append([]string(nil), values...)
|
||||
}
|
||||
input <- minio.SnowballObject{
|
||||
Key: spec.Key,
|
||||
Size: int64(len(spec.Body)),
|
||||
ModTime: modTime,
|
||||
Content: bytes.NewReader([]byte(spec.Body)),
|
||||
VersionID: spec.VersionID,
|
||||
Headers: headers,
|
||||
}
|
||||
}
|
||||
close(input)
|
||||
|
||||
err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{
|
||||
Opts: minio.PutObjectOptions{
|
||||
ContentType: "application/octet-stream",
|
||||
},
|
||||
InMemory: true,
|
||||
Compress: compressed,
|
||||
}, input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate snowball request: %w", err)
|
||||
}
|
||||
return <-body, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
outDir := flag.String("out", "..", "fixture output directory")
|
||||
flag.Parse()
|
||||
|
||||
specs := objects()
|
||||
archives := make([]fixtureArchive, 0, 2)
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
compressed bool
|
||||
}{
|
||||
{name: "snowball.tar"},
|
||||
{name: "snowball.tar.s2", compressed: true},
|
||||
} {
|
||||
payload, err := captureSnowball(fixture.compressed, specs)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
path := filepath.Join(*outDir, fixture.name)
|
||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
||||
panic(fmt.Errorf("write %s: %w", path, err))
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
archives = append(archives, fixtureArchive{
|
||||
File: fixture.name,
|
||||
Compressed: fixture.compressed,
|
||||
Length: len(payload),
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
})
|
||||
}
|
||||
|
||||
manifest := fixtureManifest{
|
||||
Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
|
||||
MinioGo: minioGoVersion,
|
||||
GeneratedAt: "2026-09-05T00:00:00Z",
|
||||
Objects: specs,
|
||||
Archives: archives,
|
||||
}
|
||||
payload, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
payload = append(payload, '\n')
|
||||
path := filepath.Join(*outDir, "manifest.json")
|
||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
||||
panic(fmt.Errorf("write %s: %w", path, err))
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
{
|
||||
"generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
|
||||
"minio_go": "v7.3.0",
|
||||
"generated_at": "2026-09-05T00:00:00Z",
|
||||
"objects": [
|
||||
{
|
||||
"key": "alpha.txt",
|
||||
"body": "alpha-body",
|
||||
"mod_time": "2024-01-02T03:04:05Z",
|
||||
"version_id": "018cc251-f400-7c22-9e8d-8b1800000001",
|
||||
"headers": {
|
||||
"Content-Type": [
|
||||
"text/plain"
|
||||
],
|
||||
"X-Amz-Meta-Owner": [
|
||||
"snowball-fixture"
|
||||
],
|
||||
"X-Amz-Tagging": [
|
||||
"project=rustfs\u0026source=minio-go"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "nested/世界.txt",
|
||||
"body": "bravo-body",
|
||||
"mod_time": "2024-01-02T03:05:05Z",
|
||||
"headers": {
|
||||
"Content-Language": [
|
||||
"zh-CN"
|
||||
],
|
||||
"X-Amz-Meta-Note": [
|
||||
"unicode-path"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"archives": [
|
||||
{
|
||||
"file": "snowball.tar",
|
||||
"compressed": false,
|
||||
"length": 4096,
|
||||
"sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e"
|
||||
},
|
||||
{
|
||||
"file": "snowball.tar.s2",
|
||||
"compressed": true,
|
||||
"length": 528,
|
||||
"sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,548 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Cursor;
|
||||
|
||||
use futures::StreamExt;
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive};
|
||||
use tar_framing::{
|
||||
FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind,
|
||||
logical::{MemberExtensions, PaxState, TarReader},
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0";
|
||||
const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar");
|
||||
const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2");
|
||||
const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json");
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureManifest {
|
||||
generator: String,
|
||||
minio_go: String,
|
||||
generated_at: String,
|
||||
objects: Vec<FixtureObject>,
|
||||
archives: Vec<FixtureArchive>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureObject {
|
||||
key: String,
|
||||
body: String,
|
||||
mod_time: String,
|
||||
#[serde(default)]
|
||||
version_id: String,
|
||||
#[serde(default)]
|
||||
headers: BTreeMap<String, Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct FixtureArchive {
|
||||
file: String,
|
||||
compressed: bool,
|
||||
length: usize,
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
struct ParsedMember {
|
||||
path: String,
|
||||
size: u64,
|
||||
mtime: Option<u64>,
|
||||
body: Vec<u8>,
|
||||
minio_pax: BTreeMap<String, Option<Vec<u8>>>,
|
||||
}
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
let mut encoded = String::with_capacity(64);
|
||||
for byte in Sha256::digest(bytes) {
|
||||
write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail");
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
async fn decode_s2(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut decoder = CompressionFormat::S2
|
||||
.get_decoder(Cursor::new(bytes.to_vec()))
|
||||
.expect("S2 fixture decoder should be available");
|
||||
let mut decoded = Vec::new();
|
||||
decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode");
|
||||
decoded
|
||||
}
|
||||
|
||||
async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec<ParsedMember> {
|
||||
let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec()));
|
||||
let mut entries = archive.entries().expect("tokio-tar should create an entry stream");
|
||||
let mut parsed = Vec::new();
|
||||
|
||||
while let Some(entry) = entries.next().await {
|
||||
let mut entry = entry.expect("tokio-tar should parse the fixture member");
|
||||
let kind = entry.header().entry_type();
|
||||
if kind == tokio_tar::EntryType::XGlobalHeader {
|
||||
continue;
|
||||
}
|
||||
|
||||
let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path");
|
||||
let path = std::str::from_utf8(path_bytes.as_ref())
|
||||
.expect("fixture paths should be UTF-8")
|
||||
.to_owned();
|
||||
let size = entry.effective_size();
|
||||
let mtime = entry.header().mtime().ok();
|
||||
let mut minio_pax = BTreeMap::new();
|
||||
if let Some(extensions) = entry
|
||||
.pax_extensions()
|
||||
.await
|
||||
.expect("tokio-tar should parse local PAX records")
|
||||
{
|
||||
for extension in extensions {
|
||||
let extension = extension.expect("fixture PAX record should be valid");
|
||||
let key = extension.key().expect("fixture PAX keys should be UTF-8");
|
||||
if key.starts_with("minio.") {
|
||||
minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec()));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
entry
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("tokio-tar should read the fixture body");
|
||||
parsed.push(ParsedMember {
|
||||
path,
|
||||
size,
|
||||
mtime,
|
||||
body,
|
||||
minio_pax,
|
||||
});
|
||||
}
|
||||
parsed
|
||||
}
|
||||
|
||||
fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec<PaxKeyword>) -> BTreeMap<String, Option<Vec<u8>>> {
|
||||
for extension in state.extensions() {
|
||||
for record in extension.records() {
|
||||
let keyword = record.keyword();
|
||||
if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio")
|
||||
&& !known_keywords.contains(&keyword)
|
||||
{
|
||||
known_keywords.push(keyword);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
known_keywords
|
||||
.iter()
|
||||
.filter_map(|keyword| {
|
||||
let record = state.effective_record(keyword)?;
|
||||
let PaxRecord::Vendor { vendor, name, value } = record else {
|
||||
return None;
|
||||
};
|
||||
let key = format!("{vendor}.{name}");
|
||||
let value = match value {
|
||||
PaxValue::Value(value) => Some(value.to_vec()),
|
||||
PaxValue::Deleted => None,
|
||||
};
|
||||
Some((key, value))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn effective_mtime(header_mtime: Option<u64>, extensions: &MemberExtensions<'_>) -> Option<u64> {
|
||||
let MemberExtensions::Pax(state) = extensions else {
|
||||
return header_mtime;
|
||||
};
|
||||
match state.effective_record(&PaxKeyword::Mtime) {
|
||||
Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value),
|
||||
Some(PaxRecord::Mtime(PaxValue::Deleted)) => None,
|
||||
_ => header_mtime,
|
||||
}
|
||||
}
|
||||
|
||||
fn padded_member_end(position: u64, size: u64) -> u64 {
|
||||
let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512;
|
||||
position
|
||||
.checked_add(512)
|
||||
.and_then(|position| position.checked_add(padded_size))
|
||||
.expect("fixture member end should not overflow")
|
||||
}
|
||||
|
||||
fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option<u64>, request_body_complete: bool) -> bool {
|
||||
// The production gate must source `request_body_complete` from RustFS's
|
||||
// length, checksum, and trailing-header validation state.
|
||||
request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position)
|
||||
}
|
||||
|
||||
fn candidate_snowball_decode_policy() -> DecodePolicy {
|
||||
DecodePolicy::default()
|
||||
.allow_gnu(true)
|
||||
.allow_all_nul_numeric_fields(true)
|
||||
.max_gnu_extension_size(1_048_576)
|
||||
.pax_policy(
|
||||
PaxDecodePolicy::default()
|
||||
.max_extension_size(1_048_576)
|
||||
.max_global_extensions_size(67_108_864)
|
||||
.allow_global_pax_extensions(false)
|
||||
.allow_non_utf8_pax_vendor_values(false)
|
||||
.allow_duplicate_pax_records(false)
|
||||
.allow_global_pax_member_metadata(false)
|
||||
.vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])),
|
||||
)
|
||||
}
|
||||
|
||||
async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec<ParsedMember>, Option<FrameError>, Option<u64>) {
|
||||
let policy = StreamPolicy::default()
|
||||
.max_pax_extension_size(1024 * 1024)
|
||||
.max_global_pax_extensions_size(4 * 1024 * 1024)
|
||||
.max_gnu_extension_size(128 * 1024);
|
||||
let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy);
|
||||
let mut parsed = Vec::new();
|
||||
let mut known_minio_keywords = Vec::new();
|
||||
let mut last_member_end = None;
|
||||
|
||||
loop {
|
||||
let mut frame = match reader.next_frame().await {
|
||||
Ok(Some(frame)) => frame,
|
||||
Ok(None) => return (parsed, None, last_member_end),
|
||||
Err(error) => return (parsed, Some(error), last_member_end),
|
||||
};
|
||||
assert_eq!(frame.header.kind, UstarKind::Regular);
|
||||
let path = String::from_utf8(
|
||||
frame
|
||||
.effective_path()
|
||||
.expect("tar-framing should resolve the fixture path")
|
||||
.into_owned(),
|
||||
)
|
||||
.expect("fixture paths should be UTF-8");
|
||||
let size = frame.header.effective_size;
|
||||
let mtime = effective_mtime(frame.header.mtime, &frame.extensions);
|
||||
let minio_pax = match &frame.extensions {
|
||||
MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords),
|
||||
MemberExtensions::Gnu { .. } => BTreeMap::new(),
|
||||
};
|
||||
let mut body = Vec::new();
|
||||
let mut chunk = Vec::new();
|
||||
while frame
|
||||
.payload
|
||||
.next_chunk(&mut chunk, 64 * 1024)
|
||||
.await
|
||||
.expect("tar-framing should read the fixture body")
|
||||
{
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
last_member_end = Some(padded_member_end(frame.header.position, size));
|
||||
parsed.push(ParsedMember {
|
||||
path,
|
||||
size,
|
||||
mtime,
|
||||
body,
|
||||
minio_pax,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_in_fixtures_match_the_minio_go_manifest() {
|
||||
let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON");
|
||||
assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball");
|
||||
assert_eq!(manifest.minio_go, "v7.3.0");
|
||||
assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z");
|
||||
assert_eq!(manifest.objects.len(), 2);
|
||||
assert_eq!(manifest.objects[0].key, "alpha.txt");
|
||||
assert_eq!(manifest.objects[0].body, "alpha-body");
|
||||
assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z");
|
||||
assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001");
|
||||
assert_eq!(
|
||||
manifest.objects[0].headers.get("X-Amz-Meta-Owner"),
|
||||
Some(&vec!["snowball-fixture".to_owned()])
|
||||
);
|
||||
|
||||
for archive in &manifest.archives {
|
||||
let bytes = match archive.file.as_str() {
|
||||
"snowball.tar" => RAW_FIXTURE,
|
||||
"snowball.tar.s2" => S2_FIXTURE,
|
||||
file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"),
|
||||
};
|
||||
assert_eq!(bytes.len(), archive.length);
|
||||
assert_eq!(sha256_hex(bytes), archive.sha256);
|
||||
assert_eq!(archive.compressed, archive.file.ends_with(".s2"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() {
|
||||
assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE);
|
||||
assert_eq!(RAW_FIXTURE.len() % 512, 0);
|
||||
assert!(RAW_FIXTURE.len() >= 1024);
|
||||
assert!(
|
||||
!RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0),
|
||||
"minio-go Flush output should not contain the standard two-block terminator"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() {
|
||||
let expected = parse_with_tokio_tar(RAW_FIXTURE).await;
|
||||
let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
|
||||
let error = error.expect("footerless minio-go fixture should fail strict termination");
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
assert_eq!(
|
||||
actual,
|
||||
[
|
||||
ParsedMember {
|
||||
path: "alpha.txt".to_owned(),
|
||||
size: 10,
|
||||
mtime: Some(1_704_164_645),
|
||||
body: b"alpha-body".to_vec(),
|
||||
minio_pax: BTreeMap::from([
|
||||
("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),),
|
||||
("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),),
|
||||
(
|
||||
"minio.metadata.X-Amz-Tagging".to_owned(),
|
||||
Some(b"project=rustfs&source=minio-go".to_vec()),
|
||||
),
|
||||
("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),),
|
||||
]),
|
||||
},
|
||||
ParsedMember {
|
||||
path: "nested/世界.txt".to_owned(),
|
||||
size: 10,
|
||||
mtime: Some(1_704_164_705),
|
||||
body: b"bravo-body".to_vec(),
|
||||
minio_pax: BTreeMap::from([
|
||||
("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),),
|
||||
("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),),
|
||||
]),
|
||||
},
|
||||
]
|
||||
);
|
||||
assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
|
||||
assert_eq!(
|
||||
error.position,
|
||||
u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")
|
||||
);
|
||||
assert_eq!(last_member_end, Some(error.position));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() {
|
||||
let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
|
||||
let error = error.expect("the real fixture should be footerless");
|
||||
assert!(is_authenticated_footerless_end(&error, last_member_end, true));
|
||||
assert!(!is_authenticated_footerless_end(&error, last_member_end, false));
|
||||
|
||||
let mut one_zero_block = RAW_FIXTURE.to_vec();
|
||||
one_zero_block.extend([0; 512]);
|
||||
let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await;
|
||||
let error = error.expect("one zero block is not a valid TAR terminator");
|
||||
assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
|
||||
assert_eq!(
|
||||
last_member_end,
|
||||
Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64"))
|
||||
);
|
||||
assert_eq!(
|
||||
error.position,
|
||||
u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64")
|
||||
);
|
||||
assert!(!is_authenticated_footerless_end(&error, last_member_end, true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() {
|
||||
let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("the default policy should reject minio vendor records"),
|
||||
};
|
||||
assert!(default_error.to_string().contains("pax vendor extension minio."));
|
||||
|
||||
let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec()))
|
||||
.with_policy(candidate_snowball_decode_policy())
|
||||
.members();
|
||||
let mut bodies = Vec::new();
|
||||
loop {
|
||||
let member = match members.next().await {
|
||||
Ok(Some(member)) => member,
|
||||
Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"),
|
||||
Err(error) => {
|
||||
assert!(error.to_string().contains("missing two-block end-of-archive marker"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
let Member::File { mut payload, .. } = member else {
|
||||
panic!("fixture should contain only regular files");
|
||||
};
|
||||
let mut body = Vec::new();
|
||||
let mut chunk = Vec::new();
|
||||
while payload
|
||||
.next_chunk(&mut chunk, 64 * 1024)
|
||||
.await
|
||||
.expect("tar-codec should read the fixture body")
|
||||
{
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
bodies.push(body);
|
||||
}
|
||||
assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]);
|
||||
assert!(
|
||||
members
|
||||
.next()
|
||||
.await
|
||||
.expect("the member cursor should be fused after an error")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
fn pax_record(key: &str, value: &str) -> Vec<u8> {
|
||||
let payload = format!("{key}={value}\n");
|
||||
let mut len = payload.len() + 3;
|
||||
loop {
|
||||
let record = format!("{len} {payload}");
|
||||
if record.len() == len {
|
||||
return record.into_bytes();
|
||||
}
|
||||
len = record.len();
|
||||
}
|
||||
}
|
||||
|
||||
async fn append_pax_header(
|
||||
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
|
||||
entry_type: tokio_tar::EntryType,
|
||||
records: &[(&str, &str)],
|
||||
) {
|
||||
let mut payload = Vec::new();
|
||||
for (key, value) in records {
|
||||
payload.extend(pax_record(key, value));
|
||||
}
|
||||
let mut header = tokio_tar::Header::new_ustar();
|
||||
header.set_entry_type(entry_type);
|
||||
header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64"));
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
|
||||
.await
|
||||
.expect("PAX test header should be written");
|
||||
}
|
||||
|
||||
async fn append_regular(builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>, path: &str) {
|
||||
let body = path.as_bytes();
|
||||
let mut header = tokio_tar::Header::new_ustar();
|
||||
header.set_entry_type(tokio_tar::EntryType::Regular);
|
||||
header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64"));
|
||||
header.set_mode(0o644);
|
||||
header.set_mtime(1_704_164_645);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, path, Cursor::new(body))
|
||||
.await
|
||||
.expect("ordinary test member should be written");
|
||||
}
|
||||
|
||||
async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec<u8> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await;
|
||||
append_regular(&mut builder, "member.txt").await;
|
||||
builder.into_inner().await.expect("policy archive should finish").into_inner()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() {
|
||||
let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await;
|
||||
let error = match TarArchive::new(Cursor::new(unknown_vendor))
|
||||
.with_policy(candidate_snowball_decode_policy())
|
||||
.members()
|
||||
.next()
|
||||
.await
|
||||
{
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"),
|
||||
};
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("pax vendor extension acme.metadata.owner is not allowed")
|
||||
);
|
||||
|
||||
let duplicate = archive_with_local_pax(&[
|
||||
("minio.metadata.x-amz-meta-owner", "first"),
|
||||
("minio.metadata.x-amz-meta-owner", "second"),
|
||||
])
|
||||
.await;
|
||||
let error = match TarArchive::new(Cursor::new(duplicate))
|
||||
.with_policy(candidate_snowball_decode_policy())
|
||||
.members()
|
||||
.next()
|
||||
.await
|
||||
{
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"),
|
||||
};
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_pax_header(
|
||||
&mut builder,
|
||||
tokio_tar::EntryType::XGlobalHeader,
|
||||
&[("minio.metadata.x-amz-meta-owner", "global")],
|
||||
)
|
||||
.await;
|
||||
append_pax_header(
|
||||
&mut builder,
|
||||
tokio_tar::EntryType::XHeader,
|
||||
&[("minio.metadata.x-amz-meta-owner", "local")],
|
||||
)
|
||||
.await;
|
||||
append_regular(&mut builder, "local.txt").await;
|
||||
append_regular(&mut builder, "inherited.txt").await;
|
||||
let archive = builder
|
||||
.into_inner()
|
||||
.await
|
||||
.expect("precedence archive should finish")
|
||||
.into_inner();
|
||||
|
||||
let legacy = parse_with_tokio_tar(&archive).await;
|
||||
let (framing, error, _) = parse_with_tar_framing(&archive).await;
|
||||
assert!(error.is_none());
|
||||
assert_eq!(legacy.len(), 2);
|
||||
assert_eq!(framing.len(), 2);
|
||||
|
||||
let owner_key = "minio.metadata.x-amz-meta-owner";
|
||||
assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
|
||||
assert!(!legacy[1].minio_pax.contains_key(owner_key));
|
||||
assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
|
||||
assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec())));
|
||||
|
||||
let error = match TarArchive::new(Cursor::new(archive))
|
||||
.with_policy(candidate_snowball_decode_policy())
|
||||
.members()
|
||||
.next()
|
||||
.await
|
||||
{
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"),
|
||||
};
|
||||
assert!(error.to_string().contains("global pax extended headers are not allowed"));
|
||||
}
|
||||
@@ -37,8 +37,8 @@ unknown-git = "deny"
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
allow-git = [
|
||||
# Temporary tokio-tar fork pinned to the reviewed parser limits,
|
||||
# cancellation safety, and error-fusing change while Snowball is
|
||||
# prototyped against tar-codec and Swift retains its current reader.
|
||||
# cancellation safety, and error-fusing change while
|
||||
# astral-sh/tokio-tar#118 awaits an upstream release.
|
||||
# owner: cxymds review: 2026-10
|
||||
"https://github.com/cxymds/tokio-tar.git",
|
||||
# Official s3s repository. Temporarily pinned to the merged generic REST
|
||||
|
||||
@@ -48,7 +48,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
|
||||
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
|
||||
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
|
||||
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
|
||||
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |
|
||||
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format and its mixed-version rules |
|
||||
| [tier-stats-contract.md](tier-stats-contract.md) | changing what `GET /rustfs/admin/v3/tier-stats` returns, adding a tier accounting source, or wiring a metric to a remote tier request |
|
||||
|
||||
## Support and compatibility matrices (release-facing, keep current)
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
- `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
|
||||
- `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
|
||||
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork.
|
||||
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
|
||||
- `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
|
||||
- `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
|
||||
- `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.
|
||||
|
||||
@@ -7,7 +7,7 @@ For crate ownership, read [crate-boundaries.md](crate-boundaries.md): ECStore ow
|
||||
|
||||
## Model
|
||||
|
||||
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. This describes the intended lock scope while the guard remains valid; it does not prove rejection of an already-dispatched disk syscall after distributed lease loss. The authority, delayed-mutation, and recovery boundary is specified in [unified-object-generation.md](unified-object-generation.md). MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
|
||||
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
|
||||
|
||||
| Layer | Mechanism | Owner |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# ILM And Tiering Persistence Contracts
|
||||
|
||||
**Use this when:** changing an ILM transition, tier configuration mutation, manual transition job, tier-delete recovery path, pool decommission, or any code that can create, transfer, or destroy ownership of a remote-tier object.
|
||||
**Source of truth:** `TransitionTransaction` and `process_transition_transaction_record` in `crates/ecstore/src/bucket/lifecycle/transition_transaction.rs`; `TierMutationIntent` and its conditional store helpers in `crates/ecstore/src/services/tier/tier_mutation_intent.rs`; the dormant validation-probe record and conditional primitives in `crates/ecstore/src/services/tier/tier_probe_intent.rs`; `TierConfigMgr::update_candidate_with_config_lock` and mutation recovery in `crates/ecstore/src/services/tier/tier.rs`; `handle_tier_mutation_peer_request` in `crates/ecstore/src/services/tier/tier_mutation_peer.rs`; the record encoders and CAS helpers in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`; manual-job execution/recovery and `cleanup_free_version_exact` in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`; `process_tier_delete_journal_entry` and manifest recovery in `crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs`; the free-version scan/re-enqueue path `recover_tier_free_versions` in `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`; `DURABLE_ILM_NAMESPACES` and `validate_durable_ilm_record` in `crates/ecstore/src/bucket/lifecycle/durable_namespace.rs`; and `record_durable_ilm_decommission_progress`, receipt verification, and receipt cleanup in `crates/ecstore/src/core/pools.rs`.
|
||||
**Source of truth:** `TransitionTransaction` and `process_transition_transaction_record` in `crates/ecstore/src/bucket/lifecycle/transition_transaction.rs`; `TierMutationIntent` and its conditional store helpers in `crates/ecstore/src/services/tier/tier_mutation_intent.rs`; `TierConfigMgr::update_candidate_with_config_lock` and mutation recovery in `crates/ecstore/src/services/tier/tier.rs`; `handle_tier_mutation_peer_request` in `crates/ecstore/src/services/tier/tier_mutation_peer.rs`; the record encoders and CAS helpers in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`; manual-job execution/recovery and `cleanup_free_version_exact` in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`; `process_tier_delete_journal_entry` and manifest recovery in `crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs`; the free-version scan/re-enqueue path `recover_tier_free_versions` in `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`; `DURABLE_ILM_NAMESPACES` and `validate_durable_ilm_record` in `crates/ecstore/src/bucket/lifecycle/durable_namespace.rs`; and `record_durable_ilm_decommission_progress`, receipt verification, and receipt cleanup in `crates/ecstore/src/core/pools.rs`.
|
||||
|
||||
This document separates three kinds of statement:
|
||||
|
||||
@@ -44,7 +44,6 @@ All keys below are objects in the internal metadata bucket. The table gives the
|
||||
| Transition transaction | `rustfs-transition-transaction-v1`; successor v2 is approved below but not implemented | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed v1 `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Create-only maximum-parity write; exact record and ETag read before successor `If-Match`; terminal receipt followed by exact ETag conditional delete |
|
||||
| Tier mutation peer intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/records/<aa>/<bb>/<mutation-id>.json` | The receiving peer creates and converges it; the mutation recovery path cleans it | Immutable: mutation ID/kind, old config ETag, candidate digest, sorted affected target identities, expiry. Mutable: revision, state, committed config ETag | Create with `If-None-Match: *`; transition/delete with ETag `If-Match`; maximum parity |
|
||||
| Tier mutation coordinator intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/coordinators/<aa>/<bb>/<mutation-id>.json` | The initiating node creates it; coordinator recovery cleans it after peer convergence | Same mutation identity and mutable fields as the peer record | Same conditional-write contract as the peer intent |
|
||||
| Tier validation probe intent | Dormant `rustfs-tier-probe-intent-v1`; no writer or recovery is enabled | `ilm/tier-probe-intents/records/<aa>/<bb>/<probe-id>.json` | No current runtime owner because no path creates the record; v1 permits only the immutable creator as owner | Immutable probe, operation-generation, destination, random remote object, creator identity, and v1 owner fence. Mutable: revision, state, and monotonic remote-version proof | Conditional create/CAS/delete primitives exist but are not called by Add/Edit/Verify or recovery |
|
||||
| Manual job | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/jobs/<aa>/<bb>/<job-id>.json` | The admin run creates it; the active owner or recovery lease advances it. There is no current record GC owner | Immutable: job ID, bucket-level scope, options, creation time. Mutable: owner/lease, state, cancel bit, cursor, progress/report, queue snapshot, timestamps/error | Initial UUID-key write uses maximum parity without create-only precondition; later updates use ETag CAS |
|
||||
| Manual scope admission | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/scopes/<aa>/<bb>/<scope-digest>.json` | Job admission creates/renews it; the job owner removes it after terminalization | Immutable bucket/run-vs-dry-run scope; mutable job/lease ownership and expiry | Create-only, renew/delete by ETag CAS |
|
||||
| Manual task | `rustfs-manual-transition-task-v1` | `ilm/manual-transition/tasks/<job shards>/<job-id>/<task-key>.json` | The scanner persists it before queue admission; no current GC owner | Immutable job plus exact bucket/object/version/tier work identity | Append-only create with `If-None-Match: *` and maximum parity |
|
||||
@@ -57,7 +56,7 @@ All keys below are objects in the internal metadata bucket. The table gives the
|
||||
| Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/<run-token>.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal |
|
||||
| Recovery control, export, and disposition | `rustfs-ilm-recovery-control-v1`, `rustfs-ilm-recovery-export-v1`, and `rustfs-ilm-recovery-disposition-v1` are approved below but not implemented | `ilm/recovery-controls/...`, `ilm/recovery-exports/...`, and `ilm/recovery-dispositions/...` under protocol/shard/operation identities | Recovery owns one control for an exact source generation; the authenticated operator creates immutable export/disposition evidence; their collectors never own remote DELETE | Source protocol/path, all-pool copy-set manifest, ETags/content digests, owner lease, retry state, redacted error code, action, actor/reason, and terminal proof | Create-only, ETag CAS, all-pool strong readback, terminal receipt when covered by decommission, and exact conditional cleanup |
|
||||
|
||||
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transition-transaction namespace, the dormant tier-validation-probe namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
|
||||
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
|
||||
|
||||
## Durable fences and write primitives
|
||||
|
||||
@@ -191,35 +190,6 @@ Intent transitions retry an ETag race at most three times before returning a ret
|
||||
- Lease drain, peer Prepare, and reference proof run outside both exclusive guards after the coordinator Prepared record and local fence are durable. The commit path reacquires namespace WRITE then `admin_updates` and repeats the full generation/identity proof. Expiry is checked as an additional rejection boundary and never replaces config-generation proof.
|
||||
- **Open:** a dedicated operator reconcile/status surface and bounded retention for irreconcilable coordinator/peer records.
|
||||
|
||||
## Tier validation probe intent
|
||||
|
||||
### Dormant current contract
|
||||
|
||||
`TierProbeIntent` defines a strict, checksum-protected `rustfs-tier-probe-intent-v1` envelope and the canonical key `ilm/tier-probe-intents/records/<aa>/<bb>/<probe-id>.json`. The same non-nil probe UUID also determines the remote object name `rustfs-tier-probe-<probe-id>`. The path parser rejects uppercase UUID aliases, wrong shards, extra components, and path/payload/object-name disagreement. The durable namespace registry validates this record so pool decommission cannot silently treat it as an ordinary object.
|
||||
|
||||
The format binds Add and Edit to the same durable mutation identity tuple `(mutation_id, old_config_etag, candidate_digest)`. The old config ETag is required even for Add because it identifies the complete persisted tier configuration generation, not whether the named destination tier already exists. Verify instead requires the current persisted config ETag and credential-independent backend identity. These are mutually exclusive tagged variants. Every record also requires the tier name, matching destination identity, immutable creator identity/epoch, positive creation time, and an owner fence with nonempty owner, non-nil epoch, and later `not_after` timestamp. In v1 that owner identity must remain exactly equal to the immutable creator identity. It never persists the process-local driver revision or credential-bearing driver fingerprint.
|
||||
|
||||
The dormant state graph is:
|
||||
|
||||
```text
|
||||
UploadOutcomeUnknown -> Uploaded -> CleanupPending -> Completed
|
||||
\-> CleanupPending -> Completed
|
||||
\-> AbortedNoRemote
|
||||
```
|
||||
|
||||
`UploadOutcomeUnknown` and `AbortedNoRemote` carry no remote version. `Uploaded`, `CleanupPending`, and `Completed` carry either explicit unversioned semantics or one exact nonempty opaque version. Once known, that remote version cannot change. Revision advances by one for each edge: `UploadOutcomeUnknown` is revision 1, `Uploaded` and `AbortedNoRemote` are revision 2, `CleanupPending` is revision 2 or 3, and `Completed` is revision 3 or 4. The strict decoder rejects any other state/revision pairing. The direct `UploadOutcomeUnknown -> CleanupPending` edge is reserved for a future authoritative provider probe that discovers the exact cleanup candidate after the original PUT response was lost.
|
||||
|
||||
Create-only, ETag CAS, exact-ETag delete, and read-with-ETag primitives exist for the record, plus a crate-level read-only inspection result that always reports both writer and destructive recovery as disabled. No Add, Edit, Verify, startup loop, periodic loop, admin HTTP route, or remote backend operation currently calls these mutation primitives. Consequently this version creates no records and authorizes no remote PUT or DELETE.
|
||||
|
||||
### Activation requirements
|
||||
|
||||
- Add/Edit must create the mutation identity before validation and reread the same `(mutation_id, old_config_etag, candidate_digest)` before every probe-intent successor. Verify must reread the same config ETag, tier, and backend identity. A credential rotation may supply usable current credentials only when the persisted destination identity remains exact; it may not weaken operation-generation checks.
|
||||
- Before enabling any writer, every required node must advertise a probe-intent-specific read/retain/recovery capability. This protocol does not reuse the legacy transition-state reconciliation capability or its token. Unknown or older nodes keep validation in the current process-local mode and no durable v1 record is written.
|
||||
- The v1 owner fence is immutable and must equal the creator identity. Any future takeover requires a new schema with explicit takeover proof, plus an approved lease duration, clock-skew allowance, durable owner/epoch CAS, and revalidation order. Expiry alone never permits remote DELETE.
|
||||
- Remote-version discovery and deletion must use the provider-bound, destination-bound implementation and bounded request API approved for that target. Unknown, multiple, changing, unsupported, or unavailable results retain the record.
|
||||
- A successful or response-lost state write must strongly reread the exact record. Remote DELETE requires a current operation-generation proof, exact destination, current credentials for that same destination, a valid fleet fence, durable takeover, and the same known remote version immediately before and after the call.
|
||||
- Terminal retention, bounded scanning, metrics, and any HTTP inspect/reconcile route remain unapproved. Raw age is never cleanup evidence, and this dormant core API must not be presented as an operator endpoint.
|
||||
|
||||
## Manual transition job, task, result, and checkpoint
|
||||
|
||||
### Current contract
|
||||
|
||||
@@ -1,151 +1,52 @@
|
||||
# Remote Credential Sealing ADR
|
||||
|
||||
**Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another.
|
||||
**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `crates/ecstore/src/bucket/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
|
||||
|
||||
## Recommendation
|
||||
|
||||
Seal the credentials, ship the write side off by default, and claim a security benefit only for deployments running the Vault Transit or AWS KMS backend — everywhere else recommend full-disk encryption and the fail-closed parse fix below, which cost no code and cover strictly more.
|
||||
|
||||
## Whether encryption buys anything here
|
||||
|
||||
This decides the whole question, so it comes before the design. Sealing converts "read the drives" into "read the drives **and** hold an authenticated path to the key". How much that is worth depends entirely on the KMS backend, and [../operations/kms-backend-security.md](../operations/kms-backend-security.md) is explicit about the difference.
|
||||
|
||||
| Backend | Where the key that unwraps these credentials lives | What sealing is worth |
|
||||
|---|---|---|
|
||||
| Vault Transit, AWS KMS | Inside Vault or AWS; only ciphertext ever leaves | Real. An offline copy of `.rustfs.sys` is inert. Each unwrap is a live authenticated call that is logged, rate-limitable and revocable, and revoking the node's identity retroactively protects every copy already taken |
|
||||
| Vault KV2 | In Vault KV v2, Base64-encoded, not wrapped | Thin. The referenced document states that KV read access is equivalent to holding the master keys, so the boundary is the Vault ACL on the key prefix — worth something only when that ACL is genuinely narrower than access to the drives, and worth nothing against anyone holding both |
|
||||
| Local, Static | In `key_dir` on the node's own filesystem, or in the process environment | Close to nothing. Whoever reads the drives on a node usually reads the host too. The only gap it covers is media taken away from the host — the same gap full-disk encryption covers better |
|
||||
|
||||
Two honest limits hold on every backend. Sealing is **not** a defense against code execution on a node: the sealer runs in-process on every node that has to build a remote client, so an attacker at that level asks it to unseal and gets the plaintext. And it is not a defense against an authorized admin, because an admin who can rewrite a target can point it at a remote they control instead of reading the old secret.
|
||||
|
||||
What it does remove is the media-level read: a decommissioned or RMA'd drive, a drive-level backup or volume snapshot, a host path exposed by a bad mount, a copy of a drive taken for support. That threat is real, and it is the only one this design addresses.
|
||||
**Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, its fail-closed rules, and the mixed-version compatibility matrix.
|
||||
**Source of truth:** the three stores that hold remote credentials today — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, and `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — plus the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier.rs`, and `crates/ecstore/src/bucket/on_demand_migration/config.rs`.
|
||||
|
||||
## Decision
|
||||
|
||||
Remote credentials are sealed **per field, into an added field, behind one shared seam**, and ECStore reaches KMS through an installed hook rather than a crate dependency.
|
||||
|
||||
1. **One seam, three consumers.** `BucketTargetSys`, `TierConfigMgr` and `OnDemandMigrationSys` seal and unseal through `crates/ecstore/src/bucket/sealed_credentials.rs`. No consumer talks to KMS, and no consumer defines its own ciphertext layout.
|
||||
2. **Only secret material is sealed,** and the enumeration comes from the redaction code, not from a pair of field names — see [Which fields are sealed](#which-fields-are-sealed). Endpoint, region, ARN, bucket, prefixes, path style, TLS flags and the custom CA bundle stay in clear text: they are needed for validation, listing and support diagnosis, and none of them is a secret.
|
||||
3. **Sealed material lives in an added field, and the plaintext field is emptied rather than removed.** A reader that does not understand the sealed field must see a credential that is *present and empty*, so it takes a missing-credential path rather than a parse failure. Removing the field instead is what turns this design into an outage; [Compatibility](#compatibility-per-store-because-the-three-differ) explains why.
|
||||
4. **Unsealing happens at client construction, not at parse time.** `build_remote_s3_client` in `crates/ecstore/src/bucket/remote_s3_client.rs` is the only place that needs plaintext, so admin reads, listings, validation and status paths never call KMS — and a KMS outage never changes which targets or tiers *exist*.
|
||||
|
||||
## What is stored today, and where
|
||||
|
||||
Two of the three are not files at all. `bucket-targets.json` and `on-demand-migration.json` are named sub-configurations inside one msgpack blob per bucket, and only the tier configuration is its own object.
|
||||
|
||||
| Store | Reached as | Actually persisted at | Written by | Container |
|
||||
|---|---|---|---|---|
|
||||
| Replication and ILM targets | `BUCKET_TARGETS_FILE` | `BucketMetadata::bucket_targets_config_json`, msgpack field `BucketTargetsConfigJSON` | `BucketMetadata::update_config`, then `BucketMetadata::save_with_store`; `crates/ecstore/src/bucket/metadata_sys.rs` serializes the update under a transaction lock | `{BUCKET_META_PREFIX}/{bucket}/{BUCKET_METADATA_FILE}` in `RUSTFS_META_BUCKET` (`crates/ecstore/src/disk/mod.rs`) |
|
||||
| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; `update_config` additionally refuses a blob this build cannot parse | same blob as above |
|
||||
| Remote tiers | `TIER_CONFIG_FILE` | its own object, a four-byte `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header followed by an `rmp_serde` payload of `ExternalTierConfigMgr` | `TierConfigMgr` through `encode_external_tiering_config_blob`, under `tier_config_lock_path` | `tier_config_path` under `CONFIG_PREFIX` in `RUSTFS_META_BUCKET` |
|
||||
|
||||
The consequence of the first two sharing a blob is that any change to how that blob parses has a blast radius covering policy, lifecycle, versioning, object lock and everything else in `BucketMetadata` — not just credentials.
|
||||
|
||||
## The at-rest boundary as it stands
|
||||
|
||||
Three things hold the line today, and all three keep working whether or not sealing ships.
|
||||
|
||||
- **The reserved bucket.** `RUSTFS_META_BUCKET` is `.rustfs.sys`; `is_reserved_or_invalid_bucket` keeps it off the S3 surface, and the admin inspect archive in `rustfs/src/admin/handlers/inspect_archive.rs` runs its request through a strict bucket-name check that a dot-prefixed reserved name does not pass.
|
||||
- **Admin authorization** on every route that can read or write one of the three configurations.
|
||||
- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
|
||||
|
||||
So no API returns a stored secret. The bytes are reachable by reading the drives, and that is the boundary sealing is proposed to move.
|
||||
|
||||
## Which fields are sealed
|
||||
|
||||
The authoritative list of what this codebase treats as secret is the redaction functions above, and it is wider than `secret_key` plus `session_token`.
|
||||
|
||||
| Store | Sealed | Left in clear text although redacted |
|
||||
|---|---|---|
|
||||
| Targets | `Credentials::secret_key`, `Credentials::session_token` | — |
|
||||
| On-demand migration | `SourceCredentials::secret_key`, `SourceCredentials::session_token` | — |
|
||||
| Tiers | `secret_key` on each of the nine S3-family backends in `crates/ecstore/src/services/tier/tier_config.rs`, `TierAzure::sp_auth.client_secret`, and `TierGCS::creds` | `TierS3::aws_role_web_identity_token_file`, which is a path rather than a secret |
|
||||
|
||||
`TierGCS::creds` carries a whole service-account key and is the largest single secret of the three stores; a design that sealed only fields literally named `secret_key` would leave it in clear text. `aws_role_web_identity_token_file` points at a file outside `.rustfs.sys`, so sealing it would protect nothing — and a tier configured that way stores no long-lived secret at all, which is the cheapest mitigation available and should be preferred where the remote supports it.
|
||||
1. **One seam, three consumers.** `BucketTargetSys`, `TierConfigMgr`, and `OnDemandMigrationSys` seal and unseal through a single ECStore-owned envelope type. No consumer talks to KMS, and no consumer defines its own ciphertext layout.
|
||||
2. **Only secret material is sealed.** `secret_key` and `session_token` are sealed. Endpoint, region, ARN, bucket, prefixes, path style, TLS flags, and the custom CA bundle stay in clear text: they are needed for validation, listing, and support diagnosis, and none of them is a secret.
|
||||
3. **Sealed material lives in an added field, never in place of the plaintext field.** A record carries either the plaintext field or the sealed field. A reader that does not understand the sealed field therefore finds the credential *absent* rather than finding a ciphertext string it would sign requests with.
|
||||
4. **Unsealing happens at client construction, not at parse time.** `build_remote_s3_client` in `crates/ecstore/src/bucket/remote_s3_client.rs` is the single point that needs plaintext, so admin reads, listings, validation, and status paths never call KMS.
|
||||
|
||||
## Envelope format
|
||||
|
||||
`SealedCredential` in `crates/ecstore/src/bucket/sealed_credentials.rs`: envelope version, KMS key id, optional KMS key version, algorithm label, and the ciphertext produced by the sealer. It is stored base64 in the two JSON stores and as bytes alongside the tier payload. `SEALED_CREDENTIAL_VERSION` is checked by `SealedCredential::check_version` *before* the sealer is consulted, so an envelope from a newer build is refused here rather than inside a backend.
|
||||
A versioned, self-describing record: envelope version, KMS key id, KMS key version, algorithm, nonce, and ciphertext. It is stored base64 in the two JSON stores and as raw bytes inside the msgpack payload of the tier blob; the tier blob's own `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header constants are unchanged, because the envelope carries its own version.
|
||||
|
||||
The encryption context binds each ciphertext to the record that owns it. `SealScope` renders store kind, owner (bucket name, tier name or target ARN) and field name into the context, so a ciphertext copied into another bucket, another tier or another field fails to decrypt instead of silently authorizing a different remote. Those context keys are part of the on-disk contract: changing one makes every existing ciphertext undecryptable.
|
||||
|
||||
The envelope deliberately does **not** carry its own scope. A scope read out of the stored bytes would be attacker-controlled, and checking a ciphertext against a context it supplied itself proves nothing. The scope is always re-derived from where the ciphertext was found, which is also a constraint on any rewrap job — see [Rotation](#rotation).
|
||||
The KMS encryption context binds each ciphertext to the record that owns it — store kind, owning bucket or tier name, and field name — so a ciphertext copied into another bucket, another tier, or another field fails to decrypt instead of silently authorizing a different remote.
|
||||
|
||||
## Why a hook instead of a dependency
|
||||
|
||||
`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
|
||||
`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering. The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup: `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs`, installed by `install_ecstore_event_dispatch_hook` in `rustfs/src/server/event.rs`, and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`. Sealing uses the same shape, with the binary supplying an implementation backed by `get_global_kms_service_manager` in `crates/kms/src/service_manager.rs`.
|
||||
|
||||
## Compatibility, per store, because the three differ
|
||||
## Compatibility matrix
|
||||
|
||||
The generic matrix is short: a plaintext record reads unchanged on any node; a sealed record reads on a new node with a sealer installed; a sealed record on a new node without one is a typed error and never a default. Everything difficult is in what an **old** node does, and the three stores behave differently enough that a single answer would be wrong.
|
||||
| Stored form | Reader | Behavior |
|
||||
|---|---|---|
|
||||
| Plaintext (today's format) | Old node | Unchanged. |
|
||||
| Plaintext | New node | Read as plaintext, no KMS call. Carries a `RUSTFS_COMPAT_TODO` marker per [compat-cleanup-register.md](compat-cleanup-register.md). |
|
||||
| Sealed | New node, hook installed | Unsealed at client construction. |
|
||||
| Sealed | New node, no hook or decrypt failure | Typed error; the target, tier, or source is unusable and reports why. Never a default, an empty credential, or the ciphertext bytes. |
|
||||
| Sealed | Old node | The credential field is absent, so the old node fails closed on its existing "missing credentials" path. This is the migration hazard the rollout gate exists for. |
|
||||
|
||||
| Store | Old node meets an added sealed field | Old node meets an emptied plaintext field | Verdict |
|
||||
|---|---|---|---|
|
||||
| Targets | Ignored. `BucketTarget` and `Credentials` do not use `deny_unknown_fields` | `Credentials` has no struct-level `serde(default)`, so a **missing** `secretKey` is a hard parse error for the whole document — but an **empty** one parses | Safe only if the plaintext field is emptied rather than removed |
|
||||
| On-demand migration | **Rejected.** `OnDemandMigrationConfig`, `SourceConfig` and `SourceCredentials` all carry `deny_unknown_fields`, so the whole configuration becomes unreadable, and `BucketMetadata::update_config` also refuses to persist it | Parses | Needs a reader-first release before any node writes the field |
|
||||
| Tiers | The payload is compact `rmp_serde`, which encodes structs positionally; an added field is an arity change a reader built for the previous struct cannot skip. `decode_external_tiering_config_blob` also rejects any `TIER_CONFIG_VERSION` it does not know | Parses | The sealed value must not be added to any struct inside the existing payload |
|
||||
## Rollout gate
|
||||
|
||||
Two of those rows are load-bearing enough to spell out.
|
||||
|
||||
**Targets.** `BucketMetadata::parse_all_configs` responds to an unparseable `bucket-targets.json` by logging `bucket_metadata_parse_failed` and setting `bucket_target_config` to `BucketTargets::default()` — an empty target list. So on an old node a record whose `secretKey` was removed does not fail per target: **every target in that bucket disappears, replication stops, and no caller sees an error.** The raw bytes survive in the blob, so it is recoverable, but the silence is the hazard. Emptying the field instead of removing it avoids triggering it, and the substitution itself should be replaced by a retained parse failure before any of this ships — see [Prerequisites](#prerequisites-in-this-order).
|
||||
|
||||
An emptied `secretKey` is not yet a clean local failure either. `build_remote_s3_client` raises `RemoteS3ClientError::MissingCredentials` only when the whole credentials object is absent, and `remote_sdk_credentials` passes an empty secret to the SDK, so today an emptied field signs a request that the remote rejects. That is loud rather than silent, and therefore acceptable as a floor, but the reader-first release should turn an empty access key or secret key into the same typed local error so the failure is attributable to this node instead of to the remote.
|
||||
|
||||
**Tiers.** A format change to `tier-config.bin` takes out every tier at once, and tiers are not only a write-path concern: an object already transitioned to a tier cannot be read without that tier's configuration, so the failure reaches GETs of data that has been there for months. The sealed values therefore belong in a companion object under the same prefix, covered by the same `tier_config_lock_path`, keyed by tier name and field name, leaving `tier-config.bin` byte-shaped exactly as it is with an empty `SecretKey`. Putting the envelope *into* `SecretKey` was considered and rejected: an old node would sign requests with the ciphertext, producing remote 403s and ciphertext in signature-related logs, instead of taking its missing-credential path. Confirm the exact decode behaviour against the encode/decode tests in `crates/ecstore/src/services/tier/tier.rs` before writing a byte of the new layout, and do not bump `TIER_CONFIG_VERSION` until every node in the supported upgrade range reads it.
|
||||
|
||||
**Downgrade** is the same event as "old node reads new bytes", with one addition: a node that has been downgraded keeps writing the old shape, so a configuration re-submitted through it loses the sealed field and returns to plaintext. That is a security regression, not a correctness one, and it is silent — which is another reason the write side is gated rather than defaulted on.
|
||||
|
||||
## KMS unavailable: read time versus write time
|
||||
|
||||
These two are not symmetric, and conflating them is how this design would cause an outage.
|
||||
|
||||
**At write time** the answer is easy: sealing fails, the admin write is refused with the typed error, and nothing is persisted. A configuration is never stored with the secret dropped, and never stored in clear text after the operator asked for sealing. The cost is that configuration cannot be changed while the KMS is down, which is acceptable and visible.
|
||||
|
||||
**At read time** the rule is that a credential which cannot be unsealed makes a remote *unusable*, never *absent*.
|
||||
|
||||
- Because unsealing happens at `build_remote_s3_client`, a KMS outage does not change which targets or tiers exist. Listings, status and admin reads keep returning them; each attempt to use one fails with a typed, retryable error that names the KMS as the cause.
|
||||
- Startup must not treat "cannot unseal" as "no such tier". A tier whose credential is unavailable stays present in `TierConfigMgr`, so a GET of an object transitioned to it fails with a retryable error rather than presenting as missing data, and nothing re-drives a transition elsewhere. The same holds for a replication target: it stays configured and reports why it is not working.
|
||||
- **A write must refuse to rewrite a configuration it could not fully read.** This is the sharpest edge in the whole design. If a partially-unreadable configuration can be re-serialized from a partially-populated in-memory view, then a KMS outage plus one unrelated admin edit persists the configuration with the unreadable records dropped — and that is the only mechanism by which a target or tier really would disappear for good. Today's code does not have this hazard, because both stores keep raw bytes or fail the whole decode; any per-record sealed handling that skips undecodable records would introduce it.
|
||||
Sealing is written only when KMS is configured **and** a module switch in `rustfs/src/module_switches.rs` is on, defaulting off in the release that introduces it. Reading sealed records is always supported; writing them is what waits. Operators enable the switch after every node in the cluster can read the format, and existing plaintext records are sealed by re-submitting the configuration through its admin API — this task ships no in-place migration sweep.
|
||||
|
||||
## Rotation
|
||||
|
||||
The envelope records the key id and, when the backend reports one, the key version. Re-wrapping is the KMS side's job, follows [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) and rustfs/backlog#1637 and #1642, and nothing here rotates, re-wraps or expires a key on its own. Two properties make that division workable, and both are constraints on the rewrap job rather than on this design.
|
||||
|
||||
- A rewrap must reproduce the encryption context, and the envelope does not carry it. The job must therefore reach a ciphertext **through its store** — enumerate targets, tiers and migration sources and derive the `SealScope` from the record's own position — rather than by scanning for envelope-shaped bytes.
|
||||
- `key_version` is absent for backends that report none. An absent version means "not known to be current", never "current"; a rewrap sweep must be able to act on it, and a completed sweep is evidence about scanned sources only, exactly as the referenced contract already says about key deletion.
|
||||
The envelope records the key id and key version it was wrapped under. Re-wrapping is the KMS side's job and follows [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md); nothing in this design rotates, re-wraps, or expires a key on its own.
|
||||
|
||||
## Fail-closed rules
|
||||
|
||||
- A missing sealer, a malformed envelope, an unknown envelope version, a failed decrypt or an encryption-context mismatch is a typed error, per the root `AGENTS.md` rule that a required value returns a typed error when it is absent or corrupt. `SealedCredentialError` has no variant that degrades to a default, an empty credential, or the raw ciphertext.
|
||||
- A seal failure fails the admin write.
|
||||
- Redaction is unchanged and independent: admin responses keep returning `REDACTED`, and `Debug` implementations keep hiding secret fields whether or not the stored form is sealed. `SealedCredential`'s own `Debug` prints the key id and a byte count, not the ciphertext.
|
||||
- Logs may carry the key id and the envelope version. They never carry ciphertext, plaintext, or an encryption-context value.
|
||||
- A sealed value never enters an equality or fingerprint comparison. `tier_config_fingerprint` hashes a tier configuration to decide whether an edit is a no-op, and `OnDemandMigrationConfig` derives `PartialEq`; a fresh nonce per seal would make every write look like a change and churn the tier driver cache. Compare plaintext configurations, then seal.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
**Do not encrypt; harden the existing boundary instead.** This is the strongest alternative, not a foil. Its parts: keep `.rustfs.sys` off every request surface, which already holds; make an unparseable `bucket-targets.json` fail closed instead of becoming an empty list, which is a bug fix worth doing regardless; prefer keyless credentials where the remote supports them, as `TierS3::aws_role_web_identity_token_file` already allows; and encrypt the drives, which removes the media threat completely, covers all three stores plus every other secret in `.rustfs.sys`, and costs no code. Against the media threat, full-disk encryption strictly dominates application-level sealing. Sealing wins only where the KMS is Transit or AWS **and** the operator wants each unwrap to be individually authenticated, logged and revocable — which is exactly the scope this ADR claims and no more.
|
||||
|
||||
**Encrypt the whole blob, as MinIO does for its tier configuration.** Rejected. `tier-config.bin`'s header is what tells a reader the format, and a whole-blob ciphertext makes every tier unreadable whenever the KMS is unreachable; for the bucket metadata blob it would take policy, lifecycle, versioning and object lock down with the credential. Per-field sealing keeps the blast radius at one credential.
|
||||
|
||||
**Keep the credential in the KMS and store only a reference.** Rejected. It makes the KMS the durability authority for configuration, adds a second lifecycle with its own orphans when a bucket or tier is deleted, and none of the supported backends is a general secret store — the backends documented in [../operations/kms-backend-security.md](../operations/kms-backend-security.md) manage keys, not arbitrary secrets.
|
||||
|
||||
**Deterministic encryption so ciphertext is stable across writes.** Rejected. It weakens the encryption to make ciphertext comparable, and the thing that wanted comparable bytes — configuration-change detection — is correctly solved by comparing plaintext configurations before sealing.
|
||||
|
||||
**Seal inside `TierConfig` rather than at the persistence boundary.** Rejected. That type's `Clone` is `redacted()`, so cloning drops secrets, and `tier_config_fingerprint` hashes the type; a nondeterministic sealed field inside it would be both lossy and churn-inducing.
|
||||
|
||||
**Encrypt with a node-local key instead of the KMS.** Rejected. The key would sit on the same host as the data, so it removes nothing the reserved path does not already remove, and it creates key material that nothing rotates.
|
||||
|
||||
## Prerequisites, in this order
|
||||
|
||||
1. Make an unparseable `bucket-targets.json` fail closed in `BucketMetadata::parse_all_configs` instead of substituting `BucketTargets::default()`. This is independently correct and it is what keeps a later mistake from being silent.
|
||||
2. Make an empty access key or secret key a typed `RemoteS3ClientError` in `remote_sdk_credentials`, so an emptied plaintext field fails on this node rather than as a signature rejection at the remote.
|
||||
3. Ship a reader-first release: every store tolerates the sealed field and the emptied plaintext field, and nothing writes either. For on-demand migration this means relaxing `deny_unknown_fields` for exactly that field name; for tiers it means reading the companion object when present.
|
||||
4. Only then enable writing, gated on KMS being configured and on a module switch in `rustfs/src/module_switches.rs` that defaults off in the release introducing it. Operators turn it on once every node reads the format. Existing plaintext records convert by re-submitting the configuration through its admin API; this work ships no in-place migration sweep.
|
||||
|
||||
Steps 1 through 3 each introduce a compatibility path that needs a `RUSTFS_COMPAT_TODO` marker and a matching entry in [compat-cleanup-register.md](compat-cleanup-register.md) when the code lands. This document adds neither, because the guard matches markers and register entries in both directions and an entry without a marker fails it.
|
||||
- A missing hook, a malformed envelope, an unknown envelope version, a failed decrypt, or an encryption-context mismatch is a typed error, per the AGENTS.md rule that required values return a typed error when absent or corrupt.
|
||||
- A seal failure fails the admin write. A configuration is never persisted with the secret dropped or left in clear text after the operator asked for sealing.
|
||||
- Redaction is unchanged and independent: admin responses keep returning `REDACTED`, and `Debug` implementations keep hiding secret fields whether or not the stored form is sealed.
|
||||
- Logs may carry the key id and envelope version. They never carry ciphertext, plaintext, or the encryption context's secret-adjacent values.
|
||||
|
||||
## Non-goals
|
||||
|
||||
Sealing the server configuration, IAM credentials or object data keys; changing which principals may read a configuration; migrating key material between KMS backends; and any at-rest protection when KMS is not configured — without KMS the stored form stays plaintext and the boundary described above is unchanged.
|
||||
Sealing the server config, IAM credentials, or object data keys; changing which principals may read a configuration; key material migration between KMS backends; and any at-rest protection when KMS is not configured — without KMS the stored form stays plaintext and the existing trust boundary (reserved bucket paths plus admin authorization) is unchanged.
|
||||
|
||||
@@ -1,177 +1,112 @@
|
||||
# Object Generation Authority And Recovery Contract
|
||||
# Object Transaction UUID And Generation-Fencing Contract
|
||||
|
||||
**Use this when:** changing object commit fencing, rollback, old-directory cleanup, prepared reads, quota settlement, or the metadata and RPC fields used by those operations.
|
||||
**Source of truth:** `crates/ecstore/src/set_disk/ops/object.rs` (`assign_object_transaction_epoch`, `verify_object_transaction_epoch_fence`); `crates/ecstore/src/set_disk/core/io_primitives.rs` (`rename_data_owned_with_fence`, `commit_rename_data_dir`); `crates/ecstore/src/disk/local.rs` (`rename_data`, `write_all_meta`); `crates/lock/src/distributed_lock.rs` (`DistributedLockGuard`, `LockLostSignal`). The implementation boundary below distinguishes existing behavior from the selected design.
|
||||
**Use this when:** adding or changing anything that fences a commit, scopes a read lease, gates old-directory cleanup, binds prepared pool reads, or settles quota against "the current version of an object", or when adding a field that rides internode RPC or `xl.meta`.
|
||||
**Source of truth:** `assign_object_transaction_epoch` in `crates/ecstore/src/set_disk/ops/object.rs` and `crates/ecstore/src/set_disk/ops/multipart.rs`; `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs`; `commit_rename_data_dir` and `RenameConvergence` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `PreparedPoolReadFallbackBarrier` in `crates/ecstore/src/store/rebalance.rs`; `crates/protos/src/node.proto`; env constants in `crates/config/src/constants/object.rs` and `crates/config/src/constants/internode.rs`.
|
||||
|
||||
## Decision And Implementation Boundary
|
||||
Design tracking lives in `rustfs/backlog#1326`. This document holds only the invariants.
|
||||
|
||||
The selected minimum authority is a **durable, ordered per-object decision protocol attached to the existing namespace-lock participant group**. The object transaction UUID remains an opaque operation/idempotency identifier. It is not an ordered lock epoch. Extending the existing lock group requires durable promises, accepted values, quorum decisions, and recovery; adding a counter to today's lock response is insufficient.
|
||||
## Authority
|
||||
|
||||
An independent service holding every object's full manifest is not selected. It would add a new routing, membership, availability, and metadata ownership system and require a wider read/write migration. The selected protocol stores the current decision and recoverable outstanding successor with the existing lock participants; object payload and prepared metadata remain on the existing storage disks. This is still new consensus and persistence work, not a small `RenameData` patch.
|
||||
The target contract requires **one per-object commit identity** consumed by commit fencing, read leases, cleanup, prepared reads, and quota settlement. No consumer may mint a second value and call it the same generation.
|
||||
|
||||
**Implementation status:** this document does not implement or claim distributed generation authority. Existing fencing remains an opt-in coordinator equality recheck. `rustfs/backlog#2251` cannot be completed by forwarding the UUID to disks and adding local CAS. Its implementation must be split at the protocol boundaries in [Required Implementation Boundaries](#required-implementation-boundaries), with the availability and rollout changes reviewed before strict activation. The two original requirements “commit with a quorum while a disk is unreachable” and “every disk immediately rejects every older request” cannot both hold; the precise target below preserves quorum availability.
|
||||
What exists today is an **object transaction UUID**, not the target authority:
|
||||
|
||||
Related contracts remain authoritative for their domains: [erasure-coding.md](erasure-coding.md) defines data durability and voting, [heal-concurrency-model.md](heal-concurrency-model.md) defines namespace-lock scope, [placement-repair-invariants.md](placement-repair-invariants.md) defines placement and repair admission, and [minio-file-format-compat.md](minio-file-format-compat.md) defines format interoperability.
|
||||
|
||||
## Current Guarantee And Counterexamples
|
||||
|
||||
`assign_object_transaction_epoch` generates a random UUID for gated PUT and CompleteMultipartUpload. `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs` stores it under both internal metadata prefixes. `verify_object_transaction_epoch_fence` re-reads quorum metadata before the rename fanout, outside the eventual per-disk mutation critical section. `ObjectTransactionEpochFence::Absent` currently covers both an absent object and existing metadata without a UUID. Cleanup receipts compare UUID equality. None of these operations is a durable distributed CAS.
|
||||
|
||||
The lock implementation already bounds lease validity. `LockLostSignal::is_lost` includes the conservative deadline; `DistributedLockGuard::run_heartbeat` retains prior deadlines after transient RPC failure and reports loss when refresh quorum is no longer valid. `LocalClient` in `crates/lock/src/client/local.rs` keeps `LocalGuardEntry` in an in-memory map; `LockResponse` in `crates/lock/src/types.rs` contains no persisted ballot or accepted object decision. Restarting a lock participant therefore cannot supply the durable order required here.
|
||||
|
||||
These schedules disprove a local UUID-CAS replacement, even assuming a perfect local mutex and atomic metadata replacement. They are protocol counterexamples, not claims that a multi-node fault test has already been run.
|
||||
|
||||
| Schedule | Result and implication |
|
||||
| Property | Current implementation |
|
||||
|---|---|
|
||||
| Four disks start at X; A captures expected X and stalls. B commits B on d1–d3, satisfying W=3. d4 has not heard from B. A reaches d4 with expected X. | d4's exact-CAS accepts A. A durable local `highest_ballot` also accepts if d4 never received B's ballot. Neither mechanism proves rejection on every disk after a quorum commit. |
|
||||
| A changes d1,d2 from X to A; after lock loss B changes d3,d4 from X to B. Each refuses the other's disks because expected X no longer matches. | Neither reaches W=3, even with all disks now reachable. Equality-CAS alone has no rule for choosing recovery, retaining uncertain work, or safely retiring it. This is a recovery/liveness counterexample, not proof of two successful intersecting write quorums. |
|
||||
| A reaches W=3; its reply is lost. A's coordinator restarts and sees a partial or changed disk view. A's rollback runs after B has replaced A. | Timeout is not evidence of abort. Restoring A's backup can erase B unless rollback names its own committed effect and consults a durable decision. |
|
||||
| A checks a lease/UUID; B later commits; A's blocking syscall resumes. | Another coordinator-side check, cancellation token, or process-local mutex cannot establish an atomic cross-node order. The mutation itself must consume the protocol state. |
|
||||
| Minting | `assign_object_transaction_epoch` mints a random non-nil UUID for PUT and CompleteMultipartUpload when the object-transaction gate is active. |
|
||||
| Persistence | Written through `FileInfo::set_object_transaction_epoch` into the version's internal metadata map under the dual-key contract (`x-rustfs-internal-*` / `x-minio-internal-*`). |
|
||||
| Fence check | The coordinator reads the current UUID (or `Absent`) and revalidates exact equality immediately before `rename_data`. |
|
||||
| Cleanup | Old-data cleanup receipts carry the committed UUID; reconciliation deletes only when the receipt UUID still equals the current object UUID. |
|
||||
|
||||
A lower-ballot write on an isolated stale disk cannot become the authoritative object. A disk that has applied B must never replace B with A. An uncontacted disk may retain older committed materialization until recovery; it must not vote that state as a newer decision or authorize cleanup. Requiring all disks to learn B before acknowledging it would change W to N or require successful isolation of every unreachable disk. That availability change is rejected for the selected design and must not be hidden inside E03's tests.
|
||||
This is an equality-CAS fence and cleanup identity. It is not a monotonic epoch, is not minted by the distributed lock grant, and is not compared atomically at each disk's `xl.meta` commit point. Documents and issues must call it the *object transaction UUID*, not proof that the generation authority exists.
|
||||
|
||||
## Authority, Identity, And State
|
||||
### Authority modes (one must be selected)
|
||||
|
||||
The authority key is `(bucket incarnation, bucket, object key)`, covering the whole object version set. It is not the S3 version ID. Deleting a noncurrent version, updating tags, or recording replication status can change the authoritative revision while the current S3 version remains the same. The bucket incarnation prevents reuse after bucket deletion/recreation.
|
||||
| Mode | Contract | Persistence requirement |
|
||||
|---|---|---|
|
||||
| Total-ordered fencing epoch | A lock grant returns a durable per-object `(term, counter)`; every disk rejects a lower epoch at the atomic metadata commit point; the value never regresses across lock-plane restart, failover, or minority recovery. | Quorum-persisted before grant, or derived from a durable term whose full comparison cannot regress. The in-memory distributed lock entry alone is insufficient. |
|
||||
| Opaque commit-generation identity | Consumers compare exact identity only; no `<` / `>` semantics. The authoritative commit performs an atomic expected-generation CAS; lease, cleanup, prepared-read, and quota contracts are phrased as "references this exact generation". | Atomic expected-identity comparison plus durable crash recovery. |
|
||||
|
||||
The protocol has separate typed values:
|
||||
The current UUID proves neither a durable total order nor a per-disk atomic CAS, so it does not decide between the modes.
|
||||
|
||||
- `Ballot = (configuration epoch, counter, durable proposer ID)`, compared lexicographically only within the specified authority configuration. The proposer persists a counter before use and raises it above every observed promise. Restart never resets it; exhaustion is an error. A different process boot gets a new transport epoch, not permission to reuse a ballot for different bytes.
|
||||
- `Generation = (object revision, operation UUID)`, allocated by a chosen successor decision. Revisions increase from the committed predecessor; UUIDs are compared for equality only. Never infer generation order from modification time, version ID, or UUID bytes.
|
||||
- `DecisionValue = (authority key, predecessor generation, successor generation, operation kind, semantic metadata digest, per-disk prepared metadata/data receipts, outcome identity)`. Disk-specific erasure indices, checksums, and metadata blobs are bound by individual receipts, not assumed byte-identical across disks. The semantic digest includes the full version set and relevant metadata, including fields omitted from ordinary read voting.
|
||||
## Consumer Binding
|
||||
|
||||
A decision does not require a new `FileInfo` positional field. The operation UUID stays in the existing metadata map. Durable authority records and local recovery records carry the revision, predecessor, ballot, and complete decision identity. They use separately versioned records; they cannot be inferred from a version's UUID alone. `xl.meta` remains a recoverable materialization of the chosen decision in strict mode.
|
||||
| Consumer | Binds generation how | Key invariant | Current state |
|
||||
|---|---|---|---|
|
||||
| Commit fence (PUT / CompleteMultipartUpload) | Checked at `rename`, rollback restore/delete, and cleanup mutation points using the selected rule | A stale writer is rejected on **all** disks; an already-ACK'd write is never rolled back | Opt-in UUID equality recheck before rename; no per-disk atomic comparison |
|
||||
| Read lease | Lease binds the exact generation observed at read time; GC runs only after every lease on that generation is released | Lease visible across nodes; crashed reader's lease reclaimed by TTL | Streaming/multipart GET holds the namespace read lock through EOF/drop (part-boundary coverage: `#6887`); no cross-node generation-bound registry |
|
||||
| Old-dir GC | Cleanup job carries the committed generation and confirms no lease owns `old_dir` before deleting | `old_dir != committed_dir`; a still-referenced directory is never deleted | UUID receipt equality (`#6077`); no lease consultation |
|
||||
| Prepared pool read | The prepared bundle carries the generation resolved during pool lookup; the chosen pool reuses it only after a match | Mismatch forces fallback to full metadata fanout | `PreparedPoolReadFallbackBarrier` (`#6889`) is a pool-local identity that fails closed / refetches on pool state change; it is not a cross-pool authority |
|
||||
| Quota reservation | Reserve / settle record binds the exact object generation (and the ordered epoch too, if selected) | A late commit cannot settle quota for a different committed generation | Durable per-bucket ledger with independent snapshot-lease fence tokens (`#6058`); not bound to the transaction UUID |
|
||||
|
||||
| Input/state | Required treatment |
|
||||
## Fence Coverage: Three Disk-Write Points
|
||||
|
||||
Checking generation only before the `rename` fanout is insufficient. The commit sequence is `tmp sync → data-dir rename → xl.meta commit → directory sync` in `crates/ecstore/src/disk/local.rs`, and `crates/ecstore/src/set_disk/core/io_primitives.rs` has two further detachable disk-write points:
|
||||
|
||||
1. **Rollback restore/delete.** On quorum failure each disk can restore backup metadata or delete the failed version. A stale writer's rollback must compare the expected generation, or it can overwrite or delete the winner's committed metadata. Panic, cancel, and timeout outcomes must be reaped into coordinator convergence rather than skip rollback through an early return.
|
||||
2. **`commit_rename_data_dir`.** A cancel-then-detach disk-write point; the coordinator's "reap all child tasks" must include it so a cancelled writer cannot bypass fence or lease and keep deleting directories.
|
||||
|
||||
If generation is validated only after the data-dir rename, a fenced writer may already have renamed its data-dir into the object path, leaving a staged orphan. Either move the fence ahead of the data-dir rename, or declare that orphan an accepted residue accounted for by GC metrics.
|
||||
|
||||
`RenameConvergence` (`AllSuccessIdentical` / `PartialCommit` / `SignatureDivergent` / `Unknown`) is a *post-commit* heal signal on the same `rename_data` path; the fence is a *commit* gate. They compose: the fence decides whether a convergence is produced, `RenameConvergence` classifies it. A fence-aware convergence variant would be an additive enum change.
|
||||
|
||||
## Transport And Security
|
||||
|
||||
Generation and derived tokens (lease, reservation) cross node boundaries in internode RPC bodies; every such flow must be signature-bound.
|
||||
|
||||
| Rule | Detail |
|
||||
|---|---|
|
||||
| Object never present | An explicit absent bootstrap state, established during strict cutover; no client-supplied `None` may authorize creation. |
|
||||
| Object deleted, including deletion of its last version | A durable tombstone head with its own revision; never revert to never-present. This prevents ABA and resurrection after a laggard rejoins. |
|
||||
| Null version | A real version-set member, distinct from absent. Replacing/removing it changes the object head. |
|
||||
| Delete marker | A real version-set member and, when latest, a deleted-current state. Preserve marker type and existing S3 behavior. |
|
||||
| Noncurrent version mutation | Compare the whole-object predecessor and the selected version's identity; commit a successor even if the latest S3 version is unchanged. |
|
||||
| Valid legacy metadata with no UUID | Import once during fenced cutover after ordinary quorum/format validation; assign a bootstrap generation in the authority. Never equate this with absent or silently import while old writers are still admitted. |
|
||||
| Missing, malformed, nil, or conflicting dual generation keys after strict enrollment | Typed corrupt/unsupported-state error; retain data for repair. No default to zero, absent, or a new UUID. |
|
||||
| Missing/lagging/replacement disk | A non-authoritative materialization target. Recover the chosen decision and validate its data before it can contribute; do not demand that it already equals the predecessor or blindly overwrite it. |
|
||||
| HMAC scope | Target audience, exact service/method, timestamp, nonce, canonical body digest, receiver replay (boot) epoch. The receiver consumes the nonce in a bounded replay cache; a transmitted-but-unconsumed nonce is not replay protection. |
|
||||
| Current substrate | RPC v2/v3 in `crates/ecstore/src/cluster/rpc/http_auth.rs` binds all of the above. Body-bound policy covers mutating disk RPCs including `RenameData`, whose versioned canonical body includes every `RenameDataRequest` field, so the `FileInfo` metadata map carrying the UUID is authenticated. |
|
||||
| Strict switches | `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` (`crates/config/src/constants/internode.rs`) are default-off rollout gates governed by [compat-cleanup-register.md](compat-cleanup-register.md). A generation capability may claim strong transport binding only after the relevant strict modes have converged fleet-wide. |
|
||||
| Acceptance tests per consumer | Method substitution, canonical body tamper, nonce replay, receiver restart, stripped-strict-metadata negatives. |
|
||||
|
||||
## Durable Decision Protocol
|
||||
## Encoding Rules
|
||||
|
||||
Use the existing namespace-lock participant identities and routing, with an explicitly persisted authority configuration. Lock voters and erasure disks are different sets: `Qlock = floor(lock_participants / 2) + 1` decides authority; the existing per-operation `Wdata` decides recoverable object durability. A lock vote is not a shard receipt. One-node deployments still persist their single voter's state. Membership changes cannot be inferred from whichever RPC endpoints answered.
|
||||
|
||||
Each participant persists, per authority key and successor slot, its highest promise, highest accepted `(ballot, DecisionValue)`, and the last learned committed head. Promises and accepted records survive unlock, TTL expiry, restart, and log compaction. Durable records are written through a storage boundary below the object API; writing them through PUT would recursively acquire the same authority. `LocalClient` must not become a filesystem implementation: the lock crate consumes an injected durability interface, while the storage owner implements it.
|
||||
|
||||
The following is a protocol contract, not pseudocode to paste into the current rename implementation:
|
||||
|
||||
1. Acquire the existing object namespace write lock for admission. Read/recover the latest authority head through a fresh quorum promise/recovery barrier; reading only cached learned-head markers is insufficient because a quorum may have accepted a value before its commit notification arrived. Resolve any accepted successor before returning a head or allocating another slot. An unavailable decision quorum is an explicit failure. The lock still prevents ordinary competing work, but it is not the safety proof after lease loss.
|
||||
2. Stage the new shards and exact replacement metadata in transaction-owned paths. Obtain Wdata receipts only after the required file and directory syncs. Receipts bind disk identity/incarnation, authority configuration, key, operation UUID, blob digests, and data directories. Preparing must not replace live metadata, remove old directories, or reuse a winner's directory. Metadata-only/delete operations also stage a recoverable replacement version set.
|
||||
3. For successor slot `head.revision + 1`, obtain Qlock durable promises for a unique ballot. Each response returns its accepted value, if any. Adopt the value with the highest accepted ballot among the promise quorum. Only if none was accepted may the proposer offer its own candidate with that predecessor. A recovered candidate is not replaced merely because its coordinator timed out or its lease expired.
|
||||
4. Validate the candidate's Wdata preparation receipts and predecessor, then obtain Qlock durable accepts of the **same** value at that ballot. A participant accepts only at or above its promise and must reject a different value at the same ballot. A value becomes chosen at Qlock acceptance. Quorum intersection plus adoption of the highest accepted value prevents a different value from being chosen in that slot. Retrying the same operation cannot allocate a second successor.
|
||||
5. Learn/persist the chosen decision and publish it on storage disks through the guarded local recovery protocol below. Return S3 success only after both a durable chosen decision and Wdata durable materializations satisfy the existing operation's rules. Decision chosen but publication incomplete is `OutcomeUnknown/PendingRecovery`, never authorization to roll back the decision. A later proposer first resolves the prior slot before allocating the next one.
|
||||
6. On a lost ACK, resolve by operation UUID and exact request digest. The results are `NotChosen`, `ChosenPendingPublication`, `Committed`, or `SupersededAfterCommit`. Reusing a UUID for different input is invalid. A timeout without a recovered decision stays unknown. Record idempotency outcomes until the client retry horizon and all dependent cleanup/accounting records have passed a durable retirement watermark; old requests then fail as expired rather than being treated as new.
|
||||
|
||||
The decision and payload retention lifetimes are coupled. Accepted/staged data is not garbage merely because no coordinator is alive. A new promise quorum can adopt a previously accepted value and finish it. If the required payload has been physically lost, fail closed and repair; never choose a different value for an already chosen slot. Persisted voter state that is lost or corrupt requires catch-up/replacement, not an empty voter with the same identity.
|
||||
|
||||
This is the minimum extension that turns the lock grant into a recoverable authority. A promise-only grant lacks outcome recovery; per-disk promises without a common decision allow minority uncertainty to escape into reads. The protocol requires review and executable state-machine tests before production integration. It does not imply that the current lock RPC is already a consensus implementation.
|
||||
|
||||
## Single-Disk Publication And Recovery
|
||||
|
||||
Local and remote disks execute the same guarded primitive. A remote RPC handler must decode and authenticate the request, then call that primitive; checking only in the RPC handler leaves local callers and deferred syscalls uncovered. All operations touching the object's metadata, backup, or referenced directories participate.
|
||||
|
||||
Under one object mutation guard, re-read local durable recovery state, verify the decision/configuration and local nonregression condition, record a write-ahead intent, sync it, perform data-directory rename and atomic metadata replacement, sync affected directories, then persist applied outcome. The guard, including ownership of any namespace/deletion lease, stays with the blocking syscall until it completes, even if its caller is cancelled. An async task disappearing must not release a guard while its syscall still runs.
|
||||
|
||||
A single filesystem rename does not atomically commit a sidecar plus `xl.meta`. The write-ahead record binds predecessor/successor identities and exact metadata bytes; startup recovery runs before disk readiness. Recovery replays a chosen intent forward and completes syncs. An unchosen staged operation stays private until authority recovery makes its retirement safe. Old snapshots never overwrite a newer applied local revision. Conflicting bytes for the same decision are corruption. Treat write/fsync errors and torn records as unknown until decoded and reconciled, not as successful rollback.
|
||||
|
||||
A laggard need not contain the predecessor. Recovery fetches the chosen decision and its verified metadata, reconstructs or validates its shards under [erasure-coding.md](erasure-coding.md), and installs that state. An empty replacement uses a fresh disk incarnation and cannot reuse old preparation receipts. A disk whose durable state claims a later decision than the supplied one rejects the operation; a conflicting same-revision digest is quarantined. Never erase a divergent disk merely because it is in the minority.
|
||||
|
||||
`rollback_committed_rename_std`, `rollback_inline_metadata_commit_std`, and `restore_metadata_backup` in `crates/ecstore/src/disk/local.rs` must become decision-aware before strict mode includes them. The permitted rollback is limited to a transaction's unchosen private preparation, or restoration proven by recovery to be necessary before any newer local effect. A chosen operation is repaired forward. Neither a client timeout nor a rename-tail error permits reverting an acknowledged decision.
|
||||
|
||||
`RenameConvergence` remains a post-publication repair signal. `PartialCommit`, `SignatureDivergent`, and `Unknown` do not decide which transaction won. Keep their diagnostics and quorum accounting; resolve authority first. Early ACK may still precede minority-tail completion after the two quorum conditions hold. Tests that inspect all disks must synchronize the tail or assert the permitted minority residue separately.
|
||||
|
||||
## Writer Participation
|
||||
|
||||
Every semantic metadata change advances the whole-object generation, including metadata-only writes. A physical repair that reproduces exactly the already chosen bytes preserves the generation and consumes that chosen decision; it must not create a new semantic value. The table specifies participation, not a generated inventory of every call site.
|
||||
|
||||
| Writer and current code boundary | Required generation behavior |
|
||||
| Rule | Reason |
|
||||
|---|---|
|
||||
| PUT / data COPY: `put_object_with_old_current_size_inner`, `copy_object` in `crates/ecstore/src/set_disk/ops/object.rs` | Stage, choose, publish a successor; preserve source read protection. A metadata-only COPY is also a semantic successor, even if data directories are shared. |
|
||||
| MPU: `complete_multipart_upload`, `new_multipart_upload`, `abort_multipart_upload` in `crates/ecstore/src/set_disk/ops/multipart.rs` | Complete chooses the destination object's successor. Part staging/upload metadata and abort remain in the upload namespace; they cannot delete a directory transferred to a chosen object decision. |
|
||||
| DELETE, batch DELETE, null/marker removal: `delete_object`, `delete_objects_with_accounting`, `delete_object_version` in `crates/ecstore/src/set_disk/ops/object.rs`; lifecycle callers in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | Each object has its own predecessor/decision; retain tombstone authority after the last version. Prefix deletion must enumerate decisions or prove a bucket-incarnation retirement barrier; a recursive bypass is forbidden in strict mode. |
|
||||
| Heal: `heal_object_with_explicit_version_regen` in `crates/ecstore/src/set_disk/ops/heal.rs` | Exact repair preserves the chosen generation and verifies full metadata identity. A version-list or semantic metadata change needs a successor. `no_lock` may skip admission only; it cannot bypass authority. In-place directory repair cannot remove a reader's live directory. |
|
||||
| Transition / restore: `transition_object`, `restore_transitioned_object`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`; `finalize_restore_metadata`, `update_restore_metadata` in `crates/ecstore/src/set_disk/replication.rs` | Each metadata transition is a successor, preserving existing operation-ID, remote tuple, and tier lease checks. Bind the transition transaction to the exact predecessor/successor; a late finalizer cannot rebase onto another restore operation. |
|
||||
| Replication status and metadata/tag/retention writeback: `put_object_metadata`, `put_object_tags`, `delete_object_tags`, `merge_replication_metadata_lww` in `crates/ecstore/src/set_disk/ops/object.rs`; callers in `crates/ecstore/src/bucket/replication/replication_resyncer.rs` | Commit a field-scoped successor conditional on the exact version/content identity. On conflict, reload and revalidate the mutation; never replay a full stale `FileInfo`. Existing LWW category rules remain applicable within that validation. |
|
||||
| Rebalance/decommission: `migrate_entry_version` in `crates/ecstore/src/services/rebalance/migration.rs`; `decommission_tier_free_version`, `decommission_tiered_object` in `crates/ecstore/src/set_disk/mod.rs`; `crates/ecstore/src/data_movement/mod.rs` | The authority key and lock group remain stable across pools. Stage the destination, choose the location/ownership successor, then retire the exact source receipt. Do not mint independent source and destination authorities. Existing placement and tier ownership fences remain required. |
|
||||
| Generic metadata entry points: `write_unique_file_info`, `update_object_meta_with_opts` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `LocalDisk::write_metadata`, `update_metadata`, `delete_version`, `delete_versions_internal`, `write_all_meta` in `crates/ecstore/src/disk/local.rs` | Consume a validated decision/recovery context or reject strict writes to enrolled objects. None may invent a generation, reset it through `fresh`, replace corrupt metadata with an unproven empty version set, or bypass durability with `no_persistence`. |
|
||||
| Rollback and GC: `rename_data_owned_with_fence`, `commit_rename_data_dir`, `reclaim_orphan_data_dirs` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `reconcile_old_data_cleanup_receipts` in `crates/ecstore/src/set_disk/ops/object.rs` | Consume the owning decision and exact directory references. Cleanup does not change object contents or grant a new semantic generation. It must use a durable retirement decision and local reader/deletion guards. |
|
||||
| **Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION`** (`crates/filemeta/src/filemeta.rs`). | `decode_xl_headers` in `crates/filemeta/src/filemeta/codec.rs` rejects newer values outright; a bump makes every new `xl.meta` unreadable by rolling-upgrade old nodes and by MinIO. See [minio-file-format-compat.md](minio-file-format-compat.md). |
|
||||
| **Do not add generation as a `FileInfo` struct field.** | Internode RPC serializes `FileInfo` with two msgpack encoders: positional-array encoding for the `read_version` family (a new positional field breaks mixed-version decode) and `encode_msgpack_named` (named-map) for `rename_data` in `rustfs/src/storage/rpc/node_service/disk.rs`. A field would have to be correct under both plus the JSON compatibility twin. Use the metadata map, which rides every encoder unchanged. |
|
||||
| **Metadata-map dual key.** | The UUID lives under `x-rustfs-internal-*` / `x-minio-internal-*`; missing, malformed, nil, or conflicting dual values fail closed when fencing is active. |
|
||||
| **No sidecar unless atomic.** | An epoch sidecar outside `xl.meta` is admissible only if it commits at the same atomic/CAS point as `xl.meta` with a specified crash-recovery protocol. None is implemented. |
|
||||
| **Regression guard.** | The real-MinIO `xl.meta` interop fixtures in `crates/filemeta/src/filemeta.rs` must keep passing: objects written by a new node stay readable by old RustFS nodes and by MinIO in both upgrade directions. |
|
||||
|
||||
Strict capability is withheld until every raw writer in `DiskAPI`, its local implementation, `DiskStore`, remote adapters, and server handlers has an enforced path. Internal authority persistence must use its own narrow storage primitive, not evade this rule by recursively calling generic object metadata writes.
|
||||
### Wire-encoding window (JSON and msgpack)
|
||||
|
||||
## Reads, Garbage Collection, And Accounting
|
||||
- Dual-encoded RPC fields exist twice in `crates/protos/src/node.proto`: a JSON `string` field and a msgpack `bytes *_bin` field (e.g. `file_info` and `file_info_bin` on `RenameDataRequest`). Senders emit both; receivers (`decode_msgpack_or_json` in `crates/ecstore/src/cluster/rpc/remote_disk.rs`) prefer `_bin` and fall back to JSON only when `_bin` is empty.
|
||||
- `rustfs_protos::internode_rpc_msgpack_only()` drops the JSON copy only when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` are set after the JSON-fallback metric reads zero fleet-wide.
|
||||
- Generation inside the `FileInfo` metadata map is carried in both copies automatically. Any new *top-level* generation datum must be added to both encodings and be safe under both msgpack encoders; a field in only one encoding is silently lost when a peer falls back.
|
||||
- `RenameDataRequest` has a versioned, injective canonical-body encoder covering both compatibility fields; a strict generation-capable request must reject missing or mismatched canonical-body metadata rather than downgrade to the unauthenticated JSON twin.
|
||||
|
||||
Strict reads need the chosen head, not a majority of arbitrary prepared/live UUIDs. Resolve the decision under the namespace read lock before accepting object metadata; validate the selected current or explicit version against it, and wait for or repair missing materialization. HEAD, GET, ListObjects/ListObjectVersions, scanner reads used for deletion, and prepared pool reads all need this distinction. A query may return an error while a chosen write is recovering; it must not expose an unchosen candidate or resurrect a retired version. This read-decision adapter is part of the strict-mode scope and is a reason E03 is larger than disk CAS.
|
||||
### Proto evolution
|
||||
|
||||
Keep the current namespace read lock through EOF/drop, including multipart part boundaries. This design does not replace it with a new cross-node generation lease registry. A strict implementation must also bind every deferred local/remote part open to the resolved generation and acquire protection on the disk that owns the directory before handing the read capability out. A reader that loses authority or cannot renew its disk protection must fail before another open; it cannot continue on an unvalidated cached pathname.
|
||||
No top-level proto field is required by the metadata-map UUID. If an ordered epoch or explicit expected-generation is ever added to proto, it uses **proto3 `optional`** (explicit presence). A non-optional scalar is forbidden: an old coordinator talking to a new disk decodes absence as a plausible zero.
|
||||
|
||||
`LocalDisk::acquire_snapshot_lease`, `renew_snapshot_lease`, `release_snapshot_lease`, and `delete_data_dir` in `crates/ecstore/src/disk/local.rs` provide disk-local path protection and deletion deferral. They are not proof of a fleet-wide object generation. Before reuse, their token must bind disk incarnation and exact generation/directory, and strict reads must reject a pre-restart token. Existing open file descriptors may finish reading an unlinked inode, but later part opens need a valid protected generation. This preserves streaming behavior without assuming a local mutex protects another node.
|
||||
## Mixed-Version Gate: One Direction
|
||||
|
||||
GC consumes a durable retirement authorization for exact directories no longer referenced by **any** retained version or pending accepted decision. Include retirement in the chosen successor that removes the last reference; if it was not recorded there, choose a metadata-neutral successor that records it before deletion. That successor advances the authority revision while preserving the S3 version contents; the cleanup syscall itself never mints an identity. Retirement prevents future repairs/reads from creating new references; a new reference requires a new decision and cannot revive a retired directory. At the destructive syscall, hold the object/directory guard, recheck the local chosen metadata references, `old_dir != committed_dir`, retirement/configuration identity, and local snapshot protection. Any uncertainty defers deletion. A stale cleanup receipt matching an earlier UUID is not enough. Across executor restart, replay the same retirement ID idempotently; do not convert a lost reply into a broader recursive delete.
|
||||
When generation enforcement is not explicitly requested, or fleet confirmation is absent, behavior falls back to current semantics. Fail-closed is reserved for an explicit administrator-confirmed strict rollout.
|
||||
|
||||
Prepared pool reads remain a separate optimization domain. In `crates/ecstore/src/store/rebalance.rs`, `prepare_latest_object_metadata_with_idx` collects candidates and revalidates a refetched winner with `validate_prepared_pool_refetch_identity` from `crates/ecstore/src/store/rebalance/support.rs`. `PreparedPoolReadFallbackBarrier` is a `#[cfg(test)]` scheduling fixture, not a production identity. Keep the all-pool resolution rules until a chosen location decision supplies equivalent evidence. The prepared bundle binds authority generation plus pool identity; mismatch requires a complete refetch or a typed failure, never selection of a different generation using the old bundle.
|
||||
| Flag (`crates/config/src/constants/object.rs`) | Default | Effect |
|
||||
|---|---|---|
|
||||
| `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` | false | With either flag absent, PUT/MPU neither persists nor consumes the transaction UUID. |
|
||||
| `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` | false | With both enabled, failure to obtain or retain the live fleet proof rejects the commit before rename. |
|
||||
|
||||
Quota remains a separate per-bucket arbitration domain. `QuotaLedger` and `settle` in `crates/ecstore/src/bucket/quota/reservation.rs` key reservations by operation UUID, validate object/size, and update under the ledger fence. A late settlement cannot remove a different reservation key; an absent/mismatched key fails or follows the existing idempotent abort rule. That proves key isolation, not that current generation A committed or that a ledger storage write is immune to stale-disk mutation.
|
||||
The fleet proof is currently borrowed from the remote-version-state writer rollout. It proves membership/process-epoch convergence for that feature only; it does not prove an epoch type, per-disk CAS support, or RPC strict-mode convergence, and must not be treated as the final generation handshake.
|
||||
|
||||
For strict mode, add the decision identity to the reservation/settlement binding. Settle a committed **historical** decision even if it has since been superseded, but only against its original reservation and recorded old/new sizes; demanding that it still be current would leak valid reservations. Abort only a recovered unchosen/retired operation. Unknown outcomes stay reserved and reconcile. Retain the existing conservative usage floor and `commit_started` recovery behavior. Ledger writes themselves require the authority protocol, with a documented lock order and no recursion through their own reservation path. No quota token is compared numerically to an object ballot or used to revoke a newer object's leases.
|
||||
### Capability negotiation (target)
|
||||
|
||||
## Restart, Membership, And Rollout
|
||||
Generation enforcement requires one **live fleet proof** containing at least: the selected authority version and comparison mode; the current membership/topology fingerprint and process epochs; support for every required disk mutation point; RPC signature/body/replay strict convergence; and the on-disk encoding version (the metadata-map UUID is version 1). Membership change or an old-node rejoin revokes the proof; revocation before commit fails an explicitly strict request and never rewrites or lowers a persisted generation. The proof may extend the authenticated fleet-proof machinery in `notification_sys` or the runtime capability contract; this document requires one shared token, not a mechanism.
|
||||
|
||||
Authority configuration is durable and includes participant identities, routing, bucket incarnation, protocol version, and quorum rules. Changing storage pool placement must not remap the authority key. A replaced voter starts as a non-voter, catches up durable promise/accepted/chosen state, and only joins through a quorum-approved configuration transition. Configuration change requires intersecting old/new decision quorums; losing the old quorum is a recovery incident, not permission to bootstrap a new empty authority. Offline data disks rejoin through generation-aware catch-up, independent of voter admission.
|
||||
## Open Decisions
|
||||
|
||||
A live capability proof must bind the authority configuration, topology, every participant process boot epoch, disk incarnations, writer/read/recovery protocol support, encoding version, and RPC signature/body/replay strictness. Restart, membership change, disk replacement, protocol downgrade, or a strict-transport setting change revokes it. Receivers revalidate before entering the publication critical section; accepted durable decisions survive proof revocation and are recovered under a fresh valid proof, never replayed as unvalidated requests. The remote-version-state fleet proof does not prove any of these generation capabilities.
|
||||
Blockers for calling the contract implemented:
|
||||
|
||||
The existing `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` and `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` flags in `crates/config/src/constants/object.rs` retain their current default-off behavior. They do not become a claim that the new protocol exists. If generation strictness is explicitly selected, missing capability is an error, never silent downgrade. No new environment variable is introduced by this document; a production gate must be documented with its implementation.
|
||||
|
||||
Strict enrollment requires quiescing old writers and readers for the enrolled namespace, recovering ambiguous operations, validating/importing legacy heads, persisting a strict-format/protocol marker, and enabling the complete fleet. New disks reject unbound legacy mutation RPCs for that namespace. Old binaries must be prevented from opening a strict-enrolled drive by a startup compatibility gate they understand before enrollment; an environment flag known only to new binaries is insufficient. Until that prerequisite is deployed, do not activate strict mode in a mixed fleet. Disabling flags after enrollment cannot drop durable authority; downgrade requires a separately verified quiescent materialization/export operation. Ordinary un-enrolled compatibility deployments keep their current behavior.
|
||||
|
||||
## Encoding And Transport
|
||||
|
||||
- Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION` in `crates/filemeta/src/filemeta.rs`. Do not add fields to positional-msgpack `FileInfo`; carry the UUID through the metadata map and protocol records through explicit versioned envelopes.
|
||||
- Write RustFS/MinIO internal metadata dual keys using `crates/utils/src/http/metadata_compat.rs`. Reject conflicting, nil, or malformed generation values; validate every persisted/RPC record again at consumption.
|
||||
- New proto values in `crates/protos/src/node.proto` require explicit presence (`optional` scalars or a present message), including absent/tombstone state. Bind expected/new generation, ballot, receipts, configuration, and outcome identity in the canonical body. The JSON and msgpack representations must carry identical semantics; absent data from an old peer cannot decode as a valid zero ballot.
|
||||
- `crates/ecstore/src/cluster/rpc/remote_disk.rs` and `rustfs/src/storage/rpc/node_service/disk.rs` must share local protocol behavior. Extend canonical encoders for every affected mutation, not only `RenameData`. Authenticate both compatibility representations and reject disagreement rather than falling back to a weaker JSON twin.
|
||||
- `crates/ecstore/src/cluster/rpc/http_auth.rs` supplies signature, canonical-body, and replay-scope checks. Strict generation capability requires fleet convergence of `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, and `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` from `crates/config/src/constants/internode.rs`. Cover method substitution, body tamper, stripped metadata, consumed nonce replay, and receiver restart.
|
||||
- Preserve real-MinIO metadata fixture decoding and supported old-RustFS compatibility before strict enrollment. Do not promise that a live MinIO binary can start a RustFS-written drive set; [minio-file-format-compat.md](minio-file-format-compat.md) explicitly excludes that direction. Strict authority records also impose a new deployment boundary even though the `xl.meta` container version is unchanged.
|
||||
|
||||
## Required Failure Outcomes
|
||||
|
||||
Every scenario must first prove its intended barrier, quorum, or crash point was reached, then inspect authority records, decoded metadata, directory references, return/error categories, and full GET bytes. Single-process barriers are insufficient evidence for the network-partition cases.
|
||||
|
||||
| Interleaving | Unique permitted outcome |
|
||||
|---|---|
|
||||
| A pauses after coordinator verification; A loses lock quorum while data RPC remains reachable; B chooses and publishes; A resumes rename. | A cannot become the chosen successor for B's predecessor or overwrite B on a disk that applied B. Isolated older materialization cannot vote as current. Read B exactly; reconcile laggards under B's decision. |
|
||||
| A's publication result is unknown; B succeeds; A's rollback/cleanup resumes. | Recover A's decision. Never restore/delete B's metadata or referenced data; retire only A-owned unchosen paths or separately authorized dead directories. |
|
||||
| Reader acquires generation G, consumes part 1; replacement retires G; GC attempts to delete part 2's directory. | The reader's valid disk protection defers GC; otherwise the reader fails before its next open. Never silently serve another generation or delete a directory still covered by valid protection. Repeat across reader and disk restarts. |
|
||||
| A and B each stage or partially publish on two of four disks; one coordinator dies; all disks return. | The promise/accept recovery rule preserves any chosen value or adopts the highest accepted candidate and finishes that slot. No guessing from UUID order, no permanent exact-CAS split, no replacement of a chosen value. |
|
||||
| Process dies after staging sync, intent sync, data rename, metadata replace, directory sync, or accepted/decision reply. | Reopen durable records before readiness. Unchosen work stays private; chosen work is replayed forward; lost ACK resolves to the original operation identity. Torn/insufficient evidence fails closed. |
|
||||
| Null version becomes a delete marker; a noncurrent version is deleted; delayed heal/metadata write resumes. | Whole-object predecessor no longer matches. Exact repair uses the chosen version set; no resurrection, marker-to-object conversion, or reset to never-present. |
|
||||
| Old coordinator reaches a new strict disk, new coordinator reaches an old disk, voter restarts, or transport strictness changes. | Compatibility behavior only in an un-enrolled namespace. Strict admission fails until a fresh complete proof and supported startup gate exist; no zero/missing-field fallback. |
|
||||
| Replacement disk is empty, or a restored minority disk has an old promise and old metadata. | It cannot vote as an initialized authority. Catch up chosen state and reconstruct data under a fresh incarnation; stale requests cannot bypass enrollment by presenting absent metadata. |
|
||||
| Quota settlement for A arrives after B commits; prepared pool refetch sees B instead of A. | Only A's original chosen outcome may settle A's reservation; B is unaffected. Prepared A cannot supply metadata/data for B without a new validated preparation. |
|
||||
|
||||
## Required Implementation Boundaries
|
||||
|
||||
These are durable ownership and acceptance boundaries, not permission to close the disk-fencing work before the protocol exists. The decision-model and availability changes require architecture review before production implementation. No unrelated external consensus service or full-manifest rewrite is authorized by this contract.
|
||||
|
||||
| Boundary | Required implementation and exit evidence |
|
||||
|---|---|
|
||||
| Durable authority substrate | Injected lock-participant persistence; typed ballot/configuration/decision values; prepare/accept/recover state machine; restart-safe proposer identities; corruption and voter replacement handling. Model/exhaustively test two competing proposers, lost replies, minority recovery, and every durable transition. The same-slot different-value property must be impossible. |
|
||||
| Disk publication boundary | Separate private preparation from publication; implement write-ahead intent, decision receipts, atomic guarded mutation, idempotent recovery, and directory retirement. Include inline/non-inline, every crash point, canceled blocking syscalls, ACK loss, and empty/lagging disks. Existing rollback helpers cannot remain an unguarded alternate route. |
|
||||
| Writer and read integration | Route every writer in the table and every strict read/scan decision through the authority; preserve data quorum and S3 version semantics. Bind prepared reads, MPU ownership transfer, tier operations, and quota outcomes. Demonstrate no raw metadata entry point bypasses strict mode. |
|
||||
| Fleet activation | Deploy the startup downgrade barrier first; import legacy/absent heads during quiescence; implement configuration/proof revocation and both RPC encodings. Run real multi-node lock/data-plane partitions and mixed-binary/restart tests. Only then can strict E03 acceptance run and activation be considered. |
|
||||
|
||||
The conservative immediate action is to keep the existing compatibility behavior and improve its local convergence/recovery independently. Those fixes must describe their smaller guarantee and must not advertise E03's distributed safety. A strict-only local CAS helper can be built behind the inactive capability boundary, but it cannot enable the feature or close the authority work.
|
||||
|
||||
## Performance And Activation Criteria
|
||||
|
||||
Measure the existing implementation and the full proposed path on identical machines, disk/filesystem, durability settings, network, object population, concurrency, and warmup. Include single hot-key and many-key 4 KiB PUT, 1 MiB PUT, metadata-only writes, and CompleteMultipartUpload with fixed part counts. Report throughput, p50/p95/p99, peak retained preparation/recovery bytes, recovery time, per-operation RPCs/fsyncs, and the object mutation critical-section duration. Include a slow minority disk, one voter loss, and restart recovery; a throughput result alone is insufficient.
|
||||
|
||||
The unoptimized proposal adds a lock-quorum promise round and an accept round with durable writes, plus decision learning/publication and local intent/applied-state persistence. Wdata staging remains separate. Read resolution may add an authority quorum round. Record actual overlapping rounds and fsync group commits; do not claim these costs disappear because the existing lock RPC is reused. Never hold a global lock across shard I/O, wait for all disks on the successful path, or weaken fsync/bitrot/quorum to recover throughput.
|
||||
|
||||
Activation requires all failure scenarios to pass with no acknowledged-data loss or wrong-generation read; no unexplained RPC/fsync amplification beyond the implemented phase budget; and an explicit performance acceptance recorded with the review. Use a conservative review trigger of more than 10% throughput loss or 15% p99 growth in any fixed-workload comparison: exceeding it blocks default activation until the architecture/operations owners accept the measured tradeoff or the implementation removes it. These are proposed rollout budgets, not measurements or performance claims. Without a reproducible baseline, leave strict mode unavailable.
|
||||
1. **Authority mode.** Total order or opaque exact-CAS. Do not retrofit ordering semantics onto the existing random UUID.
|
||||
2. **Complete `xl.meta`-writer coverage.** Enumerate commit rename, rollback restore/delete, cleanup, heal, transition, restore, replication, and data movement; each path compares/carries the selected generation or is proved incapable of replacing the authoritative identity.
|
||||
3. **Rollback as expected-generation CAS.** The quorum-failure rollback in `rename_data` restores backup metadata, not just a private temp file; it must run only when the stored generation still matches the failed writer's expectation.
|
||||
4. **Generation capability proof.** Extend the fleet proof or the runtime capability contract; one revalidatable token.
|
||||
5. **Read-lease and GC crash recovery.** Cross-node registry, TTL reclamation, lease-holder crash behavior, GC-executor recovery.
|
||||
6. **Quota reserve → commit → settle binding.** Relate the ledger's independent mutation tokens to the selected generation, with a concrete late-settle rejection test, or prove the fence is a separate arbitration domain that cannot cross-settle.
|
||||
7. **Prepared reads stay pool-local.** `PreparedPoolReadFallbackBarrier` validates freshness only within the pool that produced it; cross-pool ordering requires a common authority, and the multi-pool wait cannot be short-circuited without one.
|
||||
8. **Hot-path cost is a blocking metric.** Measure any added consensus write, fsync, fleet-proof lookup, lease operation, or centralized serialization under 4 KiB and hot-key/hot-bucket A/B.
|
||||
9. **Test infrastructure.** Multi-node, multi-pool, directed network-fault, and large-object budget for restart, mixed-version, and cross-node lease acceptance.
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
| Persisted free-version scan and re-enqueue after local-first expiry | `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs` |
|
||||
| Fenced free-version remote delete, local-marker cleanup, and rescan | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` (`cleanup_free_version_exact`) |
|
||||
| Durable manual transition job/task/result records | `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs` |
|
||||
| Dormant tier validation probe intent format and read-only core inspection | `crates/ecstore/src/services/tier/tier_probe_intent.rs` |
|
||||
| Manual run/status/cancel and transition-transaction reconcile admin routes | `rustfs/src/admin/handlers/ilm_transition.rs` |
|
||||
| `ObjectInfo` / `TransitionedObject` types | `crates/ecstore/src/object_api/types.rs` |
|
||||
| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
|
||||
@@ -120,12 +119,6 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --dry-run
|
||||
rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-objects 1000 --max-duration-seconds 30
|
||||
```
|
||||
|
||||
## Validation probe crash recovery status
|
||||
|
||||
Tier Add, Edit, and Verify currently validate a destination with a unique `rustfs-tier-probe-<uuid>` object and perform bounded compensation while the process remains alive. The `rustfs-tier-probe-intent-v1` decoder, canonical durable namespace, conditional storage primitives, state machine, and crate-level inspection type are present only as a dormant foundation. No validation path writes this record, no startup or periodic recovery scans it, and no admin HTTP route exposes it. V1 requires the owner to remain exactly equal to the immutable creator; takeover would require a new schema with explicit proof. Both durable writing and destructive recovery remain disabled until the fleet capability, operation-generation revalidation, provider timeout, retention, and operator contracts are approved.
|
||||
|
||||
Do not search the internal metadata bucket for these records as evidence that validation is crash recoverable: a current server does not create them. If a process is killed after the remote probe PUT but before cleanup, inspect the destination provider manually and retain ambiguous candidates. Never delete an empty or guessed version, and do not hand-create a probe intent to authorize cleanup.
|
||||
|
||||
Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
|
||||
|
||||
## Reconcile an unknown transition upload
|
||||
|
||||
@@ -21,8 +21,6 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
|
||||
|
||||
Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md).
|
||||
|
||||
The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation.
|
||||
|
||||
## Naming conventions
|
||||
|
||||
### Reserved test-name substrings (migration gate)
|
||||
|
||||
@@ -109,13 +109,3 @@ Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches
|
||||
## Change checklist
|
||||
|
||||
Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here.
|
||||
|
||||
## ECStore invariant selection
|
||||
|
||||
The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact.
|
||||
|
||||
The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered.
|
||||
|
||||
The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability.
|
||||
|
||||
Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review.
|
||||
|
||||
@@ -54,22 +54,6 @@ Fail-closed invariants every row enforces:
|
||||
|
||||
Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection.
|
||||
|
||||
### PUT completion fixtures
|
||||
|
||||
`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking.
|
||||
|
||||
The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup.
|
||||
|
||||
| Fixture | Completion boundary |
|
||||
|---|---|
|
||||
| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. |
|
||||
| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. |
|
||||
| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. |
|
||||
| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. |
|
||||
| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. |
|
||||
|
||||
Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks.
|
||||
|
||||
### Coverage gate
|
||||
|
||||
`full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is:
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
# Scanner Checkpoint Fixture
|
||||
|
||||
The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark.
|
||||
|
||||
Run the fixture and confirm the test filter selects a nonzero number of tests:
|
||||
|
||||
```sh
|
||||
cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list
|
||||
RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture
|
||||
```
|
||||
|
||||
The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well:
|
||||
|
||||
```sh
|
||||
RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture
|
||||
```
|
||||
|
||||
A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome.
|
||||
|
||||
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
|
||||
|
||||
The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches.
|
||||
@@ -121,11 +121,6 @@ fn map_bucket_target_error(err: BucketTargetError) -> S3Error {
|
||||
| BucketTargetError::BucketRemoteRemoveDisallowed { .. } => {
|
||||
S3Error::with_message(S3ErrorCode::InvalidRequest, err.to_string())
|
||||
}
|
||||
// A stored target configuration this node cannot decode is a
|
||||
// server-side data fault, not a bad request (rustfs/backlog#2282).
|
||||
BucketTargetError::BucketRemoteTargetsUnreadable { .. } => {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, err.to_string())
|
||||
}
|
||||
BucketTargetError::Io(io_err) => S3Error::with_message(S3ErrorCode::InternalError, io_err.to_string()),
|
||||
}
|
||||
}
|
||||
@@ -758,12 +753,7 @@ impl Operation for ListRemoteTargetHandler {
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let sys = BucketTargetSys::get();
|
||||
// An unreadable targets configuration must not be reported as an
|
||||
// empty target list (rustfs/backlog#2282).
|
||||
let targets = sys.list_targets(bucket, "").await.map_err(|e| {
|
||||
error!("list remote targets failed: {}", e);
|
||||
map_bucket_target_error(e)
|
||||
})?;
|
||||
let targets = sys.list_targets(bucket, "").await;
|
||||
|
||||
let targets: Vec<_> = targets
|
||||
.iter()
|
||||
|
||||
@@ -190,8 +190,7 @@ fn site_replicator_service_account_policy() -> S3Result<Policy> {
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}")))
|
||||
}
|
||||
|
||||
// Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation
|
||||
// -> bucket operation -> repair admission -> state -> per-bucket metadata.
|
||||
// Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata.
|
||||
// "state" is the distributed state-object lock in
|
||||
// crate::site_replication::state_lock, entered through
|
||||
// update_site_replication_state (P1-15). There is no process-local state
|
||||
@@ -435,7 +434,6 @@ pub fn register_site_replication_route(r: &mut S3Router<AdminOperation>) -> std:
|
||||
// into this module: startup sits below this layer and must not depend upwards. The admin
|
||||
// router is built before startup reconciles, so the hook is always installed in time.
|
||||
crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring);
|
||||
crate::site_replication_reconcile::register_site_replication_retry_drainer(reconcile_site_replication_retry_drain);
|
||||
|
||||
for (method, path, operation) in [
|
||||
(Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})),
|
||||
@@ -1805,61 +1803,28 @@ async fn reconcile_site_replication_buckets() -> S3Result<()> {
|
||||
/// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets
|
||||
/// from the stale endpoint. The pending marker in the persisted state closes that window.
|
||||
/// Skipping costs nothing — the timer comes back.
|
||||
async fn site_replication_reconcile_prerequisites_ready() -> bool {
|
||||
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
|
||||
return false;
|
||||
}
|
||||
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
result = "retry_queue_migration_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn reconcile_site_replication_retry_drain() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
|
||||
Box::pin(async {
|
||||
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
|
||||
return;
|
||||
};
|
||||
if !site_replication_reconcile_prerequisites_ready().await {
|
||||
return;
|
||||
}
|
||||
match load_site_replication_state().await {
|
||||
Ok(state) => {
|
||||
if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some()
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
Err(_) => return,
|
||||
}
|
||||
// Admission above observes a lifecycle-stable state. The lightweight
|
||||
// drain itself handles only idempotent bucket setup, reloads state
|
||||
// under the distributed repair lock, and shares that lock with bucket
|
||||
// deletion. Do not hold this process-local guard across peer I/O: an
|
||||
// outage recovery must not make admin add/edit/remove time out.
|
||||
drop(lifecycle);
|
||||
drain_site_replication_retry_queue_lightweight().await;
|
||||
})
|
||||
}
|
||||
|
||||
fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
|
||||
Box::pin(async {
|
||||
// The scheduler starts before IAM and the object store are guaranteed ready (IAM
|
||||
// bootstrap may still be recovering), so an early tick returns quietly instead of
|
||||
// logging a failure for every reconciler.
|
||||
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
|
||||
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(_lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if !site_replication_reconcile_prerequisites_ready().await {
|
||||
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
result = "retry_queue_migration_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1913,9 +1878,8 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Fut
|
||||
"admin site replication state"
|
||||
);
|
||||
}
|
||||
// The retry path re-checks membership from distributed state before
|
||||
// each request; release lifecycle before a potentially large replay.
|
||||
drop(lifecycle);
|
||||
// Failed peer deliveries recorded in the retry queue; runs behind the
|
||||
// same lifecycle guard and pending_* gates as the reconcilers above.
|
||||
drain_site_replication_retry_queue().await;
|
||||
})
|
||||
}
|
||||
@@ -3082,7 +3046,6 @@ fn set_pending_endpoint_refresh(state: &mut SiteReplicationState, pending: Pendi
|
||||
last_error: "endpoint target refresh pending".to_string(),
|
||||
updated_at: Some(OffsetDateTime::now_utc()),
|
||||
edit_generation: None,
|
||||
peer_unreachable: false,
|
||||
deletions_recorded: false,
|
||||
});
|
||||
state.pending_endpoint_refresh = Some(pending);
|
||||
@@ -3629,15 +3592,16 @@ const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000
|
||||
/// must be a site this state currently replicates with — the same membership
|
||||
/// rule the load-time mark pruning applies, so every mark recorded behind
|
||||
/// this check is one a reload would keep — and not this site itself, which
|
||||
/// never delivers edits to itself. The caller acknowledges an inadmissible
|
||||
/// fenced request without applying it: after a remove commits, an older
|
||||
/// in-flight retry from the departed origin must not recreate topology. Old
|
||||
/// peers remain compatible because their unstamped edits still follow the
|
||||
/// pre-fence path. The generation itself is NOT bounded here: a genuine
|
||||
/// origin whose hybrid clock persisted a wall-clock excursion allocates
|
||||
/// arbitrarily far in the future, and refusing to record its marks would
|
||||
/// strip the ordering fence from exactly the deliveries that still race —
|
||||
/// the staleness window on the read side is what defuses forged marks instead.
|
||||
/// never delivers edits to itself. The caller IGNORES an inadmissible fence
|
||||
/// rather than failing the request: the delivery applies exactly as an
|
||||
/// unstamped (pre-fence) delivery would, no high-water mark is read or
|
||||
/// written, and the worst a forged fence achieves is forfeiting an ordering
|
||||
/// guarantee its sender was never owed. The generation itself is NOT
|
||||
/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock
|
||||
/// excursion allocates arbitrarily far in the future, and refusing to
|
||||
/// record its marks would strip the ordering fence from exactly the
|
||||
/// deliveries that still race — the staleness window on the read side is
|
||||
/// what defuses forged marks instead.
|
||||
fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool {
|
||||
let (origin, generation) = fence;
|
||||
if origin != local_deployment_id && state.peers.contains_key(origin) {
|
||||
@@ -4827,135 +4791,105 @@ async fn backfill_existing_buckets_after_add(
|
||||
|
||||
let resync_id = Uuid::new_v4().to_string();
|
||||
for bucket in &buckets {
|
||||
let operation_name = bucket.name.clone();
|
||||
let lock_bucket = operation_name.clone();
|
||||
let operation_state = state.clone();
|
||||
let operation_local_peer = local_peer.clone();
|
||||
let operation_resync_id = resync_id.clone();
|
||||
let operation_bootstrap_token = bootstrap_token.map(str::to_owned);
|
||||
let bucket_errors = with_site_replication_bucket_mutation_lock(store.clone(), &lock_bucket, move || async move {
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
let name = &operation_name;
|
||||
let name = &bucket.name;
|
||||
|
||||
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
|
||||
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_versioning_setup_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: versioning setup failed: {err}"));
|
||||
continue;
|
||||
}
|
||||
match ensure_site_replication_bucket_setup(name).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
// Runtime targets unavailable: the setup silently no-ops, which would make the
|
||||
// downstream make-bucket broadcast and resync fail. Record it and skip so the
|
||||
// operator sees this bucket was not propagated instead of an unqualified success.
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_versioning_setup_failed",
|
||||
error = ?err,
|
||||
result = "backfill_bucket_setup_skipped",
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: versioning setup failed: {err}"));
|
||||
return errors;
|
||||
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
|
||||
continue;
|
||||
}
|
||||
match ensure_site_replication_bucket_setup(name).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
// Runtime targets unavailable: the setup silently no-ops, which would make the
|
||||
// downstream make-bucket broadcast and resync fail. Record it and skip so the
|
||||
// operator sees this bucket was not propagated instead of an unqualified success.
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_bucket_setup_skipped",
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
|
||||
return errors;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_bucket_setup_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: bucket setup failed: {err}"));
|
||||
}
|
||||
}
|
||||
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
|
||||
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
|
||||
// setting — object lock cannot be added after bucket creation.
|
||||
let lock_enabled = match metadata_sys::get(name).await {
|
||||
Ok(bm) => bm.lock_enabled,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_bucket_metadata_read_failed",
|
||||
fallback = "lock_enabled=false",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
if let Err(err) =
|
||||
broadcast_site_replication_make_bucket(name, lock_enabled, None, operation_bootstrap_token.as_deref()).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_make_bucket_broadcast_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
|
||||
}
|
||||
// Kick a resync toward every remote peer so existing objects travel across.
|
||||
for peer in operation_state.peers.values() {
|
||||
if peer.deployment_id == operation_local_peer.deployment_id
|
||||
|| same_identity_endpoint(&peer.endpoint, &operation_local_peer.endpoint)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
|
||||
let result = if manifest.target_arn.is_empty() {
|
||||
manifest
|
||||
} else {
|
||||
start_site_bucket_resync(name, &manifest.target_arn, &operation_resync_id).await
|
||||
};
|
||||
if result.status == "failed" {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
peer = %peer.endpoint,
|
||||
result = "backfill_resync_kick_failed",
|
||||
detail = %result.err_detail,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
|
||||
}
|
||||
}
|
||||
errors
|
||||
})
|
||||
.await;
|
||||
match bucket_errors {
|
||||
Ok(bucket_errors) => errors.extend(bucket_errors),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %lock_bucket,
|
||||
result = "backfill_bucket_mutation_lock_failed",
|
||||
bucket = %name,
|
||||
result = "backfill_bucket_setup_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{lock_bucket}: bucket mutation lock failed: {err}"));
|
||||
errors.push(format!("{name}: bucket setup failed: {err}"));
|
||||
}
|
||||
}
|
||||
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
|
||||
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
|
||||
// setting — object lock cannot be added after bucket creation.
|
||||
let lock_enabled = match metadata_sys::get(name).await {
|
||||
Ok(bm) => bm.lock_enabled,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_bucket_metadata_read_failed",
|
||||
fallback = "lock_enabled=false",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
if let Err(err) = broadcast_site_replication_make_bucket(name, lock_enabled, None, bootstrap_token).await {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
result = "backfill_make_bucket_broadcast_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
|
||||
}
|
||||
// Kick a resync toward every remote peer so existing objects travel across.
|
||||
for peer in state.peers.values() {
|
||||
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
|
||||
continue;
|
||||
}
|
||||
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
|
||||
let result = if manifest.target_arn.is_empty() {
|
||||
manifest
|
||||
} else {
|
||||
start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await
|
||||
};
|
||||
if result.status == "failed" {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
bucket = %name,
|
||||
peer = %peer.endpoint,
|
||||
result = "backfill_resync_kick_failed",
|
||||
detail = %result.err_detail,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6138,204 +6072,146 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result<SRPe
|
||||
serde_json::from_slice(body)
|
||||
}
|
||||
|
||||
fn ensure_add_bucket_set_matches_preflight(expected: &HashSet<String>, present: &HashSet<String>) -> S3Result<()> {
|
||||
let mut missing = expected.difference(present).cloned().collect::<Vec<_>>();
|
||||
if !missing.is_empty() {
|
||||
missing.sort_unstable();
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!(
|
||||
"bucket `{}` disappeared while site replication was being added; peers may already be joined — re-run replicate add",
|
||||
missing[0]
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let mut unexpected = present.difference(expected).cloned().collect::<Vec<_>>();
|
||||
if !unexpected.is_empty() {
|
||||
unexpected.sort_unstable();
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!(
|
||||
"bucket `{}` appeared while site replication was being added; peers may already be joined — re-run replicate add",
|
||||
unexpected[0]
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for SiteReplicationAddHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?;
|
||||
reject_site_replicator_on_public_admin(&cred)?;
|
||||
let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri);
|
||||
let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers);
|
||||
let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?;
|
||||
// Everything up to the commit below is preflight: peer probes, IAM
|
||||
// work and the join fan-out all talk to the network, so none of it may
|
||||
// run inside the state transaction. The snapshot read here is what the
|
||||
// `updated_at` CAS in the commit validates.
|
||||
let current_state = load_site_replication_state().await?;
|
||||
if pending_endpoint_refresh(¤t_state).is_some() {
|
||||
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
|
||||
}
|
||||
let local_peer = current_local_peer(&req, ¤t_state);
|
||||
let mut sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
let admin_access_key = cred.access_key.clone();
|
||||
let admission_store = current_object_store_handle()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
|
||||
let list_store = admission_store.clone();
|
||||
let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) =
|
||||
with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move {
|
||||
// The writer starts before the local bucket snapshot and stays
|
||||
// held through every peer join and the topology commit. A
|
||||
// delete followed by a same-name create therefore cannot hide
|
||||
// behind an unchanged final name set. Peer bootstrap callbacks
|
||||
// use their internal path and do not acquire this public-
|
||||
// mutation admission lock.
|
||||
let current_state = load_site_replication_state().await?;
|
||||
if pending_endpoint_refresh(¤t_state).is_some() {
|
||||
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
|
||||
}
|
||||
let local_peer = local_peer_at_endpoint(local_endpoint, ¤t_state);
|
||||
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
|
||||
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
|
||||
ensure_local_site_present(&mut sites, &local_peer);
|
||||
validate_add_sites(&sites, &local_peer)?;
|
||||
let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?;
|
||||
validate_add_preflight_topology(&preflight_infos, &local_peer)?;
|
||||
let expected_updated_at = current_state.updated_at;
|
||||
require_add_peer_tls_capability(&sites, &local_peer).await?;
|
||||
// Early exit on a state that moved under the preflight probes, BEFORE
|
||||
// the IAM write and the join fan-out change anything remote. Advisory
|
||||
// only — the binding check is the CAS inside the commit — but it fences
|
||||
// the common race off the side-effect path and refreshes the merge
|
||||
// base so the CAS window is only the join round trips.
|
||||
let latest_state = load_site_replication_state().await?;
|
||||
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
|
||||
let current_state = latest_state;
|
||||
let (service_account_access_key, service_account_secret_key) =
|
||||
ensure_site_replicator_service_account(&admin_access_key, false).await?;
|
||||
let expected_buckets: HashSet<String> =
|
||||
preflight_infos.iter().flat_map(|info| info.buckets.keys().cloned()).collect();
|
||||
let bootstrap_buckets: HashSet<String> = preflight_infos
|
||||
.iter()
|
||||
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
|
||||
.flat_map(|info| info.buckets.keys().cloned())
|
||||
.collect();
|
||||
let add_in_progress_guard =
|
||||
SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets.clone())?;
|
||||
let mut state = merge_add_sites(
|
||||
current_state,
|
||||
local_peer.clone(),
|
||||
sites.clone(),
|
||||
service_account_access_key.clone(),
|
||||
admin_access_key,
|
||||
replicate_ilm_expiry,
|
||||
);
|
||||
state.sync_state_initialized = true;
|
||||
let join_req = SRPeerJoinEnvelope {
|
||||
request: SRPeerJoinReq {
|
||||
svc_acct_access_key: service_account_access_key,
|
||||
svc_acct_secret_key: service_account_secret_key.clone(),
|
||||
svc_acct_parent: String::new(),
|
||||
peers: state.peers.clone(),
|
||||
updated_at: state.updated_at,
|
||||
},
|
||||
defer_sync_state_enable: true,
|
||||
};
|
||||
let peer_join_path = with_site_replication_bootstrap_token(
|
||||
SITE_REPLICATION_PEER_JOIN_PATH,
|
||||
&add_in_progress_guard.token.to_string(),
|
||||
);
|
||||
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
|
||||
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
|
||||
ensure_local_site_present(&mut sites, &local_peer);
|
||||
validate_add_sites(&sites, &local_peer)?;
|
||||
let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?;
|
||||
validate_add_preflight_topology(&preflight_infos, &local_peer)?;
|
||||
let expected_updated_at = current_state.updated_at;
|
||||
require_add_peer_tls_capability(&sites, &local_peer).await?;
|
||||
// Early exit on a state that moved under the preflight probes, BEFORE
|
||||
// the IAM write and the join fan-out change anything remote. Advisory
|
||||
// only — the binding check is the CAS inside the commit — but it fences
|
||||
// the common race off the side-effect path and refreshes the merge
|
||||
// base so the CAS window is only the join round trips.
|
||||
let latest_state = load_site_replication_state().await?;
|
||||
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
|
||||
let current_state = latest_state;
|
||||
let (service_account_access_key, service_account_secret_key) =
|
||||
ensure_site_replicator_service_account(&cred.access_key, false).await?;
|
||||
let bootstrap_buckets = preflight_infos
|
||||
.iter()
|
||||
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
|
||||
.flat_map(|info| info.buckets.keys().cloned())
|
||||
.collect();
|
||||
let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?;
|
||||
let mut state = merge_add_sites(
|
||||
current_state,
|
||||
local_peer.clone(),
|
||||
sites.clone(),
|
||||
service_account_access_key.clone(),
|
||||
cred.access_key.clone(),
|
||||
replicate_ilm_expiry,
|
||||
);
|
||||
state.sync_state_initialized = true;
|
||||
let join_req = SRPeerJoinEnvelope {
|
||||
request: SRPeerJoinReq {
|
||||
svc_acct_access_key: service_account_access_key,
|
||||
svc_acct_secret_key: service_account_secret_key.clone(),
|
||||
svc_acct_parent: String::new(),
|
||||
peers: state.peers.clone(),
|
||||
updated_at: state.updated_at,
|
||||
},
|
||||
defer_sync_state_enable: true,
|
||||
};
|
||||
let peer_join_path =
|
||||
with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string());
|
||||
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
|
||||
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
|
||||
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
|
||||
.send(&site.secret_key, &peer_join_req)
|
||||
.await?;
|
||||
|
||||
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
|
||||
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
|
||||
fallback_peer.deployment_id = preflight.deployment_id.clone();
|
||||
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
}
|
||||
// An explicit no-op join. The peer answered 200 but wrote nothing —
|
||||
// its persisted state is already newer than the snapshot it was
|
||||
// sent — so the add is only PARTIALLY configured and saying
|
||||
// "configured successfully" would be a lie (rustfs/rustfs#5963).
|
||||
// `None` (a MinIO peer, or one older than the field) is not a
|
||||
// no-op signal and is deliberately not reported.
|
||||
if join_response.applied == Some(false) {
|
||||
initial_sync_errors.push(format!(
|
||||
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
|
||||
the site is not configured against this peer",
|
||||
site.endpoint
|
||||
));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("peer join response from {} did not identify the requested site", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
validate_proposed_peer(&reconciled_peer).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("invalid peer join response from {}: {err}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
mark_unknown_peer_sync_enabled(&mut state.peers);
|
||||
|
||||
// Commit. The state transaction's CAS still fences topology
|
||||
// writers that do not use bucket admission. By this point
|
||||
// remote sites may already have accepted their joins, so a
|
||||
// mismatch asks the operator to re-run add and reconverge.
|
||||
let next_state = state;
|
||||
let present = list_store
|
||||
.list_bucket(&BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
.into_iter()
|
||||
.map(|bucket| bucket.name)
|
||||
.collect::<HashSet<_>>();
|
||||
ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?;
|
||||
let (state, edit_generation) = update_site_replication_state(move |state| {
|
||||
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
|
||||
));
|
||||
}
|
||||
adopt_add_commit_state(state, next_state);
|
||||
let edit_generation = next_peer_edit_generation(state);
|
||||
Ok((state.clone(), edit_generation))
|
||||
})
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
|
||||
.send(&site.secret_key, &peer_join_req)
|
||||
.await?;
|
||||
Ok((
|
||||
state,
|
||||
edit_generation,
|
||||
local_peer,
|
||||
service_account_secret_key,
|
||||
initial_sync_errors,
|
||||
add_in_progress_guard,
|
||||
))
|
||||
})
|
||||
.await?;
|
||||
|
||||
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
|
||||
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
|
||||
fallback_peer.deployment_id = preflight.deployment_id.clone();
|
||||
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
}
|
||||
// An explicit no-op join. The peer answered 200 but wrote nothing —
|
||||
// its persisted state is already newer than the snapshot it was
|
||||
// sent — so the add is only PARTIALLY configured and saying
|
||||
// "configured successfully" would be a lie (rustfs/rustfs#5963).
|
||||
// `None` (a MinIO peer, or one older than the field) is not a
|
||||
// no-op signal and is deliberately not reported.
|
||||
if join_response.applied == Some(false) {
|
||||
initial_sync_errors.push(format!(
|
||||
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
|
||||
the site is not configured against this peer",
|
||||
site.endpoint
|
||||
));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("peer join response from {} did not identify the requested site", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
validate_proposed_peer(&reconciled_peer).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("invalid peer join response from {}: {err}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
mark_unknown_peer_sync_enabled(&mut state.peers);
|
||||
|
||||
// Commit. The CAS runs inside the transaction, against the state the
|
||||
// transaction itself loaded — the peer round trips above took however
|
||||
// long they took, and only this check can tell whether the topology
|
||||
// this add was planned against is still the current one. The error
|
||||
// says so: by this point the remote sites already accepted their
|
||||
// joins, and re-running the add is what reconverges the local side.
|
||||
let next_state = state;
|
||||
let (state, edit_generation) = update_site_replication_state(move |state| {
|
||||
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
|
||||
));
|
||||
}
|
||||
adopt_add_commit_state(state, next_state);
|
||||
let edit_generation = next_peer_edit_generation(state);
|
||||
Ok((state.clone(), edit_generation))
|
||||
})
|
||||
.await?;
|
||||
|
||||
// The finalize fan-out delivers peer-edit payloads, so it carries the
|
||||
// generation allocated in the commit above: the receiving site orders
|
||||
@@ -7309,14 +7185,8 @@ impl Operation for SRPeerEditHandler {
|
||||
// The fence is self-reported — the shared service account means
|
||||
// the sender cannot be identified — so it is honoured only after
|
||||
// the admissibility check, against the same state it will gate.
|
||||
let commit_fence = match commit_fence {
|
||||
Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence),
|
||||
// A fenced edit can only come from a current remote peer. If
|
||||
// that origin left while the retry was in flight, applying
|
||||
// its body here would resurrect the removed topology.
|
||||
Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)),
|
||||
None => None,
|
||||
};
|
||||
let commit_fence =
|
||||
commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence));
|
||||
// Ordering fence: the sending site allocates the generation under
|
||||
// its state-object lock, so a delivery that lost the race carries
|
||||
// a generation this site has already passed. Applying it would
|
||||
@@ -9016,41 +8886,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_admission_starts_before_preflight_and_rejects_bucket_set_changes() {
|
||||
let expected = HashSet::from(["remote-owned".to_string(), "shared".to_string()]);
|
||||
let present = HashSet::from(["shared".to_string()]);
|
||||
|
||||
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
|
||||
.expect_err("a missing bootstrap bucket must reject the topology commit");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
let present = HashSet::from([
|
||||
"remote-owned".to_string(),
|
||||
"shared".to_string(),
|
||||
"created-during-add".to_string(),
|
||||
]);
|
||||
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
|
||||
.expect_err("a bucket created during add must reject the topology commit");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
let src = include_str!("site_replication.rs");
|
||||
let add = src
|
||||
.split("impl Operation for SiteReplicationAddHandler")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next())
|
||||
.expect("add handler block");
|
||||
let admission = add
|
||||
.find("with_site_replication_bucket_mutation_admission_lock")
|
||||
.expect("distributed mutation admission");
|
||||
let preflight = add.find("add_preflight_infos").expect("bucket preflight");
|
||||
let validation = add
|
||||
.find("ensure_add_bucket_set_matches_preflight")
|
||||
.expect("bucket-set validation");
|
||||
let commit = add.find("adopt_add_commit_state").expect("topology commit");
|
||||
assert!(admission < preflight && preflight < validation && validation < commit);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() {
|
||||
let src = include_str!("site_replication.rs");
|
||||
@@ -9347,19 +9182,13 @@ mod tests {
|
||||
);
|
||||
// Fence hardening: origin and generation are self-reported by a
|
||||
// caller the shared service account cannot identify, so the handler
|
||||
// must admit the fence against the same state it gates. An origin
|
||||
// removed while a retry was in flight is acknowledged without
|
||||
// applying the stale body; otherwise it could recreate topology.
|
||||
// must pass the fence through the admissibility check — against the
|
||||
// same state the fence gates, i.e. inside the transaction — before
|
||||
// reading or raising any high-water mark.
|
||||
assert!(
|
||||
handler_block.contains(
|
||||
"Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence)"
|
||||
),
|
||||
handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"),
|
||||
"SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction"
|
||||
);
|
||||
assert!(
|
||||
handler_block.contains("Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked))"),
|
||||
"SRPeerEditHandler must not apply a fenced edit after its origin leaves the current topology"
|
||||
);
|
||||
// P1-15 PR2: both halves of the fence and the edit they fence share
|
||||
// ONE transaction. Checking the fence against a state read outside the
|
||||
// lock would let the check pass on one snapshot and the write land on
|
||||
@@ -10523,9 +10352,8 @@ mod tests {
|
||||
/// A fence is self-reported: every site authenticates peer traffic with
|
||||
/// the same site-replicator credential, so a compromised peer can stamp
|
||||
/// ANY origin with ANY generation. An origin the receiver does not
|
||||
/// replicate with — or the receiver itself — is inadmissible and plants
|
||||
/// no mark; the handler acknowledges such a request without applying its
|
||||
/// body. A mark a compromised peer plants for a CURRENT origin cannot
|
||||
/// replicate with — or the receiver itself — is ignored and plants no
|
||||
/// mark; a mark a compromised peer plants for a CURRENT origin cannot
|
||||
/// silence that origin, because the staleness window refuses to fence on
|
||||
/// a mark implausibly far above the genuine deliveries.
|
||||
#[test]
|
||||
@@ -12963,7 +12791,6 @@ mod tests {
|
||||
last_error: "site replication is not enabled".to_string(),
|
||||
updated_at: Some(OffsetDateTime::now_utc()),
|
||||
edit_generation: None,
|
||||
peer_unreachable: false,
|
||||
deletions_recorded: false,
|
||||
}],
|
||||
..Default::default()
|
||||
@@ -13162,7 +12989,6 @@ mod tests {
|
||||
last_error: "peer offline".to_string(),
|
||||
updated_at: Some(OffsetDateTime::now_utc()),
|
||||
edit_generation: None,
|
||||
peer_unreachable: false,
|
||||
deletions_recorded: false,
|
||||
}],
|
||||
..Default::default()
|
||||
|
||||
@@ -252,16 +252,8 @@ pub(crate) async fn merged_list_objects_v2(
|
||||
.filter(|entry| merger.accepts(&entry.key().name))
|
||||
.collect();
|
||||
let keys: Vec<ListEntryKey> = kept.iter().map(SideEntry::key).collect();
|
||||
if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) {
|
||||
match fetch.side {
|
||||
MergeSide::Source => {
|
||||
degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
|
||||
continue;
|
||||
}
|
||||
MergeSide::Local => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
|
||||
}
|
||||
}
|
||||
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
|
||||
merger.push_page(fetch.side, keys, is_truncated, next_token);
|
||||
}
|
||||
|
||||
let outcome = merger.finish();
|
||||
@@ -348,12 +340,10 @@ async fn fetch_source_page(
|
||||
continuation_token: token,
|
||||
max_keys: params.max_keys,
|
||||
},
|
||||
// Everything under `filter.prefix` rolls into one common prefix. An
|
||||
// empty truncated probe must still follow its cursor before declaring
|
||||
// that prefix absent.
|
||||
// Everything under `filter.prefix` rolls into one common prefix, so a
|
||||
// single bounded listing settles whether it exists.
|
||||
SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest {
|
||||
prefix: Some(probe_prefix.as_str()),
|
||||
continuation_token: token,
|
||||
max_keys: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -378,8 +368,8 @@ async fn fetch_source_page(
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
!exists && page.is_truncated,
|
||||
if exists { None } else { page.next_continuation_token },
|
||||
false,
|
||||
None,
|
||||
))
|
||||
}
|
||||
_ => {
|
||||
@@ -427,17 +417,6 @@ async fn local_delete_markers(store: &Arc<ECStore>, bucket: &str, keys: &[String
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
|
||||
use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
|
||||
FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
|
||||
};
|
||||
use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
|
||||
use crate::app::storage_api::test::StoragePutObjReader;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::contract::object::ObjectIO as _;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
fn token(local: Option<&str>, local_done: bool) -> ListThroughToken {
|
||||
ListThroughToken {
|
||||
@@ -547,332 +526,4 @@ mod tests {
|
||||
assert!(degraded);
|
||||
assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local));
|
||||
}
|
||||
|
||||
/// Serves exactly the scripted S3 pages and joins every connection before
|
||||
/// returning. A source retry or unexpected operation fails the test.
|
||||
async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind listing source");
|
||||
let address = listener.local_addr().expect("listing source address");
|
||||
let server = tokio::spawn(async move {
|
||||
let mut requests = Vec::new();
|
||||
for body in pages {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept source listing");
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0; 4096];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let count = stream.read(&mut chunk).await.expect("read signed listing request");
|
||||
assert!(count > 0, "source request must include complete headers");
|
||||
request.extend_from_slice(&chunk[..count]);
|
||||
assert!(request.len() <= 32 * 1024, "listing request headers must be bounded");
|
||||
}
|
||||
let first_line = String::from_utf8_lossy(&request)
|
||||
.lines()
|
||||
.next()
|
||||
.expect("request line")
|
||||
.to_string();
|
||||
// The SDK joins the bucket endpoint with the LIST operation's `/` path.
|
||||
assert!(
|
||||
first_line.starts_with("GET /source-bucket/?"),
|
||||
"expected a path-style bucket-root LIST request, got {first_line:?}"
|
||||
);
|
||||
assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}");
|
||||
requests.push(first_line);
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(response.as_bytes()).await.expect("write source page");
|
||||
stream.shutdown().await.expect("finish source response");
|
||||
}
|
||||
requests
|
||||
});
|
||||
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
|
||||
}
|
||||
|
||||
fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
|
||||
let next = next
|
||||
.map(|token| format!("<NextContinuationToken>{token}</NextContinuationToken>"))
|
||||
.unwrap_or_default();
|
||||
let contents = key
|
||||
.map(|key| format!("<Contents><Key>{key}</Key><Size>1</Size></Contents>"))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next}{contents}</ListBucketResult>"
|
||||
)
|
||||
}
|
||||
|
||||
struct ListThroughTestState {
|
||||
bucket: String,
|
||||
module_enabled: bool,
|
||||
}
|
||||
|
||||
impl Drop for ListThroughTestState {
|
||||
fn drop(&mut self) {
|
||||
let sys = OnDemandMigrationSys::get();
|
||||
sys.remove(&self.bucket);
|
||||
sys.set_module_enabled(self.module_enabled);
|
||||
}
|
||||
}
|
||||
|
||||
async fn source_policy_request(
|
||||
pages: Vec<String>,
|
||||
policy: SourceErrorPolicy,
|
||||
resume_source: Option<&str>,
|
||||
filter_prefix: Option<&str>,
|
||||
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
|
||||
let store = shared_gating_ecstore().await;
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create list-through bucket");
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"z-local",
|
||||
&mut StoragePutObjReader::from_vec(vec![1]),
|
||||
&StorageObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("seed real local listing");
|
||||
let (endpoint, server) = scripted_list_source(pages).await;
|
||||
let sys = OnDemandMigrationSys::get();
|
||||
let _state_guard = ListThroughTestState {
|
||||
bucket: bucket.clone(),
|
||||
module_enabled: sys.is_module_enabled(),
|
||||
};
|
||||
sys.set_module_enabled(true);
|
||||
let config = OnDemandMigrationConfig {
|
||||
version: 1,
|
||||
enabled: true,
|
||||
source: SourceConfig {
|
||||
provider: Provider::Minio,
|
||||
endpoint: Some(endpoint),
|
||||
region: "us-east-1".into(),
|
||||
bucket: "source-bucket".into(),
|
||||
path_style: PathStyle::Path,
|
||||
credentials: Some(SourceCredentials {
|
||||
access_key: "test-access".into(),
|
||||
secret_key: "test-secret".into(),
|
||||
session_token: None,
|
||||
}),
|
||||
tls: TlsConfig::default(),
|
||||
},
|
||||
filter: FilterConfig {
|
||||
prefix: filter_prefix.map(str::to_string),
|
||||
..Default::default()
|
||||
},
|
||||
policy: PolicyConfig {
|
||||
list_through: true,
|
||||
source_error: policy,
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
sys.apply(&bucket, Some(&config)).await;
|
||||
assert!(
|
||||
sys.state(&bucket).expect("ODM state installed").client().is_ok(),
|
||||
"fake source client must build"
|
||||
);
|
||||
let continuation_token = resume_source.map(|source| {
|
||||
let token = ListThroughToken {
|
||||
t: "odm-list".into(),
|
||||
v: 1,
|
||||
local: None,
|
||||
local_done: false,
|
||||
source: Some(source.into()),
|
||||
source_done: false,
|
||||
last_key: None,
|
||||
};
|
||||
base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
|
||||
});
|
||||
let input = ListObjectsV2Input {
|
||||
bucket,
|
||||
max_keys: Some(2),
|
||||
continuation_token,
|
||||
delimiter: filter_prefix.map(|_| "/".to_string()),
|
||||
encoding_type: None,
|
||||
expected_bucket_owner: None,
|
||||
fetch_owner: None,
|
||||
optional_object_attributes: None,
|
||||
prefix: None,
|
||||
request_payer: None,
|
||||
start_after: None,
|
||||
};
|
||||
let request = S3Request {
|
||||
input,
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/?list-type=2"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
|
||||
)
|
||||
.await
|
||||
.expect("listing must complete within its bounded source budget");
|
||||
let requests = tokio::time::timeout(Duration::from_secs(5), server)
|
||||
.await
|
||||
.expect("source connections must finish")
|
||||
.expect("source server must not panic");
|
||||
(result, requests)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() {
|
||||
run_large_stack_test("list-through-source-policy", || async {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
|
||||
("HTTP_PROXY", None),
|
||||
("HTTPS_PROXY", None),
|
||||
("ALL_PROXY", None),
|
||||
("http_proxy", None),
|
||||
("https_proxy", None),
|
||||
("all_proxy", None),
|
||||
("NO_PROXY", Some("*")),
|
||||
("no_proxy", Some("*")),
|
||||
],
|
||||
async {
|
||||
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
|
||||
for next in [None, Some(""), Some("stuck")] {
|
||||
for key in [None, Some("a-source")] {
|
||||
let (result, requests) =
|
||||
source_policy_request(vec![source_xml(next, true, key)], policy, Some("stuck"), None).await;
|
||||
assert_eq!(requests.len(), 1, "a malformed source page must not be retried");
|
||||
assert!(requests[0].contains("continuation-token=stuck"));
|
||||
assert_source_policy_result(result, policy);
|
||||
}
|
||||
}
|
||||
let (result, requests) = source_policy_request(
|
||||
vec![
|
||||
source_xml(Some("stuck"), true, Some("a-source")),
|
||||
source_xml(Some("stuck"), true, None),
|
||||
],
|
||||
policy,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(requests.len(), 2, "the failure must occur during a real refill");
|
||||
assert!(!requests[0].contains("continuation-token="));
|
||||
assert!(requests[1].contains("continuation-token=stuck"));
|
||||
assert_source_policy_result(result, policy);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_through_empty_advancing_source_pages_reach_eof_on_the_handler_path() {
|
||||
run_large_stack_test("list-through-empty-source-pages", || async {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
|
||||
("HTTP_PROXY", None),
|
||||
("HTTPS_PROXY", None),
|
||||
("ALL_PROXY", None),
|
||||
("http_proxy", None),
|
||||
("https_proxy", None),
|
||||
("all_proxy", None),
|
||||
("NO_PROXY", Some("*")),
|
||||
("no_proxy", Some("*")),
|
||||
],
|
||||
async {
|
||||
for filter_prefix in [None, Some("photos/2024/")] {
|
||||
let source_key = if filter_prefix.is_some() {
|
||||
"photos/2024/a-source"
|
||||
} else {
|
||||
"a-source"
|
||||
};
|
||||
let (result, requests) = source_policy_request(
|
||||
vec![
|
||||
source_xml(Some("opaque-next"), true, None),
|
||||
source_xml(None, false, Some(source_key)),
|
||||
],
|
||||
SourceErrorPolicy::Propagate,
|
||||
None,
|
||||
filter_prefix,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(requests.len(), 2, "an empty truncated source page must reach its successor");
|
||||
assert!(requests[1].contains("continuation-token=opaque-next"));
|
||||
let response = result.expect("empty progressing source page is valid");
|
||||
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
|
||||
let output = response.output;
|
||||
let objects: Vec<_> = output
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|object| object.key.expect("listed object key"))
|
||||
.collect();
|
||||
if filter_prefix.is_some() {
|
||||
assert_eq!(objects, vec!["z-local"]);
|
||||
assert_eq!(
|
||||
output
|
||||
.common_prefixes
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|prefix| prefix.prefix.expect("rolled-up prefix"))
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["photos/"]
|
||||
);
|
||||
} else {
|
||||
assert_eq!(objects, vec!["a-source", "z-local"]);
|
||||
assert!(output.common_prefixes.unwrap_or_default().is_empty());
|
||||
}
|
||||
assert_eq!(output.key_count, Some(2));
|
||||
assert_eq!(output.is_truncated, Some(false));
|
||||
assert!(output.next_continuation_token.is_none());
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
|
||||
match policy {
|
||||
SourceErrorPolicy::Propagate => {
|
||||
let error = result.expect_err("propagate must expose malformed pagination");
|
||||
assert_eq!(error.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY));
|
||||
assert_eq!(error.code(), &S3ErrorCode::Custom("SourceUnavailable".into()));
|
||||
assert_eq!(error.message(), Some("invalid_pagination"));
|
||||
}
|
||||
SourceErrorPolicy::NotFound => {
|
||||
let response = result.expect("not_found must preserve the local listing");
|
||||
assert_eq!(
|
||||
response
|
||||
.headers
|
||||
.get("x-rustfs-on-demand-migration-list")
|
||||
.expect("local_only header"),
|
||||
"local_only"
|
||||
);
|
||||
let output = response.output;
|
||||
assert_eq!(
|
||||
output
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|object| object.key.expect("local key"))
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["z-local"]
|
||||
);
|
||||
assert_eq!(output.is_truncated, Some(false));
|
||||
assert_eq!(output.key_count, Some(1));
|
||||
assert!(output.next_continuation_token.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,8 +75,7 @@ use crate::auth::get_condition_values_with_client_info;
|
||||
use crate::error::ApiError;
|
||||
use crate::shared_types::RemoteAddr;
|
||||
use crate::site_replication::{
|
||||
cancel_site_replication_delete_bucket, commit_site_replication_delete_bucket, prepare_site_replication_delete_bucket,
|
||||
site_replication_bucket_meta_hook, site_replication_make_bucket_hook, with_site_replication_bucket_mutation_lock,
|
||||
site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook,
|
||||
};
|
||||
use crate::storage::storage_api::lock_bucket_targets_metadata;
|
||||
use http::StatusCode;
|
||||
@@ -1332,34 +1331,23 @@ impl DefaultBucketUsecase {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// Keep the local namespace mutation and its peer hook ordered across
|
||||
// every node in this site. Otherwise a delete waiting for repair
|
||||
// coordination can arrive after this create on remote sites.
|
||||
let operation_bucket = bucket.clone();
|
||||
let operation_store = store.clone();
|
||||
let make_result = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
|
||||
let make_result = operation_store
|
||||
.make_bucket(
|
||||
&operation_bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: false,
|
||||
lock_enabled,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if make_result.is_ok() {
|
||||
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
|
||||
if let Err(err) = site_replication_make_bucket_hook(&operation_bucket, lock_enabled).await {
|
||||
warn!(bucket = %operation_bucket, error = ?err, "site replication make bucket hook failed");
|
||||
}
|
||||
}
|
||||
make_result
|
||||
})
|
||||
.await?;
|
||||
let make_result = store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: false,
|
||||
lock_enabled,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
match make_result {
|
||||
Ok(()) => {}
|
||||
Ok(()) => {
|
||||
// Invalidate the bucket validation cache so subsequent GETs
|
||||
// see the newly created bucket immediately.
|
||||
crate::storage::invalidate_bucket_validation_cache(&bucket);
|
||||
}
|
||||
Err(StorageError::BucketExists(_)) => {
|
||||
// Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK;
|
||||
// non-owner gets 409 BucketAlreadyExists.
|
||||
@@ -1370,6 +1358,10 @@ impl DefaultBucketUsecase {
|
||||
Err(e) => return Err(ApiError::from(e).into()),
|
||||
}
|
||||
|
||||
if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed");
|
||||
}
|
||||
|
||||
let output = CreateBucketOutput::default();
|
||||
counter!("rustfs_create_bucket_total").increment(1);
|
||||
let result = Ok(S3Response::new(output));
|
||||
@@ -1405,41 +1397,16 @@ impl DefaultBucketUsecase {
|
||||
authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
|
||||
}
|
||||
|
||||
// Keep the local namespace mutation and its peer hook ordered across
|
||||
// every node in this site so an older delete cannot overtake a new
|
||||
// same-name make while it waits for repair coordination.
|
||||
let operation_bucket = input.bucket.clone();
|
||||
let operation_store = store.clone();
|
||||
with_site_replication_bucket_mutation_lock(store, &input.bucket, move || async move {
|
||||
let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?;
|
||||
let delete_result = operation_store
|
||||
.delete_bucket(
|
||||
&operation_bucket,
|
||||
&DeleteBucketOptions {
|
||||
force,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match delete_result {
|
||||
Ok(()) => {
|
||||
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
|
||||
if let Some(intent) = intent
|
||||
&& let Err(err) = commit_site_replication_delete_bucket(&intent).await
|
||||
{
|
||||
warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed");
|
||||
}
|
||||
Ok::<(), S3Error>(())
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(intent) = intent {
|
||||
cancel_site_replication_delete_bucket(intent).await;
|
||||
}
|
||||
Err(S3Error::from(ApiError::from(err)))
|
||||
}
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
store
|
||||
.delete_bucket(
|
||||
&input.bucket,
|
||||
&DeleteBucketOptions {
|
||||
force,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// Drop every cached object body for the now-deleted bucket so dead
|
||||
// bytes do not sit resident until TTL. Covers both the normal and the
|
||||
@@ -1448,9 +1415,16 @@ impl DefaultBucketUsecase {
|
||||
let cache_adapter = current_object_data_cache_for_context(self.context.as_deref());
|
||||
let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await;
|
||||
|
||||
// Invalidate bucket validation cache
|
||||
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
|
||||
|
||||
// Re-evaluate lifecycle and replication after bucket removal.
|
||||
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
|
||||
|
||||
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
|
||||
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
|
||||
}
|
||||
|
||||
// Notify peers to drop their cached metadata for the now-deleted bucket.
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
notify_bucket_metadata_delete(input.bucket.clone(), request_context);
|
||||
|
||||
@@ -394,7 +394,7 @@ impl DefaultObjectUsecase {
|
||||
// Bucket metadata uses the bucket name as its namespace-lock key. Load
|
||||
// every copy-time bucket snapshot before a same-object key can collide
|
||||
// with that key (for example, copying `bucket/bucket` onto itself).
|
||||
let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?;
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
|
||||
if cp_src_dst_same && key == bucket {
|
||||
dst_opts.object_lock_config_snapshot =
|
||||
@@ -1388,48 +1388,4 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_copy_object_refuses_a_bucket_whose_encryption_config_is_unreadable() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("copy-sse-unreadable-{}", Uuid::new_v4());
|
||||
let source = "source.bin";
|
||||
let destination = "destination.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unreadable-encryption copy bucket must be created");
|
||||
let mut reader = PutObjReader::from_vec(b"copied while the bucket still had a readable configuration".to_vec());
|
||||
store
|
||||
.put_object(&bucket, source, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copy source object must be written");
|
||||
install_unreadable_bucket_sse_config(&bucket).await;
|
||||
|
||||
let input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: source.into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(destination.to_string())
|
||||
.build()
|
||||
.expect("copy input must build");
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
|
||||
let err = Box::pin(usecase.execute_copy_object(build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect_err("an unreadable bucket encryption configuration must refuse the copy");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let lookup_err = store
|
||||
.get_object_info(&bucket, destination, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a refused copy must not leave a destination object behind");
|
||||
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2037,7 +2037,7 @@ impl DefaultObjectUsecase {
|
||||
let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5);
|
||||
|
||||
let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?);
|
||||
let bucket_sse_config = load_bucket_default_sse_config(&bucket).await?;
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse(
|
||||
bucket_sse_config.as_ref().map(|(config, _timestamp)| config),
|
||||
original_sse,
|
||||
|
||||
@@ -1485,9 +1485,8 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let bucket_sse_config = load_bucket_default_sse_config(&bucket).await;
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start);
|
||||
let bucket_sse_config = bucket_sse_config?;
|
||||
debug!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
component = "app",
|
||||
@@ -3913,121 +3912,4 @@ mod tests {
|
||||
.expect_err("writes after the zero-byte quota update must be denied");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_put_object_refuses_a_bucket_whose_encryption_config_is_unreadable() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("put-sse-unreadable-{}", Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unreadable-encryption PUT bucket must be created");
|
||||
install_unreadable_bucket_sse_config(&bucket).await;
|
||||
|
||||
let payload = Bytes::from_static(b"an operator mandated encryption for this bucket");
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
|
||||
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
|
||||
.build()
|
||||
.expect("PUT input must build");
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect_err("an unreadable bucket encryption configuration must refuse the write");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let lookup_err = store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a refused PUT must not leave an object behind");
|
||||
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_put_object_still_writes_plaintext_without_bucket_encryption() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("put-sse-absent-{}", Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("plaintext PUT bucket must be created");
|
||||
|
||||
let payload = Bytes::from_static(b"no default encryption is configured for this bucket");
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
|
||||
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
|
||||
.build()
|
||||
.expect("PUT input must build");
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
|
||||
Box::pin(usecase.execute_put_object(&FS::new(), build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect("a bucket without default encryption must still accept a plaintext write");
|
||||
|
||||
let stored = store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the plaintext object must be readable");
|
||||
assert_eq!(stored.size, i64::try_from(payload.len()).expect("test payload length must fit i64"));
|
||||
assert!(
|
||||
!stored
|
||||
.user_defined
|
||||
.keys()
|
||||
.any(|key| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION)
|
||||
|| key.starts_with("x-rustfs-encryption-")
|
||||
|| key.starts_with("x-minio-encryption-")),
|
||||
"the object must carry no encryption metadata: {:?}",
|
||||
stored.user_defined
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_put_object_extract_refuses_a_bucket_whose_encryption_config_is_unreadable() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("extract-sse-unreadable-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unreadable-encryption extract bucket must be created");
|
||||
install_unreadable_bucket_sse_config(&bucket).await;
|
||||
|
||||
let payload = Bytes::from_static(b"archive bytes that must never be unpacked in plaintext");
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("archive.tar".to_string())
|
||||
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
|
||||
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
|
||||
.build()
|
||||
.expect("extract PUT input must build");
|
||||
let mut req = build_request(input, Method::PUT);
|
||||
req.headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true"));
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&FS::new(), req))
|
||||
.await
|
||||
.expect_err("an unreadable bucket encryption configuration must refuse the extract upload");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let lookup_err = store
|
||||
.get_object_info(&bucket, "archive.tar", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a refused extract upload must not leave an object behind");
|
||||
assert!(is_err_object_not_found(&lookup_err), "{lookup_err}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,129 +269,6 @@ pub(super) fn resolve_bucket_default_sse(
|
||||
(effective_sse, effective_kms_key_id)
|
||||
}
|
||||
|
||||
/// The bucket's default encryption configuration for a write path.
|
||||
///
|
||||
/// `Ok(None)` carries one meaning only — this bucket has no default encryption
|
||||
/// — and the write proceeds in plaintext exactly as before. Every other
|
||||
/// outcome refuses the write rather than collapsing onto that same value: an
|
||||
/// encryption blob that exists but cannot be read fails closed in
|
||||
/// `get_sse_config` since rustfs/rustfs#7172, and swallowing that error here
|
||||
/// stores plaintext into a bucket whose operator mandated encryption, with
|
||||
/// nothing returned to the client and nothing in the object to tell it apart
|
||||
/// afterwards (rustfs/backlog#2287).
|
||||
///
|
||||
/// The states the lookup can report, and what each one does:
|
||||
///
|
||||
/// * configured and readable — apply the bucket default;
|
||||
/// * no encryption blob at all, including a bucket that does not exist and a
|
||||
/// bucket whose metadata document is absent — `ConfigNotFound`, so a cold
|
||||
/// cache and a missing bucket are never turned into a refusal, and the write
|
||||
/// still fails later with its own `NoSuchBucket`;
|
||||
/// * blob present but unparseable — deterministic, so retrying cannot help;
|
||||
/// surfaces as `InternalError` until an operator repairs or removes it;
|
||||
/// * the metadata read itself failed (namespace lock, quorum, disk, an
|
||||
/// uninitialized metadata system) — transient, and the typed error maps to
|
||||
/// the retryable `ServiceUnavailable`.
|
||||
///
|
||||
/// The last two are distinguished by the typed error the accessor returns, not
|
||||
/// re-derived here: [`ApiError`] already separates them. This mirrors
|
||||
/// `prepare_sse_configuration` in `storage::sse`, the resolver the multipart
|
||||
/// writer uses, which has always failed closed on the same lookup.
|
||||
pub(super) async fn load_bucket_default_sse_config(
|
||||
bucket: &str,
|
||||
) -> S3Result<Option<(ServerSideEncryptionConfiguration, OffsetDateTime)>> {
|
||||
classify_bucket_default_sse_lookup(bucket, metadata_sys::get_sse_config(bucket).await)
|
||||
}
|
||||
|
||||
fn classify_bucket_default_sse_lookup(
|
||||
bucket: &str,
|
||||
lookup: Result<(ServerSideEncryptionConfiguration, OffsetDateTime), StorageError>,
|
||||
) -> S3Result<Option<(ServerSideEncryptionConfiguration, OffsetDateTime)>> {
|
||||
match lookup {
|
||||
Ok(config) => Ok(Some(config)),
|
||||
Err(err) if err == StorageError::ConfigNotFound => Ok(None),
|
||||
Err(err) => {
|
||||
let api_error = ApiError::from(err);
|
||||
error!(
|
||||
event = "bucket_sse_config_lookup_failed",
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
result = "write_refused",
|
||||
bucket = %bucket,
|
||||
code = %api_error.code.as_str(),
|
||||
error = %api_error,
|
||||
"Bucket default encryption is unreadable; refusing the write instead of storing plaintext"
|
||||
);
|
||||
Err(api_error.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod bucket_default_sse_lookup_tests {
|
||||
use super::*;
|
||||
use s3s::dto::{ServerSideEncryptionByDefault, ServerSideEncryptionRule};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
fn sse_config() -> ServerSideEncryptionConfiguration {
|
||||
ServerSideEncryptionConfiguration {
|
||||
rules: vec![ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault {
|
||||
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
kms_master_key_id: None,
|
||||
}),
|
||||
blocked_encryption_types: None,
|
||||
bucket_key_enabled: None,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absent_configuration_still_writes_plaintext() {
|
||||
let resolved = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ConfigNotFound))
|
||||
.expect("a bucket without default encryption must keep writing plaintext");
|
||||
|
||||
assert!(resolved.is_none());
|
||||
assert_eq!(resolve_bucket_default_sse(None, None, None, false), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_readable_configuration_is_returned() {
|
||||
let resolved = classify_bucket_default_sse_lookup("bucket", Ok((sse_config(), OffsetDateTime::UNIX_EPOCH)))
|
||||
.expect("a readable configuration must not refuse the write")
|
||||
.expect("a readable configuration must be applied");
|
||||
|
||||
assert_eq!(resolved.0.rules.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_configuration_refuses_the_write() {
|
||||
let err = classify_bucket_default_sse_lookup(
|
||||
"bucket",
|
||||
Err(StorageError::other("persisted bucket encryption configuration is invalid")),
|
||||
)
|
||||
.expect_err("a corrupt encryption blob must never degrade to plaintext");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unavailable_metadata_read_refuses_the_write_as_retryable() {
|
||||
let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ErasureReadQuorum))
|
||||
.expect_err("an unreadable metadata subsystem must never degrade to plaintext");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_bucket_keeps_its_own_error() {
|
||||
let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::BucketNotFound("bucket".to_string())))
|
||||
.expect_err("a bucket-not-found lookup must not be reported as an encryption failure");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod deadlock_request_guard_tests {
|
||||
use super::DeadlockRequestGuard;
|
||||
|
||||
@@ -96,36 +96,3 @@ pub(super) fn real_cold_fill_plan(
|
||||
};
|
||||
plan
|
||||
}
|
||||
|
||||
/// A store with an ambient `AppContext`, for tests that drive a handler end to
|
||||
/// end without the object-data-cache overrides of
|
||||
/// [`real_cold_fill_test_context`].
|
||||
pub(super) async fn real_store_test_context() -> (Arc<ECStore>, Arc<AppContext>) {
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
if current_app_context().is_none() {
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
}
|
||||
let ambient = current_app_context().expect("real-store tests require an ambient AppContext");
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
(store, context)
|
||||
}
|
||||
|
||||
/// Leave the bucket in the state a damaged encryption blob produces: the raw
|
||||
/// document is retained and the typed configuration stays `None`, which is the
|
||||
/// durable "exists but cannot be read" signal `get_sse_config` fails closed on
|
||||
/// (rustfs/rustfs#7172).
|
||||
pub(super) async fn install_unreadable_bucket_sse_config(bucket: &str) {
|
||||
use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system must be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket).await.expect("bucket metadata must be cached")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration>truncated".to_vec();
|
||||
metadata.sse_config = None;
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("unreadable bucket encryption configuration must be installed");
|
||||
}
|
||||
|
||||
@@ -29,13 +29,11 @@ pub(crate) fn EndpointServerPools(
|
||||
pub(crate) mod s3 {
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input,
|
||||
ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
|
||||
};
|
||||
pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::{S3Request, S3Response};
|
||||
}
|
||||
|
||||
pub(crate) mod admin {
|
||||
|
||||
@@ -22,57 +22,6 @@ pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "confi
|
||||
|
||||
pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock<RwLock<()>> = LazyLock::new(|| RwLock::new(()));
|
||||
|
||||
const SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX: &str = "config/site-replication/bucket-mutation";
|
||||
pub(crate) const SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH: &str =
|
||||
"config/site-replication/bucket-mutation-admission.lock";
|
||||
|
||||
pub(crate) fn site_replication_bucket_mutation_lock_path(bucket: &str) -> String {
|
||||
format!("{SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX}/{bucket}.lock")
|
||||
}
|
||||
|
||||
pub(crate) async fn with_site_replication_bucket_mutation_lock<F, Fut, T>(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
operation: F,
|
||||
) -> S3Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let mutation_store = store.clone();
|
||||
let mutation_path = site_replication_bucket_mutation_lock_path(bucket);
|
||||
with_config_object_read_lock(
|
||||
store,
|
||||
SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(),
|
||||
move || async move {
|
||||
with_config_object_write_lock(mutation_store, mutation_path, operation)
|
||||
.await
|
||||
.map_err(|err| S3Error::from(ApiError::from(err)))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| S3Error::from(ApiError::from(err)))?
|
||||
}
|
||||
|
||||
/// Exclude every local bucket namespace mutation from an add's local preflight
|
||||
/// snapshot until its topology commit. Peer bootstrap callbacks do not enter
|
||||
/// this public-mutation admission path, so they can finish while the writer is
|
||||
/// held; post-commit fan-out and backfill must run after it is released.
|
||||
pub(crate) async fn with_site_replication_bucket_mutation_admission_lock<F, Fut, T>(
|
||||
store: Arc<ECStore>,
|
||||
operation: F,
|
||||
) -> S3Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation)
|
||||
.await
|
||||
.map_err(|err| S3Error::from(ApiError::from(err)))?
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct SiteReplicationBootstrapPlan {
|
||||
pub(crate) iam_items: Vec<SRIAMItem>,
|
||||
@@ -380,91 +329,6 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
/// Build only the two bucket operations needed by the lightweight retry
|
||||
/// drain. The full bootstrap plan scans every bucket and IAM record; doing
|
||||
/// that on a 30-second recovery cadence would make lifecycle admission scale
|
||||
/// with the whole site instead of the one queued bucket.
|
||||
pub(crate) fn site_replication_bucket_retry_plan_for(
|
||||
bucket: &SRBucketInfo,
|
||||
replicate_ilm_expiry: bool,
|
||||
) -> S3Result<SiteReplicationBootstrapPlan> {
|
||||
let mut plan = SiteReplicationBootstrapPlan {
|
||||
bucket_make_ops: vec![bootstrap_bucket_make_op_path(bucket)],
|
||||
bucket_configure_ops: vec![bootstrap_bucket_op_path(
|
||||
&bucket.bucket,
|
||||
SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION,
|
||||
)],
|
||||
..Default::default()
|
||||
};
|
||||
append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?;
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
pub(crate) fn site_replication_bucket_retry_plan_from_info(
|
||||
bucket: &SRBucketInfo,
|
||||
replicate_ilm_expiry: bool,
|
||||
) -> S3Result<SiteReplicationBootstrapPlan> {
|
||||
let mut plan = site_replication_bucket_retry_plan_for(bucket, replicate_ilm_expiry)?;
|
||||
// Omit only metadata the make/configure operations can reproduce exactly.
|
||||
// Non-default versioning fields and operator-authored replication rules
|
||||
// remain in the plan; their extra request cost intentionally defers the
|
||||
// event to the complete drain when the lightweight budget is too small.
|
||||
plan.bucket_items.retain(|item| !retry_bucket_metadata_is_redundant(item));
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
fn retry_bucket_metadata_is_redundant(item: &SRBucketMeta) -> bool {
|
||||
match item.r#type.as_str() {
|
||||
"version-config" => item.versioning.as_deref().is_some_and(|raw| {
|
||||
deserialize::<VersioningConfiguration>(&decode_bucket_meta_wire_value(raw)).is_ok_and(|config| {
|
||||
config
|
||||
== VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
}
|
||||
})
|
||||
}),
|
||||
"replication-config" => item.replication_config.as_deref().is_some_and(|raw| {
|
||||
deserialize::<ReplicationConfiguration>(&decode_bucket_meta_wire_value(raw))
|
||||
.is_ok_and(|config| config.role.trim().is_empty() && config.rules.iter().all(is_derived_site_replication_rule))
|
||||
}),
|
||||
// `Some("")` is the in-memory sentinel used when the bucket is lock
|
||||
// enabled but has no object-lock configuration body. The make query
|
||||
// carries lockEnabled=true; sending an empty metadata body is neither
|
||||
// useful nor parseable.
|
||||
"object-lock-config" => item.object_lock_config.as_deref() == Some(""),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn site_replication_bucket_retry_plan(
|
||||
bucket: &str,
|
||||
replicate_ilm_expiry: bool,
|
||||
) -> S3Result<SiteReplicationBootstrapPlan> {
|
||||
let Some(store) = current_object_store_handle() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let bucket_info = match store.get_bucket_info(bucket, &BucketOptions::default()).await {
|
||||
Ok(bucket_info) => bucket_info,
|
||||
Err(err) if is_err_bucket_not_found(&err) => return Ok(SiteReplicationBootstrapPlan::default()),
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let lock_enabled = bucket_info.object_locking;
|
||||
let metadata = metadata_sys::get(bucket).await.map_err(ApiError::from)?;
|
||||
let mut bucket_info = SRBucketInfo {
|
||||
bucket: bucket.to_string(),
|
||||
created_at: bucket_info.created,
|
||||
location: current_region().map(|region| region.to_string()).unwrap_or_default(),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
populate_sr_bucket_info_from_metadata(&mut bucket_info, &metadata).await;
|
||||
if lock_enabled && bucket_info.object_lock_config.is_none() {
|
||||
bucket_info.object_lock_config = Some(String::new());
|
||||
}
|
||||
site_replication_bucket_retry_plan_from_info(&bucket_info, replicate_ilm_expiry)
|
||||
}
|
||||
|
||||
pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> {
|
||||
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
|
||||
let runtime = {
|
||||
@@ -529,273 +393,20 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
|
||||
broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
|
||||
}
|
||||
|
||||
const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
|
||||
"bucket deletion reserved; local completion and peer delivery are not yet known";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SiteReplicationDeleteBucketReservation {
|
||||
peer: PeerInfo,
|
||||
previous: Option<SiteReplicationRetryEvent>,
|
||||
observed: SiteReplicationRetryEvent,
|
||||
}
|
||||
|
||||
pub(crate) struct SiteReplicationDeleteBucketIntent {
|
||||
path: String,
|
||||
reservations: Vec<SiteReplicationDeleteBucketReservation>,
|
||||
displaced: Vec<SiteReplicationRetryEvent>,
|
||||
}
|
||||
|
||||
fn site_replication_delete_bucket_path(bucket: &str, force_delete: bool) -> String {
|
||||
pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> {
|
||||
let operation = if force_delete {
|
||||
"force-delete-bucket"
|
||||
} else {
|
||||
"delete-bucket"
|
||||
};
|
||||
format!(
|
||||
let path = format!(
|
||||
"/rustfs/admin/v3/site-replication/peer/bucket-ops?{}",
|
||||
form_urlencoded::Serializer::new(String::new())
|
||||
.append_pair("bucket", bucket)
|
||||
.append_pair("operation", operation)
|
||||
.finish()
|
||||
)
|
||||
}
|
||||
|
||||
/// Reserve every destructive peer delivery before the local namespace is
|
||||
/// changed. The state transaction either persists the complete set or writes
|
||||
/// nothing, so a full/unreadable queue fails the S3 delete closed.
|
||||
pub(crate) async fn prepare_site_replication_delete_bucket(
|
||||
bucket: &str,
|
||||
force_delete: bool,
|
||||
) -> S3Result<Option<SiteReplicationDeleteBucketIntent>> {
|
||||
let path = site_replication_delete_bucket_path(bucket, force_delete);
|
||||
let reservation_path = path.clone();
|
||||
update_site_replication_state_when_changed(move |state| {
|
||||
if !state.enabled() {
|
||||
return Ok(StateCommit::Unchanged(None));
|
||||
}
|
||||
let local_peer = current_local_runtime_peer(state);
|
||||
let peers = state
|
||||
.peers
|
||||
.values()
|
||||
.filter(|peer| {
|
||||
peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if peers.is_empty() {
|
||||
return Ok(StateCommit::Unchanged(None));
|
||||
}
|
||||
|
||||
let mut reservations = Vec::with_capacity(peers.len());
|
||||
let mut displaced = Vec::new();
|
||||
for peer in peers {
|
||||
let previous = state
|
||||
.retry_queue
|
||||
.iter()
|
||||
.find(|event| retry_event_matches(event, &peer, &reservation_path))
|
||||
.cloned();
|
||||
displaced.extend(upsert_site_replication_retry_event(
|
||||
&mut state.retry_queue,
|
||||
&peer,
|
||||
&reservation_path,
|
||||
SITE_REPLICATION_DELETE_INTENT_PENDING,
|
||||
None,
|
||||
)?);
|
||||
let observed = state
|
||||
.retry_queue
|
||||
.iter()
|
||||
.find(|event| retry_event_matches(event, &peer, &reservation_path))
|
||||
.cloned()
|
||||
.ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
"site replication delete reservation disappeared before commit".to_string(),
|
||||
)
|
||||
})?;
|
||||
reservations.push(SiteReplicationDeleteBucketReservation {
|
||||
peer,
|
||||
previous,
|
||||
observed,
|
||||
});
|
||||
}
|
||||
Ok(StateCommit::Changed(Some(SiteReplicationDeleteBucketIntent {
|
||||
path: reservation_path,
|
||||
reservations,
|
||||
displaced,
|
||||
})))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Roll back a reservation when the local storage delete definitively failed.
|
||||
/// A concurrently revised reservation is preserved; it belongs to a newer
|
||||
/// observation and this operation has no authority to settle it.
|
||||
pub(crate) async fn cancel_site_replication_delete_bucket(intent: SiteReplicationDeleteBucketIntent) {
|
||||
let path = intent.path.clone();
|
||||
let result = update_site_replication_state_when_changed(move |state| {
|
||||
let mut changed = false;
|
||||
for reservation in intent.reservations {
|
||||
let Some(index) = state.retry_queue.iter().position(|event| {
|
||||
retry_event_matches(event, &reservation.peer, &reservation.observed.path)
|
||||
&& event.id == reservation.observed.id
|
||||
&& event.updated_at == reservation.observed.updated_at
|
||||
}) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(previous) = reservation.previous {
|
||||
state.retry_queue[index] = previous;
|
||||
} else {
|
||||
state.retry_queue.remove(index);
|
||||
}
|
||||
changed = true;
|
||||
}
|
||||
|
||||
let mut restored_all = true;
|
||||
for displaced in intent.displaced {
|
||||
let duplicate = state.retry_queue.iter().any(|event| {
|
||||
event.id == displaced.id
|
||||
|| (event.peer_deployment_id == displaced.peer_deployment_id && event.path == displaced.path)
|
||||
});
|
||||
if duplicate {
|
||||
continue;
|
||||
}
|
||||
if state.retry_queue.len() >= SITE_REPLICATION_RETRY_QUEUE_LIMIT {
|
||||
restored_all = false;
|
||||
continue;
|
||||
}
|
||||
state.retry_queue.push(displaced);
|
||||
changed = true;
|
||||
}
|
||||
Ok(if changed {
|
||||
StateCommit::Changed(restored_all)
|
||||
} else {
|
||||
StateCommit::Unchanged(restored_all)
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(true) => {}
|
||||
Ok(false) => warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
path,
|
||||
result = "delete_intent_cancel_incomplete",
|
||||
"admin site replication state"
|
||||
),
|
||||
Err(err) => warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
path,
|
||||
result = "delete_intent_cancel_failed",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn broadcast_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
|
||||
let sends = intent.reservations.iter().cloned().map(|reservation| {
|
||||
let request_path = intent.path.clone();
|
||||
async move {
|
||||
let fallback_peer = reservation.peer.clone();
|
||||
let observed = reservation.observed.clone();
|
||||
let delivery_path = request_path.clone();
|
||||
let delivery = with_site_replication_state_read_lock(move |state| async move {
|
||||
let Some(current_peer) = state.peers.get(&fallback_peer.deployment_id).cloned() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let service_account_secret_key =
|
||||
match site_replicator_service_account_secret(&state.service_account_access_key).await {
|
||||
Ok(secret) => secret,
|
||||
Err(err) => {
|
||||
let Some(secret) = legacy_site_replicator_state_secret(&state) else {
|
||||
return Err(err);
|
||||
};
|
||||
warn!(
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
result = "legacy_state_service_account_secret_fallback",
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
secret
|
||||
}
|
||||
};
|
||||
let result = async {
|
||||
let transport = PeerTransport::for_runtime_peer(¤t_peer).await?;
|
||||
PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)
|
||||
.with_client(&transport.client)
|
||||
.send(&service_account_secret_key, &serde_json::json!({}))
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
Ok(Some((current_peer, result)))
|
||||
})
|
||||
.await;
|
||||
match delivery {
|
||||
Ok(Some((current_peer, Ok(_)))) => {
|
||||
dequeue_observed_site_replication_retry_event(¤t_peer, &observed).await;
|
||||
None
|
||||
}
|
||||
Ok(Some((current_peer, Err(err)))) => {
|
||||
// Keep the failed deletion operator-visible, but never
|
||||
// replay it automatically: without a bucket-incarnation
|
||||
// fence, a delayed delete could erase a recreated bucket.
|
||||
enqueue_site_replication_retry_event(¤t_peer, &request_path, &err).await;
|
||||
Some(err)
|
||||
}
|
||||
Ok(None) => {
|
||||
dequeue_observed_site_replication_retry_event(&reservation.peer, &observed).await;
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
enqueue_site_replication_retry_event(&reservation.peer, &request_path, &err).await;
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
futures::future::join_all(sends)
|
||||
.await
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.map_or(Ok(()), Err)
|
||||
}
|
||||
|
||||
pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
|
||||
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
|
||||
let store =
|
||||
current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
|
||||
let retry_peers = intent
|
||||
.reservations
|
||||
.iter()
|
||||
.map(|reservation| reservation.peer.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let retry_path = intent.path.clone();
|
||||
let delivery_intent = SiteReplicationDeleteBucketIntent {
|
||||
path: intent.path.clone(),
|
||||
reservations: intent.reservations.clone(),
|
||||
displaced: Vec::new(),
|
||||
};
|
||||
match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move {
|
||||
broadcast_site_replication_delete_bucket(&delivery_intent).await
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
let err: S3Error = ApiError::from(err).into();
|
||||
for peer in &retry_peers {
|
||||
enqueue_site_replication_retry_event(peer, &retry_path, &err).await;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
);
|
||||
broadcast_site_replication_json(&path, &serde_json::json!({})).await
|
||||
}
|
||||
|
||||
pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> {
|
||||
@@ -904,39 +515,6 @@ pub(crate) fn maybe_time(value: OffsetDateTime) -> Option<OffsetDateTime> {
|
||||
(value != OffsetDateTime::UNIX_EPOCH).then_some(value)
|
||||
}
|
||||
|
||||
async fn populate_sr_bucket_info_from_metadata(entry: &mut SRBucketInfo, metadata: &BucketMetadata) {
|
||||
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
|
||||
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
|
||||
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
|
||||
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
|
||||
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
|
||||
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
|
||||
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
|
||||
// Expiry subset only: this entry feeds both the bootstrap/repair plan
|
||||
// (peers must not receive transition rules) and cross-site consistency
|
||||
// views (transition rules are site-local and would read as false
|
||||
// mismatches). A deleted expiry state is a `None` value with the
|
||||
// deletion's axis so repair can converge peers that missed the live
|
||||
// delete.
|
||||
let expiry_statement = lifecycle_expiry_statement(metadata);
|
||||
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
|
||||
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
|
||||
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
|
||||
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
|
||||
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
|
||||
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
|
||||
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
|
||||
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
|
||||
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
|
||||
// The expiry axis, not the whole-config write time: local transition-only
|
||||
// edits inflate the latter, and a repair item stamped with it could
|
||||
// out-rank a newer real expiry edit on a third site.
|
||||
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
|
||||
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
|
||||
entry.replication_targets_online =
|
||||
Some(site_replication_targets_online(&entry.bucket, &metadata.replication_config_xml).await);
|
||||
}
|
||||
|
||||
pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result<SRInfo> {
|
||||
let Some(store) = current_object_store_handle() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
@@ -968,7 +546,37 @@ pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &Pee
|
||||
};
|
||||
|
||||
if let Some(metadata) = metadata {
|
||||
populate_sr_bucket_info_from_metadata(&mut entry, &metadata).await;
|
||||
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
|
||||
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
|
||||
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
|
||||
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
|
||||
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
|
||||
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
|
||||
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
|
||||
// Expiry subset only: this entry feeds both the bootstrap/repair
|
||||
// plan (peers must not receive transition rules) and cross-site
|
||||
// consistency views (transition rules are site-local and would
|
||||
// read as false mismatches). A deleted expiry state is a `None`
|
||||
// value with the deletion's axis so repair can converge peers
|
||||
// that missed the live delete.
|
||||
let expiry_statement = lifecycle_expiry_statement(&metadata);
|
||||
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
|
||||
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
|
||||
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
|
||||
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
|
||||
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
|
||||
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
|
||||
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
|
||||
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
|
||||
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
|
||||
// The expiry axis, not the whole-config write time: local
|
||||
// transition-only edits inflate the latter, and a repair item
|
||||
// stamped with it could out-rank a newer real expiry edit on a
|
||||
// third site.
|
||||
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
|
||||
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
|
||||
entry.replication_targets_online =
|
||||
Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await);
|
||||
}
|
||||
|
||||
info.buckets.insert(bucket.name, entry);
|
||||
|
||||
@@ -47,7 +47,6 @@ use self::identity::{
|
||||
canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with,
|
||||
same_identity_endpoint,
|
||||
};
|
||||
pub(crate) use self::state_lock::with_site_replication_state_read_lock;
|
||||
use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock};
|
||||
use crate::auth::constant_time_eq;
|
||||
use crate::config::get_config_snapshot;
|
||||
@@ -65,12 +64,12 @@ use crate::storage_api::site_replication::s3::{
|
||||
#[cfg(test)]
|
||||
use crate::storage_api::site_replication::save_config as save_admin_config;
|
||||
use crate::storage_api::site_replication::{
|
||||
ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, BucketOperations,
|
||||
BucketOptions, BucketTarget, BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract,
|
||||
StorageError, VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize,
|
||||
is_err_bucket_not_found, is_site_replication_role, lock_bucket_targets_metadata, metadata_sys,
|
||||
read_config as read_admin_config, read_config_no_lock, replication_target_arn_deployment_id, save_config_no_lock, serialize,
|
||||
site_replication_rule_deployment_id, with_config_object_read_lock, with_config_object_write_lock,
|
||||
ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketTarget,
|
||||
BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, StorageError,
|
||||
VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, is_site_replication_role,
|
||||
lock_bucket_targets_metadata, metadata_sys, read_config as read_admin_config, read_config_no_lock,
|
||||
replication_target_arn_deployment_id, save_config_no_lock, serialize, site_replication_rule_deployment_id,
|
||||
with_config_object_read_lock, with_config_object_write_lock,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use base64_simd::URL_SAFE_NO_PAD;
|
||||
|
||||
@@ -649,9 +649,7 @@ pub(crate) async fn persist_site_replication_repair_task(
|
||||
let path = path.to_string();
|
||||
update_site_replication_state(move |state| {
|
||||
match failure.as_deref() {
|
||||
Some(error) => {
|
||||
upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None)?;
|
||||
}
|
||||
Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None),
|
||||
None => {
|
||||
dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path);
|
||||
// A repair is the operator's accountability transfer for the
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user