Compare commits

..

2 Commits

Author SHA1 Message Date
马登山 169e071aec fix(ecstore): release GET locks at content length 2026-07-28 17:20:09 +08:00
马登山 cf1e1a67e0 fix(ecstore): retain GET locks through streaming 2026-07-28 12:22:06 +08:00
98 changed files with 1519 additions and 6924 deletions
+2 -8
View File
@@ -10,16 +10,10 @@ never weaken a check to get green.
## `check_layer_dependencies.sh` — layer DAG in `rustfs/src`
Enforces `composition (server, startup/init) → interface (admin,
storage/ecfs, storage/s3_api) → app → infra`; no upward imports. Server source
files are composition roots, while imports of their exported HTTP contracts
are classified as interface dependencies. Known legacy violations live in
Enforces `interface (admin, storage/ecfs, storage/s3_api) → app → infra`; no
upward imports. Known legacy violations live in
`scripts/layer-dependency-baseline.txt`.
Dedicated `*_test.rs` and `tests/` modules are outside this production guard.
Inline `#[cfg(test)]` imports remain checked under their source file's layer;
move architecture-crossing test scaffolding into a dedicated test module.
- **New violation**: restructure your change so the dependency points
downward (move the shared type/function to the lower layer).
- **You legitimately removed a baseline entry**: run
+24 -9
View File
@@ -1,14 +1,17 @@
# nextest configuration for RustFS.
#
# Serialize the ecstore tests that share the process-wide disk registry or
# exercise a multi-disk commit handoff across nextest process boundaries.
# Serialize two known load-sensitive / global-state-sharing ecstore test groups
# so the full parallel nextest suite stops producing spurious failures
# (backlog #937). These tests pass in isolation but flake under the loaded
# parallel run for two distinct reasons:
#
# * store::bucket::tests::bucket_delete_* share process/global state (disk
# registry, lock client) and race make_bucket into InsufficientWriteQuorum
# when run concurrently with other ecstore tests.
# * bucket_lifecycle_ops::tests::concurrent_resend_same_part_commits_one_generation
# uses the shared multipart fixture and a deterministic uploadId-lock
# handoff, so it must not overlap another process mutating that fixture.
# asserts a lock-acquire correctness property whose serialized cross-disk
# commits exceed the (already max'd, 60s) acquire deadline only when the
# suite saturates disk I/O.
#
# serial_test's #[serial] attribute does NOT serialize these across runs:
# nextest executes each test in its own process, where the in-process
@@ -52,6 +55,12 @@ test-group = 'ecstore-serial-flaky'
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
test-group = 'ecstore-serial-flaky'
# This regression test builds three multipart objects on a four-disk set and
# coordinates live GET/DELETE races under both lock modes.
[[profile.default.overrides]]
filter = 'package(rustfs-ecstore) & test(multipart_get_keeps_delete_blocked_at_part_boundary_in_both_lock_modes)'
test-group = 'ecstore-serial-flaky'
# Serialize the durable manual-transition checkpoint test across nextest's
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
[[profile.default.overrides]]
@@ -97,6 +106,13 @@ path = "junit.xml"
# profile's own overrides list, not the default profile's).
# ===========================================================================
# QUARANTINE: OPEN backlog#937 — concurrent_resend lock-acquire deadline flakes
# under saturated disk I/O in the full parallel suite.
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
test-group = 'ecstore-serial-flaky'
retries = 2
# QUARANTINE: OPEN backlog#937 — store::bucket::tests::bucket_delete_* race
# make_bucket into InsufficientWriteQuorum via shared global state under load.
[[profile.ci.overrides]]
@@ -104,11 +120,6 @@ filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(
test-group = 'ecstore-serial-flaky'
retries = 2
# Keep the deterministic multipart handoff isolated across nextest processes.
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
test-group = 'ecstore-serial-flaky'
# QUARANTINE: OPEN rustfs#4690 — walk_dir stall-budget accounting test depends
# on producer/consumer timing windows that stretch past the budget on loaded
# CI runners (regression test for rustfs#4644; failed on a zero-Rust-diff PR).
@@ -131,6 +142,10 @@ test-group = 'e2e-reliability'
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
test-group = 'ecstore-serial-flaky'
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & test(multipart_get_keeps_delete_blocked_at_part_boundary_in_both_lock_modes)'
test-group = 'ecstore-serial-flaky'
# Serialize the durable manual-transition checkpoint test under the ci profile
# too. No retries: failures stay visible.
[[profile.ci.overrides]]
+1 -1
View File
@@ -141,7 +141,7 @@ jobs:
name: Test and Lint
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: sm-standard-4
timeout-minutes: 90
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
+2 -25
View File
@@ -172,7 +172,7 @@
],
},
{
"name": "Debug executable target/debug/rustfs with sse kms",
"name": "Debug executable target/debug/rustfs with sse",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/rustfs",
@@ -200,7 +200,7 @@
// 2. kms local backend test key
// "RUSTFS_KMS_ENABLE": "true",
// "RUSTFS_KMS_BACKEND": "local",
// "RUSTFS_KMS_KEY_DIR": "/tmp/kms-key-dir",
// "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
// "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
// "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
@@ -228,29 +228,6 @@
"rust"
],
},
{
"name": "Debug executable target/debug/rustfs with local sse",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [],
"cwd": "${workspaceFolder}",
"env": {
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
"RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
"RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
"RUSTFS_SSE_S3_MASTER_KEY": "xGb3aYSp825j2tPpg8JrUzghiXsIkfdOtmrsJ/iafiM=",
"RUST_LOG": "rustfs=debug,ecstore=debug,s3s=debug,iam=debug",
},
"sourceLanguages": [
"rust"
],
},
{
"type": "lldb",
"request": "launch",
Generated
+22 -26
View File
@@ -1703,9 +1703,9 @@ dependencies = [
[[package]]
name = "camino"
version = "1.2.5"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb1307f12aa967b5a58416e87b3653360e0fd614a016b6e970db08fecbb1b80d"
checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0"
dependencies = [
"serde_core",
]
@@ -3625,13 +3625,13 @@ dependencies = [
[[package]]
name = "displaydoc"
version = "0.2.7"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
"syn 2.0.119",
]
[[package]]
@@ -3946,10 +3946,11 @@ dependencies = [
[[package]]
name = "event-listener"
version = "5.4.2"
version = "5.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2"
checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
dependencies = [
"concurrent-queue",
"parking",
"pin-project-lite",
]
@@ -6109,9 +6110,9 @@ dependencies = [
[[package]]
name = "metrique"
version = "0.1.29"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2e394c63e2d1a30aeb3b9392ecf3439d8475d2df810a8f4f6e66d6866754017"
checksum = "aa466af30a9fe0b1db1dae097e0ce7ddac4b666b1d3a9f5c682e889272511dd8"
dependencies = [
"itoa",
"jiff",
@@ -6139,9 +6140,9 @@ dependencies = [
[[package]]
name = "metrique-macro"
version = "0.1.20"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "786df1fd0abebd0db685f7e9a353c78756d4b370fb98a52376c2015fa55f141f"
checksum = "1f5febfaf14fea234b60e0ad43c728db274033893a38d0fa1f87d9b7056d3d61"
dependencies = [
"Inflector",
"darling 0.23.0",
@@ -6168,9 +6169,9 @@ checksum = "2faca4e4480069ff02b1763b3b79f5cec7e8628e24d9dc5b6073f53d2577a4d9"
[[package]]
name = "metrique-writer"
version = "0.1.25"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82cdde44d241dab7fc8b7a32e0eb5dae6cd28f8de80b59f9a1e9f2f0b05e485e"
checksum = "124326a2ac4c4f61562fa4d071735a1c463f9ba0317d1564f75dc01313dc12d7"
dependencies = [
"ahash",
"crossbeam-queue",
@@ -6189,9 +6190,9 @@ dependencies = [
[[package]]
name = "metrique-writer-core"
version = "0.1.19"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e57379b7ee2272efaeaaa6de062503563e57333b24aadc7f2255b3d602899e8b"
checksum = "55b5bbb6d88bde29f6ed74a574cbe49e51ea0c36cccc7e63eee2040db5675b15"
dependencies = [
"derive-where",
"itertools 0.14.0",
@@ -9476,8 +9477,6 @@ name = "rustfs-lifecycle"
version = "1.0.0-beta.11"
dependencies = [
"async-trait",
"metrics",
"metrics-util",
"proptest",
"rustfs-common",
"rustfs-config",
@@ -9715,7 +9714,6 @@ dependencies = [
"http-body-util",
"hyper",
"hyper-util",
"ipnetwork",
"libunftp",
"md5",
"percent-encoding",
@@ -9732,9 +9730,7 @@ dependencies = [
"rustfs-policy",
"rustfs-rio",
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-tls-runtime",
"rustfs-trusted-proxies",
"rustfs-utils",
"rustls",
"s3s",
@@ -10355,7 +10351,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.14.1"
source = "git+https://github.com/cxymds/s3s.git?rev=fe3941d91fa1c69956f209a9145995c9f0235bff#fe3941d91fa1c69956f209a9145995c9f0235bff"
source = "git+https://github.com/s3s-project/s3s.git?rev=8136db4ac8253c0ccfd86f8216e00e0235747757#8136db4ac8253c0ccfd86f8216e00e0235747757"
dependencies = [
"arc-swap",
"arrayvec",
@@ -10461,9 +10457,9 @@ dependencies = [
[[package]]
name = "schemars"
version = "1.2.2"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
dependencies = [
"dyn-clone",
"ref-cast",
@@ -10696,7 +10692,7 @@ dependencies = [
"indexmap 1.9.3",
"indexmap 2.14.0",
"schemars 0.9.0",
"schemars 1.2.2",
"schemars 1.2.1",
"serde_core",
"serde_json",
"serde_with_macros",
@@ -11876,9 +11872,9 @@ dependencies = [
[[package]]
name = "toml_parser"
version = "1.1.3+spec-1.1.0"
version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
dependencies = [
"winnow",
]
+1 -1
View File
@@ -288,7 +288,7 @@ redis = { version = "1.4.1" }
rustix = { version = "1.1.4" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/cxymds/s3s.git", rev = "fe3941d91fa1c69956f209a9145995c9f0235bff" }
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "8136db4ac8253c0ccfd86f8216e00e0235747757" }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.3"
+1 -1
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM rust:1.97.1-trixie
FROM rust:1.97-trixie
RUN set -eux; \
export DEBIAN_FRONTEND=noninteractive; \
+1 -1
View File
@@ -32,7 +32,7 @@ ARG RUSTFS_BUILD_FEATURES=""
# -----------------------------
# Build stage
# -----------------------------
FROM rust:1.97.1-trixie AS builder
FROM rust:1.97-trixie AS builder
# Re-declare args after FROM
ARG TARGETPLATFORM
@@ -45,10 +45,10 @@
//! * Parity reconstruction: one data disk is taken offline
//! (`take_disk_offline`) and the SAME object matrix is GET both ways while
//! the EC 2+2 set rebuilds each large object from the surviving shards. The
//! eager first/single-part setup may keep its conservative whole-request
//! fallback when shard placement makes codec streaming unsafe, so this phase
//! asserts byte- and header-equality vs the legacy path rather than requiring
//! zero duplex fallbacks under degraded drive health.
//! codec-streaming reader gate never inspects drive health, so the codec
//! fast path is exercised end-to-end through reconstruction; the test
//! asserts byte- and header-equality vs the legacy path AND that the codec
//! phase never fell back to a duplex pipe while reconstructing.
//! * Missing object: a GET for an absent key is compared across both phases
//! to prove the error semantics (HTTP status + S3 error code) are identical
//! — the codec env must not perturb the NoSuchKey negative path.
@@ -475,14 +475,15 @@ mod tests {
"ranged GET length diverged with codec streaming enabled"
);
// ---- Phase B degraded: the same reconstruction, with codec gates open ----
// Re-run the reconstruction A/B with codec-streaming enabled. If eager
// first/single-part setup cannot prove the codec path is safe for the
// surviving shards, the implementation intentionally preserves the
// whole-request legacy fallback; later multipart parts can degrade in
// place. This phase verifies parity-reconstructed bytes and headers,
// while the healthy phase above remains the strict zero-duplex path
// confirmation.
// ---- Phase B degraded: the same reconstruction, now on the codec path ----
// Re-run the reconstruction A/B with the codec-streaming gates still
// open. The reader gate decision is independent of drive health (it
// never inspects disk state), so the codec fast path is exercised
// end-to-end while the EC set rebuilds each large object from the
// surviving shards — this is a real codec-vs-legacy reconstruction test,
// not legacy-vs-legacy. Snapshot the duplex count first (the range GET
// above already used the duplex path) so we can measure only the markers
// these degraded codec GETs add.
let dup_codec_before_degraded = count_marker(&codec_log, DUPLEX_MARKER);
harness.take_disk_offline(0)?;
let mut codec_degraded: BTreeMap<String, GetView> = BTreeMap::new();
@@ -510,11 +511,16 @@ mod tests {
);
}
// Keep degraded duplex markers as diagnostic evidence only: eager setup
// may fall back before streaming when shard safety cannot be proven.
// Path confirmation under reconstruction: the codec fast path must have
// served the reconstructed large objects without ever falling back to
// the legacy duplex pipe. Without this, the equivalence above could be
// legacy-vs-legacy and prove nothing about codec reconstruction.
sleep(Duration::from_millis(300)).await;
let dup_codec_degraded = count_marker(&codec_log, DUPLEX_MARKER).saturating_sub(dup_codec_before_degraded);
info!(dup_codec_degraded, "codec phase degraded-read legacy duplex marker count");
assert_eq!(
dup_codec_degraded, 0,
"codec phase created {dup_codec_degraded} duplex pipe(s) while reconstructing large objects with disk0 offline; the codec fast path was not exercised under degraded reads (see {codec_log})"
);
info!(
objects = baseline.len(),
@@ -520,33 +520,28 @@ async fn assert_msgpack_fallback_unchanged(
Ok(())
}
async fn assert_msgpack_decode_observed(collector: &OtlpMetricCollector, before: &BTreeMap<String, u64>) -> TestResult {
async fn assert_msgpack_decode_observed(
collector: &OtlpMetricCollector,
before: &BTreeMap<String, u64>,
series: &[(&str, &str)],
) -> TestResult {
let deadline = Instant::now() + Duration::from_secs(20);
loop {
let after = collector.msgpack_json_decode_totals().await;
let missing = [FALLBACK_REQUEST_DIRECTION, FALLBACK_RESPONSE_DIRECTION]
let missing = series
.iter()
.filter_map(|direction| {
let prefix = format!("{direction}\u{1f}");
let suffix = format!("\u{1f}{MSGPACK_CODEC_MSGPACK}");
let before_total = before
.iter()
.filter(|(key, _)| key.starts_with(&prefix) && key.ends_with(&suffix))
.map(|(_, value)| *value)
.sum::<u64>();
let after_total = after
.iter()
.filter(|(key, _)| key.starts_with(&prefix) && key.ends_with(&suffix))
.map(|(_, value)| *value)
.sum::<u64>();
(after_total <= before_total).then_some(format!("{direction}/{}", MSGPACK_CODEC_MSGPACK))
.filter_map(|(direction, message)| {
let key = msgpack_decode_metric_key(direction, message, MSGPACK_CODEC_MSGPACK);
let before_value = before.get(&key).copied().unwrap_or_default();
let after_value = after.get(&key).copied().unwrap_or_default();
(after_value <= before_value).then_some(format!("{direction}/{message}/{}", MSGPACK_CODEC_MSGPACK))
})
.collect::<Vec<_>>();
if missing.is_empty() {
return Ok(());
}
if Instant::now() >= deadline {
return Err(format!("timed out waiting for msgpack decode traffic for {missing:?}; totals={after:?}").into());
return Err(format!("timed out waiting for msgpack decode traffic on {missing:?}; totals={after:?}").into());
}
sleep(Duration::from_millis(100)).await;
}
@@ -1862,7 +1857,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
)
.await?;
assert_msgpack_decode_observed(&collector, &decode_before).await?;
assert_msgpack_decode_observed(&collector, &decode_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
assert_msgpack_fallback_unchanged(&collector, &fallback_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
assert_msgpack_decode_errors_unchanged(&collector, &decode_errors_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
@@ -1953,8 +1948,8 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
hot.start().await?;
let tier_name = unique_tier_name();
let add_tier_response = submit_rustfs_tier(&hot, &cold, &tier_name).await?;
hot.stop_node(3)?;
let add_tier_response = submit_rustfs_tier(&hot, &cold, &tier_name).await?;
hot.start_node(3).await?;
wait_for_tier_converged(&hot, &tier_name, &add_tier_response).await
@@ -2008,7 +2003,7 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
assert_eq!(cancel_value["job_id"].as_str(), Some(job_id.as_str()));
assert_eq!(cancel_value["bucket"].as_str(), Some(bucket.as_str()));
assert_eq!(cancel_value["dry_run"].as_bool(), Some(true));
assert_eq!(cancel_value["cancel_requested"].as_bool(), Some(false));
assert_eq!(cancel_value["cancel_requested"].as_bool(), Some(true));
let missing_job_id = Uuid::new_v4();
let (missing_status, missing_body) = signed_admin_request(
@@ -2330,7 +2325,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), REMOTE, REMOTE_TRANSITION),
)
.await?;
assert_msgpack_decode_observed(&collector, &decode_before).await?;
assert_msgpack_decode_observed(&collector, &decode_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
let encrypted_key = "transition/encrypted-sse.bin";
let encrypted_body = payload(16 * KIB, 0xAB);
@@ -400,20 +400,6 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
async fn prepare_part_transaction(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::PreparePartTransactionRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::PreparePartTransactionResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn settle_part_transaction(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::SettlePartTransactionRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::SettlePartTransactionResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_file(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameFileRequest>,
+4 -4
View File
@@ -128,7 +128,7 @@ pub mod bucket {
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
update_bucket_targets_under_transaction_lock, update_config_with,
update_bucket_targets_under_transaction_lock,
};
}
@@ -303,9 +303,9 @@ pub mod disk {
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, new_disk, validate_batch_read_version_item_count,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
@@ -1181,12 +1181,8 @@ impl TransitionState {
}
fn new_with_capacity(capacity: usize) -> Arc<Self> {
let queue_send_timeout = resolve_transition_queue_send_timeout();
Self::new_with_capacity_and_timeout(capacity, queue_send_timeout)
}
fn new_with_capacity_and_timeout(capacity: usize, queue_send_timeout: StdDuration) -> Arc<Self> {
let capacity = capacity.max(1);
let queue_send_timeout = resolve_transition_queue_send_timeout();
let (tx1, rx1) = bounded(capacity);
Arc::new(Self {
transition_tx: tx1,
@@ -1253,12 +1249,13 @@ impl TransitionState {
return false;
}
let bucket = bucket.to_string();
let scheduled = Arc::clone(&self.compensation_buckets);
let state = Arc::clone(self);
tokio::spawn(async move {
Self::inc_counter(&state.compensation_running_tasks);
state.record_scanner_transition_state();
let Some(api) = runtime_sources::object_store_handle() else {
state.finish_bucket_compensation(&bucket);
scheduled.lock().unwrap().remove(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
debug!(
@@ -1294,25 +1291,13 @@ impl TransitionState {
);
}
state.finish_bucket_compensation(&bucket);
scheduled.lock().unwrap().remove(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
});
true
}
fn finish_bucket_compensation(&self, bucket: &str) {
match self.compensation_buckets.lock() {
Ok(mut scheduled) => {
scheduled.remove(bucket);
}
Err(poisoned) => {
poisoned.into_inner().remove(bucket);
self.compensation_buckets.clear_poison();
}
}
}
#[inline]
fn inc_counter(counter: &AtomicI64) {
Self::add_counter(counter, 1);
@@ -1549,35 +1534,17 @@ impl TransitionState {
let outcome = match self.transition_tx.try_send(Some(task)) {
Ok(()) => TransitionEnqueueOutcome::Queued,
Err(async_channel::TrySendError::Full(task)) => {
Err(async_channel::TrySendError::Full(_)) => {
Self::inc_counter(&self.queue_full_tasks);
let send_timeout = self.transition_queue_send_timeout;
match tokio::time::timeout(send_timeout, self.transition_tx.send(task)).await {
Ok(Ok(())) => TransitionEnqueueOutcome::Queued,
Ok(Err(_)) => {
self.schedule_bucket_compensation(&oi.bucket);
TransitionEnqueueOutcome::QueueClosed
}
Err(_) => {
Self::inc_counter(&self.queue_send_timeout_tasks);
self.schedule_bucket_compensation(&oi.bucket);
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
timeout_ms = send_timeout.as_millis() as u64,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_send_timed_out",
"Scanner transition enqueue timed out; scheduled bucket compensation"
);
TransitionEnqueueOutcome::QueueFull
}
}
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
"transition queue is full; deferring to scanner/backfill"
);
TransitionEnqueueOutcome::QueueFull
}
Err(async_channel::TrySendError::Closed(_)) => {
self.schedule_bucket_compensation(&oi.bucket);
debug!(
bucket = %oi.bucket,
object = %oi.name,
@@ -1762,7 +1729,7 @@ impl TransitionState {
}
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
let mut tier_stats = self.lock_last_day_stats();
let mut tier_stats = self.last_day_stats.lock().unwrap();
tier_stats
.entry(tier.to_string())
.and_modify(|e| e.add_stats(ts))
@@ -1770,7 +1737,7 @@ impl TransitionState {
}
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
let tier_stats = self.lock_last_day_stats();
let tier_stats = self.last_day_stats.lock().unwrap();
let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
for (tier, st) in tier_stats.iter() {
res.insert(tier.clone(), st.clone());
@@ -1778,18 +1745,6 @@ impl TransitionState {
res
}
fn lock_last_day_stats(&self) -> std::sync::MutexGuard<'_, HashMap<String, LastDayTierStats>> {
match self.last_day_stats.lock() {
Ok(stats) => stats,
Err(poisoned) => {
let mut stats = poisoned.into_inner();
stats.clear();
self.last_day_stats.clear_poison();
stats
}
}
}
pub async fn update_workers(api: Arc<ECStore>, n: i64) {
Self::update_workers_inner(api, n).await;
}
@@ -1811,27 +1766,7 @@ impl TransitionState {
fn resize_workers_to(api: Arc<ECStore>, n: i64, requested: i64, absolute_max: i64) {
let target = n as usize;
let transition_state = runtime_sources::transition_state_handle();
let runtime = match tokio::runtime::Handle::try_current() {
Ok(runtime) => runtime,
Err(err) => {
warn!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = %err,
state = "resize_failed",
"Lifecycle worker pool requires a Tokio runtime"
);
return;
}
};
// Runtime lookup happens before locking, and the guard is dropped before
// metrics/logging callbacks. Poison therefore means a Vec mutation may
// have unwound and worker tracking cannot be reconstructed safely.
let mut workers = transition_state
.workers
.lock()
.expect("transition worker tracking mutex poisoned");
let mut workers = transition_state.workers.lock().unwrap();
let tracked_workers = workers.len();
workers.retain(|worker| !worker.handle.is_finished());
let pruned_finished_workers = tracked_workers.saturating_sub(workers.len());
@@ -1841,7 +1776,7 @@ impl TransitionState {
let clone_api = api.clone();
let cancel = CancellationToken::new();
let worker_cancel = cancel.clone();
let handle = runtime.spawn(async move {
let handle = tokio::spawn(async move {
TransitionState::worker_with_cancel(clone_api, worker_cancel).await;
});
workers.push(TransitionWorker { cancel, handle });
@@ -1855,7 +1790,6 @@ impl TransitionState {
let current_workers = workers.len() as i64;
transition_state.num_workers.store(current_workers, Ordering::SeqCst);
drop(workers);
transition_state.record_scanner_transition_state();
debug!(
@@ -4950,7 +4884,6 @@ mod tests {
FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook,
};
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
use crate::bucket::lifecycle::tier_sweeper::Jentry;
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use crate::bucket::metadata_sys;
@@ -4984,7 +4917,6 @@ mod tests {
use http::HeaderMap;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_data_usage::TierStats;
use rustfs_filemeta::{FileInfo, FileMeta};
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, OutputLocation,
@@ -6527,96 +6459,6 @@ mod tests {
assert_eq!(state.transition_rx.len(), 1);
}
#[tokio::test]
#[serial]
async fn scanner_transition_enqueue_waits_for_saturated_queue_to_recover() {
let state = TransitionState::new_with_capacity(1);
let first_object = ObjectInfo {
bucket: "bucket".to_string(),
name: "first".to_string(),
..Default::default()
};
let deferred_object = ObjectInfo {
bucket: "bucket".to_string(),
name: "deferred".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::TransitionAction,
..Default::default()
};
assert!(
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
"first scanner transition should fill the queue"
);
let deferred = state.queue_transition_task(&deferred_object, &event, &LcEventSrc::Scanner);
tokio::pin!(deferred);
assert!(
(&mut deferred).now_or_never().is_none(),
"a saturated scanner queue should apply bounded backpressure instead of dropping the task"
);
let first_task = state
.transition_rx
.recv()
.await
.expect("queue should remain open")
.expect("first queued transition task should be present");
state.release_transition(&first_task.obj_info);
assert!(
deferred.await,
"the deferred scanner transition should enqueue as soon as capacity recovers"
);
let recovered_task = state
.transition_rx
.recv()
.await
.expect("queue should remain open")
.expect("deferred transition task should be present");
assert_eq!(recovered_task.obj_info.name, deferred_object.name);
}
#[tokio::test]
#[serial]
async fn scanner_transition_sustained_saturation_schedules_compensation() {
let state = TransitionState::new_with_capacity_and_timeout(1, StdDuration::ZERO);
let first_object = ObjectInfo {
bucket: "saturated-bucket".to_string(),
name: "first".to_string(),
..Default::default()
};
let missed_object = ObjectInfo {
bucket: "saturated-bucket".to_string(),
name: "missed".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::TransitionAction,
..Default::default()
};
assert!(
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
"first scanner transition should fill the queue"
);
assert!(
!state
.queue_transition_task(&missed_object, &event, &LcEventSrc::Scanner)
.await,
"a continuously saturated queue should report that the object was not admitted"
);
assert_eq!(state.queue_full_tasks(), 1);
assert_eq!(state.queue_send_timeout_tasks(), 1);
assert_eq!(
state.compensation_scheduled_tasks(),
1,
"timed-out scanner work must schedule a bounded bucket backfill"
);
}
#[tokio::test]
#[serial]
async fn scanner_transition_enqueue_updates_transition_status() {
@@ -7142,46 +6984,6 @@ mod tests {
assert_eq!(state.compensation_pending_tasks(), 1);
}
#[test]
fn poisoned_compensation_set_can_release_completed_bucket() {
let state = TransitionState::new_with_capacity(1);
state
.compensation_buckets
.lock()
.expect("fresh mutex should lock")
.insert("bucket-a".to_string());
let poison_target = Arc::clone(&state.compensation_buckets);
let _ = std::thread::spawn(move || {
let _guard = poison_target.lock().expect("fresh mutex should lock");
panic!("poison compensation set");
})
.join();
state.finish_bucket_compensation("bucket-a");
assert_eq!(state.compensation_pending_tasks(), 0);
assert!(
state.compensation_buckets.lock().is_ok(),
"validated compensation state must clear poison"
);
}
#[test]
fn poisoned_tier_stats_are_reset_before_reuse() {
let state = TransitionState::new_with_capacity(1);
let poison_target = Arc::clone(&state.last_day_stats);
let _ = std::thread::spawn(move || {
let mut stats = poison_target.lock().expect("fresh mutex should lock");
stats.insert("stale".to_string(), LastDayTierStats::default());
panic!("poison tier stats");
})
.join();
state.add_lastday_stats("fresh", TierStats::default());
let stats = state.get_daily_all_tier_stats();
assert!(!stats.contains_key("stale"), "possibly partial statistics must be discarded");
assert!(stats.contains_key("fresh"), "statistics must accept new samples after recovery");
}
#[tokio::test(flavor = "current_thread")]
async fn scanner_transition_state_reports_compensation_pending_buckets() {
let state = TransitionState::new_with_capacity(1);
@@ -7472,23 +7274,6 @@ mod tests {
TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max);
}
#[tokio::test]
#[serial]
async fn transition_worker_resize_without_runtime_does_not_poison_tracking() {
let (_paths, ecstore) = setup_test_env().await;
let transition_state = runtime_sources::transition_state_handle();
let resize = std::thread::spawn(move || {
TransitionState::resize_workers_to(ecstore, 1, 1, resolve_transition_workers_absolute_max());
})
.join();
assert!(resize.is_ok(), "missing Tokio runtime must not panic while worker tracking is locked");
assert!(
transition_state.workers.lock().is_ok(),
"failed resize must leave worker tracking unpoisoned"
);
}
#[test]
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
let now = OffsetDateTime::now_utc();
@@ -11112,7 +10897,6 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn concurrent_resend_same_part_commits_one_generation() {
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
use crate::storage_api_contracts::object::ObjectIO as _;
let (_paths, ecstore) = setup_test_env().await;
@@ -11134,36 +10918,51 @@ mod tests {
})
.collect();
let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
let mut tasks = tokio::task::JoinSet::new();
for payload in candidates.iter().cloned() {
let store = ecstore.clone();
let bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let start = Arc::clone(&start);
tasks.spawn(async move {
start.wait().await;
let mut data = PutObjReader::from_vec(payload.clone());
store
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
.await
.map(|info| (info, payload))
});
}
start.wait().await;
// Two independent causes can produce a spurious lock-acquire timeout
// here, and both must stay covered:
// 1. A lost/stolen fast-lock wakeup could strand a waiter until the
// deadline — fixed for real in fast_lock::shard by bounding each
// notification wait (NOTIFY_WAIT_CAP re-polling).
// 2. Under the full nextest suite on loaded CI disks, the
// *legitimately serialized* cross-disk commits can exceed the
// acquire deadline all by themselves — observed on CI at the 5s
// default and the 30s production default with six resends, and
// again at 60s, which is a hard ceiling: fast_lock clamps every
// requested timeout to MAX_ACQUIRE_TIMEOUT (60s), so raising the
// env override higher is a no-op (the Timeout error still reports
// the requested value). Keep the guard about the correctness
// property, not disk latency: request the full 60s ceiling and cap
// the queue depth at three resends, so the last waiter sits behind
// at most two serialized commits (~12s each on the slowest observed
// CI runner, comfortably inside the deadline). Three concurrent
// resends still race the streaming phase and contend on the commit
// lock, which is all the generation-mixing regression needs.
// `#[serial]` keeps the process-wide env override isolated.
let results = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60"))], async {
let mut tasks = tokio::task::JoinSet::new();
for payload in candidates.iter().cloned() {
let store = ecstore.clone();
let bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
tasks.spawn(async move {
let mut data = PutObjReader::from_vec(payload.clone());
store
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
.await
.map(|info| (info, payload))
});
}
// The first writer holds the uploadId commit lock while the other
// resends reach the same critical section. Releasing it proves the
// handoff without depending on saturated CI disk latency.
commit_barrier.wait_until_paused().await;
commit_barrier.release();
let mut results = Vec::new();
while let Some(joined) = tasks.join_next().await {
let outcome = joined.expect("put_object_part task should not panic");
results.push(outcome.expect("every concurrent same-part resend must succeed without lock timeout"));
}
// Every concurrent resend must succeed; the commit lock must never
// starve a waiter into a timeout.
let mut results = Vec::new();
while let Some(joined) = tasks.join_next().await {
let outcome = joined.expect("put_object_part task should not panic");
results.push(outcome.expect("every concurrent same-part resend must succeed without lock timeout"));
}
results
})
.await;
assert_eq!(results.len(), candidates.len());
// Exactly one generation is visible after the serialized commits, and its
-27
View File
@@ -737,9 +737,6 @@ impl BucketMetadata {
}
BUCKET_TAGGING_CONFIG => {
self.tagging_config_xml = data;
// Drop the parsed form (like lifecycle above) so clearing the
// payload can't leave stale parsed tags to be cached.
self.tagging_config = None;
self.tagging_config_updated_at = updated;
}
BUCKET_QUOTA_CONFIG_FILE => {
@@ -1321,30 +1318,6 @@ mod test {
assert!(bm.lifecycle_config.is_none());
}
/// Companion to the lifecycle case above. `parse_all_configs` skips empty
/// XML rather than clearing, so without the explicit reset a cleared
/// tagging config would keep serving the previously parsed tags.
#[test]
fn tagging_update_config_clears_parsed_config_on_delete() {
let mut bm = BucketMetadata::new("test-bucket");
let tagging_xml = br#"<Tagging><TagSet><Tag><Key>env</Key><Value>prod</Value></Tag></TagSet></Tagging>"#;
bm.update_config(BUCKET_TAGGING_CONFIG, tagging_xml.to_vec())
.expect("tagging config should update");
bm.parse_all_configs().expect("tagging config should parse");
assert!(bm.tagging_config.is_some());
bm.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
.expect("tagging config delete should update metadata");
assert!(bm.tagging_config_xml.is_empty());
assert!(bm.tagging_config.is_none());
// A re-parse must not resurrect them either.
bm.parse_all_configs().expect("cleared tagging should parse");
assert!(bm.tagging_config.is_none());
}
#[tokio::test]
async fn marshal_msg_complete_example() {
// Create a complete BucketMetadata with various configurations
+11 -200
View File
@@ -239,35 +239,6 @@ pub async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Ve
bucket_meta_sys.update(bucket, BUCKET_TARGETS_FILE, data).await
}
/// Read-modify-write one bucket config file under the metadata system's
/// outer write guard.
///
/// `mutate` sees the freshly loaded on-disk metadata and returns the
/// replacement payload for `config_file` (empty clears it, like
/// [`delete`]). Both the read and the persisted write happen inside the
/// same guard that [`update`] uses, so within this process the rewrite can
/// neither clobber a concurrent update to another config file nor lose a
/// concurrent write to the same one — unlike caching a mutated clone of
/// previously read metadata.
///
/// This guard is process-local. Writers on other nodes still race, exactly
/// as they do for [`update`]: each rewrites the whole metadata file, so the
/// later save wins. What this narrows is the window — from "as stale as the
/// local cache" down to a single metadata read plus write.
pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
where
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
{
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let _targets_guard = if config_file == BUCKET_TARGETS_FILE {
Some(acquire_bucket_targets_transaction_lock(bucket).await?)
} else {
None
};
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
bucket_meta_sys.update_config_with(bucket, config_file, mutate).await
}
pub async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let api = bucket_meta_sys_lock.read().await.object_store();
@@ -632,56 +603,30 @@ impl BucketMetadataSys {
return Err(Error::other("errServerNotInitialized"));
};
let mut bm = Self::load_bucket_metadata_for_update(store, bucket, parse).await?;
let updated = bm.update_config(config_file, data)?;
self.save(bm).await?;
Ok(updated)
}
/// See the free [`update_config_with`]: same load-mutate-persist cycle as
/// [`Self::update`], with the payload computed from the loaded metadata
/// instead of supplied up front. Loads through this system's own store so
/// the read and the persisted write target the same instance.
async fn update_config_with<F>(&mut self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
where
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
{
let mut bm = Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true).await?;
let data = mutate(&bm)?;
let updated = bm.update_config(config_file, data)?;
self.save(bm).await?;
Ok(updated)
}
/// Load a bucket's on-disk metadata as the base of a config rewrite.
/// Outside erasure setups a missing metadata file degrades to a fresh
/// default (legacy buckets without one); erasure setups fail instead of
/// fabricating state that a quorum may still hold.
async fn load_bucket_metadata_for_update(store: Arc<ECStore>, bucket: &str, parse: bool) -> Result<BucketMetadata> {
if is_meta_bucketname(bucket) {
return Err(Error::other("errInvalidArgument"));
}
match load_bucket_metadata_parse(store, bucket, parse).await {
Ok(res) => Ok(res),
let mut bm = match load_bucket_metadata_parse(store, bucket, parse).await {
Ok(res) => res,
Err(err) => {
if !runtime_sources::setup_is_erasure().await
&& !runtime_sources::setup_is_dist_erasure().await
&& is_err_bucket_not_found(&err)
{
Ok(BucketMetadata::new(bucket))
BucketMetadata::new(bucket)
} else {
error!("load bucket metadata failed: {}", err);
Err(err)
return Err(err);
}
}
}
};
let updated = bm.update_config(config_file, data)?;
self.save(bm).await?;
Ok(updated)
}
async fn save(&self, bm: BucketMetadata) -> Result<()> {
@@ -796,8 +741,6 @@ impl BucketMetadataSys {
if let Some(config) = &bm.policy_config {
Ok((config.clone(), bm.policy_config_updated_at))
} else if !bm.policy_config_json.is_empty() {
Ok((serde_json::from_slice(&bm.policy_config_json)?, bm.policy_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
@@ -1108,138 +1051,6 @@ mod tests {
);
}
#[tokio::test]
async fn get_bucket_policy_rejects_malformed_cached_policy() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let mut metadata = BucketMetadata::new("malformed-policy");
metadata.policy_config_json = b"{".to_vec();
sys.set("malformed-policy".to_string(), Arc::new(metadata)).await;
let err = sys
.get_bucket_policy("malformed-policy")
.await
.expect_err("malformed persisted policy must not be treated as missing");
assert!(matches!(err, Error::Io(_)), "malformed persisted policy must surface its parse failure");
}
/// A tagging rewrite through `update_config_with` (the Swift metadata
/// POST path) is persisted: it survives a metadata reload from disk, and
/// an emptied rewrite clears the config in the cached copy too instead of
/// leaving stale parsed tags behind.
#[tokio::test]
async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
use s3s::dto::Tag;
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let mut sys = BucketMetadataSys::new(ecstore);
let bucket = "swift-tagging-bucket";
sys.persist_and_set(BucketMetadata::new(bucket))
.await
.expect("initial metadata should persist");
let tagging = Tagging {
tag_set: vec![Tag {
key: Some("swift-meta-color".to_string()),
value: Some("blue".to_string()),
}],
};
let xml = crate::bucket::utils::serialize::<Tagging>(&tagging).expect("tagging should serialize");
sys.update_config_with(bucket, BUCKET_TAGGING_CONFIG, move |bm| {
assert!(bm.tagging_config.is_none(), "rewrite must see the on-disk state");
Ok(xml)
})
.await
.expect("tagging rewrite should persist");
// Simulate the disk-truth reload that used to lose Swift writes: drop
// the cached entry and lazily re-load from the metadata file.
sys.metadata_map.write().await.clear();
let (tags, _) = sys
.get_tagging_config(bucket)
.await
.expect("tagging must survive a reload from disk");
assert_eq!(tags.tag_set.len(), 1);
assert_eq!(tags.tag_set[0].key.as_deref(), Some("swift-meta-color"));
assert_eq!(tags.tag_set[0].value.as_deref(), Some("blue"));
// An emptied rewrite clears the config everywhere.
sys.update_config_with(bucket, BUCKET_TAGGING_CONFIG, |bm| {
assert!(bm.tagging_config.is_some(), "rewrite must see the persisted tags");
Ok(Vec::new())
})
.await
.expect("clearing rewrite should persist");
assert_eq!(
sys.get_tagging_config(bucket).await.unwrap_err(),
Error::ConfigNotFound,
"cleared tagging must not be served from the cache"
);
sys.metadata_map.write().await.clear();
assert_eq!(
sys.get_tagging_config(bucket).await.unwrap_err(),
Error::ConfigNotFound,
"cleared tagging must not reappear after a reload from disk"
);
}
/// The load and the persisted write share one write guard, so concurrent
/// rewrites of the same config compose instead of clobbering each other.
/// Moving the load outside that guard loses all but the last tag.
#[tokio::test]
async fn concurrent_update_config_with_calls_do_not_lose_writes() {
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
use s3s::dto::Tag;
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
let bucket = "swift-tagging-concurrent";
sys.read()
.await
.persist_and_set(BucketMetadata::new(bucket))
.await
.expect("initial metadata should persist");
const WRITERS: usize = 8;
let mut handles = Vec::with_capacity(WRITERS);
for idx in 0..WRITERS {
let sys = sys.clone();
handles.push(tokio::spawn(async move {
sys.write()
.await
.update_config_with(bucket, BUCKET_TAGGING_CONFIG, move |bm| {
// Each writer merges its own tag onto whatever is
// currently persisted — the Swift rewrite shape.
let mut tagging = bm.tagging_config.clone().unwrap_or_else(|| Tagging { tag_set: vec![] });
tagging.tag_set.push(Tag {
key: Some(format!("swift-meta-key{idx}")),
value: Some(idx.to_string()),
});
crate::bucket::utils::serialize::<Tagging>(&tagging).map_err(|e| Error::other(e.to_string()))
})
.await
}));
}
for handle in handles {
handle
.await
.expect("writer task should join")
.expect("rewrite should persist");
}
let (tags, _) = sys
.read()
.await
.get_tagging_config(bucket)
.await
.expect("tagging should be readable");
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
}
fn target(bucket: &str, id: &str) -> BucketTarget {
BucketTarget {
source_bucket: bucket.to_string(),
+10 -101
View File
@@ -15,26 +15,23 @@
use super::metadata_sys::get_bucket_metadata_sys;
use crate::error::{Result, StorageError};
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
use tracing::info;
pub struct PolicySys {}
impl PolicySys {
pub async fn is_allowed(args: &BucketPolicyArgs<'_>) -> bool {
matches!(Self::try_is_allowed(args).await, Ok(true))
}
pub async fn try_is_allowed(args: &BucketPolicyArgs<'_>) -> Result<bool> {
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
}
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
match policy {
Ok(policy) => Ok(policy.is_allowed(args).await),
Err(StorageError::ConfigNotFound) => Ok(args.is_owner),
Err(err) => Err(err),
match Self::get(args.bucket).await {
Ok(cfg) => return cfg.is_allowed(args).await,
Err(err) => {
if err != StorageError::ConfigNotFound {
info!("config get err {:?}", err);
}
}
}
}
args.is_owner
}
pub async fn get(bucket: &str) -> Result<BucketPolicy> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -44,91 +41,3 @@ impl PolicySys {
Ok(cfg)
}
}
#[cfg(test)]
mod tests {
use super::{PolicySys, StorageError};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
use std::collections::HashMap;
fn args<'a>(
is_owner: bool,
groups: &'a Option<Vec<String>>,
conditions: &'a HashMap<String, Vec<String>>,
) -> BucketPolicyArgs<'a> {
BucketPolicyArgs {
bucket: "bucket",
action: Action::S3Action(S3Action::GetObjectAction),
is_owner,
account: "account",
groups,
conditions,
object: "object",
}
}
#[tokio::test]
async fn missing_policy_preserves_owner_and_iam_fallback_semantics() {
let groups = None;
let conditions = HashMap::new();
assert!(
PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(StorageError::ConfigNotFound),)
.await
.expect("missing policy should preserve owner access")
);
assert!(
!PolicySys::is_allowed_with_policy(&args(false, &groups, &conditions), Err(StorageError::ConfigNotFound),)
.await
.expect("missing policy should defer non-owner access to IAM")
);
}
#[tokio::test]
async fn policy_load_failures_propagate() {
let groups = None;
let conditions = HashMap::new();
for (failure, expected_message) in [
(StorageError::Io(std::io::Error::other("policy read failed")), "policy read failed"),
(
StorageError::other("bucket metadata sys not initialized for this instance"),
"bucket metadata sys not initialized for this instance",
),
] {
let result = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(failure)).await;
assert!(
matches!(result, Err(StorageError::Io(ref err)) if err.to_string().contains(expected_message)),
"policy I/O and uninitialized metadata failures must propagate instead of granting owner access"
);
}
}
#[tokio::test]
async fn explicit_bucket_deny_precedes_iam_allow() {
let groups = None;
let conditions = HashMap::new();
let policy: BucketPolicy = serde_json::from_str(
r#"{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Deny",
"Principal":{"AWS":"*"},
"Action":["s3:GetObject"],
"Resource":["arn:aws:s3:::bucket/*"]
}]
}"#,
)
.expect("deny policy should parse");
let bucket_allowed = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Ok(policy))
.await
.expect("loaded bucket policy should evaluate");
let iam_allowed = true;
let request_allowed = bucket_allowed && iam_allowed;
assert!(iam_allowed, "test precondition: IAM grants the action");
assert!(!bucket_allowed, "test precondition: bucket policy explicitly denies the action");
assert!(!request_allowed, "explicit bucket Deny must reject before IAM Allow fallback");
}
}
@@ -417,7 +417,7 @@ impl TransitionClient {
bucket: complete_multipart_upload_result.bucket,
key: complete_multipart_upload_result.key,
etag: trim_etag(&complete_multipart_upload_result.etag),
version_id: self.legacy_remote_version_id(&h)?,
version_id: self.raw_version_id(&h)?.unwrap_or_default().to_string(),
location: complete_multipart_upload_result.location,
expiration: exp_time,
expiration_rule_id: rule_id,
@@ -22,7 +22,7 @@ use bytes::Bytes;
use futures::future::join_all;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use std::io::Error;
use std::sync::{Mutex, MutexGuard, RwLock};
use std::sync::RwLock;
use std::{collections::HashMap, sync::Arc};
use time::{OffsetDateTime, format_description};
use tokio::io::AsyncReadExt;
@@ -46,14 +46,6 @@ use crate::client::utils::base64_encode;
use rustfs_utils::path::trim_etag;
use s3s::header::X_AMZ_EXPIRATION;
fn lock_md5_hasher(
md5_hasher: &Mutex<Option<rustfs_utils::hash::HashAlgorithm>>,
) -> Result<MutexGuard<'_, Option<rustfs_utils::hash::HashAlgorithm>>, std::io::Error> {
md5_hasher
.lock()
.map_err(|_| std::io::Error::other("MD5 hasher state is unavailable"))
}
/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
/// reaches EOF first. Advances the reader so the next call returns the following
/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
@@ -185,7 +177,7 @@ impl TransitionClient {
let length = buf.len();
if opts.send_content_md5 {
let mut md5_hasher = lock_md5_hasher(&self.md5_hasher)?;
let mut md5_hasher = self.md5_hasher.lock().unwrap();
let md5_hash = match md5_hasher.as_mut() {
Some(hasher) => hasher,
None => return Err(std::io::Error::other("MD5 hasher not initialized")),
@@ -378,7 +370,7 @@ impl TransitionClient {
let mut md5_base64: String = "".to_string();
if opts.send_content_md5 {
let mut md5_hasher = lock_md5_hasher(&clone_self.md5_hasher)?;
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
let md5_hash = match md5_hasher.as_mut() {
Some(hasher) => hasher,
None => {
@@ -426,9 +418,6 @@ impl TransitionClient {
}
let results = join_all(futures).await;
for result in results {
result?;
}
select! {
err = err_rx.recv() => {
@@ -578,7 +567,7 @@ impl TransitionClient {
key: object_name.to_string(),
etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
version_id: self.legacy_remote_version_id(h)?,
version_id: self.raw_version_id(h)?.unwrap_or_default().to_string(),
size,
expiration: exp_time,
expiration_rule_id: rule_id,
@@ -631,12 +620,10 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou
#[cfg(test)]
mod tests {
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part};
use crate::object_api::GetObjectReader;
use bytes::Bytes;
use rustfs_utils::hash::HashAlgorithm;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
// Drive a reader through the same per-part loop the multipart stream uses and
// collect the size of every part. Regression for rustfs/rustfs#4811: the old
@@ -746,18 +733,4 @@ mod tests {
"a gap in the parts map must be an error, not a panic"
);
}
#[test]
fn poisoned_md5_state_fails_closed() {
let hasher = Arc::new(Mutex::new(Some(HashAlgorithm::Md5)));
let poison_target = Arc::clone(&hasher);
let _ = std::thread::spawn(move || {
let _guard = poison_target.lock().expect("fresh mutex should lock");
panic!("poison MD5 state");
})
.join();
let error = lock_md5_hasher(&hasher).expect_err("poisoned hash state must not be reused");
assert_eq!(error.kind(), std::io::ErrorKind::Other);
}
}
+1 -1
View File
@@ -201,7 +201,7 @@ impl TransitionClient {
object_name: object_name.to_string(),
object_version_id: opts.version_id,
delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
delete_marker_version_id: self.legacy_remote_version_id(resp.headers())?,
delete_marker_version_id: self.raw_version_id(resp.headers())?.unwrap_or_default().to_string(),
..Default::default()
})
}
+1 -119
View File
@@ -46,33 +46,11 @@ impl RemoteVersion {
Self::Unknown | Self::Disabled => None,
}
}
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
match self {
Self::Unknown => Err(Error::new(
ErrorKind::InvalidData,
"remote object version is unknown; exact version routing is unsafe",
)),
Self::Disabled => Ok(None),
Self::SuspendedNull => Ok(Some("null")),
Self::Exact(version_id) => Ok(Some(version_id)),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ConditionalCreateCapability {
Unsupported,
IfNoneMatchStar,
GenerationMatchZero,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) struct ProviderVersionCapabilities {
raw_version_header: Option<&'static str>,
pub(crate) bucket_versioning_state: bool,
pub(crate) list_object_versions: bool,
pub(crate) conditional_create: ConditionalCreateCapability,
pub(crate) exact_get_delete: bool,
}
@@ -84,59 +62,28 @@ impl ProviderVersionCapabilities {
|| tier_type.eq_ignore_ascii_case("r2")
|| tier_type.eq_ignore_ascii_case("wasabi")
{
let list_object_versions = tier_type.eq_ignore_ascii_case("s3")
|| tier_type.eq_ignore_ascii_case("rustfs")
|| tier_type.eq_ignore_ascii_case("minio")
|| tier_type.eq_ignore_ascii_case("r2");
Self {
raw_version_header: Some(X_AMZ_VERSION_ID),
bucket_versioning_state: list_object_versions,
list_object_versions,
conditional_create: if tier_type.eq_ignore_ascii_case("s3") || tier_type.eq_ignore_ascii_case("r2") {
ConditionalCreateCapability::IfNoneMatchStar
} else {
ConditionalCreateCapability::Unsupported
},
exact_get_delete: true,
}
} else if tier_type.eq_ignore_ascii_case("aliyun") {
Self {
raw_version_header: Some(X_OSS_VERSION_ID),
bucket_versioning_state: false,
list_object_versions: false,
conditional_create: ConditionalCreateCapability::Unsupported,
exact_get_delete: true,
}
} else if tier_type.eq_ignore_ascii_case("tencent") {
Self {
raw_version_header: Some(X_COS_VERSION_ID),
bucket_versioning_state: false,
list_object_versions: false,
conditional_create: ConditionalCreateCapability::Unsupported,
exact_get_delete: true,
}
} else if tier_type.eq_ignore_ascii_case("huaweicloud") {
Self {
raw_version_header: Some(X_OBS_VERSION_ID),
bucket_versioning_state: false,
list_object_versions: false,
conditional_create: ConditionalCreateCapability::Unsupported,
exact_get_delete: true,
}
} else if tier_type.eq_ignore_ascii_case("gcs") {
Self {
raw_version_header: None,
bucket_versioning_state: false,
list_object_versions: false,
conditional_create: ConditionalCreateCapability::GenerationMatchZero,
exact_get_delete: false,
}
} else {
Self {
raw_version_header: None,
bucket_versioning_state: false,
list_object_versions: false,
conditional_create: ConditionalCreateCapability::Unsupported,
exact_get_delete: false,
}
}
@@ -196,7 +143,7 @@ fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
#[cfg(test)]
mod tests {
use super::{BucketVersioningState, ConditionalCreateCapability, ProviderVersionCapabilities, RemoteVersion};
use super::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
use http::{HeaderMap, HeaderValue};
#[test]
@@ -272,71 +219,6 @@ mod tests {
);
}
#[test]
fn provider_capability_matrix_is_conservative_and_provider_specific() {
for (tier_type, state, list, conditional_create, exact_get_delete) in [
("s3", true, true, ConditionalCreateCapability::IfNoneMatchStar, true),
("rustfs", true, true, ConditionalCreateCapability::Unsupported, true),
("minio", true, true, ConditionalCreateCapability::Unsupported, true),
("r2", true, true, ConditionalCreateCapability::IfNoneMatchStar, true),
("wasabi", false, false, ConditionalCreateCapability::Unsupported, true),
("aliyun", false, false, ConditionalCreateCapability::Unsupported, true),
("tencent", false, false, ConditionalCreateCapability::Unsupported, true),
("huaweicloud", false, false, ConditionalCreateCapability::Unsupported, true),
("gcs", false, false, ConditionalCreateCapability::GenerationMatchZero, false),
("azure", false, false, ConditionalCreateCapability::Unsupported, false),
("unsupported", false, false, ConditionalCreateCapability::Unsupported, false),
] {
let capabilities = ProviderVersionCapabilities::for_tier_type(tier_type);
assert_eq!(capabilities.bucket_versioning_state, state, "{tier_type} versioning state");
assert_eq!(capabilities.list_object_versions, list, "{tier_type} version listing");
assert_eq!(capabilities.conditional_create, conditional_create, "{tier_type} conditional create");
assert_eq!(capabilities.exact_get_delete, exact_get_delete, "{tier_type} exact routing");
}
}
#[test]
fn remote_version_states_preserve_unknown_disabled_suspended_and_exact() {
let capabilities = ProviderVersionCapabilities::for_tier_type("s3");
let empty = HeaderMap::new();
let mut null = HeaderMap::new();
null.insert("x-amz-version-id", HeaderValue::from_static("null"));
let mut exact = HeaderMap::new();
exact.insert("x-amz-version-id", HeaderValue::from_static("opaque.generation-7"));
for (headers, state, expected) in [
(&empty, BucketVersioningState::Unknown, RemoteVersion::Unknown),
(&empty, BucketVersioningState::Disabled, RemoteVersion::Disabled),
(&empty, BucketVersioningState::Suspended, RemoteVersion::Unknown),
(&empty, BucketVersioningState::Enabled, RemoteVersion::Unknown),
(&null, BucketVersioningState::Suspended, RemoteVersion::SuspendedNull),
(
&exact,
BucketVersioningState::Enabled,
RemoteVersion::Exact("opaque.generation-7".to_string()),
),
] {
assert_eq!(
capabilities
.remote_version(headers, state)
.expect("version state should normalize"),
expected
);
}
}
#[test]
fn exact_request_routing_fails_closed_for_unknown_versions() {
for (version, expected) in [
(RemoteVersion::Disabled, None),
(RemoteVersion::SuspendedNull, Some("null")),
(RemoteVersion::Exact("opaque-v1".to_string()), Some("opaque-v1")),
] {
assert_eq!(version.exact_request_id().expect("known version state"), expected);
}
assert!(RemoteVersion::Unknown.exact_request_id().is_err());
}
#[test]
fn provider_version_rejects_empty_or_oversized_headers() {
let oversized = "v".repeat(1025);
+2 -33
View File
@@ -31,7 +31,7 @@ use crate::client::{
},
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{CredContext, Credentials, SignatureType, Static},
provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion},
provider_versions::{BucketVersioningState, ProviderVersionCapabilities},
signer_error,
};
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
@@ -332,22 +332,6 @@ impl TransitionClient {
self.provider_version_capabilities().raw_version_id(headers)
}
pub(crate) fn remote_version(
&self,
headers: &HeaderMap,
versioning: BucketVersioningState,
) -> Result<RemoteVersion, std::io::Error> {
self.provider_version_capabilities().remote_version(headers, versioning)
}
pub(crate) fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result<String, std::io::Error> {
Ok(self
.remote_version(headers, BucketVersioningState::Unknown)?
.exact_id()
.unwrap_or_default()
.to_string())
}
fn trace_errors_only_off(&self) {
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
*trace_errors_only = false;
@@ -1111,16 +1095,6 @@ impl Default for ObjectInfo {
}
}
impl ObjectInfo {
pub(crate) fn remote_version(
&self,
capabilities: ProviderVersionCapabilities,
versioning: BucketVersioningState,
) -> Result<RemoteVersion, std::io::Error> {
capabilities.remote_version(&self.metadata, versioning)
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct RestoreInfo {
ongoing_restore: bool,
@@ -1440,7 +1414,7 @@ mod tests {
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
};
use crate::client::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
use crate::client::provider_versions::ProviderVersionCapabilities;
use http::{HeaderMap, HeaderValue};
use http_body_util::Full;
use hyper::body::Bytes;
@@ -1565,11 +1539,6 @@ mod tests {
.expect("opaque provider version should parse");
assert_eq!(info.version_id, None);
assert_eq!(
info.remote_version(ProviderVersionCapabilities::for_tier_type("tencent"), BucketVersioningState::Enabled,)
.expect("opaque response version should remain available"),
RemoteVersion::Exact("opaque.version_01".to_string())
);
assert_eq!(
info.metadata.get("x-cos-version-id").and_then(|value| value.to_str().ok()),
Some("opaque.version_01")
+5 -71
View File
@@ -24,8 +24,8 @@ use crate::cluster::rpc::internode_data_transport::{
use crate::disk::error::{Error, Result};
use crate::disk::{
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
disk_store::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -48,10 +48,9 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -2482,71 +2481,6 @@ impl DiskAPI for RemoteDisk {
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(PreparePartTransactionRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let canonical_body = rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "prepare_part_transaction")?;
let response = client.prepare_part_transaction(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(SettlePartTransactionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
rollback: action == PartTransactionAction::Rollback,
});
let canonical_body = rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "settle_part_transaction")?;
let response = client.settle_part_transaction(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
trace!(
+4 -242
View File
@@ -16,7 +16,6 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
use crate::multipart_listing::paginate_multipart_listing;
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -50,10 +49,7 @@ use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
use std::{collections::HashMap, sync::Arc};
use tokio::sync::RwLock;
use tokio::sync::broadcast::{Receiver, Sender};
use tokio::time::Duration;
@@ -67,8 +63,6 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
@@ -805,52 +799,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
delimiter: Option<String>,
max_uploads: usize,
) -> Result<ListMultipartsInfo> {
let per_set_limit = max_uploads.saturating_add(1);
let results = futures::stream::iter(self.disk_set.iter().cloned())
.map(|set| {
let key_marker = key_marker.clone();
let upload_id_marker = upload_id_marker.clone();
let delimiter = delimiter.clone();
async move {
set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit)
.await
}
})
.buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY)
.collect::<Vec<_>>()
.await;
let mut uploads = Vec::new();
let mut common_prefixes = HashSet::new();
let mut source_truncated = false;
for result in results {
let page = result?;
uploads.extend(page.uploads);
common_prefixes.extend(page.common_prefixes);
source_truncated |= page.is_truncated;
}
let page = paginate_multipart_listing(
uploads,
common_prefixes.into_iter().collect(),
key_marker.as_deref(),
key_marker.as_ref().and(upload_id_marker.as_deref()),
max_uploads,
source_truncated,
);
Ok(ListMultipartsInfo {
key_marker,
upload_id_marker,
next_key_marker: page.next_key_marker,
next_upload_id_marker: page.next_upload_id_marker,
max_uploads,
is_truncated: page.is_truncated,
uploads: page.uploads,
common_prefixes: page.common_prefixes,
prefix: prefix.to_owned(),
delimiter,
})
self.get_disks_by_key(prefix)
.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
.await
}
#[tracing::instrument(skip(self))]
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
@@ -1160,7 +1111,6 @@ mod tests {
use crate::layout::endpoints::SetupType;
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::list::ListOperations as _;
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use rustfs_lock::client::local::LocalClient;
use serial_test::serial;
@@ -1298,194 +1248,6 @@ mod tests {
assert_eq!(result, (Some(3), Some(1), Some(0)));
}
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
let format = FormatV3::new(2, 2);
let mut temp_dirs = Vec::new();
let mut all_endpoints = Vec::new();
let mut disk_sets = Vec::new();
for set_index in 0..2 {
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..2 {
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
.expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(set_index);
endpoint.set_disk_index(disk_index);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("disk should be created");
let mut disk_format = format.clone();
disk_format.erasure.this = format.erasure.sets[set_index][disk_index];
save_format_file(&Some(disk.clone()), &Some(disk_format))
.await
.expect("format should be saved");
temp_dirs.push(temp_dir);
all_endpoints.push(endpoint.clone());
endpoints.push(endpoint);
disks.push(Some(disk));
}
disk_sets.push(
SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(disks)),
2,
1,
0,
set_index,
endpoints,
format.clone(),
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
)
.await,
);
}
let sets = Arc::new(Sets {
id: format.id,
disk_set: disk_sets,
pool_idx: 0,
endpoints: PoolEndpoints {
legacy: false,
set_count: 2,
drives_per_set: 2,
endpoints: Endpoints::from(all_endpoints),
cmd_line: String::new(),
platform: String::new(),
},
format,
parity_count: 1,
set_count: 2,
set_drive_count: 2,
default_parity_count: 1,
distribution_algo: DistributionAlgoVersion::V1,
exit_signal: None,
ctx: bootstrap_ctx(),
});
(temp_dirs, sets)
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
sets.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut keys_by_set = [Vec::new(), Vec::new()];
for index in 0..100 {
let key = format!("logs/{index:03}.bin");
let set_index = sets.get_hashed_set_index(&key);
if keys_by_set[set_index].len() < 2 {
keys_by_set[set_index].push(key);
}
if keys_by_set.iter().all(|keys| keys.len() == 2) {
break;
}
}
assert!(keys_by_set.iter().all(|keys| keys.len() == 2), "test keys must span both sets");
let repeated_key = keys_by_set[0][0].clone();
let mut expected = Vec::new();
for key in keys_by_set.iter().flatten() {
let upload = sets
.new_multipart_upload(&bucket, key, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
expected.push((key.clone(), upload.upload_id));
}
let second = sets
.new_multipart_upload(&bucket, &repeated_key, &ObjectOptions::default())
.await
.expect("second upload for the same key should be created");
expected.push((repeated_key, second.upload_id));
expected.sort();
let mut actual = Vec::new();
let mut key_marker = None;
let mut upload_id_marker = None;
for _ in 0..expected.len() + 1 {
let page = sets
.list_multipart_uploads(&bucket, "logs/", key_marker.clone(), upload_id_marker.clone(), None, 2)
.await
.expect("multipart page should list across every set");
assert!(page.uploads.len() <= 2);
actual.extend(
page.uploads
.iter()
.map(|upload| (upload.object.clone(), upload.upload_id.clone())),
);
if !page.is_truncated {
break;
}
key_marker = page.next_key_marker;
upload_id_marker = page.next_upload_id_marker;
}
assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
let mut deduped = actual.clone();
deduped.dedup();
assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");
let mut nested_by_set = [None, None];
for index in 0..100 {
let key = format!("nested/group-{index:03}/file.bin");
let set_index = sets.get_hashed_set_index(&key);
nested_by_set[set_index].get_or_insert(key);
if nested_by_set.iter().all(Option::is_some) {
break;
}
}
for key in nested_by_set.iter().flatten() {
sets.new_multipart_upload(&bucket, key, &ObjectOptions::default())
.await
.expect("nested multipart upload should be created");
}
let mut expected_prefixes = nested_by_set
.iter()
.flatten()
.map(|key| {
key.rsplit_once('/')
.expect("nested key should contain a delimiter")
.0
.to_string()
+ "/"
})
.collect::<Vec<_>>();
expected_prefixes.sort();
let first = sets
.list_multipart_uploads(&bucket, "nested/", None, None, Some("/".to_string()), 1)
.await
.expect("first delimiter page should list across every set");
assert!(first.is_truncated);
assert_eq!(first.common_prefixes, expected_prefixes[..1]);
let second = sets
.list_multipart_uploads(
&bucket,
"nested/",
first.next_key_marker,
first.next_upload_id_marker,
Some("/".to_string()),
1,
)
.await
.expect("second delimiter page should list across every set");
assert!(!second.is_truncated);
assert_eq!(second.common_prefixes, expected_prefixes[1..]);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn sets_list_objects_v2_lists_objects_within_the_pool() {
+42 -125
View File
@@ -35,7 +35,7 @@ use std::{
Arc,
atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
},
time::{Duration, SystemTime, UNIX_EPOCH},
time::Duration,
};
use tokio::{sync::RwLock, time};
use tokio_util::sync::CancellationToken;
@@ -339,19 +339,21 @@ impl Drop for DiskHealthWaitingGuard<'_> {
impl DiskHealthTracker {
/// Create a new disk health tracker
pub fn new() -> Self {
let now = current_unix_time();
let now_nanos = unix_nanos(now);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
Self {
last_success: AtomicI64::new(now_nanos),
last_started: AtomicI64::new(now_nanos),
last_success: AtomicI64::new(now),
last_started: AtomicI64::new(now),
status: AtomicU32::new(DISK_HEALTH_OK),
waiting: AtomicU32::new(0),
runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
consecutive_failures: AtomicU32::new(0),
consecutive_successes: AtomicU32::new(0),
offline_since_unix_secs: AtomicI64::new(0),
last_transition_unix_secs: AtomicI64::new(unix_secs_i64(now)),
last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
last_capacity_total: AtomicU64::new(0),
last_capacity_used: AtomicU64::new(0),
last_capacity_free: AtomicU64::new(0),
@@ -361,7 +363,11 @@ impl DiskHealthTracker {
/// Log a successful operation
pub fn log_success(&self) {
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.last_success.store(now, Ordering::Relaxed);
}
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
@@ -423,14 +429,11 @@ impl DiskHealthTracker {
}
pub fn offline_duration(&self) -> Option<Duration> {
self.offline_duration_at(current_unix_secs())
}
fn offline_duration_at(&self, now: u64) -> Option<Duration> {
let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
if offline_since <= 0 {
return None;
}
let now = current_unix_secs();
Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
}
@@ -489,12 +492,6 @@ impl DiskHealthTracker {
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
}
fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) {
let now_nanos = unix_nanos(now);
let now_secs = unix_secs_i64(now);
self.status.store(DISK_HEALTH_OK, Ordering::Release);
self.runtime_state
.store(RuntimeDriveHealthState::Online as u32, Ordering::Release);
@@ -502,9 +499,11 @@ impl DiskHealthTracker {
self.consecutive_successes.store(0, Ordering::Release);
self.offline_since_unix_secs.store(0, Ordering::Release);
self.waiting.store(0, Ordering::Release);
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap();
let now_nanos = now.as_nanos() as i64;
self.last_success.store(now_nanos, Ordering::Relaxed);
self.last_started.store(now_nanos, Ordering::Relaxed);
self.last_transition_unix_secs.store(now_secs, Ordering::Release);
self.last_transition_unix_secs.store(now.as_secs() as i64, Ordering::Release);
record_drive_runtime_state(endpoint, RuntimeDriveHealthState::Online);
}
@@ -613,33 +612,10 @@ impl DiskHealthTracker {
}
fn current_unix_secs() -> u64 {
// Zero is reserved as "not recorded" by health timestamp atomics.
current_unix_time().as_secs().max(1)
}
fn current_unix_nanos() -> i64 {
unix_nanos(current_unix_time())
}
fn current_unix_time() -> Duration {
unix_time_since_epoch(SystemTime::now())
}
fn unix_time_since_epoch(time: SystemTime) -> Duration {
time.duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO)
}
fn unix_nanos(time: Duration) -> i64 {
i64::try_from(time.as_nanos()).unwrap_or(i64::MAX)
}
fn unix_secs_i64(time: Duration) -> i64 {
i64::try_from(time.as_secs()).unwrap_or(i64::MAX)
}
fn elapsed_since(last_nanos: i64, now_nanos: i64) -> Duration {
let elapsed_nanos = now_nanos.saturating_sub(last_nanos).max(0);
Duration::from_nanos(u64::try_from(elapsed_nanos).unwrap_or(u64::MAX))
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs()
}
impl Default for DiskHealthTracker {
@@ -659,7 +635,11 @@ struct HealthDiskCtxValue {
impl HealthDiskCtxValue {
fn log_success(&self) {
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.last_success.store(now, Ordering::Relaxed);
}
}
@@ -794,7 +774,12 @@ impl LocalDiskWrapper {
}
let last_success_nanos = health.last_success.load(Ordering::Relaxed);
let elapsed = elapsed_since(last_success_nanos, current_unix_nanos());
let elapsed = Duration::from_nanos(
(std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64 - last_success_nanos) as u64
);
if elapsed < SKIP_IF_SUCCESS_BEFORE {
continue;
@@ -1106,7 +1091,11 @@ impl LocalDiskWrapper {
self.check_disk_stale().await?;
// Record operation start
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.health.last_started.store(now, Ordering::Relaxed);
let _waiting_guard = self.health.waiting_guard();
if timeout_duration == Duration::ZERO {
@@ -1328,7 +1317,11 @@ impl DiskAPI for LocalDiskWrapper {
}
// Record operation start
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.health.last_started.store(now, Ordering::Relaxed);
self.health.increment_waiting();
// Execute the operation
@@ -1480,33 +1473,6 @@ impl DiskAPI for LocalDiskWrapper {
.await
}
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
self.track_disk_health(
|| async {
self.disk
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn settle_part_transaction(&self, volume: &str, path: &str, action: crate::disk::PartTransactionAction) -> Result<()> {
self.track_disk_health(
|| async { self.disk.settle_part_transaction(volume, path, action).await },
get_max_timeout_duration(),
)
.await
}
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
self.track_disk_health(|| async { self.disk.delete(volume, path, opt).await }, get_max_timeout_duration())
.await
@@ -2277,53 +2243,4 @@ mod tests {
assert!(health.mark_offline(&endpoint, "again"));
assert!(health.is_faulty());
}
#[test]
fn unix_time_clamps_epoch_and_pre_epoch_to_zero() {
let before_epoch = UNIX_EPOCH
.checked_sub(Duration::from_nanos(1))
.expect("one nanosecond before the Unix epoch should be representable");
assert_eq!(unix_time_since_epoch(UNIX_EPOCH), Duration::ZERO);
assert_eq!(unix_time_since_epoch(before_epoch), Duration::ZERO);
}
#[test]
fn elapsed_and_offline_duration_saturate_on_clock_rollback() {
let health = DiskHealthTracker::new();
health.offline_since_unix_secs.store(10, Ordering::Release);
assert_eq!(elapsed_since(10, 12), Duration::from_nanos(2));
assert_eq!(elapsed_since(10, 9), Duration::ZERO);
assert_eq!(health.offline_duration_at(9), Some(Duration::ZERO));
}
#[test]
fn pre_epoch_retry_reset_updates_the_complete_health_state() {
let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry-pre-epoch").expect("endpoint should parse");
let health = DiskHealthTracker::new();
health.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
health
.runtime_state
.store(RuntimeDriveHealthState::Offline as u32, Ordering::Release);
health.consecutive_failures.store(3, Ordering::Release);
health.consecutive_successes.store(2, Ordering::Release);
health.offline_since_unix_secs.store(11, Ordering::Release);
health.waiting.store(4, Ordering::Release);
health.last_success.store(12, Ordering::Release);
health.last_started.store(13, Ordering::Release);
health.last_transition_unix_secs.store(14, Ordering::Release);
health.reset_for_store_init_retry_at(&endpoint, unix_time_since_epoch(UNIX_EPOCH - Duration::from_nanos(1)));
assert_eq!(health.status.load(Ordering::Acquire), DISK_HEALTH_OK);
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 0);
assert_eq!(health.consecutive_successes.load(Ordering::Acquire), 0);
assert_eq!(health.offline_since_unix_secs.load(Ordering::Acquire), 0);
assert_eq!(health.waiting.load(Ordering::Acquire), 0);
assert_eq!(health.last_success.load(Ordering::Acquire), 0);
assert_eq!(health.last_started.load(Ordering::Acquire), 0);
assert_eq!(health.last_transition_unix_secs.load(Ordering::Acquire), 0);
}
}
+2 -297
View File
@@ -19,8 +19,7 @@ use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_r
use crate::disk::{
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, PART_TRANSACTION_NEW_META,
PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
endpoint::Endpoint,
@@ -81,10 +80,6 @@ const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_
const ENV_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS";
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_COUNT: u64 = 2;
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: u64 = 100;
const PART_TRANSACTION_OLD_DATA: &str = "old.data";
const PART_TRANSACTION_OLD_DATA_ABSENT: &str = "old.data.absent";
const PART_TRANSACTION_OLD_META_ABSENT: &str = "old.meta.absent";
const PART_TRANSACTION_PUBLISH_META: &str = "publish.meta";
enum ReadAllError {
Open(std::io::Error),
Disk(DiskError),
@@ -146,28 +141,6 @@ fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
}
}
fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> std::io::Result<()> {
match std::fs::symlink_metadata(src) {
Ok(metadata) if metadata.is_file() => std::fs::hard_link(src, backup),
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file")),
Err(err) if err.kind() == ErrorKind::NotFound => std::fs::write(absent, []),
Err(err) => Err(err),
}
}
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
match std::fs::symlink_metadata(backup) {
Ok(metadata) if metadata.is_file() => {
remove_file_if_exists(restore)?;
std::fs::hard_link(backup, restore)?;
std::fs::rename(restore, current)
}
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction backup is not a file")),
Err(err) if err.kind() == ErrorKind::NotFound && absent.is_file() => remove_file_if_exists(current),
Err(err) => Err(err),
}
}
fn rollback_committed_rename_std(
dst_file_path: &Path,
new_data_path: Option<&Path>,
@@ -6474,151 +6447,6 @@ impl DiskAPI for LocalDisk {
Ok(resp)
}
#[tracing::instrument(level = "trace", skip_all)]
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
let src_volume_dir = self.get_bucket_path(src_volume)?;
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
if !skip_access_checks(src_volume) {
super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
}
if !skip_access_checks(dst_volume) {
super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
}
let src_file_path = self.get_object_path(src_volume, src_path)?;
let dst_file_path = self.get_object_path(dst_volume, dst_path)?;
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] {
check_path_length(path.to_string_lossy().as_ref())?;
}
let durability = effective_durability(dst_volume);
tokio::task::spawn_blocking(move || {
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
if !source.is_file() {
return Err(DiskError::FileAccessDenied);
}
if transaction_path.exists() {
return Err(DiskError::FileAccessDenied);
}
let Some(transaction_parent) = transaction_path.parent() else {
return Err(DiskError::InvalidPath);
};
std::fs::create_dir_all(transaction_parent).map_err(to_file_error)?;
let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4()));
std::fs::create_dir(&staging_path).map_err(to_file_error)?;
let prepare_result = (|| -> std::io::Result<()> {
snapshot_part_transaction_file(
&dst_file_path,
&staging_path.join(PART_TRANSACTION_OLD_DATA),
&staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
)?;
snapshot_part_transaction_file(
&dst_meta_path,
&staging_path.join(PART_TRANSACTION_OLD_META),
&staging_path.join(PART_TRANSACTION_OLD_META_ABSENT),
)?;
let mut new_meta = std::fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(staging_path.join(PART_TRANSACTION_NEW_META))?;
std::io::Write::write_all(&mut new_meta, &meta)?;
if durability.syncs_commit_metadata() {
new_meta.sync_data()?;
os::fsync_dir_std(&staging_path)?;
}
std::fs::rename(&staging_path, &transaction_path)?;
if durability.syncs_commit_metadata() {
os::fsync_dir_std(transaction_parent)?;
}
Ok(())
})();
if let Err(err) = prepare_result {
let _ = remove_dir_all_if_exists(&staging_path);
return Err(to_file_error(err).into());
}
Ok(())
})
.await
.map_err(DiskError::from)?
}
#[tracing::instrument(level = "trace", skip_all)]
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
self.get_bucket_path(volume)?;
let current_data_path = self.get_object_path(volume, path)?;
let current_meta_path = self.get_object_path(volume, &format!("{path}.meta"))?;
let transaction_path = self.get_object_path(volume, &crate::disk::part_transaction_path(path))?;
for candidate in [&current_data_path, &current_meta_path, &transaction_path] {
check_path_length(candidate.to_string_lossy().as_ref())?;
}
let durability = effective_durability(volume);
tokio::task::spawn_blocking(move || {
match std::fs::symlink_metadata(&transaction_path) {
Ok(metadata) if metadata.is_dir() => {}
Ok(_) => return Err(DiskError::FileCorrupt),
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()),
Err(err) => return Err(to_file_error(err).into()),
}
if action == PartTransactionAction::Rollback {
std::fs::write(transaction_path.join(PART_TRANSACTION_ROLLBACK), []).map_err(to_file_error)?;
if durability.syncs_commit_metadata() {
os::fsync_dir_std(&transaction_path).map_err(to_file_error)?;
}
restore_part_transaction_file(
&current_data_path,
&transaction_path.join(PART_TRANSACTION_OLD_DATA),
&transaction_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
&transaction_path.join("restore.data"),
)
.map_err(to_file_error)?;
restore_part_transaction_file(
&current_meta_path,
&transaction_path.join(PART_TRANSACTION_OLD_META),
&transaction_path.join(PART_TRANSACTION_OLD_META_ABSENT),
&transaction_path.join("restore.meta"),
)
.map_err(to_file_error)?;
if durability.syncs_commit_metadata()
&& let Some(parent) = current_data_path.parent()
{
os::fsync_dir_std(parent).map_err(to_file_error)?;
}
}
let Some(parent) = transaction_path.parent() else {
return Err(DiskError::InvalidPath);
};
let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4()));
std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?;
if durability.syncs_commit_metadata() {
os::fsync_dir_std(parent).map_err(to_file_error)?;
}
remove_dir_all_if_exists(&cleanup_path).map_err(to_file_error)?;
Ok(())
})
.await
.map_err(DiskError::from)??;
self.io_backend.invalidate_cached_fd(volume, path).await;
Ok(())
}
#[tracing::instrument(level = "trace", skip_all)]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
let src_volume_dir = self.get_bucket_path(src_volume)?;
@@ -6700,42 +6528,6 @@ impl DiskAPI for LocalDisk {
}
}
let transaction_publish_meta = if src_is_dir {
None
} else {
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META);
match fs::read(&transaction_meta_path).await {
Ok(expected_meta) => {
if expected_meta.as_slice() != meta.as_ref() {
return Err(DiskError::FileCorrupt);
}
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
let source_meta_path = transaction_meta_path.clone();
let publish_path = publish_meta_path.clone();
tokio::task::spawn_blocking(move || {
remove_file_if_exists(&publish_path)?;
std::fs::hard_link(source_meta_path, &publish_path)
})
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
Some(publish_meta_path)
}
// Old peers know only RenamePart. The new coordinator never
// reaches rename unless prepare succeeded, so an absent
// transaction directory identifies the rolling-upgrade legacy
// path. A present directory without new.meta is corruption.
Err(err) if err.kind() == ErrorKind::NotFound => match fs::metadata(&transaction_path).await {
Ok(_) => return Err(DiskError::FileCorrupt),
Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None,
Err(meta_err) => return Err(to_file_error(meta_err).into()),
},
Err(err) => return Err(to_file_error(err).into()),
}
};
// UploadPart is acknowledged once this rename lands, so the part data and
// its directory entry must be durable before we return. Relaxed keeps the
// part payload fdatasync but leaves the directory entry to the page cache.
@@ -6769,17 +6561,7 @@ impl DiskAPI for LocalDisk {
return Err(DiskError::FileAccessDenied);
}
if let Some(transaction_publish_meta) = transaction_publish_meta {
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
rename_all(&transaction_publish_meta, &dst_meta_path, &dst_volume_dir).await?;
if durability.syncs_commit_metadata()
&& let Some(parent) = dst_meta_path.parent()
{
os::fsync_dir(parent).await.map_err(to_file_error)?;
}
} else {
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
}
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
if let Some(parent) = src_file_path.parent() {
self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?;
@@ -8500,12 +8282,6 @@ mod test {
file_info.data_dir = data_dir;
file_info.data = data;
file_info.size = size;
file_info.parts = vec![ObjectPartInfo {
number: 1,
size: usize::try_from(size).expect("test object size should fit usize"),
actual_size: size,
..Default::default()
}];
file_info.mod_time = Some(OffsetDateTime::now_utc());
file_info
}
@@ -9594,15 +9370,9 @@ mod test {
.await
.expect("source part should be written");
disk.prepare_part_transaction(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
.await
.expect("part transaction should be prepared");
disk.rename_part(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
.await
.expect("rename_part should commit part");
disk.settle_part_transaction(bucket, "object/part.1", PartTransactionAction::Commit)
.await
.expect("part transaction should be committed");
assert_eq!(
disk.read_all(bucket, "object/part.1")
@@ -9620,71 +9390,6 @@ mod test {
matches!(disk.read_all(tmp_volume, "upload/part.1").await, Err(DiskError::FileNotFound)),
"source part must be removed after a successful commit"
);
let legacy_payload = Bytes::from_static(b"legacy peer payload");
let legacy_meta = Bytes::from_static(b"legacy peer metadata");
disk.write_all(tmp_volume, "legacy/part.1", legacy_payload.clone())
.await
.expect("legacy source part should be written");
disk.rename_part(tmp_volume, "legacy/part.1", bucket, "object/part.1", legacy_meta.clone())
.await
.expect("pre-transaction peer RenamePart should remain supported");
assert_eq!(
disk.read_all(bucket, "object/part.1")
.await
.expect("legacy destination part should be readable"),
legacy_payload
);
assert_eq!(
disk.read_all(bucket, "object/part.1.meta")
.await
.expect("legacy destination metadata should be readable"),
legacy_meta
);
}
#[tokio::test]
async fn test_part_transaction_rolls_back_data_published_before_metadata() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
ensure_test_volume(&disk, "tmp").await;
ensure_test_volume(&disk, "bucket").await;
disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data"))
.await
.expect("new part should be staged");
disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data"))
.await
.expect("old part data should be staged");
disk.write_all("bucket", "object/part.1.meta", Bytes::from_static(b"old metadata"))
.await
.expect("old part metadata should be staged");
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata"))
.await
.expect("part transaction should be prepared");
disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1")
.await
.expect("data publication should succeed");
disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Rollback)
.await
.expect("part transaction should roll back");
assert_eq!(
disk.read_all("bucket", "object/part.1")
.await
.expect("old part data should be restored"),
Bytes::from_static(b"old data")
);
assert_eq!(
disk.read_all("bucket", "object/part.1.meta")
.await
.expect("old part metadata should be restored"),
Bytes::from_static(b"old metadata")
);
}
struct BlockingScanWriter {
-58
View File
@@ -38,16 +38,6 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
pub const HEALING_MARKER_PATH: &str = "healing.bin";
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"),
None => format!(".{part_path}.rustfs-txn"),
}
}
use crate::cluster::rpc::RemoteDisk;
use crate::cluster::rpc::build_internode_data_transport_from_env;
@@ -72,12 +62,6 @@ pub type DiskStore = Arc<Disk>;
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PartTransactionAction {
Commit,
Rollback,
}
#[derive(Clone, Copy, Debug)]
pub struct MmapCopyStageMetrics {
pub(crate) path: &'static str,
@@ -397,35 +381,6 @@ impl DiskAPI for Disk {
}
}
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
match self {
Disk::Local(local_disk) => {
local_disk
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
Disk::Remote(remote_disk) => {
remote_disk
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
}
}
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.settle_part_transaction(volume, path, action).await,
Disk::Remote(remote_disk) => remote_disk.settle_part_transaction(volume, path, action).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
match self {
@@ -704,19 +659,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
// ReadFileStream
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>;
async fn prepare_part_transaction(
&self,
_src_volume: &str,
_src_path: &str,
_dst_volume: &str,
_dst_path: &str,
_meta: Bytes,
) -> Result<()> {
Err(DiskError::MethodNotAllowed)
}
async fn settle_part_transaction(&self, _volume: &str, _path: &str, _action: PartTransactionAction) -> Result<()> {
Err(DiskError::MethodNotAllowed)
}
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
// VerifyFile
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
+2 -47
View File
@@ -443,9 +443,7 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith
size.div_ceil(shard_size) * algo.size() + size
}
/// Verify an interleaved per-block bitrot shard file and consume the reader
/// through EOF. Bytes beyond the encoded length are corruption, even when every
/// expected block has a valid hash.
/// Verify an interleaved per-block bitrot shard file.
///
/// The read loop below assumes every block on disk is `[hash][data]` (streaming
/// bitrot). It is therefore only valid for the streaming Highway variants, whose
@@ -489,11 +487,6 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
left -= read;
}
let mut trailing = [0u8; 1];
if r.read(&mut trailing).await? != 0 {
return Err(std::io::Error::other("bitrot shard file has trailing data"));
}
Ok(())
}
@@ -867,19 +860,12 @@ mod tests {
.await
.expect("valid bitrot shard file should verify");
let mut truncated = written.clone();
truncated.pop();
let err = bitrot_verify(Cursor::new(truncated), written.len(), data.len(), algo.clone(), shard_size)
.await
.expect_err("one-byte-short shard file must be rejected while reading");
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
let err = bitrot_verify(Cursor::new(written.clone()), written.len() - 1, data.len(), algo.clone(), shard_size)
.await
.expect_err("wrong file size must be rejected before reading data");
assert!(err.to_string().contains("size mismatch"));
let mut corrupt = written.clone();
let mut corrupt = written;
let last = corrupt.len() - 1;
corrupt[last] ^= 0x80;
let err = bitrot_verify(
@@ -892,21 +878,6 @@ mod tests {
.await
.expect_err("hash mismatch must reject corrupted data");
assert!(err.to_string().contains("hash mismatch"));
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = written.clone();
oversized.extend_from_slice(&trailing);
let err = bitrot_verify(
Cursor::new(oversized),
written.len(),
data.len(),
HashAlgorithm::HighwayHash256S,
shard_size,
)
.await
.expect_err("trailing bytes after a valid encoded shard must be rejected");
assert!(err.to_string().contains("trailing data"));
}
}
#[tokio::test]
@@ -938,22 +909,6 @@ mod tests {
assert!(err.to_string().contains("hash mismatch"));
}
#[tokio::test]
async fn bitrot_verify_accepts_exact_legacy_streaming_layout() {
let data = b"legacy streaming bitrot";
let shard_size = 8;
let algo = HashAlgorithm::HighwayHash256SLegacy;
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, algo.clone());
for chunk in data.chunks(shard_size) {
writer.write(chunk).await.expect("legacy streaming shard should encode");
}
let written = writer.into_inner().into_inner();
bitrot_verify(Cursor::new(written.clone()), written.len(), data.len(), algo, shard_size)
.await
.expect("exact legacy streaming shard should remain valid");
}
#[tokio::test]
async fn write_all_vectored_retries_partial_hash_and_data_writes_and_rejects_zero_write() {
let mut writer = LimitedVectoredWriter {
+7 -57
View File
@@ -28,6 +28,7 @@ use md5::{Digest, Md5};
use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
use rustfs_utils::path::path_join_buf;
#[cfg(feature = "rio-v2")]
use serde::Deserialize;
#[cfg(feature = "rio-v2")]
use sha2::Sha256;
@@ -43,7 +44,6 @@ const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size";
const DEFAULT_SSE_ALGORITHM: &str = "AES256";
const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
#[cfg(feature = "rio-v2")]
const DARE_PAYLOAD_SIZE: i64 = 64 * 1024;
#[cfg(feature = "rio-v2")]
@@ -1716,39 +1716,16 @@ fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context
fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct LocalSseDekEnvelope<'a> {
version: u8,
nonce: &'a str,
ciphertext: &'a str,
let parts: Vec<&str> = encrypted_dek.split(':').collect();
if parts.len() != 2 {
return Err(Error::other("invalid managed DEK format"));
}
let (nonce, ciphertext) = match serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek) {
Ok(envelope) => {
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version)));
}
(envelope.nonce, envelope.ciphertext)
}
Err(_) => {
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
return Err(Error::other("invalid managed DEK format"));
};
if ciphertext.contains(':') {
return Err(Error::other("invalid managed DEK format"));
}
(nonce, ciphertext)
}
};
let nonce_vec = BASE64_STANDARD
.decode(nonce)
.decode(parts[0])
.map_err(|_| Error::other("invalid managed DEK nonce"))?;
let ciphertext = BASE64_STANDARD
.decode(ciphertext)
.decode(parts[1])
.map_err(|_| Error::other("invalid managed DEK ciphertext"))?;
let nonce_array: [u8; 12] = nonce_vec
@@ -2347,36 +2324,9 @@ mod tests {
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::from([0u8; 12]);
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek");
serde_json::json!({
"version": LOCAL_SSE_DEK_FORMAT_VERSION,
"nonce": BASE64_STANDARD.encode(nonce),
"ciphertext": BASE64_STANDARD.encode(ciphertext),
})
.to_string()
}
fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String {
let key = Key::<Aes256Gcm>::from(master_key);
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::from([0u8; 12]);
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek");
format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext))
}
#[test]
fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() {
let envelope = serde_json::json!({
"version": LOCAL_SSE_DEK_FORMAT_VERSION + 1,
"nonce": BASE64_STANDARD.encode([0u8; 12]),
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
})
.to_string();
let error =
decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed");
assert!(error.to_string().contains("unsupported managed DEK format version"));
}
#[cfg(feature = "rio-v2")]
fn seal_managed_s3_object_key_for_test(
bucket: &str,
@@ -2483,7 +2433,7 @@ mod tests {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async {
let data_key = [0x24; 32];
let base_nonce = [0x14; 12];
let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]);
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [7u8; 32]);
let metadata = HashMap::from([
(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
+15 -213
View File
@@ -46,10 +46,7 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::{
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
part_transaction_path,
};
use crate::disk::OldCurrentSize;
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
use crate::io_support::bitrot::{
@@ -3177,155 +3174,6 @@ impl SetDisks {
}
}
async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result<bool> {
let disks = self.get_disks_internal().await;
let transaction_path = part_transaction_path(dst_object);
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
let current_meta_path = format!("{dst_object}.meta");
let reads = disks.iter().map(|disk| {
let disk = disk.clone();
let transaction_meta_path = transaction_meta_path.clone();
let rollback_path = rollback_path.clone();
let current_meta_path = current_meta_path.clone();
async move {
let Some(disk) = disk else {
return Ok((None, None, false));
};
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound) => None,
Err(err) => return Err(err),
};
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
Ok(_) => true,
Err(DiskError::FileNotFound) => false,
Err(err) => return Err(err),
};
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &current_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
Err(_) => None,
};
Ok((transaction_meta, current_meta, rollback))
}
});
let observations = join_all(reads).await.into_iter().collect::<disk::error::Result<Vec<_>>>()?;
if observations.iter().all(|(transaction, _, _)| transaction.is_none()) {
return Ok(false);
}
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
for (_, current, _) in &observations {
if let Some(current) = current {
*current_counts.entry(current.clone()).or_default() += 1;
}
}
let current_quorum = current_counts
.into_iter()
.find_map(|(meta, count)| (count >= write_quorum).then_some(meta));
let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}");
let old_meta_absent_path = format!("{transaction_path}/old.meta.absent");
let decisions = observations
.iter()
.enumerate()
.filter_map(|(index, (transaction_meta, _, rollback))| {
transaction_meta.as_ref().map(|meta| (index, meta.clone(), *rollback))
})
.map(|(index, transaction_meta, rollback)| {
let disk = disks[index].clone();
let old_meta_path = old_meta_path.clone();
let old_meta_absent_path = old_meta_absent_path.clone();
let current_quorum = current_quorum.clone();
async move {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let action = if rollback {
PartTransactionAction::Rollback
} else if current_quorum.as_ref() == Some(&transaction_meta) {
PartTransactionAction::Commit
} else if let Some(current_quorum) = current_quorum {
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
Ok(_) => PartTransactionAction::Commit,
Err(DiskError::FileNotFound) => {
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
Ok(_) => PartTransactionAction::Commit,
Err(_) => return Err(DiskError::FileCorrupt),
}
}
Err(err) => return Err(err),
}
} else {
PartTransactionAction::Rollback
};
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
.await?;
Ok(action == PartTransactionAction::Commit)
}
});
let results = join_all(decisions).await;
if let Some(err) = results.iter().find_map(|result| result.as_ref().err()) {
return Err(err.clone());
}
Ok(results.iter().any(|result| matches!(result, Ok(true))))
}
pub(in crate::set_disk) async fn recover_part_transactions(
&self,
part_path: &str,
read_quorum: usize,
write_quorum: usize,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let listings = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
async move {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
.await
}
}))
.await;
let mut transaction_parts = HashSet::new();
let mut errs = Vec::with_capacity(listings.len());
for listing in listings {
match listing {
Ok(entries) => {
errs.push(None);
for entry in entries {
let name = entry.trim_end_matches('/');
let Some(part_number) = name
.strip_prefix(".part.")
.and_then(|name| name.strip_suffix(".rustfs-txn"))
.and_then(|number| number.parse::<usize>().ok())
else {
continue;
};
transaction_parts.insert(part_number);
}
}
Err(err) => errs.push(Some(err)),
}
}
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
return Err(err);
}
for part_number in transaction_parts {
self.recover_part_transaction(&format!("{part_path}part.{part_number}"), write_quorum)
.await?;
}
Ok(())
}
#[tracing::instrument(skip(disks, meta))]
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn rename_part(
@@ -3339,46 +3187,23 @@ impl SetDisks {
write_quorum: usize,
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
) -> disk::error::Result<Vec<Option<DiskStore>>> {
self.recover_part_transaction(dst_object, write_quorum).await?;
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
let prepare_tasks = disks.iter().map(|disk| {
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
let meta = meta.clone();
async move {
let disk = disk?;
Some(
disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await,
)
}
});
let prepare_results = join_all(prepare_tasks).await;
let prepare_errs = prepare_results
.into_iter()
.map(|result| match result {
Some(Ok(())) => None,
Some(Err(err)) => Some(err),
None => Some(DiskError::DiskNotFound),
})
.collect::<Vec<_>>();
let prepared_disks = Self::eval_disks(disks, &prepare_errs);
if reduce_write_quorum_errs(&prepare_errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some() {
self.recover_part_transaction(&dst_object, write_quorum).await?;
return Err(DiskError::ErasureWriteQuorum);
}
// Do NOT pre-delete the destination part before renaming: the per-disk
// `rename_part` replaces `part.N` atomically (std::fs::rename) and rewrites
// `part.N.meta`, so the pre-delete is redundant — and destructive. It
// opened a window where an already-committed (ACKed) part was removed on
// every disk before the new rename landed, so a re-upload that then failed
// quorum destroyed the committed part outright (backlog#853 / #799 B4).
// The atomic rename overwrites in place; on quorum failure below we roll
// the destination back.
let mut errs = Vec::with_capacity(disks.len());
let futures = prepared_disks.iter().map(|disk| {
let futures = disks.iter().map(|disk| {
let disk = disk.clone();
let meta = meta.clone();
let src_bucket = src_bucket.clone();
@@ -3428,42 +3253,19 @@ impl SetDisks {
);
}
let reduced_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
if let Some(err) = reduced_err {
let rollbacks = prepared_disks.iter().filter_map(|disk| {
disk.clone().map(|disk| {
let dst_object = dst_object.clone();
async move {
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Rollback)
.await
}
})
});
let rollback_results = join_all(rollbacks).await;
self.recover_part_transaction(&dst_object, write_quorum).await?;
if let Some(rollback_err) = rollback_results.iter().find_map(|result| result.as_ref().err()) {
warn!(error = %rollback_err, "rename_part rollback did not settle on every prepared disk");
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
if let Some(context) = quorum_context {
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
} else {
warn!("rename_part errs {:?}", &errs);
}
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
.await;
return Err(err);
}
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
if !committed {
let err = DiskError::ErasureWriteQuorum;
if let Some(context) = quorum_context {
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
} else {
warn!("rename_part errs {:?}", &errs);
}
return Err(err);
}
Ok(Self::eval_disks(&prepared_disks, &errs))
let disks = Self::eval_disks(disks, &errs);
Ok(disks)
}
pub(in crate::set_disk) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
+21 -6
View File
@@ -357,6 +357,7 @@ impl Drop for ObjectLockDiagGuard {
struct SetDiskLockGuardedReader {
inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
guard: Option<ObjectLockDiagGuard>,
remaining: Option<usize>,
}
impl AsyncRead for SetDiskLockGuardedReader {
@@ -364,8 +365,23 @@ impl AsyncRead for SetDiskLockGuardedReader {
let had_capacity = buf.remaining() > 0;
let filled_before = buf.filled().len();
let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
self.guard.take();
match &poll {
Poll::Ready(Ok(())) if buf.filled().len() > filled_before => {
let produced = buf.filled().len() - filled_before;
if let Some(remaining) = self.remaining.as_mut() {
*remaining = remaining.saturating_sub(produced);
if *remaining == 0 {
self.guard.take();
}
}
}
Poll::Ready(Ok(())) if had_capacity => {
self.guard.take();
}
Poll::Ready(Err(_)) => {
self.guard.take();
}
_ => {}
}
poll
}
@@ -374,19 +390,20 @@ impl AsyncRead for SetDiskLockGuardedReader {
fn finish_set_disk_read_lock(
mut reader: GetObjectReader,
read_lock_guard: Option<ObjectLockDiagGuard>,
lock_optimization_enabled: bool,
bucket: &str,
object: &str,
) -> GetObjectReader {
if lock_optimization_enabled || reader.buffered_body.is_some() {
if reader.buffered_body.is_some() || reader.object_info.size == 0 {
release_materialized_read_lock(bucket, object, read_lock_guard);
return reader;
}
if let Some(guard) = read_lock_guard {
let remaining = usize::try_from(reader.object_info.size).ok();
reader.stream = Box::new(SetDiskLockGuardedReader {
inner: reader.stream,
guard: Some(guard),
remaining,
});
}
reader
@@ -687,8 +704,6 @@ mod core;
mod ctx;
mod metadata;
mod ops;
#[cfg(test)]
pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
@@ -199,32 +199,6 @@ mod tests {
.expect("healthy temp shard should verify");
assert_eq!(verified, 1);
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = encoded.to_vec();
oversized.extend_from_slice(&trailing);
disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(oversized))
.await
.expect("oversized temp shard should be staged");
let err = verify_written_bitrot_shards(
&[Some(disk.clone())],
None,
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path,
logical_shard_size,
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("disk shard with trailing bytes must not be committed");
assert!(err.to_string().contains("trailing data"));
}
let mut corrupt = encoded.to_vec();
let last = corrupt.len() - 1;
corrupt[last] ^= 0x01;
@@ -251,40 +225,4 @@ mod tests {
.expect_err("corrupt no-parity temp shard must not be committed");
assert!(err.to_string().contains("bitrot self-verify failed"));
}
#[tokio::test]
async fn no_parity_inline_self_verify_rejects_trailing_bytes() {
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
let shard_size = 16usize;
let payload = b"inline bitrot payload";
let encoded = encode_streaming_shard(payload, shard_size).await;
let disks = disks.into_iter().map(Some).collect::<Vec<_>>();
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = encoded.to_vec();
oversized.extend_from_slice(&trailing);
let parts = [FileInfo {
data: Some(Bytes::from(oversized)),
..Default::default()
}];
let err = verify_written_bitrot_shards(
&disks,
Some(&parts),
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "inline-object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path: "unused-for-inline",
logical_shard_size: payload.len(),
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("inline shard with trailing bytes must not be committed");
assert!(err.to_string().contains("trailing data"));
}
}
}
+8 -257
View File
@@ -19,71 +19,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_HEAL: &str = "heal";
const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename";
const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete";
#[cfg(test)]
static HEAL_RENAME_FAILURES: std::sync::Mutex<Vec<(String, String, usize)>> = std::sync::Mutex::new(Vec::new());
#[cfg(test)]
struct HealRenameFailureScope {
bucket: String,
object: String,
}
#[cfg(test)]
impl HealRenameFailureScope {
fn install(bucket: &str, object: &str, disk_indexes: &[usize]) -> Self {
let mut failures = HEAL_RENAME_FAILURES
.lock()
.expect("heal rename failure registry should not poison");
assert!(
!failures
.iter()
.any(|(registered_bucket, registered_object, _)| { registered_bucket == bucket && registered_object == object }),
"heal rename failures must be installed once per object"
);
failures.extend(
disk_indexes
.iter()
.map(|index| (bucket.to_string(), object.to_string(), *index)),
);
Self {
bucket: bucket.to_string(),
object: object.to_string(),
}
}
}
#[cfg(test)]
impl Drop for HealRenameFailureScope {
fn drop(&mut self) {
HEAL_RENAME_FAILURES
.lock()
.expect("heal rename failure registry should not poison")
.retain(|(bucket, object, _)| bucket != &self.bucket || object != &self.object);
}
}
#[cfg(test)]
fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> bool {
let mut failures = HEAL_RENAME_FAILURES
.lock()
.expect("heal rename failure registry should not poison");
if let Some(position) = failures
.iter()
.position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index))
{
failures.swap_remove(position);
true
} else {
false
}
}
#[cfg(not(test))]
fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) -> bool {
false
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct PartFailureSummary {
@@ -724,12 +659,8 @@ impl SetDisks {
}
}
// Rename from tmp location to the actual location.
// MinIO stops on the first RenameData error. RustFS intentionally
// continues per target, but reports any residue after all attempts
// so successful repairs survive and failed targets remain retryable.
let mut rename_attempts = 0usize;
let mut rename_successes = 0usize;
let mut healed_disks = vec![None; out_dated_disks.len()];
for (index, outdated_disk) in out_dated_disks.iter().enumerate() {
if let Some(disk) = outdated_disk {
rename_attempts += 1;
@@ -738,18 +669,9 @@ impl SetDisks {
// Attempt a rename now from healed data to final location.
parts_metadata[index].set_healing();
let rename_result = if should_fail_heal_rename(bucket, object, index) {
Err(DiskError::Unexpected)
} else {
disk.rename_data(
RUSTFS_META_TMP_BUCKET,
&tmp_id,
parts_metadata[index].clone(),
bucket,
object,
)
.await
};
let rename_result = disk
.rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object)
.await;
if let Err(err) = &rename_result {
warn!(
@@ -768,7 +690,6 @@ impl SetDisks {
);
} else {
rename_successes += 1;
healed_disks[index] = Some(disk.clone());
if parts_metadata[index].is_remote() {
let rm_data_dir =
parts_metadata[index].data_dir.expect("operation should succeed").to_string();
@@ -813,18 +734,15 @@ impl SetDisks {
.await
.map_err(DiskError::other)?;
self.record_healed_capacity_scope(&healed_disks);
if rename_successes < rename_attempts {
if rename_attempts > 0 && rename_successes == 0 {
return Ok((
result,
Some(DiskError::other(format!(
"{HEAL_RENAME_INCOMPLETE}: {rename_successes} of {rename_attempts} targets committed for \
{bucket}/{object}"
))),
Some(DiskError::other(format!("all healed data rename attempts failed for {bucket}/{object}"))),
));
}
self.record_healed_capacity_scope(&out_dated_disks);
// The object is healthy here; sweep any data dirs left behind
// by pre-#3510 unversioned overwrites, which the dangling paths
// above never touch (issues #3231, #3191). Best effort — a
@@ -1394,13 +1312,11 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
#[cfg(test)]
mod heal_result_report_tests {
use super::SetDisks;
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
use crate::disk::endpoint::Endpoint;
use crate::disk::error::DiskError;
use crate::disk::format::FormatV3;
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
use crate::disk::{DiskOption, DiskStore, new_disk};
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::{config::storageclass, store::init_format::save_format_file};
@@ -1490,171 +1406,6 @@ mod heal_result_report_tests {
(dir, set)
}
async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
let mut entries = Vec::new();
for dir in temp_dirs {
let tmp = dir.path().join(RUSTFS_META_TMP_BUCKET);
let mut read_dir = match tokio::fs::read_dir(&tmp).await {
Ok(read_dir) => read_dir,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
Err(err) => panic!("tmp directory should be readable: {err}"),
};
while let Some(entry) = read_dir.next_entry().await.expect("tmp entry should be readable") {
let name = entry.file_name().to_string_lossy().into_owned();
if name != ".trash" {
entries.push(name);
}
}
}
entries
}
#[tokio::test]
#[serial_test::serial]
async fn heal_rename_outcome_matrix_reports_partial_and_retries_failed_targets() {
for (case, failed_attempts, expect_error) in [
("ok-ok", Vec::new(), false),
("ok-err", vec![1], true),
("err-ok", vec![0], true),
("err-err", vec![0, 1], true),
] {
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = format!("heal-rename-{case}");
let object = "object.bin";
for disk in &disks {
disk.make_volume(&bucket).await.expect("bucket volume should be created");
}
let payload = vec![0x5a; 1024 * 1024];
let mut reader = PutObjReader::from_vec(payload);
set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("source object should be written");
let source = disks[2]
.read_version("", &bucket, object, "", &ReadOptions::default())
.await
.expect("source metadata should be readable");
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
let tmp_entries_before_heal = non_trash_tmp_entries(&temp_dirs).await;
let target_slots = {
let mut slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
slots.sort_unstable();
slots
};
let failed_slots = failed_attempts
.iter()
.map(|attempt| target_slots[*attempt])
.collect::<Vec<_>>();
let failed_physical_indexes = [0, 1]
.into_iter()
.filter(|index| failed_slots.contains(&(source.erasure.distribution[*index] - 1)))
.collect::<Vec<_>>();
for index in [0, 1] {
tokio::fs::remove_file(
temp_dirs[index]
.path()
.join(&bucket)
.join(object)
.join(data_dir.to_string())
.join("part.1"),
)
.await
.expect("target shard should be removed before heal");
}
let failure_scope = HealRenameFailureScope::install(&bucket, object, &failed_slots);
let (first_result, first_error) = set
.heal_object(
&bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("heal should report its per-target rename outcome");
drop(failure_scope);
assert_eq!(first_error.is_some(), expect_error, "{case}: aggregate status must match target outcomes");
if let Some(error) = first_error {
let error = error.to_string();
assert!(
error.contains(HEAL_RENAME_INCOMPLETE),
"{case}: partial/all failure must have an explicit retryable status: {error}"
);
assert!(
error.contains(&format!("{} of 2 targets committed", 2 - failed_slots.len())),
"{case}: aggregate status must distinguish partial from all-target failure: {error}"
);
}
for index in [0, 1] {
let expected = if failed_physical_indexes.contains(&index) {
DriveState::Missing
} else {
DriveState::Ok
};
assert_eq!(
first_result.after.drives[index].state,
expected.to_string(),
"{case}: after.drives must reflect the actual rename outcome at index {index}"
);
assert_eq!(
temp_dirs[index]
.path()
.join(&bucket)
.join(object)
.join(data_dir.to_string())
.join("part.1")
.exists(),
!failed_physical_indexes.contains(&index),
"{case}: tmp cleanup must neither delete committed shards nor expose failed targets"
);
}
let tmp_entries_after_heal = non_trash_tmp_entries(&temp_dirs).await;
assert!(
tmp_entries_after_heal
.iter()
.all(|entry| tmp_entries_before_heal.contains(entry)),
"{case}: first heal must not leave a new temporary shard: {tmp_entries_after_heal:?}"
);
if !failed_slots.is_empty() {
let (retry_result, retry_error) = set
.heal_object(
&bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("second heal should retry failed targets");
assert!(retry_error.is_none(), "{case}: second heal should complete remaining targets");
for index in [0, 1] {
assert_eq!(
retry_result.after.drives[index].state,
DriveState::Ok.to_string(),
"{case}: second heal must converge target {index}"
);
}
let tmp_entries_after_retry = non_trash_tmp_entries(&temp_dirs).await;
assert!(
tmp_entries_after_retry
.iter()
.all(|entry| tmp_entries_before_heal.contains(entry)),
"{case}: retry must not leave a new temporary shard: {tmp_entries_after_retry:?}"
);
}
}
}
// Regression for #955: an offline disk must contribute exactly one drive
// record. Before the fix the offline branch fell through and pushed a second
// (Corrupt) record for the same disk, so `before/after.drives` grew to
+2 -38
View File
@@ -69,13 +69,6 @@ struct HealWalkCollector {
}
impl HealWalkCollector {
fn lock_objects(&self) -> disk::error::Result<std::sync::MutexGuard<'_, Vec<HealWalkObject>>> {
self.objects.lock().map_err(|_| {
self.cancel.cancel();
DiskError::FileCorrupt
})
}
/// Expand one resolved entry into its versions and record it. Cancels the
/// walk once EITHER page bound (distinct object names OR expanded versions)
/// is met — always at a sorted object-key boundary so a heavily-versioned
@@ -112,9 +105,7 @@ impl HealWalkCollector {
let added = versions.len();
let (objs_len, ver_total) = {
let Ok(mut objects) = self.lock_objects() else {
return;
};
let mut objects = self.objects.lock().unwrap();
objects.push(HealWalkObject {
name: entry.name,
versions,
@@ -248,10 +239,7 @@ impl SetDisks {
Err(err) => return Err(err),
}
let objects = {
let mut objects = collector.lock_objects()?;
std::mem::take(&mut *objects)
};
let objects = std::mem::take(&mut *collector.objects.lock().unwrap());
let truncated = collector.truncated.load(Ordering::SeqCst);
Ok(finalize_heal_walk_page(objects, forward_to, truncated))
}
@@ -322,28 +310,4 @@ mod tests {
assert_eq!(next_forward, None, "a complete final page must not carry a resume cursor");
assert!(!truncated);
}
#[test]
fn poisoned_collector_state_cancels_the_walk() {
let collector = Arc::new(HealWalkCollector {
bucket: "bucket".to_string(),
batch_objects: 2,
version_budget: 2,
objects: Mutex::new(Vec::new()),
version_total: AtomicUsize::new(0),
truncated: AtomicBool::new(false),
cancel: CancellationToken::new(),
});
let poison_target = Arc::clone(&collector);
let _ = std::thread::spawn(move || {
let _guard = poison_target.objects.lock().expect("fresh mutex should lock");
panic!("poison heal collector");
})
.join();
let error = collector.lock_objects().expect_err("poisoned collection must fail closed");
assert_eq!(error, DiskError::FileCorrupt);
assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk");
assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed");
}
}
-23
View File
@@ -29,12 +29,6 @@ impl SetDisks {
pub async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
ListOperations::new(self.ctx()).delete_all(bucket, prefix).await
}
pub(crate) async fn delete_all_with_quorum(&self, bucket: &str, prefix: &str, write_quorum: usize) -> Result<()> {
ListOperations::new(self.ctx())
.delete_all_with_quorum(bucket, prefix, write_quorum)
.await
}
}
/// List/prefix maintenance operations, borrowing the `SetDisks` core state
@@ -54,14 +48,6 @@ impl<'a> ListOperations<'a> {
}
pub(crate) async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
self.delete_all_inner(bucket, prefix, None).await
}
async fn delete_all_with_quorum(&self, bucket: &str, prefix: &str, write_quorum: usize) -> Result<()> {
self.delete_all_inner(bucket, prefix, Some(write_quorum)).await
}
async fn delete_all_inner(&self, bucket: &str, prefix: &str, write_quorum: Option<usize>) -> Result<()> {
let disks = self.ctx.disks().read().await;
let disks = disks.clone();
@@ -93,9 +79,6 @@ impl<'a> ListOperations<'a> {
Ok(_) => {
errors.push(None);
}
Err(DiskError::FileNotFound | DiskError::PathNotFound | DiskError::VolumeNotFound) => {
errors.push(None);
}
Err(e) => {
errors.push(Some(e));
}
@@ -114,12 +97,6 @@ impl<'a> ListOperations<'a> {
);
}
if let Some(write_quorum) = write_quorum
&& let Some(err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum)
{
return Err(err.into());
}
Ok(())
}
}
+196 -525
View File
@@ -26,8 +26,6 @@ use crate::crash_inject::{self, CrashPoint};
use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream};
use std::future::Future;
#[cfg(test)]
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::task::JoinSet;
@@ -35,7 +33,7 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
#[cfg(test)]
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum MultipartCommitPause {
enum MultipartCommitPause {
PutPartBeforeLockLost,
PutPartAfterRename,
BeforeLockLost,
@@ -47,13 +45,12 @@ struct MultipartCommitBarrierState {
bucket: String,
object: String,
pause: MultipartCommitPause,
armed: AtomicBool,
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[cfg(test)]
pub(crate) struct MultipartCommitBarrier {
struct MultipartCommitBarrier {
state: Arc<MultipartCommitBarrierState>,
}
@@ -63,12 +60,11 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
#[cfg(test)]
impl MultipartCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
let state = Arc::new(MultipartCommitBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
pause,
armed: AtomicBool::new(true),
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
});
@@ -82,13 +78,13 @@ impl MultipartCommitBarrier {
Self { state }
}
pub(crate) async fn wait_until_paused(&self) {
async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("multipart completion should reach the deterministic commit barrier");
}
pub(crate) fn release(&self) {
fn release(&self) {
self.state.release.notify_one();
}
}
@@ -116,9 +112,7 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
.cloned();
if let Some(barrier) = barrier
&& barrier.armed.swap(false, Ordering::AcqRel)
{
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
@@ -783,9 +777,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
mut max_parts: usize,
opts: &ObjectOptions,
) -> Result<ListPartsInfo> {
let _upload_guard = self
.acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts)
.await?;
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
@@ -1263,15 +1254,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let _upload_guard = self
.acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts)
.await?;
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
self.check_upload_id_exists(bucket, object, upload_id, false).await?;
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
self.delete_all_with_quorum(
RUSTFS_META_MULTIPART_BUCKET,
&upload_id_path,
fi.write_quorum(self.default_write_quorum()),
)
.await
self.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await
}
// complete_multipart_upload finished
#[tracing::instrument(skip(self))]
@@ -1303,18 +1289,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
if !opts.no_lock && object_lock_guard.is_none() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
.await?,
);
}
let upload_guard = self
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
.await?;
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let (mut fi, mut files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let has_layout_candidate = range_seek_rollout_enabled
&& fi
.data_dir
@@ -1327,7 +1303,22 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
&token,
)
});
let quorum_validated_layout_token = if has_layout_candidate {
let upload_guard = if has_layout_candidate {
if object_lock_guard.is_none() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
.await?,
);
}
let guard = self
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
.await?;
(fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
guard
} else {
None
};
let quorum_validated_layout_token = upload_guard.as_ref().and_then(|_| {
fi.data_dir
.filter(|data_dir| !data_dir.is_nil())
.map(|data_dir| data_dir.to_string())
@@ -1338,9 +1329,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
token,
)
})
} else {
None
};
});
rustfs_utils::http::metadata_compat::remove_str(
&mut fi.metadata,
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
@@ -1367,9 +1356,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
let part_meta_paths = uploaded_parts
.iter()
@@ -1727,6 +1713,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
if !opts.no_lock && object_lock_guard.is_none() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
.await?,
);
}
// Phase 2 (backlog#899): fence the commit on lock loss before any destructive
// step. If the refresh heartbeat has observed a refresh-quorum loss, another
// writer may have re-acquired this object's lock; proceeding would race a
@@ -1829,36 +1821,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
#[cfg(test)]
pause_multipart_commit(bucket, object, MultipartCommitPause::AfterRename).await;
let cleanup_store = self.clone();
let cleanup_upload_id_path = upload_id_path.clone();
let cleanup_bucket = bucket.to_owned();
let cleanup_object = object.to_owned();
let cleanup_upload_id = upload_id.to_owned();
let cleanup_handle = tokio::spawn(async move {
let _upload_guard = upload_guard;
if let Err(err) = cleanup_store
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_id_path, write_quorum)
.await
{
warn!(
bucket = %cleanup_bucket,
object = %cleanup_object,
upload_id = %cleanup_upload_id,
error = ?err,
"completed multipart upload staging cleanup did not reach write quorum"
);
}
});
if let Err(err) = cleanup_handle.await {
warn!(
bucket = %bucket,
object = %object,
upload_id = %upload_id,
error = ?err,
"completed multipart upload staging cleanup task failed"
);
}
drop(upload_guard);
drop(object_lock_guard); // drop object lock guard to release the lock
// backlog#1321: enqueue heal only when the committed replicas actually
@@ -1903,6 +1866,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
});
}
let upload_id_path = upload_id_path.clone();
let store = self.clone();
let _cleanup_handle = tokio::spawn(async move {
let _ = store.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await;
});
for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk
&& disk.is_online().await
@@ -2178,6 +2147,163 @@ mod tests {
}
}
async fn put_two_part_object(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, first: &[u8], second: &[u8]) {
use crate::storage_api_contracts::multipart::MultipartOperations as _;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let mut completed = Vec::new();
for (part_number, payload) in [(1, first), (2, second)] {
let payload_len = i64::try_from(payload.len()).expect("test payload length should fit i64");
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload.to_vec()), payload_len, payload_len, None, None, false)
.expect("part hash reader should be created"),
);
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default())
.await
.expect("multipart part should be written");
completed.push(CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
});
}
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, completed, &ObjectOptions::default())
.await
.expect("multipart upload should complete");
}
async fn assert_get_blocks_delete_at_part_boundary(
set_disks: &Arc<SetDisks>,
bucket: &str,
object: &'static str,
range: Option<HTTPRangeSpec>,
expected: Vec<u8>,
) {
use crate::set_disk::read::get_part_boundary_barrier;
use http::HeaderMap;
use tokio::io::AsyncReadExt as _;
let barrier = get_part_boundary_barrier::arm(bucket, object, 0);
let mut reader = set_disks
.get_object_reader(bucket, object, range, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("GET should return a body reader");
let body_complete = Arc::new(Notify::new());
let body_complete_task = Arc::clone(&body_complete);
let allow_reader_drop = Arc::new(Notify::new());
let allow_reader_drop_task = Arc::clone(&allow_reader_drop);
let expected_len = expected.len();
let read = tokio::spawn(async move {
let mut body = vec![0_u8; expected_len];
reader.stream.read_exact(&mut body).await?;
body_complete_task.notify_one();
allow_reader_drop_task.notified().await;
Ok::<_, std::io::Error>(body)
});
barrier.wait_until_paused().await;
let delete_set = Arc::clone(set_disks);
let delete_bucket = bucket.to_string();
let delete_started = Arc::new(Notify::new());
let delete_started_task = Arc::clone(&delete_started);
let mut delete = tokio::spawn(async move {
delete_started_task.notify_one();
delete_set
.delete_object(&delete_bucket, object, ObjectOptions::default())
.await
});
delete_started.notified().await;
tokio::task::yield_now().await;
assert!(!delete.is_finished(), "DELETE must wait behind the streaming GET read lock");
barrier.release();
body_complete.notified().await;
let delete_result = tokio::time::timeout(Duration::from_secs(2), &mut delete).await;
allow_reader_drop.notify_one();
delete_result
.expect("DELETE must acquire the lock after the exact Content-Length without waiting for reader drop")
.expect("DELETE task should not panic")
.expect("DELETE should proceed after GET delivers its exact Content-Length");
let body = read
.await
.expect("GET task should not panic")
.expect("GET stream should deliver its exact Content-Length");
assert_eq!(body.len(), expected.len(), "successful GET body must match the requested length");
assert_eq!(body, expected, "DELETE must not truncate or mix the multipart snapshot");
}
#[tokio::test]
#[serial]
async fn multipart_get_keeps_delete_blocked_at_part_boundary_in_both_lock_modes() {
use crate::set_disk::read::get_part_boundary_barrier;
use tokio::io::AsyncReadExt as _;
for enabled in ["false", "true"] {
temp_env::async_with_vars(
[
(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some(enabled)),
(ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("false")),
],
async {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = format!("get-delete-part-boundary-{enabled}");
let object = "object";
make_bucket_on_all(&disk_stores, &bucket).await;
let first = vec![0x41; GLOBAL_MIN_PART_SIZE.as_u64() as usize];
let second = vec![0x42; 64 * 1024];
let expected: Vec<u8> = first.iter().chain(&second).copied().collect();
put_two_part_object(&set_disks, &bucket, object, &first, &second).await;
assert_get_blocks_delete_at_part_boundary(&set_disks, &bucket, object, None, expected).await;
let range_object = "range-object";
put_two_part_object(&set_disks, &bucket, range_object, &first, &second).await;
let range_start = i64::try_from(first.len() - 4096).expect("range start should fit i64");
let range_end = i64::try_from(first.len() + 4095).expect("range end should fit i64");
let range = HTTPRangeSpec {
is_suffix_length: false,
start: range_start,
end: range_end,
};
let range_expected = first[first.len() - 4096..].iter().chain(&second[..4096]).copied().collect();
assert_get_blocks_delete_at_part_boundary(&set_disks, &bucket, range_object, Some(range), range_expected)
.await;
let cancelled_object = "cancelled-object";
put_two_part_object(&set_disks, &bucket, cancelled_object, &first, &second).await;
let cancel_barrier = get_part_boundary_barrier::arm(&bucket, cancelled_object, 0);
let mut cancelled_reader = set_disks
.get_object_reader(&bucket, cancelled_object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("cancelled GET should return a body reader");
let cancelled_read = tokio::spawn(async move {
let mut body = Vec::new();
cancelled_reader.stream.read_to_end(&mut body).await
});
cancel_barrier.wait_until_paused().await;
cancelled_read.abort();
cancelled_read.await.expect_err("aborted GET task should report cancellation");
cancel_barrier.release();
tokio::time::timeout(
Duration::from_secs(10),
set_disks.delete_object(&bucket, cancelled_object, ObjectOptions::default()),
)
.await
.expect("DELETE should promptly acquire the lock after GET cancellation")
.expect("DELETE after cancellation should succeed");
},
)
.await;
}
}
async fn stage_upload_with_create_opts(
set_disks: &Arc<SetDisks>,
bucket: &str,
@@ -2214,317 +2340,6 @@ mod tests {
)
}
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x47; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(2).await;
assert!(!abort.is_finished(), "abort must wait for the completion upload lock");
barrier.release();
complete
.await
.expect("completion task should not panic")
.expect("completion should win the upload finalization");
let abort_err = abort
.await
.expect("abort task should not panic")
.expect_err("abort must observe the upload as finalized");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("complete-first must leave the committed object readable");
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
}
async fn assert_abort_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x48; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let object_holder = set_disks
.new_ns_lock(bucket, object)
.await
.expect("object namespace lock should be created")
.get_write_lock(Duration::from_secs(5))
.await
.expect("test should hold the object lock");
let holder = set_disks
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
.await
.expect("upload namespace lock should be created")
.get_write_lock(Duration::from_secs(5))
.await
.expect("test should hold the upload lock");
signaling.wait_for_attempts(1).await;
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(2).await;
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
drop(holder);
abort
.await
.expect("abort task should not panic")
.expect("abort should win the upload finalization");
drop(object_holder);
let complete_err = complete
.await
.expect("completion task should not panic")
.expect_err("completion must observe the aborted upload");
assert!(matches!(complete_err, StorageError::InvalidUploadID(..)));
let object_err = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect_err("abort-first must not publish an object");
assert!(matches!(object_err, StorageError::ObjectNotFound(..)));
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
}
async fn assert_quorum_minus_one_retry_preserves_completable_part(
disk_count: usize,
parity: usize,
success_indices: &[usize],
) {
use tokio::io::AsyncReadExt as _;
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await;
let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]);
let object = "object";
make_bucket_on_all(&disk_stores, &bucket).await;
let payload = vec![0x41; 1 << 20];
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, &bucket, object, &payload, &ObjectOptions::default()).await;
let (upload_meta, _) = set_disks
.check_upload_id_exists(&bucket, object, &upload_id, false)
.await
.expect("staged upload metadata should be readable");
let upload_path = SetDisks::get_upload_id_dir(&bucket, object, &upload_id);
let write_quorum = upload_meta.write_quorum(set_disks.default_write_quorum());
assert_eq!(success_indices.len() + 1, write_quorum);
let part_path = format!(
"{}/{}/part.1",
upload_path,
upload_meta.data_dir.expect("multipart upload should have a data directory")
);
let acknowledged_meta = disk_stores[0]
.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{part_path}.meta"))
.await
.expect("acknowledged part metadata should be readable");
let retry_path = format!("{}/part.1", Uuid::new_v4());
let mut retry_disks = vec![None; disk_stores.len()];
for &index in success_indices {
disk_stores[index]
.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"retry shard"))
.await
.expect("retry shard should be staged");
retry_disks[index] = Some(disk_stores[index].clone());
}
let err = set_disks
.rename_part(
&retry_disks,
RUSTFS_META_TMP_BUCKET,
&retry_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
acknowledged_meta,
write_quorum,
None,
)
.await
.expect_err("quorum-minus-one renamed shards must remain below write quorum");
assert_eq!(err, DiskError::ErasureWriteQuorum);
for (index, disk) in disk_stores.iter().enumerate() {
assert!(
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path).await.is_ok(),
"old acknowledged shard on disk {index} must survive"
);
}
set_disks
.clone()
.complete_multipart_upload(&bucket, object, &upload_id, parts, &ObjectOptions::default())
.await
.expect("the old acknowledged part should remain completable");
let mut reader = set_disks
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed object should be readable");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("completed object should stream fully");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn upload_part_retry_quorum_failure_preserves_old_part_across_ec_geometries() {
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[0, 1]).await;
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[2, 3]).await;
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[0, 1, 2]).await;
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[3, 4, 5]).await;
}
#[tokio::test]
async fn complete_multipart_upload_recovers_interrupted_part_retry() {
use tokio::io::AsyncReadExt as _;
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await;
let bucket = "multipart-retry-recovery";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let payload = vec![0x42; 1 << 20];
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
let (upload_meta, _) = set_disks
.check_upload_id_exists(bucket, object, &upload_id, false)
.await
.expect("staged upload metadata should be readable");
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let part_path = format!(
"{}/{}/part.1",
upload_path,
upload_meta.data_dir.expect("multipart upload should have a data directory")
);
let retry_meta = Bytes::from_static(b"interrupted retry metadata");
for (index, disk) in disk_stores.iter().enumerate().take(3) {
let retry_path = format!("{}/part.1", Uuid::new_v4());
disk.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"interrupted retry shard"))
.await
.expect("retry shard should be staged");
disk.prepare_part_transaction(
RUSTFS_META_TMP_BUCKET,
&retry_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
retry_meta.clone(),
)
.await
.expect("part transaction should be prepared");
disk.rename_part(
RUSTFS_META_TMP_BUCKET,
&retry_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
retry_meta.clone(),
)
.await
.unwrap_or_else(|err| panic!("retry shard {index} should be published: {err}"));
}
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default())
.await
.expect("completion should roll back the interrupted quorum-minus-one retry");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed object should be readable");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("completed object should stream fully");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn rename_part_quorum_failure_without_old_part_removes_new_shards() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
let src_path = format!("{}/part.1", Uuid::new_v4());
let dst_path = format!("{}/part.1", Uuid::new_v4());
let mut retry_disks = vec![None; disk_stores.len()];
for index in [0, 1] {
disk_stores[index]
.write_all(RUSTFS_META_TMP_BUCKET, &src_path, Bytes::from_static(b"new shard"))
.await
.expect("new shard should be staged");
retry_disks[index] = Some(disk_stores[index].clone());
}
let err = set_disks
.rename_part(
&retry_disks,
RUSTFS_META_TMP_BUCKET,
&src_path,
RUSTFS_META_MULTIPART_BUCKET,
&dst_path,
Bytes::from_static(b"retry metadata"),
3,
None,
)
.await
.expect_err("two renamed shards must remain below write quorum");
assert_eq!(err, DiskError::ErasureWriteQuorum);
for (index, disk) in disk_stores.iter().enumerate() {
assert!(
matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &dst_path).await, Err(DiskError::FileNotFound)),
"failed first upload must not leave a destination on disk {index}"
);
}
}
async fn make_multipart_lock_test_set_disks() -> Arc<SetDisks> {
let endpoints = vec![
Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
@@ -3192,81 +3007,6 @@ mod tests {
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn abort_and_complete_linearize_for_plain_sse_and_legacy_layouts() {
assert_complete_first_linearizes("multipart-complete-first-plain", "object", ObjectOptions::default()).await;
assert_abort_first_linearizes("multipart-abort-first-plain", "object", ObjectOptions::default()).await;
let encrypted_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
assert_complete_first_linearizes("multipart-complete-first-sse", "object", encrypted_opts.clone()).await;
assert_abort_first_linearizes("multipart-abort-first-sse", "object", encrypted_opts.clone()).await;
})
.await;
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("false"))], async {
assert_complete_first_linearizes("multipart-complete-first-legacy", "object", encrypted_opts.clone()).await;
assert_abort_first_linearizes("multipart-abort-first-legacy", "object", encrypted_opts).await;
})
.await;
}
#[tokio::test]
async fn abort_enforces_delete_write_quorum_boundary() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-abort-delete-quorum";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let quorum_upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let saved_disks = {
let mut disks = set_disks.disks.write().await;
let saved = disks.clone();
disks[3] = None;
saved
};
set_disks
.abort_multipart_upload(bucket, object, &quorum_upload.upload_id, &ObjectOptions::default())
.await
.expect("abort should succeed at the exact delete write quorum");
*set_disks.disks.write().await = saved_disks;
assert!(matches!(
set_disks
.check_upload_id_exists(bucket, object, &quorum_upload.upload_id, false)
.await,
Err(StorageError::InvalidUploadID(..))
));
let below_quorum_upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("second multipart upload should be created");
let saved_disks = {
let mut disks = set_disks.disks.write().await;
let saved = disks.clone();
disks[2] = None;
disks[3] = None;
saved
};
let err = set_disks
.abort_multipart_upload(bucket, object, &below_quorum_upload.upload_id, &ObjectOptions::default())
.await
.expect_err("abort must report a delete below write quorum");
assert!(matches!(err, StorageError::ErasureWriteQuorum));
*set_disks.disks.write().await = saved_disks;
set_disks
.check_upload_id_exists(bucket, object, &below_quorum_upload.upload_id, false)
.await
.expect("failed abort must leave quorum-visible staging on the restored disks");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_revalidates_layout_candidate_after_upload_lock() {
@@ -3398,17 +3138,6 @@ mod tests {
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
.await
@@ -3419,68 +3148,10 @@ mod tests {
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
let list_err = list
.await
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_validates_parts_after_an_inflight_upload_part_commit() {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-complete-put-part-race-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, original_parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x49; 4096], &ObjectOptions::default()).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
let put_store = set_disks.clone();
let put_upload_id = upload_id.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x4a; 4096]);
put_store
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, original_parts, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!complete.is_finished(), "completion must wait for the UploadPart commit lock");
barrier.release();
put.await
.expect("UploadPart task should not panic")
.expect("UploadPart replacement should commit");
let err = complete
.await
.expect("completion task should not panic")
.expect_err("completion must reject the stale ETag after UploadPart wins");
assert!(matches!(err, StorageError::InvalidPart(..)));
set_disks
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
.expect("failed completion must leave the upload retryable");
}
#[tokio::test(start_paused = true)]
#[serial]
async fn complete_fences_upload_lock_loss_before_commit() {
File diff suppressed because it is too large Load Diff
+94 -167
View File
@@ -207,9 +207,11 @@ impl SetDisks {
version_id: &str,
opts: &ReadOptions,
) -> Result<Vec<FileInfo>> {
let disks = self.disks.read().await.clone();
let required_reads = self.default_read_quorum();
// Use existing disk selection logic
let disks = self.disks.read().await;
let required_reads = self.format.erasure.sets.len();
// Clone parameters outside the closure to avoid lifetime issues
let bucket = bucket.to_string();
let object = object.to_string();
let version_id = version_id.to_string();
@@ -218,6 +220,7 @@ impl SetDisks {
let processor = runtime_sources::batch_processors().read_processor();
let tasks: Vec<_> = disks
.iter()
.take(required_reads + 2) // Read a few extra for reliability
.filter_map(|disk| {
disk.as_ref().map(|d| {
let disk = d.clone();
@@ -231,10 +234,10 @@ impl SetDisks {
})
.collect();
processor
.execute_batch_with_quorum(tasks, required_reads)
.await
.map_err(Into::into)
match processor.execute_batch_with_quorum(tasks, required_reads).await {
Ok(results) => Ok(results),
Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type
}
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -1114,6 +1117,10 @@ impl SetDisks {
total_read += part_length;
part_offset = 0;
#[cfg(test)]
if current_part < last_part_index {
get_part_boundary_barrier::checkpoint(bucket, object, current_part).await;
}
}
// debug!("read end");
@@ -1817,6 +1824,79 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
}
#[cfg(test)]
pub(in crate::set_disk) mod get_part_boundary_barrier {
use std::collections::HashMap;
use std::sync::{Arc, Mutex, OnceLock};
use tokio::sync::Notify;
struct Armed {
part_index: usize,
arrived: Arc<Notify>,
release: Arc<Notify>,
}
fn registry() -> &'static Mutex<HashMap<(String, String), Armed>> {
static REGISTRY: OnceLock<Mutex<HashMap<(String, String), Armed>>> = OnceLock::new();
REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
}
pub struct BarrierHandle {
key: (String, String),
arrived: Arc<Notify>,
release: Arc<Notify>,
}
pub fn arm(bucket: &str, object: &str, part_index: usize) -> BarrierHandle {
let key = (bucket.to_string(), object.to_string());
let arrived = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let previous = registry().lock().expect("GET part barrier registry poisoned").insert(
key.clone(),
Armed {
part_index,
arrived: Arc::clone(&arrived),
release: Arc::clone(&release),
},
);
assert!(previous.is_none(), "GET part barrier already armed for object");
BarrierHandle { key, arrived, release }
}
impl BarrierHandle {
pub async fn wait_until_paused(&self) {
self.arrived.notified().await;
}
pub fn release(&self) {
self.release.notify_one();
}
}
impl Drop for BarrierHandle {
fn drop(&mut self) {
self.release.notify_one();
registry()
.lock()
.expect("GET part barrier registry poisoned")
.remove(&self.key);
}
}
pub async fn checkpoint(bucket: &str, object: &str, part_index: usize) {
let state = registry()
.lock()
.expect("GET part barrier registry poisoned")
.get(&(bucket.to_string(), object.to_string()))
.filter(|armed| armed.part_index == part_index)
.map(|armed| (Arc::clone(&armed.arrived), Arc::clone(&armed.release)));
if let Some((arrived, release)) = state {
arrived.notify_one();
release.notified().await;
}
}
}
#[cfg(test)]
mod metadata_cache_tests {
use super::*;
@@ -1957,42 +2037,6 @@ mod metadata_cache_tests {
(dir, disk)
}
async fn read_version_quorum_test_set(
bucket: &str,
object: &str,
pool_set_count: usize,
readable_disks: usize,
) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
let mut dirs = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..4 {
let (dir, disk) = new_read_version_test_disk(bucket).await;
if disk_index < readable_disks {
let mut fi = valid_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
fi.erasure.index = fi.erasure.distribution[disk_index];
disk.write_metadata(bucket, bucket, object, fi)
.await
.expect("metadata should be written before quorum read");
}
dirs.push(dir);
disks.push(Some(disk));
}
let set = SetDisks::new(
"read-version-quorum-test".to_string(),
Arc::new(RwLock::new(disks)),
4,
2,
0,
0,
Vec::new(),
FormatV3::new(pool_set_count, 4),
Vec::new(),
)
.await;
(dirs, set)
}
#[test]
#[serial]
fn get_object_metadata_cache_capacity_uses_default_and_env_override() {
@@ -2016,40 +2060,9 @@ mod metadata_cache_tests {
fi.size = 1;
fi.erasure.index = 1;
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
fi.add_object_part(1, "part-etag".to_string(), 1, fi.mod_time, 1, None, None);
fi
}
#[tokio::test]
async fn get_object_with_fileinfo_rejects_positive_size_without_parts() {
let mut fi = valid_test_fileinfo("object");
fi.parts.clear();
let mut output = Vec::new();
let err = SetDisks::get_object_with_fileinfo(
"bucket",
"object",
0,
1,
&mut output,
fi,
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("positive-size metadata without parts must fail without panicking");
assert_eq!(err, Error::FileCorrupt);
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_rejects_invalid_ranges_before_reader_setup() {
let bucket = "bucket";
@@ -2151,38 +2164,6 @@ mod metadata_cache_tests {
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_accepts_zero_size_without_parts() {
let bucket = "bucket";
let object = "empty";
let mut fi = valid_test_fileinfo(object);
fi.size = 0;
fi.parts.clear();
let mut output = Vec::new();
SetDisks::get_object_with_fileinfo(
bucket,
object,
0,
0,
&mut output,
fi,
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"empty",
)
.await
.expect("zero-byte object without parts must remain readable");
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_fails_closed_without_read_quorum() {
let bucket = "bucket";
@@ -2190,6 +2171,12 @@ mod metadata_cache_tests {
let mut fi = valid_test_fileinfo(object);
fi.erasure.block_size = 1;
fi.erasure.distribution = vec![1, 2, 3, 4];
fi.parts.push(ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
});
let mut output = Vec::new();
let err = SetDisks::get_object_with_fileinfo(
@@ -2223,7 +2210,6 @@ mod metadata_cache_tests {
let object = "object";
let (_dir, disk) = new_read_version_test_disk(bucket).await;
let mut fi = valid_test_fileinfo(object);
fi.size = 0;
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
@@ -2256,70 +2242,11 @@ mod metadata_cache_tests {
.await
.expect_err("empty disk set must fail closed");
assert!(
missing_quorum.to_string().contains("Insufficient successful results"),
"optimized read failure should preserve the batch quorum diagnostic: {missing_quorum}"
missing_quorum.to_string().to_ascii_lowercase().contains("file"),
"optimized read failure should map to file-not-found style error: {missing_quorum}"
);
}
#[tokio::test]
async fn read_version_optimized_uses_current_set_drive_quorum_not_pool_set_count() {
let bucket = "read-version-layout-quorum-bucket";
let object = "object";
let (_single_set_dirs, single_set) = read_version_quorum_test_set(bucket, object, 1, 1).await;
single_set
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect_err("one metadata copy must not satisfy a four-drive 2+2 set");
let (_multi_set_dirs, multi_set) = read_version_quorum_test_set(bucket, object, 3, 2).await;
let versions = multi_set
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect("two metadata copies should satisfy the current set's read quorum");
assert_eq!(versions.len(), 2);
assert!(versions.iter().all(|version| version.name == object));
}
#[tokio::test]
async fn read_version_optimized_counts_only_valid_metadata_toward_quorum() {
let bucket = "read-version-corrupt-quorum-bucket";
let object = "object";
let (quorum_minus_one_dirs, quorum_minus_one) = read_version_quorum_test_set(bucket, object, 1, 2).await;
tokio::fs::write(
quorum_minus_one_dirs[1]
.path()
.join(bucket)
.join(object)
.join(crate::disk::STORAGE_FORMAT_FILE),
b"corrupt metadata",
)
.await
.expect("metadata should be corrupted for the test");
quorum_minus_one
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect_err("one valid and one corrupt metadata copy must not satisfy read quorum");
let (quorum_dirs, quorum) = read_version_quorum_test_set(bucket, object, 1, 3).await;
tokio::fs::write(
quorum_dirs[2]
.path()
.join(bucket)
.join(object)
.join(crate::disk::STORAGE_FORMAT_FILE),
b"corrupt metadata",
)
.await
.expect("metadata should be corrupted for the test");
let versions = quorum
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect("two valid metadata copies must satisfy read quorum despite one corrupt copy");
assert_eq!(versions.len(), 2);
}
#[tokio::test]
async fn get_object_info_and_quorum_maps_delete_marker_and_purge_states() {
let bucket = "get-object-info-marker-bucket";
+1 -13
View File
@@ -6696,7 +6696,7 @@ mod test {
use crate::object_api::ObjectInfo;
use rustfs_filemeta::{
FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker,
ObjectPartInfo, VersionType,
VersionType,
};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
@@ -6899,12 +6899,6 @@ mod test {
fi.volume = "bucket".to_owned();
fi.name = "object".to_owned();
fi.size = 1;
fi.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
fi.fresh = true;
fi.erasure.index = 1;
fi.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"));
@@ -6960,12 +6954,6 @@ mod test {
fi.version_id = Some(Uuid::from_u128(version_idx));
fi.versioned = true;
fi.size = 1;
fi.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
fi.mod_time = Some(*mod_time);
fi.metadata = metadata;
+84 -48
View File
@@ -16,7 +16,7 @@ use super::*;
use crate::disk::OldCurrentSize;
use crate::set_disk::{
get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
is_lock_optimization_enabled, is_object_lock_diag_enabled,
is_object_lock_diag_enabled,
};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_io_metrics::{
@@ -89,6 +89,7 @@ impl PreparedGetObjectReader {
struct LockGuardedReader {
inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
guard: Option<ObjectLockDiagGuard>,
remaining: Option<usize>,
}
impl AsyncRead for LockGuardedReader {
@@ -96,8 +97,23 @@ impl AsyncRead for LockGuardedReader {
let had_capacity = buf.remaining() > 0;
let filled_before = buf.filled().len();
let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
self.guard.take();
match &poll {
Poll::Ready(Ok(())) if buf.filled().len() > filled_before => {
let produced = buf.filled().len() - filled_before;
if let Some(remaining) = self.remaining.as_mut() {
*remaining = remaining.saturating_sub(produced);
if *remaining == 0 {
self.guard.take();
}
}
}
Poll::Ready(Ok(())) if had_capacity => {
self.guard.take();
}
Poll::Ready(Err(_)) => {
self.guard.take();
}
_ => {}
}
poll
}
@@ -641,14 +657,16 @@ impl ECStore {
}
fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
if reader.buffered_body.is_some() || reader.object_info.size == 0 {
return reader;
}
if let Some(guard) = guard {
let remaining = usize::try_from(reader.object_info.size).ok();
reader.stream = Box::new(LockGuardedReader {
inner: reader.stream,
guard: Some(guard),
remaining,
});
}
@@ -2601,10 +2619,10 @@ mod tests {
// (backlog#1304): restore entry no longer serializes on the object lock.
// The replacement semantics — non-blocking reads during the copy-back and
// fast rejection of a concurrent restore — are covered end-to-end by
// `restore_object_usecase_reports_ongoing_conflict`
// (rustfs/src/app/lifecycle_transition_api_test.rs), while the SetDisks
// transition matrix covers the final local commit. Restore-vs-reader data
// protection lives in the inner put_object/complete_multipart_upload locks.
// `restore_object_usecase_reports_ongoing_conflict_and_completion`
// (rustfs/src/app/lifecycle_transition_api_test.rs) and at the lock level
// by the accept-guard test below; restore-vs-reader data protection lives
// in the inner put_object/complete_multipart_upload commit locks.
#[tokio::test]
#[serial_test::serial]
async fn restore_accept_guard_serializes_concurrent_accepts() {
@@ -2649,8 +2667,11 @@ mod tests {
ObjectLockDiagMode::Read,
);
let reader = GetObjectReader {
stream: Box::new(Cursor::new(Vec::<u8>::new())),
object_info: ObjectInfo::default(),
stream: Box::new(Cursor::new(vec![1])),
object_info: ObjectInfo {
size: 1,
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
};
@@ -2670,7 +2691,7 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
async fn reader_lock_is_held_for_stream_when_optimization_is_enabled() {
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
@@ -2690,17 +2711,23 @@ mod tests {
);
let reader = GetObjectReader {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo::default(),
object_info: ObjectInfo {
size: 3,
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
};
let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
.await
.expect_err("streaming readers must retain the read lock when lock optimization is enabled");
drop(reader);
lock.get_write_lock(key, "writer", Duration::from_secs(1))
.await
.expect("lock optimization should release the read lock before returning the stream");
drop(reader);
.expect("cancelling the reader should release the read lock");
})
.await;
}
@@ -2744,41 +2771,50 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn reader_lock_is_released_after_stream_eof() {
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
let key = rustfs_lock::ObjectKey::new("bucket", "object");
let read_guard = lock
.get_read_lock(key.clone(), "reader", Duration::from_secs(1))
.await
.expect("read lock should be acquired");
let read_guard = ObjectLockDiagGuard::new(
read_guard,
true,
"test_get_object",
Some("bucket".to_string()),
Some("object".to_string()),
Some("reader".to_string()),
ObjectLockDiagMode::Read,
);
let reader = GetObjectReader {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo::default(),
buffered_body: None,
body_source: Default::default(),
};
async fn reader_lock_is_released_after_exact_content_length_without_extra_eof_poll() {
for enabled in ["false", "true"] {
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some(enabled))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
let key = rustfs_lock::ObjectKey::new("bucket", "object");
let read_guard = lock
.get_read_lock(key.clone(), "reader", Duration::from_secs(1))
.await
.expect("read lock should be acquired");
let read_guard = ObjectLockDiagGuard::new(
read_guard,
true,
"test_get_object",
Some("bucket".to_string()),
Some("object".to_string()),
Some("reader".to_string()),
ObjectLockDiagMode::Read,
);
let reader = GetObjectReader {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo {
size: 3,
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
};
let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
let mut output = Vec::new();
reader.stream.read_to_end(&mut output).await.expect("reader should reach EOF");
assert_eq!(output, vec![1, 2, 3]);
let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
let mut output = [0_u8; 3];
reader
.stream
.read_exact(&mut output)
.await
.expect("reader should deliver the exact content length");
assert_eq!(output, [1, 2, 3]);
lock.get_write_lock(key, "writer", Duration::from_secs(1))
.await
.expect("EOF should release the read lock before the reader is dropped");
drop(reader);
})
.await;
lock.get_write_lock(key, "writer", Duration::from_secs(1))
.await
.expect("the final response byte should release the read lock without an extra EOF poll");
drop(reader);
})
.await;
}
}
}
+1 -1
View File
@@ -20,5 +20,5 @@ Example:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
cargo +1.96.0 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
```
-23
View File
@@ -415,10 +415,6 @@ impl FileInfo {
}
fn validate_collection_contents(&self, layout: Option<&ValidatedErasureLayout>) -> Result<()> {
if layout.is_some() && self.size > 0 && self.parts.is_empty() {
return Err(Error::FileCorrupt);
}
let mut part_numbers = [0u64; FILEINFO_PART_BITMAP_WORDS];
for part in &self.parts {
if let Some(layout) = layout {
@@ -696,10 +692,6 @@ impl FileInfo {
// to_part_offset gets the part index where offset is located, returns part index and offset
pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> {
if self.size > 0 && self.parts.is_empty() {
return Err(Error::FileCorrupt);
}
if offset == 0 {
return Ok((0, 0));
}
@@ -1193,21 +1185,6 @@ mod tests {
assert_eq!(layout.expect("strict layout must be present").data_blocks, 16);
}
#[test]
fn validate_require_erasure_rejects_positive_size_without_parts() {
let mut fi = FileInfo::new("bucket/object", 4, 2);
fi.erasure.index = 1;
fi.size = 1;
assert_eq!(fi.validate_for_metadata_read(), Err(Error::FileCorrupt));
assert_eq!(fi.to_part_offset(0), Err(Error::FileCorrupt));
fi.size = 0;
fi.validate_for_metadata_read()
.expect("zero-byte object metadata may omit parts");
assert_eq!(fi.to_part_offset(0), Ok((0, 0)));
}
#[test]
fn validate_for_erasure_write_only_relaxes_pending_shard_index() {
let mut fi = validation_test_fileinfo();
+14 -117
View File
@@ -182,9 +182,14 @@ impl FileMeta {
// TODO: use old buf
let meta_buf = ver.marshal_msg()?;
let pre_mod_time = self.versions[idx].header.mod_time;
self.versions[idx].header = ver.header();
self.versions[idx].meta = meta_buf;
self.sort_by_mod_time();
if pre_mod_time != self.versions[idx].header.mod_time {
self.sort_by_mod_time();
}
Ok(())
}
@@ -351,10 +356,15 @@ impl FileMeta {
return self.set_idx(fidx, version);
}
let mod_time = version.get_mod_time();
let new_shallow = FileMetaShallowVersion::try_from(version)?;
let insert_pos = self
.versions
.partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
let insert_pos = match mod_time {
Some(nm) => self.versions.partition_point(|exist| match exist.header.mod_time {
Some(em) => em > nm,
None => false,
}),
None => self.versions.partition_point(|exist| exist.header.mod_time.is_some()),
};
self.versions.insert(insert_pos, new_shallow);
Ok(())
@@ -1102,119 +1112,6 @@ mod test {
assert!(fm.is_sorted_by_mod_time());
}
fn version_for_ordering(
version_type: VersionType,
version_id: Uuid,
mod_time: OffsetDateTime,
marker: u8,
) -> FileMetaVersion {
match version_type {
VersionType::Object => FileMetaVersion {
version_type,
object: Some(MetaObject {
version_id: Some(version_id),
erasure_algorithm: ErasureAlgo::ReedSolomon,
erasure_m: 2,
erasure_n: 2,
erasure_block_size: 1 << 20,
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
mod_time: Some(mod_time),
meta_sys: HashMap::from([("ordering-marker".to_string(), vec![marker])]),
..Default::default()
}),
..Default::default()
},
VersionType::Delete => FileMetaVersion {
version_type,
delete_marker: Some(MetaDeleteMarker {
version_id: Some(version_id),
mod_time: Some(mod_time),
meta_sys: HashMap::from([("ordering-marker".to_string(), vec![marker])]),
}),
..Default::default()
},
_ => unreachable!("ordering regression only constructs object and delete versions"),
}
}
#[test]
fn add_version_filemata_uses_canonical_equal_time_order() {
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
let object = version_for_ordering(VersionType::Object, Uuid::from_u128(1), mod_time, 1);
let delete_marker = version_for_ordering(VersionType::Delete, Uuid::from_u128(2), mod_time, 2);
for versions in [[object.clone(), delete_marker.clone()], [delete_marker, object]] {
let mut fm = FileMeta::new();
for version in versions {
fm.add_version_filemata(version).expect("add equal-time version");
}
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
assert_eq!(fm.versions[1].header.version_type, VersionType::Delete);
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
}
}
#[test]
fn add_version_filemata_preserves_tie_breaks_after_reload() {
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
let versions = [
version_for_ordering(VersionType::Object, Uuid::from_u128(10), mod_time, 10),
version_for_ordering(VersionType::Object, Uuid::from_u128(20), mod_time, 20),
version_for_ordering(VersionType::Delete, Uuid::from_u128(30), mod_time, 30),
];
let mut expected = versions.iter().map(FileMetaVersion::header).collect::<Vec<_>>();
expected.sort_by(|a, b| {
if a.sorts_before(b) {
Ordering::Less
} else if b.sorts_before(a) {
Ordering::Greater
} else {
Ordering::Equal
}
});
let mut fm = FileMeta::new();
for version in versions.into_iter().rev() {
fm.add_version_filemata(version).expect("add equal-time version");
}
let loaded = FileMeta::load(&fm.marshal_msg().expect("serialize file metadata")).expect("reload file metadata");
let actual = loaded
.versions
.iter()
.map(|version| version.header.clone())
.collect::<Vec<_>>();
assert_ne!(expected[0].signature, expected[1].signature, "fixture must exercise signature ordering");
assert_eq!(actual, expected);
assert!(actual.windows(2).all(|pair| pair[0].sorts_before(&pair[1])));
}
#[test]
fn add_version_filemata_reorders_equal_time_replacement() {
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
let first = version_for_ordering(VersionType::Object, Uuid::from_u128(40), mod_time, 40);
let second = version_for_ordering(VersionType::Object, Uuid::from_u128(50), mod_time, 50);
let (target, peer) = if first.header().sorts_before(&second.header()) {
(first, second)
} else {
(second, first)
};
let target_id = target.header().version_id.expect("test version id");
let mut fm = FileMeta::new();
fm.add_version_filemata(peer).expect("add peer object");
fm.add_version_filemata(target).expect("add target object");
assert_eq!(fm.versions[0].header.version_id, Some(target_id));
fm.add_version_filemata(version_for_ordering(VersionType::Delete, target_id, mod_time, 60))
.expect("replace target with equal-time delete marker");
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
assert_eq!(fm.versions[1].header.version_type, VersionType::Delete);
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
}
/// Regression for backlog#799 B16: replication reset state must be
/// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
/// key (produced by `ObjectInfo::replication_state`) has no internal prefix,
-20
View File
@@ -144,7 +144,6 @@ fn is_recoverable_heal_error_message(error: &str) -> bool {
"connection refused",
"operation canceled",
"quorum not reached",
"heal rename incomplete",
]
.iter()
.any(|pattern| error.contains(pattern))
@@ -155,22 +154,3 @@ impl From<Error> for std::io::Error {
std::io::Error::other(err)
}
}
#[cfg(test)]
mod tests {
use super::Error;
use crate::heal::EcstoreError;
#[test]
fn incomplete_target_rename_is_recoverable() {
let task_error = Error::TaskExecutionFailed {
message: "heal rename incomplete: 1 of 2 targets committed".to_string(),
};
let storage_error = Error::Storage(EcstoreError::Io(std::io::Error::other(
"heal rename incomplete: 1 of 2 targets committed",
)));
assert!(task_error.is_recoverable_heal());
assert!(storage_error.is_recoverable_heal());
}
}
-8
View File
@@ -161,14 +161,6 @@ pub struct ConfigureStaticKmsRequest {
pub allow_insecure_dev_defaults: Option<bool>,
}
impl Drop for ConfigureStaticKmsRequest {
fn drop(&mut self) {
use zeroize::Zeroize;
self.secret_key.zeroize();
}
}
impl fmt::Debug for ConfigureStaticKmsRequest {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ConfigureStaticKmsRequest")
+24 -156
View File
@@ -23,17 +23,16 @@
use crate::backends::{BackendInfo, KmsBackend, KmsClient};
use crate::config::{BackendConfig, KmsConfig};
use crate::encryption::DataKeyEnvelope;
use crate::error::{KmsError, Result};
use crate::types::*;
use aes_gcm::{
Aes256Gcm, Key, Nonce,
aead::{Aead, KeyInit, Payload},
aead::{Aead, KeyInit},
};
use async_trait::async_trait;
use jiff::Zoned;
use rand::RngExt;
use std::collections::{BTreeMap, HashMap};
use std::collections::HashMap;
use tracing::debug;
use zeroize::Zeroizing;
@@ -42,11 +41,6 @@ const NONCE_SIZE: usize = 12;
/// AES-256 key size in bytes.
const KEY_SIZE: usize = 32;
fn context_aad(context: &HashMap<String, String>) -> Result<Vec<u8>> {
let canonical: BTreeMap<&str, &str> = context.iter().map(|(key, value)| (key.as_str(), value.as_str())).collect();
serde_json::to_vec(&canonical).map_err(Into::into)
}
/// Static single-key KMS backend.
///
/// Uses a pre-configured AES-256 key to derive data encryption keys. This is a
@@ -117,35 +111,21 @@ impl KmsClient for StaticKmsBackend {
let key = Key::<Aes256Gcm>::from(*self.key);
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::from(nonce_bytes);
let aad = context_aad(&request.encryption_context)?;
let encrypted = cipher
.encrypt(
&nonce,
Payload {
msg: plaintext.as_ref(),
aad: &aad,
},
)
.encrypt(&nonce, plaintext.as_ref())
.map_err(|e| KmsError::cryptographic_error("AES-256-GCM encrypt", e.to_string()))?;
let envelope = DataKeyEnvelope {
key_id: uuid::Uuid::new_v4().to_string(),
master_key_id: request.master_key_id.clone(),
key_spec: request.key_spec.clone(),
encrypted_key: encrypted,
nonce: nonce_bytes.to_vec(),
encryption_context: request.encryption_context.clone(),
created_at: Zoned::now(),
};
let ciphertext = serde_json::to_vec(&envelope)?;
// Ciphertext format: encrypted_dek || nonce
let mut ciphertext = encrypted;
ciphertext.extend_from_slice(&nonce_bytes);
Ok(DataKeyInfo::new(
self.key_id.clone(),
0,
0, // version is always 0 for static KMS
Some(plaintext.to_vec()),
ciphertext,
request.key_spec.clone(),
"AES_256".to_string(),
))
}
@@ -161,28 +141,14 @@ impl KmsClient for StaticKmsBackend {
let key = Key::<Aes256Gcm>::from(*self.key);
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::from(nonce_bytes);
let aad = context_aad(&request.encryption_context)?;
let encrypted = cipher
.encrypt(
&nonce,
Payload {
msg: request.plaintext.as_ref(),
aad: &aad,
},
)
.encrypt(&nonce, request.plaintext.as_ref())
.map_err(|e| KmsError::cryptographic_error("AES-256-GCM encrypt", e.to_string()))?;
let envelope = DataKeyEnvelope {
key_id: uuid::Uuid::new_v4().to_string(),
master_key_id: request.key_id.clone(),
key_spec: "AES_256".to_string(),
encrypted_key: encrypted,
nonce: nonce_bytes.to_vec(),
encryption_context: request.encryption_context.clone(),
created_at: Zoned::now(),
};
let ciphertext = serde_json::to_vec(&envelope)?;
// Ciphertext format: encrypted_data || nonce
let mut ciphertext = encrypted;
ciphertext.extend_from_slice(&nonce_bytes);
Ok(EncryptResponse {
ciphertext,
@@ -193,39 +159,21 @@ impl KmsClient for StaticKmsBackend {
}
async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> {
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
.map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?;
if envelope.master_key_id != self.key_id {
return Err(KmsError::key_not_found(&envelope.master_key_id));
if request.ciphertext.len() < NONCE_SIZE + 1 {
return Err(KmsError::cryptographic_error("decrypt", "Ciphertext too short for static KMS format"));
}
for (key, expected_value) in &envelope.encryption_context {
match request.encryption_context.get(key) {
Some(actual_value) if actual_value == expected_value => {}
Some(actual_value) => {
return Err(KmsError::context_mismatch(format!(
"Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
)));
}
None if request.encryption_context.is_empty() => {}
None => return Err(KmsError::context_mismatch(format!("Missing context key '{key}'"))),
}
}
// Split ciphertext: encrypted_data || nonce(12)
let split_at = request.ciphertext.len() - NONCE_SIZE;
let encrypted = &request.ciphertext[..split_at];
let nonce_slice = &request.ciphertext[split_at..];
let key = Key::<Aes256Gcm>::from(*self.key);
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::try_from(envelope.nonce.as_slice())
.map_err(|_| KmsError::cryptographic_error("nonce", "invalid nonce length"))?;
let aad = context_aad(&envelope.encryption_context)?;
let nonce = Nonce::try_from(nonce_slice).map_err(|_| KmsError::cryptographic_error("nonce", "invalid nonce length"))?;
let plaintext = cipher
.decrypt(
&nonce,
Payload {
msg: envelope.encrypted_key.as_ref(),
aad: &aad,
},
)
.decrypt(&nonce, encrypted)
.map_err(|e| KmsError::cryptographic_error("AES-256-GCM decrypt", e.to_string()))?;
Ok(plaintext)
@@ -369,13 +317,7 @@ impl KmsBackend for StaticKmsBackend {
}
async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
let gen_req = GenerateKeyRequest {
master_key_id: request.key_id.clone(),
key_spec: request.key_spec.as_str().to_string(),
key_length: None,
encryption_context: request.encryption_context,
grant_tokens: Vec::new(),
};
let gen_req = GenerateKeyRequest::new(request.key_id.clone(), request.key_spec.as_str().to_string());
let data_key = <Self as KmsClient>::generate_data_key(self, &gen_req, None).await?;
let plaintext_key = data_key
@@ -436,9 +378,8 @@ impl KmsBackend for StaticKmsBackend {
#[cfg(test)]
mod tests {
use super::*;
use crate::backends::{KmsBackend as KmsBackendTrait, KmsClient};
use crate::backends::KmsClient;
use crate::config::{BackendConfig, KmsBackend, StaticConfig};
use crate::encryption::is_data_key_envelope;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
@@ -489,12 +430,8 @@ mod tests {
assert_eq!(data_key.version, 0);
assert!(data_key.plaintext.is_some());
assert_eq!(data_key.plaintext.as_ref().expect("plaintext should be set").len(), 32);
let envelope: DataKeyEnvelope =
serde_json::from_slice(&data_key.ciphertext).expect("static data key should use a KMS envelope");
assert_eq!(envelope.master_key_id, key_id);
assert_eq!(envelope.encrypted_key.len(), 32 + 16);
assert_eq!(envelope.nonce.len(), NONCE_SIZE);
assert_eq!(envelope.encryption_context.get("bucket").map(String::as_str), Some("test-bucket"));
// Ciphertext should be: encrypted(32) + tag(16) + nonce(12)
assert_eq!(data_key.ciphertext.len(), 32 + 16 + NONCE_SIZE);
// Decrypt the data key
let decrypt_request =
@@ -506,75 +443,6 @@ mod tests {
assert_eq!(decrypted.as_slice(), data_key.plaintext.as_deref().expect("plaintext should exist"));
}
#[tokio::test]
async fn generated_data_key_uses_kms_envelope_for_sse_read_routing() {
let (backend, key_id, _key) = create_test_backend().await;
let request = GenerateKeyRequest::new(key_id, "AES_256".to_string())
.with_context("bucket".to_string(), "source-bucket".to_string())
.with_context("object".to_string(), "source-object".to_string());
let data_key = KmsClient::generate_data_key(&backend, &request, None)
.await
.expect("generate static KMS data key");
assert!(
is_data_key_envelope(&data_key.ciphertext),
"static KMS ciphertext must use the KMS envelope recognized by the SSE read path"
);
}
#[tokio::test]
async fn generated_data_key_rejects_a_different_encryption_context() {
let (backend, key_id, _key) = create_test_backend().await;
let generate_request = GenerateDataKeyRequest {
key_id,
key_spec: KeySpec::Aes256,
encryption_context: HashMap::from([
("bucket".to_string(), "source-bucket".to_string()),
("object".to_string(), "source-object".to_string()),
]),
};
let generated = KmsBackendTrait::generate_data_key(&backend, generate_request)
.await
.expect("generate context-bound static KMS data key");
let decrypt_request = DecryptRequest {
ciphertext: generated.ciphertext_blob,
encryption_context: HashMap::from([
("bucket".to_string(), "different-bucket".to_string()),
("object".to_string(), "different-object".to_string()),
]),
grant_tokens: Vec::new(),
};
let error = KmsBackendTrait::decrypt(&backend, decrypt_request)
.await
.expect_err("a static KMS data key must not decrypt under a different object context");
assert!(matches!(error, KmsError::ContextMismatch { .. }));
}
#[tokio::test]
async fn generated_data_key_rejects_tampered_envelope_context() {
let (backend, key_id, _key) = create_test_backend().await;
let request = GenerateKeyRequest::new(key_id, "AES_256".to_string())
.with_context("bucket".to_string(), "source-bucket".to_string());
let generated = KmsClient::generate_data_key(&backend, &request, None)
.await
.expect("generate context-bound data key");
let mut envelope: DataKeyEnvelope = serde_json::from_slice(&generated.ciphertext).expect("parse static KMS envelope");
envelope
.encryption_context
.insert("bucket".to_string(), "different-bucket".to_string());
let decrypt_request = DecryptRequest::new(serde_json::to_vec(&envelope).expect("serialize tampered envelope"))
.with_context("bucket".to_string(), "different-bucket".to_string());
let error = KmsClient::decrypt(&backend, &decrypt_request, None)
.await
.expect_err("tampering with authenticated envelope context must fail");
assert!(matches!(error, KmsError::CryptographicError { .. }));
}
#[tokio::test]
async fn test_generate_data_key_wrong_key_id() {
let (backend, _key_id, _key) = create_test_backend().await;
-24
View File
@@ -214,18 +214,9 @@ pub struct StaticConfig {
/// Key identifier (name) for the single configured key
pub key_id: String,
/// Base64-encoded 32-byte AES-256 key material (zeroed on drop)
#[serde(skip_serializing, default)]
pub secret_key: String,
}
impl Drop for StaticConfig {
fn drop(&mut self) {
use zeroize::Zeroize;
self.secret_key.zeroize();
}
}
impl fmt::Debug for StaticConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("StaticConfig")
@@ -1063,21 +1054,6 @@ mod tests {
assert!(serialized.contains("persisted-token-secret"));
}
#[test]
fn static_kms_config_serialization_does_not_expose_key_material() {
use base64::Engine as _;
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone());
let serialized = serde_json::to_string(&config).expect("static KMS config should serialize");
assert!(
!serialized.contains(&encoded_key),
"persisted static KMS configuration must not contain plaintext key material"
);
}
#[test]
fn test_config_validation() {
let mut config = KmsConfig {
+10 -16
View File
@@ -31,25 +31,19 @@ use tokio::sync::RwLock;
pub struct KmsManager {
backend: Arc<dyn KmsBackend>,
cache: Arc<RwLock<KmsCache>>,
default_key_id: Option<String>,
enable_cache: bool,
config: KmsConfig,
}
impl KmsManager {
/// Create a new KMS manager with the given backend and config
pub fn new(backend: Arc<dyn KmsBackend>, config: KmsConfig) -> Self {
let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64)));
Self {
backend,
cache,
default_key_id: config.default_key_id,
enable_cache: config.enable_cache,
}
Self { backend, cache, config }
}
/// Get the default key ID if configured
pub fn get_default_key_id(&self) -> Option<&String> {
self.default_key_id.as_ref()
self.config.default_key_id.as_ref()
}
/// Create a new master key
@@ -57,7 +51,7 @@ impl KmsManager {
let response = self.backend.create_key(request).await?;
// Cache the key metadata if enabled
if self.enable_cache {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.put_key_metadata(&response.key_id, &response.key_metadata).await;
}
@@ -83,7 +77,7 @@ impl KmsManager {
/// Describe a key
pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
// Check cache first if enabled
if self.enable_cache {
if self.config.enable_cache {
let cache = self.cache.read().await;
if let Some(cached_metadata) = cache.get_key_metadata(&request.key_id).await {
return Ok(DescribeKeyResponse {
@@ -95,7 +89,7 @@ impl KmsManager {
// Get from backend and cache
let response = self.backend.describe_key(request).await?;
if self.enable_cache {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache
.put_key_metadata(&response.key_metadata.key_id, &response.key_metadata)
@@ -112,7 +106,7 @@ impl KmsManager {
/// Get cache statistics
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
if self.enable_cache {
if self.config.enable_cache {
let cache = self.cache.read().await;
Some(cache.stats())
} else {
@@ -122,7 +116,7 @@ impl KmsManager {
/// Clear the cache
pub async fn clear_cache(&self) -> Result<()> {
if self.enable_cache {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.clear().await;
}
@@ -134,7 +128,7 @@ impl KmsManager {
let response = self.backend.delete_key(request).await?;
// Remove from cache if enabled and key is being deleted
if self.enable_cache {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.remove_key_metadata(&response.key_id).await;
}
@@ -147,7 +141,7 @@ impl KmsManager {
let response = self.backend.cancel_key_deletion(request).await?;
// Update cache if enabled
if self.enable_cache {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.put_key_metadata(&response.key_id, &response.key_metadata).await;
}
+10 -2
View File
@@ -350,7 +350,11 @@ impl ObjectEncryptionService {
encryption_context: context.clone(),
};
let data_key = self.kms_manager.generate_data_key(request).await?;
let data_key = self
.kms_manager
.generate_data_key(request)
.await
.map_err(|e| KmsError::backend_error(format!("Failed to generate data key: {e}")))?;
let plaintext_key = data_key.plaintext_key;
@@ -427,7 +431,11 @@ impl ObjectEncryptionService {
grant_tokens: Vec::new(),
};
let decrypt_response = self.kms_manager.decrypt(decrypt_request).await?;
let decrypt_response = self
.kms_manager
.decrypt(decrypt_request)
.await
.map_err(|e| KmsError::backend_error(format!("Failed to decrypt data key: {e}")))?;
// Create cipher
let cipher = create_cipher(&algorithm, &decrypt_response.plaintext)?;
-28
View File
@@ -94,16 +94,6 @@ impl KmsServiceManager {
self.config.read().await.clone()
}
/// Get configuration for status and management responses without static key material.
pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
let mut config = self.config.read().await.clone()?;
if let BackendConfig::Static(static_config) = &mut config.backend_config {
use zeroize::Zeroize;
static_config.secret_key.zeroize();
}
Some(config)
}
/// Configure KMS with new configuration
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
new_config.validate()?;
@@ -459,22 +449,4 @@ mod tests {
assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
assert!(manager.get_config().await.is_none());
}
#[tokio::test]
async fn redacted_config_omits_static_key_material() {
use base64::Engine as _;
let manager = KmsServiceManager::new();
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
manager
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
.await
.expect("configure static KMS");
let config = manager.get_redacted_config().await.expect("redacted config");
let BackendConfig::Static(static_config) = config.backend_config else {
panic!("expected static config");
};
assert!(static_config.secret_key.is_empty());
}
}
-2
View File
@@ -27,7 +27,6 @@ documentation = "https://docs.rs/rustfs-lifecycle/latest/rustfs_lifecycle/"
[dependencies]
async-trait.workspace = true
metrics.workspace = true
rustfs-common.workspace = true
rustfs-config = { workspace = true, features = ["constants"] }
rustfs-replication.workspace = true
@@ -39,7 +38,6 @@ url.workspace = true
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
[dev-dependencies]
metrics-util = { version = "0.20", features = ["debugging"] }
proptest = "1"
serial_test.workspace = true
temp-env.workspace = true
+20 -115
View File
@@ -34,7 +34,6 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_EXPIRY_COMPUTED: &str = "lifecycle_expiry_computed";
const EVENT_LIFECYCLE_DEBUG_DAY_SECS: &str = "lifecycle_debug_day_secs";
const EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED: &str = "lifecycle_noncurrent_expiry_skipped";
const METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL: &str = "rustfs_lifecycle_invalid_mod_time_total";
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
const _ERR_XML_NOT_WELL_FORMED: &str =
@@ -205,21 +204,6 @@ fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> {
filter.and.as_ref().and_then(|and| and.prefix.as_deref())
}
fn lifecycle_mod_time(obj: &ObjectOpts) -> Option<OffsetDateTime> {
let reason = match obj.mod_time {
None => "missing",
Some(mod_time) if mod_time < OffsetDateTime::UNIX_EPOCH => "before_unix_epoch",
Some(mod_time) if mod_time == OffsetDateTime::UNIX_EPOCH => "unix_epoch",
Some(mod_time) => return Some(mod_time),
};
metrics::counter!(
METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL,
"reason" => reason
)
.increment(1);
None
}
#[async_trait::async_trait]
pub trait Lifecycle {
async fn has_transition(&self) -> bool;
@@ -450,8 +434,14 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
async fn predict_expiration(&self, obj: &ObjectOpts) -> Event {
let Some(mod_time) = lifecycle_mod_time(obj) else {
return Event::default();
let mod_time = match obj.mod_time {
Some(time) => {
if time.unix_timestamp() == 0 {
return Event::default();
}
time
}
None => return Event::default(),
};
if obj.delete_marker || !(obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) {
@@ -505,10 +495,20 @@ impl Lifecycle for BucketLifecycleConfiguration {
obj.name, obj.mod_time, obj.successor_mod_time, now, obj.is_latest, obj.delete_marker
);
let Some(mod_time) = lifecycle_mod_time(obj) else {
return Event::default();
// Gracefully handle missing mod_time instead of panicking
let mod_time = match obj.mod_time {
Some(t) => t,
None => {
debug!("eval_inner: mod_time is None for object={}, returning default event", obj.name);
return Event::default();
}
};
if mod_time.unix_timestamp() == 0 {
debug!("eval_inner: mod_time is 0, returning default event");
return Event::default();
}
if let Some(restore_expires) = obj.restore_expires
&& restore_expires.unix_timestamp() != 0
&& now.unix_timestamp() > restore_expires.unix_timestamp()
@@ -1082,8 +1082,6 @@ impl Default for TransitionOptions {
#[cfg(test)]
mod tests {
use super::*;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass};
use serial_test::serial;
use std::sync::Arc;
@@ -1099,99 +1097,6 @@ mod tests {
});
}
#[test]
fn eval_inner_reports_invalid_mod_time_without_expiring_object() {
let lifecycle = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-after-one-day".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
};
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("current-thread runtime should build");
let actions = metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let now = OffsetDateTime::from_unix_timestamp(10 * 86_400).expect("fixed timestamp should be valid");
let mut opts = ObjectOpts {
name: "legacy-object".to_string(),
is_latest: true,
..Default::default()
};
let mut actions = Vec::new();
for mod_time in [
None,
Some(OffsetDateTime::UNIX_EPOCH),
Some(OffsetDateTime::UNIX_EPOCH - Duration::nanoseconds(1)),
Some(OffsetDateTime::UNIX_EPOCH + Duration::nanoseconds(1)),
] {
opts.mod_time = mod_time;
actions.push((
lifecycle.eval_inner(&opts, now, 0).await.action,
lifecycle.predict_expiration(&opts).await.action,
));
}
actions
})
});
assert_eq!(
actions,
[
(IlmAction::NoneAction, IlmAction::NoneAction),
(IlmAction::NoneAction, IlmAction::NoneAction),
(IlmAction::NoneAction, IlmAction::NoneAction),
(IlmAction::DeleteAction, IlmAction::DeleteAction)
]
);
let invalid_mod_time_metrics = snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter_map(|(composite, _unit, _description, value)| {
(composite.kind() == MetricKind::Counter && composite.key().name() == METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL)
.then(|| {
let reason = composite
.key()
.labels()
.find(|label| label.key() == "reason")
.map(|label| label.value().to_string())
.expect("invalid mod_time metric should have a reason");
(reason, value)
})
})
.collect::<Vec<_>>();
assert_eq!(invalid_mod_time_metrics.len(), 3);
assert!(
invalid_mod_time_metrics
.iter()
.all(|(_, value)| matches!(value, DebugValue::Counter(2)))
);
assert!(invalid_mod_time_metrics.iter().any(|(reason, _)| reason == "missing"));
assert!(invalid_mod_time_metrics.iter().any(|(reason, _)| reason == "unix_epoch"));
assert!(
invalid_mod_time_metrics
.iter()
.any(|(reason, _)| reason == "before_unix_epoch")
);
}
#[tokio::test]
#[serial]
async fn validate_rejects_zero_expiration_days() {
+4 -4
View File
@@ -45,15 +45,15 @@ pub(super) fn rules() -> Vec<Rule> {
)
},
Rule {
anchors: strings(["heal rename incomplete"]),
anchors: strings(["all healed data rename attempts failed"]),
..base(
"heal-rename-failed",
P1Unavailable,
"heal",
"heal 数据落位失败",
contains("heal rename incomplete"),
"heal 数据落位(rename)未在全部目标盘完成",
"检查失败目标盘的写权限与空间,修复后重跑 heal",
contains("all healed data rename attempts failed"),
"heal 数据落位(rename)全部失败",
"检查盘写权限与空间",
)
},
Rule {
+1 -1
View File
@@ -177,7 +177,7 @@ fn every_rule_has_a_positive_sample() {
msg("heal: latest metadata for b/o has no data_dir, cannot heal object data"),
),
("heal-all-writes-failed", msg("all drives had write errors, unable to heal b/o")),
("heal-rename-failed", msg("heal rename incomplete: 1 of 2 targets committed for b/o")),
("heal-rename-failed", msg("all healed data rename attempts failed for b/o")),
(
"heal-xlmeta-regen-failed",
msg("heal_object: failed to regenerate recoverable xl.meta on disk"),
-15
View File
@@ -25,9 +25,6 @@ keywords = ["file-system", "notification", "real-time", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "filesystem"]
documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
[features]
demo-examples = []
[dependencies]
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
rustfs-ecstore = { workspace = true }
@@ -74,18 +71,6 @@ workspace = true
[lib]
doctest = false
[[example]]
name = "full_demo"
required-features = ["demo-examples"]
[[example]]
name = "full_demo_one"
required-features = ["demo-examples"]
[[example]]
name = "webhook"
required-features = ["demo-examples"]
[[bench]]
name = "snapshot_mode_scan"
harness = false
-5
View File
@@ -49,8 +49,6 @@ swift = [
"dep:hmac",
"dep:sha1",
"dep:hex",
"dep:ipnetwork",
"dep:rustfs-trusted-proxies",
"dep:astral-tokio-tar",
"dep:base64",
"dep:async-compression",
@@ -113,8 +111,6 @@ quick-xml = { workspace = true, optional = true, features = ["serialize"] }
hmac = { workspace = true, optional = true }
sha1 = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
ipnetwork = { workspace = true, optional = true }
rustfs-trusted-proxies = { workspace = true, optional = true }
astral-tokio-tar = { workspace = true, optional = true }
base64 = { workspace = true, optional = true }
async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] }
@@ -134,7 +130,6 @@ socket2 = { workspace = true, optional = true, features = ["all"] }
[dev-dependencies]
tempfile = { workspace = true }
proptest = "1"
rustfs-test-utils = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
tokio = { workspace = true, features = ["test-util", "macros", "fs"] }
+51 -41
View File
@@ -16,11 +16,12 @@
use super::storage_api::account::{BucketOperations, MakeBucketOptions};
use super::{SwiftError, SwiftResult};
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging, validate_metadata};
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
use rustfs_credentials::Credentials;
use s3s::dto::{Tag, Tagging};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use time;
/// Validate that the authenticated user has access to the requested account
///
@@ -147,33 +148,15 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
/// Updates account-level metadata such as TempURL keys.
/// Only updates swift-account-meta-* tags, preserving other tags.
///
/// The caller must own the account: this metadata holds the account's TempURL
/// signing key, so writing it for someone else's account would let the writer
/// mint valid pre-signed URLs against that account's objects. Reads
/// ([`get_account_metadata`]) stay unauthenticated on purpose — TempURL
/// signature validation has to resolve the key before any credentials exist.
///
/// # Arguments
/// * `account` - Account identifier
/// * `metadata` - Metadata key-value pairs to store (keys will be prefixed with `swift-account-meta-`)
/// * `credentials` - Keystone credentials of the caller
/// * `credentials` - S3 credentials
pub async fn update_account_metadata(
account: &str,
metadata: &HashMap<String, String>,
credentials: &Option<Credentials>,
_credentials: &Option<Credentials>,
) -> SwiftResult<()> {
let Some(credentials) = credentials.as_ref() else {
return Err(SwiftError::Unauthorized(
"Keystone authentication required to update account metadata".to_string(),
));
};
validate_account_access(account, credentials)?;
// These tags are persisted into the bucket metadata file, which every
// later config write rewrites in full — so unbounded metadata inflates
// the cost of unrelated writes for the life of the account.
validate_metadata(metadata)?;
let bucket_name = get_account_metadata_bucket_name(account);
let Some(store) = resolve_swift_object_store_handle() else {
@@ -190,30 +173,57 @@ pub async fn update_account_metadata(
.map_err(|e| SwiftError::InternalServerError(format!("Failed to create account metadata bucket: {}", e)))?;
}
// Rewrite the persisted tags: replace swift-account-meta-* tags with the
// new metadata while preserving other tags. An empty result clears the
// tagging config.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
// Load current bucket metadata
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-account-meta-")
} else {
true
}
});
let mut bucket_meta_clone = (*bucket_meta).clone();
for (key, value) in metadata {
tagging.tag_set.push(Tag {
key: Some(format!("swift-account-meta-{}", key)),
value: Some(value.clone()),
});
// Get existing tags, preserving non-Swift tags
let mut existing_tagging = bucket_meta_clone
.tagging_config
.clone()
.unwrap_or_else(|| Tagging { tag_set: vec![] });
// Remove old swift-account-meta-* tags while preserving other tags
existing_tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-account-meta-")
} else {
true
}
});
tagging
})
.await?;
// Add new metadata tags
for (key, value) in metadata {
existing_tagging.tag_set.push(Tag {
key: Some(format!("swift-account-meta-{}", key)),
value: Some(value.clone()),
});
}
let now = time::OffsetDateTime::now_utc();
if existing_tagging.tag_set.is_empty() {
// No tags remain; clear tagging config
bucket_meta_clone.tagging_config_xml = Vec::new();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = None;
} else {
// Serialize tags to XML
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = Some(existing_tagging);
}
// Save updated metadata
set_swift_bucket_metadata(bucket_name.clone(), bucket_meta_clone)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
Ok(())
}
+226 -184
View File
@@ -22,22 +22,12 @@ use super::storage_api::container::{
};
use super::types::Container;
use super::{SwiftError, SwiftResult};
use super::{
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
validate_metadata,
};
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
use rustfs_credentials::Credentials;
use s3s::dto::{Tag, Tagging};
use sha2::{Digest, Sha256};
use tracing::{debug, error};
fn required_bucket_usage(usage: &std::collections::HashMap<String, (u64, u64)>, bucket: &str) -> SwiftResult<(u64, u64)> {
usage
.get(bucket)
.copied()
.ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))
}
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
const LOG_SUBSYSTEM_SWIFT_CONTAINER: &str = "swift_container";
const EVENT_SWIFT_CONTAINER_STORAGE_STATE: &str = "swift_container_storage_state";
@@ -262,24 +252,13 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR
.list_bucket(&BucketOptions::default())
.await
.map_err(|e| sanitize_storage_error("Container listing", e))?;
let usage = get_swift_bucket_usage()
.await
.map_err(|e| sanitize_storage_error("Container usage retrieval", e))?
.ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))?;
// Filter and convert buckets to containers
let containers: Vec<Container> = bucket_infos
.iter()
.filter(|info| mapper.bucket_belongs_to_project(&info.name, &project_id))
.filter_map(|info| {
bucket_info_to_container(info, &mapper, &project_id).map(|mut container| {
let (count, bytes) = required_bucket_usage(&usage, &info.name)?;
container.count = count;
container.bytes = bytes;
Ok(container)
})
})
.collect::<SwiftResult<Vec<_>>>()?;
.filter_map(|info| bucket_info_to_container(info, &mapper, &project_id))
.collect();
debug!(
event = EVENT_SWIFT_CONTAINER_STORAGE_STATE,
@@ -389,44 +368,6 @@ pub struct ContainerMetadata {
pub custom_metadata: std::collections::HashMap<String, String>,
}
async fn get_container_metadata_base(
account: &str,
container: &str,
credentials: &Credentials,
) -> SwiftResult<(String, BucketInfo, std::collections::HashMap<String, String>)> {
let project_id = validate_account_access(account, credentials)?;
validate_container_name(container)?;
let bucket_name = ContainerMapper::default().swift_to_s3_bucket(container, &project_id);
let Some(store) = resolve_swift_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
let bucket_info = store
.get_bucket_info(&bucket_name, &BucketOptions::default())
.await
.map_err(|e| {
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
SwiftError::NotFound(format!("Container '{}' not found", container))
} else {
sanitize_storage_error("Container metadata retrieval", e)
}
})?;
let custom_metadata = get_swift_bucket_metadata(&bucket_name)
.await
.ok()
.and_then(|bucket_meta| bucket_meta.tagging_config.as_ref().map(s3_tags_to_swift_metadata))
.unwrap_or_default();
Ok((bucket_name, bucket_info, custom_metadata))
}
pub(crate) async fn get_container_custom_metadata(
account: &str,
container: &str,
credentials: &Credentials,
) -> SwiftResult<std::collections::HashMap<String, String>> {
let (_, _, custom_metadata) = get_container_metadata_base(account, container, credentials).await?;
Ok(custom_metadata)
}
/// Get container metadata (for HEAD operation)
///
/// This function:
@@ -441,22 +382,60 @@ pub(crate) async fn get_container_custom_metadata(
/// - Returns 404 Not Found if container doesn't exist
#[allow(dead_code)] // Used by handler
pub async fn get_container_metadata(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<ContainerMetadata> {
let (bucket_name, bucket_info, custom_metadata) = get_container_metadata_base(account, container, credentials).await?;
// Validate account access and extract project_id
let project_id = validate_account_access(account, credentials)?;
// Ecstore serves the persisted scanner snapshot through a bounded in-process cache.
// Single-node deployments overlay successful in-process mutations; distributed
// deployments use only the cluster-wide persisted snapshot.
let usage = get_swift_bucket_usage()
// Validate container name
validate_container_name(container)?;
// Create mapper with default config (tenant prefixing enabled)
let mapper = ContainerMapper::default();
// Convert Swift container name to S3 bucket name
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
// Get storage layer
let Some(store) = resolve_swift_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
// Get bucket info
let bucket_info = store
.get_bucket_info(&bucket_name, &BucketOptions::default())
.await
.map_err(|e| sanitize_storage_error("Container usage retrieval", e))?
.ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))?;
let (object_count, bytes_used) = required_bucket_usage(&usage, &bucket_name)?;
.map_err(|e| {
// Check if bucket not found
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
SwiftError::NotFound(format!("Container '{}' not found", container))
} else {
sanitize_storage_error("Container metadata retrieval", e)
}
})?;
// Load bucket metadata to get custom metadata from tags
let custom_metadata = match get_swift_bucket_metadata(&bucket_name).await {
Ok(bucket_meta) => {
if let Some(tagging) = &bucket_meta.tagging_config {
s3_tags_to_swift_metadata(tagging)
} else {
std::collections::HashMap::new()
}
}
Err(_) => {
// If metadata not available, return empty (container may be newly created)
std::collections::HashMap::new()
}
};
// Currently returns basic metadata with limitations:
// 1. Object count requires iterating all objects (expensive)
// 2. Bytes used requires summing all object sizes (expensive)
// 3. Custom metadata is now loaded from bucket tags ✅
Ok(ContainerMetadata {
object_count,
bytes_used,
object_count: 0, // TODO: implement object counting in backend
bytes_used: 0, // TODO: implement size aggregation in backend
created: bucket_info.created,
custom_metadata,
custom_metadata, // ✅ Now populated from bucket tags!
})
}
@@ -486,11 +465,6 @@ pub async fn update_container_metadata(
// Validate container name
validate_container_name(container)?;
// These tags are persisted into the bucket metadata file, which every
// later config write rewrites in full — so unbounded metadata inflates
// the cost of unrelated writes for the life of the container.
validate_metadata(&metadata)?;
// Create mapper with default config (tenant prefixing enabled)
let mapper = ContainerMapper::default();
@@ -514,30 +488,57 @@ pub async fn update_container_metadata(
}
})?;
// Rewrite the persisted tags: replace swift-meta-* tags with the new
// metadata while preserving non-Swift tags. An empty result clears the
// tagging config.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
// Load current bucket metadata
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-meta-")
} else {
true // Keep tags with no key (shouldn't happen, but be safe)
}
});
let mut bucket_meta_clone = (*bucket_meta).clone();
if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
tagging.tag_set.append(&mut new_tagging.tag_set);
// Get existing tags, preserving non-Swift tags
let mut existing_tagging = bucket_meta_clone
.tagging_config
.clone()
.unwrap_or_else(|| Tagging { tag_set: vec![] });
// Remove old swift-meta-* tags while preserving other tags
existing_tagging.tag_set.retain(|tag| {
if let Some(key) = &tag.key {
!key.starts_with("swift-meta-")
} else {
true // Keep tags with no key (shouldn't happen, but be safe)
}
// If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
// we've already removed swift-meta-* tags above, so only non-Swift
// tags remain
});
tagging
})
.await?;
// Add new Swift metadata tags if provided
if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
// Merge: existing non-Swift tags + new Swift tags
existing_tagging.tag_set.append(&mut new_tagging.tag_set);
}
// If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
// we've already removed swift-meta-* tags above, so only non-Swift tags remain
let now = time::OffsetDateTime::now_utc();
if existing_tagging.tag_set.is_empty() {
// No tags remain after removing swift-meta-* tags; clear tagging config
bucket_meta_clone.tagging_config_xml = Vec::new();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = None;
} else {
// Serialize the merged tags to XML
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = Some(existing_tagging);
}
// Save updated metadata
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
Ok(())
}
@@ -800,23 +801,44 @@ pub async fn enable_versioning(
}
})?;
// Rewrite the persisted tags: replace any versioning tag with the new
// archive location while preserving all other tags.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
// Load current bucket metadata
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
tagging
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
let mut bucket_meta_clone = (*bucket_meta).clone();
tagging.tag_set.push(Tag {
key: Some("swift-versions-location".to_string()),
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
});
// Get existing tags
let mut existing_tagging = bucket_meta_clone
.tagging_config
.clone()
.unwrap_or_else(|| Tagging { tag_set: vec![] });
tagging
})
.await?;
// Remove old versioning tag if present
existing_tagging
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
// Add new versioning tag
existing_tagging.tag_set.push(Tag {
key: Some("swift-versions-location".to_string()),
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
});
let now = time::OffsetDateTime::now_utc();
// Serialize tags to XML
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = Some(existing_tagging);
// Save updated metadata
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
Ok(())
}
@@ -842,38 +864,50 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
let mapper = ContainerMapper::default();
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
let Some(store) = resolve_swift_object_store_handle() else {
// Verify container exists
let Some(_store) = resolve_swift_object_store_handle() else {
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
};
// Verify container exists. Without this the rewrite below would persist a
// fabricated default for a container that does not exist: the metadata
// loader turns "no metadata on disk" into a fresh BucketMetadata, and
// writing that creates an orphan .metadata.bin and caches a fabricated
// default as authoritative.
store
.get_bucket_info(&bucket_name, &BucketOptions::default())
// Load current bucket metadata
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
.await
.map_err(|e| {
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
SwiftError::NotFound(format!("Container '{}' not found", container))
} else {
sanitize_storage_error("Container verification", e)
}
})?;
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
// Rewrite the persisted tags: drop the versioning tag while preserving
// all other tags. An empty result clears the tagging config.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
let mut bucket_meta_clone = (*bucket_meta).clone();
tagging
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
// Get existing tags
let mut existing_tagging = bucket_meta_clone
.tagging_config
.clone()
.unwrap_or_else(|| Tagging { tag_set: vec![] });
tagging
})
.await?;
// Remove versioning tag
existing_tagging
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
let now = time::OffsetDateTime::now_utc();
if existing_tagging.tag_set.is_empty() {
// No tags remain; clear tagging config
bucket_meta_clone.tagging_config_xml = Vec::new();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = None;
} else {
// Serialize remaining tags to XML
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = Some(existing_tagging);
}
// Save updated metadata
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
Ok(())
}
@@ -998,37 +1032,65 @@ pub async fn set_container_acl(
}
})?;
// Rewrite the persisted tags: replace the ACL tags with the new grants
// while preserving all other tags. An empty result clears the tagging
// config.
update_swift_bucket_tagging(bucket_name, |current| {
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
// Load current bucket metadata
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
tagging
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-acl-read") && tag.key.as_deref() != Some("swift-acl-write"));
let mut bucket_meta_clone = (*bucket_meta).clone();
if let Some(read) = read_acl
&& !read.trim().is_empty()
{
tagging.tag_set.push(Tag {
key: Some("swift-acl-read".to_string()),
value: Some(read.to_string()),
});
}
// Get existing tags
let mut existing_tagging = bucket_meta_clone
.tagging_config
.clone()
.unwrap_or_else(|| Tagging { tag_set: vec![] });
if let Some(write) = write_acl
&& !write.trim().is_empty()
{
tagging.tag_set.push(Tag {
key: Some("swift-acl-write".to_string()),
value: Some(write.to_string()),
});
}
// Remove old ACL tags
existing_tagging
.tag_set
.retain(|tag| tag.key.as_deref() != Some("swift-acl-read") && tag.key.as_deref() != Some("swift-acl-write"));
tagging
})
.await?;
// Add new read ACL tag if provided
if let Some(read) = read_acl
&& !read.trim().is_empty()
{
existing_tagging.tag_set.push(Tag {
key: Some("swift-acl-read".to_string()),
value: Some(read.to_string()),
});
}
// Add new write ACL tag if provided
if let Some(write) = write_acl
&& !write.trim().is_empty()
{
existing_tagging.tag_set.push(Tag {
key: Some("swift-acl-write".to_string()),
value: Some(write.to_string()),
});
}
let now = time::OffsetDateTime::now_utc();
if existing_tagging.tag_set.is_empty() {
// No tags remain; clear tagging config
bucket_meta_clone.tagging_config_xml = Vec::new();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = None;
} else {
// Serialize tags to XML
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
bucket_meta_clone.tagging_config_updated_at = now;
bucket_meta_clone.tagging_config = Some(existing_tagging);
}
// Save updated metadata
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
.await
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
debug!(
"Set ACLs for container {}/{}: read={:?}, write={:?}",
@@ -1781,24 +1843,4 @@ mod tests {
assert_eq!(tagging.tag_set.len(), 1);
assert_eq!(tagging.tag_set[0].value.as_deref(), Some("new-archive"));
}
#[test]
fn nonempty_container_usage_preserves_authoritative_totals() {
let usage = std::collections::HashMap::from([("tenant-container".to_string(), (7, 4097))]);
assert_eq!(
required_bucket_usage(&usage, "tenant-container").expect("authoritative bucket usage"),
(7, 4097)
);
}
#[test]
fn missing_container_usage_fails_closed() {
let usage = std::collections::HashMap::new();
assert!(matches!(
required_bucket_usage(&usage, "tenant-container"),
Err(SwiftError::ServiceUnavailable(_))
));
}
}
-4
View File
@@ -32,8 +32,6 @@ pub enum SwiftError {
NotFound(String),
/// 409 Conflict
Conflict(String),
/// 411 Length Required
LengthRequired(String),
/// 413 Request Entity Too Large (Payload Too Large)
RequestEntityTooLarge(String),
/// 422 Unprocessable Entity
@@ -56,7 +54,6 @@ impl fmt::Display for SwiftError {
SwiftError::Forbidden(msg) => write!(f, "Forbidden: {}", msg),
SwiftError::NotFound(msg) => write!(f, "Not Found: {}", msg),
SwiftError::Conflict(msg) => write!(f, "Conflict: {}", msg),
SwiftError::LengthRequired(msg) => write!(f, "Length Required: {}", msg),
SwiftError::RequestEntityTooLarge(msg) => write!(f, "Request Entity Too Large: {}", msg),
SwiftError::UnprocessableEntity(msg) => write!(f, "Unprocessable Entity: {}", msg),
SwiftError::TooManyRequests { retry_after, .. } => {
@@ -79,7 +76,6 @@ impl SwiftError {
SwiftError::Forbidden(_) => StatusCode::FORBIDDEN,
SwiftError::NotFound(_) => StatusCode::NOT_FOUND,
SwiftError::Conflict(_) => StatusCode::CONFLICT,
SwiftError::LengthRequired(_) => StatusCode::LENGTH_REQUIRED,
SwiftError::RequestEntityTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE,
SwiftError::UnprocessableEntity(_) => StatusCode::UNPROCESSABLE_ENTITY,
SwiftError::TooManyRequests { .. } => StatusCode::TOO_MANY_REQUESTS,
+10 -59
View File
@@ -28,9 +28,7 @@ use axum::http::{Method, Request, Response, StatusCode};
use futures::Future;
use rustfs_credentials::Credentials;
use rustfs_keystone::KEYSTONE_CREDENTIALS;
use rustfs_trusted_proxies::ClientInfo;
use s3s::Body;
use std::net::{IpAddr, SocketAddr};
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio_util::io::StreamReader;
@@ -42,14 +40,6 @@ const LOG_SUBSYSTEM_SWIFT_HANDLER: &str = "swift_handler";
const EVENT_SWIFT_ROUTE_STATE: &str = "swift_route_state";
const EVENT_SWIFT_TEMPURL_STATE: &str = "swift_tempurl_state";
fn trusted_client_ip<B>(req: &Request<B>) -> Option<IpAddr> {
req.extensions()
.get::<ClientInfo>()
.map(|info| info.real_ip)
.or_else(|| req.extensions().get::<SocketAddr>().map(SocketAddr::ip))
.filter(|ip| !ip.is_unspecified())
}
/// Swift-aware service that routes to Swift handlers or S3 service
#[derive(Clone)]
pub struct SwiftService<S> {
@@ -138,7 +128,6 @@ async fn handle_swift_request(
route: SwiftRoute,
credentials: Option<Credentials>,
) -> Result<Response<Body>, SwiftError> {
let client_ip = trusted_client_ip(&req);
// Extract parts
let (parts, body) = req.into_parts();
let method = parts.method.clone();
@@ -175,7 +164,7 @@ async fn handle_swift_request(
let tempurl = tempurl::TempURL::new(key);
let path = uri.path();
tempurl.validate_request(method.as_str(), path, &tempurl_params, client_ip)?;
tempurl.validate_request(method.as_str(), path, &tempurl_params)?;
// TempURL is valid - proceed with request (no credentials needed)
debug!(
@@ -652,11 +641,14 @@ async fn handle_authenticated_request(
.await;
}
let object_size = headers
.get("content-length")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u64>().ok());
super::quota::check_upload_quota(&account, &container, object_size, &credentials).await?;
// Check quota before upload (if Content-Length provided)
if let Some(content_length) = headers.get("content-length")
&& let Ok(size_str) = content_length.to_str()
&& let Ok(object_size) = size_str.parse::<u64>()
{
// Check if upload would exceed quota
super::quota::check_upload_quota(&account, &container, object_size, &credentials).await?;
}
// Check if versioning is enabled for this container
if let Some(archive_container) = container::get_versions_location(&account, &container, &credentials).await? {
@@ -1483,7 +1475,6 @@ fn swift_error_to_response(error: SwiftError) -> Response<Body> {
SwiftError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.as_str()),
SwiftError::NotFound(msg) => (StatusCode::NOT_FOUND, msg.as_str()),
SwiftError::Conflict(msg) => (StatusCode::CONFLICT, msg.as_str()),
SwiftError::LengthRequired(msg) => (StatusCode::LENGTH_REQUIRED, msg.as_str()),
SwiftError::RequestEntityTooLarge(msg) => (StatusCode::PAYLOAD_TOO_LARGE, msg.as_str()),
SwiftError::UnprocessableEntity(msg) => (StatusCode::UNPROCESSABLE_ENTITY, msg.as_str()),
SwiftError::TooManyRequests { .. } => (StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded"),
@@ -1506,10 +1497,7 @@ fn swift_error_to_response(error: SwiftError) -> Response<Body> {
#[cfg(test)]
mod tests {
use super::{parse_range_header, trusted_client_ip};
use axum::http::Request;
use rustfs_trusted_proxies::ClientInfo;
use std::net::SocketAddr;
use super::parse_range_header;
#[test]
fn test_parse_range_header_start_end() {
// bytes=100-199
@@ -1600,41 +1588,4 @@ mod tests {
let result = parse_range_header("bytes=0-999", 1000);
assert_eq!(result, Some((0, 999)));
}
#[test]
fn trusted_client_ip_ignores_untrusted_forwarded_headers() {
let mut request = Request::builder()
.header("x-forwarded-for", "198.51.100.9")
.header("x-real-ip", "198.51.100.10")
.body(())
.expect("request");
request
.extensions_mut()
.insert("192.0.2.7:9000".parse::<SocketAddr>().expect("socket address"));
assert_eq!(trusted_client_ip(&request), Some("192.0.2.7".parse().expect("peer IP address")));
}
#[test]
fn trusted_client_ip_prefers_validated_proxy_identity() {
let mut request = Request::new(());
request
.extensions_mut()
.insert("192.0.2.7:9000".parse::<SocketAddr>().expect("socket address"));
request.extensions_mut().insert(ClientInfo::direct(
"203.0.113.8:443".parse::<SocketAddr>().expect("validated client address"),
));
assert_eq!(trusted_client_ip(&request), Some("203.0.113.8".parse().expect("validated client IP")));
}
#[test]
fn trusted_client_ip_rejects_unspecified_fallback() {
let mut request = Request::new(());
request
.extensions_mut()
.insert(ClientInfo::direct("0.0.0.0:0".parse::<SocketAddr>().expect("unspecified fallback")));
assert_eq!(trusted_client_ip(&request), None);
}
}
+1 -44
View File
@@ -58,53 +58,10 @@ pub mod versioning;
pub use errors::{SwiftError, SwiftResult};
pub use router::{SwiftRoute, SwiftRouter};
/// Maximum number of metadata headers allowed per resource (Swift standard)
pub(crate) const MAX_METADATA_COUNT: usize = 90;
/// Maximum size in bytes for a single metadata value (Swift standard)
pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
/// Validate metadata against Swift limits
///
/// Checks that:
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
///
/// Applies to object, container and account metadata alike: all three are
/// persisted, and container/account metadata additionally lands in the
/// bucket metadata file that every later config write rewrites in full.
///
/// Returns error if limits are exceeded.
pub(crate) fn validate_metadata(metadata: &std::collections::HashMap<String, String>) -> SwiftResult<()> {
// Check total metadata count
if metadata.len() > MAX_METADATA_COUNT {
return Err(SwiftError::BadRequest(format!(
"Too many metadata headers: {} (max: {})",
metadata.len(),
MAX_METADATA_COUNT
)));
}
// Check individual value sizes
for (key, value) in metadata.iter() {
if value.len() > MAX_METADATA_VALUE_SIZE {
return Err(SwiftError::BadRequest(format!(
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
key,
value.len(),
MAX_METADATA_VALUE_SIZE
)));
}
}
Ok(())
}
// Note: Container, Object, and SwiftMetadata types used by Swift implementation
pub use storage_api::public_api::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
pub(crate) use storage_api::public_api::{
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata,
};
#[allow(unused_imports)]
pub use types::{Container, Object, SwiftMetadata};
+39 -1
View File
@@ -53,7 +53,7 @@ use super::account::validate_account_access;
use super::container::ContainerMapper;
use super::expiration_worker::{track_object_expiration, untrack_object_expiration};
use super::storage_api::object::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle, validate_metadata};
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle};
use axum::http::HeaderMap;
use rustfs_credentials::Credentials;
use rustfs_rio::HashReader;
@@ -68,6 +68,12 @@ const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
/// Maximum number of metadata headers allowed per object (Swift standard)
const MAX_METADATA_COUNT: usize = 90;
/// Maximum size in bytes for a single metadata value (Swift standard)
const MAX_METADATA_VALUE_SIZE: usize = 256;
/// Maximum object size in bytes (5GB - Swift default)
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
@@ -228,6 +234,38 @@ impl Default for ObjectKeyMapper {
}
}
/// Validate metadata against Swift limits
///
/// Checks that:
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
///
/// Returns error if limits are exceeded.
fn validate_metadata(metadata: &HashMap<String, String>) -> SwiftResult<()> {
// Check total metadata count
if metadata.len() > MAX_METADATA_COUNT {
return Err(SwiftError::BadRequest(format!(
"Too many metadata headers: {} (max: {})",
metadata.len(),
MAX_METADATA_COUNT
)));
}
// Check individual value sizes
for (key, value) in metadata.iter() {
if value.len() > MAX_METADATA_VALUE_SIZE {
return Err(SwiftError::BadRequest(format!(
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
key,
value.len(),
MAX_METADATA_VALUE_SIZE
)));
}
}
Ok(())
}
fn metadata_delete_at(metadata: &HashMap<String, String>) -> Option<u64> {
metadata
.get(SWIFT_DELETE_AT_METADATA)
+21 -42
View File
@@ -87,17 +87,18 @@ pub struct QuotaConfig {
impl QuotaConfig {
/// Load quota configuration from container metadata
pub async fn load(account: &str, container_name: &str, credentials: &Credentials) -> SwiftResult<Self> {
let custom_metadata = container::get_container_custom_metadata(account, container_name, credentials).await?;
// Get container metadata
let container_info = container::get_container_metadata(account, container_name, credentials).await?;
let mut config = QuotaConfig::default();
// Parse Quota-Bytes
if let Some(quota_bytes_str) = custom_metadata.get("x-container-meta-quota-bytes") {
if let Some(quota_bytes_str) = container_info.custom_metadata.get("x-container-meta-quota-bytes") {
config.quota_bytes = quota_bytes_str.parse().ok();
}
// Parse Quota-Count
if let Some(quota_count_str) = custom_metadata.get("x-container-meta-quota-count") {
if let Some(quota_count_str) = container_info.custom_metadata.get("x-container-meta-quota-count") {
config.quota_count = quota_count_str.parse().ok();
}
@@ -112,12 +113,9 @@ impl QuotaConfig {
/// Check if adding an object would exceed quotas
///
/// Returns Ok(()) if within quota, Err with 413 if exceeded
pub fn check_quota(&self, current_bytes: u64, current_count: u64, additional_bytes: Option<u64>) -> SwiftResult<()> {
pub fn check_quota(&self, current_bytes: u64, current_count: u64, additional_bytes: u64) -> SwiftResult<()> {
// Check byte quota
if let Some(max_bytes) = self.quota_bytes {
let additional_bytes = additional_bytes.ok_or_else(|| {
SwiftError::LengthRequired("Content-Length is required when a byte quota is configured".to_string())
})?;
let new_bytes = current_bytes.saturating_add(additional_bytes);
if new_bytes > max_bytes {
return Err(SwiftError::RequestEntityTooLarge(format!(
@@ -149,7 +147,7 @@ impl QuotaConfig {
pub async fn check_upload_quota(
account: &str,
container_name: &str,
object_size: Option<u64>,
object_size: u64,
credentials: &Credentials,
) -> SwiftResult<()> {
// Load quota config
@@ -159,6 +157,7 @@ pub async fn check_upload_quota(
if !quota.is_enabled() {
return Ok(());
}
// Get current container usage
let metadata = container::get_container_metadata(account, container_name, credentials).await?;
@@ -174,7 +173,7 @@ pub async fn check_upload_quota(
quota_bytes = ?quota.quota_bytes,
current_count = metadata.object_count,
quota_count = ?quota.quota_count,
object_size = ?object_size,
object_size,
"swift quota state changed"
);
@@ -236,7 +235,7 @@ mod tests {
};
// Current: 500 bytes, adding 400 bytes = 900 total (within 1000 limit)
let result = config.check_quota(500, 0, Some(400));
let result = config.check_quota(500, 0, 400);
assert!(result.is_ok());
}
@@ -248,7 +247,7 @@ mod tests {
};
// Current: 500 bytes, adding 600 bytes = 1100 total (exceeds 1000 limit)
let result = config.check_quota(500, 0, Some(600));
let result = config.check_quota(500, 0, 600);
assert!(result.is_err());
match result {
Err(SwiftError::RequestEntityTooLarge(msg)) => {
@@ -266,7 +265,7 @@ mod tests {
};
// Current: 500 bytes, adding 500 bytes = 1000 total (exactly at limit)
let result = config.check_quota(500, 0, Some(500));
let result = config.check_quota(500, 0, 500);
assert!(result.is_ok());
}
@@ -278,7 +277,7 @@ mod tests {
};
// Current: 5 objects, adding 1 = 6 total (within 10 limit)
let result = config.check_quota(0, 5, Some(100));
let result = config.check_quota(0, 5, 100);
assert!(result.is_ok());
}
@@ -290,7 +289,7 @@ mod tests {
};
// Current: 10 objects, adding 1 = 11 total (exceeds 10 limit)
let result = config.check_quota(0, 10, Some(100));
let result = config.check_quota(0, 10, 100);
assert!(result.is_err());
match result {
Err(SwiftError::RequestEntityTooLarge(msg)) => {
@@ -308,7 +307,7 @@ mod tests {
};
// Current: 9 objects, adding 1 = 10 total (exactly at limit)
let result = config.check_quota(0, 9, Some(100));
let result = config.check_quota(0, 9, 100);
assert!(result.is_ok());
}
@@ -320,7 +319,7 @@ mod tests {
};
// Both within limits
let result = config.check_quota(500, 5, Some(400));
let result = config.check_quota(500, 5, 400);
assert!(result.is_ok());
}
@@ -332,7 +331,7 @@ mod tests {
};
// Bytes exceeded, count within
let result = config.check_quota(500, 5, Some(600));
let result = config.check_quota(500, 5, 600);
assert!(result.is_err());
}
@@ -344,7 +343,7 @@ mod tests {
};
// Count exceeded, bytes within
let result = config.check_quota(500, 10, Some(100));
let result = config.check_quota(500, 10, 100);
assert!(result.is_err());
}
@@ -356,7 +355,7 @@ mod tests {
};
// No limits, should always pass
let result = config.check_quota(999999, 999999, Some(999999));
let result = config.check_quota(999999, 999999, 999999);
assert!(result.is_ok());
}
@@ -368,7 +367,7 @@ mod tests {
};
// Zero limit means no uploads allowed
let result = config.check_quota(0, 0, Some(1));
let result = config.check_quota(0, 0, 1);
assert!(result.is_err());
}
@@ -380,7 +379,7 @@ mod tests {
};
// Zero limit means no objects allowed
let result = config.check_quota(0, 0, Some(100));
let result = config.check_quota(0, 0, 100);
assert!(result.is_err());
}
@@ -392,28 +391,8 @@ mod tests {
};
// Test saturating_add protection
let result = config.check_quota(u64::MAX - 100, 0, Some(200));
let result = config.check_quota(u64::MAX - 100, 0, 200);
// Should saturate to u64::MAX and compare against quota
assert!(result.is_ok());
}
#[test]
fn byte_quota_rejects_upload_without_content_length() {
let config = QuotaConfig {
quota_bytes: Some(1000),
quota_count: None,
};
assert!(matches!(config.check_quota(500, 0, None), Err(SwiftError::LengthRequired(_))));
}
#[test]
fn count_only_quota_checks_upload_without_content_length() {
let config = QuotaConfig {
quota_bytes: None,
quota_count: Some(10),
};
assert!(matches!(config.check_quota(0, 10, None), Err(SwiftError::RequestEntityTooLarge(_))));
}
}
+6 -110
View File
@@ -12,22 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::sync::Arc;
use rustfs_ecstore::api::bucket::metadata::BUCKET_TAGGING_CONFIG;
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
use rustfs_ecstore::api::bucket::metadata_sys::{get as get_swift_bucket_metadata_from_backend, update_config_with};
use rustfs_ecstore::api::bucket::utils::serialize as serialize_bucket_config;
use rustfs_ecstore::api::error::Error as SwiftStorageError;
use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
};
pub(crate) use rustfs_ecstore::api::error::Result as SwiftStorageResult;
use rustfs_ecstore::api::notification::get_global_notification_sys;
pub(crate) use rustfs_ecstore::api::runtime::object_store_handle as resolve_swift_object_store_handle;
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
use rustfs_storage_api as storage_contracts;
use s3s::dto::Tagging;
use super::{SwiftError, SwiftResult};
pub(crate) mod account {
pub(crate) use super::storage_contracts::{BucketOperations, MakeBucketOptions};
@@ -49,24 +43,13 @@ pub(crate) mod object {
pub(crate) mod public_api {
pub use super::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
pub(crate) use super::{
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
};
pub(crate) use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
}
pub(crate) mod versioning {
pub(crate) use super::storage_contracts::{ListOperations, ObjectOperations};
}
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
const LOG_SUBSYSTEM_SWIFT_STORAGE: &str = "swift_storage";
const EVENT_SWIFT_BUCKET_TAGGING_UPDATE: &str = "swift_bucket_tagging_update";
/// Marks the refusal to rewrite an unreadable persisted tagging config, so the
/// caller can turn it into an actionable client error rather than a generic
/// storage failure. Carried through the ecstore error, which is a string type.
const UNREADABLE_TAGGING_SENTINEL: &str = "swift: persisted tagging config could not be parsed";
pub type SwiftGetObjectReader = <SwiftStore as storage_contracts::ObjectIO>::GetObjectReader;
pub type SwiftObjectInfo = <SwiftStore as storage_contracts::ObjectOperations>::ObjectInfo;
pub type SwiftObjectOptions = <SwiftStore as storage_contracts::ObjectOperations>::ObjectOptions;
@@ -76,93 +59,6 @@ pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResul
get_swift_bucket_metadata_from_backend(bucket).await
}
/// Rewrite the bucket's tagging config through the persisting
/// bucket-metadata path.
///
/// `rewrite` sees the tag set currently persisted on disk (`None` when the
/// bucket has none) and returns the full replacement; an empty tag set
/// clears the config. The read-modify-write runs under the bucket metadata
/// system's write guard — serialized against every other config update —
/// and the result is written to the bucket metadata file before the cache
/// is refreshed, so a Swift metadata POST survives process restarts and
/// disk-truth reloads. Peers are then told to reload, matching what the S3
/// handlers do after a config write.
///
/// Storage failures are logged in full and reported to the client as a
/// generic error: these now carry real disk and quorum detail, which does not
/// belong in a Swift response body. The one exception is an unreadable
/// persisted config, which is reported specifically because the operator has
/// to act on it.
pub(crate) async fn update_swift_bucket_tagging<F>(bucket: String, rewrite: F) -> SwiftResult<()>
where
F: FnOnce(Option<&Tagging>) -> Tagging + Send,
{
let result = update_config_with(&bucket, BUCKET_TAGGING_CONFIG, |bm| {
// Merging onto an unparseable tag set would silently drop every tag
// the bucket has — including the container ACL and versioning tags —
// because the rewrite closures treat "no parsed tags" as "no tags".
// Refuse instead: the persisted config is intact, just unreadable.
if !bm.tagging_config_xml.is_empty() && bm.tagging_config.is_none() {
return Err(SwiftStorageError::other(UNREADABLE_TAGGING_SENTINEL));
}
let tagging = rewrite(bm.tagging_config.as_ref());
if tagging.tag_set.is_empty() {
Ok(Vec::new())
} else {
// The S3 XML serializer, not quick_xml: the metadata loader's
// parse step must be able to round-trip what we persist.
serialize_bucket_config(&tagging)
.map_err(|e| SwiftStorageError::other(format!("failed to serialize bucket tagging: {e}")))
}
})
.await;
if let Err(err) = result {
let unreadable = err.to_string().contains(UNREADABLE_TAGGING_SENTINEL);
tracing::error!(
event = EVENT_SWIFT_BUCKET_TAGGING_UPDATE,
component = LOG_COMPONENT_PROTOCOLS,
subsystem = LOG_SUBSYSTEM_SWIFT_STORAGE,
bucket = %bucket,
error = %err,
reason = if unreadable { "unreadable_persisted_config" } else { "storage_failure" },
result = "failed",
"swift bucket tagging update failed"
);
// A Swift-only client has no way to repair this itself, so say what
// happened and name the remedy instead of a bare storage error.
return Err(if unreadable {
SwiftError::Conflict(format!(
"The persisted tagging configuration for container store '{bucket}' cannot be parsed, so metadata cannot be updated without discarding it. Reset it with the S3 DeleteBucketTagging API."
))
} else {
SwiftError::InternalServerError("Metadata update operation failed".to_string())
});
}
if let Some(notification_sys) = get_global_notification_sys() {
tokio::spawn(async move {
if let Err(err) = notification_sys.load_bucket_metadata(&bucket).await {
tracing::warn!(bucket = %bucket, error = %err, "failed to notify peers after swift bucket tagging update");
}
});
}
Ok(())
}
pub(crate) async fn get_swift_bucket_usage() -> SwiftStorageResult<Option<HashMap<String, (u64, u64)>>> {
let Some(store) = resolve_swift_object_store_handle() else {
return Ok(None);
};
let mut data_usage = rustfs_ecstore::api::data_usage::load_data_usage_from_backend_cached(store).await?;
rustfs_ecstore::api::data_usage::apply_bucket_usage_memory_overlay(&mut data_usage).await;
Ok(Some(
data_usage
.buckets_usage
.into_iter()
.map(|(bucket, usage)| (bucket, (usage.objects_count, usage.size)))
.collect(),
))
pub(crate) async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
set_swift_bucket_metadata_in_backend(bucket, metadata).await
}
+18 -156
View File
@@ -21,11 +21,7 @@
use crate::swift::errors::SwiftError;
use hmac::{Hmac, KeyInit, Mac};
use ipnetwork::IpNetwork;
use percent_encoding::percent_decode_str;
use sha1::Sha1;
use std::net::IpAddr;
use std::str::FromStr;
use std::time::{SystemTime, UNIX_EPOCH};
type HmacSha1 = Hmac<Sha1>;
@@ -54,12 +50,12 @@ impl TempURLParams {
let mut ip_range = None;
for param in query.split('&') {
if let Some((name, value)) = param.split_once('=') {
let value = percent_decode_str(value).decode_utf8().ok()?;
match name {
"temp_url_sig" => sig = Some(value.into_owned()),
"temp_url_expires" => expires = value.parse().ok(),
"temp_url_ip_range" => ip_range = Some(value.into_owned()),
let parts: Vec<&str> = param.split('=').collect();
if parts.len() == 2 {
match parts[0] {
"temp_url_sig" => sig = Some(parts[1].to_string()),
"temp_url_expires" => expires = parts[1].parse().ok(),
"temp_url_ip_range" => ip_range = Some(parts[1].to_string()),
_ => {}
}
}
@@ -101,22 +97,9 @@ impl TempURL {
/// # Returns
/// Hex-encoded HMAC-SHA1 signature
pub fn generate_signature(&self, method: &str, expires: u64, path: &str) -> Result<String, SwiftError> {
self.generate_signature_with_ip_range(method, expires, path, None)
}
/// Generate a TempURL signature, optionally binding it to an IP range.
pub fn generate_signature_with_ip_range(
&self,
method: &str,
expires: u64,
path: &str,
ip_range: Option<&str>,
) -> Result<String, SwiftError> {
// Construct message for HMAC
let message = match ip_range {
Some(ip_range) => format!("ip={}\n{}\n{}\n{}", ip_range, method.to_uppercase(), expires, path),
None => format!("{}\n{}\n{}", method.to_uppercase(), expires, path),
};
// Format: "{METHOD}\n{expires}\n{path}"
let message = format!("{}\n{}\n{}", method.to_uppercase(), expires, path);
// Calculate HMAC-SHA1
let mut mac = HmacSha1::new_from_slice(self.key.as_bytes())
@@ -144,13 +127,7 @@ impl TempURL {
/// # Returns
/// - `Ok(())` if signature is valid and not expired
/// - `Err(SwiftError::Unauthorized)` if invalid or expired
pub fn validate_request(
&self,
method: &str,
path: &str,
params: &TempURLParams,
client_ip: Option<IpAddr>,
) -> Result<(), SwiftError> {
pub fn validate_request(&self, method: &str, path: &str, params: &TempURLParams) -> Result<(), SwiftError> {
// 1. Check expiration first (fast path for expired URLs)
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -162,25 +139,17 @@ impl TempURL {
}
// 2. Generate expected signature
let expected_sig =
self.generate_signature_with_ip_range(method, params.temp_url_expires, path, params.temp_url_ip_range.as_deref())?;
let expected_sig = self.generate_signature(method, params.temp_url_expires, path)?;
// 3. Constant-time comparison to prevent timing attacks
if !constant_time_compare(params.temp_url_sig.as_bytes(), expected_sig.as_bytes()) {
return Err(SwiftError::Unauthorized("Invalid TempURL signature".to_string()));
}
if let Some(ip_range) = &params.temp_url_ip_range {
let client_ip =
client_ip.ok_or_else(|| SwiftError::Unauthorized("Trusted client address unavailable".to_string()))?;
let allowed = IpAddr::from_str(ip_range)
.map(|ip| ip == client_ip)
.or_else(|_| IpNetwork::from_str(ip_range).map(|network| network.contains(client_ip)))
.map_err(|_| SwiftError::Unauthorized("Invalid TempURL IP range".to_string()))?;
if !allowed {
return Err(SwiftError::Unauthorized("Client address is outside the TempURL IP range".to_string()));
}
}
// 4. TODO: Validate IP range if specified (future enhancement)
// if let Some(ip_range) = &params.temp_url_ip_range {
// validate_ip_range(client_ip, ip_range)?;
// }
Ok(())
}
@@ -339,7 +308,7 @@ mod tests {
// Should validate successfully
assert!(
tempurl
.validate_request("GET", "/v1/AUTH_test/container/object", &params, None)
.validate_request("GET", "/v1/AUTH_test/container/object", &params)
.is_ok()
);
}
@@ -362,7 +331,7 @@ mod tests {
};
// Should reject expired URL
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", &params, None);
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", &params);
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_)));
}
@@ -380,7 +349,7 @@ mod tests {
};
// Should reject invalid signature
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", &params, None);
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", &params);
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_)));
}
@@ -403,7 +372,7 @@ mod tests {
};
// Try to validate with PUT method
let result = tempurl.validate_request("PUT", "/v1/AUTH_test/container/object", &params, None);
let result = tempurl.validate_request("PUT", "/v1/AUTH_test/container/object", &params);
assert!(result.is_err());
assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_)));
}
@@ -454,14 +423,6 @@ mod tests {
assert_eq!(params.temp_url_ip_range.as_deref(), Some("192.168.1.0/24"));
}
#[test]
fn test_parse_percent_encoded_ipv6_range() {
let query = "temp_url_sig=abc123&temp_url_expires=1609459200&temp_url_ip_range=2001%3Adb8%3A%3A%2F32";
let params = TempURLParams::from_query(query).expect("encoded IPv6 TempURL parameters");
assert_eq!(params.temp_url_ip_range.as_deref(), Some("2001:db8::/32"));
}
#[test]
fn test_parse_tempurl_params_missing_sig() {
let query = "temp_url_expires=1609459200";
@@ -514,103 +475,4 @@ mod tests {
.unwrap();
assert_eq!(sig, sig2);
}
#[test]
fn ip_bound_signature_matches_openstack_message_format() {
let tempurl = TempURL::new("mykey".to_string());
let signature = tempurl
.generate_signature_with_ip_range("GET", 1648082711, "/v1/AUTH_account/container/object", Some("1.2.3.0/24"))
.expect("IP-bound signature generation");
assert_eq!(signature, "8698990d811e64cff1c8a2151340874fd5bda0c5");
}
fn ip_bound_params(tempurl: &TempURL, ip_range: &str) -> TempURLParams {
let expires = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock after Unix epoch")
.as_secs()
+ 3600;
let signature = tempurl
.generate_signature_with_ip_range("GET", expires, "/v1/AUTH_test/container/object", Some(ip_range))
.expect("IP-bound signature generation");
TempURLParams {
temp_url_sig: signature,
temp_url_expires: expires,
temp_url_ip_range: Some(ip_range.to_string()),
}
}
#[test]
fn ip_bound_tempurl_accepts_ipv4_inside_range_and_rejects_outside() {
let tempurl = TempURL::new("mykey".to_string());
let params = ip_bound_params(&tempurl, "192.0.2.0/24");
assert!(
tempurl
.validate_request(
"GET",
"/v1/AUTH_test/container/object",
&params,
Some("192.0.2.42".parse().expect("IPv4 address")),
)
.is_ok()
);
assert!(matches!(
tempurl.validate_request(
"GET",
"/v1/AUTH_test/container/object",
&params,
Some("198.51.100.42".parse().expect("IPv4 address")),
),
Err(SwiftError::Unauthorized(_))
));
}
#[test]
fn ip_bound_tempurl_accepts_ipv6_inside_range_and_rejects_outside() {
let tempurl = TempURL::new("mykey".to_string());
let params = ip_bound_params(&tempurl, "2001:db8::/32");
assert!(
tempurl
.validate_request(
"GET",
"/v1/AUTH_test/container/object",
&params,
Some("2001:db8::42".parse().expect("IPv6 address")),
)
.is_ok()
);
assert!(matches!(
tempurl.validate_request(
"GET",
"/v1/AUTH_test/container/object",
&params,
Some("2001:db9::42".parse().expect("IPv6 address")),
),
Err(SwiftError::Unauthorized(_))
));
}
#[test]
fn ip_bound_tempurl_rejects_missing_client_ip_and_invalid_range() {
let tempurl = TempURL::new("mykey".to_string());
let params = ip_bound_params(&tempurl, "192.0.2.0/24");
assert!(matches!(
tempurl.validate_request("GET", "/v1/AUTH_test/container/object", &params, None),
Err(SwiftError::Unauthorized(_))
));
let invalid_params = ip_bound_params(&tempurl, "not-a-network");
assert!(matches!(
tempurl.validate_request(
"GET",
"/v1/AUTH_test/container/object",
&invalid_params,
Some("192.0.2.42".parse().expect("IPv4 address")),
),
Err(SwiftError::Unauthorized(_))
));
}
}
@@ -1,25 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! ECStore facade boundary for the protocols integration tests.
//!
//! The Swift tests need to drive bucket-metadata reloads the way the peer
//! `LoadBucketMetadata` RPC does. Everything they touch from `rustfs_ecstore`
//! is aliased here so the tests themselves hold no raw facade subpaths, the
//! same boundary `crates/protocols/src/swift/storage_api.rs` provides for the
//! Swift implementation.
pub use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
pub use rustfs_ecstore::api::bucket::metadata_sys::{get as get_bucket_metadata, set_bucket_metadata};
pub use rustfs_ecstore::api::runtime::object_store_handle;
@@ -1,312 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Regression tests: a Swift metadata POST must be persisted to the bucket
//! metadata file, not just the in-memory cache. The metadata has to survive
//! the disk-truth reloads performed by peer LoadBucketMetadata notifications
//! and the periodic refresh loop — and, transitively, a process restart.
#![cfg(feature = "swift")]
use std::collections::HashMap;
use rustfs_credentials::Credentials;
use rustfs_protocols::swift::SwiftError;
use rustfs_protocols::swift::container::{ContainerMapper, update_container_metadata};
use rustfs_protocols::swift::{account, container};
use rustfs_test_utils::TestECStoreEnv;
use serde_json::json;
use sha2::{Digest, Sha256};
mod ecstore_test_compat;
use ecstore_test_compat::{get_bucket_metadata, load_bucket_metadata, object_store_handle, set_bucket_metadata};
fn keystone_credentials(project_id: &str) -> Credentials {
let mut claims = HashMap::new();
claims.insert("keystone_project_id".to_string(), json!(project_id));
claims.insert("keystone_roles".to_string(), json!(["member"]));
Credentials {
access_key: "keystone:swift-test".to_string(),
claims: Some(claims),
..Default::default()
}
}
/// The account-metadata bucket name scheme from `swift::account`
/// (`swift-account-{sha256(account)[0..16]}`), mirrored here so the test can
/// reload that bucket's metadata from disk.
fn account_metadata_bucket_name(account: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(account.as_bytes());
let hash = hex::encode(hasher.finalize());
format!("swift-account-{}", &hash[0..16])
}
/// Replace the cached bucket metadata with what is actually on disk — the
/// same thing a peer LoadBucketMetadata notification or the periodic refresh
/// loop does. Before the fix this silently discarded every Swift metadata
/// POST, because those writes only ever touched the cache.
async fn reload_bucket_metadata_from_disk(bucket: &str) {
let store = object_store_handle().expect("test store should be published");
let bm = load_bucket_metadata(store, bucket)
.await
.expect("bucket metadata should load from disk");
set_bucket_metadata(bucket.to_string(), bm)
.await
.expect("reloaded metadata should install");
}
/// The `swift-meta-*` tags currently persisted for a container, keyed the way
/// a Swift client sees them. `get_container_metadata` would be the natural
/// reader, but it additionally requires a data-usage snapshot for the object
/// count and byte total, which a bare test store has none of — and that is
/// orthogonal to whether the metadata itself was persisted.
async fn persisted_container_metadata(bucket: &str) -> HashMap<String, String> {
let bm = get_bucket_metadata(bucket).await.expect("bucket metadata should be cached");
let mut out = HashMap::new();
if let Some(tagging) = &bm.tagging_config {
for tag in &tagging.tag_set {
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
&& let Some(meta_key) = key.strip_prefix("swift-meta-")
{
out.insert(meta_key.to_string(), value.clone());
}
}
}
out
}
/// Every scenario that needs a store runs against ONE environment: the Swift
/// handlers resolve the ambient object store and bucket-metadata system, so a
/// second `TestECStoreEnv` in this process would race the first.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn swift_metadata_writes_are_durable() {
let env = TestECStoreEnv::builder().prefix("swift_meta_persist").build().await;
posts_survive_disk_truth_reload(&env).await;
tag_writers_preserve_each_others_state(&env).await;
versioning_writes_reject_missing_containers().await;
}
async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
// --- Container metadata POST (X-Container-Meta-*) ---
let project_id = "swiftpersistproj";
let swift_account = format!("AUTH_{project_id}");
let credentials = keystone_credentials(project_id);
let swift_container = "photos";
let bucket = ContainerMapper::default().swift_to_s3_bucket(swift_container, project_id);
env.make_bucket(&bucket, false).await;
let mut metadata = HashMap::new();
metadata.insert("color".to_string(), "blue".to_string());
update_container_metadata(&swift_account, swift_container, &credentials, metadata)
.await
.expect("container metadata POST should succeed");
reload_bucket_metadata_from_disk(&bucket).await;
let container_meta = persisted_container_metadata(&bucket).await;
assert_eq!(
container_meta.get("color").map(String::as_str),
Some("blue"),
"container metadata POST must survive a disk-truth metadata reload"
);
// A follow-up POST replaces the Swift metadata and that replacement must
// survive a reload too (the rewrite merges against disk state, so the
// previous value must actually be gone).
let mut metadata = HashMap::new();
metadata.insert("season".to_string(), "summer".to_string());
update_container_metadata(&swift_account, swift_container, &credentials, metadata)
.await
.expect("second container metadata POST should succeed");
reload_bucket_metadata_from_disk(&bucket).await;
let container_meta = persisted_container_metadata(&bucket).await;
assert_eq!(container_meta.get("season").map(String::as_str), Some("summer"));
assert!(
!container_meta.contains_key("color"),
"replaced container metadata must not resurrect on reload"
);
// --- Container versioning POST (X-Versions-Location) ---
let archive_container = "photos-archive";
let archive_bucket = ContainerMapper::default().swift_to_s3_bucket(archive_container, project_id);
env.make_bucket(&archive_bucket, false).await;
container::enable_versioning(&swift_account, swift_container, archive_container, &credentials)
.await
.expect("enable versioning should succeed");
reload_bucket_metadata_from_disk(&bucket).await;
let location = container::get_versions_location(&swift_account, swift_container, &credentials)
.await
.expect("versions location should load");
assert_eq!(
location.as_deref(),
Some(archive_container),
"versions location must survive a disk-truth metadata reload"
);
// --- Account metadata POST (TempURL keys etc.) ---
let mut account_meta = HashMap::new();
account_meta.insert("temp-url-key".to_string(), "s3cr3t".to_string());
account::update_account_metadata(&swift_account, &account_meta, &Some(credentials.clone()))
.await
.expect("account metadata POST should succeed");
reload_bucket_metadata_from_disk(&account_metadata_bucket_name(&swift_account)).await;
let loaded = account::get_account_metadata(&swift_account, &None)
.await
.expect("account metadata should load");
assert_eq!(
loaded.get("temp-url-key").map(String::as_str),
Some("s3cr3t"),
"account metadata POST must survive a disk-truth metadata reload"
);
}
/// The rewrites all share one tag set, so a closure that ignored the current
/// state would still pass a single-feature test. Drive ACLs, versioning and
/// container metadata over the same container and assert each survives the
/// others — and that clearing one leaves the rest alone.
async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
let project_id = "swiftcrosstagproj";
let swift_account = format!("AUTH_{project_id}");
let credentials = keystone_credentials(project_id);
let container = "shared";
let archive = "shared-archive";
let bucket = ContainerMapper::default().swift_to_s3_bucket(container, project_id);
env.make_bucket(&bucket, false).await;
env.make_bucket(&ContainerMapper::default().swift_to_s3_bucket(archive, project_id), false)
.await;
let mut metadata = HashMap::new();
metadata.insert("color".to_string(), "blue".to_string());
update_container_metadata(&swift_account, container, &credentials, metadata)
.await
.expect("container metadata POST should succeed");
container::enable_versioning(&swift_account, container, archive, &credentials)
.await
.expect("enable versioning should succeed");
container::set_container_acl(&swift_account, container, Some(".r:*"), Some("AUTH_other"), &credentials)
.await
.expect("set container ACL should succeed");
reload_bucket_metadata_from_disk(&bucket).await;
// All three writers' state coexists after a disk-truth reload.
let meta = persisted_container_metadata(&bucket).await;
assert_eq!(meta.get("color").map(String::as_str), Some("blue"));
assert_eq!(
container::get_versions_location(&swift_account, container, &credentials)
.await
.expect("versions location should load")
.as_deref(),
Some(archive)
);
let acl = container::get_container_acl(&swift_account, container, &credentials)
.await
.expect("container ACL should load");
assert!(!acl.read.is_empty(), "read ACL must survive the reload");
assert!(!acl.write.is_empty(), "write ACL must survive the reload");
// Disabling versioning drops only the versioning tag.
container::disable_versioning(&swift_account, container, &credentials)
.await
.expect("disable versioning should succeed");
reload_bucket_metadata_from_disk(&bucket).await;
assert_eq!(
container::get_versions_location(&swift_account, container, &credentials)
.await
.expect("versions location should load"),
None,
"disable_versioning must clear the versioning tag durably"
);
let meta = persisted_container_metadata(&bucket).await;
assert_eq!(
meta.get("color").map(String::as_str),
Some("blue"),
"disable_versioning must not disturb container metadata"
);
let acl = container::get_container_acl(&swift_account, container, &credentials)
.await
.expect("container ACL should load");
assert!(!acl.read.is_empty(), "disable_versioning must not disturb the ACL");
}
/// A container that does not exist must not get metadata persisted for it:
/// the metadata loader turns "nothing on disk" into a fresh default, so an
/// unguarded rewrite would create an orphan metadata file and cache a
/// fabricated default as authoritative.
async fn versioning_writes_reject_missing_containers() {
let project_id = "swiftmissingproj";
let swift_account = format!("AUTH_{project_id}");
let credentials = keystone_credentials(project_id);
let missing = "no-such-container";
let err = container::disable_versioning(&swift_account, missing, &credentials)
.await
.expect_err("disabling versioning on a missing container must fail");
assert!(
matches!(err, SwiftError::NotFound(_)),
"expected NotFound for a missing container, got {err:?}"
);
let bucket = ContainerMapper::default().swift_to_s3_bucket(missing, project_id);
assert!(
get_bucket_metadata(&bucket).await.is_err(),
"a rejected write must not have cached metadata for a nonexistent container"
);
}
/// Account metadata holds the account's TempURL signing key, and it is now
/// durable — so a write for someone else's account would be a persistent,
/// cluster-wide takeover of that account's pre-signed URLs, not a cache blip.
/// The write path must reject both a foreign account and a missing token,
/// while reads stay open for pre-auth TempURL signature validation.
///
/// Deliberately builds no store: both rejections must happen before the write
/// path resolves storage at all, and a second `TestECStoreEnv` in this process
/// would race the other test over the ambient store handle.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn account_metadata_write_rejects_foreign_and_anonymous_callers() {
let victim_account = "AUTH_victimproject";
let attacker_credentials = keystone_credentials("attackerproject");
let mut poisoned = HashMap::new();
poisoned.insert("temp-url-key".to_string(), "attacker-key".to_string());
let err = account::update_account_metadata(victim_account, &poisoned, &Some(attacker_credentials))
.await
.expect_err("writing another account's metadata must be rejected");
assert!(
matches!(err, SwiftError::Forbidden(_)),
"cross-account metadata write must be Forbidden, got {err:?}"
);
let err = account::update_account_metadata(victim_account, &poisoned, &None)
.await
.expect_err("anonymous account metadata write must be rejected");
assert!(
matches!(err, SwiftError::Unauthorized(_)),
"anonymous metadata write must be Unauthorized, got {err:?}"
);
}
@@ -233,46 +233,6 @@ pub struct RenamePartResponse {
pub error: ::core::option::Option<Error>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct PreparePartTransactionRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub src_volume: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub src_path: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub dst_volume: ::prost::alloc::string::String,
#[prost(string, tag = "5")]
pub dst_path: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "6")]
pub meta: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct PreparePartTransactionResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(message, optional, tag = "2")]
pub error: ::core::option::Option<Error>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct SettlePartTransactionRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub volume: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub path: ::prost::alloc::string::String,
#[prost(bool, tag = "4")]
pub rollback: bool,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct SettlePartTransactionResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(message, optional, tag = "2")]
pub error: ::core::option::Option<Error>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RenameFileRequest {
#[prost(string, tag = "1")]
pub disk: ::prost::alloc::string::String,
@@ -1640,21 +1600,6 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "CheckParts"));
self.inner.unary(req, path, codec).await
}
pub async fn prepare_part_transaction(
&mut self,
request: impl tonic::IntoRequest<super::PreparePartTransactionRequest>,
) -> std::result::Result<tonic::Response<super::PreparePartTransactionResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/PreparePartTransaction");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "PreparePartTransaction"));
self.inner.unary(req, path, codec).await
}
pub async fn rename_part(
&mut self,
request: impl tonic::IntoRequest<super::RenamePartRequest>,
@@ -1670,21 +1615,6 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "RenamePart"));
self.inner.unary(req, path, codec).await
}
pub async fn settle_part_transaction(
&mut self,
request: impl tonic::IntoRequest<super::SettlePartTransactionRequest>,
) -> std::result::Result<tonic::Response<super::SettlePartTransactionResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/SettlePartTransaction");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "SettlePartTransaction"));
self.inner.unary(req, path, codec).await
}
pub async fn rename_file(
&mut self,
request: impl tonic::IntoRequest<super::RenameFileRequest>,
@@ -2796,18 +2726,10 @@ pub mod node_service_server {
&self,
request: tonic::Request<super::CheckPartsRequest>,
) -> std::result::Result<tonic::Response<super::CheckPartsResponse>, tonic::Status>;
async fn prepare_part_transaction(
&self,
request: tonic::Request<super::PreparePartTransactionRequest>,
) -> std::result::Result<tonic::Response<super::PreparePartTransactionResponse>, tonic::Status>;
async fn rename_part(
&self,
request: tonic::Request<super::RenamePartRequest>,
) -> std::result::Result<tonic::Response<super::RenamePartResponse>, tonic::Status>;
async fn settle_part_transaction(
&self,
request: tonic::Request<super::SettlePartTransactionRequest>,
) -> std::result::Result<tonic::Response<super::SettlePartTransactionResponse>, tonic::Status>;
async fn rename_file(
&self,
request: tonic::Request<super::RenameFileRequest>,
@@ -3510,34 +3432,6 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/PreparePartTransaction" => {
#[allow(non_camel_case_types)]
struct PreparePartTransactionSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::PreparePartTransactionRequest> for PreparePartTransactionSvc<T> {
type Response = super::PreparePartTransactionResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::PreparePartTransactionRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::prepare_part_transaction(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = PreparePartTransactionSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/RenamePart" => {
#[allow(non_camel_case_types)]
struct RenamePartSvc<T: NodeService>(pub Arc<T>);
@@ -3566,34 +3460,6 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/SettlePartTransaction" => {
#[allow(non_camel_case_types)]
struct SettlePartTransactionSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::SettlePartTransactionRequest> for SettlePartTransactionSvc<T> {
type Response = super::SettlePartTransactionResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::SettlePartTransactionRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::settle_part_transaction(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = SettlePartTransactionSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/RenameFile" => {
#[allow(non_camel_case_types)]
struct RenameFileSvc<T: NodeService>(pub Arc<T>);
+2 -68
View File
@@ -600,30 +600,6 @@ pub fn canonical_rename_part_request_body(
Ok(body.finish())
}
pub fn canonical_prepare_part_transaction_request_body(
request: &proto_gen::node_service::PreparePartTransactionRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-prepare-part-transaction-request-v1\0");
body.push_str(&request.disk)?;
body.push_str(&request.src_volume)?;
body.push_str(&request.src_path)?;
body.push_str(&request.dst_volume)?;
body.push_str(&request.dst_path)?;
body.push_bytes(&request.meta)?;
Ok(body.finish())
}
pub fn canonical_settle_part_transaction_request_body(
request: &proto_gen::node_service::SettlePartTransactionRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-settle-part-transaction-request-v1\0");
body.push_str(&request.disk)?;
body.push_str(&request.volume)?;
body.push_str(&request.path)?;
body.push_bool(request.rollback);
Ok(body.finish())
}
pub fn canonical_delete_volume_request_body(
request: &proto_gen::node_service::DeleteVolumeRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -661,8 +637,8 @@ pub fn canonical_make_volumes_request_body(
mod disk_mutation_canonical_tests {
use super::proto_gen::node_service::{
DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest,
MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest,
SettlePartTransactionRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
MakeVolumesRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, UpdateMetadataRequest, WriteAllRequest,
WriteMetadataRequest,
};
use super::*;
@@ -910,48 +886,6 @@ mod disk_mutation_canonical_tests {
}
assert_all_distinct(&bodies);
let prepare_part = PreparePartTransactionRequest {
disk: "d".into(),
src_volume: "sv".into(),
src_path: "sp".into(),
dst_volume: "dv".into(),
dst_path: "dp".into(),
meta: vec![0x01].into(),
};
let mut bodies = vec![canonical_prepare_part_transaction_request_body(&prepare_part).unwrap()];
for mutate in [
|r: &mut PreparePartTransactionRequest| r.disk = "d2".into(),
|r: &mut PreparePartTransactionRequest| r.src_volume = "sv2".into(),
|r: &mut PreparePartTransactionRequest| r.src_path = "sp2".into(),
|r: &mut PreparePartTransactionRequest| r.dst_volume = "dv2".into(),
|r: &mut PreparePartTransactionRequest| r.dst_path = "dp2".into(),
|r: &mut PreparePartTransactionRequest| r.meta = vec![0x02].into(),
] {
let mut request = prepare_part.clone();
mutate(&mut request);
bodies.push(canonical_prepare_part_transaction_request_body(&request).unwrap());
}
assert_all_distinct(&bodies);
let settle_part = SettlePartTransactionRequest {
disk: "d".into(),
volume: "v".into(),
path: "p".into(),
rollback: false,
};
let mut bodies = vec![canonical_settle_part_transaction_request_body(&settle_part).unwrap()];
for mutate in [
|r: &mut SettlePartTransactionRequest| r.disk = "d2".into(),
|r: &mut SettlePartTransactionRequest| r.volume = "v2".into(),
|r: &mut SettlePartTransactionRequest| r.path = "p2".into(),
|r: &mut SettlePartTransactionRequest| r.rollback = true,
] {
let mut request = settle_part.clone();
mutate(&mut request);
bodies.push(canonical_settle_part_transaction_request_body(&request).unwrap());
}
assert_all_distinct(&bodies);
let rename_part = RenamePartRequest {
disk: "d".into(),
src_volume: "sv".into(),
-28
View File
@@ -170,32 +170,6 @@ message RenamePartResponse {
optional Error error = 2;
}
message PreparePartTransactionRequest {
string disk = 1;
string src_volume = 2;
string src_path = 3;
string dst_volume = 4;
string dst_path = 5;
bytes meta = 6;
}
message PreparePartTransactionResponse {
bool success = 1;
optional Error error = 2;
}
message SettlePartTransactionRequest {
string disk = 1;
string volume = 2;
string path = 3;
bool rollback = 4;
}
message SettlePartTransactionResponse {
bool success = 1;
optional Error error = 2;
}
message RenameFileRequest {
string disk = 1;
string src_volume = 2;
@@ -969,9 +943,7 @@ service NodeService {
rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {};
rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {};
rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {};
rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {};
rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {};
rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {};
rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {};
rpc Write(WriteRequest) returns (WriteResponse) {};
rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {};
+1 -1
View File
@@ -30,7 +30,7 @@ Or point the tests at another generated root:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-smoke'
cargo +1.97.1 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored
cargo +1.96.0 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored
```
## Scope
+1 -14
View File
@@ -271,11 +271,7 @@ impl EventName {
EventName::ObjectCreatedPut,
],
EventName::ObjectTaggingAll => vec![EventName::ObjectTaggingPut, EventName::ObjectTaggingDelete],
EventName::ObjectRemovedAll => vec![
EventName::ObjectRemovedDelete,
EventName::ObjectRemovedDeleteMarkerCreated,
EventName::ObjectRemovedNoOP,
],
EventName::ObjectRemovedAll => vec![EventName::ObjectRemovedDelete, EventName::ObjectRemovedDeleteMarkerCreated],
EventName::ObjectReplicationAll => vec![
EventName::ObjectReplicationFailed,
EventName::ObjectReplicationComplete,
@@ -643,15 +639,6 @@ mod tests {
assert_eq!(EventName::parse("s3:Scanner:*").unwrap(), EventName::ObjectScannerAll);
}
#[test]
fn test_object_removed_all_includes_noop_extension() {
let expanded = EventName::ObjectRemovedAll.expand();
assert!(expanded.contains(&EventName::ObjectRemovedDelete));
assert!(expanded.contains(&EventName::ObjectRemovedDeleteMarkerCreated));
assert!(expanded.contains(&EventName::ObjectRemovedNoOP));
}
/// `is_removed` must be true for every `ObjectRemoved*` variant and false
/// for everything else.
#[test]
-3
View File
@@ -40,9 +40,6 @@ allow-git = [
# SigV4 payload-checksum fix until it is available in a crates.io release.
# owner: marshawcoco review: 2026-10
"https://github.com/s3s-project/s3s.git",
# Presigned expiry and constant-time authentication fixes pending upstream merge.
# owner: cxymds review: 2026-10
"https://github.com/cxymds/s3s.git",
"https://github.com/apache/datafusion.git",
]
@@ -20,7 +20,6 @@ for later deletion.
- `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
- `backlog-1316` legacy encrypted multipart range seek: the feature remains opt-in until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently.
- `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
## Review Checklist
-10
View File
@@ -9,16 +9,6 @@ failure pattern reported in rustfs/rustfs#4304.
> format. Replacing the executable and restarting is safe; no migration step
> runs on startup.
> [!WARNING]
> The release that switches local SSE wrapped DEKs from the legacy
> `base64(nonce):base64(ciphertext)` representation to the versioned JSON
> envelope is a deliberate exception. Do not run that release together with
> an older RustFS version: older nodes cannot read objects written with the
> JSON envelope. Freeze every source of object mutation, including client
> writes and background lifecycle or replication work, upgrade every node,
> and then resume traffic. Downgrading or rolling back after new encrypted
> objects are written is not supported.
## TL;DR
- **Rolling restart (no downtime):** restart **one node at a time**, and wait
+1
View File
@@ -30,6 +30,7 @@
systems = [
"x86_64-linux"
"aarch64-linux"
"x86_64-darwin"
"aarch64-darwin"
];
forAllSystems = nixpkgs.lib.genAttrs systems;
+4 -42
View File
@@ -36,8 +36,6 @@ use tracing::{error, info, instrument, warn};
/// Path to store KMS configuration in the cluster metadata
const KMS_CONFIG_PATH: &str = "config/kms_config.json";
const STATIC_KMS_LOCAL_CONFIG_REQUIRED: &str =
"Static KMS must be configured through RUSTFS_KMS_STATIC_SECRET_KEY or RUSTFS_KMS_STATIC_SECRET_KEY_FILE";
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_KMS: &str = "kms";
const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state";
@@ -108,25 +106,9 @@ fn normalize_configure_request_auth(
Ok(())
}
fn ensure_kms_config_persistable(config: &KmsConfig) -> Result<(), String> {
if matches!(&config.backend_config, rustfs_kms::BackendConfig::Static(_)) {
return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string());
}
Ok(())
}
fn ensure_kms_request_persistable(request: &ConfigureKmsRequest) -> Result<(), String> {
if matches!(request, ConfigureKmsRequest::Static(_)) {
return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string());
}
Ok(())
}
/// Save KMS configuration to cluster storage
#[instrument(skip(config))]
async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
ensure_kms_config_persistable(config)?;
let context = current_app_context();
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
return Err("Storage layer not initialized".to_string());
@@ -350,16 +332,12 @@ impl Operation for ConfigureKmsHandler {
);
let service_manager = kms_service_manager_from_context();
let existing_config = service_manager.get_redacted_config().await;
let existing_config = service_manager.get_config().await;
if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
if let Err(e) = ensure_kms_request_persistable(&configure_request) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
// Convert request to KmsConfig
let kms_config = configure_request.to_kms_config();
@@ -775,7 +753,7 @@ impl Operation for GetKmsStatusHandler {
let service_manager = kms_service_manager_from_context();
let status = service_manager.get_status().await;
let config = service_manager.get_redacted_config().await;
let config = service_manager.get_config().await;
// Get backend type and health status
let backend_type = config.as_ref().map(|c| c.backend.clone());
@@ -895,16 +873,12 @@ impl Operation for ReconfigureKmsHandler {
);
let service_manager = kms_service_manager_from_context();
let existing_config = service_manager.get_redacted_config().await;
let existing_config = service_manager.get_config().await;
if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
if let Err(e) = ensure_kms_request_persistable(&configure_request) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
// Convert request to KmsConfig
let kms_config = configure_request.to_kms_config();
@@ -986,7 +960,7 @@ impl Operation for ReconfigureKmsHandler {
#[cfg(test)]
mod tests {
use super::{decode_persisted_kms_config, ensure_kms_config_persistable, kms_configure_actions, kms_service_control_actions};
use super::{decode_persisted_kms_config, kms_configure_actions, kms_service_control_actions};
use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
use tempfile::TempDir;
@@ -1072,16 +1046,4 @@ mod tests {
assert!(!config.allow_insecure_dev_defaults);
assert!(config.validate().is_ok());
}
#[test]
fn static_kms_config_is_not_persisted_with_cluster_configuration() {
use base64::Engine as _;
let config = rustfs_kms::KmsConfig::static_kms(
"static-key".to_string(),
base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]),
);
assert!(ensure_kms_config_persistable(&config).is_err());
}
}
+2 -2
View File
@@ -193,7 +193,7 @@ impl Operation for KmsStatusHandler {
hit_count: hits,
miss_count: misses,
});
let config = kms_service_manager_from_context().get_redacted_config().await;
let config = kms_service_manager_from_context().get_config().await;
let response = KmsStatusResponse {
backend_type: config
@@ -243,7 +243,7 @@ impl Operation for KmsConfigHandler {
};
let config = kms_service_manager_from_context()
.get_redacted_config()
.get_config()
.await
.ok_or_else(|| s3_error!(InternalError, "KMS config not available"))?;
+2 -37
View File
@@ -33,7 +33,7 @@ use crate::admin::runtime_sources::{
};
use crate::admin::storage_api::access::{ReqInfo, authorize_request, spawn_traced};
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::auth::{check_key_valid, constant_time_eq, get_session_token};
use crate::auth::{check_key_valid, get_session_token};
use crate::error::ApiError;
use crate::license::license_check;
use crate::server::{
@@ -905,9 +905,7 @@ fn validate_object_lambda_response_auth_headers(headers: &HeaderMap, output_rout
.and_then(|value| value.to_str().ok())
.map(str::trim);
if route.is_some_and(|route| constant_time_eq(route, output_route))
&& token.is_some_and(|token| constant_time_eq(token, output_token))
{
if route == Some(output_route) && token == Some(output_token) {
return Ok(());
}
@@ -4353,39 +4351,6 @@ mod tests {
let err = validate_object_lambda_response_auth_headers(&mismatched, "route-123", "token-456")
.expect_err("mismatched auth headers should fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
for (route, token) in [
("Route-123", "token-456"),
("route-124", "token-456"),
("route-1234", "token-456"),
("route-123", "Token-456"),
("route-123", "token-457"),
("route-123", "token-4567"),
] {
let mut headers = HeaderMap::new();
headers.insert(
"x-amz-request-route",
HeaderValue::try_from(route).expect("test route must be a valid header"),
);
headers.insert(
"x-amz-request-token",
HeaderValue::try_from(token).expect("test token must be a valid header"),
);
assert!(
validate_object_lambda_response_auth_headers(&headers, "route-123", "token-456").is_err(),
"first-byte, last-byte, and length mismatches must fail: {route}/{token}"
);
}
}
#[test]
fn object_lambda_auth_headers_use_constant_time_helper() {
let source = include_str!("router.rs");
let production = source.split_once("#[cfg(test)]").map_or(source, |(production, _)| production);
assert!(!production.contains("route == Some(output_route)"));
assert!(!production.contains("token == Some(output_token)"));
assert!(production.contains("constant_time_eq(route, output_route)"));
assert!(production.contains("constant_time_eq(token, output_token)"));
}
#[test]
+4 -6
View File
@@ -1810,7 +1810,7 @@ impl DefaultBucketUsecase {
client_info,
);
let read_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
let read_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
bucket: &bucket,
action: Action::S3Action(S3Action::ListBucketAction),
is_owner: false,
@@ -1819,10 +1819,9 @@ impl DefaultBucketUsecase {
conditions: &conditions,
object: "",
})
.await
.map_err(ApiError::from)?;
.await;
let write_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
let write_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
bucket: &bucket,
action: Action::S3Action(S3Action::PutObjectAction),
is_owner: false,
@@ -1831,8 +1830,7 @@ impl DefaultBucketUsecase {
conditions: &conditions,
object: "",
})
.await
.map_err(ApiError::from)?;
.await;
let mut is_public = read_allowed || write_allowed;
let ignore_public_acls = match metadata_sys::get_public_access_block_config(&bucket).await {
@@ -198,9 +198,13 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
first_info.total_free_capacity
);
assert_eq!(
second_info.total_used_capacity,
second_info.total_capacity.saturating_sub(second_info.total_free_capacity),
"server used capacity must stay internally consistent on repeated requests"
(first_info.total_capacity, first_info.total_free_capacity, first_info.total_used_capacity,),
(
second_info.total_capacity,
second_info.total_free_capacity,
second_info.total_used_capacity,
),
"repeated data usage requests must report stable capacity values"
);
// The endpoint must serve the seeded snapshot numbers, not recomputed ones.
+102 -80
View File
@@ -169,34 +169,6 @@ async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
.expect("Failed to upload test object")
}
async fn transition_uploaded_object_directly(
ecstore: &Arc<ECStore>,
bucket: &str,
object: &str,
tier_name: &str,
uploaded: &ObjectInfo,
) -> ObjectInfo {
let transition_opts = ObjectOptions {
transition: lifecycle::lifecycle_contract::TransitionOptions {
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
tier: tier_name.to_string(),
etag: uploaded.etag.clone().unwrap_or_default(),
..Default::default()
},
version_id: uploaded.version_id.map(|version| version.to_string()),
versioned: uploaded.version_id.is_some(),
mod_time: uploaded.mod_time,
..Default::default()
};
ecstore
.transition_object(bucket, object, &transition_opts)
.await
.expect("Failed to transition object directly");
wait_for_transition(ecstore, bucket, object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("object should transition before restore assertions")
}
async fn set_bucket_lifecycle_transition_with_tier(
bucket_name: &str,
storage_class: &str,
@@ -2054,12 +2026,13 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
/// backlog#1148 ilm-8: the RestoreObject API surface on a transitioned object.
///
/// POST restore(days=1) is accepted and flips the object to
/// `x-amz-restore: ongoing-request="true"` while the mock tier GET barrier
/// proves the background copy-back has reached the remote read; a second POST
/// during that window is rejected with 409 `RestoreAlreadyInProgress`.
/// Synchronous SetDisks transition tests cover copy-back completion, restore
/// metadata, and local byte-identical reads.
/// POST restore(days=1) is accepted and immediately flips the object to
/// `x-amz-restore: ongoing-request="true"` (the mock tier's injected GET
/// latency keeps the background copy-back in flight); a second POST during
/// that window is rejected with 409 `RestoreAlreadyInProgress`; once the
/// copy-back completes the object reports `ongoing-request="false"` with a
/// future expiry-date; and a full GET is then served from the local restored
/// copy (the mock tier records no further `get` calls).
///
/// Re-enabled in the serial lane by backlog#1304: the accept path now flips
/// the ongoing flag under a short compare-and-set guard and the copy-back
@@ -2069,7 +2042,7 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
async fn restore_object_usecase_reports_ongoing_conflict() {
async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultObjectUsecase::from_global();
@@ -2077,17 +2050,29 @@ async fn restore_object_usecase_reports_ongoing_conflict() {
let backend = register_mock_tier(&tier_name).await;
let bucket = format!("test-api-restore-{}", &Uuid::new_v4().simple().to_string()[..8]);
// Keep the object under the shared ILM test prefix even though this setup
// transitions it directly; it keeps diagnostics aligned with sibling tests.
// Must live under the `test/` prefix: `set_bucket_lifecycle_transition_with_tier`
// scopes the transition rule to `<Filter><Prefix>test/</Prefix>`, so an object
// outside it never matches, is never enqueued, and never transitions — the
// setup `wait_for_transition` would then time out before the restore assertions.
let object = "test/restore/api-object.bin";
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
create_test_bucket(&ecstore, bucket.as_str()).await;
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
backend.clear_op_log().await;
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
.await
.expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
let get_barrier = backend.arm_get_barrier().await;
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
.await
.expect("Failed to enqueue transitioned object");
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("object should transition before the restore API runs");
// Slow the tier GET so the background copy-back stays in flight long
// enough to observe the ongoing state and the conflict rejection.
backend.set_latency(Some(Duration::from_millis(1500))).await;
let restore_request = || RestoreRequest {
days: Some(1),
@@ -2111,18 +2096,15 @@ async fn restore_object_usecase_reports_ongoing_conflict() {
.await
.expect("restore request should be accepted");
get_barrier.wait_until_paused().await;
// The barrier proves the detached copy-back reached the tier GET and is
// still paused, so the ongoing state and conflict rejection are not timing
// assumptions about task scheduling.
// The accepted restore is immediately visible as ongoing (the metadata is
// written synchronously before the copy-back is spawned).
let ongoing = ecstore
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
.await
.expect("Failed to load object info during restore");
assert!(
ongoing.restore_ongoing,
"x-amz-restore must report ongoing-request=true while the copy-back tier GET is paused"
"x-amz-restore must report ongoing-request=true right after the restore is accepted"
);
// A second restore while one is in flight is rejected.
@@ -2135,7 +2117,45 @@ async fn restore_object_usecase_reports_ongoing_conflict() {
"unexpected rejection for a repeated restore: {err:?}"
);
get_barrier.release();
// Completion: ongoing flips to false and a future expiry-date appears.
let mut completed = None;
for _ in 0..40 {
let info = ecstore
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
.await
.expect("Failed to poll object info for restore completion");
if !info.restore_ongoing && info.restore_expires.is_some() {
completed = Some(info);
break;
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
let completed = completed.expect("restore copy-back should complete within the poll window");
backend.clear_faults().await;
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock before unix epoch")
.as_secs() as i64;
let expires = completed.restore_expires.expect("completed restore carries an expiry");
assert!(
expires.unix_timestamp() > now_secs,
"restore expiry-date must be in the future, got {expires}"
);
assert_eq!(
completed.transitioned_object.status, "complete",
"restore must not clear the transitioned state"
);
// The restored copy serves GET locally: no further tier GETs.
let tier_gets_after_restore = backend.get_count().await;
let data = read_object_bytes(&ecstore, bucket.as_str(), object).await;
assert_eq!(data, payload, "restored GET must return the original bytes");
assert_eq!(
backend.get_count().await,
tier_gets_after_restore,
"GET of a restored object must be served locally, not from the tier"
);
}
/// backlog#1304: the restore-accept compare-and-set itself, under real
@@ -2156,18 +2176,27 @@ async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores()
let backend = register_mock_tier(&tier_name).await;
let bucket = format!("test-api-restore-cas-{}", &Uuid::new_v4().simple().to_string()[..8]);
// Keep the object under the shared ILM test prefix for diagnostics parity.
// Must live under the `test/` prefix — the shared transition rule filters on it.
let object = "test/restore/cas-object.bin";
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
create_test_bucket(&ecstore, bucket.as_str()).await;
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
backend.clear_op_log().await;
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
.await
.expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
// Hold the accepted copy-back at the tier GET until both accept attempts
// return, so the loser cannot observe an already-restored object.
let get_barrier = backend.arm_get_barrier().await;
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
.await
.expect("Failed to enqueue transitioned object");
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("object should transition before the concurrent restores run");
// Keep the winner's copy-back in flight while the loser's accept runs, so
// the loser cannot slip into the already-restored path after a completed
// copy-back.
backend.set_latency(Some(Duration::from_millis(1500))).await;
let tier_gets_before_restore = backend.get_count().await;
@@ -2212,34 +2241,27 @@ async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores()
"the losing concurrent restore must be rejected as already in progress: {rejection:?}"
);
get_barrier.wait_until_paused().await;
get_barrier.release();
// This test is scoped to the accept CAS: completion, expiry metadata, and
// local restored GET service are covered by the single-request restore test
// above. Here it is enough to prove exactly one copy-back was admitted.
let expected_tier_gets = tier_gets_before_restore + 1;
let deadline = tokio::time::Instant::now() + TRANSITION_WAIT_TIMEOUT;
loop {
let actual_tier_gets = backend.get_count().await;
if actual_tier_gets >= expected_tier_gets {
assert_eq!(
actual_tier_gets - tier_gets_before_restore,
1,
"two concurrent restore requests must trigger exactly one tier copy-back GET"
);
// Let the single accepted copy-back complete, then verify the tier saw
// exactly one restore read — a second GET means a double copy-back.
let mut completed = false;
for _ in 0..40 {
let info = ecstore
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
.await
.expect("Failed to poll object info for restore completion");
if !info.restore_ongoing && info.restore_expires.is_some() {
completed = true;
break;
}
if tokio::time::Instant::now() >= deadline {
let op_log = backend.op_log().await;
panic!(
"mock tier should record exactly one restore GET within {TRANSITION_WAIT_TIMEOUT:?}; \
tier_gets_before_restore={tier_gets_before_restore}, actual_tier_gets={actual_tier_gets}, op_log={op_log:?}"
);
}
tokio::time::sleep(Duration::from_millis(50)).await;
tokio::time::sleep(Duration::from_millis(500)).await;
}
backend.clear_faults().await;
assert!(completed, "the accepted restore copy-back should complete within the poll window");
assert_eq!(
backend.get_count().await - tier_gets_before_restore,
1,
"two concurrent restore requests must trigger exactly one tier copy-back GET"
);
}
/// rustfs/backlog#1320: a single PUT must compute the replication decision
+6 -128
View File
@@ -440,12 +440,6 @@ impl DefaultMultipartUsecase {
}
let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) };
let Some(parts) = multipart_upload.parts.filter(|parts| !parts.is_empty()) else {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
"You must specify at least one part".to_string(),
));
};
let mut opts = get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?;
let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
@@ -455,7 +449,12 @@ impl DefaultMultipartUsecase {
let capacity_scope_token = Uuid::new_v4();
opts.capacity_scope_token = Some(capacity_scope_token);
let uploaded_parts_vec = parts.into_iter().map(complete_part_from_s3).collect::<Vec<_>>();
let uploaded_parts_vec = multipart_upload
.parts
.unwrap_or_default()
.into_iter()
.map(complete_part_from_s3)
.collect::<Vec<_>>();
let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?;
@@ -1794,127 +1793,6 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
}
#[tokio::test]
async fn execute_complete_multipart_upload_rejects_missing_parts_list() {
use s3s::xml::Deserialize as _;
let mut deserializer = s3s::xml::Deserializer::new(b"<CompleteMultipartUpload/>");
let multipart_upload =
CompletedMultipartUpload::deserialize(&mut deserializer).expect("empty multipart XML should decode");
deserializer
.expect_eof()
.expect("empty multipart XML should be fully consumed");
assert!(multipart_upload.parts.is_none());
let input = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.multipart_upload(Some(multipart_upload))
.build()
.expect("complete multipart input should build");
let req = build_request(input, Method::POST);
let err = Box::pin(make_usecase().execute_complete_multipart_upload(req))
.await
.expect_err("missing parts list must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
#[tokio::test]
#[serial_test::serial]
async fn rejected_empty_parts_preserve_existing_object_and_staging() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("empty-complete-{}", Uuid::new_v4());
let object = "existing-object";
let existing_payload = b"existing object must survive";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create multipart regression bucket");
let mut existing_reader = PutObjReader::from_vec(existing_payload.to_vec());
let existing_info = store
.put_object(&bucket, object, &mut existing_reader, &ObjectOptions::default())
.await
.expect("write existing object");
let upload = store
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
.await
.expect("create multipart staging upload");
let mut part_reader = PutObjReader::from_vec(b"staged part".to_vec());
let staged_part = store
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
.await
.expect("write staged multipart part");
let input = CompleteMultipartUploadInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.upload_id(upload.upload_id.clone())
.multipart_upload(Some(CompletedMultipartUpload { parts: Some(Vec::new()) }))
.build()
.expect("complete multipart input should build");
let err = DefaultMultipartUsecase::from_global()
.execute_complete_multipart_upload(build_request(input, Method::POST))
.await
.expect_err("empty parts list must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let current_info = store
.get_object_info(&bucket, object, &ObjectOptions::default())
.await
.expect("existing object should remain readable");
assert_eq!(current_info.size, existing_info.size);
assert_eq!(current_info.etag, existing_info.etag);
let staged = store
.list_object_parts(&bucket, object, &upload.upload_id, None, 1000, &ObjectOptions::default())
.await
.expect("multipart staging should remain available");
assert_eq!(staged.parts.len(), 1);
assert_eq!(staged.parts[0].part_num, 1);
assert_eq!(staged.parts[0].etag, staged_part.etag);
let staged_etag = staged_part.etag.expect("staged part should have an ETag");
let input = CompleteMultipartUploadInput::builder()
.bucket(bucket)
.key(object.to_string())
.upload_id(upload.upload_id)
.multipart_upload(Some(CompletedMultipartUpload {
parts: Some(vec![CompletedPart {
part_number: Some(1),
e_tag: Some(to_s3s_etag(&staged_etag)),
..Default::default()
}]),
}))
.build()
.expect("complete multipart input should build");
let completed = DefaultMultipartUsecase::from_global()
.execute_complete_multipart_upload(build_request(input, Method::POST))
.await
.expect("staged part should remain completable after empty request rejection");
assert!(completed.output.e_tag.is_some());
let completed_info = store
.get_object_info(
completed
.output
.bucket
.as_deref()
.expect("complete response should include bucket"),
object,
&ObjectOptions::default(),
)
.await
.expect("completed object should be readable");
assert_eq!(completed_info.size, b"staged part".len() as i64);
}
#[tokio::test]
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
let multipart_upload = CompletedMultipartUpload {
+4 -56
View File
@@ -3051,25 +3051,6 @@ fn can_skip_delete_objects_pre_stat(
!bucket_lock_enabled && !replicate_deletes && delete_creates_delete_marker(opts) && accounting_creates_delete_marker
}
fn complete_delete_noop(
helper: OperationHelper,
bucket: String,
key: String,
version_id: Option<String>,
) -> (S3Result<S3Response<DeleteObjectOutput>>, OperationHelper) {
let helper = helper
.event_name(EventName::ObjectRemovedNoOP)
.object(ObjectInfo {
name: key,
bucket,
..Default::default()
})
.version_id(version_id.unwrap_or_default());
let result = Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT));
let helper = helper.complete(&result);
(result, helper)
}
fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
.map(|value| normalize_snowball_prefix(&value))
@@ -7076,7 +7057,7 @@ impl DefaultObjectUsecase {
// TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
// 1. Add a lightweight get_object_lock_info() that only fetches retention metadata
// 2. Or use combined get-and-delete in storage layer with retention check callback
let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone.clone(), None, &req.headers)
let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone, None, &req.headers)
.await
.map_err(ApiError::from)?;
let existing_object_info = match store.get_object_info(&bucket, &key, &get_opts).await {
@@ -7119,8 +7100,9 @@ impl DefaultObjectUsecase {
}
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
let (result, _helper) = complete_delete_noop(helper, bucket, key, version_id_clone);
return result;
// TODO: send event
return Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT));
}
return Err(ApiError::from(err).into());
@@ -13105,40 +13087,6 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
#[test]
fn delete_not_found_completes_noop_event_with_version_context() {
temp_env::with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || {
crate::server::refresh_notify_module_enabled();
for (version_id, expected_version) in [(None, ""), (Some("requested-version".to_string()), "requested-version")] {
let input = DeleteObjectInput::builder()
.bucket("test-bucket".to_string())
.key("missing-key".to_string())
.version_id(version_id.clone())
.build()
.expect("delete input should build");
let mut req = build_request(input, Method::DELETE);
req.extensions.insert(crate::storage::access::ReqInfo {
bucket: Some("test-bucket".to_string()),
object: Some("missing-key".to_string()),
version_id: version_id.clone(),
..Default::default()
});
let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObject);
let (result, helper) =
complete_delete_noop(helper, "test-bucket".to_string(), "missing-key".to_string(), version_id);
let event = helper.event_args().expect("successful no-op delete should retain an event");
assert_eq!(result.expect("no-op delete should succeed").status, Some(StatusCode::NO_CONTENT));
assert_eq!(event.event_name, EventName::ObjectRemovedNoOP);
assert_eq!(event.bucket_name, "test-bucket");
assert_eq!(event.object.name, "missing-key");
assert_eq!(event.version_id, expected_version);
}
});
crate::server::refresh_notify_module_enabled();
}
#[test]
fn expected_current_version_header_normalizes_uuid_and_null() {
let version = Uuid::new_v4();
+4 -17
View File
@@ -452,7 +452,7 @@ pub fn check_claims_from_token(token: &str, cred: &Credentials) -> S3Result<Hash
return Err(s3_error!(InvalidRequest, "invalid token2"));
}
if !cred.is_service_account() && cred.is_temp() && !constant_time_eq(token, &cred.session_token) {
if !cred.is_service_account() && cred.is_temp() && token != cred.session_token {
return Err(s3_error!(InvalidRequest, "invalid token3"));
}
@@ -1801,10 +1801,9 @@ mod tests {
#[test]
fn test_constant_time_eq() {
assert!(constant_time_eq("test", "test"));
assert!(!constant_time_eq("Test", "test"), "first-byte mismatch must fail");
assert!(!constant_time_eq("tesu", "test"), "last-byte mismatch must fail");
assert!(!constant_time_eq("test", "test1"), "longer candidate must fail");
assert!(!constant_time_eq("test1", "test"), "shorter candidate must fail");
assert!(!constant_time_eq("test", "Test"));
assert!(!constant_time_eq("test", "test1"));
assert!(!constant_time_eq("test1", "test"));
assert!(!constant_time_eq("", "test"));
assert!(constant_time_eq("", ""));
@@ -1816,18 +1815,6 @@ mod tests {
assert!(!constant_time_eq(key1, key3));
}
#[test]
fn session_token_comparison_uses_constant_time_helper() {
let source = include_str!("auth.rs");
let production = source.split_once("#[cfg(test)]").map_or(source, |(production, _)| production);
let ordinary_comparison = ["token ", "!=", " cred.session_token"].concat();
assert!(
!production.contains(&ordinary_comparison),
"temporary session tokens must not use ordinary string comparison"
);
assert!(production.contains("!constant_time_eq(token, &cred.session_token)"));
}
#[test]
fn test_get_condition_values_source_ip() {
let mut headers = HeaderMap::new();
-41
View File
@@ -245,21 +245,6 @@ impl From<StorageError> for ApiError {
};
}
if let StorageError::Io(ref io_err) = err
&& matches!(
io_err
.get_ref()
.and_then(|inner| inner.downcast_ref::<rustfs_kms::KmsError>()),
Some(rustfs_kms::KmsError::BackendError { .. })
)
{
return ApiError {
code: S3ErrorCode::ServiceUnavailable,
message: ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable),
source: Some(Box::new(err)),
};
}
let code = match &err {
StorageError::NotImplemented => S3ErrorCode::NotImplemented,
StorageError::InvalidArgument(_, _, _) => S3ErrorCode::InvalidArgument,
@@ -494,14 +479,6 @@ mod tests {
assert!(downcast_storage_error.is_some());
}
#[test]
fn policy_metadata_read_failure_maps_to_internal_error() {
let api_error = ApiError::from(StorageError::Io(IoError::other("policy read failed")));
assert_eq!(api_error.code, S3ErrorCode::InternalError);
assert!(api_error.source.is_some());
}
#[test]
fn test_kms_service_unavailable_maps_to_retryable_error() {
let api_error = ApiError::from(StorageError::other(KmsUnavailableError));
@@ -510,14 +487,6 @@ mod tests {
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn test_kms_backend_unavailable_maps_to_retryable_error() {
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::backend_error("Vault connection refused")));
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() {
let api_error = ApiError::from(QuotaError::UsageUnavailable {
@@ -528,16 +497,6 @@ mod tests {
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn test_kms_cryptographic_error_is_not_retryable() {
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error(
"decrypt",
"authentication failed",
)));
assert_eq!(api_error.code, S3ErrorCode::InternalError);
}
#[test]
fn test_api_error_from_storage_error_mappings() {
let test_cases = vec![
+18 -106
View File
@@ -14,7 +14,7 @@
use super::{
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
refresh_persisted_module_switches_from, runtime_sources,
refresh_persisted_module_switches_from_store, runtime_sources,
};
use crate::storage_api::server::event::{
EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
@@ -309,36 +309,30 @@ pub async fn shutdown_event_notifier() -> Result<(), NotificationError> {
#[instrument]
pub async fn init_event_notifier() -> Result<(), NotificationError> {
init_event_notifier_with_store(runtime_sources::current_object_store_handle).await
}
async fn init_event_notifier_with_store<CurrentStore>(current_store: CurrentStore) -> Result<(), NotificationError>
where
CurrentStore: FnOnce() -> Option<std::sync::Arc<rustfs_notify::NotifyStore>>,
{
mark_event_notifier_unreconciled();
validate_notify_module_env().map_err(NotificationError::Initialization)?;
let system = ensure_live_events_initialized();
let Some(store) = current_store() else {
let enabled = refresh_notify_module_enabled();
if enabled {
return Err(NotificationError::Initialization(
"failed to refresh notify module switch: storage layer not initialized".to_string(),
));
}
initialize_live_event_support(&system, None).await?;
return Ok(());
};
refresh_persisted_module_switches_from(store.clone())
refresh_persisted_module_switches_from_store()
.await
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
let enabled = refresh_notify_module_enabled();
if !enabled {
initialize_live_event_support(&system, Some(store)).await?;
info!(
target: "rustfs::main::init_event_notifier",
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
rustfs_config::ENV_NOTIFY_ENABLE
);
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
system.set_targets_enabled(false, None).await?;
}
system.reload_persisted_config().await?;
info!(
target: "rustfs::main::init_event_notifier",
"Live event stream support initialized successfully."
);
ensure_event_notifier_converged(&system)?;
mark_event_notifier_reconciled();
return Ok(());
}
@@ -353,7 +347,7 @@ where
"Event notifier configuration found, proceeding with initialization."
);
system.reload_persisted_config_from_store(store).await?;
system.reload_persisted_config().await?;
let runtime_state = system.runtime_lifecycle_state();
if !matches!(runtime_state, NotificationRuntimeState::TargetsEnabled { .. }) {
system.set_targets_enabled(true, None).await?;
@@ -367,53 +361,13 @@ where
Ok(())
}
async fn initialize_live_event_support(
system: &NotificationSystem,
store: Option<std::sync::Arc<rustfs_notify::NotifyStore>>,
) -> Result<(), NotificationError> {
if store.is_some() {
info!(
target: "rustfs::main::init_event_notifier",
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
rustfs_config::ENV_NOTIFY_ENABLE
);
} else {
info!(
target: "rustfs::main::init_event_notifier",
"Notify module is disabled, initializing live event stream support only. Persisted notification target reconciliation is deferred until storage is initialized. Set {}=true to enable notification targets.",
rustfs_config::ENV_NOTIFY_ENABLE
);
}
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
system.set_targets_enabled(false, None).await?;
}
if let Some(store) = store {
system.reload_persisted_config_from_store(store).await?;
}
info!(
target: "rustfs::main::init_event_notifier",
"Live event stream support initialized successfully."
);
ensure_event_notifier_converged(system)
}
#[cfg(test)]
mod tests {
use super::super::module_switch::{
PersistedModuleSwitches, current_persisted_module_switches, persisted_module_switches_configured,
set_persisted_module_switches,
};
use super::{
convert_ecstore_object_info, init_event_notifier_with_store, parse_host_and_port, run_persisted_event_notifier_reconciler,
};
use crate::server::is_event_notifier_reconciled;
use super::{convert_ecstore_object_info, parse_host_and_port, run_persisted_event_notifier_reconciler};
use crate::storage_api::server::event::StorageObjectInfo;
use crate::storage_api::server::event::contract::lifecycle::TransitionedObject;
use chrono::{DateTime, Utc};
use rustfs_notify::NotificationError;
use rustfs_notify::NotificationRuntimeState;
use serial_test::serial;
use std::{
collections::HashMap,
future::pending,
@@ -427,48 +381,6 @@ mod tests {
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
#[tokio::test]
#[serial]
async fn disabled_notify_without_storage_initializes_live_events_without_reconcile() {
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"))], async {
let previous = current_persisted_module_switches();
let previous_configured = persisted_module_switches_configured();
set_persisted_module_switches(PersistedModuleSwitches::default(), false);
init_event_notifier_with_store(|| None)
.await
.expect("disabled notify should not require storage during early bootstrap");
let system = rustfs_notify::notification_system().expect("live event container should be initialized");
assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
assert!(
!is_event_notifier_reconciled(),
"early live-only bootstrap must not claim persisted notify config is reconciled"
);
set_persisted_module_switches(previous, previous_configured);
})
.await;
}
#[tokio::test]
#[serial]
async fn enabled_notify_without_storage_still_fails_visible() {
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async {
let previous = current_persisted_module_switches();
let previous_configured = persisted_module_switches_configured();
set_persisted_module_switches(PersistedModuleSwitches::default(), false);
let err = init_event_notifier_with_store(|| None)
.await
.expect_err("enabled notify should still fail when storage is unavailable");
assert!(err.to_string().contains("storage layer not initialized"));
set_persisted_module_switches(previous, previous_configured);
})
.await;
}
#[test]
fn parse_host_and_port_with_ipv4_and_port() {
let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string());
+1 -1
View File
@@ -80,7 +80,7 @@ pub(crate) fn current_persisted_module_switches() -> PersistedModuleSwitches {
}
}
pub(crate) fn persisted_module_switches_configured() -> bool {
fn persisted_module_switches_configured() -> bool {
PERSISTED_MODULE_SWITCH_CONFIGURED.load(Ordering::Relaxed)
}
+21 -29
View File
@@ -505,7 +505,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
let owner_can_bypass_deny = owner_can_bypass_policy_deny(is_owner, &action);
if !bucket_name.is_empty()
&& !owner_can_bypass_deny
&& !PolicySys::try_is_allowed(&BucketPolicyArgs {
&& !PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket_name,
action,
// Early explicit-deny gate for bucket policy: use owner short-circuit path so
@@ -517,7 +517,6 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
object: object.as_str(),
})
.await
.map_err(ApiError::from)?
{
return Err(s3_error!(AccessDenied, "Access Denied"));
}
@@ -536,7 +535,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
};
let delete_version_allowed = iam_store.eval_prepared(&prepared, &delete_version_args).await;
if !delete_version_allowed
&& !PolicySys::try_is_allowed(&BucketPolicyArgs {
&& !PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket.as_str(),
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
is_owner,
@@ -546,7 +545,6 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
object: object.as_str(),
})
.await
.map_err(ApiError::from)?
{
return Err(s3_error!(AccessDenied, "Access Denied"));
}
@@ -573,7 +571,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
return Ok(());
}
let policy_allowed_fallback = PolicySys::try_is_allowed(&BucketPolicyArgs {
let policy_allowed_fallback = PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket.as_str(),
action,
is_owner,
@@ -582,8 +580,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
conditions: &conditions,
object: object.as_str(),
})
.await
.map_err(ApiError::from)?;
.await;
if policy_allowed_fallback {
authorize_table_data_plane_if_needed(action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions, claims)
@@ -608,7 +605,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
return Ok(());
}
if PolicySys::try_is_allowed(&BucketPolicyArgs {
if PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket.as_str(),
action: Action::S3Action(S3Action::ListBucketAction),
is_owner,
@@ -618,7 +615,6 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
object: object.as_str(),
})
.await
.map_err(ApiError::from)?
{
return Ok(());
}
@@ -692,7 +688,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
let bucket_name = bucket.as_str();
if !bucket_name.is_empty()
&& !PolicySys::try_is_allowed(&BucketPolicyArgs {
&& !PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket_name,
action,
// Early explicit-deny gate for bucket policy in anonymous path.
@@ -703,14 +699,13 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
object: object.as_str(),
})
.await
.map_err(ApiError::from)?
{
return Err(s3_error!(AccessDenied, "Access Denied"));
}
if action != Action::S3Action(S3Action::ListAllMyBucketsAction) {
if action == Action::S3Action(S3Action::DeleteObjectAction) && version_id.is_some() {
let delete_version_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
let delete_version_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket.as_str(),
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
is_owner: false,
@@ -719,14 +714,13 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
conditions: &conditions,
object: object.as_str(),
})
.await
.map_err(ApiError::from)?;
.await;
if !delete_version_allowed {
return Err(s3_error!(AccessDenied, "Access Denied"));
}
}
let policy_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
let policy_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket.as_str(),
action,
is_owner: false,
@@ -735,8 +729,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
conditions: &conditions,
object: object.as_str(),
})
.await
.map_err(ApiError::from)?;
.await;
// A bucket policy granting s3:ListBucket also covers listing versions. This
// fallback has to feed the same post-authorization gates as the direct grant
@@ -744,7 +737,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
// ListObjectVersions after RestrictPublicBuckets is turned on.
let policy_allowed = policy_allowed
|| (action == Action::S3Action(S3Action::ListBucketVersionsAction)
&& PolicySys::try_is_allowed(&BucketPolicyArgs {
&& PolicySys::is_allowed(&BucketPolicyArgs {
bucket: bucket.as_str(),
action: Action::S3Action(S3Action::ListBucketAction),
is_owner: false,
@@ -753,8 +746,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
conditions: &conditions,
object: "",
})
.await
.map_err(ApiError::from)?);
.await);
if policy_allowed {
deny_anonymous_table_data_plane_if_needed(action, bucket.as_str(), object.as_str()).await?;
@@ -2771,7 +2763,7 @@ mod tests {
}
#[tokio::test]
async fn abort_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
async fn abort_multipart_upload_rejects_unauthorized_request() {
let fs = FS::new();
let mut req = build_request(
AbortMultipartUploadInput::builder()
@@ -2787,12 +2779,12 @@ mod tests {
let err = fs
.abort_multipart_upload(&mut req)
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
.expect_err("missing credentials should reject access");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
#[tokio::test]
async fn complete_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
async fn complete_multipart_upload_rejects_unauthorized_request() {
let fs = FS::new();
let mut req = build_request(
CompleteMultipartUploadInput::builder()
@@ -2809,12 +2801,12 @@ mod tests {
let err = fs
.complete_multipart_upload(&mut req)
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
.expect_err("missing credentials should reject access");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
#[tokio::test]
async fn upload_part_copy_fails_closed_when_policy_metadata_is_unavailable() {
async fn upload_part_copy_rejects_unauthorized_request() {
let fs = FS::new();
let mut req = build_request(
UploadPartCopyInput::builder()
@@ -2836,7 +2828,7 @@ mod tests {
let err = fs
.upload_part_copy(&mut req)
.await
.expect_err("unavailable policy metadata must fail closed");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
.expect_err("missing credentials should reject access");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
}
-8
View File
@@ -243,14 +243,6 @@ impl OperationHelper {
matches!(self, Self::Enabled(state) if state.event_builder.is_some())
}
#[cfg(test)]
pub(crate) fn event_args(&self) -> Option<rustfs_notify::EventArgs> {
match self {
Self::Enabled(state) => state.event_builder.clone().map(|builder| builder.build()),
Self::Disabled => None,
}
}
/// Sets the ObjectInfo for event notification.
pub fn object(mut self, object_info: ObjectInfo) -> Self {
if let Self::Enabled(state) = &mut self
+6 -75
View File
@@ -926,24 +926,10 @@ impl Node for NodeService {
self.handle_check_parts(request).await
}
async fn prepare_part_transaction(
&self,
request: Request<PreparePartTransactionRequest>,
) -> Result<Response<PreparePartTransactionResponse>, Status> {
self.handle_prepare_part_transaction(request).await
}
async fn rename_part(&self, request: Request<RenamePartRequest>) -> Result<Response<RenamePartResponse>, Status> {
self.handle_rename_part(request).await
}
async fn settle_part_transaction(
&self,
request: Request<SettlePartTransactionRequest>,
) -> Result<Response<SettlePartTransactionResponse>, Status> {
self.handle_settle_part_transaction(request).await
}
async fn rename_file(&self, request: Request<RenameFileRequest>) -> Result<Response<RenameFileResponse>, Status> {
self.handle_rename_file(request).await
}
@@ -2060,12 +2046,12 @@ mod tests {
HealBucketRequest, HealControlRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest,
LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SettlePartTransactionRequest,
SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState,
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, WriteRequest,
MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest,
ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest,
RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest,
StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -2612,28 +2598,6 @@ mod tests {
},
rustfs_protos::canonical_rename_part_request_body
);
assert_gated!(
prepare_part_transaction,
PreparePartTransactionRequest {
disk: disk.clone(),
src_volume: "src".into(),
src_path: "sp".into(),
dst_volume: "dst".into(),
dst_path: "dp".into(),
meta: vec![0x04].into(),
},
rustfs_protos::canonical_prepare_part_transaction_request_body
);
assert_gated!(
settle_part_transaction,
SettlePartTransactionRequest {
disk: disk.clone(),
volume: "dst".into(),
path: "dp".into(),
rollback: true,
},
rustfs_protos::canonical_settle_part_transaction_request_body
);
assert_gated!(
delete_volume,
DeleteVolumeRequest {
@@ -3352,39 +3316,6 @@ mod tests {
assert!(rename_response.error.is_some());
}
#[tokio::test]
async fn test_part_transaction_invalid_disk() {
let service = create_test_node_service();
let prepare = service
.prepare_part_transaction(Request::new(PreparePartTransactionRequest {
disk: "invalid-disk-path".to_string(),
src_volume: "src-volume".to_string(),
src_path: "src-path".to_string(),
dst_volume: "dst-volume".to_string(),
dst_path: "dst-path".to_string(),
meta: Bytes::new(),
}))
.await
.expect("prepare RPC should return a structured disk error")
.into_inner();
assert!(!prepare.success);
assert!(prepare.error.is_some());
let settle = service
.settle_part_transaction(Request::new(SettlePartTransactionRequest {
disk: "invalid-disk-path".to_string(),
volume: "dst-volume".to_string(),
path: "dst-path".to_string(),
rollback: true,
}))
.await
.expect("settle RPC should return a structured disk error")
.into_inner();
assert!(!settle.success);
assert!(settle.error.is_some());
}
#[tokio::test]
async fn test_rename_file_invalid_disk() {
let service = create_test_node_service();
+1 -73
View File
@@ -18,7 +18,7 @@ use crate::storage::storage_api::rpc_consumer::node_service::{
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use crate::storage::storage_api::{PartTransactionAction, verify_tonic_mutation_body_digest};
use crate::storage::storage_api::verify_tonic_mutation_body_digest;
use bytes::Bytes;
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::{
@@ -1038,78 +1038,6 @@ impl NodeService {
}
}
pub(super) async fn handle_prepare_part_transaction(
&self,
request: Request<PreparePartTransactionRequest>,
) -> Result<Response<PreparePartTransactionResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref()),
"prepare_part_transaction",
)?;
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
match disk
.prepare_part_transaction(
&request.src_volume,
&request.src_path,
&request.dst_volume,
&request.dst_path,
request.meta,
)
.await
{
Ok(()) => Ok(Response::new(PreparePartTransactionResponse {
success: true,
error: None,
})),
Err(err) => Ok(Response::new(PreparePartTransactionResponse {
success: false,
error: Some(err.into()),
})),
}
} else {
Ok(Response::new(PreparePartTransactionResponse {
success: false,
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
pub(super) async fn handle_settle_part_transaction(
&self,
request: Request<SettlePartTransactionRequest>,
) -> Result<Response<SettlePartTransactionResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref()),
"settle_part_transaction",
)?;
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let action = if request.rollback {
PartTransactionAction::Rollback
} else {
PartTransactionAction::Commit
};
match disk.settle_part_transaction(&request.volume, &request.path, action).await {
Ok(()) => Ok(Response::new(SettlePartTransactionResponse {
success: true,
error: None,
})),
Err(err) => Ok(Response::new(SettlePartTransactionResponse {
success: false,
error: Some(err.into()),
})),
}
} else {
Ok(Response::new(SettlePartTransactionResponse {
success: false,
error: Some(DiskError::other("cannot find disk".to_string()).into()),
}))
}
}
pub(super) async fn handle_check_parts(
&self,
request: Request<CheckPartsRequest>,
+20 -95
View File
@@ -91,7 +91,6 @@ use rustfs_kms::{DataKey, KmsUnavailableError, is_data_key_envelope, types::Obje
use rustfs_utils::get_env_opt_str;
use s3s::S3ErrorCode;
use s3s::dto::ServerSideEncryption;
use serde::{Deserialize, Serialize};
#[cfg(feature = "rio-v2")]
use sha2::Sha256;
use std::collections::HashMap;
@@ -1886,10 +1885,6 @@ struct KmsSseDekProvider {
service_manager: Option<Arc<rustfs_kms::KmsServiceManager>>,
}
fn kms_operation_error(error: rustfs_kms::KmsError) -> ApiError {
ApiError::from(StorageError::other(error))
}
impl KmsSseDekProvider {
/// Create a new KMS-backed provider
pub async fn new() -> Result<Self, ApiError> {
@@ -1941,7 +1936,7 @@ impl SseDekProvider for KmsSseDekProvider {
let (data_key, encrypted_data_key) = service
.create_data_key(&kms_key_option, context)
.await
.map_err(kms_operation_error)?;
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {}", e))))?;
Ok((data_key, encrypted_data_key))
}
@@ -1959,7 +1954,7 @@ impl SseDekProvider for KmsSseDekProvider {
let data_key = service
.decrypt_data_key(encrypted_dek, context)
.await
.map_err(kms_operation_error)?;
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {}", e))))?;
Ok(data_key.plaintext_key)
}
@@ -1978,7 +1973,7 @@ impl SseDekProvider for KmsSseDekProvider {
let data_key = service
.decrypt_legacy_data_key(encrypted_dek)
.await
.map_err(kms_operation_error)?;
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt legacy data key: {e}"))))?;
Ok(data_key.plaintext_key)
}
@@ -2003,16 +1998,6 @@ pub(crate) struct LocalSseDekProvider {
master_key: [u8; 32],
}
const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct LocalSseDekEnvelope<'a> {
version: u8,
nonce: &'a str,
ciphertext: &'a str,
}
/// Test-only alias so existing test code that references `TestSseDekProvider`
/// continues to compile without changes.
#[cfg(test)]
@@ -2114,51 +2099,22 @@ impl LocalSseDekProvider {
.encrypt(&nonce, dek.as_slice())
.map_err(|_| ApiError::from(StorageError::other("Failed to encrypt DEK")))?;
let nonce = BASE64_STANDARD.encode(nonce);
let ciphertext = BASE64_STANDARD.encode(ciphertext);
serde_json::to_string(&LocalSseDekEnvelope {
version: LOCAL_SSE_DEK_FORMAT_VERSION,
nonce: &nonce,
ciphertext: &ciphertext,
})
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to serialize encrypted DEK: {e}"))))
// nonce:ciphertext
Ok(format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext)))
}
// Simple decryption of DEK
pub(crate) fn decrypt_dek(encrypted_dek: &str, cmk_value: [u8; 32]) -> Result<[u8; 32], ApiError> {
let envelope = serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek);
let (nonce, ciphertext) = match envelope {
Ok(envelope) => {
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
return Err(ApiError::from(StorageError::other(format!(
"Unsupported encrypted DEK format version: {}",
envelope.version
))));
}
(envelope.nonce, envelope.ciphertext)
}
Err(json_error) if encrypted_dek.trim_start().starts_with('{') => {
return Err(ApiError::from(StorageError::other(format!(
"Invalid encrypted DEK JSON format: {json_error}"
))));
}
Err(_) => {
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format")));
};
if ciphertext.contains(':') {
return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format")));
}
(nonce, ciphertext)
}
};
let parts: Vec<&str> = encrypted_dek.split(':').collect();
if parts.len() != 2 {
return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format")));
}
let nonce_vec = BASE64_STANDARD
.decode(nonce)
.decode(parts[0])
.map_err(|_| ApiError::from(StorageError::other("Invalid nonce format")))?;
let ciphertext = BASE64_STANDARD
.decode(ciphertext)
.decode(parts[1])
.map_err(|_| ApiError::from(StorageError::other("Invalid ciphertext format")))?;
let key = Key::<Aes256Gcm>::from(cmk_value);
@@ -2640,10 +2596,10 @@ mod tests {
SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material,
apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata,
reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption,
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
validate_ssec_params, verify_ssec_key_match,
map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata, reset_sse_dek_provider,
resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata,
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params,
verify_ssec_key_match,
};
#[cfg(feature = "rio-v2")]
use super::{
@@ -2666,15 +2622,6 @@ mod tests {
assert!(super::parse_simple_sse_cmk(&zero).is_err());
}
#[test]
fn kms_operation_errors_preserve_retryability_classification() {
let unavailable = kms_operation_error(rustfs_kms::KmsError::backend_error("connection refused"));
let corrupt = kms_operation_error(rustfs_kms::KmsError::cryptographic_error("decrypt", "authentication failed"));
assert_eq!(unavailable.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(corrupt.code, S3ErrorCode::InternalError);
}
#[test]
fn parse_simple_sse_cmk_accepts_valid_32_byte_key() {
let mut key = [0u8; 32];
@@ -4076,25 +4023,17 @@ mod tests {
}
#[test]
fn test_encrypt_dek_writes_json_with_random_nonces() {
fn test_encrypt_dek_uses_random_nonce_prefixes() {
let dek = [0x11u8; 32];
let cmk = [0x22u8; 32];
let encrypted_a = TestSseDekProvider::encrypt_dek(dek, cmk).expect("first DEK wrap should succeed");
let encrypted_b = TestSseDekProvider::encrypt_dek(dek, cmk).expect("second DEK wrap should succeed");
let envelope_a: super::LocalSseDekEnvelope =
serde_json::from_str(&encrypted_a).expect("first wrapped DEK should be a JSON envelope");
let envelope_b: super::LocalSseDekEnvelope =
serde_json::from_str(&encrypted_b).expect("second wrapped DEK should be a JSON envelope");
let nonce_a = encrypted_a.split(':').next().expect("wrapped DEK should contain nonce");
let nonce_b = encrypted_b.split(':').next().expect("wrapped DEK should contain nonce");
assert_eq!(envelope_a.version, super::LOCAL_SSE_DEK_FORMAT_VERSION);
assert_eq!(envelope_b.version, super::LOCAL_SSE_DEK_FORMAT_VERSION);
assert_ne!(envelope_a.nonce, envelope_b.nonce, "each DEK wrap should use a distinct nonce");
assert_eq!(
TestSseDekProvider::decrypt_dek(&encrypted_a, cmk).expect("first JSON envelope should decrypt"),
dek
);
assert_ne!(nonce_a, nonce_b, "each DEK wrap should use a distinct nonce prefix");
}
#[test]
@@ -4112,20 +4051,6 @@ mod tests {
assert_eq!(decrypted, dek);
}
#[test]
fn test_decrypt_dek_rejects_unknown_json_version() {
let envelope = serde_json::json!({
"version": super::LOCAL_SSE_DEK_FORMAT_VERSION + 1,
"nonce": BASE64_STANDARD.encode([0u8; 12]),
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
})
.to_string();
let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32])
.expect_err("unknown JSON envelope versions must fail closed");
assert!(error.message.contains("Unsupported encrypted DEK format version"));
}
#[tokio::test]
async fn test_sse_encryption_fails_closed_without_local_sse_master_key() {
let _guard = lock_sse_test_state().await;
+3 -28
View File
@@ -430,9 +430,9 @@ pub(crate) mod ecstore_data_usage {
pub(crate) mod ecstore_disk {
pub(crate) use rustfs_ecstore::api::disk::{
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
get_object_disk_read_timeout, validate_batch_read_version_item_count,
FileInfoVersions, FileReader, FileWriter, OldCurrentSize, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout,
validate_batch_read_version_item_count,
};
pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
}
@@ -629,7 +629,6 @@ pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq;
pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp;
pub(crate) type ReadOptions = ecstore_disk::ReadOptions;
pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize;
pub(crate) type PartTransactionAction = ecstore_disk::PartTransactionAction;
pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp;
pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType;
pub(crate) type ReplicationStats = StorageReplicationStatsHandle;
@@ -1098,15 +1097,6 @@ pub(crate) trait StorageDiskRpcExt {
dst_path: &str,
meta: bytes::Bytes,
) -> DiskResult<()>;
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: bytes::Bytes,
) -> DiskResult<()>;
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()>;
async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>;
async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult<CheckPartsResp>;
async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult<CheckPartsResp>;
@@ -1251,21 +1241,6 @@ where
ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await
}
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: bytes::Bytes,
) -> DiskResult<()> {
ecstore_disk::DiskAPI::prepare_part_transaction(self, src_volume, src_path, dst_volume, dst_path, meta).await
}
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()> {
ecstore_disk::DiskAPI::settle_part_transaction(self, volume, path, action).await
}
async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> {
ecstore_disk::DiskAPI::delete(self, volume, path, options).await
}
+1 -1
View File
@@ -90,7 +90,7 @@ their issue closes.
| `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — |
| `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — |
| `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — |
| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence with strict/baseline/blocked result classification | `test_manual_transition_runbooks.sh` |
| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence | `test_manual_transition_runbooks.sh` |
| `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — |
| `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — |
| `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — |
+6 -88
View File
@@ -13,14 +13,7 @@ fi
classify_source_layer() {
local file="$1"
if [[ "$file" == rustfs/src/server/* ]] ||
[[ "$file" == rustfs/src/startup_*.rs ]] ||
[[ "$file" == rustfs/src/init.rs ]] ||
[[ "$file" == rustfs/src/main.rs ]] ||
[[ "$file" == rustfs/src/lib.rs ]] ||
[[ "$file" == rustfs/src/embedded.rs ]]; then
printf 'composition'
elif [[ "$file" == rustfs/src/app/* ]]; then
if [[ "$file" == rustfs/src/app/* ]]; then
printf 'app'
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]]; then
printf 'interface'
@@ -37,14 +30,6 @@ classify_target_layer() {
local storage_path
case "$root" in
init | main | lib | embedded | startup_*)
printf 'composition'
;;
server)
# Server files are composition roots when they import lower layers, but
# their exported HTTP contracts belong to the interface boundary.
printf 'interface'
;;
app)
printf 'app'
;;
@@ -68,9 +53,6 @@ classify_target_layer() {
layer_rank() {
case "$1" in
composition)
printf '4'
;;
interface)
printf '3'
;;
@@ -86,48 +68,6 @@ layer_rank() {
esac
}
is_reverse_dependency() {
local source_rank target_rank
source_rank="$(layer_rank "$1")"
target_rank="$(layer_rank "$2")"
(( source_rank < target_rank ))
}
assert_dependency_direction() {
local expected="$1"
local source="$2"
local target="$3"
local actual='allowed'
if is_reverse_dependency "$source" "$target"; then
actual='reverse'
fi
if [[ "$actual" != "$expected" ]]; then
printf 'Layer dependency guard self-test failed: %s -> %s (expected %s, got %s)\n' \
"$source" "$target" "$expected" "$actual" >&2
exit 1
fi
}
run_layer_model_self_tests() {
local server_source app_source storage_source server_target admin_target app_target
server_source="$(classify_source_layer rustfs/src/server/http.rs)"
app_source="$(classify_source_layer rustfs/src/app/bucket_usecase.rs)"
storage_source="$(classify_source_layer rustfs/src/storage/rpc/node_service.rs)"
server_target="$(classify_target_layer server::http)"
admin_target="$(classify_target_layer admin::router)"
app_target="$(classify_target_layer app::bucket_usecase)"
assert_dependency_direction 'allowed' "$server_source" "$admin_target"
assert_dependency_direction 'allowed' "$server_source" "$app_target"
assert_dependency_direction 'reverse' "$app_source" "$server_target"
assert_dependency_direction 'reverse' "$storage_source" "$server_target"
assert_dependency_direction 'reverse' "$app_source" "$admin_target"
assert_dependency_direction 'reverse' "$storage_source" "$admin_target"
}
normalize_import_group_item() {
local prefix="$1"
local item="$2"
@@ -242,11 +182,6 @@ normalize_import_path() {
emit_crate_use_statements() {
(cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do
# The guard is file-scoped: dedicated test modules are excluded, while
# inline #[cfg(test)] imports remain subject to the source file's layer.
if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then
continue
fi
perl -0777 -ne '
while (/\buse\s+crate::.*?;/sg) {
my $statement = $&;
@@ -259,26 +194,6 @@ emit_crate_use_statements() {
done)
}
write_baseline_file() {
local entries="$1"
cat >"$BASELINE_FILE" <<'EOF'
# Layer dependency baseline for the rustfs binary crate.
#
# The guard models production imports as:
# composition -> interface -> app -> infra
#
# Canonical dependency entry:
# dep|source_file|source_layer->target_layer|crate::imported_symbol
#
# Canonical conceptual cycle entry:
# cycle|left_layer<->right_layer
EOF
cat "$entries" >>"$BASELINE_FILE"
}
run_layer_model_self_tests
normalize_baseline_file() {
local input="$1"
local output="$2"
@@ -350,7 +265,10 @@ while IFS= read -r line; do
printf '%s->%s\n' "$source_layer" "$target_layer" >>"$EDGES_RAW"
fi
if is_reverse_dependency "$source_layer" "$target_layer"; then
source_rank="$(layer_rank "$source_layer")"
target_rank="$(layer_rank "$target_layer")"
if (( source_rank < target_rank )); then
printf 'dep|%s|%s->%s|crate::%s\n' "$file" "$source_layer" "$target_layer" "$import_path" >>"$VIOLATIONS_RAW"
fi
done < <(normalize_import_path "$text")
@@ -374,7 +292,7 @@ done <"${TMP_DIR}/edges_sorted.txt" | sort -u >"${TMP_DIR}/cycles_sorted.txt"
cat "${TMP_DIR}/violations_sorted.txt" "${TMP_DIR}/cycles_sorted.txt" | sort -u >"$CURRENT_BASELINE"
if [[ "$MODE" == "update" ]]; then
write_baseline_file "$CURRENT_BASELINE"
cp "$CURRENT_BASELINE" "$BASELINE_FILE"
echo "Updated baseline: $BASELINE_FILE"
exit 0
fi
+20 -29
View File
@@ -1,44 +1,35 @@
# Layer dependency baseline for the rustfs binary crate.
#
# The guard models production imports as:
# composition -> interface -> app -> infra
# These are intra-crate module references within rustfs/ that cross the
# conceptual layer boundaries (app, infra/server, interface/admin).
# Since they live inside one Cargo crate, Rust doesn't enforce separation.
# The list is maintained for architectural awareness during code review.
#
# Canonical dependency entry:
# dep|source_file|source_layer->target_layer|crate::imported_symbol
# Format for dependency entries:
# status|source_file|direction|imported_symbol|reason
#
# Canonical conceptual cycle entry:
# cycle|left_layer<->right_layer
# Format for conceptual cycles:
# status|cycle|direction_pair|reason
#
# Status:
# accepted - reviewed and intentionally allowed
# todo - should be resolved in a future refactor
cycle|app<->infra
cycle|app<->interface
cycle|composition<->infra
cycle|composition<->interface
cycle|infra<->interface
dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::ENV_SCANNER_ENABLED
dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::scanner_enabled_from_env
dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::DependencyReadiness
dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::collect_dependency_readiness_report
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::server::RemoteAddr
dep|rustfs/src/app/object_usecase.rs|app->interface|crate::server::convert_ecstore_object_info
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::DependencyReadiness
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::DependencyReadinessReport
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::ReadinessDegradedReason
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::snapshot_dependency_readiness_report
dep|rustfs/src/init.rs|infra->interface|crate::admin
dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context
dep|rustfs/src/storage/access.rs|infra->interface|crate::server::RemoteAddr
dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::server::cors
dep|rustfs/src/server/http.rs|infra->interface|crate::admin
dep|rustfs/src/server/layer.rs|infra->interface|crate::admin::console::is_console_path
dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::storage::ecfs::ListObjectUnorderedQuery
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::convert_ecstore_object_info
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_audit_module_enabled
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_notify_module_enabled
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_audit_module_enabled
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_notify_module_enabled
dep|rustfs/src/storage/rpc/http_service.rs|infra->interface|crate::server::RPC_PREFIX
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::ecfs::FS
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::ecfs::validate_object_lock_configuration_input
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::s3_api::common::rustfs_initiator
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::s3_api::common::rustfs_owner
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_dynamic_config_runtime_state
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_runtime_config_snapshot
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::site_replication::reload_site_replication_runtime_state
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM
dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::heal_enabled_from_env
dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::scanner_enabled_from_env
@@ -25,13 +25,9 @@ COLD_IMAGE="${COLD_IMAGE:-${NEW_IMAGE}}"
BASE_PORT="${BASE_PORT:-19400}"
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/manual-transition-1508-docker/$(date +%Y%m%dT%H%M%S)}"
KEEP_UP=false
ROLLBACK_PHASE="${ROLLBACK_PHASE:-after-terminal}"
OLD_NODE_PHASE="${OLD_NODE_PHASE:-initial}"
ROLLBACK_NEW2_TO_OLD=true
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
POLL_SECONDS="${POLL_SECONDS:-180}"
TRANSITION_WORKERS="${TRANSITION_WORKERS:-2}"
TRANSITION_QUEUE_CAPACITY="${TRANSITION_QUEUE_CAPACITY:-64}"
FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT:-false}"
HOT_ACCESS_KEY="${HOT_ACCESS_KEY:-mvadmin}"
HOT_SECRET_KEY="${HOT_SECRET_KEY:-mvsecret}"
@@ -62,31 +58,18 @@ Options:
--object-count <n> Non-empty probe object count
--tier <name> Remote tier name
--keep-up Leave Docker services running
--rollback-phase <phase> Rollback node2 timing: after-terminal, in-flight, none
--old-node-phase <phase> Old node1 timing: initial, before-job
--no-rollback Alias for --rollback-phase none
--no-rollback Do not replace node2 with old image after job admission
-h, --help Show help
Environment:
PROJECT_NAME OLD_IMAGE NEW_IMAGE COLD_IMAGE BASE_PORT OUT_DIR KEEP_UP
HOT_ACCESS_KEY HOT_SECRET_KEY COLD_ACCESS_KEY COLD_SECRET_KEY
TIER_NAME TIER_BUCKET TIER_PREFIX JOB_BUCKET JOB_PREFIX OBJECT_COUNT
ROLLBACK_PHASE WAIT_TIMEOUT_SECS POLL_SECONDS TRANSITION_WORKERS
TRANSITION_QUEUE_CAPACITY OLD_NODE_PHASE FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT AWS_SIGV4_SCOPE
WAIT_TIMEOUT_SECS POLL_SECONDS AWS_SIGV4_SCOPE
Artifacts:
compose.yml, image inspect files, health/readiness logs, API responses,
terminal status, old-node readback, container logs, summary.env.
Result classifications:
strict_mixed_rollout_pass Real old/new images, non-empty completed transition, zero failures
baseline_tiered_storage_pass Same old/new image completed transition; useful baseline, not #1508 closure
blocked_manual_api_not_implemented Manual transition API returned 501 before job admission
blocked_manual_api_unavailable Manual transition API did not return a usable job_id
blocked_cluster_readiness_failed Docker cluster did not reach health/readiness before admission
blocked_empty_scan_or_lifecycle Job completed without lifecycle-matching transition work
blocked_manual_job_preempted_by_lifecycle_queue Lifecycle/immediate transition queued work before the job
strict_mixed_rollout_fail Mixed rollout ran but did not satisfy the strict #1508 gate
USAGE
}
@@ -128,28 +111,6 @@ parse_positive_int() {
fi
}
validate_rollback_phase() {
case "$ROLLBACK_PHASE" in
after-terminal|in-flight|none)
;;
*)
log_error "--rollback-phase must be one of: after-terminal, in-flight, none"
exit 1
;;
esac
}
validate_old_node_phase() {
case "$OLD_NODE_PHASE" in
initial|before-job)
;;
*)
log_error "--old-node-phase must be one of: initial, before-job"
exit 1
;;
esac
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -189,16 +150,8 @@ parse_args() {
KEEP_UP=true
shift
;;
--rollback-phase)
ROLLBACK_PHASE="$(arg_value "$1" "${2:-}")"
shift 2
;;
--old-node-phase)
OLD_NODE_PHASE="$(arg_value "$1" "${2:-}")"
shift 2
;;
--no-rollback)
ROLLBACK_PHASE=none
ROLLBACK_NEW2_TO_OLD=false
shift
;;
-h|--help)
@@ -245,18 +198,12 @@ cleanup() {
}
write_compose_file() {
local node1_image
COMPOSE_FILE="${OUT_DIR}/compose.yml"
node1_image="$OLD_IMAGE"
if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then
node1_image="$NEW_IMAGE"
fi
cat >"$COMPOSE_FILE" <<EOF
services:
cold:
image: ${COLD_IMAGE}
hostname: cold
user: "0:0"
environment:
- RUSTFS_ADDRESS=:9000
- RUSTFS_ACCESS_KEY=${COLD_ACCESS_KEY}
@@ -275,20 +222,14 @@ services:
- rustfs-1508-net
node1:
image: ${node1_image}
image: ${OLD_IMAGE}
hostname: node1
user: "0:0"
environment: &hot-env
- RUSTFS_ADDRESS=:9000
- RUSTFS_ACCESS_KEY=${HOT_ACCESS_KEY}
- RUSTFS_SECRET_KEY=${HOT_SECRET_KEY}
- RUSTFS_VOLUMES=$(hot_volumes)
- RUSTFS_SCANNER_ENABLED=false
- RUSTFS_SCANNER_CYCLE=3600
- RUSTFS_SCANNER_START_DELAY_SECS=3600
- RUSTFS_MAX_TRANSITION_WORKERS=${TRANSITION_WORKERS}
- RUSTFS_TRANSITION_QUEUE_CAPACITY=${TRANSITION_QUEUE_CAPACITY}
- RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT=${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
- RUSTFS_OBS_LOGGER_LEVEL=warn
volumes:
@@ -304,7 +245,6 @@ services:
node2:
image: ${NEW_IMAGE}
hostname: node2
user: "0:0"
environment: *hot-env
volumes:
- node2_data_0:/data/rustfs0
@@ -319,7 +259,6 @@ services:
node3:
image: ${NEW_IMAGE}
hostname: node3
user: "0:0"
environment: *hot-env
volumes:
- node3_data_0:/data/rustfs0
@@ -334,7 +273,6 @@ services:
node4:
image: ${NEW_IMAGE}
hostname: node4
user: "0:0"
environment: *hot-env
volumes:
- node4_data_0:/data/rustfs0
@@ -495,19 +433,16 @@ seed_objects() {
}
start_transition_job() {
local query response job_id run_http_code
local query response job_id
query="bucket=$(url_encode "$JOB_BUCKET")"
query="${query}&prefix=$(url_encode "${JOB_PREFIX}/")"
query="${query}&tier=$(url_encode "$TIER_NAME")"
query="${query}&dryRun=false&maxObjects=${OBJECT_COUNT}&mode=async"
curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}" \
-o "${OUT_DIR}/run-response.json" \
-w "%{http_code}\n" >"${OUT_DIR}/run-response.http_code" || true
response="$(cat "${OUT_DIR}/run-response.json" 2>/dev/null || true)"
run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)"
response="$(curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}")"
printf '%s\n' "$response" >"${OUT_DIR}/run-response.json"
job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
if [[ -z "$job_id" ]]; then
log_warn "manual transition response omitted job_id, http_code=${run_http_code:-unknown}"
log_error "manual transition response omitted job_id"
return 1
fi
printf '%s\n' "$job_id" >"${OUT_DIR}/job-id.txt"
@@ -516,25 +451,19 @@ start_transition_job() {
replace_node2_with_old_image() {
local network
network="$(network_name)"
log_info "Replacing node2 with old image ${OLD_IMAGE} for ${ROLLBACK_PHASE} rollback readback"
log_info "Replacing node2 with old image ${OLD_IMAGE} for in-flight rollback readback"
compose stop node2 >/dev/null
compose rm -f node2 >/dev/null
docker run -d \
--name "${PROJECT_NAME}-node2-rollback-old" \
--network "$network" \
--network-alias node2 \
--user 0:0 \
-p "$((BASE_PORT + 2)):9000" \
-e RUSTFS_ADDRESS=:9000 \
-e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
-e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
-e RUSTFS_VOLUMES="$(hot_volumes)" \
-e RUSTFS_SCANNER_ENABLED=false \
-e RUSTFS_SCANNER_CYCLE=3600 \
-e RUSTFS_SCANNER_START_DELAY_SECS=3600 \
-e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \
-e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \
-e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
-e RUSTFS_OBS_LOGGER_LEVEL=warn \
-v "${PROJECT_NAME}_node2_data_0:/data/rustfs0" \
@@ -544,37 +473,6 @@ replace_node2_with_old_image() {
"$OLD_IMAGE" >/dev/null
}
replace_node1_with_old_image() {
local network
network="$(network_name)"
log_info "Replacing node1 with old image ${OLD_IMAGE} before manual transition job"
compose stop node1 >/dev/null
compose rm -f node1 >/dev/null
docker run -d \
--name "${PROJECT_NAME}-node1-before-job-old" \
--network "$network" \
--network-alias node1 \
--user 0:0 \
-p "$((BASE_PORT + 1)):9000" \
-e RUSTFS_ADDRESS=:9000 \
-e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
-e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
-e RUSTFS_VOLUMES="$(hot_volumes)" \
-e RUSTFS_SCANNER_ENABLED=false \
-e RUSTFS_SCANNER_CYCLE=3600 \
-e RUSTFS_SCANNER_START_DELAY_SECS=3600 \
-e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \
-e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \
-e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
-e RUSTFS_OBS_LOGGER_LEVEL=warn \
-v "${PROJECT_NAME}_node1_data_0:/data/rustfs0" \
-v "${PROJECT_NAME}_node1_data_1:/data/rustfs1" \
-v "${PROJECT_NAME}_node1_data_2:/data/rustfs2" \
-v "${PROJECT_NAME}_node1_data_3:/data/rustfs3" \
"$OLD_IMAGE" >/dev/null
}
poll_terminal_status() {
local job_id="$1"
local status_url status_json terminal_state
@@ -614,85 +512,6 @@ head_probe() {
-w "%{http_code}\n" >"${OUT_DIR}/head-object.http_code" || true
}
image_id() {
local file="$1"
jq -r '.[0].Id // ""' "$file" 2>/dev/null || true
}
is_compat_readback_code() {
case "$1" in
200|501)
return 0
;;
*)
return 1
;;
esac
}
classify_result() {
local terminal_state="$1"
local transition_completed="$2"
local transition_failed="$3"
local tier_failure="$4"
local old_code="$5"
local rollback_code="$6"
local run_http_code="$7"
local lifecycle_config_found="$8"
local scanned="$9"
local eligible="${10}"
local skipped_already_transitioned="${11}"
local skipped_already_in_flight="${12}"
local old_image_id new_image_id images_are_mixed readback_ok
if [[ -f "${OUT_DIR}/readiness-failed" ]]; then
printf 'blocked_cluster_readiness_failed\n'
return
fi
old_image_id="$(image_id "${OUT_DIR}/old-image.inspect.json")"
new_image_id="$(image_id "${OUT_DIR}/new-image.inspect.json")"
images_are_mixed=false
if [[ "$OLD_IMAGE" != "$NEW_IMAGE" && -n "$old_image_id" && -n "$new_image_id" && "$old_image_id" != "$new_image_id" ]]; then
images_are_mixed=true
fi
readback_ok=false
if is_compat_readback_code "$old_code"; then
if [[ "$ROLLBACK_PHASE" == "none" ]] || is_compat_readback_code "$rollback_code"; then
readback_ok=true
fi
fi
if [[ "$run_http_code" == "501" ]]; then
printf 'blocked_manual_api_not_implemented\n'
return
fi
if [[ -z "$(cat "${OUT_DIR}/job-id.txt" 2>/dev/null || true)" ]]; then
printf 'blocked_manual_api_unavailable\n'
return
fi
if [[ "$terminal_state" == "completed" && ( "$lifecycle_config_found" != "true" || "$scanned" == "0" || "$eligible" == "0" || "$transition_completed" == "0" ) ]]; then
printf 'blocked_empty_scan_or_lifecycle\n'
return
fi
if [[ "$transition_completed" == "0" && "$eligible" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then
if [[ "$skipped_already_transitioned" != "0" || "$skipped_already_in_flight" != "0" ]]; then
printf 'blocked_manual_job_preempted_by_lifecycle_queue\n'
return
fi
fi
if [[ "$terminal_state" == "completed" && "$transition_completed" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then
if [[ "$images_are_mixed" == "true" && "$readback_ok" == "true" ]]; then
printf 'strict_mixed_rollout_pass\n'
return
fi
printf 'baseline_tiered_storage_pass\n'
return
fi
printf 'strict_mixed_rollout_fail\n'
}
collect_logs() {
local service
if [[ -z "$COMPOSE_FILE" || ! -f "$COMPOSE_FILE" ]]; then
@@ -701,63 +520,37 @@ collect_logs() {
for service in cold node1 node2 node3 node4; do
compose logs --no-color "$service" >"${OUT_DIR}/${service}.log" 2>/dev/null || true
done
docker logs "${PROJECT_NAME}-node1-before-job-old" >"${OUT_DIR}/node1-before-job-old.log" 2>/dev/null || true
docker logs "${PROJECT_NAME}-node2-rollback-old" >"${OUT_DIR}/node2-rollback-old.log" 2>/dev/null || true
}
summarize() {
local terminal_state transition_completed tier_failure transition_failed old_code rollback_code run_http_code lifecycle_config_found scanned eligible skipped_already_transitioned skipped_already_in_flight queue_queued queue_active result_classification
local terminal_state transition_completed tier_failure transition_failed old_code rollback_code
terminal_state="$(cat "${OUT_DIR}/terminal-state.txt" 2>/dev/null || true)"
transition_completed="$(jq -r '.report.transition_completed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
tier_failure="$(jq -r '.report.tier_failure // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
transition_failed="$(jq -r '.report.transition_failed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
old_code="$(cat "${OUT_DIR}/old-node-status.http_code" 2>/dev/null || true)"
rollback_code="$(cat "${OUT_DIR}/rollback-node2-status.http_code" 2>/dev/null || true)"
run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)"
lifecycle_config_found="$(jq -r '.report.lifecycle_config_found // false' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf 'false')"
scanned="$(jq -r '.report.scanned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
eligible="$(jq -r '.report.eligible // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
skipped_already_transitioned="$(jq -r '.report.skipped_already_transitioned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
skipped_already_in_flight="$(jq -r '.report.skipped_already_in_flight // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
queue_queued="$(jq -r '.queue_snapshot.queued // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
queue_active="$(jq -r '.queue_snapshot.active // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
result_classification="$(classify_result "$terminal_state" "$transition_completed" "$transition_failed" "$tier_failure" "$old_code" "$rollback_code" "$run_http_code" "$lifecycle_config_found" "$scanned" "$eligible" "$skipped_already_transitioned" "$skipped_already_in_flight")"
cat >"${OUT_DIR}/summary.env" <<EOF
project_name=${PROJECT_NAME}
old_image=${OLD_IMAGE}
new_image=${NEW_IMAGE}
cold_image=${COLD_IMAGE}
old_image_id=$(image_id "${OUT_DIR}/old-image.inspect.json")
new_image_id=$(image_id "${OUT_DIR}/new-image.inspect.json")
base_port=${BASE_PORT}
tier=${TIER_NAME}
job_bucket=${JOB_BUCKET}
job_prefix=${JOB_PREFIX}
object_count=${OBJECT_COUNT}
transition_workers=${TRANSITION_WORKERS}
transition_queue_capacity=${TRANSITION_QUEUE_CAPACITY}
force_immediate_transition_enqueue_timeout=${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}
run_http_code=${run_http_code}
terminal_state=${terminal_state}
lifecycle_config_found=${lifecycle_config_found}
scanned=${scanned}
eligible=${eligible}
skipped_already_transitioned=${skipped_already_transitioned}
skipped_already_in_flight=${skipped_already_in_flight}
transition_completed=${transition_completed}
transition_failed=${transition_failed}
tier_failure=${tier_failure}
queue_queued=${queue_queued}
queue_active=${queue_active}
old_node_status_http_code=${old_code}
rollback_node2_status_http_code=${rollback_code}
rollback_phase=${ROLLBACK_PHASE}
old_node_phase=${OLD_NODE_PHASE}
rollback_new2_to_old=$([[ "$ROLLBACK_PHASE" == "none" ]] && printf 'false' || printf 'true')
result_classification=${result_classification}
rollback_new2_to_old=${ROLLBACK_NEW2_TO_OLD}
EOF
cat "${OUT_DIR}/summary.env"
if [[ "$result_classification" != "strict_mixed_rollout_pass" ]]; then
if [[ "$terminal_state" != "completed" || "$transition_completed" == "0" || "$transition_failed" != "0" || "$tier_failure" != "0" ]]; then
log_error "strict #1508 mixed-version transition gate failed; see ${OUT_DIR}"
return 1
fi
@@ -767,8 +560,6 @@ main() {
parse_args "$@"
parse_positive_int "--base-port" "$BASE_PORT"
parse_positive_int "--object-count" "$OBJECT_COUNT"
validate_rollback_phase
validate_old_node_phase
require_cmd docker
require_cmd curl
require_cmd jq
@@ -783,37 +574,20 @@ main() {
docker image inspect "$COLD_IMAGE" >"${OUT_DIR}/cold-image.inspect.json"
compose up -d
if ! wait_cluster_ready; then
printf 'cluster readiness failed before manual transition admission\n' >"${OUT_DIR}/readiness-failed"
collect_logs
summarize
return 1
fi
wait_cluster_ready
curl_cold PUT "$(cold_endpoint)/${TIER_BUCKET}" -o "${OUT_DIR}/create-cold-bucket.response" -w "%{http_code}\n" >"${OUT_DIR}/create-cold-bucket.http_code"
create_bucket "$(hot_endpoint 2)" "$JOB_BUCKET" "${OUT_DIR}/create-hot-bucket"
add_tier
seed_objects
put_lifecycle
if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then
replace_node1_with_old_image
wait_http_ok "$(hot_endpoint 1)/health" "node1-old-live" || true
wait_http_ok "$(hot_endpoint 1)/health/ready" "node1-old-ready" || true
fi
if start_transition_job; then
if [[ "$ROLLBACK_PHASE" == "in-flight" ]]; then
replace_node2_with_old_image
fi
poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")" || true
if [[ "$ROLLBACK_PHASE" == "after-terminal" ]]; then
replace_node2_with_old_image
wait_http_ok "$(hot_endpoint 2)/health" "rollback-node2-live" || true
fi
capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
head_probe
else
log_warn "Skipping terminal polling because no manual transition job was admitted"
seed_objects
start_transition_job
if [[ "$ROLLBACK_NEW2_TO_OLD" == "true" ]]; then
replace_node2_with_old_image
fi
poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")"
capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
head_probe
collect_logs
summarize
}
@@ -208,16 +208,7 @@ bash "$MIXED_DOCKER_HARNESS" --help >/tmp/manual_transition_mixed_version_docker
rg -q "mixed_version_docker_harness" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q -- "--old-image" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q -- "--new-image" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q -- "--rollback-phase" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q -- "--old-node-phase" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q -- "--no-rollback" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "strict_mixed_rollout_pass" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "baseline_tiered_storage_pass" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "blocked_manual_api_not_implemented" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "blocked_cluster_readiness_failed" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "blocked_manual_job_preempted_by_lifecycle_queue" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "OLD_NODE_PHASE" /tmp/manual_transition_mixed_version_docker_harness.help
rg -q "FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" /tmp/manual_transition_mixed_version_docker_harness.help
if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then
echo "failure samples script should fail when --sample has no value" >&2
exit 1