mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-29 09:38:59 +00:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| debd664016 | |||
| 957080bea5 | |||
| c1538cf1c3 | |||
| 5af56cbb02 | |||
| f99956eade | |||
| 775279b6fd | |||
| eb755e2b97 | |||
| 7f146fc5de | |||
| 5426237a49 | |||
| d7afa4e38e | |||
| a3f5a8eaf9 | |||
| 547c678eed | |||
| df945b275a | |||
| 2e78a49c95 | |||
| 7ad0e726db | |||
| 45c3386b68 | |||
| 7af92b4f54 | |||
| daa627ee0e | |||
| 5c3d3a8220 | |||
| 6bc5fc77b5 | |||
| e822fc1552 | |||
| 2f6115e058 | |||
| 882e1a71b1 | |||
| b432f31c2c | |||
| 6e0640444e | |||
| 4fb9b0dc7f | |||
| 2216f00cfd | |||
| fd2a87d47e | |||
| 1e10d752b9 | |||
| 362f6026ac |
@@ -10,10 +10,16 @@ never weaken a check to get green.
|
||||
|
||||
## `check_layer_dependencies.sh` — layer DAG in `rustfs/src`
|
||||
|
||||
Enforces `interface (admin, storage/ecfs, storage/s3_api) → app → infra`; no
|
||||
upward imports. Known legacy violations live in
|
||||
Enforces `composition (server, startup/init) → interface (admin,
|
||||
storage/ecfs, storage/s3_api) → app → infra`; no upward imports. Server source
|
||||
files are composition roots, while imports of their exported HTTP contracts
|
||||
are classified as interface dependencies. Known legacy violations live in
|
||||
`scripts/layer-dependency-baseline.txt`.
|
||||
|
||||
Dedicated `*_test.rs` and `tests/` modules are outside this production guard.
|
||||
Inline `#[cfg(test)]` imports remain checked under their source file's layer;
|
||||
move architecture-crossing test scaffolding into a dedicated test module.
|
||||
|
||||
- **New violation**: restructure your change so the dependency points
|
||||
downward (move the shared type/function to the lower layer).
|
||||
- **You legitimately removed a baseline entry**: run
|
||||
|
||||
+9
-14
@@ -1,17 +1,14 @@
|
||||
# nextest configuration for RustFS.
|
||||
#
|
||||
# Serialize two known load-sensitive / global-state-sharing ecstore test groups
|
||||
# so the full parallel nextest suite stops producing spurious failures
|
||||
# (backlog #937). These tests pass in isolation but flake under the loaded
|
||||
# parallel run for two distinct reasons:
|
||||
# Serialize the ecstore tests that share the process-wide disk registry or
|
||||
# exercise a multi-disk commit handoff across nextest process boundaries.
|
||||
#
|
||||
# * store::bucket::tests::bucket_delete_* share process/global state (disk
|
||||
# registry, lock client) and race make_bucket into InsufficientWriteQuorum
|
||||
# when run concurrently with other ecstore tests.
|
||||
# * bucket_lifecycle_ops::tests::concurrent_resend_same_part_commits_one_generation
|
||||
# asserts a lock-acquire correctness property whose serialized cross-disk
|
||||
# commits exceed the (already max'd, 60s) acquire deadline only when the
|
||||
# suite saturates disk I/O.
|
||||
# uses the shared multipart fixture and a deterministic uploadId-lock
|
||||
# handoff, so it must not overlap another process mutating that fixture.
|
||||
#
|
||||
# serial_test's #[serial] attribute does NOT serialize these across runs:
|
||||
# nextest executes each test in its own process, where the in-process
|
||||
@@ -100,13 +97,6 @@ path = "junit.xml"
|
||||
# profile's own overrides list, not the default profile's).
|
||||
# ===========================================================================
|
||||
|
||||
# QUARANTINE: OPEN backlog#937 — concurrent_resend lock-acquire deadline flakes
|
||||
# under saturated disk I/O in the full parallel suite.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
retries = 2
|
||||
|
||||
# QUARANTINE: OPEN backlog#937 — store::bucket::tests::bucket_delete_* race
|
||||
# make_bucket into InsufficientWriteQuorum via shared global state under load.
|
||||
[[profile.ci.overrides]]
|
||||
@@ -114,6 +104,11 @@ filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
retries = 2
|
||||
|
||||
# Keep the deterministic multipart handoff isolated across nextest processes.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# QUARANTINE: OPEN rustfs#4690 — walk_dir stall-budget accounting test depends
|
||||
# on producer/consumer timing windows that stretch past the budget on loaded
|
||||
# CI runners (regression test for rustfs#4644; failed on a zero-Rust-diff PR).
|
||||
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
name: Test and Lint
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
|
||||
Vendored
+25
-2
@@ -172,7 +172,7 @@
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Debug executable target/debug/rustfs with sse",
|
||||
"name": "Debug executable target/debug/rustfs with sse kms",
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/target/debug/rustfs",
|
||||
@@ -200,7 +200,7 @@
|
||||
// 2. kms local backend test key
|
||||
// "RUSTFS_KMS_ENABLE": "true",
|
||||
// "RUSTFS_KMS_BACKEND": "local",
|
||||
// "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
|
||||
// "RUSTFS_KMS_KEY_DIR": "/tmp/kms-key-dir",
|
||||
// "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
|
||||
// "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
|
||||
|
||||
@@ -228,6 +228,29 @@
|
||||
"rust"
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Debug executable target/debug/rustfs with local sse",
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/target/debug/rustfs",
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}",
|
||||
"env": {
|
||||
"RUSTFS_ACCESS_KEY": "rustfsadmin",
|
||||
"RUSTFS_SECRET_KEY": "rustfsadmin",
|
||||
"RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
|
||||
"RUSTFS_ADDRESS": ":9000",
|
||||
"RUSTFS_CONSOLE_ENABLE": "true",
|
||||
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
|
||||
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
|
||||
"RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
|
||||
"RUSTFS_SSE_S3_MASTER_KEY": "xGb3aYSp825j2tPpg8JrUzghiXsIkfdOtmrsJ/iafiM=",
|
||||
"RUST_LOG": "rustfs=debug,ecstore=debug,s3s=debug,iam=debug",
|
||||
},
|
||||
"sourceLanguages": [
|
||||
"rust"
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "lldb",
|
||||
"request": "launch",
|
||||
|
||||
Generated
+26
-22
@@ -1703,9 +1703,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "camino"
|
||||
version = "1.2.4"
|
||||
version = "1.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0"
|
||||
checksum = "bb1307f12aa967b5a58416e87b3653360e0fd614a016b6e970db08fecbb1b80d"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -3625,13 +3625,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.6"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3946,11 +3946,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "5.4.1"
|
||||
version = "5.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
|
||||
checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2"
|
||||
dependencies = [
|
||||
"concurrent-queue",
|
||||
"parking",
|
||||
"pin-project-lite",
|
||||
]
|
||||
@@ -6110,9 +6109,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique"
|
||||
version = "0.1.28"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa466af30a9fe0b1db1dae097e0ce7ddac4b666b1d3a9f5c682e889272511dd8"
|
||||
checksum = "d2e394c63e2d1a30aeb3b9392ecf3439d8475d2df810a8f4f6e66d6866754017"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"jiff",
|
||||
@@ -6140,9 +6139,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-macro"
|
||||
version = "0.1.19"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f5febfaf14fea234b60e0ad43c728db274033893a38d0fa1f87d9b7056d3d61"
|
||||
checksum = "786df1fd0abebd0db685f7e9a353c78756d4b370fb98a52376c2015fa55f141f"
|
||||
dependencies = [
|
||||
"Inflector",
|
||||
"darling 0.23.0",
|
||||
@@ -6169,9 +6168,9 @@ checksum = "2faca4e4480069ff02b1763b3b79f5cec7e8628e24d9dc5b6073f53d2577a4d9"
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "124326a2ac4c4f61562fa4d071735a1c463f9ba0317d1564f75dc01313dc12d7"
|
||||
checksum = "82cdde44d241dab7fc8b7a32e0eb5dae6cd28f8de80b59f9a1e9f2f0b05e485e"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"crossbeam-queue",
|
||||
@@ -6190,9 +6189,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer-core"
|
||||
version = "0.1.18"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55b5bbb6d88bde29f6ed74a574cbe49e51ea0c36cccc7e63eee2040db5675b15"
|
||||
checksum = "e57379b7ee2272efaeaaa6de062503563e57333b24aadc7f2255b3d602899e8b"
|
||||
dependencies = [
|
||||
"derive-where",
|
||||
"itertools 0.14.0",
|
||||
@@ -9477,6 +9476,8 @@ name = "rustfs-lifecycle"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"proptest",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
@@ -9714,6 +9715,7 @@ dependencies = [
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"ipnetwork",
|
||||
"libunftp",
|
||||
"md5",
|
||||
"percent-encoding",
|
||||
@@ -9730,7 +9732,9 @@ dependencies = [
|
||||
"rustfs-policy",
|
||||
"rustfs-rio",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-tls-runtime",
|
||||
"rustfs-trusted-proxies",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"s3s",
|
||||
@@ -10351,7 +10355,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.14.1"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=8136db4ac8253c0ccfd86f8216e00e0235747757#8136db4ac8253c0ccfd86f8216e00e0235747757"
|
||||
source = "git+https://github.com/cxymds/s3s.git?rev=fe3941d91fa1c69956f209a9145995c9f0235bff#fe3941d91fa1c69956f209a9145995c9f0235bff"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -10457,9 +10461,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.1"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
|
||||
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
@@ -10692,7 +10696,7 @@ dependencies = [
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.14.0",
|
||||
"schemars 0.9.0",
|
||||
"schemars 1.2.1",
|
||||
"schemars 1.2.2",
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
"serde_with_macros",
|
||||
@@ -11872,9 +11876,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
|
||||
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
|
||||
dependencies = [
|
||||
"winnow",
|
||||
]
|
||||
|
||||
+1
-1
@@ -288,7 +288,7 @@ redis = { version = "1.4.1" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "8136db4ac8253c0ccfd86f8216e00e0235747757" }
|
||||
s3s = { git = "https://github.com/cxymds/s3s.git", rev = "fe3941d91fa1c69956f209a9145995c9f0235bff" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.97-trixie
|
||||
FROM rust:1.97.1-trixie
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ ARG RUSTFS_BUILD_FEATURES=""
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.97-trixie AS builder
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
@@ -45,10 +45,10 @@
|
||||
//! * Parity reconstruction: one data disk is taken offline
|
||||
//! (`take_disk_offline`) and the SAME object matrix is GET both ways while
|
||||
//! the EC 2+2 set rebuilds each large object from the surviving shards. The
|
||||
//! codec-streaming reader gate never inspects drive health, so the codec
|
||||
//! fast path is exercised end-to-end through reconstruction; the test
|
||||
//! asserts byte- and header-equality vs the legacy path AND that the codec
|
||||
//! phase never fell back to a duplex pipe while reconstructing.
|
||||
//! eager first/single-part setup may keep its conservative whole-request
|
||||
//! fallback when shard placement makes codec streaming unsafe, so this phase
|
||||
//! asserts byte- and header-equality vs the legacy path rather than requiring
|
||||
//! zero duplex fallbacks under degraded drive health.
|
||||
//! * Missing object: a GET for an absent key is compared across both phases
|
||||
//! to prove the error semantics (HTTP status + S3 error code) are identical
|
||||
//! — the codec env must not perturb the NoSuchKey negative path.
|
||||
@@ -475,15 +475,14 @@ mod tests {
|
||||
"ranged GET length diverged with codec streaming enabled"
|
||||
);
|
||||
|
||||
// ---- Phase B degraded: the same reconstruction, now on the codec path ----
|
||||
// Re-run the reconstruction A/B with the codec-streaming gates still
|
||||
// open. The reader gate decision is independent of drive health (it
|
||||
// never inspects disk state), so the codec fast path is exercised
|
||||
// end-to-end while the EC set rebuilds each large object from the
|
||||
// surviving shards — this is a real codec-vs-legacy reconstruction test,
|
||||
// not legacy-vs-legacy. Snapshot the duplex count first (the range GET
|
||||
// above already used the duplex path) so we can measure only the markers
|
||||
// these degraded codec GETs add.
|
||||
// ---- Phase B degraded: the same reconstruction, with codec gates open ----
|
||||
// Re-run the reconstruction A/B with codec-streaming enabled. If eager
|
||||
// first/single-part setup cannot prove the codec path is safe for the
|
||||
// surviving shards, the implementation intentionally preserves the
|
||||
// whole-request legacy fallback; later multipart parts can degrade in
|
||||
// place. This phase verifies parity-reconstructed bytes and headers,
|
||||
// while the healthy phase above remains the strict zero-duplex path
|
||||
// confirmation.
|
||||
let dup_codec_before_degraded = count_marker(&codec_log, DUPLEX_MARKER);
|
||||
harness.take_disk_offline(0)?;
|
||||
let mut codec_degraded: BTreeMap<String, GetView> = BTreeMap::new();
|
||||
@@ -511,16 +510,11 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// Path confirmation under reconstruction: the codec fast path must have
|
||||
// served the reconstructed large objects without ever falling back to
|
||||
// the legacy duplex pipe. Without this, the equivalence above could be
|
||||
// legacy-vs-legacy and prove nothing about codec reconstruction.
|
||||
// Keep degraded duplex markers as diagnostic evidence only: eager setup
|
||||
// may fall back before streaming when shard safety cannot be proven.
|
||||
sleep(Duration::from_millis(300)).await;
|
||||
let dup_codec_degraded = count_marker(&codec_log, DUPLEX_MARKER).saturating_sub(dup_codec_before_degraded);
|
||||
assert_eq!(
|
||||
dup_codec_degraded, 0,
|
||||
"codec phase created {dup_codec_degraded} duplex pipe(s) while reconstructing large objects with disk0 offline; the codec fast path was not exercised under degraded reads (see {codec_log})"
|
||||
);
|
||||
info!(dup_codec_degraded, "codec phase degraded-read legacy duplex marker count");
|
||||
|
||||
info!(
|
||||
objects = baseline.len(),
|
||||
|
||||
@@ -520,28 +520,33 @@ async fn assert_msgpack_fallback_unchanged(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_msgpack_decode_observed(
|
||||
collector: &OtlpMetricCollector,
|
||||
before: &BTreeMap<String, u64>,
|
||||
series: &[(&str, &str)],
|
||||
) -> TestResult {
|
||||
async fn assert_msgpack_decode_observed(collector: &OtlpMetricCollector, before: &BTreeMap<String, u64>) -> TestResult {
|
||||
let deadline = Instant::now() + Duration::from_secs(20);
|
||||
loop {
|
||||
let after = collector.msgpack_json_decode_totals().await;
|
||||
let missing = series
|
||||
let missing = [FALLBACK_REQUEST_DIRECTION, FALLBACK_RESPONSE_DIRECTION]
|
||||
.iter()
|
||||
.filter_map(|(direction, message)| {
|
||||
let key = msgpack_decode_metric_key(direction, message, MSGPACK_CODEC_MSGPACK);
|
||||
let before_value = before.get(&key).copied().unwrap_or_default();
|
||||
let after_value = after.get(&key).copied().unwrap_or_default();
|
||||
(after_value <= before_value).then_some(format!("{direction}/{message}/{}", MSGPACK_CODEC_MSGPACK))
|
||||
.filter_map(|direction| {
|
||||
let prefix = format!("{direction}\u{1f}");
|
||||
let suffix = format!("\u{1f}{MSGPACK_CODEC_MSGPACK}");
|
||||
let before_total = before
|
||||
.iter()
|
||||
.filter(|(key, _)| key.starts_with(&prefix) && key.ends_with(&suffix))
|
||||
.map(|(_, value)| *value)
|
||||
.sum::<u64>();
|
||||
let after_total = after
|
||||
.iter()
|
||||
.filter(|(key, _)| key.starts_with(&prefix) && key.ends_with(&suffix))
|
||||
.map(|(_, value)| *value)
|
||||
.sum::<u64>();
|
||||
(after_total <= before_total).then_some(format!("{direction}/{}", MSGPACK_CODEC_MSGPACK))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if missing.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!("timed out waiting for msgpack decode traffic on {missing:?}; totals={after:?}").into());
|
||||
return Err(format!("timed out waiting for msgpack decode traffic for {missing:?}; totals={after:?}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
@@ -1857,7 +1862,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||
)
|
||||
.await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
||||
assert_msgpack_fallback_unchanged(&collector, &fallback_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
|
||||
assert_msgpack_decode_errors_unchanged(&collector, &decode_errors_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
|
||||
|
||||
@@ -1948,8 +1953,8 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
|
||||
hot.start().await?;
|
||||
|
||||
let tier_name = unique_tier_name();
|
||||
hot.stop_node(3)?;
|
||||
let add_tier_response = submit_rustfs_tier(&hot, &cold, &tier_name).await?;
|
||||
hot.stop_node(3)?;
|
||||
hot.start_node(3).await?;
|
||||
|
||||
wait_for_tier_converged(&hot, &tier_name, &add_tier_response).await
|
||||
@@ -2003,7 +2008,7 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
|
||||
assert_eq!(cancel_value["job_id"].as_str(), Some(job_id.as_str()));
|
||||
assert_eq!(cancel_value["bucket"].as_str(), Some(bucket.as_str()));
|
||||
assert_eq!(cancel_value["dry_run"].as_bool(), Some(true));
|
||||
assert_eq!(cancel_value["cancel_requested"].as_bool(), Some(true));
|
||||
assert_eq!(cancel_value["cancel_requested"].as_bool(), Some(false));
|
||||
|
||||
let missing_job_id = Uuid::new_v4();
|
||||
let (missing_status, missing_body) = signed_admin_request(
|
||||
@@ -2325,7 +2330,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), REMOTE, REMOTE_TRANSITION),
|
||||
)
|
||||
.await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before, &MSGPACK_FALLBACK_CONTROL_SERIES).await?;
|
||||
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
||||
|
||||
let encrypted_key = "transition/encrypted-sse.bin";
|
||||
let encrypted_body = payload(16 * KIB, 0xAB);
|
||||
|
||||
@@ -400,6 +400,20 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::PreparePartTransactionRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::PreparePartTransactionResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::SettlePartTransactionRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::SettlePartTransactionResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_file(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameFileRequest>,
|
||||
|
||||
@@ -128,7 +128,7 @@ pub mod bucket {
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -303,9 +303,9 @@ pub mod disk {
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo,
|
||||
WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
|
||||
@@ -1181,8 +1181,12 @@ impl TransitionState {
|
||||
}
|
||||
|
||||
fn new_with_capacity(capacity: usize) -> Arc<Self> {
|
||||
let capacity = capacity.max(1);
|
||||
let queue_send_timeout = resolve_transition_queue_send_timeout();
|
||||
Self::new_with_capacity_and_timeout(capacity, queue_send_timeout)
|
||||
}
|
||||
|
||||
fn new_with_capacity_and_timeout(capacity: usize, queue_send_timeout: StdDuration) -> Arc<Self> {
|
||||
let capacity = capacity.max(1);
|
||||
let (tx1, rx1) = bounded(capacity);
|
||||
Arc::new(Self {
|
||||
transition_tx: tx1,
|
||||
@@ -1249,13 +1253,12 @@ impl TransitionState {
|
||||
return false;
|
||||
}
|
||||
let bucket = bucket.to_string();
|
||||
let scheduled = Arc::clone(&self.compensation_buckets);
|
||||
let state = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
Self::inc_counter(&state.compensation_running_tasks);
|
||||
state.record_scanner_transition_state();
|
||||
let Some(api) = runtime_sources::object_store_handle() else {
|
||||
scheduled.lock().unwrap().remove(&bucket);
|
||||
state.finish_bucket_compensation(&bucket);
|
||||
Self::add_counter(&state.compensation_running_tasks, -1);
|
||||
state.record_scanner_transition_state();
|
||||
debug!(
|
||||
@@ -1291,13 +1294,25 @@ impl TransitionState {
|
||||
);
|
||||
}
|
||||
|
||||
scheduled.lock().unwrap().remove(&bucket);
|
||||
state.finish_bucket_compensation(&bucket);
|
||||
Self::add_counter(&state.compensation_running_tasks, -1);
|
||||
state.record_scanner_transition_state();
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
fn finish_bucket_compensation(&self, bucket: &str) {
|
||||
match self.compensation_buckets.lock() {
|
||||
Ok(mut scheduled) => {
|
||||
scheduled.remove(bucket);
|
||||
}
|
||||
Err(poisoned) => {
|
||||
poisoned.into_inner().remove(bucket);
|
||||
self.compensation_buckets.clear_poison();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn inc_counter(counter: &AtomicI64) {
|
||||
Self::add_counter(counter, 1);
|
||||
@@ -1534,17 +1549,35 @@ impl TransitionState {
|
||||
|
||||
let outcome = match self.transition_tx.try_send(Some(task)) {
|
||||
Ok(()) => TransitionEnqueueOutcome::Queued,
|
||||
Err(async_channel::TrySendError::Full(_)) => {
|
||||
Err(async_channel::TrySendError::Full(task)) => {
|
||||
Self::inc_counter(&self.queue_full_tasks);
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
source = ?src,
|
||||
"transition queue is full; deferring to scanner/backfill"
|
||||
);
|
||||
TransitionEnqueueOutcome::QueueFull
|
||||
let send_timeout = self.transition_queue_send_timeout;
|
||||
match tokio::time::timeout(send_timeout, self.transition_tx.send(task)).await {
|
||||
Ok(Ok(())) => TransitionEnqueueOutcome::Queued,
|
||||
Ok(Err(_)) => {
|
||||
self.schedule_bucket_compensation(&oi.bucket);
|
||||
TransitionEnqueueOutcome::QueueClosed
|
||||
}
|
||||
Err(_) => {
|
||||
Self::inc_counter(&self.queue_send_timeout_tasks);
|
||||
self.schedule_bucket_compensation(&oi.bucket);
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
source = ?src,
|
||||
timeout_ms = send_timeout.as_millis() as u64,
|
||||
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "queue_send_timed_out",
|
||||
"Scanner transition enqueue timed out; scheduled bucket compensation"
|
||||
);
|
||||
TransitionEnqueueOutcome::QueueFull
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(async_channel::TrySendError::Closed(_)) => {
|
||||
self.schedule_bucket_compensation(&oi.bucket);
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
@@ -1729,7 +1762,7 @@ impl TransitionState {
|
||||
}
|
||||
|
||||
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
|
||||
let mut tier_stats = self.last_day_stats.lock().unwrap();
|
||||
let mut tier_stats = self.lock_last_day_stats();
|
||||
tier_stats
|
||||
.entry(tier.to_string())
|
||||
.and_modify(|e| e.add_stats(ts))
|
||||
@@ -1737,7 +1770,7 @@ impl TransitionState {
|
||||
}
|
||||
|
||||
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
|
||||
let tier_stats = self.last_day_stats.lock().unwrap();
|
||||
let tier_stats = self.lock_last_day_stats();
|
||||
let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
|
||||
for (tier, st) in tier_stats.iter() {
|
||||
res.insert(tier.clone(), st.clone());
|
||||
@@ -1745,6 +1778,18 @@ impl TransitionState {
|
||||
res
|
||||
}
|
||||
|
||||
fn lock_last_day_stats(&self) -> std::sync::MutexGuard<'_, HashMap<String, LastDayTierStats>> {
|
||||
match self.last_day_stats.lock() {
|
||||
Ok(stats) => stats,
|
||||
Err(poisoned) => {
|
||||
let mut stats = poisoned.into_inner();
|
||||
stats.clear();
|
||||
self.last_day_stats.clear_poison();
|
||||
stats
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn update_workers(api: Arc<ECStore>, n: i64) {
|
||||
Self::update_workers_inner(api, n).await;
|
||||
}
|
||||
@@ -1766,7 +1811,27 @@ impl TransitionState {
|
||||
fn resize_workers_to(api: Arc<ECStore>, n: i64, requested: i64, absolute_max: i64) {
|
||||
let target = n as usize;
|
||||
let transition_state = runtime_sources::transition_state_handle();
|
||||
let mut workers = transition_state.workers.lock().unwrap();
|
||||
let runtime = match tokio::runtime::Handle::try_current() {
|
||||
Ok(runtime) => runtime,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
error = %err,
|
||||
state = "resize_failed",
|
||||
"Lifecycle worker pool requires a Tokio runtime"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
// Runtime lookup happens before locking, and the guard is dropped before
|
||||
// metrics/logging callbacks. Poison therefore means a Vec mutation may
|
||||
// have unwound and worker tracking cannot be reconstructed safely.
|
||||
let mut workers = transition_state
|
||||
.workers
|
||||
.lock()
|
||||
.expect("transition worker tracking mutex poisoned");
|
||||
let tracked_workers = workers.len();
|
||||
workers.retain(|worker| !worker.handle.is_finished());
|
||||
let pruned_finished_workers = tracked_workers.saturating_sub(workers.len());
|
||||
@@ -1776,7 +1841,7 @@ impl TransitionState {
|
||||
let clone_api = api.clone();
|
||||
let cancel = CancellationToken::new();
|
||||
let worker_cancel = cancel.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
let handle = runtime.spawn(async move {
|
||||
TransitionState::worker_with_cancel(clone_api, worker_cancel).await;
|
||||
});
|
||||
workers.push(TransitionWorker { cancel, handle });
|
||||
@@ -1790,6 +1855,7 @@ impl TransitionState {
|
||||
|
||||
let current_workers = workers.len() as i64;
|
||||
transition_state.num_workers.store(current_workers, Ordering::SeqCst);
|
||||
drop(workers);
|
||||
transition_state.record_scanner_transition_state();
|
||||
|
||||
debug!(
|
||||
@@ -4884,6 +4950,7 @@ mod tests {
|
||||
FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
|
||||
set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
use crate::bucket::metadata_sys;
|
||||
@@ -4917,6 +4984,7 @@ mod tests {
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::metrics::{IlmAction, global_metrics};
|
||||
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, OutputLocation,
|
||||
@@ -6459,6 +6527,96 @@ mod tests {
|
||||
assert_eq!(state.transition_rx.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_transition_enqueue_waits_for_saturated_queue_to_recover() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
let first_object = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "first".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let deferred_object = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "deferred".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
|
||||
"first scanner transition should fill the queue"
|
||||
);
|
||||
|
||||
let deferred = state.queue_transition_task(&deferred_object, &event, &LcEventSrc::Scanner);
|
||||
tokio::pin!(deferred);
|
||||
assert!(
|
||||
(&mut deferred).now_or_never().is_none(),
|
||||
"a saturated scanner queue should apply bounded backpressure instead of dropping the task"
|
||||
);
|
||||
|
||||
let first_task = state
|
||||
.transition_rx
|
||||
.recv()
|
||||
.await
|
||||
.expect("queue should remain open")
|
||||
.expect("first queued transition task should be present");
|
||||
state.release_transition(&first_task.obj_info);
|
||||
|
||||
assert!(
|
||||
deferred.await,
|
||||
"the deferred scanner transition should enqueue as soon as capacity recovers"
|
||||
);
|
||||
let recovered_task = state
|
||||
.transition_rx
|
||||
.recv()
|
||||
.await
|
||||
.expect("queue should remain open")
|
||||
.expect("deferred transition task should be present");
|
||||
assert_eq!(recovered_task.obj_info.name, deferred_object.name);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_transition_sustained_saturation_schedules_compensation() {
|
||||
let state = TransitionState::new_with_capacity_and_timeout(1, StdDuration::ZERO);
|
||||
let first_object = ObjectInfo {
|
||||
bucket: "saturated-bucket".to_string(),
|
||||
name: "first".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let missed_object = ObjectInfo {
|
||||
bucket: "saturated-bucket".to_string(),
|
||||
name: "missed".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
|
||||
"first scanner transition should fill the queue"
|
||||
);
|
||||
assert!(
|
||||
!state
|
||||
.queue_transition_task(&missed_object, &event, &LcEventSrc::Scanner)
|
||||
.await,
|
||||
"a continuously saturated queue should report that the object was not admitted"
|
||||
);
|
||||
assert_eq!(state.queue_full_tasks(), 1);
|
||||
assert_eq!(state.queue_send_timeout_tasks(), 1);
|
||||
assert_eq!(
|
||||
state.compensation_scheduled_tasks(),
|
||||
1,
|
||||
"timed-out scanner work must schedule a bounded bucket backfill"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_transition_enqueue_updates_transition_status() {
|
||||
@@ -6984,6 +7142,46 @@ mod tests {
|
||||
assert_eq!(state.compensation_pending_tasks(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_compensation_set_can_release_completed_bucket() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
state
|
||||
.compensation_buckets
|
||||
.lock()
|
||||
.expect("fresh mutex should lock")
|
||||
.insert("bucket-a".to_string());
|
||||
let poison_target = Arc::clone(&state.compensation_buckets);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poison_target.lock().expect("fresh mutex should lock");
|
||||
panic!("poison compensation set");
|
||||
})
|
||||
.join();
|
||||
|
||||
state.finish_bucket_compensation("bucket-a");
|
||||
assert_eq!(state.compensation_pending_tasks(), 0);
|
||||
assert!(
|
||||
state.compensation_buckets.lock().is_ok(),
|
||||
"validated compensation state must clear poison"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_tier_stats_are_reset_before_reuse() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
let poison_target = Arc::clone(&state.last_day_stats);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let mut stats = poison_target.lock().expect("fresh mutex should lock");
|
||||
stats.insert("stale".to_string(), LastDayTierStats::default());
|
||||
panic!("poison tier stats");
|
||||
})
|
||||
.join();
|
||||
|
||||
state.add_lastday_stats("fresh", TierStats::default());
|
||||
let stats = state.get_daily_all_tier_stats();
|
||||
assert!(!stats.contains_key("stale"), "possibly partial statistics must be discarded");
|
||||
assert!(stats.contains_key("fresh"), "statistics must accept new samples after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scanner_transition_state_reports_compensation_pending_buckets() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
@@ -7274,6 +7472,23 @@ mod tests {
|
||||
TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn transition_worker_resize_without_runtime_does_not_poison_tracking() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let transition_state = runtime_sources::transition_state_handle();
|
||||
let resize = std::thread::spawn(move || {
|
||||
TransitionState::resize_workers_to(ecstore, 1, 1, resolve_transition_workers_absolute_max());
|
||||
})
|
||||
.join();
|
||||
|
||||
assert!(resize.is_ok(), "missing Tokio runtime must not panic while worker tracking is locked");
|
||||
assert!(
|
||||
transition_state.workers.lock().is_ok(),
|
||||
"failed resize must leave worker tracking unpoisoned"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
@@ -10897,6 +11112,7 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn concurrent_resend_same_part_commits_one_generation() {
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
@@ -10918,51 +11134,36 @@ mod tests {
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Two independent causes can produce a spurious lock-acquire timeout
|
||||
// here, and both must stay covered:
|
||||
// 1. A lost/stolen fast-lock wakeup could strand a waiter until the
|
||||
// deadline — fixed for real in fast_lock::shard by bounding each
|
||||
// notification wait (NOTIFY_WAIT_CAP re-polling).
|
||||
// 2. Under the full nextest suite on loaded CI disks, the
|
||||
// *legitimately serialized* cross-disk commits can exceed the
|
||||
// acquire deadline all by themselves — observed on CI at the 5s
|
||||
// default and the 30s production default with six resends, and
|
||||
// again at 60s, which is a hard ceiling: fast_lock clamps every
|
||||
// requested timeout to MAX_ACQUIRE_TIMEOUT (60s), so raising the
|
||||
// env override higher is a no-op (the Timeout error still reports
|
||||
// the requested value). Keep the guard about the correctness
|
||||
// property, not disk latency: request the full 60s ceiling and cap
|
||||
// the queue depth at three resends, so the last waiter sits behind
|
||||
// at most two serialized commits (~12s each on the slowest observed
|
||||
// CI runner, comfortably inside the deadline). Three concurrent
|
||||
// resends still race the streaming phase and contend on the commit
|
||||
// lock, which is all the generation-mixing regression needs.
|
||||
// `#[serial]` keeps the process-wide env override isolated.
|
||||
let results = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60"))], async {
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for payload in candidates.iter().cloned() {
|
||||
let store = ecstore.clone();
|
||||
let bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
tasks.spawn(async move {
|
||||
let mut data = PutObjReader::from_vec(payload.clone());
|
||||
store
|
||||
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.map(|info| (info, payload))
|
||||
});
|
||||
}
|
||||
let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for payload in candidates.iter().cloned() {
|
||||
let store = ecstore.clone();
|
||||
let bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let start = Arc::clone(&start);
|
||||
tasks.spawn(async move {
|
||||
start.wait().await;
|
||||
let mut data = PutObjReader::from_vec(payload.clone());
|
||||
store
|
||||
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.map(|info| (info, payload))
|
||||
});
|
||||
}
|
||||
start.wait().await;
|
||||
|
||||
// Every concurrent resend must succeed; the commit lock must never
|
||||
// starve a waiter into a timeout.
|
||||
let mut results = Vec::new();
|
||||
while let Some(joined) = tasks.join_next().await {
|
||||
let outcome = joined.expect("put_object_part task should not panic");
|
||||
results.push(outcome.expect("every concurrent same-part resend must succeed without lock timeout"));
|
||||
}
|
||||
results
|
||||
})
|
||||
.await;
|
||||
// The first writer holds the uploadId commit lock while the other
|
||||
// resends reach the same critical section. Releasing it proves the
|
||||
// handoff without depending on saturated CI disk latency.
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release();
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(joined) = tasks.join_next().await {
|
||||
let outcome = joined.expect("put_object_part task should not panic");
|
||||
results.push(outcome.expect("every concurrent same-part resend must succeed without lock timeout"));
|
||||
}
|
||||
assert_eq!(results.len(), candidates.len());
|
||||
|
||||
// Exactly one generation is visible after the serialized commits, and its
|
||||
|
||||
@@ -737,6 +737,9 @@ impl BucketMetadata {
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
self.tagging_config = None;
|
||||
self.tagging_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_QUOTA_CONFIG_FILE => {
|
||||
@@ -1318,6 +1321,30 @@ mod test {
|
||||
assert!(bm.lifecycle_config.is_none());
|
||||
}
|
||||
|
||||
/// Companion to the lifecycle case above. `parse_all_configs` skips empty
|
||||
/// XML rather than clearing, so without the explicit reset a cleared
|
||||
/// tagging config would keep serving the previously parsed tags.
|
||||
#[test]
|
||||
fn tagging_update_config_clears_parsed_config_on_delete() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
let tagging_xml = br#"<Tagging><TagSet><Tag><Key>env</Key><Value>prod</Value></Tag></TagSet></Tagging>"#;
|
||||
|
||||
bm.update_config(BUCKET_TAGGING_CONFIG, tagging_xml.to_vec())
|
||||
.expect("tagging config should update");
|
||||
bm.parse_all_configs().expect("tagging config should parse");
|
||||
assert!(bm.tagging_config.is_some());
|
||||
|
||||
bm.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("tagging config delete should update metadata");
|
||||
|
||||
assert!(bm.tagging_config_xml.is_empty());
|
||||
assert!(bm.tagging_config.is_none());
|
||||
|
||||
// A re-parse must not resurrect them either.
|
||||
bm.parse_all_configs().expect("cleared tagging should parse");
|
||||
assert!(bm.tagging_config.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn marshal_msg_complete_example() {
|
||||
// Create a complete BucketMetadata with various configurations
|
||||
|
||||
@@ -239,6 +239,35 @@ pub async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Ve
|
||||
bucket_meta_sys.update(bucket, BUCKET_TARGETS_FILE, data).await
|
||||
}
|
||||
|
||||
/// Read-modify-write one bucket config file under the metadata system's
|
||||
/// outer write guard.
|
||||
///
|
||||
/// `mutate` sees the freshly loaded on-disk metadata and returns the
|
||||
/// replacement payload for `config_file` (empty clears it, like
|
||||
/// [`delete`]). Both the read and the persisted write happen inside the
|
||||
/// same guard that [`update`] uses, so within this process the rewrite can
|
||||
/// neither clobber a concurrent update to another config file nor lose a
|
||||
/// concurrent write to the same one — unlike caching a mutated clone of
|
||||
/// previously read metadata.
|
||||
///
|
||||
/// This guard is process-local. Writers on other nodes still race, exactly
|
||||
/// as they do for [`update`]: each rewrites the whole metadata file, so the
|
||||
/// later save wins. What this narrows is the window — from "as stale as the
|
||||
/// local cache" down to a single metadata read plus write.
|
||||
pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let _targets_guard = if config_file == BUCKET_TARGETS_FILE {
|
||||
Some(acquire_bucket_targets_transaction_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
|
||||
bucket_meta_sys.update_config_with(bucket, config_file, mutate).await
|
||||
}
|
||||
|
||||
pub async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let api = bucket_meta_sys_lock.read().await.object_store();
|
||||
@@ -603,24 +632,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other("errServerNotInitialized"));
|
||||
};
|
||||
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
}
|
||||
|
||||
let mut bm = match load_bucket_metadata_parse(store, bucket, parse).await {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
if !runtime_sources::setup_is_erasure().await
|
||||
&& !runtime_sources::setup_is_dist_erasure().await
|
||||
&& is_err_bucket_not_found(&err)
|
||||
{
|
||||
BucketMetadata::new(bucket)
|
||||
} else {
|
||||
error!("load bucket metadata failed: {}", err);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut bm = Self::load_bucket_metadata_for_update(store, bucket, parse).await?;
|
||||
|
||||
let updated = bm.update_config(config_file, data)?;
|
||||
|
||||
@@ -629,6 +641,49 @@ impl BucketMetadataSys {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
/// See the free [`update_config_with`]: same load-mutate-persist cycle as
|
||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||
/// instead of supplied up front. Loads through this system's own store so
|
||||
/// the read and the persisted write target the same instance.
|
||||
async fn update_config_with<F>(&mut self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
let mut bm = Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true).await?;
|
||||
|
||||
let data = mutate(&bm)?;
|
||||
let updated = bm.update_config(config_file, data)?;
|
||||
|
||||
self.save(bm).await?;
|
||||
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
/// Load a bucket's on-disk metadata as the base of a config rewrite.
|
||||
/// Outside erasure setups a missing metadata file degrades to a fresh
|
||||
/// default (legacy buckets without one); erasure setups fail instead of
|
||||
/// fabricating state that a quorum may still hold.
|
||||
async fn load_bucket_metadata_for_update(store: Arc<ECStore>, bucket: &str, parse: bool) -> Result<BucketMetadata> {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
}
|
||||
|
||||
match load_bucket_metadata_parse(store, bucket, parse).await {
|
||||
Ok(res) => Ok(res),
|
||||
Err(err) => {
|
||||
if !runtime_sources::setup_is_erasure().await
|
||||
&& !runtime_sources::setup_is_dist_erasure().await
|
||||
&& is_err_bucket_not_found(&err)
|
||||
{
|
||||
Ok(BucketMetadata::new(bucket))
|
||||
} else {
|
||||
error!("load bucket metadata failed: {}", err);
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn save(&self, bm: BucketMetadata) -> Result<()> {
|
||||
if is_meta_bucketname(&bm.name) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
@@ -741,6 +796,8 @@ impl BucketMetadataSys {
|
||||
|
||||
if let Some(config) = &bm.policy_config {
|
||||
Ok((config.clone(), bm.policy_config_updated_at))
|
||||
} else if !bm.policy_config_json.is_empty() {
|
||||
Ok((serde_json::from_slice(&bm.policy_config_json)?, bm.policy_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
@@ -1051,6 +1108,138 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_bucket_policy_rejects_malformed_cached_policy() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let mut metadata = BucketMetadata::new("malformed-policy");
|
||||
metadata.policy_config_json = b"{".to_vec();
|
||||
sys.set("malformed-policy".to_string(), Arc::new(metadata)).await;
|
||||
|
||||
let err = sys
|
||||
.get_bucket_policy("malformed-policy")
|
||||
.await
|
||||
.expect_err("malformed persisted policy must not be treated as missing");
|
||||
|
||||
assert!(matches!(err, Error::Io(_)), "malformed persisted policy must surface its parse failure");
|
||||
}
|
||||
/// A tagging rewrite through `update_config_with` (the Swift metadata
|
||||
/// POST path) is persisted: it survives a metadata reload from disk, and
|
||||
/// an emptied rewrite clears the config in the cached copy too instead of
|
||||
/// leaving stale parsed tags behind.
|
||||
#[tokio::test]
|
||||
async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let mut sys = BucketMetadataSys::new(ecstore);
|
||||
|
||||
let bucket = "swift-tagging-bucket";
|
||||
sys.persist_and_set(BucketMetadata::new(bucket))
|
||||
.await
|
||||
.expect("initial metadata should persist");
|
||||
|
||||
let tagging = Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some("swift-meta-color".to_string()),
|
||||
value: Some("blue".to_string()),
|
||||
}],
|
||||
};
|
||||
let xml = crate::bucket::utils::serialize::<Tagging>(&tagging).expect("tagging should serialize");
|
||||
sys.update_config_with(bucket, BUCKET_TAGGING_CONFIG, move |bm| {
|
||||
assert!(bm.tagging_config.is_none(), "rewrite must see the on-disk state");
|
||||
Ok(xml)
|
||||
})
|
||||
.await
|
||||
.expect("tagging rewrite should persist");
|
||||
|
||||
// Simulate the disk-truth reload that used to lose Swift writes: drop
|
||||
// the cached entry and lazily re-load from the metadata file.
|
||||
sys.metadata_map.write().await.clear();
|
||||
let (tags, _) = sys
|
||||
.get_tagging_config(bucket)
|
||||
.await
|
||||
.expect("tagging must survive a reload from disk");
|
||||
assert_eq!(tags.tag_set.len(), 1);
|
||||
assert_eq!(tags.tag_set[0].key.as_deref(), Some("swift-meta-color"));
|
||||
assert_eq!(tags.tag_set[0].value.as_deref(), Some("blue"));
|
||||
|
||||
// An emptied rewrite clears the config everywhere.
|
||||
sys.update_config_with(bucket, BUCKET_TAGGING_CONFIG, |bm| {
|
||||
assert!(bm.tagging_config.is_some(), "rewrite must see the persisted tags");
|
||||
Ok(Vec::new())
|
||||
})
|
||||
.await
|
||||
.expect("clearing rewrite should persist");
|
||||
assert_eq!(
|
||||
sys.get_tagging_config(bucket).await.unwrap_err(),
|
||||
Error::ConfigNotFound,
|
||||
"cleared tagging must not be served from the cache"
|
||||
);
|
||||
sys.metadata_map.write().await.clear();
|
||||
assert_eq!(
|
||||
sys.get_tagging_config(bucket).await.unwrap_err(),
|
||||
Error::ConfigNotFound,
|
||||
"cleared tagging must not reappear after a reload from disk"
|
||||
);
|
||||
}
|
||||
|
||||
/// The load and the persisted write share one write guard, so concurrent
|
||||
/// rewrites of the same config compose instead of clobbering each other.
|
||||
/// Moving the load outside that guard loses all but the last tag.
|
||||
#[tokio::test]
|
||||
async fn concurrent_update_config_with_calls_do_not_lose_writes() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
|
||||
let bucket = "swift-tagging-concurrent";
|
||||
sys.read()
|
||||
.await
|
||||
.persist_and_set(BucketMetadata::new(bucket))
|
||||
.await
|
||||
.expect("initial metadata should persist");
|
||||
|
||||
const WRITERS: usize = 8;
|
||||
let mut handles = Vec::with_capacity(WRITERS);
|
||||
for idx in 0..WRITERS {
|
||||
let sys = sys.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
sys.write()
|
||||
.await
|
||||
.update_config_with(bucket, BUCKET_TAGGING_CONFIG, move |bm| {
|
||||
// Each writer merges its own tag onto whatever is
|
||||
// currently persisted — the Swift rewrite shape.
|
||||
let mut tagging = bm.tagging_config.clone().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some(format!("swift-meta-key{idx}")),
|
||||
value: Some(idx.to_string()),
|
||||
});
|
||||
crate::bucket::utils::serialize::<Tagging>(&tagging).map_err(|e| Error::other(e.to_string()))
|
||||
})
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
handle
|
||||
.await
|
||||
.expect("writer task should join")
|
||||
.expect("rewrite should persist");
|
||||
}
|
||||
|
||||
let (tags, _) = sys
|
||||
.read()
|
||||
.await
|
||||
.get_tagging_config(bucket)
|
||||
.await
|
||||
.expect("tagging should be readable");
|
||||
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
|
||||
}
|
||||
|
||||
fn target(bucket: &str, id: &str) -> BucketTarget {
|
||||
BucketTarget {
|
||||
source_bucket: bucket.to_string(),
|
||||
|
||||
@@ -15,23 +15,26 @@
|
||||
use super::metadata_sys::get_bucket_metadata_sys;
|
||||
use crate::error::{Result, StorageError};
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
use tracing::info;
|
||||
|
||||
pub struct PolicySys {}
|
||||
|
||||
impl PolicySys {
|
||||
pub async fn is_allowed(args: &BucketPolicyArgs<'_>) -> bool {
|
||||
match Self::get(args.bucket).await {
|
||||
Ok(cfg) => return cfg.is_allowed(args).await,
|
||||
Err(err) => {
|
||||
if err != StorageError::ConfigNotFound {
|
||||
info!("config get err {:?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
args.is_owner
|
||||
matches!(Self::try_is_allowed(args).await, Ok(true))
|
||||
}
|
||||
|
||||
pub async fn try_is_allowed(args: &BucketPolicyArgs<'_>) -> Result<bool> {
|
||||
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
|
||||
}
|
||||
|
||||
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
|
||||
match policy {
|
||||
Ok(policy) => Ok(policy.is_allowed(args).await),
|
||||
Err(StorageError::ConfigNotFound) => Ok(args.is_owner),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(bucket: &str) -> Result<BucketPolicy> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -41,3 +44,91 @@ impl PolicySys {
|
||||
Ok(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{PolicySys, StorageError};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn args<'a>(
|
||||
is_owner: bool,
|
||||
groups: &'a Option<Vec<String>>,
|
||||
conditions: &'a HashMap<String, Vec<String>>,
|
||||
) -> BucketPolicyArgs<'a> {
|
||||
BucketPolicyArgs {
|
||||
bucket: "bucket",
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
is_owner,
|
||||
account: "account",
|
||||
groups,
|
||||
conditions,
|
||||
object: "object",
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_policy_preserves_owner_and_iam_fallback_semantics() {
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
assert!(
|
||||
PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(StorageError::ConfigNotFound),)
|
||||
.await
|
||||
.expect("missing policy should preserve owner access")
|
||||
);
|
||||
assert!(
|
||||
!PolicySys::is_allowed_with_policy(&args(false, &groups, &conditions), Err(StorageError::ConfigNotFound),)
|
||||
.await
|
||||
.expect("missing policy should defer non-owner access to IAM")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn policy_load_failures_propagate() {
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
for (failure, expected_message) in [
|
||||
(StorageError::Io(std::io::Error::other("policy read failed")), "policy read failed"),
|
||||
(
|
||||
StorageError::other("bucket metadata sys not initialized for this instance"),
|
||||
"bucket metadata sys not initialized for this instance",
|
||||
),
|
||||
] {
|
||||
let result = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(failure)).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::Io(ref err)) if err.to_string().contains(expected_message)),
|
||||
"policy I/O and uninitialized metadata failures must propagate instead of granting owner access"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_bucket_deny_precedes_iam_allow() {
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let policy: BucketPolicy = serde_json::from_str(
|
||||
r#"{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{
|
||||
"Effect":"Deny",
|
||||
"Principal":{"AWS":"*"},
|
||||
"Action":["s3:GetObject"],
|
||||
"Resource":["arn:aws:s3:::bucket/*"]
|
||||
}]
|
||||
}"#,
|
||||
)
|
||||
.expect("deny policy should parse");
|
||||
|
||||
let bucket_allowed = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Ok(policy))
|
||||
.await
|
||||
.expect("loaded bucket policy should evaluate");
|
||||
let iam_allowed = true;
|
||||
let request_allowed = bucket_allowed && iam_allowed;
|
||||
|
||||
assert!(iam_allowed, "test precondition: IAM grants the action");
|
||||
assert!(!bucket_allowed, "test precondition: bucket policy explicitly denies the action");
|
||||
assert!(!request_allowed, "explicit bucket Deny must reject before IAM Allow fallback");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -417,7 +417,7 @@ impl TransitionClient {
|
||||
bucket: complete_multipart_upload_result.bucket,
|
||||
key: complete_multipart_upload_result.key,
|
||||
etag: trim_etag(&complete_multipart_upload_result.etag),
|
||||
version_id: self.raw_version_id(&h)?.unwrap_or_default().to_string(),
|
||||
version_id: self.legacy_remote_version_id(&h)?,
|
||||
location: complete_multipart_upload_result.location,
|
||||
expiration: exp_time,
|
||||
expiration_rule_id: rule_id,
|
||||
|
||||
@@ -22,7 +22,7 @@ use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use std::io::Error;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::{Mutex, MutexGuard, RwLock};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{OffsetDateTime, format_description};
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -46,6 +46,14 @@ use crate::client::utils::base64_encode;
|
||||
use rustfs_utils::path::trim_etag;
|
||||
use s3s::header::X_AMZ_EXPIRATION;
|
||||
|
||||
fn lock_md5_hasher(
|
||||
md5_hasher: &Mutex<Option<rustfs_utils::hash::HashAlgorithm>>,
|
||||
) -> Result<MutexGuard<'_, Option<rustfs_utils::hash::HashAlgorithm>>, std::io::Error> {
|
||||
md5_hasher
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("MD5 hasher state is unavailable"))
|
||||
}
|
||||
|
||||
/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
|
||||
/// reaches EOF first. Advances the reader so the next call returns the following
|
||||
/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
|
||||
@@ -177,7 +185,7 @@ impl TransitionClient {
|
||||
let length = buf.len();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = self.md5_hasher.lock().unwrap();
|
||||
let mut md5_hasher = lock_md5_hasher(&self.md5_hasher)?;
|
||||
let md5_hash = match md5_hasher.as_mut() {
|
||||
Some(hasher) => hasher,
|
||||
None => return Err(std::io::Error::other("MD5 hasher not initialized")),
|
||||
@@ -370,7 +378,7 @@ impl TransitionClient {
|
||||
let mut md5_base64: String = "".to_string();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
|
||||
let mut md5_hasher = lock_md5_hasher(&clone_self.md5_hasher)?;
|
||||
let md5_hash = match md5_hasher.as_mut() {
|
||||
Some(hasher) => hasher,
|
||||
None => {
|
||||
@@ -418,6 +426,9 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
result?;
|
||||
}
|
||||
|
||||
select! {
|
||||
err = err_rx.recv() => {
|
||||
@@ -567,7 +578,7 @@ impl TransitionClient {
|
||||
key: object_name.to_string(),
|
||||
etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
|
||||
|
||||
version_id: self.raw_version_id(h)?.unwrap_or_default().to_string(),
|
||||
version_id: self.legacy_remote_version_id(h)?,
|
||||
size,
|
||||
expiration: exp_time,
|
||||
expiration_rule_id: rule_id,
|
||||
@@ -620,10 +631,12 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part};
|
||||
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
|
||||
use crate::object_api::GetObjectReader;
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::hash::HashAlgorithm;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
// Drive a reader through the same per-part loop the multipart stream uses and
|
||||
// collect the size of every part. Regression for rustfs/rustfs#4811: the old
|
||||
@@ -733,4 +746,18 @@ mod tests {
|
||||
"a gap in the parts map must be an error, not a panic"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_md5_state_fails_closed() {
|
||||
let hasher = Arc::new(Mutex::new(Some(HashAlgorithm::Md5)));
|
||||
let poison_target = Arc::clone(&hasher);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poison_target.lock().expect("fresh mutex should lock");
|
||||
panic!("poison MD5 state");
|
||||
})
|
||||
.join();
|
||||
|
||||
let error = lock_md5_hasher(&hasher).expect_err("poisoned hash state must not be reused");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::Other);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ impl TransitionClient {
|
||||
object_name: object_name.to_string(),
|
||||
object_version_id: opts.version_id,
|
||||
delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
|
||||
delete_marker_version_id: self.raw_version_id(resp.headers())?.unwrap_or_default().to_string(),
|
||||
delete_marker_version_id: self.legacy_remote_version_id(resp.headers())?,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -46,11 +46,33 @@ impl RemoteVersion {
|
||||
Self::Unknown | Self::Disabled => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||
match self {
|
||||
Self::Unknown => Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
"remote object version is unknown; exact version routing is unsafe",
|
||||
)),
|
||||
Self::Disabled => Ok(None),
|
||||
Self::SuspendedNull => Ok(Some("null")),
|
||||
Self::Exact(version_id) => Ok(Some(version_id)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum ConditionalCreateCapability {
|
||||
Unsupported,
|
||||
IfNoneMatchStar,
|
||||
GenerationMatchZero,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct ProviderVersionCapabilities {
|
||||
raw_version_header: Option<&'static str>,
|
||||
pub(crate) bucket_versioning_state: bool,
|
||||
pub(crate) list_object_versions: bool,
|
||||
pub(crate) conditional_create: ConditionalCreateCapability,
|
||||
pub(crate) exact_get_delete: bool,
|
||||
}
|
||||
|
||||
@@ -62,28 +84,59 @@ impl ProviderVersionCapabilities {
|
||||
|| tier_type.eq_ignore_ascii_case("r2")
|
||||
|| tier_type.eq_ignore_ascii_case("wasabi")
|
||||
{
|
||||
let list_object_versions = tier_type.eq_ignore_ascii_case("s3")
|
||||
|| tier_type.eq_ignore_ascii_case("rustfs")
|
||||
|| tier_type.eq_ignore_ascii_case("minio")
|
||||
|| tier_type.eq_ignore_ascii_case("r2");
|
||||
Self {
|
||||
raw_version_header: Some(X_AMZ_VERSION_ID),
|
||||
bucket_versioning_state: list_object_versions,
|
||||
list_object_versions,
|
||||
conditional_create: if tier_type.eq_ignore_ascii_case("s3") || tier_type.eq_ignore_ascii_case("r2") {
|
||||
ConditionalCreateCapability::IfNoneMatchStar
|
||||
} else {
|
||||
ConditionalCreateCapability::Unsupported
|
||||
},
|
||||
exact_get_delete: true,
|
||||
}
|
||||
} else if tier_type.eq_ignore_ascii_case("aliyun") {
|
||||
Self {
|
||||
raw_version_header: Some(X_OSS_VERSION_ID),
|
||||
bucket_versioning_state: false,
|
||||
list_object_versions: false,
|
||||
conditional_create: ConditionalCreateCapability::Unsupported,
|
||||
exact_get_delete: true,
|
||||
}
|
||||
} else if tier_type.eq_ignore_ascii_case("tencent") {
|
||||
Self {
|
||||
raw_version_header: Some(X_COS_VERSION_ID),
|
||||
bucket_versioning_state: false,
|
||||
list_object_versions: false,
|
||||
conditional_create: ConditionalCreateCapability::Unsupported,
|
||||
exact_get_delete: true,
|
||||
}
|
||||
} else if tier_type.eq_ignore_ascii_case("huaweicloud") {
|
||||
Self {
|
||||
raw_version_header: Some(X_OBS_VERSION_ID),
|
||||
bucket_versioning_state: false,
|
||||
list_object_versions: false,
|
||||
conditional_create: ConditionalCreateCapability::Unsupported,
|
||||
exact_get_delete: true,
|
||||
}
|
||||
} else if tier_type.eq_ignore_ascii_case("gcs") {
|
||||
Self {
|
||||
raw_version_header: None,
|
||||
bucket_versioning_state: false,
|
||||
list_object_versions: false,
|
||||
conditional_create: ConditionalCreateCapability::GenerationMatchZero,
|
||||
exact_get_delete: false,
|
||||
}
|
||||
} else {
|
||||
Self {
|
||||
raw_version_header: None,
|
||||
bucket_versioning_state: false,
|
||||
list_object_versions: false,
|
||||
conditional_create: ConditionalCreateCapability::Unsupported,
|
||||
exact_get_delete: false,
|
||||
}
|
||||
}
|
||||
@@ -143,7 +196,7 @@ fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
|
||||
use super::{BucketVersioningState, ConditionalCreateCapability, ProviderVersionCapabilities, RemoteVersion};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
|
||||
#[test]
|
||||
@@ -219,6 +272,71 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_capability_matrix_is_conservative_and_provider_specific() {
|
||||
for (tier_type, state, list, conditional_create, exact_get_delete) in [
|
||||
("s3", true, true, ConditionalCreateCapability::IfNoneMatchStar, true),
|
||||
("rustfs", true, true, ConditionalCreateCapability::Unsupported, true),
|
||||
("minio", true, true, ConditionalCreateCapability::Unsupported, true),
|
||||
("r2", true, true, ConditionalCreateCapability::IfNoneMatchStar, true),
|
||||
("wasabi", false, false, ConditionalCreateCapability::Unsupported, true),
|
||||
("aliyun", false, false, ConditionalCreateCapability::Unsupported, true),
|
||||
("tencent", false, false, ConditionalCreateCapability::Unsupported, true),
|
||||
("huaweicloud", false, false, ConditionalCreateCapability::Unsupported, true),
|
||||
("gcs", false, false, ConditionalCreateCapability::GenerationMatchZero, false),
|
||||
("azure", false, false, ConditionalCreateCapability::Unsupported, false),
|
||||
("unsupported", false, false, ConditionalCreateCapability::Unsupported, false),
|
||||
] {
|
||||
let capabilities = ProviderVersionCapabilities::for_tier_type(tier_type);
|
||||
assert_eq!(capabilities.bucket_versioning_state, state, "{tier_type} versioning state");
|
||||
assert_eq!(capabilities.list_object_versions, list, "{tier_type} version listing");
|
||||
assert_eq!(capabilities.conditional_create, conditional_create, "{tier_type} conditional create");
|
||||
assert_eq!(capabilities.exact_get_delete, exact_get_delete, "{tier_type} exact routing");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_states_preserve_unknown_disabled_suspended_and_exact() {
|
||||
let capabilities = ProviderVersionCapabilities::for_tier_type("s3");
|
||||
let empty = HeaderMap::new();
|
||||
let mut null = HeaderMap::new();
|
||||
null.insert("x-amz-version-id", HeaderValue::from_static("null"));
|
||||
let mut exact = HeaderMap::new();
|
||||
exact.insert("x-amz-version-id", HeaderValue::from_static("opaque.generation-7"));
|
||||
|
||||
for (headers, state, expected) in [
|
||||
(&empty, BucketVersioningState::Unknown, RemoteVersion::Unknown),
|
||||
(&empty, BucketVersioningState::Disabled, RemoteVersion::Disabled),
|
||||
(&empty, BucketVersioningState::Suspended, RemoteVersion::Unknown),
|
||||
(&empty, BucketVersioningState::Enabled, RemoteVersion::Unknown),
|
||||
(&null, BucketVersioningState::Suspended, RemoteVersion::SuspendedNull),
|
||||
(
|
||||
&exact,
|
||||
BucketVersioningState::Enabled,
|
||||
RemoteVersion::Exact("opaque.generation-7".to_string()),
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
capabilities
|
||||
.remote_version(headers, state)
|
||||
.expect("version state should normalize"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_request_routing_fails_closed_for_unknown_versions() {
|
||||
for (version, expected) in [
|
||||
(RemoteVersion::Disabled, None),
|
||||
(RemoteVersion::SuspendedNull, Some("null")),
|
||||
(RemoteVersion::Exact("opaque-v1".to_string()), Some("opaque-v1")),
|
||||
] {
|
||||
assert_eq!(version.exact_request_id().expect("known version state"), expected);
|
||||
}
|
||||
assert!(RemoteVersion::Unknown.exact_request_id().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_version_rejects_empty_or_oversized_headers() {
|
||||
let oversized = "v".repeat(1025);
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::client::{
|
||||
},
|
||||
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
|
||||
credentials::{CredContext, Credentials, SignatureType, Static},
|
||||
provider_versions::{BucketVersioningState, ProviderVersionCapabilities},
|
||||
provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion},
|
||||
signer_error,
|
||||
};
|
||||
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
|
||||
@@ -332,6 +332,22 @@ impl TransitionClient {
|
||||
self.provider_version_capabilities().raw_version_id(headers)
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version(
|
||||
&self,
|
||||
headers: &HeaderMap,
|
||||
versioning: BucketVersioningState,
|
||||
) -> Result<RemoteVersion, std::io::Error> {
|
||||
self.provider_version_capabilities().remote_version(headers, versioning)
|
||||
}
|
||||
|
||||
pub(crate) fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result<String, std::io::Error> {
|
||||
Ok(self
|
||||
.remote_version(headers, BucketVersioningState::Unknown)?
|
||||
.exact_id()
|
||||
.unwrap_or_default()
|
||||
.to_string())
|
||||
}
|
||||
|
||||
fn trace_errors_only_off(&self) {
|
||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||
*trace_errors_only = false;
|
||||
@@ -1095,6 +1111,16 @@ impl Default for ObjectInfo {
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectInfo {
|
||||
pub(crate) fn remote_version(
|
||||
&self,
|
||||
capabilities: ProviderVersionCapabilities,
|
||||
versioning: BucketVersioningState,
|
||||
) -> Result<RemoteVersion, std::io::Error> {
|
||||
capabilities.remote_version(&self.metadata, versioning)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct RestoreInfo {
|
||||
ongoing_restore: bool,
|
||||
@@ -1414,7 +1440,7 @@ mod tests {
|
||||
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
|
||||
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
|
||||
};
|
||||
use crate::client::provider_versions::ProviderVersionCapabilities;
|
||||
use crate::client::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::Full;
|
||||
use hyper::body::Bytes;
|
||||
@@ -1539,6 +1565,11 @@ mod tests {
|
||||
.expect("opaque provider version should parse");
|
||||
|
||||
assert_eq!(info.version_id, None);
|
||||
assert_eq!(
|
||||
info.remote_version(ProviderVersionCapabilities::for_tier_type("tencent"), BucketVersioningState::Enabled,)
|
||||
.expect("opaque response version should remain available"),
|
||||
RemoteVersion::Exact("opaque.version_01".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
info.metadata.get("x-cos-version-id").and_then(|value| value.to_str().ok()),
|
||||
Some("opaque.version_01")
|
||||
|
||||
@@ -24,8 +24,8 @@ use crate::cluster::rpc::internode_data_transport::{
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -48,9 +48,10 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
|
||||
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
|
||||
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
|
||||
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -2481,6 +2482,71 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(PreparePartTransactionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
meta,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "prepare_part_transaction")?;
|
||||
|
||||
let response = client.prepare_part_transaction(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SettlePartTransactionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
rollback: action == PartTransactionAction::Rollback,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "settle_part_transaction")?;
|
||||
|
||||
let response = client.settle_part_transaction(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
trace!(
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
use crate::disk::error_reduce::count_errs;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||
@@ -49,7 +50,10 @@ use rustfs_filemeta::FileInfo;
|
||||
use rustfs_lock::NamespaceLockWrapper;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::broadcast::{Receiver, Sender};
|
||||
use tokio::time::Duration;
|
||||
@@ -63,6 +67,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Sets {
|
||||
pub id: Uuid,
|
||||
@@ -799,9 +805,52 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
|
||||
delimiter: Option<String>,
|
||||
max_uploads: usize,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
self.get_disks_by_key(prefix)
|
||||
.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
|
||||
.await
|
||||
let per_set_limit = max_uploads.saturating_add(1);
|
||||
let results = futures::stream::iter(self.disk_set.iter().cloned())
|
||||
.map(|set| {
|
||||
let key_marker = key_marker.clone();
|
||||
let upload_id_marker = upload_id_marker.clone();
|
||||
let delimiter = delimiter.clone();
|
||||
async move {
|
||||
set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let mut uploads = Vec::new();
|
||||
let mut common_prefixes = HashSet::new();
|
||||
let mut source_truncated = false;
|
||||
for result in results {
|
||||
let page = result?;
|
||||
uploads.extend(page.uploads);
|
||||
common_prefixes.extend(page.common_prefixes);
|
||||
source_truncated |= page.is_truncated;
|
||||
}
|
||||
|
||||
let page = paginate_multipart_listing(
|
||||
uploads,
|
||||
common_prefixes.into_iter().collect(),
|
||||
key_marker.as_deref(),
|
||||
key_marker.as_ref().and(upload_id_marker.as_deref()),
|
||||
max_uploads,
|
||||
source_truncated,
|
||||
);
|
||||
|
||||
Ok(ListMultipartsInfo {
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
next_key_marker: page.next_key_marker,
|
||||
next_upload_id_marker: page.next_upload_id_marker,
|
||||
max_uploads,
|
||||
is_truncated: page.is_truncated,
|
||||
uploads: page.uploads,
|
||||
common_prefixes: page.common_prefixes,
|
||||
prefix: prefix.to_owned(),
|
||||
delimiter,
|
||||
})
|
||||
}
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
|
||||
@@ -1111,6 +1160,7 @@ mod tests {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::list::ListOperations as _;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use serial_test::serial;
|
||||
|
||||
@@ -1248,6 +1298,194 @@ mod tests {
|
||||
assert_eq!(result, (Some(3), Some(1), Some(0)));
|
||||
}
|
||||
|
||||
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
let format = FormatV3::new(2, 2);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut all_endpoints = Vec::new();
|
||||
let mut disk_sets = Vec::new();
|
||||
|
||||
for set_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..2 {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
|
||||
.expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
let mut disk_format = format.clone();
|
||||
disk_format.erasure.this = format.erasure.sets[set_index][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("format should be saved");
|
||||
temp_dirs.push(temp_dir);
|
||||
all_endpoints.push(endpoint.clone());
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
disk_sets.push(
|
||||
SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
set_index,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
|
||||
let sets = Arc::new(Sets {
|
||||
id: format.id,
|
||||
disk_set: disk_sets,
|
||||
pool_idx: 0,
|
||||
endpoints: PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 2,
|
||||
endpoints: Endpoints::from(all_endpoints),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
},
|
||||
format,
|
||||
parity_count: 1,
|
||||
set_count: 2,
|
||||
set_drive_count: 2,
|
||||
default_parity_count: 1,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
ctx: bootstrap_ctx(),
|
||||
});
|
||||
(temp_dirs, sets)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
|
||||
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
|
||||
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let mut keys_by_set = [Vec::new(), Vec::new()];
|
||||
for index in 0..100 {
|
||||
let key = format!("logs/{index:03}.bin");
|
||||
let set_index = sets.get_hashed_set_index(&key);
|
||||
if keys_by_set[set_index].len() < 2 {
|
||||
keys_by_set[set_index].push(key);
|
||||
}
|
||||
if keys_by_set.iter().all(|keys| keys.len() == 2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(keys_by_set.iter().all(|keys| keys.len() == 2), "test keys must span both sets");
|
||||
|
||||
let repeated_key = keys_by_set[0][0].clone();
|
||||
let mut expected = Vec::new();
|
||||
for key in keys_by_set.iter().flatten() {
|
||||
let upload = sets
|
||||
.new_multipart_upload(&bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
expected.push((key.clone(), upload.upload_id));
|
||||
}
|
||||
let second = sets
|
||||
.new_multipart_upload(&bucket, &repeated_key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("second upload for the same key should be created");
|
||||
expected.push((repeated_key, second.upload_id));
|
||||
expected.sort();
|
||||
|
||||
let mut actual = Vec::new();
|
||||
let mut key_marker = None;
|
||||
let mut upload_id_marker = None;
|
||||
for _ in 0..expected.len() + 1 {
|
||||
let page = sets
|
||||
.list_multipart_uploads(&bucket, "logs/", key_marker.clone(), upload_id_marker.clone(), None, 2)
|
||||
.await
|
||||
.expect("multipart page should list across every set");
|
||||
assert!(page.uploads.len() <= 2);
|
||||
actual.extend(
|
||||
page.uploads
|
||||
.iter()
|
||||
.map(|upload| (upload.object.clone(), upload.upload_id.clone())),
|
||||
);
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
key_marker = page.next_key_marker;
|
||||
upload_id_marker = page.next_upload_id_marker;
|
||||
}
|
||||
|
||||
assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
|
||||
let mut deduped = actual.clone();
|
||||
deduped.dedup();
|
||||
assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");
|
||||
|
||||
let mut nested_by_set = [None, None];
|
||||
for index in 0..100 {
|
||||
let key = format!("nested/group-{index:03}/file.bin");
|
||||
let set_index = sets.get_hashed_set_index(&key);
|
||||
nested_by_set[set_index].get_or_insert(key);
|
||||
if nested_by_set.iter().all(Option::is_some) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for key in nested_by_set.iter().flatten() {
|
||||
sets.new_multipart_upload(&bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("nested multipart upload should be created");
|
||||
}
|
||||
let mut expected_prefixes = nested_by_set
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|key| {
|
||||
key.rsplit_once('/')
|
||||
.expect("nested key should contain a delimiter")
|
||||
.0
|
||||
.to_string()
|
||||
+ "/"
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
expected_prefixes.sort();
|
||||
|
||||
let first = sets
|
||||
.list_multipart_uploads(&bucket, "nested/", None, None, Some("/".to_string()), 1)
|
||||
.await
|
||||
.expect("first delimiter page should list across every set");
|
||||
assert!(first.is_truncated);
|
||||
assert_eq!(first.common_prefixes, expected_prefixes[..1]);
|
||||
let second = sets
|
||||
.list_multipart_uploads(
|
||||
&bucket,
|
||||
"nested/",
|
||||
first.next_key_marker,
|
||||
first.next_upload_id_marker,
|
||||
Some("/".to_string()),
|
||||
1,
|
||||
)
|
||||
.await
|
||||
.expect("second delimiter page should list across every set");
|
||||
assert!(!second.is_truncated);
|
||||
assert_eq!(second.common_prefixes, expected_prefixes[1..]);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn sets_list_objects_v2_lists_objects_within_the_pool() {
|
||||
|
||||
@@ -35,7 +35,7 @@ use std::{
|
||||
Arc,
|
||||
atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use tokio::{sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -339,21 +339,19 @@ impl Drop for DiskHealthWaitingGuard<'_> {
|
||||
impl DiskHealthTracker {
|
||||
/// Create a new disk health tracker
|
||||
pub fn new() -> Self {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
let now = current_unix_time();
|
||||
let now_nanos = unix_nanos(now);
|
||||
|
||||
Self {
|
||||
last_success: AtomicI64::new(now),
|
||||
last_started: AtomicI64::new(now),
|
||||
last_success: AtomicI64::new(now_nanos),
|
||||
last_started: AtomicI64::new(now_nanos),
|
||||
status: AtomicU32::new(DISK_HEALTH_OK),
|
||||
waiting: AtomicU32::new(0),
|
||||
runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
|
||||
consecutive_failures: AtomicU32::new(0),
|
||||
consecutive_successes: AtomicU32::new(0),
|
||||
offline_since_unix_secs: AtomicI64::new(0),
|
||||
last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
|
||||
last_transition_unix_secs: AtomicI64::new(unix_secs_i64(now)),
|
||||
last_capacity_total: AtomicU64::new(0),
|
||||
last_capacity_used: AtomicU64::new(0),
|
||||
last_capacity_free: AtomicU64::new(0),
|
||||
@@ -363,11 +361,7 @@ impl DiskHealthTracker {
|
||||
|
||||
/// Log a successful operation
|
||||
pub fn log_success(&self) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.last_success.store(now, Ordering::Relaxed);
|
||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
|
||||
@@ -429,11 +423,14 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn offline_duration(&self) -> Option<Duration> {
|
||||
self.offline_duration_at(current_unix_secs())
|
||||
}
|
||||
|
||||
fn offline_duration_at(&self, now: u64) -> Option<Duration> {
|
||||
let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
|
||||
if offline_since <= 0 {
|
||||
return None;
|
||||
}
|
||||
let now = current_unix_secs();
|
||||
Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
|
||||
}
|
||||
|
||||
@@ -492,6 +489,12 @@ impl DiskHealthTracker {
|
||||
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
|
||||
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
|
||||
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
|
||||
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
|
||||
}
|
||||
|
||||
fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) {
|
||||
let now_nanos = unix_nanos(now);
|
||||
let now_secs = unix_secs_i64(now);
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
self.runtime_state
|
||||
.store(RuntimeDriveHealthState::Online as u32, Ordering::Release);
|
||||
@@ -499,11 +502,9 @@ impl DiskHealthTracker {
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
self.offline_since_unix_secs.store(0, Ordering::Release);
|
||||
self.waiting.store(0, Ordering::Release);
|
||||
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap();
|
||||
let now_nanos = now.as_nanos() as i64;
|
||||
self.last_success.store(now_nanos, Ordering::Relaxed);
|
||||
self.last_started.store(now_nanos, Ordering::Relaxed);
|
||||
self.last_transition_unix_secs.store(now.as_secs() as i64, Ordering::Release);
|
||||
self.last_transition_unix_secs.store(now_secs, Ordering::Release);
|
||||
record_drive_runtime_state(endpoint, RuntimeDriveHealthState::Online);
|
||||
}
|
||||
|
||||
@@ -612,10 +613,33 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
fn current_unix_secs() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
// Zero is reserved as "not recorded" by health timestamp atomics.
|
||||
current_unix_time().as_secs().max(1)
|
||||
}
|
||||
|
||||
fn current_unix_nanos() -> i64 {
|
||||
unix_nanos(current_unix_time())
|
||||
}
|
||||
|
||||
fn current_unix_time() -> Duration {
|
||||
unix_time_since_epoch(SystemTime::now())
|
||||
}
|
||||
|
||||
fn unix_time_since_epoch(time: SystemTime) -> Duration {
|
||||
time.duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO)
|
||||
}
|
||||
|
||||
fn unix_nanos(time: Duration) -> i64 {
|
||||
i64::try_from(time.as_nanos()).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
fn unix_secs_i64(time: Duration) -> i64 {
|
||||
i64::try_from(time.as_secs()).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
fn elapsed_since(last_nanos: i64, now_nanos: i64) -> Duration {
|
||||
let elapsed_nanos = now_nanos.saturating_sub(last_nanos).max(0);
|
||||
Duration::from_nanos(u64::try_from(elapsed_nanos).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
impl Default for DiskHealthTracker {
|
||||
@@ -635,11 +659,7 @@ struct HealthDiskCtxValue {
|
||||
|
||||
impl HealthDiskCtxValue {
|
||||
fn log_success(&self) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.last_success.store(now, Ordering::Relaxed);
|
||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -774,12 +794,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
let last_success_nanos = health.last_success.load(Ordering::Relaxed);
|
||||
let elapsed = Duration::from_nanos(
|
||||
(std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64 - last_success_nanos) as u64
|
||||
);
|
||||
let elapsed = elapsed_since(last_success_nanos, current_unix_nanos());
|
||||
|
||||
if elapsed < SKIP_IF_SUCCESS_BEFORE {
|
||||
continue;
|
||||
@@ -1091,11 +1106,7 @@ impl LocalDiskWrapper {
|
||||
self.check_disk_stale().await?;
|
||||
|
||||
// Record operation start
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.health.last_started.store(now, Ordering::Relaxed);
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
let _waiting_guard = self.health.waiting_guard();
|
||||
|
||||
if timeout_duration == Duration::ZERO {
|
||||
@@ -1317,11 +1328,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Record operation start
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.health.last_started.store(now, Ordering::Relaxed);
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
self.health.increment_waiting();
|
||||
|
||||
// Execute the operation
|
||||
@@ -1473,6 +1480,33 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async {
|
||||
self.disk
|
||||
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: crate::disk::PartTransactionAction) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.settle_part_transaction(volume, path, action).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
self.track_disk_health(|| async { self.disk.delete(volume, path, opt).await }, get_max_timeout_duration())
|
||||
.await
|
||||
@@ -2243,4 +2277,53 @@ mod tests {
|
||||
assert!(health.mark_offline(&endpoint, "again"));
|
||||
assert!(health.is_faulty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unix_time_clamps_epoch_and_pre_epoch_to_zero() {
|
||||
let before_epoch = UNIX_EPOCH
|
||||
.checked_sub(Duration::from_nanos(1))
|
||||
.expect("one nanosecond before the Unix epoch should be representable");
|
||||
|
||||
assert_eq!(unix_time_since_epoch(UNIX_EPOCH), Duration::ZERO);
|
||||
assert_eq!(unix_time_since_epoch(before_epoch), Duration::ZERO);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elapsed_and_offline_duration_saturate_on_clock_rollback() {
|
||||
let health = DiskHealthTracker::new();
|
||||
health.offline_since_unix_secs.store(10, Ordering::Release);
|
||||
|
||||
assert_eq!(elapsed_since(10, 12), Duration::from_nanos(2));
|
||||
assert_eq!(elapsed_since(10, 9), Duration::ZERO);
|
||||
assert_eq!(health.offline_duration_at(9), Some(Duration::ZERO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_epoch_retry_reset_updates_the_complete_health_state() {
|
||||
let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry-pre-epoch").expect("endpoint should parse");
|
||||
let health = DiskHealthTracker::new();
|
||||
health.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
health
|
||||
.runtime_state
|
||||
.store(RuntimeDriveHealthState::Offline as u32, Ordering::Release);
|
||||
health.consecutive_failures.store(3, Ordering::Release);
|
||||
health.consecutive_successes.store(2, Ordering::Release);
|
||||
health.offline_since_unix_secs.store(11, Ordering::Release);
|
||||
health.waiting.store(4, Ordering::Release);
|
||||
health.last_success.store(12, Ordering::Release);
|
||||
health.last_started.store(13, Ordering::Release);
|
||||
health.last_transition_unix_secs.store(14, Ordering::Release);
|
||||
|
||||
health.reset_for_store_init_retry_at(&endpoint, unix_time_since_epoch(UNIX_EPOCH - Duration::from_nanos(1)));
|
||||
|
||||
assert_eq!(health.status.load(Ordering::Acquire), DISK_HEALTH_OK);
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.consecutive_successes.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.offline_since_unix_secs.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.waiting.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.last_success.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.last_started.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.last_transition_unix_secs.load(Ordering::Acquire), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_r
|
||||
use crate::disk::{
|
||||
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
||||
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
|
||||
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, PART_TRANSACTION_NEW_META,
|
||||
PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
|
||||
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
|
||||
endpoint::Endpoint,
|
||||
@@ -80,6 +81,10 @@ const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_
|
||||
const ENV_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS";
|
||||
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_COUNT: u64 = 2;
|
||||
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: u64 = 100;
|
||||
const PART_TRANSACTION_OLD_DATA: &str = "old.data";
|
||||
const PART_TRANSACTION_OLD_DATA_ABSENT: &str = "old.data.absent";
|
||||
const PART_TRANSACTION_OLD_META_ABSENT: &str = "old.meta.absent";
|
||||
const PART_TRANSACTION_PUBLISH_META: &str = "publish.meta";
|
||||
enum ReadAllError {
|
||||
Open(std::io::Error),
|
||||
Disk(DiskError),
|
||||
@@ -141,6 +146,28 @@ fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> std::io::Result<()> {
|
||||
match std::fs::symlink_metadata(src) {
|
||||
Ok(metadata) if metadata.is_file() => std::fs::hard_link(src, backup),
|
||||
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file")),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => std::fs::write(absent, []),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
|
||||
match std::fs::symlink_metadata(backup) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
remove_file_if_exists(restore)?;
|
||||
std::fs::hard_link(backup, restore)?;
|
||||
std::fs::rename(restore, current)
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction backup is not a file")),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound && absent.is_file() => remove_file_if_exists(current),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn rollback_committed_rename_std(
|
||||
dst_file_path: &Path,
|
||||
new_data_path: Option<&Path>,
|
||||
@@ -6447,6 +6474,151 @@ impl DiskAPI for LocalDisk {
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
let src_volume_dir = self.get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
if !skip_access_checks(dst_volume) {
|
||||
super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
|
||||
let src_file_path = self.get_object_path(src_volume, src_path)?;
|
||||
let dst_file_path = self.get_object_path(dst_volume, dst_path)?;
|
||||
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
|
||||
for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] {
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
}
|
||||
|
||||
let durability = effective_durability(dst_volume);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
|
||||
if !source.is_file() {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
if transaction_path.exists() {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
let Some(transaction_parent) = transaction_path.parent() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
std::fs::create_dir_all(transaction_parent).map_err(to_file_error)?;
|
||||
let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir(&staging_path).map_err(to_file_error)?;
|
||||
|
||||
let prepare_result = (|| -> std::io::Result<()> {
|
||||
snapshot_part_transaction_file(
|
||||
&dst_file_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
|
||||
)?;
|
||||
snapshot_part_transaction_file(
|
||||
&dst_meta_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META_ABSENT),
|
||||
)?;
|
||||
|
||||
let mut new_meta = std::fs::OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(staging_path.join(PART_TRANSACTION_NEW_META))?;
|
||||
std::io::Write::write_all(&mut new_meta, &meta)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
new_meta.sync_data()?;
|
||||
os::fsync_dir_std(&staging_path)?;
|
||||
}
|
||||
std::fs::rename(&staging_path, &transaction_path)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(transaction_parent)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
if let Err(err) = prepare_result {
|
||||
let _ = remove_dir_all_if_exists(&staging_path);
|
||||
return Err(to_file_error(err).into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
self.get_bucket_path(volume)?;
|
||||
let current_data_path = self.get_object_path(volume, path)?;
|
||||
let current_meta_path = self.get_object_path(volume, &format!("{path}.meta"))?;
|
||||
let transaction_path = self.get_object_path(volume, &crate::disk::part_transaction_path(path))?;
|
||||
for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] {
|
||||
check_path_length(candidate.to_string_lossy().as_ref())?;
|
||||
}
|
||||
let durability = effective_durability(volume);
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
match std::fs::symlink_metadata(&transaction_path) {
|
||||
Ok(metadata) if metadata.is_dir() => {}
|
||||
Ok(_) => return Err(DiskError::FileCorrupt),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
|
||||
if action == PartTransactionAction::Rollback {
|
||||
std::fs::write(transaction_path.join(PART_TRANSACTION_ROLLBACK), []).map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(&transaction_path).map_err(to_file_error)?;
|
||||
}
|
||||
restore_part_transaction_file(
|
||||
¤t_data_path,
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_DATA),
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
|
||||
&transaction_path.join("restore.data"),
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
restore_part_transaction_file(
|
||||
¤t_meta_path,
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_META),
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_META_ABSENT),
|
||||
&transaction_path.join("restore.meta"),
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata()
|
||||
&& let Some(parent) = current_data_path.parent()
|
||||
{
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
|
||||
let Some(parent) = transaction_path.parent() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4()));
|
||||
std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
}
|
||||
remove_dir_all_if_exists(&cleanup_path).map_err(to_file_error)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)??;
|
||||
|
||||
self.io_backend.invalidate_cached_fd(volume, path).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
|
||||
let src_volume_dir = self.get_bucket_path(src_volume)?;
|
||||
@@ -6528,6 +6700,42 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let transaction_publish_meta = if src_is_dir {
|
||||
None
|
||||
} else {
|
||||
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
|
||||
let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META);
|
||||
match fs::read(&transaction_meta_path).await {
|
||||
Ok(expected_meta) => {
|
||||
if expected_meta.as_slice() != meta.as_ref() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
|
||||
let source_meta_path = transaction_meta_path.clone();
|
||||
let publish_path = publish_meta_path.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
remove_file_if_exists(&publish_path)?;
|
||||
std::fs::hard_link(source_meta_path, &publish_path)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
Some(publish_meta_path)
|
||||
}
|
||||
// Old peers know only RenamePart. The new coordinator never
|
||||
// reaches rename unless prepare succeeded, so an absent
|
||||
// transaction directory identifies the rolling-upgrade legacy
|
||||
// path. A present directory without new.meta is corruption.
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => match fs::metadata(&transaction_path).await {
|
||||
Ok(_) => return Err(DiskError::FileCorrupt),
|
||||
Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None,
|
||||
Err(meta_err) => return Err(to_file_error(meta_err).into()),
|
||||
},
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
};
|
||||
|
||||
// UploadPart is acknowledged once this rename lands, so the part data and
|
||||
// its directory entry must be durable before we return. Relaxed keeps the
|
||||
// part payload fdatasync but leaves the directory entry to the page cache.
|
||||
@@ -6561,7 +6769,17 @@ impl DiskAPI for LocalDisk {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
|
||||
if let Some(transaction_publish_meta) = transaction_publish_meta {
|
||||
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
rename_all(&transaction_publish_meta, &dst_meta_path, &dst_volume_dir).await?;
|
||||
if durability.syncs_commit_metadata()
|
||||
&& let Some(parent) = dst_meta_path.parent()
|
||||
{
|
||||
os::fsync_dir(parent).await.map_err(to_file_error)?;
|
||||
}
|
||||
} else {
|
||||
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path.parent() {
|
||||
self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?;
|
||||
@@ -8282,6 +8500,12 @@ mod test {
|
||||
file_info.data_dir = data_dir;
|
||||
file_info.data = data;
|
||||
file_info.size = size;
|
||||
file_info.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: usize::try_from(size).expect("test object size should fit usize"),
|
||||
actual_size: size,
|
||||
..Default::default()
|
||||
}];
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
file_info
|
||||
}
|
||||
@@ -9370,9 +9594,15 @@ mod test {
|
||||
.await
|
||||
.expect("source part should be written");
|
||||
|
||||
disk.prepare_part_transaction(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_part(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
|
||||
.await
|
||||
.expect("rename_part should commit part");
|
||||
disk.settle_part_transaction(bucket, "object/part.1", PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("part transaction should be committed");
|
||||
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, "object/part.1")
|
||||
@@ -9390,6 +9620,71 @@ mod test {
|
||||
matches!(disk.read_all(tmp_volume, "upload/part.1").await, Err(DiskError::FileNotFound)),
|
||||
"source part must be removed after a successful commit"
|
||||
);
|
||||
|
||||
let legacy_payload = Bytes::from_static(b"legacy peer payload");
|
||||
let legacy_meta = Bytes::from_static(b"legacy peer metadata");
|
||||
disk.write_all(tmp_volume, "legacy/part.1", legacy_payload.clone())
|
||||
.await
|
||||
.expect("legacy source part should be written");
|
||||
disk.rename_part(tmp_volume, "legacy/part.1", bucket, "object/part.1", legacy_meta.clone())
|
||||
.await
|
||||
.expect("pre-transaction peer RenamePart should remain supported");
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, "object/part.1")
|
||||
.await
|
||||
.expect("legacy destination part should be readable"),
|
||||
legacy_payload
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, "object/part.1.meta")
|
||||
.await
|
||||
.expect("legacy destination metadata should be readable"),
|
||||
legacy_meta
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_part_transaction_rolls_back_data_published_before_metadata() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
ensure_test_volume(&disk, "tmp").await;
|
||||
ensure_test_volume(&disk, "bucket").await;
|
||||
|
||||
disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data"))
|
||||
.await
|
||||
.expect("new part should be staged");
|
||||
disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data"))
|
||||
.await
|
||||
.expect("old part data should be staged");
|
||||
disk.write_all("bucket", "object/part.1.meta", Bytes::from_static(b"old metadata"))
|
||||
.await
|
||||
.expect("old part metadata should be staged");
|
||||
|
||||
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata"))
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1")
|
||||
.await
|
||||
.expect("data publication should succeed");
|
||||
disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Rollback)
|
||||
.await
|
||||
.expect("part transaction should roll back");
|
||||
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object/part.1")
|
||||
.await
|
||||
.expect("old part data should be restored"),
|
||||
Bytes::from_static(b"old data")
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object/part.1.meta")
|
||||
.await
|
||||
.expect("old part metadata should be restored"),
|
||||
Bytes::from_static(b"old metadata")
|
||||
);
|
||||
}
|
||||
|
||||
struct BlockingScanWriter {
|
||||
|
||||
@@ -38,6 +38,16 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
|
||||
pub const HEALING_MARKER_PATH: &str = "healing.bin";
|
||||
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
|
||||
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
|
||||
pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
|
||||
pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
|
||||
pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
|
||||
|
||||
pub fn part_transaction_path(part_path: &str) -> String {
|
||||
match part_path.rsplit_once('/') {
|
||||
Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"),
|
||||
None => format!(".{part_path}.rustfs-txn"),
|
||||
}
|
||||
}
|
||||
|
||||
use crate::cluster::rpc::RemoteDisk;
|
||||
use crate::cluster::rpc::build_internode_data_transport_from_env;
|
||||
@@ -62,6 +72,12 @@ pub type DiskStore = Arc<Disk>;
|
||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PartTransactionAction {
|
||||
Commit,
|
||||
Rollback,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct MmapCopyStageMetrics {
|
||||
pub(crate) path: &'static str,
|
||||
@@ -381,6 +397,35 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.settle_part_transaction(volume, path, action).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.settle_part_transaction(volume, path, action).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
match self {
|
||||
@@ -659,6 +704,19 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
// ReadFileStream
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>;
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
_src_volume: &str,
|
||||
_src_path: &str,
|
||||
_dst_volume: &str,
|
||||
_dst_path: &str,
|
||||
_meta: Bytes,
|
||||
) -> Result<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn settle_part_transaction(&self, _volume: &str, _path: &str, _action: PartTransactionAction) -> Result<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
|
||||
// VerifyFile
|
||||
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
|
||||
|
||||
@@ -443,7 +443,9 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith
|
||||
size.div_ceil(shard_size) * algo.size() + size
|
||||
}
|
||||
|
||||
/// Verify an interleaved per-block bitrot shard file.
|
||||
/// Verify an interleaved per-block bitrot shard file and consume the reader
|
||||
/// through EOF. Bytes beyond the encoded length are corruption, even when every
|
||||
/// expected block has a valid hash.
|
||||
///
|
||||
/// The read loop below assumes every block on disk is `[hash][data]` (streaming
|
||||
/// bitrot). It is therefore only valid for the streaming Highway variants, whose
|
||||
@@ -487,6 +489,11 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||
left -= read;
|
||||
}
|
||||
|
||||
let mut trailing = [0u8; 1];
|
||||
if r.read(&mut trailing).await? != 0 {
|
||||
return Err(std::io::Error::other("bitrot shard file has trailing data"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -860,12 +867,19 @@ mod tests {
|
||||
.await
|
||||
.expect("valid bitrot shard file should verify");
|
||||
|
||||
let mut truncated = written.clone();
|
||||
truncated.pop();
|
||||
let err = bitrot_verify(Cursor::new(truncated), written.len(), data.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.expect_err("one-byte-short shard file must be rejected while reading");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
|
||||
|
||||
let err = bitrot_verify(Cursor::new(written.clone()), written.len() - 1, data.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.expect_err("wrong file size must be rejected before reading data");
|
||||
assert!(err.to_string().contains("size mismatch"));
|
||||
|
||||
let mut corrupt = written;
|
||||
let mut corrupt = written.clone();
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x80;
|
||||
let err = bitrot_verify(
|
||||
@@ -878,6 +892,21 @@ mod tests {
|
||||
.await
|
||||
.expect_err("hash mismatch must reject corrupted data");
|
||||
assert!(err.to_string().contains("hash mismatch"));
|
||||
|
||||
for trailing in [vec![0xa5], vec![0xa5; 17]] {
|
||||
let mut oversized = written.clone();
|
||||
oversized.extend_from_slice(&trailing);
|
||||
let err = bitrot_verify(
|
||||
Cursor::new(oversized),
|
||||
written.len(),
|
||||
data.len(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
shard_size,
|
||||
)
|
||||
.await
|
||||
.expect_err("trailing bytes after a valid encoded shard must be rejected");
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -909,6 +938,22 @@ mod tests {
|
||||
assert!(err.to_string().contains("hash mismatch"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_verify_accepts_exact_legacy_streaming_layout() {
|
||||
let data = b"legacy streaming bitrot";
|
||||
let shard_size = 8;
|
||||
let algo = HashAlgorithm::HighwayHash256SLegacy;
|
||||
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, algo.clone());
|
||||
for chunk in data.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("legacy streaming shard should encode");
|
||||
}
|
||||
let written = writer.into_inner().into_inner();
|
||||
|
||||
bitrot_verify(Cursor::new(written.clone()), written.len(), data.len(), algo, shard_size)
|
||||
.await
|
||||
.expect("exact legacy streaming shard should remain valid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_all_vectored_retries_partial_hash_and_data_writes_and_rejects_zero_write() {
|
||||
let mut writer = LimitedVectoredWriter {
|
||||
|
||||
@@ -28,7 +28,6 @@ use md5::{Digest, Md5};
|
||||
use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
|
||||
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use serde::Deserialize;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use sha2::Sha256;
|
||||
@@ -44,6 +43,7 @@ const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
|
||||
const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
|
||||
const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size";
|
||||
const DEFAULT_SSE_ALGORITHM: &str = "AES256";
|
||||
const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
const DARE_PAYLOAD_SIZE: i64 = 64 * 1024;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -1716,16 +1716,39 @@ fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context
|
||||
|
||||
fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
|
||||
let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?;
|
||||
let parts: Vec<&str> = encrypted_dek.split(':').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err(Error::other("invalid managed DEK format"));
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LocalSseDekEnvelope<'a> {
|
||||
version: u8,
|
||||
nonce: &'a str,
|
||||
ciphertext: &'a str,
|
||||
}
|
||||
|
||||
let (nonce, ciphertext) = match serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek) {
|
||||
Ok(envelope) => {
|
||||
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
|
||||
return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version)));
|
||||
}
|
||||
(envelope.nonce, envelope.ciphertext)
|
||||
}
|
||||
Err(_) => {
|
||||
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
|
||||
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
|
||||
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
|
||||
return Err(Error::other("invalid managed DEK format"));
|
||||
};
|
||||
if ciphertext.contains(':') {
|
||||
return Err(Error::other("invalid managed DEK format"));
|
||||
}
|
||||
(nonce, ciphertext)
|
||||
}
|
||||
};
|
||||
|
||||
let nonce_vec = BASE64_STANDARD
|
||||
.decode(parts[0])
|
||||
.decode(nonce)
|
||||
.map_err(|_| Error::other("invalid managed DEK nonce"))?;
|
||||
let ciphertext = BASE64_STANDARD
|
||||
.decode(parts[1])
|
||||
.decode(ciphertext)
|
||||
.map_err(|_| Error::other("invalid managed DEK ciphertext"))?;
|
||||
|
||||
let nonce_array: [u8; 12] = nonce_vec
|
||||
@@ -2324,9 +2347,36 @@ mod tests {
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::from([0u8; 12]);
|
||||
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek");
|
||||
serde_json::json!({
|
||||
"version": LOCAL_SSE_DEK_FORMAT_VERSION,
|
||||
"nonce": BASE64_STANDARD.encode(nonce),
|
||||
"ciphertext": BASE64_STANDARD.encode(ciphertext),
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String {
|
||||
let key = Key::<Aes256Gcm>::from(master_key);
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::from([0u8; 12]);
|
||||
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek");
|
||||
format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() {
|
||||
let envelope = serde_json::json!({
|
||||
"version": LOCAL_SSE_DEK_FORMAT_VERSION + 1,
|
||||
"nonce": BASE64_STANDARD.encode([0u8; 12]),
|
||||
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let error =
|
||||
decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed");
|
||||
assert!(error.to_string().contains("unsupported managed DEK format version"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
fn seal_managed_s3_object_key_for_test(
|
||||
bucket: &str,
|
||||
@@ -2433,7 +2483,7 @@ mod tests {
|
||||
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async {
|
||||
let data_key = [0x24; 32];
|
||||
let base_nonce = [0x14; 12];
|
||||
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [7u8; 32]);
|
||||
let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]);
|
||||
let metadata = HashMap::from([
|
||||
(
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
|
||||
@@ -46,7 +46,10 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::OldCurrentSize;
|
||||
use crate::disk::{
|
||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
use crate::io_support::bitrot::{
|
||||
@@ -3174,6 +3177,155 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result<bool> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
|
||||
let current_meta_path = format!("{dst_object}.meta");
|
||||
|
||||
let reads = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let transaction_meta_path = transaction_meta_path.clone();
|
||||
let rollback_path = rollback_path.clone();
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Ok((None, None, false));
|
||||
};
|
||||
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => true,
|
||||
Err(DiskError::FileNotFound) => false,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
|
||||
Err(_) => None,
|
||||
};
|
||||
Ok((transaction_meta, current_meta, rollback))
|
||||
}
|
||||
});
|
||||
let observations = join_all(reads).await.into_iter().collect::<disk::error::Result<Vec<_>>>()?;
|
||||
if observations.iter().all(|(transaction, _, _)| transaction.is_none()) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
|
||||
for (_, current, _) in &observations {
|
||||
if let Some(current) = current {
|
||||
*current_counts.entry(current.clone()).or_default() += 1;
|
||||
}
|
||||
}
|
||||
let current_quorum = current_counts
|
||||
.into_iter()
|
||||
.find_map(|(meta, count)| (count >= write_quorum).then_some(meta));
|
||||
|
||||
let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}");
|
||||
let old_meta_absent_path = format!("{transaction_path}/old.meta.absent");
|
||||
let decisions = observations
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, (transaction_meta, _, rollback))| {
|
||||
transaction_meta.as_ref().map(|meta| (index, meta.clone(), *rollback))
|
||||
})
|
||||
.map(|(index, transaction_meta, rollback)| {
|
||||
let disk = disks[index].clone();
|
||||
let old_meta_path = old_meta_path.clone();
|
||||
let old_meta_absent_path = old_meta_absent_path.clone();
|
||||
let current_quorum = current_quorum.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let action = if rollback {
|
||||
PartTransactionAction::Rollback
|
||||
} else if current_quorum.as_ref() == Some(&transaction_meta) {
|
||||
PartTransactionAction::Commit
|
||||
} else if let Some(current_quorum) = current_quorum {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
|
||||
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(DiskError::FileNotFound) => {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(_) => return Err(DiskError::FileCorrupt),
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
} else {
|
||||
PartTransactionAction::Rollback
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
|
||||
.await?;
|
||||
Ok(action == PartTransactionAction::Commit)
|
||||
}
|
||||
});
|
||||
|
||||
let results = join_all(decisions).await;
|
||||
if let Some(err) = results.iter().find_map(|result| result.as_ref().err()) {
|
||||
return Err(err.clone());
|
||||
}
|
||||
Ok(results.iter().any(|result| matches!(result, Ok(true))))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn recover_part_transactions(
|
||||
&self,
|
||||
part_path: &str,
|
||||
read_quorum: usize,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
|
||||
let mut transaction_parts = HashSet::new();
|
||||
let mut errs = Vec::with_capacity(listings.len());
|
||||
for listing in listings {
|
||||
match listing {
|
||||
Ok(entries) => {
|
||||
errs.push(None);
|
||||
for entry in entries {
|
||||
let name = entry.trim_end_matches('/');
|
||||
let Some(part_number) = name
|
||||
.strip_prefix(".part.")
|
||||
.and_then(|name| name.strip_suffix(".rustfs-txn"))
|
||||
.and_then(|number| number.parse::<usize>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
transaction_parts.insert(part_number);
|
||||
}
|
||||
}
|
||||
Err(err) => errs.push(Some(err)),
|
||||
}
|
||||
}
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
for part_number in transaction_parts {
|
||||
self.recover_part_transaction(&format!("{part_path}part.{part_number}"), write_quorum)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(disks, meta))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(in crate::set_disk) async fn rename_part(
|
||||
@@ -3187,23 +3339,46 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
// Do NOT pre-delete the destination part before renaming: the per-disk
|
||||
// `rename_part` replaces `part.N` atomically (std::fs::rename) and rewrites
|
||||
// `part.N.meta`, so the pre-delete is redundant — and destructive. It
|
||||
// opened a window where an already-committed (ACKed) part was removed on
|
||||
// every disk before the new rename landed, so a re-upload that then failed
|
||||
// quorum destroyed the committed part outright (backlog#853 / #799 B4).
|
||||
// The atomic rename overwrites in place; on quorum failure below we roll
|
||||
// the destination back.
|
||||
let prepare_tasks = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let meta = meta.clone();
|
||||
async move {
|
||||
let disk = disk?;
|
||||
Some(
|
||||
disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
});
|
||||
let prepare_results = join_all(prepare_tasks).await;
|
||||
let prepare_errs = prepare_results
|
||||
.into_iter()
|
||||
.map(|result| match result {
|
||||
Some(Ok(())) => None,
|
||||
Some(Err(err)) => Some(err),
|
||||
None => Some(DiskError::DiskNotFound),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let prepared_disks = Self::eval_disks(disks, &prepare_errs);
|
||||
if reduce_write_quorum_errs(&prepare_errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some() {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let futures = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let meta = meta.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
@@ -3253,19 +3428,42 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
let reduced_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
|
||||
if let Some(err) = reduced_err {
|
||||
let rollbacks = prepared_disks.iter().filter_map(|disk| {
|
||||
disk.clone().map(|disk| {
|
||||
let dst_object = dst_object.clone();
|
||||
async move {
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Rollback)
|
||||
.await
|
||||
}
|
||||
})
|
||||
});
|
||||
let rollback_results = join_all(rollbacks).await;
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
if let Some(rollback_err) = rollback_results.iter().find_map(|result| result.as_ref().err()) {
|
||||
warn!(error = %rollback_err, "rename_part rollback did not settle on every prepared disk");
|
||||
}
|
||||
if let Some(context) = quorum_context {
|
||||
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
|
||||
} else {
|
||||
warn!("rename_part errs {:?}", &errs);
|
||||
}
|
||||
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let disks = Self::eval_disks(disks, &errs);
|
||||
Ok(disks)
|
||||
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
if !committed {
|
||||
let err = DiskError::ErasureWriteQuorum;
|
||||
if let Some(context) = quorum_context {
|
||||
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
|
||||
} else {
|
||||
warn!("rename_part errs {:?}", &errs);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(Self::eval_disks(&prepared_disks, &errs))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
|
||||
|
||||
@@ -687,6 +687,8 @@ mod core;
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
|
||||
@@ -199,6 +199,32 @@ mod tests {
|
||||
.expect("healthy temp shard should verify");
|
||||
assert_eq!(verified, 1);
|
||||
|
||||
for trailing in [vec![0xa5], vec![0xa5; 17]] {
|
||||
let mut oversized = encoded.to_vec();
|
||||
oversized.extend_from_slice(&trailing);
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(oversized))
|
||||
.await
|
||||
.expect("oversized temp shard should be staged");
|
||||
let err = verify_written_bitrot_shards(
|
||||
&[Some(disk.clone())],
|
||||
None,
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
part_number: None,
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path,
|
||||
logical_shard_size,
|
||||
shard_size,
|
||||
write_quorum: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("disk shard with trailing bytes must not be committed");
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
|
||||
let mut corrupt = encoded.to_vec();
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x01;
|
||||
@@ -225,4 +251,40 @@ mod tests {
|
||||
.expect_err("corrupt no-parity temp shard must not be committed");
|
||||
assert!(err.to_string().contains("bitrot self-verify failed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_parity_inline_self_verify_rejects_trailing_bytes() {
|
||||
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
|
||||
let shard_size = 16usize;
|
||||
let payload = b"inline bitrot payload";
|
||||
let encoded = encode_streaming_shard(payload, shard_size).await;
|
||||
let disks = disks.into_iter().map(Some).collect::<Vec<_>>();
|
||||
|
||||
for trailing in [vec![0xa5], vec![0xa5; 17]] {
|
||||
let mut oversized = encoded.to_vec();
|
||||
oversized.extend_from_slice(&trailing);
|
||||
let parts = [FileInfo {
|
||||
data: Some(Bytes::from(oversized)),
|
||||
..Default::default()
|
||||
}];
|
||||
let err = verify_written_bitrot_shards(
|
||||
&disks,
|
||||
Some(&parts),
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "inline-object",
|
||||
part_number: None,
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path: "unused-for-inline",
|
||||
logical_shard_size: payload.len(),
|
||||
shard_size,
|
||||
write_quorum: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("inline shard with trailing bytes must not be committed");
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,71 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename";
|
||||
const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete";
|
||||
|
||||
#[cfg(test)]
|
||||
static HEAL_RENAME_FAILURES: std::sync::Mutex<Vec<(String, String, usize)>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
#[cfg(test)]
|
||||
struct HealRenameFailureScope {
|
||||
bucket: String,
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl HealRenameFailureScope {
|
||||
fn install(bucket: &str, object: &str, disk_indexes: &[usize]) -> Self {
|
||||
let mut failures = HEAL_RENAME_FAILURES
|
||||
.lock()
|
||||
.expect("heal rename failure registry should not poison");
|
||||
assert!(
|
||||
!failures
|
||||
.iter()
|
||||
.any(|(registered_bucket, registered_object, _)| { registered_bucket == bucket && registered_object == object }),
|
||||
"heal rename failures must be installed once per object"
|
||||
);
|
||||
failures.extend(
|
||||
disk_indexes
|
||||
.iter()
|
||||
.map(|index| (bucket.to_string(), object.to_string(), *index)),
|
||||
);
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for HealRenameFailureScope {
|
||||
fn drop(&mut self) {
|
||||
HEAL_RENAME_FAILURES
|
||||
.lock()
|
||||
.expect("heal rename failure registry should not poison")
|
||||
.retain(|(bucket, object, _)| bucket != &self.bucket || object != &self.object);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> bool {
|
||||
let mut failures = HEAL_RENAME_FAILURES
|
||||
.lock()
|
||||
.expect("heal rename failure registry should not poison");
|
||||
if let Some(position) = failures
|
||||
.iter()
|
||||
.position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index))
|
||||
{
|
||||
failures.swap_remove(position);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct PartFailureSummary {
|
||||
@@ -659,8 +724,12 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
// Rename from tmp location to the actual location.
|
||||
// MinIO stops on the first RenameData error. RustFS intentionally
|
||||
// continues per target, but reports any residue after all attempts
|
||||
// so successful repairs survive and failed targets remain retryable.
|
||||
let mut rename_attempts = 0usize;
|
||||
let mut rename_successes = 0usize;
|
||||
let mut healed_disks = vec![None; out_dated_disks.len()];
|
||||
for (index, outdated_disk) in out_dated_disks.iter().enumerate() {
|
||||
if let Some(disk) = outdated_disk {
|
||||
rename_attempts += 1;
|
||||
@@ -669,9 +738,18 @@ impl SetDisks {
|
||||
// Attempt a rename now from healed data to final location.
|
||||
parts_metadata[index].set_healing();
|
||||
|
||||
let rename_result = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object)
|
||||
.await;
|
||||
let rename_result = if should_fail_heal_rename(bucket, object, index) {
|
||||
Err(DiskError::Unexpected)
|
||||
} else {
|
||||
disk.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_id,
|
||||
parts_metadata[index].clone(),
|
||||
bucket,
|
||||
object,
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
if let Err(err) = &rename_result {
|
||||
warn!(
|
||||
@@ -690,6 +768,7 @@ impl SetDisks {
|
||||
);
|
||||
} else {
|
||||
rename_successes += 1;
|
||||
healed_disks[index] = Some(disk.clone());
|
||||
if parts_metadata[index].is_remote() {
|
||||
let rm_data_dir =
|
||||
parts_metadata[index].data_dir.expect("operation should succeed").to_string();
|
||||
@@ -734,15 +813,18 @@ impl SetDisks {
|
||||
.await
|
||||
.map_err(DiskError::other)?;
|
||||
|
||||
if rename_attempts > 0 && rename_successes == 0 {
|
||||
self.record_healed_capacity_scope(&healed_disks);
|
||||
|
||||
if rename_successes < rename_attempts {
|
||||
return Ok((
|
||||
result,
|
||||
Some(DiskError::other(format!("all healed data rename attempts failed for {bucket}/{object}"))),
|
||||
Some(DiskError::other(format!(
|
||||
"{HEAL_RENAME_INCOMPLETE}: {rename_successes} of {rename_attempts} targets committed for \
|
||||
{bucket}/{object}"
|
||||
))),
|
||||
));
|
||||
}
|
||||
|
||||
self.record_healed_capacity_scope(&out_dated_disks);
|
||||
|
||||
// The object is healthy here; sweep any data dirs left behind
|
||||
// by pre-#3510 unversioned overwrites, which the dangling paths
|
||||
// above never touch (issues #3231, #3191). Best effort — a
|
||||
@@ -1312,11 +1394,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
#[cfg(test)]
|
||||
mod heal_result_report_tests {
|
||||
use super::SetDisks;
|
||||
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::format::FormatV3;
|
||||
use crate::disk::{DiskOption, DiskStore, new_disk};
|
||||
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::{config::storageclass, store::init_format::save_format_file};
|
||||
@@ -1406,6 +1490,171 @@ mod heal_result_report_tests {
|
||||
(dir, set)
|
||||
}
|
||||
|
||||
async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
|
||||
let mut entries = Vec::new();
|
||||
for dir in temp_dirs {
|
||||
let tmp = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let mut read_dir = match tokio::fs::read_dir(&tmp).await {
|
||||
Ok(read_dir) => read_dir,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => panic!("tmp directory should be readable: {err}"),
|
||||
};
|
||||
while let Some(entry) = read_dir.next_entry().await.expect("tmp entry should be readable") {
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if name != ".trash" {
|
||||
entries.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_rename_outcome_matrix_reports_partial_and_retries_failed_targets() {
|
||||
for (case, failed_attempts, expect_error) in [
|
||||
("ok-ok", Vec::new(), false),
|
||||
("ok-err", vec![1], true),
|
||||
("err-ok", vec![0], true),
|
||||
("err-err", vec![0, 1], true),
|
||||
] {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = format!("heal-rename-{case}");
|
||||
let object = "object.bin";
|
||||
for disk in &disks {
|
||||
disk.make_volume(&bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let payload = vec![0x5a; 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let source = disks[2]
|
||||
.read_version("", &bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source metadata should be readable");
|
||||
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
|
||||
let tmp_entries_before_heal = non_trash_tmp_entries(&temp_dirs).await;
|
||||
let target_slots = {
|
||||
let mut slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
|
||||
slots.sort_unstable();
|
||||
slots
|
||||
};
|
||||
let failed_slots = failed_attempts
|
||||
.iter()
|
||||
.map(|attempt| target_slots[*attempt])
|
||||
.collect::<Vec<_>>();
|
||||
let failed_physical_indexes = [0, 1]
|
||||
.into_iter()
|
||||
.filter(|index| failed_slots.contains(&(source.erasure.distribution[*index] - 1)))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for index in [0, 1] {
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[index]
|
||||
.path()
|
||||
.join(&bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("target shard should be removed before heal");
|
||||
}
|
||||
|
||||
let failure_scope = HealRenameFailureScope::install(&bucket, object, &failed_slots);
|
||||
let (first_result, first_error) = set
|
||||
.heal_object(
|
||||
&bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal should report its per-target rename outcome");
|
||||
drop(failure_scope);
|
||||
|
||||
assert_eq!(first_error.is_some(), expect_error, "{case}: aggregate status must match target outcomes");
|
||||
if let Some(error) = first_error {
|
||||
let error = error.to_string();
|
||||
assert!(
|
||||
error.contains(HEAL_RENAME_INCOMPLETE),
|
||||
"{case}: partial/all failure must have an explicit retryable status: {error}"
|
||||
);
|
||||
assert!(
|
||||
error.contains(&format!("{} of 2 targets committed", 2 - failed_slots.len())),
|
||||
"{case}: aggregate status must distinguish partial from all-target failure: {error}"
|
||||
);
|
||||
}
|
||||
for index in [0, 1] {
|
||||
let expected = if failed_physical_indexes.contains(&index) {
|
||||
DriveState::Missing
|
||||
} else {
|
||||
DriveState::Ok
|
||||
};
|
||||
assert_eq!(
|
||||
first_result.after.drives[index].state,
|
||||
expected.to_string(),
|
||||
"{case}: after.drives must reflect the actual rename outcome at index {index}"
|
||||
);
|
||||
assert_eq!(
|
||||
temp_dirs[index]
|
||||
.path()
|
||||
.join(&bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1")
|
||||
.exists(),
|
||||
!failed_physical_indexes.contains(&index),
|
||||
"{case}: tmp cleanup must neither delete committed shards nor expose failed targets"
|
||||
);
|
||||
}
|
||||
let tmp_entries_after_heal = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
tmp_entries_after_heal
|
||||
.iter()
|
||||
.all(|entry| tmp_entries_before_heal.contains(entry)),
|
||||
"{case}: first heal must not leave a new temporary shard: {tmp_entries_after_heal:?}"
|
||||
);
|
||||
|
||||
if !failed_slots.is_empty() {
|
||||
let (retry_result, retry_error) = set
|
||||
.heal_object(
|
||||
&bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("second heal should retry failed targets");
|
||||
assert!(retry_error.is_none(), "{case}: second heal should complete remaining targets");
|
||||
for index in [0, 1] {
|
||||
assert_eq!(
|
||||
retry_result.after.drives[index].state,
|
||||
DriveState::Ok.to_string(),
|
||||
"{case}: second heal must converge target {index}"
|
||||
);
|
||||
}
|
||||
let tmp_entries_after_retry = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
tmp_entries_after_retry
|
||||
.iter()
|
||||
.all(|entry| tmp_entries_before_heal.contains(entry)),
|
||||
"{case}: retry must not leave a new temporary shard: {tmp_entries_after_retry:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Regression for #955: an offline disk must contribute exactly one drive
|
||||
// record. Before the fix the offline branch fell through and pushed a second
|
||||
// (Corrupt) record for the same disk, so `before/after.drives` grew to
|
||||
|
||||
@@ -69,6 +69,13 @@ struct HealWalkCollector {
|
||||
}
|
||||
|
||||
impl HealWalkCollector {
|
||||
fn lock_objects(&self) -> disk::error::Result<std::sync::MutexGuard<'_, Vec<HealWalkObject>>> {
|
||||
self.objects.lock().map_err(|_| {
|
||||
self.cancel.cancel();
|
||||
DiskError::FileCorrupt
|
||||
})
|
||||
}
|
||||
|
||||
/// Expand one resolved entry into its versions and record it. Cancels the
|
||||
/// walk once EITHER page bound (distinct object names OR expanded versions)
|
||||
/// is met — always at a sorted object-key boundary so a heavily-versioned
|
||||
@@ -105,7 +112,9 @@ impl HealWalkCollector {
|
||||
|
||||
let added = versions.len();
|
||||
let (objs_len, ver_total) = {
|
||||
let mut objects = self.objects.lock().unwrap();
|
||||
let Ok(mut objects) = self.lock_objects() else {
|
||||
return;
|
||||
};
|
||||
objects.push(HealWalkObject {
|
||||
name: entry.name,
|
||||
versions,
|
||||
@@ -239,7 +248,10 @@ impl SetDisks {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let objects = std::mem::take(&mut *collector.objects.lock().unwrap());
|
||||
let objects = {
|
||||
let mut objects = collector.lock_objects()?;
|
||||
std::mem::take(&mut *objects)
|
||||
};
|
||||
let truncated = collector.truncated.load(Ordering::SeqCst);
|
||||
Ok(finalize_heal_walk_page(objects, forward_to, truncated))
|
||||
}
|
||||
@@ -310,4 +322,28 @@ mod tests {
|
||||
assert_eq!(next_forward, None, "a complete final page must not carry a resume cursor");
|
||||
assert!(!truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_collector_state_cancels_the_walk() {
|
||||
let collector = Arc::new(HealWalkCollector {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 2,
|
||||
version_budget: 2,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
version_total: AtomicUsize::new(0),
|
||||
truncated: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
});
|
||||
let poison_target = Arc::clone(&collector);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poison_target.objects.lock().expect("fresh mutex should lock");
|
||||
panic!("poison heal collector");
|
||||
})
|
||||
.join();
|
||||
|
||||
let error = collector.lock_objects().expect_err("poisoned collection must fail closed");
|
||||
assert_eq!(error, DiskError::FileCorrupt);
|
||||
assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk");
|
||||
assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,12 @@ impl SetDisks {
|
||||
pub async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
ListOperations::new(self.ctx()).delete_all(bucket, prefix).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_all_with_quorum(&self, bucket: &str, prefix: &str, write_quorum: usize) -> Result<()> {
|
||||
ListOperations::new(self.ctx())
|
||||
.delete_all_with_quorum(bucket, prefix, write_quorum)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
/// List/prefix maintenance operations, borrowing the `SetDisks` core state
|
||||
@@ -48,6 +54,14 @@ impl<'a> ListOperations<'a> {
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
self.delete_all_inner(bucket, prefix, None).await
|
||||
}
|
||||
|
||||
async fn delete_all_with_quorum(&self, bucket: &str, prefix: &str, write_quorum: usize) -> Result<()> {
|
||||
self.delete_all_inner(bucket, prefix, Some(write_quorum)).await
|
||||
}
|
||||
|
||||
async fn delete_all_inner(&self, bucket: &str, prefix: &str, write_quorum: Option<usize>) -> Result<()> {
|
||||
let disks = self.ctx.disks().read().await;
|
||||
|
||||
let disks = disks.clone();
|
||||
@@ -79,6 +93,9 @@ impl<'a> ListOperations<'a> {
|
||||
Ok(_) => {
|
||||
errors.push(None);
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::PathNotFound | DiskError::VolumeNotFound) => {
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
errors.push(Some(e));
|
||||
}
|
||||
@@ -97,6 +114,12 @@ impl<'a> ListOperations<'a> {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(write_quorum) = write_quorum
|
||||
&& let Some(err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum)
|
||||
{
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
@@ -33,7 +35,7 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum MultipartCommitPause {
|
||||
pub(crate) enum MultipartCommitPause {
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
BeforeLockLost,
|
||||
@@ -45,12 +47,13 @@ struct MultipartCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
pause: MultipartCommitPause,
|
||||
armed: AtomicBool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct MultipartCommitBarrier {
|
||||
pub(crate) struct MultipartCommitBarrier {
|
||||
state: Arc<MultipartCommitBarrierState>,
|
||||
}
|
||||
|
||||
@@ -60,11 +63,12 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
|
||||
|
||||
#[cfg(test)]
|
||||
impl MultipartCommitBarrier {
|
||||
fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
let state = Arc::new(MultipartCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
pause,
|
||||
armed: AtomicBool::new(true),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
@@ -78,13 +82,13 @@ impl MultipartCommitBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
async fn wait_until_paused(&self) {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
@@ -112,7 +116,9 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
if let Some(barrier) = barrier
|
||||
&& barrier.armed.swap(false, Ordering::AcqRel)
|
||||
{
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
@@ -777,6 +783,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
mut max_parts: usize,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ListPartsInfo> {
|
||||
let _upload_guard = self
|
||||
.acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
|
||||
|
||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
@@ -1254,10 +1263,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let _upload_guard = self
|
||||
.acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
self.check_upload_id_exists(bucket, object, upload_id, false).await?;
|
||||
let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
|
||||
self.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await
|
||||
self.delete_all_with_quorum(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&upload_id_path,
|
||||
fi.write_quorum(self.default_write_quorum()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
// complete_multipart_upload finished
|
||||
#[tracing::instrument(skip(self))]
|
||||
@@ -1289,8 +1303,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.no_lock && object_lock_guard.is_none() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let upload_guard = self
|
||||
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
|
||||
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let (mut fi, mut files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
let has_layout_candidate = range_seek_rollout_enabled
|
||||
&& fi
|
||||
.data_dir
|
||||
@@ -1303,22 +1327,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
&token,
|
||||
)
|
||||
});
|
||||
let upload_guard = if has_layout_candidate {
|
||||
if object_lock_guard.is_none() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let guard = self
|
||||
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
(fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
guard
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let quorum_validated_layout_token = upload_guard.as_ref().and_then(|_| {
|
||||
let quorum_validated_layout_token = if has_layout_candidate {
|
||||
fi.data_dir
|
||||
.filter(|data_dir| !data_dir.is_nil())
|
||||
.map(|data_dir| data_dir.to_string())
|
||||
@@ -1329,7 +1338,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
token,
|
||||
)
|
||||
})
|
||||
});
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut fi.metadata,
|
||||
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
|
||||
@@ -1356,6 +1367,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
|
||||
|
||||
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let part_meta_paths = uploaded_parts
|
||||
.iter()
|
||||
@@ -1713,12 +1727,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.no_lock && object_lock_guard.is_none() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
// Phase 2 (backlog#899): fence the commit on lock loss before any destructive
|
||||
// step. If the refresh heartbeat has observed a refresh-quorum loss, another
|
||||
// writer may have re-acquired this object's lock; proceeding would race a
|
||||
@@ -1821,7 +1829,36 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::AfterRename).await;
|
||||
drop(upload_guard);
|
||||
|
||||
let cleanup_store = self.clone();
|
||||
let cleanup_upload_id_path = upload_id_path.clone();
|
||||
let cleanup_bucket = bucket.to_owned();
|
||||
let cleanup_object = object.to_owned();
|
||||
let cleanup_upload_id = upload_id.to_owned();
|
||||
let cleanup_handle = tokio::spawn(async move {
|
||||
let _upload_guard = upload_guard;
|
||||
if let Err(err) = cleanup_store
|
||||
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_id_path, write_quorum)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket = %cleanup_bucket,
|
||||
object = %cleanup_object,
|
||||
upload_id = %cleanup_upload_id,
|
||||
error = ?err,
|
||||
"completed multipart upload staging cleanup did not reach write quorum"
|
||||
);
|
||||
}
|
||||
});
|
||||
if let Err(err) = cleanup_handle.await {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
upload_id = %upload_id,
|
||||
error = ?err,
|
||||
"completed multipart upload staging cleanup task failed"
|
||||
);
|
||||
}
|
||||
drop(object_lock_guard); // drop object lock guard to release the lock
|
||||
|
||||
// backlog#1321: enqueue heal only when the committed replicas actually
|
||||
@@ -1866,12 +1903,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
let upload_id_path = upload_id_path.clone();
|
||||
let store = self.clone();
|
||||
let _cleanup_handle = tokio::spawn(async move {
|
||||
let _ = store.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await;
|
||||
});
|
||||
|
||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||
if let Some(disk) = op_disk
|
||||
&& disk.is_online().await
|
||||
@@ -2183,6 +2214,317 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x47; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
assert!(!abort.is_finished(), "abort must wait for the completion upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should win the upload finalization");
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("abort must observe the upload as finalized");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("complete-first must leave the committed object readable");
|
||||
assert!(matches!(
|
||||
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
}
|
||||
|
||||
async fn assert_abort_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x48; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let object_holder = set_disks
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.expect("object namespace lock should be created")
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("test should hold the object lock");
|
||||
let holder = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await
|
||||
.expect("upload namespace lock should be created")
|
||||
.get_write_lock(Duration::from_secs(5))
|
||||
.await
|
||||
.expect("test should hold the upload lock");
|
||||
signaling.wait_for_attempts(1).await;
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
drop(holder);
|
||||
|
||||
abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect("abort should win the upload finalization");
|
||||
drop(object_holder);
|
||||
let complete_err = complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect_err("completion must observe the aborted upload");
|
||||
assert!(matches!(complete_err, StorageError::InvalidUploadID(..)));
|
||||
let object_err = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("abort-first must not publish an object");
|
||||
assert!(matches!(object_err, StorageError::ObjectNotFound(..)));
|
||||
assert!(matches!(
|
||||
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
}
|
||||
|
||||
async fn assert_quorum_minus_one_retry_preserves_completable_part(
|
||||
disk_count: usize,
|
||||
parity: usize,
|
||||
success_indices: &[usize],
|
||||
) {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await;
|
||||
let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]);
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, &bucket).await;
|
||||
|
||||
let payload = vec![0x41; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, &bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let (upload_meta, _) = set_disks
|
||||
.check_upload_id_exists(&bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("staged upload metadata should be readable");
|
||||
let upload_path = SetDisks::get_upload_id_dir(&bucket, object, &upload_id);
|
||||
let write_quorum = upload_meta.write_quorum(set_disks.default_write_quorum());
|
||||
assert_eq!(success_indices.len() + 1, write_quorum);
|
||||
let part_path = format!(
|
||||
"{}/{}/part.1",
|
||||
upload_path,
|
||||
upload_meta.data_dir.expect("multipart upload should have a data directory")
|
||||
);
|
||||
let acknowledged_meta = disk_stores[0]
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{part_path}.meta"))
|
||||
.await
|
||||
.expect("acknowledged part metadata should be readable");
|
||||
let retry_path = format!("{}/part.1", Uuid::new_v4());
|
||||
|
||||
let mut retry_disks = vec![None; disk_stores.len()];
|
||||
for &index in success_indices {
|
||||
disk_stores[index]
|
||||
.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"retry shard"))
|
||||
.await
|
||||
.expect("retry shard should be staged");
|
||||
retry_disks[index] = Some(disk_stores[index].clone());
|
||||
}
|
||||
|
||||
let err = set_disks
|
||||
.rename_part(
|
||||
&retry_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&retry_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
acknowledged_meta,
|
||||
write_quorum,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("quorum-minus-one renamed shards must remain below write quorum");
|
||||
assert_eq!(err, DiskError::ErasureWriteQuorum);
|
||||
|
||||
for (index, disk) in disk_stores.iter().enumerate() {
|
||||
assert!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path).await.is_ok(),
|
||||
"old acknowledged shard on disk {index} must survive"
|
||||
);
|
||||
}
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(&bucket, object, &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the old acknowledged part should remain completable");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should be readable");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_part_retry_quorum_failure_preserves_old_part_across_ec_geometries() {
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[0, 1]).await;
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[2, 3]).await;
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[0, 1, 2]).await;
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[3, 4, 5]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_upload_recovers_interrupted_part_retry() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await;
|
||||
let bucket = "multipart-retry-recovery";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let payload = vec![0x42; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let (upload_meta, _) = set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("staged upload metadata should be readable");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_path = format!(
|
||||
"{}/{}/part.1",
|
||||
upload_path,
|
||||
upload_meta.data_dir.expect("multipart upload should have a data directory")
|
||||
);
|
||||
let retry_meta = Bytes::from_static(b"interrupted retry metadata");
|
||||
|
||||
for (index, disk) in disk_stores.iter().enumerate().take(3) {
|
||||
let retry_path = format!("{}/part.1", Uuid::new_v4());
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"interrupted retry shard"))
|
||||
.await
|
||||
.expect("retry shard should be staged");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&retry_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
retry_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&retry_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
retry_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("retry shard {index} should be published: {err}"));
|
||||
}
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completion should roll back the interrupted quorum-minus-one retry");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should be readable");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_part_quorum_failure_without_old_part_removes_new_shards() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
let src_path = format!("{}/part.1", Uuid::new_v4());
|
||||
let dst_path = format!("{}/part.1", Uuid::new_v4());
|
||||
|
||||
let mut retry_disks = vec![None; disk_stores.len()];
|
||||
for index in [0, 1] {
|
||||
disk_stores[index]
|
||||
.write_all(RUSTFS_META_TMP_BUCKET, &src_path, Bytes::from_static(b"new shard"))
|
||||
.await
|
||||
.expect("new shard should be staged");
|
||||
retry_disks[index] = Some(disk_stores[index].clone());
|
||||
}
|
||||
|
||||
let err = set_disks
|
||||
.rename_part(
|
||||
&retry_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&src_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&dst_path,
|
||||
Bytes::from_static(b"retry metadata"),
|
||||
3,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("two renamed shards must remain below write quorum");
|
||||
assert_eq!(err, DiskError::ErasureWriteQuorum);
|
||||
for (index, disk) in disk_stores.iter().enumerate() {
|
||||
assert!(
|
||||
matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &dst_path).await, Err(DiskError::FileNotFound)),
|
||||
"failed first upload must not leave a destination on disk {index}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_multipart_lock_test_set_disks() -> Arc<SetDisks> {
|
||||
let endpoints = vec![
|
||||
Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
|
||||
@@ -2850,6 +3192,81 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn abort_and_complete_linearize_for_plain_sse_and_legacy_layouts() {
|
||||
assert_complete_first_linearizes("multipart-complete-first-plain", "object", ObjectOptions::default()).await;
|
||||
assert_abort_first_linearizes("multipart-abort-first-plain", "object", ObjectOptions::default()).await;
|
||||
|
||||
let encrypted_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
assert_complete_first_linearizes("multipart-complete-first-sse", "object", encrypted_opts.clone()).await;
|
||||
assert_abort_first_linearizes("multipart-abort-first-sse", "object", encrypted_opts.clone()).await;
|
||||
})
|
||||
.await;
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("false"))], async {
|
||||
assert_complete_first_linearizes("multipart-complete-first-legacy", "object", encrypted_opts.clone()).await;
|
||||
assert_abort_first_linearizes("multipart-abort-first-legacy", "object", encrypted_opts).await;
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_enforces_delete_write_quorum_boundary() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-abort-delete-quorum";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let quorum_upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
|
||||
let saved_disks = {
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
let saved = disks.clone();
|
||||
disks[3] = None;
|
||||
saved
|
||||
};
|
||||
set_disks
|
||||
.abort_multipart_upload(bucket, object, &quorum_upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("abort should succeed at the exact delete write quorum");
|
||||
*set_disks.disks.write().await = saved_disks;
|
||||
assert!(matches!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &quorum_upload.upload_id, false)
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
|
||||
let below_quorum_upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("second multipart upload should be created");
|
||||
let saved_disks = {
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
let saved = disks.clone();
|
||||
disks[2] = None;
|
||||
disks[3] = None;
|
||||
saved
|
||||
};
|
||||
let err = set_disks
|
||||
.abort_multipart_upload(bucket, object, &below_quorum_upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("abort must report a delete below write quorum");
|
||||
assert!(matches!(err, StorageError::ErasureWriteQuorum));
|
||||
|
||||
*set_disks.disks.write().await = saved_disks;
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &below_quorum_upload.upload_id, false)
|
||||
.await
|
||||
.expect("failed abort must leave quorum-visible staging on the restored disks");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_revalidates_layout_candidate_after_upload_lock() {
|
||||
@@ -2981,6 +3398,17 @@ mod tests {
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
.await
|
||||
@@ -2991,10 +3419,68 @@ mod tests {
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
let list_err = list
|
||||
.await
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_validates_parts_after_an_inflight_upload_part_commit() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-complete-put-part-race-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, original_parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x49; 4096], &ObjectOptions::default()).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
let put_upload_id = upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x4a; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, original_parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!complete.is_finished(), "completion must wait for the UploadPart commit lock");
|
||||
|
||||
barrier.release();
|
||||
put.await
|
||||
.expect("UploadPart task should not panic")
|
||||
.expect("UploadPart replacement should commit");
|
||||
let err = complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect_err("completion must reject the stale ETag after UploadPart wins");
|
||||
assert!(matches!(err, StorageError::InvalidPart(..)));
|
||||
set_disks
|
||||
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("failed completion must leave the upload retryable");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn complete_fences_upload_lock_loss_before_commit() {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -207,11 +207,9 @@ impl SetDisks {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<Vec<FileInfo>> {
|
||||
// Use existing disk selection logic
|
||||
let disks = self.disks.read().await;
|
||||
let required_reads = self.format.erasure.sets.len();
|
||||
let disks = self.disks.read().await.clone();
|
||||
let required_reads = self.default_read_quorum();
|
||||
|
||||
// Clone parameters outside the closure to avoid lifetime issues
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let version_id = version_id.to_string();
|
||||
@@ -220,7 +218,6 @@ impl SetDisks {
|
||||
let processor = runtime_sources::batch_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
.iter()
|
||||
.take(required_reads + 2) // Read a few extra for reliability
|
||||
.filter_map(|disk| {
|
||||
disk.as_ref().map(|d| {
|
||||
let disk = d.clone();
|
||||
@@ -234,10 +231,10 @@ impl SetDisks {
|
||||
})
|
||||
.collect();
|
||||
|
||||
match processor.execute_batch_with_quorum(tasks, required_reads).await {
|
||||
Ok(results) => Ok(results),
|
||||
Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type
|
||||
}
|
||||
processor
|
||||
.execute_batch_with_quorum(tasks, required_reads)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
@@ -1960,6 +1957,42 @@ mod metadata_cache_tests {
|
||||
(dir, disk)
|
||||
}
|
||||
|
||||
async fn read_version_quorum_test_set(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pool_set_count: usize,
|
||||
readable_disks: usize,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let (dir, disk) = new_read_version_test_disk(bucket).await;
|
||||
if disk_index < readable_disks {
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.erasure.index = fi.erasure.distribution[disk_index];
|
||||
disk.write_metadata(bucket, bucket, object, fi)
|
||||
.await
|
||||
.expect("metadata should be written before quorum read");
|
||||
}
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = SetDisks::new(
|
||||
"read-version-quorum-test".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
Vec::new(),
|
||||
FormatV3::new(pool_set_count, 4),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
(dirs, set)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn get_object_metadata_cache_capacity_uses_default_and_env_override() {
|
||||
@@ -1983,9 +2016,40 @@ mod metadata_cache_tests {
|
||||
fi.size = 1;
|
||||
fi.erasure.index = 1;
|
||||
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
|
||||
fi.add_object_part(1, "part-etag".to_string(), 1, fi.mod_time, 1, None, None);
|
||||
fi
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_rejects_positive_size_without_parts() {
|
||||
let mut fi = valid_test_fileinfo("object");
|
||||
fi.parts.clear();
|
||||
|
||||
let mut output = Vec::new();
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
0,
|
||||
1,
|
||||
&mut output,
|
||||
fi,
|
||||
Vec::new(),
|
||||
&[],
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
)
|
||||
.await
|
||||
.expect_err("positive-size metadata without parts must fail without panicking");
|
||||
|
||||
assert_eq!(err, Error::FileCorrupt);
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_rejects_invalid_ranges_before_reader_setup() {
|
||||
let bucket = "bucket";
|
||||
@@ -2087,6 +2151,38 @@ mod metadata_cache_tests {
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_accepts_zero_size_without_parts() {
|
||||
let bucket = "bucket";
|
||||
let object = "empty";
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.size = 0;
|
||||
fi.parts.clear();
|
||||
|
||||
let mut output = Vec::new();
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
0,
|
||||
0,
|
||||
&mut output,
|
||||
fi,
|
||||
Vec::new(),
|
||||
&[],
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"empty",
|
||||
)
|
||||
.await
|
||||
.expect("zero-byte object without parts must remain readable");
|
||||
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_fails_closed_without_read_quorum() {
|
||||
let bucket = "bucket";
|
||||
@@ -2094,12 +2190,6 @@ mod metadata_cache_tests {
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.erasure.block_size = 1;
|
||||
fi.erasure.distribution = vec![1, 2, 3, 4];
|
||||
fi.parts.push(ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut output = Vec::new();
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
@@ -2133,6 +2223,7 @@ mod metadata_cache_tests {
|
||||
let object = "object";
|
||||
let (_dir, disk) = new_read_version_test_disk(bucket).await;
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.size = 0;
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
disk.write_metadata(bucket, bucket, object, fi.clone())
|
||||
.await
|
||||
@@ -2165,11 +2256,70 @@ mod metadata_cache_tests {
|
||||
.await
|
||||
.expect_err("empty disk set must fail closed");
|
||||
assert!(
|
||||
missing_quorum.to_string().to_ascii_lowercase().contains("file"),
|
||||
"optimized read failure should map to file-not-found style error: {missing_quorum}"
|
||||
missing_quorum.to_string().contains("Insufficient successful results"),
|
||||
"optimized read failure should preserve the batch quorum diagnostic: {missing_quorum}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_version_optimized_uses_current_set_drive_quorum_not_pool_set_count() {
|
||||
let bucket = "read-version-layout-quorum-bucket";
|
||||
let object = "object";
|
||||
|
||||
let (_single_set_dirs, single_set) = read_version_quorum_test_set(bucket, object, 1, 1).await;
|
||||
single_set
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("one metadata copy must not satisfy a four-drive 2+2 set");
|
||||
|
||||
let (_multi_set_dirs, multi_set) = read_version_quorum_test_set(bucket, object, 3, 2).await;
|
||||
let versions = multi_set
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("two metadata copies should satisfy the current set's read quorum");
|
||||
assert_eq!(versions.len(), 2);
|
||||
assert!(versions.iter().all(|version| version.name == object));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_version_optimized_counts_only_valid_metadata_toward_quorum() {
|
||||
let bucket = "read-version-corrupt-quorum-bucket";
|
||||
let object = "object";
|
||||
|
||||
let (quorum_minus_one_dirs, quorum_minus_one) = read_version_quorum_test_set(bucket, object, 1, 2).await;
|
||||
tokio::fs::write(
|
||||
quorum_minus_one_dirs[1]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE),
|
||||
b"corrupt metadata",
|
||||
)
|
||||
.await
|
||||
.expect("metadata should be corrupted for the test");
|
||||
quorum_minus_one
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("one valid and one corrupt metadata copy must not satisfy read quorum");
|
||||
|
||||
let (quorum_dirs, quorum) = read_version_quorum_test_set(bucket, object, 1, 3).await;
|
||||
tokio::fs::write(
|
||||
quorum_dirs[2]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE),
|
||||
b"corrupt metadata",
|
||||
)
|
||||
.await
|
||||
.expect("metadata should be corrupted for the test");
|
||||
let versions = quorum
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("two valid metadata copies must satisfy read quorum despite one corrupt copy");
|
||||
assert_eq!(versions.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_info_and_quorum_maps_delete_marker_and_purge_states() {
|
||||
let bucket = "get-object-info-marker-bucket";
|
||||
|
||||
@@ -6696,7 +6696,7 @@ mod test {
|
||||
use crate::object_api::ObjectInfo;
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker,
|
||||
VersionType,
|
||||
ObjectPartInfo, VersionType,
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
@@ -6899,6 +6899,12 @@ mod test {
|
||||
fi.volume = "bucket".to_owned();
|
||||
fi.name = "object".to_owned();
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.fresh = true;
|
||||
fi.erasure.index = 1;
|
||||
fi.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"));
|
||||
@@ -6954,6 +6960,12 @@ mod test {
|
||||
fi.version_id = Some(Uuid::from_u128(version_idx));
|
||||
fi.versioned = true;
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.mod_time = Some(*mod_time);
|
||||
fi.metadata = metadata;
|
||||
|
||||
|
||||
@@ -2601,10 +2601,10 @@ mod tests {
|
||||
// (backlog#1304): restore entry no longer serializes on the object lock.
|
||||
// The replacement semantics — non-blocking reads during the copy-back and
|
||||
// fast rejection of a concurrent restore — are covered end-to-end by
|
||||
// `restore_object_usecase_reports_ongoing_conflict_and_completion`
|
||||
// (rustfs/src/app/lifecycle_transition_api_test.rs) and at the lock level
|
||||
// by the accept-guard test below; restore-vs-reader data protection lives
|
||||
// in the inner put_object/complete_multipart_upload commit locks.
|
||||
// `restore_object_usecase_reports_ongoing_conflict`
|
||||
// (rustfs/src/app/lifecycle_transition_api_test.rs), while the SetDisks
|
||||
// transition matrix covers the final local commit. Restore-vs-reader data
|
||||
// protection lives in the inner put_object/complete_multipart_upload locks.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_accept_guard_serializes_concurrent_accepts() {
|
||||
|
||||
@@ -20,5 +20,5 @@ Example:
|
||||
```powershell
|
||||
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
|
||||
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
|
||||
cargo +1.96.0 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
|
||||
cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
|
||||
```
|
||||
|
||||
@@ -415,6 +415,10 @@ impl FileInfo {
|
||||
}
|
||||
|
||||
fn validate_collection_contents(&self, layout: Option<&ValidatedErasureLayout>) -> Result<()> {
|
||||
if layout.is_some() && self.size > 0 && self.parts.is_empty() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
let mut part_numbers = [0u64; FILEINFO_PART_BITMAP_WORDS];
|
||||
for part in &self.parts {
|
||||
if let Some(layout) = layout {
|
||||
@@ -692,6 +696,10 @@ impl FileInfo {
|
||||
|
||||
// to_part_offset gets the part index where offset is located, returns part index and offset
|
||||
pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> {
|
||||
if self.size > 0 && self.parts.is_empty() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
if offset == 0 {
|
||||
return Ok((0, 0));
|
||||
}
|
||||
@@ -1185,6 +1193,21 @@ mod tests {
|
||||
assert_eq!(layout.expect("strict layout must be present").data_blocks, 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_require_erasure_rejects_positive_size_without_parts() {
|
||||
let mut fi = FileInfo::new("bucket/object", 4, 2);
|
||||
fi.erasure.index = 1;
|
||||
fi.size = 1;
|
||||
|
||||
assert_eq!(fi.validate_for_metadata_read(), Err(Error::FileCorrupt));
|
||||
assert_eq!(fi.to_part_offset(0), Err(Error::FileCorrupt));
|
||||
|
||||
fi.size = 0;
|
||||
fi.validate_for_metadata_read()
|
||||
.expect("zero-byte object metadata may omit parts");
|
||||
assert_eq!(fi.to_part_offset(0), Ok((0, 0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_for_erasure_write_only_relaxes_pending_shard_index() {
|
||||
let mut fi = validation_test_fileinfo();
|
||||
|
||||
+117
-14
@@ -182,14 +182,9 @@ impl FileMeta {
|
||||
// TODO: use old buf
|
||||
let meta_buf = ver.marshal_msg()?;
|
||||
|
||||
let pre_mod_time = self.versions[idx].header.mod_time;
|
||||
|
||||
self.versions[idx].header = ver.header();
|
||||
self.versions[idx].meta = meta_buf;
|
||||
|
||||
if pre_mod_time != self.versions[idx].header.mod_time {
|
||||
self.sort_by_mod_time();
|
||||
}
|
||||
self.sort_by_mod_time();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -356,15 +351,10 @@ impl FileMeta {
|
||||
return self.set_idx(fidx, version);
|
||||
}
|
||||
|
||||
let mod_time = version.get_mod_time();
|
||||
let new_shallow = FileMetaShallowVersion::try_from(version)?;
|
||||
let insert_pos = match mod_time {
|
||||
Some(nm) => self.versions.partition_point(|exist| match exist.header.mod_time {
|
||||
Some(em) => em > nm,
|
||||
None => false,
|
||||
}),
|
||||
None => self.versions.partition_point(|exist| exist.header.mod_time.is_some()),
|
||||
};
|
||||
let insert_pos = self
|
||||
.versions
|
||||
.partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
|
||||
self.versions.insert(insert_pos, new_shallow);
|
||||
Ok(())
|
||||
|
||||
@@ -1112,6 +1102,119 @@ mod test {
|
||||
assert!(fm.is_sorted_by_mod_time());
|
||||
}
|
||||
|
||||
fn version_for_ordering(
|
||||
version_type: VersionType,
|
||||
version_id: Uuid,
|
||||
mod_time: OffsetDateTime,
|
||||
marker: u8,
|
||||
) -> FileMetaVersion {
|
||||
match version_type {
|
||||
VersionType::Object => FileMetaVersion {
|
||||
version_type,
|
||||
object: Some(MetaObject {
|
||||
version_id: Some(version_id),
|
||||
erasure_algorithm: ErasureAlgo::ReedSolomon,
|
||||
erasure_m: 2,
|
||||
erasure_n: 2,
|
||||
erasure_block_size: 1 << 20,
|
||||
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
|
||||
mod_time: Some(mod_time),
|
||||
meta_sys: HashMap::from([("ordering-marker".to_string(), vec![marker])]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
VersionType::Delete => FileMetaVersion {
|
||||
version_type,
|
||||
delete_marker: Some(MetaDeleteMarker {
|
||||
version_id: Some(version_id),
|
||||
mod_time: Some(mod_time),
|
||||
meta_sys: HashMap::from([("ordering-marker".to_string(), vec![marker])]),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
_ => unreachable!("ordering regression only constructs object and delete versions"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_uses_canonical_equal_time_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let object = version_for_ordering(VersionType::Object, Uuid::from_u128(1), mod_time, 1);
|
||||
let delete_marker = version_for_ordering(VersionType::Delete, Uuid::from_u128(2), mod_time, 2);
|
||||
|
||||
for versions in [[object.clone(), delete_marker.clone()], [delete_marker, object]] {
|
||||
let mut fm = FileMeta::new();
|
||||
for version in versions {
|
||||
fm.add_version_filemata(version).expect("add equal-time version");
|
||||
}
|
||||
|
||||
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
|
||||
assert_eq!(fm.versions[1].header.version_type, VersionType::Delete);
|
||||
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_preserves_tie_breaks_after_reload() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let versions = [
|
||||
version_for_ordering(VersionType::Object, Uuid::from_u128(10), mod_time, 10),
|
||||
version_for_ordering(VersionType::Object, Uuid::from_u128(20), mod_time, 20),
|
||||
version_for_ordering(VersionType::Delete, Uuid::from_u128(30), mod_time, 30),
|
||||
];
|
||||
let mut expected = versions.iter().map(FileMetaVersion::header).collect::<Vec<_>>();
|
||||
expected.sort_by(|a, b| {
|
||||
if a.sorts_before(b) {
|
||||
Ordering::Less
|
||||
} else if b.sorts_before(a) {
|
||||
Ordering::Greater
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
});
|
||||
|
||||
let mut fm = FileMeta::new();
|
||||
for version in versions.into_iter().rev() {
|
||||
fm.add_version_filemata(version).expect("add equal-time version");
|
||||
}
|
||||
let loaded = FileMeta::load(&fm.marshal_msg().expect("serialize file metadata")).expect("reload file metadata");
|
||||
let actual = loaded
|
||||
.versions
|
||||
.iter()
|
||||
.map(|version| version.header.clone())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_ne!(expected[0].signature, expected[1].signature, "fixture must exercise signature ordering");
|
||||
assert_eq!(actual, expected);
|
||||
assert!(actual.windows(2).all(|pair| pair[0].sorts_before(&pair[1])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_reorders_equal_time_replacement() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let first = version_for_ordering(VersionType::Object, Uuid::from_u128(40), mod_time, 40);
|
||||
let second = version_for_ordering(VersionType::Object, Uuid::from_u128(50), mod_time, 50);
|
||||
let (target, peer) = if first.header().sorts_before(&second.header()) {
|
||||
(first, second)
|
||||
} else {
|
||||
(second, first)
|
||||
};
|
||||
let target_id = target.header().version_id.expect("test version id");
|
||||
|
||||
let mut fm = FileMeta::new();
|
||||
fm.add_version_filemata(peer).expect("add peer object");
|
||||
fm.add_version_filemata(target).expect("add target object");
|
||||
assert_eq!(fm.versions[0].header.version_id, Some(target_id));
|
||||
|
||||
fm.add_version_filemata(version_for_ordering(VersionType::Delete, target_id, mod_time, 60))
|
||||
.expect("replace target with equal-time delete marker");
|
||||
|
||||
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
|
||||
assert_eq!(fm.versions[1].header.version_type, VersionType::Delete);
|
||||
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
|
||||
}
|
||||
|
||||
/// Regression for backlog#799 B16: replication reset state must be
|
||||
/// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
|
||||
/// key (produced by `ObjectInfo::replication_state`) has no internal prefix,
|
||||
|
||||
@@ -144,6 +144,7 @@ fn is_recoverable_heal_error_message(error: &str) -> bool {
|
||||
"connection refused",
|
||||
"operation canceled",
|
||||
"quorum not reached",
|
||||
"heal rename incomplete",
|
||||
]
|
||||
.iter()
|
||||
.any(|pattern| error.contains(pattern))
|
||||
@@ -154,3 +155,22 @@ impl From<Error> for std::io::Error {
|
||||
std::io::Error::other(err)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Error;
|
||||
use crate::heal::EcstoreError;
|
||||
|
||||
#[test]
|
||||
fn incomplete_target_rename_is_recoverable() {
|
||||
let task_error = Error::TaskExecutionFailed {
|
||||
message: "heal rename incomplete: 1 of 2 targets committed".to_string(),
|
||||
};
|
||||
let storage_error = Error::Storage(EcstoreError::Io(std::io::Error::other(
|
||||
"heal rename incomplete: 1 of 2 targets committed",
|
||||
)));
|
||||
|
||||
assert!(task_error.is_recoverable_heal());
|
||||
assert!(storage_error.is_recoverable_heal());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,6 +161,14 @@ pub struct ConfigureStaticKmsRequest {
|
||||
pub allow_insecure_dev_defaults: Option<bool>,
|
||||
}
|
||||
|
||||
impl Drop for ConfigureStaticKmsRequest {
|
||||
fn drop(&mut self) {
|
||||
use zeroize::Zeroize;
|
||||
|
||||
self.secret_key.zeroize();
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ConfigureStaticKmsRequest {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("ConfigureStaticKmsRequest")
|
||||
|
||||
@@ -23,16 +23,17 @@
|
||||
|
||||
use crate::backends::{BackendInfo, KmsBackend, KmsClient};
|
||||
use crate::config::{BackendConfig, KmsConfig};
|
||||
use crate::encryption::DataKeyEnvelope;
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::types::*;
|
||||
use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
aead::{Aead, KeyInit},
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use tracing::debug;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
@@ -41,6 +42,11 @@ const NONCE_SIZE: usize = 12;
|
||||
/// AES-256 key size in bytes.
|
||||
const KEY_SIZE: usize = 32;
|
||||
|
||||
fn context_aad(context: &HashMap<String, String>) -> Result<Vec<u8>> {
|
||||
let canonical: BTreeMap<&str, &str> = context.iter().map(|(key, value)| (key.as_str(), value.as_str())).collect();
|
||||
serde_json::to_vec(&canonical).map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Static single-key KMS backend.
|
||||
///
|
||||
/// Uses a pre-configured AES-256 key to derive data encryption keys. This is a
|
||||
@@ -111,21 +117,35 @@ impl KmsClient for StaticKmsBackend {
|
||||
let key = Key::<Aes256Gcm>::from(*self.key);
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::from(nonce_bytes);
|
||||
let aad = context_aad(&request.encryption_context)?;
|
||||
|
||||
let encrypted = cipher
|
||||
.encrypt(&nonce, plaintext.as_ref())
|
||||
.encrypt(
|
||||
&nonce,
|
||||
Payload {
|
||||
msg: plaintext.as_ref(),
|
||||
aad: &aad,
|
||||
},
|
||||
)
|
||||
.map_err(|e| KmsError::cryptographic_error("AES-256-GCM encrypt", e.to_string()))?;
|
||||
|
||||
// Ciphertext format: encrypted_dek || nonce
|
||||
let mut ciphertext = encrypted;
|
||||
ciphertext.extend_from_slice(&nonce_bytes);
|
||||
let envelope = DataKeyEnvelope {
|
||||
key_id: uuid::Uuid::new_v4().to_string(),
|
||||
master_key_id: request.master_key_id.clone(),
|
||||
key_spec: request.key_spec.clone(),
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
let ciphertext = serde_json::to_vec(&envelope)?;
|
||||
|
||||
Ok(DataKeyInfo::new(
|
||||
self.key_id.clone(),
|
||||
0, // version is always 0 for static KMS
|
||||
0,
|
||||
Some(plaintext.to_vec()),
|
||||
ciphertext,
|
||||
"AES_256".to_string(),
|
||||
request.key_spec.clone(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -141,14 +161,28 @@ impl KmsClient for StaticKmsBackend {
|
||||
let key = Key::<Aes256Gcm>::from(*self.key);
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::from(nonce_bytes);
|
||||
let aad = context_aad(&request.encryption_context)?;
|
||||
|
||||
let encrypted = cipher
|
||||
.encrypt(&nonce, request.plaintext.as_ref())
|
||||
.encrypt(
|
||||
&nonce,
|
||||
Payload {
|
||||
msg: request.plaintext.as_ref(),
|
||||
aad: &aad,
|
||||
},
|
||||
)
|
||||
.map_err(|e| KmsError::cryptographic_error("AES-256-GCM encrypt", e.to_string()))?;
|
||||
|
||||
// Ciphertext format: encrypted_data || nonce
|
||||
let mut ciphertext = encrypted;
|
||||
ciphertext.extend_from_slice(&nonce_bytes);
|
||||
let envelope = DataKeyEnvelope {
|
||||
key_id: uuid::Uuid::new_v4().to_string(),
|
||||
master_key_id: request.key_id.clone(),
|
||||
key_spec: "AES_256".to_string(),
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
let ciphertext = serde_json::to_vec(&envelope)?;
|
||||
|
||||
Ok(EncryptResponse {
|
||||
ciphertext,
|
||||
@@ -159,21 +193,39 @@ impl KmsClient for StaticKmsBackend {
|
||||
}
|
||||
|
||||
async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> {
|
||||
if request.ciphertext.len() < NONCE_SIZE + 1 {
|
||||
return Err(KmsError::cryptographic_error("decrypt", "Ciphertext too short for static KMS format"));
|
||||
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
|
||||
.map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?;
|
||||
if envelope.master_key_id != self.key_id {
|
||||
return Err(KmsError::key_not_found(&envelope.master_key_id));
|
||||
}
|
||||
|
||||
// Split ciphertext: encrypted_data || nonce(12)
|
||||
let split_at = request.ciphertext.len() - NONCE_SIZE;
|
||||
let encrypted = &request.ciphertext[..split_at];
|
||||
let nonce_slice = &request.ciphertext[split_at..];
|
||||
for (key, expected_value) in &envelope.encryption_context {
|
||||
match request.encryption_context.get(key) {
|
||||
Some(actual_value) if actual_value == expected_value => {}
|
||||
Some(actual_value) => {
|
||||
return Err(KmsError::context_mismatch(format!(
|
||||
"Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
|
||||
)));
|
||||
}
|
||||
None if request.encryption_context.is_empty() => {}
|
||||
None => return Err(KmsError::context_mismatch(format!("Missing context key '{key}'"))),
|
||||
}
|
||||
}
|
||||
|
||||
let key = Key::<Aes256Gcm>::from(*self.key);
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::try_from(nonce_slice).map_err(|_| KmsError::cryptographic_error("nonce", "invalid nonce length"))?;
|
||||
let nonce = Nonce::try_from(envelope.nonce.as_slice())
|
||||
.map_err(|_| KmsError::cryptographic_error("nonce", "invalid nonce length"))?;
|
||||
let aad = context_aad(&envelope.encryption_context)?;
|
||||
|
||||
let plaintext = cipher
|
||||
.decrypt(&nonce, encrypted)
|
||||
.decrypt(
|
||||
&nonce,
|
||||
Payload {
|
||||
msg: envelope.encrypted_key.as_ref(),
|
||||
aad: &aad,
|
||||
},
|
||||
)
|
||||
.map_err(|e| KmsError::cryptographic_error("AES-256-GCM decrypt", e.to_string()))?;
|
||||
|
||||
Ok(plaintext)
|
||||
@@ -317,7 +369,13 @@ impl KmsBackend for StaticKmsBackend {
|
||||
}
|
||||
|
||||
async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
|
||||
let gen_req = GenerateKeyRequest::new(request.key_id.clone(), request.key_spec.as_str().to_string());
|
||||
let gen_req = GenerateKeyRequest {
|
||||
master_key_id: request.key_id.clone(),
|
||||
key_spec: request.key_spec.as_str().to_string(),
|
||||
key_length: None,
|
||||
encryption_context: request.encryption_context,
|
||||
grant_tokens: Vec::new(),
|
||||
};
|
||||
let data_key = <Self as KmsClient>::generate_data_key(self, &gen_req, None).await?;
|
||||
|
||||
let plaintext_key = data_key
|
||||
@@ -378,8 +436,9 @@ impl KmsBackend for StaticKmsBackend {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::backends::KmsClient;
|
||||
use crate::backends::{KmsBackend as KmsBackendTrait, KmsClient};
|
||||
use crate::config::{BackendConfig, KmsBackend, StaticConfig};
|
||||
use crate::encryption::is_data_key_envelope;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
|
||||
@@ -430,8 +489,12 @@ mod tests {
|
||||
assert_eq!(data_key.version, 0);
|
||||
assert!(data_key.plaintext.is_some());
|
||||
assert_eq!(data_key.plaintext.as_ref().expect("plaintext should be set").len(), 32);
|
||||
// Ciphertext should be: encrypted(32) + tag(16) + nonce(12)
|
||||
assert_eq!(data_key.ciphertext.len(), 32 + 16 + NONCE_SIZE);
|
||||
let envelope: DataKeyEnvelope =
|
||||
serde_json::from_slice(&data_key.ciphertext).expect("static data key should use a KMS envelope");
|
||||
assert_eq!(envelope.master_key_id, key_id);
|
||||
assert_eq!(envelope.encrypted_key.len(), 32 + 16);
|
||||
assert_eq!(envelope.nonce.len(), NONCE_SIZE);
|
||||
assert_eq!(envelope.encryption_context.get("bucket").map(String::as_str), Some("test-bucket"));
|
||||
|
||||
// Decrypt the data key
|
||||
let decrypt_request =
|
||||
@@ -443,6 +506,75 @@ mod tests {
|
||||
assert_eq!(decrypted.as_slice(), data_key.plaintext.as_deref().expect("plaintext should exist"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generated_data_key_uses_kms_envelope_for_sse_read_routing() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
let request = GenerateKeyRequest::new(key_id, "AES_256".to_string())
|
||||
.with_context("bucket".to_string(), "source-bucket".to_string())
|
||||
.with_context("object".to_string(), "source-object".to_string());
|
||||
|
||||
let data_key = KmsClient::generate_data_key(&backend, &request, None)
|
||||
.await
|
||||
.expect("generate static KMS data key");
|
||||
|
||||
assert!(
|
||||
is_data_key_envelope(&data_key.ciphertext),
|
||||
"static KMS ciphertext must use the KMS envelope recognized by the SSE read path"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generated_data_key_rejects_a_different_encryption_context() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
let generate_request = GenerateDataKeyRequest {
|
||||
key_id,
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: HashMap::from([
|
||||
("bucket".to_string(), "source-bucket".to_string()),
|
||||
("object".to_string(), "source-object".to_string()),
|
||||
]),
|
||||
};
|
||||
let generated = KmsBackendTrait::generate_data_key(&backend, generate_request)
|
||||
.await
|
||||
.expect("generate context-bound static KMS data key");
|
||||
let decrypt_request = DecryptRequest {
|
||||
ciphertext: generated.ciphertext_blob,
|
||||
encryption_context: HashMap::from([
|
||||
("bucket".to_string(), "different-bucket".to_string()),
|
||||
("object".to_string(), "different-object".to_string()),
|
||||
]),
|
||||
grant_tokens: Vec::new(),
|
||||
};
|
||||
|
||||
let error = KmsBackendTrait::decrypt(&backend, decrypt_request)
|
||||
.await
|
||||
.expect_err("a static KMS data key must not decrypt under a different object context");
|
||||
|
||||
assert!(matches!(error, KmsError::ContextMismatch { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generated_data_key_rejects_tampered_envelope_context() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
let request = GenerateKeyRequest::new(key_id, "AES_256".to_string())
|
||||
.with_context("bucket".to_string(), "source-bucket".to_string());
|
||||
let generated = KmsClient::generate_data_key(&backend, &request, None)
|
||||
.await
|
||||
.expect("generate context-bound data key");
|
||||
let mut envelope: DataKeyEnvelope = serde_json::from_slice(&generated.ciphertext).expect("parse static KMS envelope");
|
||||
envelope
|
||||
.encryption_context
|
||||
.insert("bucket".to_string(), "different-bucket".to_string());
|
||||
let decrypt_request = DecryptRequest::new(serde_json::to_vec(&envelope).expect("serialize tampered envelope"))
|
||||
.with_context("bucket".to_string(), "different-bucket".to_string());
|
||||
|
||||
let error = KmsClient::decrypt(&backend, &decrypt_request, None)
|
||||
.await
|
||||
.expect_err("tampering with authenticated envelope context must fail");
|
||||
|
||||
assert!(matches!(error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_generate_data_key_wrong_key_id() {
|
||||
let (backend, _key_id, _key) = create_test_backend().await;
|
||||
|
||||
@@ -214,9 +214,18 @@ pub struct StaticConfig {
|
||||
/// Key identifier (name) for the single configured key
|
||||
pub key_id: String,
|
||||
/// Base64-encoded 32-byte AES-256 key material (zeroed on drop)
|
||||
#[serde(skip_serializing, default)]
|
||||
pub secret_key: String,
|
||||
}
|
||||
|
||||
impl Drop for StaticConfig {
|
||||
fn drop(&mut self) {
|
||||
use zeroize::Zeroize;
|
||||
|
||||
self.secret_key.zeroize();
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for StaticConfig {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("StaticConfig")
|
||||
@@ -1054,6 +1063,21 @@ mod tests {
|
||||
assert!(serialized.contains("persisted-token-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_kms_config_serialization_does_not_expose_key_material() {
|
||||
use base64::Engine as _;
|
||||
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone());
|
||||
|
||||
let serialized = serde_json::to_string(&config).expect("static KMS config should serialize");
|
||||
|
||||
assert!(
|
||||
!serialized.contains(&encoded_key),
|
||||
"persisted static KMS configuration must not contain plaintext key material"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_validation() {
|
||||
let mut config = KmsConfig {
|
||||
|
||||
+16
-10
@@ -31,19 +31,25 @@ use tokio::sync::RwLock;
|
||||
pub struct KmsManager {
|
||||
backend: Arc<dyn KmsBackend>,
|
||||
cache: Arc<RwLock<KmsCache>>,
|
||||
config: KmsConfig,
|
||||
default_key_id: Option<String>,
|
||||
enable_cache: bool,
|
||||
}
|
||||
|
||||
impl KmsManager {
|
||||
/// Create a new KMS manager with the given backend and config
|
||||
pub fn new(backend: Arc<dyn KmsBackend>, config: KmsConfig) -> Self {
|
||||
let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64)));
|
||||
Self { backend, cache, config }
|
||||
Self {
|
||||
backend,
|
||||
cache,
|
||||
default_key_id: config.default_key_id,
|
||||
enable_cache: config.enable_cache,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the default key ID if configured
|
||||
pub fn get_default_key_id(&self) -> Option<&String> {
|
||||
self.config.default_key_id.as_ref()
|
||||
self.default_key_id.as_ref()
|
||||
}
|
||||
|
||||
/// Create a new master key
|
||||
@@ -51,7 +57,7 @@ impl KmsManager {
|
||||
let response = self.backend.create_key(request).await?;
|
||||
|
||||
// Cache the key metadata if enabled
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let mut cache = self.cache.write().await;
|
||||
cache.put_key_metadata(&response.key_id, &response.key_metadata).await;
|
||||
}
|
||||
@@ -77,7 +83,7 @@ impl KmsManager {
|
||||
/// Describe a key
|
||||
pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
|
||||
// Check cache first if enabled
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let cache = self.cache.read().await;
|
||||
if let Some(cached_metadata) = cache.get_key_metadata(&request.key_id).await {
|
||||
return Ok(DescribeKeyResponse {
|
||||
@@ -89,7 +95,7 @@ impl KmsManager {
|
||||
// Get from backend and cache
|
||||
let response = self.backend.describe_key(request).await?;
|
||||
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let mut cache = self.cache.write().await;
|
||||
cache
|
||||
.put_key_metadata(&response.key_metadata.key_id, &response.key_metadata)
|
||||
@@ -106,7 +112,7 @@ impl KmsManager {
|
||||
|
||||
/// Get cache statistics
|
||||
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let cache = self.cache.read().await;
|
||||
Some(cache.stats())
|
||||
} else {
|
||||
@@ -116,7 +122,7 @@ impl KmsManager {
|
||||
|
||||
/// Clear the cache
|
||||
pub async fn clear_cache(&self) -> Result<()> {
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let mut cache = self.cache.write().await;
|
||||
cache.clear().await;
|
||||
}
|
||||
@@ -128,7 +134,7 @@ impl KmsManager {
|
||||
let response = self.backend.delete_key(request).await?;
|
||||
|
||||
// Remove from cache if enabled and key is being deleted
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let mut cache = self.cache.write().await;
|
||||
cache.remove_key_metadata(&response.key_id).await;
|
||||
}
|
||||
@@ -141,7 +147,7 @@ impl KmsManager {
|
||||
let response = self.backend.cancel_key_deletion(request).await?;
|
||||
|
||||
// Update cache if enabled
|
||||
if self.config.enable_cache {
|
||||
if self.enable_cache {
|
||||
let mut cache = self.cache.write().await;
|
||||
cache.put_key_metadata(&response.key_id, &response.key_metadata).await;
|
||||
}
|
||||
|
||||
@@ -350,11 +350,7 @@ impl ObjectEncryptionService {
|
||||
encryption_context: context.clone(),
|
||||
};
|
||||
|
||||
let data_key = self
|
||||
.kms_manager
|
||||
.generate_data_key(request)
|
||||
.await
|
||||
.map_err(|e| KmsError::backend_error(format!("Failed to generate data key: {e}")))?;
|
||||
let data_key = self.kms_manager.generate_data_key(request).await?;
|
||||
|
||||
let plaintext_key = data_key.plaintext_key;
|
||||
|
||||
@@ -431,11 +427,7 @@ impl ObjectEncryptionService {
|
||||
grant_tokens: Vec::new(),
|
||||
};
|
||||
|
||||
let decrypt_response = self
|
||||
.kms_manager
|
||||
.decrypt(decrypt_request)
|
||||
.await
|
||||
.map_err(|e| KmsError::backend_error(format!("Failed to decrypt data key: {e}")))?;
|
||||
let decrypt_response = self.kms_manager.decrypt(decrypt_request).await?;
|
||||
|
||||
// Create cipher
|
||||
let cipher = create_cipher(&algorithm, &decrypt_response.plaintext)?;
|
||||
|
||||
@@ -94,6 +94,16 @@ impl KmsServiceManager {
|
||||
self.config.read().await.clone()
|
||||
}
|
||||
|
||||
/// Get configuration for status and management responses without static key material.
|
||||
pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
|
||||
let mut config = self.config.read().await.clone()?;
|
||||
if let BackendConfig::Static(static_config) = &mut config.backend_config {
|
||||
use zeroize::Zeroize;
|
||||
static_config.secret_key.zeroize();
|
||||
}
|
||||
Some(config)
|
||||
}
|
||||
|
||||
/// Configure KMS with new configuration
|
||||
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
new_config.validate()?;
|
||||
@@ -449,4 +459,22 @@ mod tests {
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
|
||||
assert!(manager.get_config().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn redacted_config_omits_static_key_material() {
|
||||
use base64::Engine as _;
|
||||
|
||||
let manager = KmsServiceManager::new();
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
manager
|
||||
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
|
||||
.await
|
||||
.expect("configure static KMS");
|
||||
|
||||
let config = manager.get_redacted_config().await.expect("redacted config");
|
||||
let BackendConfig::Static(static_config) = config.backend_config else {
|
||||
panic!("expected static config");
|
||||
};
|
||||
assert!(static_config.secret_key.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ documentation = "https://docs.rs/rustfs-lifecycle/latest/rustfs_lifecycle/"
|
||||
|
||||
[dependencies]
|
||||
async-trait.workspace = true
|
||||
metrics.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-replication.workspace = true
|
||||
@@ -38,6 +39,7 @@ url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[dev-dependencies]
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
proptest = "1"
|
||||
serial_test.workspace = true
|
||||
temp-env.workspace = true
|
||||
|
||||
+115
-20
@@ -34,6 +34,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
const EVENT_LIFECYCLE_EXPIRY_COMPUTED: &str = "lifecycle_expiry_computed";
|
||||
const EVENT_LIFECYCLE_DEBUG_DAY_SECS: &str = "lifecycle_debug_day_secs";
|
||||
const EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED: &str = "lifecycle_noncurrent_expiry_skipped";
|
||||
const METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL: &str = "rustfs_lifecycle_invalid_mod_time_total";
|
||||
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
|
||||
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
|
||||
const _ERR_XML_NOT_WELL_FORMED: &str =
|
||||
@@ -204,6 +205,21 @@ fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> {
|
||||
filter.and.as_ref().and_then(|and| and.prefix.as_deref())
|
||||
}
|
||||
|
||||
fn lifecycle_mod_time(obj: &ObjectOpts) -> Option<OffsetDateTime> {
|
||||
let reason = match obj.mod_time {
|
||||
None => "missing",
|
||||
Some(mod_time) if mod_time < OffsetDateTime::UNIX_EPOCH => "before_unix_epoch",
|
||||
Some(mod_time) if mod_time == OffsetDateTime::UNIX_EPOCH => "unix_epoch",
|
||||
Some(mod_time) => return Some(mod_time),
|
||||
};
|
||||
metrics::counter!(
|
||||
METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL,
|
||||
"reason" => reason
|
||||
)
|
||||
.increment(1);
|
||||
None
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Lifecycle {
|
||||
async fn has_transition(&self) -> bool;
|
||||
@@ -434,14 +450,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
}
|
||||
|
||||
async fn predict_expiration(&self, obj: &ObjectOpts) -> Event {
|
||||
let mod_time = match obj.mod_time {
|
||||
Some(time) => {
|
||||
if time.unix_timestamp() == 0 {
|
||||
return Event::default();
|
||||
}
|
||||
time
|
||||
}
|
||||
None => return Event::default(),
|
||||
let Some(mod_time) = lifecycle_mod_time(obj) else {
|
||||
return Event::default();
|
||||
};
|
||||
|
||||
if obj.delete_marker || !(obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) {
|
||||
@@ -495,19 +505,9 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
obj.name, obj.mod_time, obj.successor_mod_time, now, obj.is_latest, obj.delete_marker
|
||||
);
|
||||
|
||||
// Gracefully handle missing mod_time instead of panicking
|
||||
let mod_time = match obj.mod_time {
|
||||
Some(t) => t,
|
||||
None => {
|
||||
debug!("eval_inner: mod_time is None for object={}, returning default event", obj.name);
|
||||
return Event::default();
|
||||
}
|
||||
};
|
||||
|
||||
if mod_time.unix_timestamp() == 0 {
|
||||
debug!("eval_inner: mod_time is 0, returning default event");
|
||||
let Some(mod_time) = lifecycle_mod_time(obj) else {
|
||||
return Event::default();
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(restore_expires) = obj.restore_expires
|
||||
&& restore_expires.unix_timestamp() != 0
|
||||
@@ -1082,6 +1082,8 @@ impl Default for TransitionOptions {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass};
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
@@ -1097,6 +1099,99 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eval_inner_reports_invalid_mod_time_without_expiring_object() {
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("expire-after-one-day".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.expect("current-thread runtime should build");
|
||||
|
||||
let actions = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let now = OffsetDateTime::from_unix_timestamp(10 * 86_400).expect("fixed timestamp should be valid");
|
||||
let mut opts = ObjectOpts {
|
||||
name: "legacy-object".to_string(),
|
||||
is_latest: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut actions = Vec::new();
|
||||
for mod_time in [
|
||||
None,
|
||||
Some(OffsetDateTime::UNIX_EPOCH),
|
||||
Some(OffsetDateTime::UNIX_EPOCH - Duration::nanoseconds(1)),
|
||||
Some(OffsetDateTime::UNIX_EPOCH + Duration::nanoseconds(1)),
|
||||
] {
|
||||
opts.mod_time = mod_time;
|
||||
actions.push((
|
||||
lifecycle.eval_inner(&opts, now, 0).await.action,
|
||||
lifecycle.predict_expiration(&opts).await.action,
|
||||
));
|
||||
}
|
||||
actions
|
||||
})
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
actions,
|
||||
[
|
||||
(IlmAction::NoneAction, IlmAction::NoneAction),
|
||||
(IlmAction::NoneAction, IlmAction::NoneAction),
|
||||
(IlmAction::NoneAction, IlmAction::NoneAction),
|
||||
(IlmAction::DeleteAction, IlmAction::DeleteAction)
|
||||
]
|
||||
);
|
||||
|
||||
let invalid_mod_time_metrics = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter_map(|(composite, _unit, _description, value)| {
|
||||
(composite.kind() == MetricKind::Counter && composite.key().name() == METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL)
|
||||
.then(|| {
|
||||
let reason = composite
|
||||
.key()
|
||||
.labels()
|
||||
.find(|label| label.key() == "reason")
|
||||
.map(|label| label.value().to_string())
|
||||
.expect("invalid mod_time metric should have a reason");
|
||||
(reason, value)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(invalid_mod_time_metrics.len(), 3);
|
||||
assert!(
|
||||
invalid_mod_time_metrics
|
||||
.iter()
|
||||
.all(|(_, value)| matches!(value, DebugValue::Counter(2)))
|
||||
);
|
||||
assert!(invalid_mod_time_metrics.iter().any(|(reason, _)| reason == "missing"));
|
||||
assert!(invalid_mod_time_metrics.iter().any(|(reason, _)| reason == "unix_epoch"));
|
||||
assert!(
|
||||
invalid_mod_time_metrics
|
||||
.iter()
|
||||
.any(|(reason, _)| reason == "before_unix_epoch")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validate_rejects_zero_expiration_days() {
|
||||
|
||||
@@ -45,15 +45,15 @@ pub(super) fn rules() -> Vec<Rule> {
|
||||
)
|
||||
},
|
||||
Rule {
|
||||
anchors: strings(["all healed data rename attempts failed"]),
|
||||
anchors: strings(["heal rename incomplete"]),
|
||||
..base(
|
||||
"heal-rename-failed",
|
||||
P1Unavailable,
|
||||
"heal",
|
||||
"heal 数据落位失败",
|
||||
contains("all healed data rename attempts failed"),
|
||||
"heal 数据落位(rename)全部失败。",
|
||||
"检查盘写权限与空间。",
|
||||
contains("heal rename incomplete"),
|
||||
"heal 数据落位(rename)未在全部目标盘完成。",
|
||||
"检查失败目标盘的写权限与空间,修复后重跑 heal。",
|
||||
)
|
||||
},
|
||||
Rule {
|
||||
|
||||
@@ -177,7 +177,7 @@ fn every_rule_has_a_positive_sample() {
|
||||
msg("heal: latest metadata for b/o has no data_dir, cannot heal object data"),
|
||||
),
|
||||
("heal-all-writes-failed", msg("all drives had write errors, unable to heal b/o")),
|
||||
("heal-rename-failed", msg("all healed data rename attempts failed for b/o")),
|
||||
("heal-rename-failed", msg("heal rename incomplete: 1 of 2 targets committed for b/o")),
|
||||
(
|
||||
"heal-xlmeta-regen-failed",
|
||||
msg("heal_object: failed to regenerate recoverable xl.meta on disk"),
|
||||
|
||||
@@ -25,6 +25,9 @@ keywords = ["file-system", "notification", "real-time", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
|
||||
|
||||
[features]
|
||||
demo-examples = []
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
@@ -71,6 +74,18 @@ workspace = true
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
[[example]]
|
||||
name = "full_demo"
|
||||
required-features = ["demo-examples"]
|
||||
|
||||
[[example]]
|
||||
name = "full_demo_one"
|
||||
required-features = ["demo-examples"]
|
||||
|
||||
[[example]]
|
||||
name = "webhook"
|
||||
required-features = ["demo-examples"]
|
||||
|
||||
[[bench]]
|
||||
name = "snapshot_mode_scan"
|
||||
harness = false
|
||||
|
||||
@@ -49,6 +49,8 @@ swift = [
|
||||
"dep:hmac",
|
||||
"dep:sha1",
|
||||
"dep:hex",
|
||||
"dep:ipnetwork",
|
||||
"dep:rustfs-trusted-proxies",
|
||||
"dep:astral-tokio-tar",
|
||||
"dep:base64",
|
||||
"dep:async-compression",
|
||||
@@ -111,6 +113,8 @@ quick-xml = { workspace = true, optional = true, features = ["serialize"] }
|
||||
hmac = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
hex = { workspace = true, optional = true }
|
||||
ipnetwork = { workspace = true, optional = true }
|
||||
rustfs-trusted-proxies = { workspace = true, optional = true }
|
||||
astral-tokio-tar = { workspace = true, optional = true }
|
||||
base64 = { workspace = true, optional = true }
|
||||
async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] }
|
||||
@@ -130,6 +134,7 @@ socket2 = { workspace = true, optional = true, features = ["all"] }
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
proptest = "1"
|
||||
rustfs-test-utils = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs"] }
|
||||
|
||||
|
||||
@@ -16,12 +16,11 @@
|
||||
|
||||
use super::storage_api::account::{BucketOperations, MakeBucketOptions};
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, update_swift_bucket_tagging, validate_metadata};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use time;
|
||||
|
||||
/// Validate that the authenticated user has access to the requested account
|
||||
///
|
||||
@@ -148,15 +147,33 @@ pub async fn get_account_metadata(account: &str, _credentials: &Option<Credentia
|
||||
/// Updates account-level metadata such as TempURL keys.
|
||||
/// Only updates swift-account-meta-* tags, preserving other tags.
|
||||
///
|
||||
/// The caller must own the account: this metadata holds the account's TempURL
|
||||
/// signing key, so writing it for someone else's account would let the writer
|
||||
/// mint valid pre-signed URLs against that account's objects. Reads
|
||||
/// ([`get_account_metadata`]) stay unauthenticated on purpose — TempURL
|
||||
/// signature validation has to resolve the key before any credentials exist.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `account` - Account identifier
|
||||
/// * `metadata` - Metadata key-value pairs to store (keys will be prefixed with `swift-account-meta-`)
|
||||
/// * `credentials` - S3 credentials
|
||||
/// * `credentials` - Keystone credentials of the caller
|
||||
pub async fn update_account_metadata(
|
||||
account: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
_credentials: &Option<Credentials>,
|
||||
credentials: &Option<Credentials>,
|
||||
) -> SwiftResult<()> {
|
||||
let Some(credentials) = credentials.as_ref() else {
|
||||
return Err(SwiftError::Unauthorized(
|
||||
"Keystone authentication required to update account metadata".to_string(),
|
||||
));
|
||||
};
|
||||
validate_account_access(account, credentials)?;
|
||||
|
||||
// These tags are persisted into the bucket metadata file, which every
|
||||
// later config write rewrites in full — so unbounded metadata inflates
|
||||
// the cost of unrelated writes for the life of the account.
|
||||
validate_metadata(metadata)?;
|
||||
|
||||
let bucket_name = get_account_metadata_bucket_name(account);
|
||||
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
@@ -173,57 +190,30 @@ pub async fn update_account_metadata(
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to create account metadata bucket: {}", e)))?;
|
||||
}
|
||||
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
// Rewrite the persisted tags: replace swift-account-meta-* tags with the
|
||||
// new metadata while preserving other tags. An empty result clears the
|
||||
// tagging config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
|
||||
// Get existing tags, preserving non-Swift tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
// Remove old swift-account-meta-* tags while preserving other tags
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-account-meta-")
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
// Add new metadata tags
|
||||
for (key, value) in metadata {
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some(format!("swift-account-meta-{}", key)),
|
||||
value: Some(value.clone()),
|
||||
tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-account-meta-")
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
for (key, value) in metadata {
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some(format!("swift-account-meta-{}", key)),
|
||||
value: Some(value.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name.clone(), bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
tagging
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -22,12 +22,22 @@ use super::storage_api::container::{
|
||||
};
|
||||
use super::types::Container;
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
use super::{
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
|
||||
validate_metadata,
|
||||
};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::{debug, error};
|
||||
|
||||
fn required_bucket_usage(usage: &std::collections::HashMap<String, (u64, u64)>, bucket: &str) -> SwiftResult<(u64, u64)> {
|
||||
usage
|
||||
.get(bucket)
|
||||
.copied()
|
||||
.ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
||||
const LOG_SUBSYSTEM_SWIFT_CONTAINER: &str = "swift_container";
|
||||
const EVENT_SWIFT_CONTAINER_STORAGE_STATE: &str = "swift_container_storage_state";
|
||||
@@ -252,13 +262,24 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR
|
||||
.list_bucket(&BucketOptions::default())
|
||||
.await
|
||||
.map_err(|e| sanitize_storage_error("Container listing", e))?;
|
||||
let usage = get_swift_bucket_usage()
|
||||
.await
|
||||
.map_err(|e| sanitize_storage_error("Container usage retrieval", e))?
|
||||
.ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))?;
|
||||
|
||||
// Filter and convert buckets to containers
|
||||
let containers: Vec<Container> = bucket_infos
|
||||
.iter()
|
||||
.filter(|info| mapper.bucket_belongs_to_project(&info.name, &project_id))
|
||||
.filter_map(|info| bucket_info_to_container(info, &mapper, &project_id))
|
||||
.collect();
|
||||
.filter_map(|info| {
|
||||
bucket_info_to_container(info, &mapper, &project_id).map(|mut container| {
|
||||
let (count, bytes) = required_bucket_usage(&usage, &info.name)?;
|
||||
container.count = count;
|
||||
container.bytes = bytes;
|
||||
Ok(container)
|
||||
})
|
||||
})
|
||||
.collect::<SwiftResult<Vec<_>>>()?;
|
||||
|
||||
debug!(
|
||||
event = EVENT_SWIFT_CONTAINER_STORAGE_STATE,
|
||||
@@ -368,6 +389,44 @@ pub struct ContainerMetadata {
|
||||
pub custom_metadata: std::collections::HashMap<String, String>,
|
||||
}
|
||||
|
||||
async fn get_container_metadata_base(
|
||||
account: &str,
|
||||
container: &str,
|
||||
credentials: &Credentials,
|
||||
) -> SwiftResult<(String, BucketInfo, std::collections::HashMap<String, String>)> {
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
validate_container_name(container)?;
|
||||
let bucket_name = ContainerMapper::default().swift_to_s3_bucket(container, &project_id);
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
|
||||
};
|
||||
let bucket_info = store
|
||||
.get_bucket_info(&bucket_name, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
|
||||
SwiftError::NotFound(format!("Container '{}' not found", container))
|
||||
} else {
|
||||
sanitize_storage_error("Container metadata retrieval", e)
|
||||
}
|
||||
})?;
|
||||
let custom_metadata = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|bucket_meta| bucket_meta.tagging_config.as_ref().map(s3_tags_to_swift_metadata))
|
||||
.unwrap_or_default();
|
||||
Ok((bucket_name, bucket_info, custom_metadata))
|
||||
}
|
||||
|
||||
pub(crate) async fn get_container_custom_metadata(
|
||||
account: &str,
|
||||
container: &str,
|
||||
credentials: &Credentials,
|
||||
) -> SwiftResult<std::collections::HashMap<String, String>> {
|
||||
let (_, _, custom_metadata) = get_container_metadata_base(account, container, credentials).await?;
|
||||
Ok(custom_metadata)
|
||||
}
|
||||
|
||||
/// Get container metadata (for HEAD operation)
|
||||
///
|
||||
/// This function:
|
||||
@@ -382,60 +441,22 @@ pub struct ContainerMetadata {
|
||||
/// - Returns 404 Not Found if container doesn't exist
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn get_container_metadata(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<ContainerMetadata> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
let (bucket_name, bucket_info, custom_metadata) = get_container_metadata_base(account, container, credentials).await?;
|
||||
|
||||
// Validate container name
|
||||
validate_container_name(container)?;
|
||||
|
||||
// Create mapper with default config (tenant prefixing enabled)
|
||||
let mapper = ContainerMapper::default();
|
||||
|
||||
// Convert Swift container name to S3 bucket name
|
||||
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
|
||||
|
||||
// Get storage layer
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
|
||||
};
|
||||
|
||||
// Get bucket info
|
||||
let bucket_info = store
|
||||
.get_bucket_info(&bucket_name, &BucketOptions::default())
|
||||
// Ecstore serves the persisted scanner snapshot through a bounded in-process cache.
|
||||
// Single-node deployments overlay successful in-process mutations; distributed
|
||||
// deployments use only the cluster-wide persisted snapshot.
|
||||
let usage = get_swift_bucket_usage()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
// Check if bucket not found
|
||||
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
|
||||
SwiftError::NotFound(format!("Container '{}' not found", container))
|
||||
} else {
|
||||
sanitize_storage_error("Container metadata retrieval", e)
|
||||
}
|
||||
})?;
|
||||
.map_err(|e| sanitize_storage_error("Container usage retrieval", e))?
|
||||
.ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))?;
|
||||
let (object_count, bytes_used) = required_bucket_usage(&usage, &bucket_name)?;
|
||||
|
||||
// Load bucket metadata to get custom metadata from tags
|
||||
let custom_metadata = match get_swift_bucket_metadata(&bucket_name).await {
|
||||
Ok(bucket_meta) => {
|
||||
if let Some(tagging) = &bucket_meta.tagging_config {
|
||||
s3_tags_to_swift_metadata(tagging)
|
||||
} else {
|
||||
std::collections::HashMap::new()
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// If metadata not available, return empty (container may be newly created)
|
||||
std::collections::HashMap::new()
|
||||
}
|
||||
};
|
||||
|
||||
// Currently returns basic metadata with limitations:
|
||||
// 1. Object count requires iterating all objects (expensive)
|
||||
// 2. Bytes used requires summing all object sizes (expensive)
|
||||
// 3. Custom metadata is now loaded from bucket tags ✅
|
||||
Ok(ContainerMetadata {
|
||||
object_count: 0, // TODO: implement object counting in backend
|
||||
bytes_used: 0, // TODO: implement size aggregation in backend
|
||||
object_count,
|
||||
bytes_used,
|
||||
created: bucket_info.created,
|
||||
custom_metadata, // ✅ Now populated from bucket tags!
|
||||
custom_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -465,6 +486,11 @@ pub async fn update_container_metadata(
|
||||
// Validate container name
|
||||
validate_container_name(container)?;
|
||||
|
||||
// These tags are persisted into the bucket metadata file, which every
|
||||
// later config write rewrites in full — so unbounded metadata inflates
|
||||
// the cost of unrelated writes for the life of the container.
|
||||
validate_metadata(&metadata)?;
|
||||
|
||||
// Create mapper with default config (tenant prefixing enabled)
|
||||
let mapper = ContainerMapper::default();
|
||||
|
||||
@@ -488,57 +514,30 @@ pub async fn update_container_metadata(
|
||||
}
|
||||
})?;
|
||||
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
// Rewrite the persisted tags: replace swift-meta-* tags with the new
|
||||
// metadata while preserving non-Swift tags. An empty result clears the
|
||||
// tagging config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-meta-")
|
||||
} else {
|
||||
true // Keep tags with no key (shouldn't happen, but be safe)
|
||||
}
|
||||
});
|
||||
|
||||
// Get existing tags, preserving non-Swift tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
// Remove old swift-meta-* tags while preserving other tags
|
||||
existing_tagging.tag_set.retain(|tag| {
|
||||
if let Some(key) = &tag.key {
|
||||
!key.starts_with("swift-meta-")
|
||||
} else {
|
||||
true // Keep tags with no key (shouldn't happen, but be safe)
|
||||
if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
|
||||
tagging.tag_set.append(&mut new_tagging.tag_set);
|
||||
}
|
||||
});
|
||||
// If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
|
||||
// we've already removed swift-meta-* tags above, so only non-Swift
|
||||
// tags remain
|
||||
|
||||
// Add new Swift metadata tags if provided
|
||||
if let Some(mut new_tagging) = swift_metadata_to_s3_tags(&metadata) {
|
||||
// Merge: existing non-Swift tags + new Swift tags
|
||||
existing_tagging.tag_set.append(&mut new_tagging.tag_set);
|
||||
}
|
||||
// If metadata.is_empty() and swift_metadata_to_s3_tags returns None,
|
||||
// we've already removed swift-meta-* tags above, so only non-Swift tags remain
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain after removing swift-meta-* tags; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize the merged tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
tagging
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -801,44 +800,23 @@ pub async fn enable_versioning(
|
||||
}
|
||||
})?;
|
||||
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
// Rewrite the persisted tags: replace any versioning tag with the new
|
||||
// archive location while preserving all other tags.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
|
||||
// Get existing tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some("swift-versions-location".to_string()),
|
||||
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
|
||||
});
|
||||
|
||||
// Remove old versioning tag if present
|
||||
existing_tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
|
||||
// Add new versioning tag
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some("swift-versions-location".to_string()),
|
||||
value: Some(archive_container.to_string()), // Store Swift container name, not S3 bucket name
|
||||
});
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
// Serialize tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
tagging
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -864,50 +842,38 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
|
||||
let mapper = ContainerMapper::default();
|
||||
let bucket_name = mapper.swift_to_s3_bucket(container, &project_id);
|
||||
|
||||
// Verify container exists
|
||||
let Some(_store) = resolve_swift_object_store_handle() else {
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string()));
|
||||
};
|
||||
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
// Verify container exists. Without this the rewrite below would persist a
|
||||
// fabricated default for a container that does not exist: the metadata
|
||||
// loader turns "no metadata on disk" into a fresh BucketMetadata, and
|
||||
// writing that creates an orphan .metadata.bin and caches a fabricated
|
||||
// default as authoritative.
|
||||
store
|
||||
.get_bucket_info(&bucket_name, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") {
|
||||
SwiftError::NotFound(format!("Container '{}' not found", container))
|
||||
} else {
|
||||
sanitize_storage_error("Container verification", e)
|
||||
}
|
||||
})?;
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
// Rewrite the persisted tags: drop the versioning tag while preserving
|
||||
// all other tags. An empty result clears the tagging config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
// Get existing tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
|
||||
// Remove versioning tag
|
||||
existing_tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-versions-location"));
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize remaining tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
tagging
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1032,65 +998,37 @@ pub async fn set_container_acl(
|
||||
}
|
||||
})?;
|
||||
|
||||
// Load current bucket metadata
|
||||
let bucket_meta = get_swift_bucket_metadata(&bucket_name)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to load bucket metadata: {}", e)))?;
|
||||
// Rewrite the persisted tags: replace the ACL tags with the new grants
|
||||
// while preserving all other tags. An empty result clears the tagging
|
||||
// config.
|
||||
update_swift_bucket_tagging(bucket_name, |current| {
|
||||
let mut tagging = current.cloned().unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
|
||||
let mut bucket_meta_clone = (*bucket_meta).clone();
|
||||
tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-acl-read") && tag.key.as_deref() != Some("swift-acl-write"));
|
||||
|
||||
// Get existing tags
|
||||
let mut existing_tagging = bucket_meta_clone
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_else(|| Tagging { tag_set: vec![] });
|
||||
if let Some(read) = read_acl
|
||||
&& !read.trim().is_empty()
|
||||
{
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-read".to_string()),
|
||||
value: Some(read.to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
// Remove old ACL tags
|
||||
existing_tagging
|
||||
.tag_set
|
||||
.retain(|tag| tag.key.as_deref() != Some("swift-acl-read") && tag.key.as_deref() != Some("swift-acl-write"));
|
||||
if let Some(write) = write_acl
|
||||
&& !write.trim().is_empty()
|
||||
{
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-write".to_string()),
|
||||
value: Some(write.to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
// Add new read ACL tag if provided
|
||||
if let Some(read) = read_acl
|
||||
&& !read.trim().is_empty()
|
||||
{
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-read".to_string()),
|
||||
value: Some(read.to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
// Add new write ACL tag if provided
|
||||
if let Some(write) = write_acl
|
||||
&& !write.trim().is_empty()
|
||||
{
|
||||
existing_tagging.tag_set.push(Tag {
|
||||
key: Some("swift-acl-write".to_string()),
|
||||
value: Some(write.to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
if existing_tagging.tag_set.is_empty() {
|
||||
// No tags remain; clear tagging config
|
||||
bucket_meta_clone.tagging_config_xml = Vec::new();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = None;
|
||||
} else {
|
||||
// Serialize tags to XML
|
||||
let tagging_xml = quick_xml::se::to_string(&existing_tagging)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to serialize tags: {}", e)))?;
|
||||
|
||||
bucket_meta_clone.tagging_config_xml = tagging_xml.into_bytes();
|
||||
bucket_meta_clone.tagging_config_updated_at = now;
|
||||
bucket_meta_clone.tagging_config = Some(existing_tagging);
|
||||
}
|
||||
|
||||
// Save updated metadata
|
||||
set_swift_bucket_metadata(bucket_name, bucket_meta_clone)
|
||||
.await
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Failed to save metadata: {}", e)))?;
|
||||
tagging
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
"Set ACLs for container {}/{}: read={:?}, write={:?}",
|
||||
@@ -1843,4 +1781,24 @@ mod tests {
|
||||
assert_eq!(tagging.tag_set.len(), 1);
|
||||
assert_eq!(tagging.tag_set[0].value.as_deref(), Some("new-archive"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonempty_container_usage_preserves_authoritative_totals() {
|
||||
let usage = std::collections::HashMap::from([("tenant-container".to_string(), (7, 4097))]);
|
||||
|
||||
assert_eq!(
|
||||
required_bucket_usage(&usage, "tenant-container").expect("authoritative bucket usage"),
|
||||
(7, 4097)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_container_usage_fails_closed() {
|
||||
let usage = std::collections::HashMap::new();
|
||||
|
||||
assert!(matches!(
|
||||
required_bucket_usage(&usage, "tenant-container"),
|
||||
Err(SwiftError::ServiceUnavailable(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ pub enum SwiftError {
|
||||
NotFound(String),
|
||||
/// 409 Conflict
|
||||
Conflict(String),
|
||||
/// 411 Length Required
|
||||
LengthRequired(String),
|
||||
/// 413 Request Entity Too Large (Payload Too Large)
|
||||
RequestEntityTooLarge(String),
|
||||
/// 422 Unprocessable Entity
|
||||
@@ -54,6 +56,7 @@ impl fmt::Display for SwiftError {
|
||||
SwiftError::Forbidden(msg) => write!(f, "Forbidden: {}", msg),
|
||||
SwiftError::NotFound(msg) => write!(f, "Not Found: {}", msg),
|
||||
SwiftError::Conflict(msg) => write!(f, "Conflict: {}", msg),
|
||||
SwiftError::LengthRequired(msg) => write!(f, "Length Required: {}", msg),
|
||||
SwiftError::RequestEntityTooLarge(msg) => write!(f, "Request Entity Too Large: {}", msg),
|
||||
SwiftError::UnprocessableEntity(msg) => write!(f, "Unprocessable Entity: {}", msg),
|
||||
SwiftError::TooManyRequests { retry_after, .. } => {
|
||||
@@ -76,6 +79,7 @@ impl SwiftError {
|
||||
SwiftError::Forbidden(_) => StatusCode::FORBIDDEN,
|
||||
SwiftError::NotFound(_) => StatusCode::NOT_FOUND,
|
||||
SwiftError::Conflict(_) => StatusCode::CONFLICT,
|
||||
SwiftError::LengthRequired(_) => StatusCode::LENGTH_REQUIRED,
|
||||
SwiftError::RequestEntityTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE,
|
||||
SwiftError::UnprocessableEntity(_) => StatusCode::UNPROCESSABLE_ENTITY,
|
||||
SwiftError::TooManyRequests { .. } => StatusCode::TOO_MANY_REQUESTS,
|
||||
|
||||
@@ -28,7 +28,9 @@ use axum::http::{Method, Request, Response, StatusCode};
|
||||
use futures::Future;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_keystone::KEYSTONE_CREDENTIALS;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use s3s::Body;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio_util::io::StreamReader;
|
||||
@@ -40,6 +42,14 @@ const LOG_SUBSYSTEM_SWIFT_HANDLER: &str = "swift_handler";
|
||||
const EVENT_SWIFT_ROUTE_STATE: &str = "swift_route_state";
|
||||
const EVENT_SWIFT_TEMPURL_STATE: &str = "swift_tempurl_state";
|
||||
|
||||
fn trusted_client_ip<B>(req: &Request<B>) -> Option<IpAddr> {
|
||||
req.extensions()
|
||||
.get::<ClientInfo>()
|
||||
.map(|info| info.real_ip)
|
||||
.or_else(|| req.extensions().get::<SocketAddr>().map(SocketAddr::ip))
|
||||
.filter(|ip| !ip.is_unspecified())
|
||||
}
|
||||
|
||||
/// Swift-aware service that routes to Swift handlers or S3 service
|
||||
#[derive(Clone)]
|
||||
pub struct SwiftService<S> {
|
||||
@@ -128,6 +138,7 @@ async fn handle_swift_request(
|
||||
route: SwiftRoute,
|
||||
credentials: Option<Credentials>,
|
||||
) -> Result<Response<Body>, SwiftError> {
|
||||
let client_ip = trusted_client_ip(&req);
|
||||
// Extract parts
|
||||
let (parts, body) = req.into_parts();
|
||||
let method = parts.method.clone();
|
||||
@@ -164,7 +175,7 @@ async fn handle_swift_request(
|
||||
let tempurl = tempurl::TempURL::new(key);
|
||||
let path = uri.path();
|
||||
|
||||
tempurl.validate_request(method.as_str(), path, &tempurl_params)?;
|
||||
tempurl.validate_request(method.as_str(), path, &tempurl_params, client_ip)?;
|
||||
|
||||
// TempURL is valid - proceed with request (no credentials needed)
|
||||
debug!(
|
||||
@@ -641,14 +652,11 @@ async fn handle_authenticated_request(
|
||||
.await;
|
||||
}
|
||||
|
||||
// Check quota before upload (if Content-Length provided)
|
||||
if let Some(content_length) = headers.get("content-length")
|
||||
&& let Ok(size_str) = content_length.to_str()
|
||||
&& let Ok(object_size) = size_str.parse::<u64>()
|
||||
{
|
||||
// Check if upload would exceed quota
|
||||
super::quota::check_upload_quota(&account, &container, object_size, &credentials).await?;
|
||||
}
|
||||
let object_size = headers
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<u64>().ok());
|
||||
super::quota::check_upload_quota(&account, &container, object_size, &credentials).await?;
|
||||
|
||||
// Check if versioning is enabled for this container
|
||||
if let Some(archive_container) = container::get_versions_location(&account, &container, &credentials).await? {
|
||||
@@ -1475,6 +1483,7 @@ fn swift_error_to_response(error: SwiftError) -> Response<Body> {
|
||||
SwiftError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.as_str()),
|
||||
SwiftError::NotFound(msg) => (StatusCode::NOT_FOUND, msg.as_str()),
|
||||
SwiftError::Conflict(msg) => (StatusCode::CONFLICT, msg.as_str()),
|
||||
SwiftError::LengthRequired(msg) => (StatusCode::LENGTH_REQUIRED, msg.as_str()),
|
||||
SwiftError::RequestEntityTooLarge(msg) => (StatusCode::PAYLOAD_TOO_LARGE, msg.as_str()),
|
||||
SwiftError::UnprocessableEntity(msg) => (StatusCode::UNPROCESSABLE_ENTITY, msg.as_str()),
|
||||
SwiftError::TooManyRequests { .. } => (StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded"),
|
||||
@@ -1497,7 +1506,10 @@ fn swift_error_to_response(error: SwiftError) -> Response<Body> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_range_header;
|
||||
use super::{parse_range_header, trusted_client_ip};
|
||||
use axum::http::Request;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::net::SocketAddr;
|
||||
#[test]
|
||||
fn test_parse_range_header_start_end() {
|
||||
// bytes=100-199
|
||||
@@ -1588,4 +1600,41 @@ mod tests {
|
||||
let result = parse_range_header("bytes=0-999", 1000);
|
||||
assert_eq!(result, Some((0, 999)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trusted_client_ip_ignores_untrusted_forwarded_headers() {
|
||||
let mut request = Request::builder()
|
||||
.header("x-forwarded-for", "198.51.100.9")
|
||||
.header("x-real-ip", "198.51.100.10")
|
||||
.body(())
|
||||
.expect("request");
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert("192.0.2.7:9000".parse::<SocketAddr>().expect("socket address"));
|
||||
|
||||
assert_eq!(trusted_client_ip(&request), Some("192.0.2.7".parse().expect("peer IP address")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trusted_client_ip_prefers_validated_proxy_identity() {
|
||||
let mut request = Request::new(());
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert("192.0.2.7:9000".parse::<SocketAddr>().expect("socket address"));
|
||||
request.extensions_mut().insert(ClientInfo::direct(
|
||||
"203.0.113.8:443".parse::<SocketAddr>().expect("validated client address"),
|
||||
));
|
||||
|
||||
assert_eq!(trusted_client_ip(&request), Some("203.0.113.8".parse().expect("validated client IP")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trusted_client_ip_rejects_unspecified_fallback() {
|
||||
let mut request = Request::new(());
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(ClientInfo::direct("0.0.0.0:0".parse::<SocketAddr>().expect("unspecified fallback")));
|
||||
|
||||
assert_eq!(trusted_client_ip(&request), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,10 +58,53 @@ pub mod versioning;
|
||||
|
||||
pub use errors::{SwiftError, SwiftResult};
|
||||
pub use router::{SwiftRoute, SwiftRouter};
|
||||
|
||||
/// Maximum number of metadata headers allowed per resource (Swift standard)
|
||||
pub(crate) const MAX_METADATA_COUNT: usize = 90;
|
||||
|
||||
/// Maximum size in bytes for a single metadata value (Swift standard)
|
||||
pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
|
||||
|
||||
/// Validate metadata against Swift limits
|
||||
///
|
||||
/// Checks that:
|
||||
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
|
||||
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
|
||||
///
|
||||
/// Applies to object, container and account metadata alike: all three are
|
||||
/// persisted, and container/account metadata additionally lands in the
|
||||
/// bucket metadata file that every later config write rewrites in full.
|
||||
///
|
||||
/// Returns error if limits are exceeded.
|
||||
pub(crate) fn validate_metadata(metadata: &std::collections::HashMap<String, String>) -> SwiftResult<()> {
|
||||
// Check total metadata count
|
||||
if metadata.len() > MAX_METADATA_COUNT {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Too many metadata headers: {} (max: {})",
|
||||
metadata.len(),
|
||||
MAX_METADATA_COUNT
|
||||
)));
|
||||
}
|
||||
|
||||
// Check individual value sizes
|
||||
for (key, value) in metadata.iter() {
|
||||
if value.len() > MAX_METADATA_VALUE_SIZE {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
|
||||
key,
|
||||
value.len(),
|
||||
MAX_METADATA_VALUE_SIZE
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Note: Container, Object, and SwiftMetadata types used by Swift implementation
|
||||
pub use storage_api::public_api::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use storage_api::public_api::{
|
||||
get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata,
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use types::{Container, Object, SwiftMetadata};
|
||||
|
||||
@@ -53,7 +53,7 @@ use super::account::validate_account_access;
|
||||
use super::container::ContainerMapper;
|
||||
use super::expiration_worker::{track_object_expiration, untrack_object_expiration};
|
||||
use super::storage_api::object::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle};
|
||||
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle, validate_metadata};
|
||||
use axum::http::HeaderMap;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_rio::HashReader;
|
||||
@@ -68,12 +68,6 @@ const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
|
||||
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
|
||||
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
|
||||
|
||||
/// Maximum number of metadata headers allowed per object (Swift standard)
|
||||
const MAX_METADATA_COUNT: usize = 90;
|
||||
|
||||
/// Maximum size in bytes for a single metadata value (Swift standard)
|
||||
const MAX_METADATA_VALUE_SIZE: usize = 256;
|
||||
|
||||
/// Maximum object size in bytes (5GB - Swift default)
|
||||
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
@@ -234,38 +228,6 @@ impl Default for ObjectKeyMapper {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate metadata against Swift limits
|
||||
///
|
||||
/// Checks that:
|
||||
/// - Total number of metadata entries doesn't exceed MAX_METADATA_COUNT
|
||||
/// - Individual metadata values don't exceed MAX_METADATA_VALUE_SIZE
|
||||
///
|
||||
/// Returns error if limits are exceeded.
|
||||
fn validate_metadata(metadata: &HashMap<String, String>) -> SwiftResult<()> {
|
||||
// Check total metadata count
|
||||
if metadata.len() > MAX_METADATA_COUNT {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Too many metadata headers: {} (max: {})",
|
||||
metadata.len(),
|
||||
MAX_METADATA_COUNT
|
||||
)));
|
||||
}
|
||||
|
||||
// Check individual value sizes
|
||||
for (key, value) in metadata.iter() {
|
||||
if value.len() > MAX_METADATA_VALUE_SIZE {
|
||||
return Err(SwiftError::BadRequest(format!(
|
||||
"Metadata value for '{}' too large: {} bytes (max: {} bytes)",
|
||||
key,
|
||||
value.len(),
|
||||
MAX_METADATA_VALUE_SIZE
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn metadata_delete_at(metadata: &HashMap<String, String>) -> Option<u64> {
|
||||
metadata
|
||||
.get(SWIFT_DELETE_AT_METADATA)
|
||||
|
||||
@@ -87,18 +87,17 @@ pub struct QuotaConfig {
|
||||
impl QuotaConfig {
|
||||
/// Load quota configuration from container metadata
|
||||
pub async fn load(account: &str, container_name: &str, credentials: &Credentials) -> SwiftResult<Self> {
|
||||
// Get container metadata
|
||||
let container_info = container::get_container_metadata(account, container_name, credentials).await?;
|
||||
let custom_metadata = container::get_container_custom_metadata(account, container_name, credentials).await?;
|
||||
|
||||
let mut config = QuotaConfig::default();
|
||||
|
||||
// Parse Quota-Bytes
|
||||
if let Some(quota_bytes_str) = container_info.custom_metadata.get("x-container-meta-quota-bytes") {
|
||||
if let Some(quota_bytes_str) = custom_metadata.get("x-container-meta-quota-bytes") {
|
||||
config.quota_bytes = quota_bytes_str.parse().ok();
|
||||
}
|
||||
|
||||
// Parse Quota-Count
|
||||
if let Some(quota_count_str) = container_info.custom_metadata.get("x-container-meta-quota-count") {
|
||||
if let Some(quota_count_str) = custom_metadata.get("x-container-meta-quota-count") {
|
||||
config.quota_count = quota_count_str.parse().ok();
|
||||
}
|
||||
|
||||
@@ -113,9 +112,12 @@ impl QuotaConfig {
|
||||
/// Check if adding an object would exceed quotas
|
||||
///
|
||||
/// Returns Ok(()) if within quota, Err with 413 if exceeded
|
||||
pub fn check_quota(&self, current_bytes: u64, current_count: u64, additional_bytes: u64) -> SwiftResult<()> {
|
||||
pub fn check_quota(&self, current_bytes: u64, current_count: u64, additional_bytes: Option<u64>) -> SwiftResult<()> {
|
||||
// Check byte quota
|
||||
if let Some(max_bytes) = self.quota_bytes {
|
||||
let additional_bytes = additional_bytes.ok_or_else(|| {
|
||||
SwiftError::LengthRequired("Content-Length is required when a byte quota is configured".to_string())
|
||||
})?;
|
||||
let new_bytes = current_bytes.saturating_add(additional_bytes);
|
||||
if new_bytes > max_bytes {
|
||||
return Err(SwiftError::RequestEntityTooLarge(format!(
|
||||
@@ -147,7 +149,7 @@ impl QuotaConfig {
|
||||
pub async fn check_upload_quota(
|
||||
account: &str,
|
||||
container_name: &str,
|
||||
object_size: u64,
|
||||
object_size: Option<u64>,
|
||||
credentials: &Credentials,
|
||||
) -> SwiftResult<()> {
|
||||
// Load quota config
|
||||
@@ -157,7 +159,6 @@ pub async fn check_upload_quota(
|
||||
if !quota.is_enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Get current container usage
|
||||
let metadata = container::get_container_metadata(account, container_name, credentials).await?;
|
||||
|
||||
@@ -173,7 +174,7 @@ pub async fn check_upload_quota(
|
||||
quota_bytes = ?quota.quota_bytes,
|
||||
current_count = metadata.object_count,
|
||||
quota_count = ?quota.quota_count,
|
||||
object_size,
|
||||
object_size = ?object_size,
|
||||
"swift quota state changed"
|
||||
);
|
||||
|
||||
@@ -235,7 +236,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Current: 500 bytes, adding 400 bytes = 900 total (within 1000 limit)
|
||||
let result = config.check_quota(500, 0, 400);
|
||||
let result = config.check_quota(500, 0, Some(400));
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
@@ -247,7 +248,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Current: 500 bytes, adding 600 bytes = 1100 total (exceeds 1000 limit)
|
||||
let result = config.check_quota(500, 0, 600);
|
||||
let result = config.check_quota(500, 0, Some(600));
|
||||
assert!(result.is_err());
|
||||
match result {
|
||||
Err(SwiftError::RequestEntityTooLarge(msg)) => {
|
||||
@@ -265,7 +266,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Current: 500 bytes, adding 500 bytes = 1000 total (exactly at limit)
|
||||
let result = config.check_quota(500, 0, 500);
|
||||
let result = config.check_quota(500, 0, Some(500));
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
@@ -277,7 +278,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Current: 5 objects, adding 1 = 6 total (within 10 limit)
|
||||
let result = config.check_quota(0, 5, 100);
|
||||
let result = config.check_quota(0, 5, Some(100));
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
@@ -289,7 +290,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Current: 10 objects, adding 1 = 11 total (exceeds 10 limit)
|
||||
let result = config.check_quota(0, 10, 100);
|
||||
let result = config.check_quota(0, 10, Some(100));
|
||||
assert!(result.is_err());
|
||||
match result {
|
||||
Err(SwiftError::RequestEntityTooLarge(msg)) => {
|
||||
@@ -307,7 +308,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Current: 9 objects, adding 1 = 10 total (exactly at limit)
|
||||
let result = config.check_quota(0, 9, 100);
|
||||
let result = config.check_quota(0, 9, Some(100));
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
@@ -319,7 +320,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Both within limits
|
||||
let result = config.check_quota(500, 5, 400);
|
||||
let result = config.check_quota(500, 5, Some(400));
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
@@ -331,7 +332,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Bytes exceeded, count within
|
||||
let result = config.check_quota(500, 5, 600);
|
||||
let result = config.check_quota(500, 5, Some(600));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -343,7 +344,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Count exceeded, bytes within
|
||||
let result = config.check_quota(500, 10, 100);
|
||||
let result = config.check_quota(500, 10, Some(100));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -355,7 +356,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// No limits, should always pass
|
||||
let result = config.check_quota(999999, 999999, 999999);
|
||||
let result = config.check_quota(999999, 999999, Some(999999));
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
@@ -367,7 +368,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Zero limit means no uploads allowed
|
||||
let result = config.check_quota(0, 0, 1);
|
||||
let result = config.check_quota(0, 0, Some(1));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -379,7 +380,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Zero limit means no objects allowed
|
||||
let result = config.check_quota(0, 0, 100);
|
||||
let result = config.check_quota(0, 0, Some(100));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -391,8 +392,28 @@ mod tests {
|
||||
};
|
||||
|
||||
// Test saturating_add protection
|
||||
let result = config.check_quota(u64::MAX - 100, 0, 200);
|
||||
let result = config.check_quota(u64::MAX - 100, 0, Some(200));
|
||||
// Should saturate to u64::MAX and compare against quota
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_quota_rejects_upload_without_content_length() {
|
||||
let config = QuotaConfig {
|
||||
quota_bytes: Some(1000),
|
||||
quota_count: None,
|
||||
};
|
||||
|
||||
assert!(matches!(config.check_quota(500, 0, None), Err(SwiftError::LengthRequired(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_only_quota_checks_upload_without_content_length() {
|
||||
let config = QuotaConfig {
|
||||
quota_bytes: None,
|
||||
quota_count: Some(10),
|
||||
};
|
||||
|
||||
assert!(matches!(config.check_quota(0, 10, None), Err(SwiftError::RequestEntityTooLarge(_))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,16 +12,22 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{get as get_swift_bucket_metadata_from_backend, update_config_with};
|
||||
use rustfs_ecstore::api::bucket::utils::serialize as serialize_bucket_config;
|
||||
use rustfs_ecstore::api::error::Error as SwiftStorageError;
|
||||
pub(crate) use rustfs_ecstore::api::error::Result as SwiftStorageResult;
|
||||
use rustfs_ecstore::api::notification::get_global_notification_sys;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::object_store_handle as resolve_swift_object_store_handle;
|
||||
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
use s3s::dto::Tagging;
|
||||
|
||||
use super::{SwiftError, SwiftResult};
|
||||
|
||||
pub(crate) mod account {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, MakeBucketOptions};
|
||||
@@ -43,13 +49,24 @@ pub(crate) mod object {
|
||||
|
||||
pub(crate) mod public_api {
|
||||
pub use super::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
pub(crate) use super::{
|
||||
get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, update_swift_bucket_tagging,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod versioning {
|
||||
pub(crate) use super::storage_contracts::{ListOperations, ObjectOperations};
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
||||
const LOG_SUBSYSTEM_SWIFT_STORAGE: &str = "swift_storage";
|
||||
const EVENT_SWIFT_BUCKET_TAGGING_UPDATE: &str = "swift_bucket_tagging_update";
|
||||
|
||||
/// Marks the refusal to rewrite an unreadable persisted tagging config, so the
|
||||
/// caller can turn it into an actionable client error rather than a generic
|
||||
/// storage failure. Carried through the ecstore error, which is a string type.
|
||||
const UNREADABLE_TAGGING_SENTINEL: &str = "swift: persisted tagging config could not be parsed";
|
||||
|
||||
pub type SwiftGetObjectReader = <SwiftStore as storage_contracts::ObjectIO>::GetObjectReader;
|
||||
pub type SwiftObjectInfo = <SwiftStore as storage_contracts::ObjectOperations>::ObjectInfo;
|
||||
pub type SwiftObjectOptions = <SwiftStore as storage_contracts::ObjectOperations>::ObjectOptions;
|
||||
@@ -59,6 +76,93 @@ pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResul
|
||||
get_swift_bucket_metadata_from_backend(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
|
||||
set_swift_bucket_metadata_in_backend(bucket, metadata).await
|
||||
/// Rewrite the bucket's tagging config through the persisting
|
||||
/// bucket-metadata path.
|
||||
///
|
||||
/// `rewrite` sees the tag set currently persisted on disk (`None` when the
|
||||
/// bucket has none) and returns the full replacement; an empty tag set
|
||||
/// clears the config. The read-modify-write runs under the bucket metadata
|
||||
/// system's write guard — serialized against every other config update —
|
||||
/// and the result is written to the bucket metadata file before the cache
|
||||
/// is refreshed, so a Swift metadata POST survives process restarts and
|
||||
/// disk-truth reloads. Peers are then told to reload, matching what the S3
|
||||
/// handlers do after a config write.
|
||||
///
|
||||
/// Storage failures are logged in full and reported to the client as a
|
||||
/// generic error: these now carry real disk and quorum detail, which does not
|
||||
/// belong in a Swift response body. The one exception is an unreadable
|
||||
/// persisted config, which is reported specifically because the operator has
|
||||
/// to act on it.
|
||||
pub(crate) async fn update_swift_bucket_tagging<F>(bucket: String, rewrite: F) -> SwiftResult<()>
|
||||
where
|
||||
F: FnOnce(Option<&Tagging>) -> Tagging + Send,
|
||||
{
|
||||
let result = update_config_with(&bucket, BUCKET_TAGGING_CONFIG, |bm| {
|
||||
// Merging onto an unparseable tag set would silently drop every tag
|
||||
// the bucket has — including the container ACL and versioning tags —
|
||||
// because the rewrite closures treat "no parsed tags" as "no tags".
|
||||
// Refuse instead: the persisted config is intact, just unreadable.
|
||||
if !bm.tagging_config_xml.is_empty() && bm.tagging_config.is_none() {
|
||||
return Err(SwiftStorageError::other(UNREADABLE_TAGGING_SENTINEL));
|
||||
}
|
||||
|
||||
let tagging = rewrite(bm.tagging_config.as_ref());
|
||||
if tagging.tag_set.is_empty() {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
// The S3 XML serializer, not quick_xml: the metadata loader's
|
||||
// parse step must be able to round-trip what we persist.
|
||||
serialize_bucket_config(&tagging)
|
||||
.map_err(|e| SwiftStorageError::other(format!("failed to serialize bucket tagging: {e}")))
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
if let Err(err) = result {
|
||||
let unreadable = err.to_string().contains(UNREADABLE_TAGGING_SENTINEL);
|
||||
tracing::error!(
|
||||
event = EVENT_SWIFT_BUCKET_TAGGING_UPDATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_SWIFT_STORAGE,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = if unreadable { "unreadable_persisted_config" } else { "storage_failure" },
|
||||
result = "failed",
|
||||
"swift bucket tagging update failed"
|
||||
);
|
||||
// A Swift-only client has no way to repair this itself, so say what
|
||||
// happened and name the remedy instead of a bare storage error.
|
||||
return Err(if unreadable {
|
||||
SwiftError::Conflict(format!(
|
||||
"The persisted tagging configuration for container store '{bucket}' cannot be parsed, so metadata cannot be updated without discarding it. Reset it with the S3 DeleteBucketTagging API."
|
||||
))
|
||||
} else {
|
||||
SwiftError::InternalServerError("Metadata update operation failed".to_string())
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(notification_sys) = get_global_notification_sys() {
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = notification_sys.load_bucket_metadata(&bucket).await {
|
||||
tracing::warn!(bucket = %bucket, error = %err, "failed to notify peers after swift bucket tagging update");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_swift_bucket_usage() -> SwiftStorageResult<Option<HashMap<String, (u64, u64)>>> {
|
||||
let Some(store) = resolve_swift_object_store_handle() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut data_usage = rustfs_ecstore::api::data_usage::load_data_usage_from_backend_cached(store).await?;
|
||||
rustfs_ecstore::api::data_usage::apply_bucket_usage_memory_overlay(&mut data_usage).await;
|
||||
Ok(Some(
|
||||
data_usage
|
||||
.buckets_usage
|
||||
.into_iter()
|
||||
.map(|(bucket, usage)| (bucket, (usage.objects_count, usage.size)))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -21,7 +21,11 @@
|
||||
|
||||
use crate::swift::errors::SwiftError;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use ipnetwork::IpNetwork;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use sha1::Sha1;
|
||||
use std::net::IpAddr;
|
||||
use std::str::FromStr;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type HmacSha1 = Hmac<Sha1>;
|
||||
@@ -50,12 +54,12 @@ impl TempURLParams {
|
||||
let mut ip_range = None;
|
||||
|
||||
for param in query.split('&') {
|
||||
let parts: Vec<&str> = param.split('=').collect();
|
||||
if parts.len() == 2 {
|
||||
match parts[0] {
|
||||
"temp_url_sig" => sig = Some(parts[1].to_string()),
|
||||
"temp_url_expires" => expires = parts[1].parse().ok(),
|
||||
"temp_url_ip_range" => ip_range = Some(parts[1].to_string()),
|
||||
if let Some((name, value)) = param.split_once('=') {
|
||||
let value = percent_decode_str(value).decode_utf8().ok()?;
|
||||
match name {
|
||||
"temp_url_sig" => sig = Some(value.into_owned()),
|
||||
"temp_url_expires" => expires = value.parse().ok(),
|
||||
"temp_url_ip_range" => ip_range = Some(value.into_owned()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -97,9 +101,22 @@ impl TempURL {
|
||||
/// # Returns
|
||||
/// Hex-encoded HMAC-SHA1 signature
|
||||
pub fn generate_signature(&self, method: &str, expires: u64, path: &str) -> Result<String, SwiftError> {
|
||||
self.generate_signature_with_ip_range(method, expires, path, None)
|
||||
}
|
||||
|
||||
/// Generate a TempURL signature, optionally binding it to an IP range.
|
||||
pub fn generate_signature_with_ip_range(
|
||||
&self,
|
||||
method: &str,
|
||||
expires: u64,
|
||||
path: &str,
|
||||
ip_range: Option<&str>,
|
||||
) -> Result<String, SwiftError> {
|
||||
// Construct message for HMAC
|
||||
// Format: "{METHOD}\n{expires}\n{path}"
|
||||
let message = format!("{}\n{}\n{}", method.to_uppercase(), expires, path);
|
||||
let message = match ip_range {
|
||||
Some(ip_range) => format!("ip={}\n{}\n{}\n{}", ip_range, method.to_uppercase(), expires, path),
|
||||
None => format!("{}\n{}\n{}", method.to_uppercase(), expires, path),
|
||||
};
|
||||
|
||||
// Calculate HMAC-SHA1
|
||||
let mut mac = HmacSha1::new_from_slice(self.key.as_bytes())
|
||||
@@ -127,7 +144,13 @@ impl TempURL {
|
||||
/// # Returns
|
||||
/// - `Ok(())` if signature is valid and not expired
|
||||
/// - `Err(SwiftError::Unauthorized)` if invalid or expired
|
||||
pub fn validate_request(&self, method: &str, path: &str, params: &TempURLParams) -> Result<(), SwiftError> {
|
||||
pub fn validate_request(
|
||||
&self,
|
||||
method: &str,
|
||||
path: &str,
|
||||
params: &TempURLParams,
|
||||
client_ip: Option<IpAddr>,
|
||||
) -> Result<(), SwiftError> {
|
||||
// 1. Check expiration first (fast path for expired URLs)
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
@@ -139,17 +162,25 @@ impl TempURL {
|
||||
}
|
||||
|
||||
// 2. Generate expected signature
|
||||
let expected_sig = self.generate_signature(method, params.temp_url_expires, path)?;
|
||||
let expected_sig =
|
||||
self.generate_signature_with_ip_range(method, params.temp_url_expires, path, params.temp_url_ip_range.as_deref())?;
|
||||
|
||||
// 3. Constant-time comparison to prevent timing attacks
|
||||
if !constant_time_compare(params.temp_url_sig.as_bytes(), expected_sig.as_bytes()) {
|
||||
return Err(SwiftError::Unauthorized("Invalid TempURL signature".to_string()));
|
||||
}
|
||||
|
||||
// 4. TODO: Validate IP range if specified (future enhancement)
|
||||
// if let Some(ip_range) = ¶ms.temp_url_ip_range {
|
||||
// validate_ip_range(client_ip, ip_range)?;
|
||||
// }
|
||||
if let Some(ip_range) = ¶ms.temp_url_ip_range {
|
||||
let client_ip =
|
||||
client_ip.ok_or_else(|| SwiftError::Unauthorized("Trusted client address unavailable".to_string()))?;
|
||||
let allowed = IpAddr::from_str(ip_range)
|
||||
.map(|ip| ip == client_ip)
|
||||
.or_else(|_| IpNetwork::from_str(ip_range).map(|network| network.contains(client_ip)))
|
||||
.map_err(|_| SwiftError::Unauthorized("Invalid TempURL IP range".to_string()))?;
|
||||
if !allowed {
|
||||
return Err(SwiftError::Unauthorized("Client address is outside the TempURL IP range".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -308,7 +339,7 @@ mod tests {
|
||||
// Should validate successfully
|
||||
assert!(
|
||||
tempurl
|
||||
.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms)
|
||||
.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
@@ -331,7 +362,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Should reject expired URL
|
||||
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms);
|
||||
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_)));
|
||||
}
|
||||
@@ -349,7 +380,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Should reject invalid signature
|
||||
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms);
|
||||
let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_)));
|
||||
}
|
||||
@@ -372,7 +403,7 @@ mod tests {
|
||||
};
|
||||
|
||||
// Try to validate with PUT method
|
||||
let result = tempurl.validate_request("PUT", "/v1/AUTH_test/container/object", ¶ms);
|
||||
let result = tempurl.validate_request("PUT", "/v1/AUTH_test/container/object", ¶ms, None);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_)));
|
||||
}
|
||||
@@ -423,6 +454,14 @@ mod tests {
|
||||
assert_eq!(params.temp_url_ip_range.as_deref(), Some("192.168.1.0/24"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_percent_encoded_ipv6_range() {
|
||||
let query = "temp_url_sig=abc123&temp_url_expires=1609459200&temp_url_ip_range=2001%3Adb8%3A%3A%2F32";
|
||||
let params = TempURLParams::from_query(query).expect("encoded IPv6 TempURL parameters");
|
||||
|
||||
assert_eq!(params.temp_url_ip_range.as_deref(), Some("2001:db8::/32"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_tempurl_params_missing_sig() {
|
||||
let query = "temp_url_expires=1609459200";
|
||||
@@ -475,4 +514,103 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(sig, sig2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_bound_signature_matches_openstack_message_format() {
|
||||
let tempurl = TempURL::new("mykey".to_string());
|
||||
let signature = tempurl
|
||||
.generate_signature_with_ip_range("GET", 1648082711, "/v1/AUTH_account/container/object", Some("1.2.3.0/24"))
|
||||
.expect("IP-bound signature generation");
|
||||
|
||||
assert_eq!(signature, "8698990d811e64cff1c8a2151340874fd5bda0c5");
|
||||
}
|
||||
|
||||
fn ip_bound_params(tempurl: &TempURL, ip_range: &str) -> TempURLParams {
|
||||
let expires = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock after Unix epoch")
|
||||
.as_secs()
|
||||
+ 3600;
|
||||
let signature = tempurl
|
||||
.generate_signature_with_ip_range("GET", expires, "/v1/AUTH_test/container/object", Some(ip_range))
|
||||
.expect("IP-bound signature generation");
|
||||
TempURLParams {
|
||||
temp_url_sig: signature,
|
||||
temp_url_expires: expires,
|
||||
temp_url_ip_range: Some(ip_range.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_bound_tempurl_accepts_ipv4_inside_range_and_rejects_outside() {
|
||||
let tempurl = TempURL::new("mykey".to_string());
|
||||
let params = ip_bound_params(&tempurl, "192.0.2.0/24");
|
||||
|
||||
assert!(
|
||||
tempurl
|
||||
.validate_request(
|
||||
"GET",
|
||||
"/v1/AUTH_test/container/object",
|
||||
¶ms,
|
||||
Some("192.0.2.42".parse().expect("IPv4 address")),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(matches!(
|
||||
tempurl.validate_request(
|
||||
"GET",
|
||||
"/v1/AUTH_test/container/object",
|
||||
¶ms,
|
||||
Some("198.51.100.42".parse().expect("IPv4 address")),
|
||||
),
|
||||
Err(SwiftError::Unauthorized(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_bound_tempurl_accepts_ipv6_inside_range_and_rejects_outside() {
|
||||
let tempurl = TempURL::new("mykey".to_string());
|
||||
let params = ip_bound_params(&tempurl, "2001:db8::/32");
|
||||
|
||||
assert!(
|
||||
tempurl
|
||||
.validate_request(
|
||||
"GET",
|
||||
"/v1/AUTH_test/container/object",
|
||||
¶ms,
|
||||
Some("2001:db8::42".parse().expect("IPv6 address")),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(matches!(
|
||||
tempurl.validate_request(
|
||||
"GET",
|
||||
"/v1/AUTH_test/container/object",
|
||||
¶ms,
|
||||
Some("2001:db9::42".parse().expect("IPv6 address")),
|
||||
),
|
||||
Err(SwiftError::Unauthorized(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_bound_tempurl_rejects_missing_client_ip_and_invalid_range() {
|
||||
let tempurl = TempURL::new("mykey".to_string());
|
||||
let params = ip_bound_params(&tempurl, "192.0.2.0/24");
|
||||
assert!(matches!(
|
||||
tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None),
|
||||
Err(SwiftError::Unauthorized(_))
|
||||
));
|
||||
|
||||
let invalid_params = ip_bound_params(&tempurl, "not-a-network");
|
||||
assert!(matches!(
|
||||
tempurl.validate_request(
|
||||
"GET",
|
||||
"/v1/AUTH_test/container/object",
|
||||
&invalid_params,
|
||||
Some("192.0.2.42".parse().expect("IPv4 address")),
|
||||
),
|
||||
Err(SwiftError::Unauthorized(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ECStore facade boundary for the protocols integration tests.
|
||||
//!
|
||||
//! The Swift tests need to drive bucket-metadata reloads the way the peer
|
||||
//! `LoadBucketMetadata` RPC does. Everything they touch from `rustfs_ecstore`
|
||||
//! is aliased here so the tests themselves hold no raw facade subpaths, the
|
||||
//! same boundary `crates/protocols/src/swift/storage_api.rs` provides for the
|
||||
//! Swift implementation.
|
||||
|
||||
pub use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub use rustfs_ecstore::api::bucket::metadata_sys::{get as get_bucket_metadata, set_bucket_metadata};
|
||||
pub use rustfs_ecstore::api::runtime::object_store_handle;
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests: a Swift metadata POST must be persisted to the bucket
|
||||
//! metadata file, not just the in-memory cache. The metadata has to survive
|
||||
//! the disk-truth reloads performed by peer LoadBucketMetadata notifications
|
||||
//! and the periodic refresh loop — and, transitively, a process restart.
|
||||
|
||||
#![cfg(feature = "swift")]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_protocols::swift::SwiftError;
|
||||
use rustfs_protocols::swift::container::{ContainerMapper, update_container_metadata};
|
||||
use rustfs_protocols::swift::{account, container};
|
||||
use rustfs_test_utils::TestECStoreEnv;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
mod ecstore_test_compat;
|
||||
use ecstore_test_compat::{get_bucket_metadata, load_bucket_metadata, object_store_handle, set_bucket_metadata};
|
||||
|
||||
fn keystone_credentials(project_id: &str) -> Credentials {
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert("keystone_project_id".to_string(), json!(project_id));
|
||||
claims.insert("keystone_roles".to_string(), json!(["member"]));
|
||||
|
||||
Credentials {
|
||||
access_key: "keystone:swift-test".to_string(),
|
||||
claims: Some(claims),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The account-metadata bucket name scheme from `swift::account`
|
||||
/// (`swift-account-{sha256(account)[0..16]}`), mirrored here so the test can
|
||||
/// reload that bucket's metadata from disk.
|
||||
fn account_metadata_bucket_name(account: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(account.as_bytes());
|
||||
let hash = hex::encode(hasher.finalize());
|
||||
format!("swift-account-{}", &hash[0..16])
|
||||
}
|
||||
|
||||
/// Replace the cached bucket metadata with what is actually on disk — the
|
||||
/// same thing a peer LoadBucketMetadata notification or the periodic refresh
|
||||
/// loop does. Before the fix this silently discarded every Swift metadata
|
||||
/// POST, because those writes only ever touched the cache.
|
||||
async fn reload_bucket_metadata_from_disk(bucket: &str) {
|
||||
let store = object_store_handle().expect("test store should be published");
|
||||
let bm = load_bucket_metadata(store, bucket)
|
||||
.await
|
||||
.expect("bucket metadata should load from disk");
|
||||
set_bucket_metadata(bucket.to_string(), bm)
|
||||
.await
|
||||
.expect("reloaded metadata should install");
|
||||
}
|
||||
|
||||
/// The `swift-meta-*` tags currently persisted for a container, keyed the way
|
||||
/// a Swift client sees them. `get_container_metadata` would be the natural
|
||||
/// reader, but it additionally requires a data-usage snapshot for the object
|
||||
/// count and byte total, which a bare test store has none of — and that is
|
||||
/// orthogonal to whether the metadata itself was persisted.
|
||||
async fn persisted_container_metadata(bucket: &str) -> HashMap<String, String> {
|
||||
let bm = get_bucket_metadata(bucket).await.expect("bucket metadata should be cached");
|
||||
let mut out = HashMap::new();
|
||||
if let Some(tagging) = &bm.tagging_config {
|
||||
for tag in &tagging.tag_set {
|
||||
if let (Some(key), Some(value)) = (&tag.key, &tag.value)
|
||||
&& let Some(meta_key) = key.strip_prefix("swift-meta-")
|
||||
{
|
||||
out.insert(meta_key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every scenario that needs a store runs against ONE environment: the Swift
|
||||
/// handlers resolve the ambient object store and bucket-metadata system, so a
|
||||
/// second `TestECStoreEnv` in this process would race the first.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn swift_metadata_writes_are_durable() {
|
||||
let env = TestECStoreEnv::builder().prefix("swift_meta_persist").build().await;
|
||||
|
||||
posts_survive_disk_truth_reload(&env).await;
|
||||
tag_writers_preserve_each_others_state(&env).await;
|
||||
versioning_writes_reject_missing_containers().await;
|
||||
}
|
||||
|
||||
async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
|
||||
// --- Container metadata POST (X-Container-Meta-*) ---
|
||||
let project_id = "swiftpersistproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let swift_container = "photos";
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(swift_container, project_id);
|
||||
env.make_bucket(&bucket, false).await;
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("color".to_string(), "blue".to_string());
|
||||
update_container_metadata(&swift_account, swift_container, &credentials, metadata)
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let container_meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(
|
||||
container_meta.get("color").map(String::as_str),
|
||||
Some("blue"),
|
||||
"container metadata POST must survive a disk-truth metadata reload"
|
||||
);
|
||||
|
||||
// A follow-up POST replaces the Swift metadata and that replacement must
|
||||
// survive a reload too (the rewrite merges against disk state, so the
|
||||
// previous value must actually be gone).
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("season".to_string(), "summer".to_string());
|
||||
update_container_metadata(&swift_account, swift_container, &credentials, metadata)
|
||||
.await
|
||||
.expect("second container metadata POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let container_meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(container_meta.get("season").map(String::as_str), Some("summer"));
|
||||
assert!(
|
||||
!container_meta.contains_key("color"),
|
||||
"replaced container metadata must not resurrect on reload"
|
||||
);
|
||||
|
||||
// --- Container versioning POST (X-Versions-Location) ---
|
||||
let archive_container = "photos-archive";
|
||||
let archive_bucket = ContainerMapper::default().swift_to_s3_bucket(archive_container, project_id);
|
||||
env.make_bucket(&archive_bucket, false).await;
|
||||
|
||||
container::enable_versioning(&swift_account, swift_container, archive_container, &credentials)
|
||||
.await
|
||||
.expect("enable versioning should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
let location = container::get_versions_location(&swift_account, swift_container, &credentials)
|
||||
.await
|
||||
.expect("versions location should load");
|
||||
assert_eq!(
|
||||
location.as_deref(),
|
||||
Some(archive_container),
|
||||
"versions location must survive a disk-truth metadata reload"
|
||||
);
|
||||
|
||||
// --- Account metadata POST (TempURL keys etc.) ---
|
||||
let mut account_meta = HashMap::new();
|
||||
account_meta.insert("temp-url-key".to_string(), "s3cr3t".to_string());
|
||||
account::update_account_metadata(&swift_account, &account_meta, &Some(credentials.clone()))
|
||||
.await
|
||||
.expect("account metadata POST should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&account_metadata_bucket_name(&swift_account)).await;
|
||||
|
||||
let loaded = account::get_account_metadata(&swift_account, &None)
|
||||
.await
|
||||
.expect("account metadata should load");
|
||||
assert_eq!(
|
||||
loaded.get("temp-url-key").map(String::as_str),
|
||||
Some("s3cr3t"),
|
||||
"account metadata POST must survive a disk-truth metadata reload"
|
||||
);
|
||||
}
|
||||
|
||||
/// The rewrites all share one tag set, so a closure that ignored the current
|
||||
/// state would still pass a single-feature test. Drive ACLs, versioning and
|
||||
/// container metadata over the same container and assert each survives the
|
||||
/// others — and that clearing one leaves the rest alone.
|
||||
async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
|
||||
let project_id = "swiftcrosstagproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let container = "shared";
|
||||
let archive = "shared-archive";
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(container, project_id);
|
||||
env.make_bucket(&bucket, false).await;
|
||||
env.make_bucket(&ContainerMapper::default().swift_to_s3_bucket(archive, project_id), false)
|
||||
.await;
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("color".to_string(), "blue".to_string());
|
||||
update_container_metadata(&swift_account, container, &credentials, metadata)
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
container::enable_versioning(&swift_account, container, archive, &credentials)
|
||||
.await
|
||||
.expect("enable versioning should succeed");
|
||||
container::set_container_acl(&swift_account, container, Some(".r:*"), Some("AUTH_other"), &credentials)
|
||||
.await
|
||||
.expect("set container ACL should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
// All three writers' state coexists after a disk-truth reload.
|
||||
let meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(meta.get("color").map(String::as_str), Some("blue"));
|
||||
assert_eq!(
|
||||
container::get_versions_location(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("versions location should load")
|
||||
.as_deref(),
|
||||
Some(archive)
|
||||
);
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("container ACL should load");
|
||||
assert!(!acl.read.is_empty(), "read ACL must survive the reload");
|
||||
assert!(!acl.write.is_empty(), "write ACL must survive the reload");
|
||||
|
||||
// Disabling versioning drops only the versioning tag.
|
||||
container::disable_versioning(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("disable versioning should succeed");
|
||||
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
|
||||
assert_eq!(
|
||||
container::get_versions_location(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("versions location should load"),
|
||||
None,
|
||||
"disable_versioning must clear the versioning tag durably"
|
||||
);
|
||||
let meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(
|
||||
meta.get("color").map(String::as_str),
|
||||
Some("blue"),
|
||||
"disable_versioning must not disturb container metadata"
|
||||
);
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("container ACL should load");
|
||||
assert!(!acl.read.is_empty(), "disable_versioning must not disturb the ACL");
|
||||
}
|
||||
|
||||
/// A container that does not exist must not get metadata persisted for it:
|
||||
/// the metadata loader turns "nothing on disk" into a fresh default, so an
|
||||
/// unguarded rewrite would create an orphan metadata file and cache a
|
||||
/// fabricated default as authoritative.
|
||||
async fn versioning_writes_reject_missing_containers() {
|
||||
let project_id = "swiftmissingproj";
|
||||
let swift_account = format!("AUTH_{project_id}");
|
||||
let credentials = keystone_credentials(project_id);
|
||||
let missing = "no-such-container";
|
||||
|
||||
let err = container::disable_versioning(&swift_account, missing, &credentials)
|
||||
.await
|
||||
.expect_err("disabling versioning on a missing container must fail");
|
||||
assert!(
|
||||
matches!(err, SwiftError::NotFound(_)),
|
||||
"expected NotFound for a missing container, got {err:?}"
|
||||
);
|
||||
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(missing, project_id);
|
||||
assert!(
|
||||
get_bucket_metadata(&bucket).await.is_err(),
|
||||
"a rejected write must not have cached metadata for a nonexistent container"
|
||||
);
|
||||
}
|
||||
|
||||
/// Account metadata holds the account's TempURL signing key, and it is now
|
||||
/// durable — so a write for someone else's account would be a persistent,
|
||||
/// cluster-wide takeover of that account's pre-signed URLs, not a cache blip.
|
||||
/// The write path must reject both a foreign account and a missing token,
|
||||
/// while reads stay open for pre-auth TempURL signature validation.
|
||||
///
|
||||
/// Deliberately builds no store: both rejections must happen before the write
|
||||
/// path resolves storage at all, and a second `TestECStoreEnv` in this process
|
||||
/// would race the other test over the ambient store handle.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn account_metadata_write_rejects_foreign_and_anonymous_callers() {
|
||||
let victim_account = "AUTH_victimproject";
|
||||
let attacker_credentials = keystone_credentials("attackerproject");
|
||||
|
||||
let mut poisoned = HashMap::new();
|
||||
poisoned.insert("temp-url-key".to_string(), "attacker-key".to_string());
|
||||
|
||||
let err = account::update_account_metadata(victim_account, &poisoned, &Some(attacker_credentials))
|
||||
.await
|
||||
.expect_err("writing another account's metadata must be rejected");
|
||||
assert!(
|
||||
matches!(err, SwiftError::Forbidden(_)),
|
||||
"cross-account metadata write must be Forbidden, got {err:?}"
|
||||
);
|
||||
|
||||
let err = account::update_account_metadata(victim_account, &poisoned, &None)
|
||||
.await
|
||||
.expect_err("anonymous account metadata write must be rejected");
|
||||
assert!(
|
||||
matches!(err, SwiftError::Unauthorized(_)),
|
||||
"anonymous metadata write must be Unauthorized, got {err:?}"
|
||||
);
|
||||
}
|
||||
@@ -233,6 +233,46 @@ pub struct RenamePartResponse {
|
||||
pub error: ::core::option::Option<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct PreparePartTransactionRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub src_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub src_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub dst_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "5")]
|
||||
pub dst_path: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "6")]
|
||||
pub meta: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct PreparePartTransactionResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SettlePartTransactionRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub path: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "4")]
|
||||
pub rollback: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct SettlePartTransactionResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct RenameFileRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
@@ -1600,6 +1640,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "CheckParts"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn prepare_part_transaction(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::PreparePartTransactionRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::PreparePartTransactionResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/PreparePartTransaction");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "PreparePartTransaction"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_part(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenamePartRequest>,
|
||||
@@ -1615,6 +1670,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenamePart"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn settle_part_transaction(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SettlePartTransactionRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SettlePartTransactionResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/SettlePartTransaction");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "SettlePartTransaction"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_file(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameFileRequest>,
|
||||
@@ -2726,10 +2796,18 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::CheckPartsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::CheckPartsResponse>, tonic::Status>;
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
request: tonic::Request<super::PreparePartTransactionRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::PreparePartTransactionResponse>, tonic::Status>;
|
||||
async fn rename_part(
|
||||
&self,
|
||||
request: tonic::Request<super::RenamePartRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenamePartResponse>, tonic::Status>;
|
||||
async fn settle_part_transaction(
|
||||
&self,
|
||||
request: tonic::Request<super::SettlePartTransactionRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SettlePartTransactionResponse>, tonic::Status>;
|
||||
async fn rename_file(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameFileRequest>,
|
||||
@@ -3432,6 +3510,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/PreparePartTransaction" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct PreparePartTransactionSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::PreparePartTransactionRequest> for PreparePartTransactionSvc<T> {
|
||||
type Response = super::PreparePartTransactionResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::PreparePartTransactionRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::prepare_part_transaction(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = PreparePartTransactionSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenamePart" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenamePartSvc<T: NodeService>(pub Arc<T>);
|
||||
@@ -3460,6 +3566,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/SettlePartTransaction" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct SettlePartTransactionSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SettlePartTransactionRequest> for SettlePartTransactionSvc<T> {
|
||||
type Response = super::SettlePartTransactionResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SettlePartTransactionRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::settle_part_transaction(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = SettlePartTransactionSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameFile" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameFileSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -600,6 +600,30 @@ pub fn canonical_rename_part_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_prepare_part_transaction_request_body(
|
||||
request: &proto_gen::node_service::PreparePartTransactionRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-prepare-part-transaction-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.meta)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_settle_part_transaction_request_body(
|
||||
request: &proto_gen::node_service::SettlePartTransactionRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-settle-part-transaction-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_bool(request.rollback);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_volume_request_body(
|
||||
request: &proto_gen::node_service::DeleteVolumeRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -637,8 +661,8 @@ pub fn canonical_make_volumes_request_body(
|
||||
mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, UpdateMetadataRequest, WriteAllRequest,
|
||||
WriteMetadataRequest,
|
||||
MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest,
|
||||
SettlePartTransactionRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
@@ -886,6 +910,48 @@ mod disk_mutation_canonical_tests {
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let prepare_part = PreparePartTransactionRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: vec![0x01].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_prepare_part_transaction_request_body(&prepare_part).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut PreparePartTransactionRequest| r.disk = "d2".into(),
|
||||
|r: &mut PreparePartTransactionRequest| r.src_volume = "sv2".into(),
|
||||
|r: &mut PreparePartTransactionRequest| r.src_path = "sp2".into(),
|
||||
|r: &mut PreparePartTransactionRequest| r.dst_volume = "dv2".into(),
|
||||
|r: &mut PreparePartTransactionRequest| r.dst_path = "dp2".into(),
|
||||
|r: &mut PreparePartTransactionRequest| r.meta = vec![0x02].into(),
|
||||
] {
|
||||
let mut request = prepare_part.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_prepare_part_transaction_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let settle_part = SettlePartTransactionRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
rollback: false,
|
||||
};
|
||||
let mut bodies = vec![canonical_settle_part_transaction_request_body(&settle_part).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut SettlePartTransactionRequest| r.disk = "d2".into(),
|
||||
|r: &mut SettlePartTransactionRequest| r.volume = "v2".into(),
|
||||
|r: &mut SettlePartTransactionRequest| r.path = "p2".into(),
|
||||
|r: &mut SettlePartTransactionRequest| r.rollback = true,
|
||||
] {
|
||||
let mut request = settle_part.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_settle_part_transaction_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let rename_part = RenamePartRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
|
||||
@@ -170,6 +170,32 @@ message RenamePartResponse {
|
||||
optional Error error = 2;
|
||||
}
|
||||
|
||||
message PreparePartTransactionRequest {
|
||||
string disk = 1;
|
||||
string src_volume = 2;
|
||||
string src_path = 3;
|
||||
string dst_volume = 4;
|
||||
string dst_path = 5;
|
||||
bytes meta = 6;
|
||||
}
|
||||
|
||||
message PreparePartTransactionResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
}
|
||||
|
||||
message SettlePartTransactionRequest {
|
||||
string disk = 1;
|
||||
string volume = 2;
|
||||
string path = 3;
|
||||
bool rollback = 4;
|
||||
}
|
||||
|
||||
message SettlePartTransactionResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
}
|
||||
|
||||
message RenameFileRequest {
|
||||
string disk = 1;
|
||||
string src_volume = 2;
|
||||
@@ -943,7 +969,9 @@ service NodeService {
|
||||
rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {};
|
||||
rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {};
|
||||
rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {};
|
||||
rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {};
|
||||
rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {};
|
||||
rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {};
|
||||
rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {};
|
||||
rpc Write(WriteRequest) returns (WriteResponse) {};
|
||||
rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {};
|
||||
|
||||
@@ -30,7 +30,7 @@ Or point the tests at another generated root:
|
||||
|
||||
```powershell
|
||||
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-smoke'
|
||||
cargo +1.96.0 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored
|
||||
cargo +1.97.1 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored
|
||||
```
|
||||
|
||||
## Scope
|
||||
|
||||
@@ -271,7 +271,11 @@ impl EventName {
|
||||
EventName::ObjectCreatedPut,
|
||||
],
|
||||
EventName::ObjectTaggingAll => vec![EventName::ObjectTaggingPut, EventName::ObjectTaggingDelete],
|
||||
EventName::ObjectRemovedAll => vec![EventName::ObjectRemovedDelete, EventName::ObjectRemovedDeleteMarkerCreated],
|
||||
EventName::ObjectRemovedAll => vec![
|
||||
EventName::ObjectRemovedDelete,
|
||||
EventName::ObjectRemovedDeleteMarkerCreated,
|
||||
EventName::ObjectRemovedNoOP,
|
||||
],
|
||||
EventName::ObjectReplicationAll => vec![
|
||||
EventName::ObjectReplicationFailed,
|
||||
EventName::ObjectReplicationComplete,
|
||||
@@ -639,6 +643,15 @@ mod tests {
|
||||
assert_eq!(EventName::parse("s3:Scanner:*").unwrap(), EventName::ObjectScannerAll);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_object_removed_all_includes_noop_extension() {
|
||||
let expanded = EventName::ObjectRemovedAll.expand();
|
||||
|
||||
assert!(expanded.contains(&EventName::ObjectRemovedDelete));
|
||||
assert!(expanded.contains(&EventName::ObjectRemovedDeleteMarkerCreated));
|
||||
assert!(expanded.contains(&EventName::ObjectRemovedNoOP));
|
||||
}
|
||||
|
||||
/// `is_removed` must be true for every `ObjectRemoved*` variant and false
|
||||
/// for everything else.
|
||||
#[test]
|
||||
|
||||
@@ -40,6 +40,9 @@ allow-git = [
|
||||
# SigV4 payload-checksum fix until it is available in a crates.io release.
|
||||
# owner: marshawcoco review: 2026-10
|
||||
"https://github.com/s3s-project/s3s.git",
|
||||
# Presigned expiry and constant-time authentication fixes pending upstream merge.
|
||||
# owner: cxymds review: 2026-10
|
||||
"https://github.com/cxymds/s3s.git",
|
||||
"https://github.com/apache/datafusion.git",
|
||||
]
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ for later deletion.
|
||||
- `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
|
||||
- `backlog-1316` legacy encrypted multipart range seek: the feature remains opt-in until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently.
|
||||
- `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection.
|
||||
- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object.
|
||||
|
||||
## Review Checklist
|
||||
|
||||
|
||||
@@ -9,6 +9,16 @@ failure pattern reported in rustfs/rustfs#4304.
|
||||
> format. Replacing the executable and restarting is safe; no migration step
|
||||
> runs on startup.
|
||||
|
||||
> [!WARNING]
|
||||
> The release that switches local SSE wrapped DEKs from the legacy
|
||||
> `base64(nonce):base64(ciphertext)` representation to the versioned JSON
|
||||
> envelope is a deliberate exception. Do not run that release together with
|
||||
> an older RustFS version: older nodes cannot read objects written with the
|
||||
> JSON envelope. Freeze every source of object mutation, including client
|
||||
> writes and background lifecycle or replication work, upgrade every node,
|
||||
> and then resume traffic. Downgrading or rolling back after new encrypted
|
||||
> objects are written is not supported.
|
||||
|
||||
## TL;DR
|
||||
|
||||
- **Rolling restart (no downtime):** restart **one node at a time**, and wait
|
||||
|
||||
@@ -30,7 +30,6 @@
|
||||
systems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
"x86_64-darwin"
|
||||
"aarch64-darwin"
|
||||
];
|
||||
forAllSystems = nixpkgs.lib.genAttrs systems;
|
||||
|
||||
@@ -36,6 +36,8 @@ use tracing::{error, info, instrument, warn};
|
||||
|
||||
/// Path to store KMS configuration in the cluster metadata
|
||||
const KMS_CONFIG_PATH: &str = "config/kms_config.json";
|
||||
const STATIC_KMS_LOCAL_CONFIG_REQUIRED: &str =
|
||||
"Static KMS must be configured through RUSTFS_KMS_STATIC_SECRET_KEY or RUSTFS_KMS_STATIC_SECRET_KEY_FILE";
|
||||
const LOG_COMPONENT_ADMIN: &str = "admin";
|
||||
const LOG_SUBSYSTEM_KMS: &str = "kms";
|
||||
const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state";
|
||||
@@ -106,9 +108,25 @@ fn normalize_configure_request_auth(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_kms_config_persistable(config: &KmsConfig) -> Result<(), String> {
|
||||
if matches!(&config.backend_config, rustfs_kms::BackendConfig::Static(_)) {
|
||||
return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_kms_request_persistable(request: &ConfigureKmsRequest) -> Result<(), String> {
|
||||
if matches!(request, ConfigureKmsRequest::Static(_)) {
|
||||
return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save KMS configuration to cluster storage
|
||||
#[instrument(skip(config))]
|
||||
async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
|
||||
ensure_kms_config_persistable(config)?;
|
||||
|
||||
let context = current_app_context();
|
||||
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
|
||||
return Err("Storage layer not initialized".to_string());
|
||||
@@ -332,12 +350,16 @@ impl Operation for ConfigureKmsHandler {
|
||||
);
|
||||
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
let existing_config = service_manager.get_config().await;
|
||||
let existing_config = service_manager.get_redacted_config().await;
|
||||
|
||||
if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) {
|
||||
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
|
||||
}
|
||||
|
||||
if let Err(e) = ensure_kms_request_persistable(&configure_request) {
|
||||
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
|
||||
}
|
||||
|
||||
// Convert request to KmsConfig
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
@@ -753,7 +775,7 @@ impl Operation for GetKmsStatusHandler {
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
|
||||
let status = service_manager.get_status().await;
|
||||
let config = service_manager.get_config().await;
|
||||
let config = service_manager.get_redacted_config().await;
|
||||
|
||||
// Get backend type and health status
|
||||
let backend_type = config.as_ref().map(|c| c.backend.clone());
|
||||
@@ -873,12 +895,16 @@ impl Operation for ReconfigureKmsHandler {
|
||||
);
|
||||
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
let existing_config = service_manager.get_config().await;
|
||||
let existing_config = service_manager.get_redacted_config().await;
|
||||
|
||||
if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) {
|
||||
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
|
||||
}
|
||||
|
||||
if let Err(e) = ensure_kms_request_persistable(&configure_request) {
|
||||
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
|
||||
}
|
||||
|
||||
// Convert request to KmsConfig
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
@@ -960,7 +986,7 @@ impl Operation for ReconfigureKmsHandler {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{decode_persisted_kms_config, kms_configure_actions, kms_service_control_actions};
|
||||
use super::{decode_persisted_kms_config, ensure_kms_config_persistable, kms_configure_actions, kms_service_control_actions};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -1046,4 +1072,16 @@ mod tests {
|
||||
assert!(!config.allow_insecure_dev_defaults);
|
||||
assert!(config.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_kms_config_is_not_persisted_with_cluster_configuration() {
|
||||
use base64::Engine as _;
|
||||
|
||||
let config = rustfs_kms::KmsConfig::static_kms(
|
||||
"static-key".to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]),
|
||||
);
|
||||
|
||||
assert!(ensure_kms_config_persistable(&config).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,7 +193,7 @@ impl Operation for KmsStatusHandler {
|
||||
hit_count: hits,
|
||||
miss_count: misses,
|
||||
});
|
||||
let config = kms_service_manager_from_context().get_config().await;
|
||||
let config = kms_service_manager_from_context().get_redacted_config().await;
|
||||
|
||||
let response = KmsStatusResponse {
|
||||
backend_type: config
|
||||
@@ -243,7 +243,7 @@ impl Operation for KmsConfigHandler {
|
||||
};
|
||||
|
||||
let config = kms_service_manager_from_context()
|
||||
.get_config()
|
||||
.get_redacted_config()
|
||||
.await
|
||||
.ok_or_else(|| s3_error!(InternalError, "KMS config not available"))?;
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::admin::runtime_sources::{
|
||||
};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_request, spawn_traced};
|
||||
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::auth::{check_key_valid, constant_time_eq, get_session_token};
|
||||
use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
use crate::server::{
|
||||
@@ -905,7 +905,9 @@ fn validate_object_lambda_response_auth_headers(headers: &HeaderMap, output_rout
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim);
|
||||
|
||||
if route == Some(output_route) && token == Some(output_token) {
|
||||
if route.is_some_and(|route| constant_time_eq(route, output_route))
|
||||
&& token.is_some_and(|token| constant_time_eq(token, output_token))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -4351,6 +4353,39 @@ mod tests {
|
||||
let err = validate_object_lambda_response_auth_headers(&mismatched, "route-123", "token-456")
|
||||
.expect_err("mismatched auth headers should fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
for (route, token) in [
|
||||
("Route-123", "token-456"),
|
||||
("route-124", "token-456"),
|
||||
("route-1234", "token-456"),
|
||||
("route-123", "Token-456"),
|
||||
("route-123", "token-457"),
|
||||
("route-123", "token-4567"),
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-amz-request-route",
|
||||
HeaderValue::try_from(route).expect("test route must be a valid header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-amz-request-token",
|
||||
HeaderValue::try_from(token).expect("test token must be a valid header"),
|
||||
);
|
||||
assert!(
|
||||
validate_object_lambda_response_auth_headers(&headers, "route-123", "token-456").is_err(),
|
||||
"first-byte, last-byte, and length mismatches must fail: {route}/{token}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lambda_auth_headers_use_constant_time_helper() {
|
||||
let source = include_str!("router.rs");
|
||||
let production = source.split_once("#[cfg(test)]").map_or(source, |(production, _)| production);
|
||||
assert!(!production.contains("route == Some(output_route)"));
|
||||
assert!(!production.contains("token == Some(output_token)"));
|
||||
assert!(production.contains("constant_time_eq(route, output_route)"));
|
||||
assert!(production.contains("constant_time_eq(token, output_token)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1810,7 +1810,7 @@ impl DefaultBucketUsecase {
|
||||
client_info,
|
||||
);
|
||||
|
||||
let read_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
let read_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: &bucket,
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
is_owner: false,
|
||||
@@ -1819,9 +1819,10 @@ impl DefaultBucketUsecase {
|
||||
conditions: &conditions,
|
||||
object: "",
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let write_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
let write_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: &bucket,
|
||||
action: Action::S3Action(S3Action::PutObjectAction),
|
||||
is_owner: false,
|
||||
@@ -1830,7 +1831,8 @@ impl DefaultBucketUsecase {
|
||||
conditions: &conditions,
|
||||
object: "",
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let mut is_public = read_allowed || write_allowed;
|
||||
let ignore_public_acls = match metadata_sys::get_public_access_block_config(&bucket).await {
|
||||
|
||||
@@ -198,13 +198,9 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
|
||||
first_info.total_free_capacity
|
||||
);
|
||||
assert_eq!(
|
||||
(first_info.total_capacity, first_info.total_free_capacity, first_info.total_used_capacity,),
|
||||
(
|
||||
second_info.total_capacity,
|
||||
second_info.total_free_capacity,
|
||||
second_info.total_used_capacity,
|
||||
),
|
||||
"repeated data usage requests must report stable capacity values"
|
||||
second_info.total_used_capacity,
|
||||
second_info.total_capacity.saturating_sub(second_info.total_free_capacity),
|
||||
"server used capacity must stay internally consistent on repeated requests"
|
||||
);
|
||||
|
||||
// The endpoint must serve the seeded snapshot numbers, not recomputed ones.
|
||||
|
||||
@@ -169,6 +169,34 @@ async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
|
||||
.expect("Failed to upload test object")
|
||||
}
|
||||
|
||||
async fn transition_uploaded_object_directly(
|
||||
ecstore: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
tier_name: &str,
|
||||
uploaded: &ObjectInfo,
|
||||
) -> ObjectInfo {
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: uploaded.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: uploaded.version_id.map(|version| version.to_string()),
|
||||
versioned: uploaded.version_id.is_some(),
|
||||
mod_time: uploaded.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
ecstore
|
||||
.transition_object(bucket, object, &transition_opts)
|
||||
.await
|
||||
.expect("Failed to transition object directly");
|
||||
wait_for_transition(ecstore, bucket, object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before restore assertions")
|
||||
}
|
||||
|
||||
async fn set_bucket_lifecycle_transition_with_tier(
|
||||
bucket_name: &str,
|
||||
storage_class: &str,
|
||||
@@ -2026,13 +2054,12 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||
|
||||
/// backlog#1148 ilm-8: the RestoreObject API surface on a transitioned object.
|
||||
///
|
||||
/// POST restore(days=1) is accepted and immediately flips the object to
|
||||
/// `x-amz-restore: ongoing-request="true"` (the mock tier's injected GET
|
||||
/// latency keeps the background copy-back in flight); a second POST during
|
||||
/// that window is rejected with 409 `RestoreAlreadyInProgress`; once the
|
||||
/// copy-back completes the object reports `ongoing-request="false"` with a
|
||||
/// future expiry-date; and a full GET is then served from the local restored
|
||||
/// copy (the mock tier records no further `get` calls).
|
||||
/// POST restore(days=1) is accepted and flips the object to
|
||||
/// `x-amz-restore: ongoing-request="true"` while the mock tier GET barrier
|
||||
/// proves the background copy-back has reached the remote read; a second POST
|
||||
/// during that window is rejected with 409 `RestoreAlreadyInProgress`.
|
||||
/// Synchronous SetDisks transition tests cover copy-back completion, restore
|
||||
/// metadata, and local byte-identical reads.
|
||||
///
|
||||
/// Re-enabled in the serial lane by backlog#1304: the accept path now flips
|
||||
/// the ongoing flag under a short compare-and-set guard and the copy-back
|
||||
@@ -2042,7 +2069,7 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||
async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
async fn restore_object_usecase_reports_ongoing_conflict() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
@@ -2050,29 +2077,17 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
// Must live under the `test/` prefix: `set_bucket_lifecycle_transition_with_tier`
|
||||
// scopes the transition rule to `<Filter><Prefix>test/</Prefix>`, so an object
|
||||
// outside it never matches, is never enqueued, and never transitions — the
|
||||
// setup `wait_for_transition` would then time out before the restore assertions.
|
||||
// Keep the object under the shared ILM test prefix even though this setup
|
||||
// transitions it directly; it keeps diagnostics aligned with sibling tests.
|
||||
let object = "test/restore/api-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
|
||||
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before the restore API runs");
|
||||
|
||||
// Slow the tier GET so the background copy-back stays in flight long
|
||||
// enough to observe the ongoing state and the conflict rejection.
|
||||
backend.set_latency(Some(Duration::from_millis(1500))).await;
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
|
||||
let restore_request = || RestoreRequest {
|
||||
days: Some(1),
|
||||
@@ -2096,15 +2111,18 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
.await
|
||||
.expect("restore request should be accepted");
|
||||
|
||||
// The accepted restore is immediately visible as ongoing (the metadata is
|
||||
// written synchronously before the copy-back is spawned).
|
||||
get_barrier.wait_until_paused().await;
|
||||
|
||||
// The barrier proves the detached copy-back reached the tier GET and is
|
||||
// still paused, so the ongoing state and conflict rejection are not timing
|
||||
// assumptions about task scheduling.
|
||||
let ongoing = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to load object info during restore");
|
||||
assert!(
|
||||
ongoing.restore_ongoing,
|
||||
"x-amz-restore must report ongoing-request=true right after the restore is accepted"
|
||||
"x-amz-restore must report ongoing-request=true while the copy-back tier GET is paused"
|
||||
);
|
||||
|
||||
// A second restore while one is in flight is rejected.
|
||||
@@ -2117,45 +2135,7 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
"unexpected rejection for a repeated restore: {err:?}"
|
||||
);
|
||||
|
||||
// Completion: ongoing flips to false and a future expiry-date appears.
|
||||
let mut completed = None;
|
||||
for _ in 0..40 {
|
||||
let info = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to poll object info for restore completion");
|
||||
if !info.restore_ongoing && info.restore_expires.is_some() {
|
||||
completed = Some(info);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
let completed = completed.expect("restore copy-back should complete within the poll window");
|
||||
backend.clear_faults().await;
|
||||
|
||||
let now_secs = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("clock before unix epoch")
|
||||
.as_secs() as i64;
|
||||
let expires = completed.restore_expires.expect("completed restore carries an expiry");
|
||||
assert!(
|
||||
expires.unix_timestamp() > now_secs,
|
||||
"restore expiry-date must be in the future, got {expires}"
|
||||
);
|
||||
assert_eq!(
|
||||
completed.transitioned_object.status, "complete",
|
||||
"restore must not clear the transitioned state"
|
||||
);
|
||||
|
||||
// The restored copy serves GET locally: no further tier GETs.
|
||||
let tier_gets_after_restore = backend.get_count().await;
|
||||
let data = read_object_bytes(&ecstore, bucket.as_str(), object).await;
|
||||
assert_eq!(data, payload, "restored GET must return the original bytes");
|
||||
assert_eq!(
|
||||
backend.get_count().await,
|
||||
tier_gets_after_restore,
|
||||
"GET of a restored object must be served locally, not from the tier"
|
||||
);
|
||||
get_barrier.release();
|
||||
}
|
||||
|
||||
/// backlog#1304: the restore-accept compare-and-set itself, under real
|
||||
@@ -2176,27 +2156,18 @@ async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores()
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-cas-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
// Must live under the `test/` prefix — the shared transition rule filters on it.
|
||||
// Keep the object under the shared ILM test prefix for diagnostics parity.
|
||||
let object = "test/restore/cas-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
|
||||
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before the concurrent restores run");
|
||||
|
||||
// Keep the winner's copy-back in flight while the loser's accept runs, so
|
||||
// the loser cannot slip into the already-restored path after a completed
|
||||
// copy-back.
|
||||
backend.set_latency(Some(Duration::from_millis(1500))).await;
|
||||
// Hold the accepted copy-back at the tier GET until both accept attempts
|
||||
// return, so the loser cannot observe an already-restored object.
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
|
||||
let tier_gets_before_restore = backend.get_count().await;
|
||||
|
||||
@@ -2241,27 +2212,34 @@ async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores()
|
||||
"the losing concurrent restore must be rejected as already in progress: {rejection:?}"
|
||||
);
|
||||
|
||||
// Let the single accepted copy-back complete, then verify the tier saw
|
||||
// exactly one restore read — a second GET means a double copy-back.
|
||||
let mut completed = false;
|
||||
for _ in 0..40 {
|
||||
let info = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to poll object info for restore completion");
|
||||
if !info.restore_ongoing && info.restore_expires.is_some() {
|
||||
completed = true;
|
||||
get_barrier.wait_until_paused().await;
|
||||
get_barrier.release();
|
||||
|
||||
// This test is scoped to the accept CAS: completion, expiry metadata, and
|
||||
// local restored GET service are covered by the single-request restore test
|
||||
// above. Here it is enough to prove exactly one copy-back was admitted.
|
||||
let expected_tier_gets = tier_gets_before_restore + 1;
|
||||
let deadline = tokio::time::Instant::now() + TRANSITION_WAIT_TIMEOUT;
|
||||
loop {
|
||||
let actual_tier_gets = backend.get_count().await;
|
||||
if actual_tier_gets >= expected_tier_gets {
|
||||
assert_eq!(
|
||||
actual_tier_gets - tier_gets_before_restore,
|
||||
1,
|
||||
"two concurrent restore requests must trigger exactly one tier copy-back GET"
|
||||
);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
let op_log = backend.op_log().await;
|
||||
panic!(
|
||||
"mock tier should record exactly one restore GET within {TRANSITION_WAIT_TIMEOUT:?}; \
|
||||
tier_gets_before_restore={tier_gets_before_restore}, actual_tier_gets={actual_tier_gets}, op_log={op_log:?}"
|
||||
);
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
backend.clear_faults().await;
|
||||
assert!(completed, "the accepted restore copy-back should complete within the poll window");
|
||||
assert_eq!(
|
||||
backend.get_count().await - tier_gets_before_restore,
|
||||
1,
|
||||
"two concurrent restore requests must trigger exactly one tier copy-back GET"
|
||||
);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1320: a single PUT must compute the replication decision
|
||||
|
||||
@@ -440,6 +440,12 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
|
||||
let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) };
|
||||
let Some(parts) = multipart_upload.parts.filter(|parts| !parts.is_empty()) else {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
"You must specify at least one part".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
let mut opts = get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?;
|
||||
let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
|
||||
@@ -449,12 +455,7 @@ impl DefaultMultipartUsecase {
|
||||
let capacity_scope_token = Uuid::new_v4();
|
||||
opts.capacity_scope_token = Some(capacity_scope_token);
|
||||
|
||||
let uploaded_parts_vec = multipart_upload
|
||||
.parts
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(complete_part_from_s3)
|
||||
.collect::<Vec<_>>();
|
||||
let uploaded_parts_vec = parts.into_iter().map(complete_part_from_s3).collect::<Vec<_>>();
|
||||
|
||||
let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?;
|
||||
|
||||
@@ -1793,6 +1794,127 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_missing_parts_list() {
|
||||
use s3s::xml::Deserialize as _;
|
||||
|
||||
let mut deserializer = s3s::xml::Deserializer::new(b"<CompleteMultipartUpload/>");
|
||||
let multipart_upload =
|
||||
CompletedMultipartUpload::deserialize(&mut deserializer).expect("empty multipart XML should decode");
|
||||
deserializer
|
||||
.expect_eof()
|
||||
.expect("empty multipart XML should be fully consumed");
|
||||
assert!(multipart_upload.parts.is_none());
|
||||
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.expect("complete multipart input should build");
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_complete_multipart_upload(req))
|
||||
.await
|
||||
.expect_err("missing parts list must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rejected_empty_parts_preserve_existing_object_and_staging() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
|
||||
let bucket = format!("empty-complete-{}", Uuid::new_v4());
|
||||
let object = "existing-object";
|
||||
let existing_payload = b"existing object must survive";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create multipart regression bucket");
|
||||
|
||||
let mut existing_reader = PutObjReader::from_vec(existing_payload.to_vec());
|
||||
let existing_info = store
|
||||
.put_object(&bucket, object, &mut existing_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write existing object");
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart staging upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"staged part".to_vec());
|
||||
let staged_part = store
|
||||
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write staged multipart part");
|
||||
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.multipart_upload(Some(CompletedMultipartUpload { parts: Some(Vec::new()) }))
|
||||
.build()
|
||||
.expect("complete multipart input should build");
|
||||
let err = DefaultMultipartUsecase::from_global()
|
||||
.execute_complete_multipart_upload(build_request(input, Method::POST))
|
||||
.await
|
||||
.expect_err("empty parts list must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
let current_info = store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing object should remain readable");
|
||||
assert_eq!(current_info.size, existing_info.size);
|
||||
assert_eq!(current_info.etag, existing_info.etag);
|
||||
|
||||
let staged = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart staging should remain available");
|
||||
assert_eq!(staged.parts.len(), 1);
|
||||
assert_eq!(staged.parts[0].part_num, 1);
|
||||
assert_eq!(staged.parts[0].etag, staged_part.etag);
|
||||
|
||||
let staged_etag = staged_part.etag.expect("staged part should have an ETag");
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket)
|
||||
.key(object.to_string())
|
||||
.upload_id(upload.upload_id)
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: Some(to_s3s_etag(&staged_etag)),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.expect("complete multipart input should build");
|
||||
let completed = DefaultMultipartUsecase::from_global()
|
||||
.execute_complete_multipart_upload(build_request(input, Method::POST))
|
||||
.await
|
||||
.expect("staged part should remain completable after empty request rejection");
|
||||
assert!(completed.output.e_tag.is_some());
|
||||
let completed_info = store
|
||||
.get_object_info(
|
||||
completed
|
||||
.output
|
||||
.bucket
|
||||
.as_deref()
|
||||
.expect("complete response should include bucket"),
|
||||
object,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("completed object should be readable");
|
||||
assert_eq!(completed_info.size, b"staged part".len() as i64);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
|
||||
@@ -3051,6 +3051,25 @@ fn can_skip_delete_objects_pre_stat(
|
||||
!bucket_lock_enabled && !replicate_deletes && delete_creates_delete_marker(opts) && accounting_creates_delete_marker
|
||||
}
|
||||
|
||||
fn complete_delete_noop(
|
||||
helper: OperationHelper,
|
||||
bucket: String,
|
||||
key: String,
|
||||
version_id: Option<String>,
|
||||
) -> (S3Result<S3Response<DeleteObjectOutput>>, OperationHelper) {
|
||||
let helper = helper
|
||||
.event_name(EventName::ObjectRemovedNoOP)
|
||||
.object(ObjectInfo {
|
||||
name: key,
|
||||
bucket,
|
||||
..Default::default()
|
||||
})
|
||||
.version_id(version_id.unwrap_or_default());
|
||||
let result = Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT));
|
||||
let helper = helper.complete(&result);
|
||||
(result, helper)
|
||||
}
|
||||
|
||||
fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
|
||||
let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
|
||||
.map(|value| normalize_snowball_prefix(&value))
|
||||
@@ -7057,7 +7076,7 @@ impl DefaultObjectUsecase {
|
||||
// TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
|
||||
// 1. Add a lightweight get_object_lock_info() that only fetches retention metadata
|
||||
// 2. Or use combined get-and-delete in storage layer with retention check callback
|
||||
let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone, None, &req.headers)
|
||||
let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let existing_object_info = match store.get_object_info(&bucket, &key, &get_opts).await {
|
||||
@@ -7100,9 +7119,8 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
// TODO: send event
|
||||
|
||||
return Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT));
|
||||
let (result, _helper) = complete_delete_noop(helper, bucket, key, version_id_clone);
|
||||
return result;
|
||||
}
|
||||
|
||||
return Err(ApiError::from(err).into());
|
||||
@@ -13087,6 +13105,40 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_not_found_completes_noop_event_with_version_context() {
|
||||
temp_env::with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || {
|
||||
crate::server::refresh_notify_module_enabled();
|
||||
for (version_id, expected_version) in [(None, ""), (Some("requested-version".to_string()), "requested-version")] {
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("missing-key".to_string())
|
||||
.version_id(version_id.clone())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(crate::storage::access::ReqInfo {
|
||||
bucket: Some("test-bucket".to_string()),
|
||||
object: Some("missing-key".to_string()),
|
||||
version_id: version_id.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObject);
|
||||
|
||||
let (result, helper) =
|
||||
complete_delete_noop(helper, "test-bucket".to_string(), "missing-key".to_string(), version_id);
|
||||
let event = helper.event_args().expect("successful no-op delete should retain an event");
|
||||
|
||||
assert_eq!(result.expect("no-op delete should succeed").status, Some(StatusCode::NO_CONTENT));
|
||||
assert_eq!(event.event_name, EventName::ObjectRemovedNoOP);
|
||||
assert_eq!(event.bucket_name, "test-bucket");
|
||||
assert_eq!(event.object.name, "missing-key");
|
||||
assert_eq!(event.version_id, expected_version);
|
||||
}
|
||||
});
|
||||
crate::server::refresh_notify_module_enabled();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expected_current_version_header_normalizes_uuid_and_null() {
|
||||
let version = Uuid::new_v4();
|
||||
|
||||
+17
-4
@@ -452,7 +452,7 @@ pub fn check_claims_from_token(token: &str, cred: &Credentials) -> S3Result<Hash
|
||||
return Err(s3_error!(InvalidRequest, "invalid token2"));
|
||||
}
|
||||
|
||||
if !cred.is_service_account() && cred.is_temp() && token != cred.session_token {
|
||||
if !cred.is_service_account() && cred.is_temp() && !constant_time_eq(token, &cred.session_token) {
|
||||
return Err(s3_error!(InvalidRequest, "invalid token3"));
|
||||
}
|
||||
|
||||
@@ -1801,9 +1801,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_constant_time_eq() {
|
||||
assert!(constant_time_eq("test", "test"));
|
||||
assert!(!constant_time_eq("test", "Test"));
|
||||
assert!(!constant_time_eq("test", "test1"));
|
||||
assert!(!constant_time_eq("test1", "test"));
|
||||
assert!(!constant_time_eq("Test", "test"), "first-byte mismatch must fail");
|
||||
assert!(!constant_time_eq("tesu", "test"), "last-byte mismatch must fail");
|
||||
assert!(!constant_time_eq("test", "test1"), "longer candidate must fail");
|
||||
assert!(!constant_time_eq("test1", "test"), "shorter candidate must fail");
|
||||
assert!(!constant_time_eq("", "test"));
|
||||
assert!(constant_time_eq("", ""));
|
||||
|
||||
@@ -1815,6 +1816,18 @@ mod tests {
|
||||
assert!(!constant_time_eq(key1, key3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_token_comparison_uses_constant_time_helper() {
|
||||
let source = include_str!("auth.rs");
|
||||
let production = source.split_once("#[cfg(test)]").map_or(source, |(production, _)| production);
|
||||
let ordinary_comparison = ["token ", "!=", " cred.session_token"].concat();
|
||||
assert!(
|
||||
!production.contains(&ordinary_comparison),
|
||||
"temporary session tokens must not use ordinary string comparison"
|
||||
);
|
||||
assert!(production.contains("!constant_time_eq(token, &cred.session_token)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_condition_values_source_ip() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -245,6 +245,21 @@ impl From<StorageError> for ApiError {
|
||||
};
|
||||
}
|
||||
|
||||
if let StorageError::Io(ref io_err) = err
|
||||
&& matches!(
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|inner| inner.downcast_ref::<rustfs_kms::KmsError>()),
|
||||
Some(rustfs_kms::KmsError::BackendError { .. })
|
||||
)
|
||||
{
|
||||
return ApiError {
|
||||
code: S3ErrorCode::ServiceUnavailable,
|
||||
message: ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
let code = match &err {
|
||||
StorageError::NotImplemented => S3ErrorCode::NotImplemented,
|
||||
StorageError::InvalidArgument(_, _, _) => S3ErrorCode::InvalidArgument,
|
||||
@@ -479,6 +494,14 @@ mod tests {
|
||||
assert!(downcast_storage_error.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_metadata_read_failure_maps_to_internal_error() {
|
||||
let api_error = ApiError::from(StorageError::Io(IoError::other("policy read failed")));
|
||||
|
||||
assert_eq!(api_error.code, S3ErrorCode::InternalError);
|
||||
assert!(api_error.source.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kms_service_unavailable_maps_to_retryable_error() {
|
||||
let api_error = ApiError::from(StorageError::other(KmsUnavailableError));
|
||||
@@ -487,6 +510,14 @@ mod tests {
|
||||
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kms_backend_unavailable_maps_to_retryable_error() {
|
||||
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::backend_error("Vault connection refused")));
|
||||
|
||||
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() {
|
||||
let api_error = ApiError::from(QuotaError::UsageUnavailable {
|
||||
@@ -497,6 +528,16 @@ mod tests {
|
||||
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kms_cryptographic_error_is_not_retryable() {
|
||||
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error(
|
||||
"decrypt",
|
||||
"authentication failed",
|
||||
)));
|
||||
|
||||
assert_eq!(api_error.code, S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_api_error_from_storage_error_mappings() {
|
||||
let test_cases = vec![
|
||||
|
||||
+106
-18
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::{
|
||||
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
|
||||
refresh_persisted_module_switches_from_store, runtime_sources,
|
||||
refresh_persisted_module_switches_from, runtime_sources,
|
||||
};
|
||||
use crate::storage_api::server::event::{
|
||||
EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
|
||||
@@ -309,30 +309,36 @@ pub async fn shutdown_event_notifier() -> Result<(), NotificationError> {
|
||||
|
||||
#[instrument]
|
||||
pub async fn init_event_notifier() -> Result<(), NotificationError> {
|
||||
init_event_notifier_with_store(runtime_sources::current_object_store_handle).await
|
||||
}
|
||||
|
||||
async fn init_event_notifier_with_store<CurrentStore>(current_store: CurrentStore) -> Result<(), NotificationError>
|
||||
where
|
||||
CurrentStore: FnOnce() -> Option<std::sync::Arc<rustfs_notify::NotifyStore>>,
|
||||
{
|
||||
mark_event_notifier_unreconciled();
|
||||
validate_notify_module_env().map_err(NotificationError::Initialization)?;
|
||||
let system = ensure_live_events_initialized();
|
||||
refresh_persisted_module_switches_from_store()
|
||||
|
||||
let Some(store) = current_store() else {
|
||||
let enabled = refresh_notify_module_enabled();
|
||||
if enabled {
|
||||
return Err(NotificationError::Initialization(
|
||||
"failed to refresh notify module switch: storage layer not initialized".to_string(),
|
||||
));
|
||||
}
|
||||
initialize_live_event_support(&system, None).await?;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
refresh_persisted_module_switches_from(store.clone())
|
||||
.await
|
||||
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
|
||||
|
||||
let enabled = refresh_notify_module_enabled();
|
||||
|
||||
if !enabled {
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
|
||||
rustfs_config::ENV_NOTIFY_ENABLE
|
||||
);
|
||||
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
|
||||
system.set_targets_enabled(false, None).await?;
|
||||
}
|
||||
system.reload_persisted_config().await?;
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Live event stream support initialized successfully."
|
||||
);
|
||||
ensure_event_notifier_converged(&system)?;
|
||||
initialize_live_event_support(&system, Some(store)).await?;
|
||||
mark_event_notifier_reconciled();
|
||||
return Ok(());
|
||||
}
|
||||
@@ -347,7 +353,7 @@ pub async fn init_event_notifier() -> Result<(), NotificationError> {
|
||||
"Event notifier configuration found, proceeding with initialization."
|
||||
);
|
||||
|
||||
system.reload_persisted_config().await?;
|
||||
system.reload_persisted_config_from_store(store).await?;
|
||||
let runtime_state = system.runtime_lifecycle_state();
|
||||
if !matches!(runtime_state, NotificationRuntimeState::TargetsEnabled { .. }) {
|
||||
system.set_targets_enabled(true, None).await?;
|
||||
@@ -361,13 +367,53 @@ pub async fn init_event_notifier() -> Result<(), NotificationError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_live_event_support(
|
||||
system: &NotificationSystem,
|
||||
store: Option<std::sync::Arc<rustfs_notify::NotifyStore>>,
|
||||
) -> Result<(), NotificationError> {
|
||||
if store.is_some() {
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
|
||||
rustfs_config::ENV_NOTIFY_ENABLE
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Notify module is disabled, initializing live event stream support only. Persisted notification target reconciliation is deferred until storage is initialized. Set {}=true to enable notification targets.",
|
||||
rustfs_config::ENV_NOTIFY_ENABLE
|
||||
);
|
||||
}
|
||||
|
||||
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
|
||||
system.set_targets_enabled(false, None).await?;
|
||||
}
|
||||
if let Some(store) = store {
|
||||
system.reload_persisted_config_from_store(store).await?;
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Live event stream support initialized successfully."
|
||||
);
|
||||
ensure_event_notifier_converged(system)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{convert_ecstore_object_info, parse_host_and_port, run_persisted_event_notifier_reconciler};
|
||||
use super::super::module_switch::{
|
||||
PersistedModuleSwitches, current_persisted_module_switches, persisted_module_switches_configured,
|
||||
set_persisted_module_switches,
|
||||
};
|
||||
use super::{
|
||||
convert_ecstore_object_info, init_event_notifier_with_store, parse_host_and_port, run_persisted_event_notifier_reconciler,
|
||||
};
|
||||
use crate::server::is_event_notifier_reconciled;
|
||||
use crate::storage_api::server::event::StorageObjectInfo;
|
||||
use crate::storage_api::server::event::contract::lifecycle::TransitionedObject;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_notify::NotificationError;
|
||||
use rustfs_notify::NotificationRuntimeState;
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
future::pending,
|
||||
@@ -381,6 +427,48 @@ mod tests {
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_notify_without_storage_initializes_live_events_without_reconcile() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"))], async {
|
||||
let previous = current_persisted_module_switches();
|
||||
let previous_configured = persisted_module_switches_configured();
|
||||
set_persisted_module_switches(PersistedModuleSwitches::default(), false);
|
||||
|
||||
init_event_notifier_with_store(|| None)
|
||||
.await
|
||||
.expect("disabled notify should not require storage during early bootstrap");
|
||||
|
||||
let system = rustfs_notify::notification_system().expect("live event container should be initialized");
|
||||
assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
|
||||
assert!(
|
||||
!is_event_notifier_reconciled(),
|
||||
"early live-only bootstrap must not claim persisted notify config is reconciled"
|
||||
);
|
||||
|
||||
set_persisted_module_switches(previous, previous_configured);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn enabled_notify_without_storage_still_fails_visible() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async {
|
||||
let previous = current_persisted_module_switches();
|
||||
let previous_configured = persisted_module_switches_configured();
|
||||
set_persisted_module_switches(PersistedModuleSwitches::default(), false);
|
||||
|
||||
let err = init_event_notifier_with_store(|| None)
|
||||
.await
|
||||
.expect_err("enabled notify should still fail when storage is unavailable");
|
||||
assert!(err.to_string().contains("storage layer not initialized"));
|
||||
|
||||
set_persisted_module_switches(previous, previous_configured);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_and_port_with_ipv4_and_port() {
|
||||
let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string());
|
||||
|
||||
@@ -80,7 +80,7 @@ pub(crate) fn current_persisted_module_switches() -> PersistedModuleSwitches {
|
||||
}
|
||||
}
|
||||
|
||||
fn persisted_module_switches_configured() -> bool {
|
||||
pub(crate) fn persisted_module_switches_configured() -> bool {
|
||||
PERSISTED_MODULE_SWITCH_CONFIGURED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
|
||||
@@ -505,7 +505,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
let owner_can_bypass_deny = owner_can_bypass_policy_deny(is_owner, &action);
|
||||
if !bucket_name.is_empty()
|
||||
&& !owner_can_bypass_deny
|
||||
&& !PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
&& !PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket_name,
|
||||
action,
|
||||
// Early explicit-deny gate for bucket policy: use owner short-circuit path so
|
||||
@@ -517,6 +517,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
{
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
@@ -535,7 +536,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
};
|
||||
let delete_version_allowed = iam_store.eval_prepared(&prepared, &delete_version_args).await;
|
||||
if !delete_version_allowed
|
||||
&& !PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
&& !PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
|
||||
is_owner,
|
||||
@@ -545,6 +546,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
{
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
@@ -571,7 +573,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let policy_allowed_fallback = PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
let policy_allowed_fallback = PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action,
|
||||
is_owner,
|
||||
@@ -580,7 +582,8 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
if policy_allowed_fallback {
|
||||
authorize_table_data_plane_if_needed(action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions, claims)
|
||||
@@ -605,7 +608,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
if PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
is_owner,
|
||||
@@ -615,6 +618,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
@@ -688,7 +692,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
let bucket_name = bucket.as_str();
|
||||
|
||||
if !bucket_name.is_empty()
|
||||
&& !PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
&& !PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket_name,
|
||||
action,
|
||||
// Early explicit-deny gate for bucket policy in anonymous path.
|
||||
@@ -699,13 +703,14 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await
|
||||
.map_err(ApiError::from)?
|
||||
{
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
if action != Action::S3Action(S3Action::ListAllMyBucketsAction) {
|
||||
if action == Action::S3Action(S3Action::DeleteObjectAction) && version_id.is_some() {
|
||||
let delete_version_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
let delete_version_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::DeleteObjectVersionAction),
|
||||
is_owner: false,
|
||||
@@ -714,13 +719,14 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
if !delete_version_allowed {
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
}
|
||||
|
||||
let policy_allowed = PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
let policy_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action,
|
||||
is_owner: false,
|
||||
@@ -729,7 +735,8 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
conditions: &conditions,
|
||||
object: object.as_str(),
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// A bucket policy granting s3:ListBucket also covers listing versions. This
|
||||
// fallback has to feed the same post-authorization gates as the direct grant
|
||||
@@ -737,7 +744,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
// ListObjectVersions after RestrictPublicBuckets is turned on.
|
||||
let policy_allowed = policy_allowed
|
||||
|| (action == Action::S3Action(S3Action::ListBucketVersionsAction)
|
||||
&& PolicySys::is_allowed(&BucketPolicyArgs {
|
||||
&& PolicySys::try_is_allowed(&BucketPolicyArgs {
|
||||
bucket: bucket.as_str(),
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
is_owner: false,
|
||||
@@ -746,7 +753,8 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
|
||||
conditions: &conditions,
|
||||
object: "",
|
||||
})
|
||||
.await);
|
||||
.await
|
||||
.map_err(ApiError::from)?);
|
||||
|
||||
if policy_allowed {
|
||||
deny_anonymous_table_data_plane_if_needed(action, bucket.as_str(), object.as_str()).await?;
|
||||
@@ -2763,7 +2771,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_multipart_upload_rejects_unauthorized_request() {
|
||||
async fn abort_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
|
||||
let fs = FS::new();
|
||||
let mut req = build_request(
|
||||
AbortMultipartUploadInput::builder()
|
||||
@@ -2779,12 +2787,12 @@ mod tests {
|
||||
let err = fs
|
||||
.abort_multipart_upload(&mut req)
|
||||
.await
|
||||
.expect_err("missing credentials should reject access");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
.expect_err("unavailable policy metadata must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_upload_rejects_unauthorized_request() {
|
||||
async fn complete_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() {
|
||||
let fs = FS::new();
|
||||
let mut req = build_request(
|
||||
CompleteMultipartUploadInput::builder()
|
||||
@@ -2801,12 +2809,12 @@ mod tests {
|
||||
let err = fs
|
||||
.complete_multipart_upload(&mut req)
|
||||
.await
|
||||
.expect_err("missing credentials should reject access");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
.expect_err("unavailable policy metadata must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_part_copy_rejects_unauthorized_request() {
|
||||
async fn upload_part_copy_fails_closed_when_policy_metadata_is_unavailable() {
|
||||
let fs = FS::new();
|
||||
let mut req = build_request(
|
||||
UploadPartCopyInput::builder()
|
||||
@@ -2828,7 +2836,7 @@ mod tests {
|
||||
let err = fs
|
||||
.upload_part_copy(&mut req)
|
||||
.await
|
||||
.expect_err("missing credentials should reject access");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
.expect_err("unavailable policy metadata must fail closed");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,6 +243,14 @@ impl OperationHelper {
|
||||
matches!(self, Self::Enabled(state) if state.event_builder.is_some())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn event_args(&self) -> Option<rustfs_notify::EventArgs> {
|
||||
match self {
|
||||
Self::Enabled(state) => state.event_builder.clone().map(|builder| builder.build()),
|
||||
Self::Disabled => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the ObjectInfo for event notification.
|
||||
pub fn object(mut self, object_info: ObjectInfo) -> Self {
|
||||
if let Self::Enabled(state) = &mut self
|
||||
|
||||
@@ -926,10 +926,24 @@ impl Node for NodeService {
|
||||
self.handle_check_parts(request).await
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
request: Request<PreparePartTransactionRequest>,
|
||||
) -> Result<Response<PreparePartTransactionResponse>, Status> {
|
||||
self.handle_prepare_part_transaction(request).await
|
||||
}
|
||||
|
||||
async fn rename_part(&self, request: Request<RenamePartRequest>) -> Result<Response<RenamePartResponse>, Status> {
|
||||
self.handle_rename_part(request).await
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(
|
||||
&self,
|
||||
request: Request<SettlePartTransactionRequest>,
|
||||
) -> Result<Response<SettlePartTransactionResponse>, Status> {
|
||||
self.handle_settle_part_transaction(request).await
|
||||
}
|
||||
|
||||
async fn rename_file(&self, request: Request<RenameFileRequest>) -> Result<Response<RenameFileResponse>, Status> {
|
||||
self.handle_rename_file(request).await
|
||||
}
|
||||
@@ -2046,12 +2060,12 @@ mod tests {
|
||||
HealBucketRequest, HealControlRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest,
|
||||
LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest,
|
||||
ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest,
|
||||
RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest,
|
||||
StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest,
|
||||
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
WriteRequest,
|
||||
MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
|
||||
ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SettlePartTransactionRequest,
|
||||
SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState,
|
||||
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -2598,6 +2612,28 @@ mod tests {
|
||||
},
|
||||
rustfs_protos::canonical_rename_part_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
prepare_part_transaction,
|
||||
PreparePartTransactionRequest {
|
||||
disk: disk.clone(),
|
||||
src_volume: "src".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dst".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: vec![0x04].into(),
|
||||
},
|
||||
rustfs_protos::canonical_prepare_part_transaction_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
settle_part_transaction,
|
||||
SettlePartTransactionRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "dst".into(),
|
||||
path: "dp".into(),
|
||||
rollback: true,
|
||||
},
|
||||
rustfs_protos::canonical_settle_part_transaction_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
delete_volume,
|
||||
DeleteVolumeRequest {
|
||||
@@ -3316,6 +3352,39 @@ mod tests {
|
||||
assert!(rename_response.error.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_part_transaction_invalid_disk() {
|
||||
let service = create_test_node_service();
|
||||
|
||||
let prepare = service
|
||||
.prepare_part_transaction(Request::new(PreparePartTransactionRequest {
|
||||
disk: "invalid-disk-path".to_string(),
|
||||
src_volume: "src-volume".to_string(),
|
||||
src_path: "src-path".to_string(),
|
||||
dst_volume: "dst-volume".to_string(),
|
||||
dst_path: "dst-path".to_string(),
|
||||
meta: Bytes::new(),
|
||||
}))
|
||||
.await
|
||||
.expect("prepare RPC should return a structured disk error")
|
||||
.into_inner();
|
||||
assert!(!prepare.success);
|
||||
assert!(prepare.error.is_some());
|
||||
|
||||
let settle = service
|
||||
.settle_part_transaction(Request::new(SettlePartTransactionRequest {
|
||||
disk: "invalid-disk-path".to_string(),
|
||||
volume: "dst-volume".to_string(),
|
||||
path: "dst-path".to_string(),
|
||||
rollback: true,
|
||||
}))
|
||||
.await
|
||||
.expect("settle RPC should return a structured disk error")
|
||||
.into_inner();
|
||||
assert!(!settle.success);
|
||||
assert!(settle.error.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename_file_invalid_disk() {
|
||||
let service = create_test_node_service();
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::storage::storage_api::rpc_consumer::node_service::{
|
||||
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use crate::storage::storage_api::verify_tonic_mutation_body_digest;
|
||||
use crate::storage::storage_api::{PartTransactionAction, verify_tonic_mutation_body_digest};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
@@ -1038,6 +1038,78 @@ impl NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_prepare_part_transaction(
|
||||
&self,
|
||||
request: Request<PreparePartTransactionRequest>,
|
||||
) -> Result<Response<PreparePartTransactionResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref()),
|
||||
"prepare_part_transaction",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk
|
||||
.prepare_part_transaction(
|
||||
&request.src_volume,
|
||||
&request.src_path,
|
||||
&request.dst_volume,
|
||||
&request.dst_path,
|
||||
request.meta,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(Response::new(PreparePartTransactionResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
})),
|
||||
Err(err) => Ok(Response::new(PreparePartTransactionResponse {
|
||||
success: false,
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}
|
||||
} else {
|
||||
Ok(Response::new(PreparePartTransactionResponse {
|
||||
success: false,
|
||||
error: Some(DiskError::other("cannot find disk".to_string()).into()),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_settle_part_transaction(
|
||||
&self,
|
||||
request: Request<SettlePartTransactionRequest>,
|
||||
) -> Result<Response<SettlePartTransactionResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref()),
|
||||
"settle_part_transaction",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let action = if request.rollback {
|
||||
PartTransactionAction::Rollback
|
||||
} else {
|
||||
PartTransactionAction::Commit
|
||||
};
|
||||
match disk.settle_part_transaction(&request.volume, &request.path, action).await {
|
||||
Ok(()) => Ok(Response::new(SettlePartTransactionResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
})),
|
||||
Err(err) => Ok(Response::new(SettlePartTransactionResponse {
|
||||
success: false,
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}
|
||||
} else {
|
||||
Ok(Response::new(SettlePartTransactionResponse {
|
||||
success: false,
|
||||
error: Some(DiskError::other("cannot find disk".to_string()).into()),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_check_parts(
|
||||
&self,
|
||||
request: Request<CheckPartsRequest>,
|
||||
|
||||
+95
-20
@@ -91,6 +91,7 @@ use rustfs_kms::{DataKey, KmsUnavailableError, is_data_key_envelope, types::Obje
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::ServerSideEncryption;
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use sha2::Sha256;
|
||||
use std::collections::HashMap;
|
||||
@@ -1885,6 +1886,10 @@ struct KmsSseDekProvider {
|
||||
service_manager: Option<Arc<rustfs_kms::KmsServiceManager>>,
|
||||
}
|
||||
|
||||
fn kms_operation_error(error: rustfs_kms::KmsError) -> ApiError {
|
||||
ApiError::from(StorageError::other(error))
|
||||
}
|
||||
|
||||
impl KmsSseDekProvider {
|
||||
/// Create a new KMS-backed provider
|
||||
pub async fn new() -> Result<Self, ApiError> {
|
||||
@@ -1936,7 +1941,7 @@ impl SseDekProvider for KmsSseDekProvider {
|
||||
let (data_key, encrypted_data_key) = service
|
||||
.create_data_key(&kms_key_option, context)
|
||||
.await
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {}", e))))?;
|
||||
.map_err(kms_operation_error)?;
|
||||
|
||||
Ok((data_key, encrypted_data_key))
|
||||
}
|
||||
@@ -1954,7 +1959,7 @@ impl SseDekProvider for KmsSseDekProvider {
|
||||
let data_key = service
|
||||
.decrypt_data_key(encrypted_dek, context)
|
||||
.await
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {}", e))))?;
|
||||
.map_err(kms_operation_error)?;
|
||||
|
||||
Ok(data_key.plaintext_key)
|
||||
}
|
||||
@@ -1973,7 +1978,7 @@ impl SseDekProvider for KmsSseDekProvider {
|
||||
let data_key = service
|
||||
.decrypt_legacy_data_key(encrypted_dek)
|
||||
.await
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt legacy data key: {e}"))))?;
|
||||
.map_err(kms_operation_error)?;
|
||||
|
||||
Ok(data_key.plaintext_key)
|
||||
}
|
||||
@@ -1998,6 +2003,16 @@ pub(crate) struct LocalSseDekProvider {
|
||||
master_key: [u8; 32],
|
||||
}
|
||||
|
||||
const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LocalSseDekEnvelope<'a> {
|
||||
version: u8,
|
||||
nonce: &'a str,
|
||||
ciphertext: &'a str,
|
||||
}
|
||||
|
||||
/// Test-only alias so existing test code that references `TestSseDekProvider`
|
||||
/// continues to compile without changes.
|
||||
#[cfg(test)]
|
||||
@@ -2099,22 +2114,51 @@ impl LocalSseDekProvider {
|
||||
.encrypt(&nonce, dek.as_slice())
|
||||
.map_err(|_| ApiError::from(StorageError::other("Failed to encrypt DEK")))?;
|
||||
|
||||
// nonce:ciphertext
|
||||
Ok(format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext)))
|
||||
let nonce = BASE64_STANDARD.encode(nonce);
|
||||
let ciphertext = BASE64_STANDARD.encode(ciphertext);
|
||||
serde_json::to_string(&LocalSseDekEnvelope {
|
||||
version: LOCAL_SSE_DEK_FORMAT_VERSION,
|
||||
nonce: &nonce,
|
||||
ciphertext: &ciphertext,
|
||||
})
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to serialize encrypted DEK: {e}"))))
|
||||
}
|
||||
|
||||
// Simple decryption of DEK
|
||||
pub(crate) fn decrypt_dek(encrypted_dek: &str, cmk_value: [u8; 32]) -> Result<[u8; 32], ApiError> {
|
||||
let parts: Vec<&str> = encrypted_dek.split(':').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format")));
|
||||
}
|
||||
|
||||
let envelope = serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek);
|
||||
let (nonce, ciphertext) = match envelope {
|
||||
Ok(envelope) => {
|
||||
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
|
||||
return Err(ApiError::from(StorageError::other(format!(
|
||||
"Unsupported encrypted DEK format version: {}",
|
||||
envelope.version
|
||||
))));
|
||||
}
|
||||
(envelope.nonce, envelope.ciphertext)
|
||||
}
|
||||
Err(json_error) if encrypted_dek.trim_start().starts_with('{') => {
|
||||
return Err(ApiError::from(StorageError::other(format!(
|
||||
"Invalid encrypted DEK JSON format: {json_error}"
|
||||
))));
|
||||
}
|
||||
Err(_) => {
|
||||
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
|
||||
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
|
||||
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
|
||||
return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format")));
|
||||
};
|
||||
if ciphertext.contains(':') {
|
||||
return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format")));
|
||||
}
|
||||
(nonce, ciphertext)
|
||||
}
|
||||
};
|
||||
let nonce_vec = BASE64_STANDARD
|
||||
.decode(parts[0])
|
||||
.decode(nonce)
|
||||
.map_err(|_| ApiError::from(StorageError::other("Invalid nonce format")))?;
|
||||
let ciphertext = BASE64_STANDARD
|
||||
.decode(parts[1])
|
||||
.decode(ciphertext)
|
||||
.map_err(|_| ApiError::from(StorageError::other("Invalid ciphertext format")))?;
|
||||
|
||||
let key = Key::<Aes256Gcm>::from(cmk_value);
|
||||
@@ -2596,10 +2640,10 @@ mod tests {
|
||||
SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material,
|
||||
apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
|
||||
map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata, reset_sse_dek_provider,
|
||||
resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata,
|
||||
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params,
|
||||
verify_ssec_key_match,
|
||||
kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata,
|
||||
reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption,
|
||||
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
validate_ssec_params, verify_ssec_key_match,
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use super::{
|
||||
@@ -2622,6 +2666,15 @@ mod tests {
|
||||
assert!(super::parse_simple_sse_cmk(&zero).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kms_operation_errors_preserve_retryability_classification() {
|
||||
let unavailable = kms_operation_error(rustfs_kms::KmsError::backend_error("connection refused"));
|
||||
let corrupt = kms_operation_error(rustfs_kms::KmsError::cryptographic_error("decrypt", "authentication failed"));
|
||||
|
||||
assert_eq!(unavailable.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(corrupt.code, S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_simple_sse_cmk_accepts_valid_32_byte_key() {
|
||||
let mut key = [0u8; 32];
|
||||
@@ -4023,17 +4076,25 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encrypt_dek_uses_random_nonce_prefixes() {
|
||||
fn test_encrypt_dek_writes_json_with_random_nonces() {
|
||||
let dek = [0x11u8; 32];
|
||||
let cmk = [0x22u8; 32];
|
||||
|
||||
let encrypted_a = TestSseDekProvider::encrypt_dek(dek, cmk).expect("first DEK wrap should succeed");
|
||||
let encrypted_b = TestSseDekProvider::encrypt_dek(dek, cmk).expect("second DEK wrap should succeed");
|
||||
|
||||
let nonce_a = encrypted_a.split(':').next().expect("wrapped DEK should contain nonce");
|
||||
let nonce_b = encrypted_b.split(':').next().expect("wrapped DEK should contain nonce");
|
||||
let envelope_a: super::LocalSseDekEnvelope =
|
||||
serde_json::from_str(&encrypted_a).expect("first wrapped DEK should be a JSON envelope");
|
||||
let envelope_b: super::LocalSseDekEnvelope =
|
||||
serde_json::from_str(&encrypted_b).expect("second wrapped DEK should be a JSON envelope");
|
||||
|
||||
assert_ne!(nonce_a, nonce_b, "each DEK wrap should use a distinct nonce prefix");
|
||||
assert_eq!(envelope_a.version, super::LOCAL_SSE_DEK_FORMAT_VERSION);
|
||||
assert_eq!(envelope_b.version, super::LOCAL_SSE_DEK_FORMAT_VERSION);
|
||||
assert_ne!(envelope_a.nonce, envelope_b.nonce, "each DEK wrap should use a distinct nonce");
|
||||
assert_eq!(
|
||||
TestSseDekProvider::decrypt_dek(&encrypted_a, cmk).expect("first JSON envelope should decrypt"),
|
||||
dek
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4051,6 +4112,20 @@ mod tests {
|
||||
assert_eq!(decrypted, dek);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decrypt_dek_rejects_unknown_json_version() {
|
||||
let envelope = serde_json::json!({
|
||||
"version": super::LOCAL_SSE_DEK_FORMAT_VERSION + 1,
|
||||
"nonce": BASE64_STANDARD.encode([0u8; 12]),
|
||||
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32])
|
||||
.expect_err("unknown JSON envelope versions must fail closed");
|
||||
assert!(error.message.contains("Unsupported encrypted DEK format version"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sse_encryption_fails_closed_without_local_sse_master_key() {
|
||||
let _guard = lock_sse_test_state().await;
|
||||
|
||||
@@ -430,9 +430,9 @@ pub(crate) mod ecstore_data_usage {
|
||||
pub(crate) mod ecstore_disk {
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, OldCurrentSize, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout,
|
||||
validate_batch_read_version_item_count,
|
||||
FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq,
|
||||
ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
get_object_disk_read_timeout, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
|
||||
}
|
||||
@@ -629,6 +629,7 @@ pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq;
|
||||
pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp;
|
||||
pub(crate) type ReadOptions = ecstore_disk::ReadOptions;
|
||||
pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize;
|
||||
pub(crate) type PartTransactionAction = ecstore_disk::PartTransactionAction;
|
||||
pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp;
|
||||
pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType;
|
||||
pub(crate) type ReplicationStats = StorageReplicationStatsHandle;
|
||||
@@ -1097,6 +1098,15 @@ pub(crate) trait StorageDiskRpcExt {
|
||||
dst_path: &str,
|
||||
meta: bytes::Bytes,
|
||||
) -> DiskResult<()>;
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: bytes::Bytes,
|
||||
) -> DiskResult<()>;
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()>;
|
||||
async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>;
|
||||
async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult<CheckPartsResp>;
|
||||
async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult<CheckPartsResp>;
|
||||
@@ -1241,6 +1251,21 @@ where
|
||||
ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: bytes::Bytes,
|
||||
) -> DiskResult<()> {
|
||||
ecstore_disk::DiskAPI::prepare_part_transaction(self, src_volume, src_path, dst_volume, dst_path, meta).await
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()> {
|
||||
ecstore_disk::DiskAPI::settle_part_transaction(self, volume, path, action).await
|
||||
}
|
||||
|
||||
async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> {
|
||||
ecstore_disk::DiskAPI::delete(self, volume, path, options).await
|
||||
}
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ their issue closes.
|
||||
| `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — |
|
||||
| `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — |
|
||||
| `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — |
|
||||
| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence | `test_manual_transition_runbooks.sh` |
|
||||
| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence with strict/baseline/blocked result classification | `test_manual_transition_runbooks.sh` |
|
||||
| `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — |
|
||||
| `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — |
|
||||
| `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — |
|
||||
|
||||
@@ -13,7 +13,14 @@ fi
|
||||
classify_source_layer() {
|
||||
local file="$1"
|
||||
|
||||
if [[ "$file" == rustfs/src/app/* ]]; then
|
||||
if [[ "$file" == rustfs/src/server/* ]] ||
|
||||
[[ "$file" == rustfs/src/startup_*.rs ]] ||
|
||||
[[ "$file" == rustfs/src/init.rs ]] ||
|
||||
[[ "$file" == rustfs/src/main.rs ]] ||
|
||||
[[ "$file" == rustfs/src/lib.rs ]] ||
|
||||
[[ "$file" == rustfs/src/embedded.rs ]]; then
|
||||
printf 'composition'
|
||||
elif [[ "$file" == rustfs/src/app/* ]]; then
|
||||
printf 'app'
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]]; then
|
||||
printf 'interface'
|
||||
@@ -30,6 +37,14 @@ classify_target_layer() {
|
||||
local storage_path
|
||||
|
||||
case "$root" in
|
||||
init | main | lib | embedded | startup_*)
|
||||
printf 'composition'
|
||||
;;
|
||||
server)
|
||||
# Server files are composition roots when they import lower layers, but
|
||||
# their exported HTTP contracts belong to the interface boundary.
|
||||
printf 'interface'
|
||||
;;
|
||||
app)
|
||||
printf 'app'
|
||||
;;
|
||||
@@ -53,6 +68,9 @@ classify_target_layer() {
|
||||
|
||||
layer_rank() {
|
||||
case "$1" in
|
||||
composition)
|
||||
printf '4'
|
||||
;;
|
||||
interface)
|
||||
printf '3'
|
||||
;;
|
||||
@@ -68,6 +86,48 @@ layer_rank() {
|
||||
esac
|
||||
}
|
||||
|
||||
is_reverse_dependency() {
|
||||
local source_rank target_rank
|
||||
|
||||
source_rank="$(layer_rank "$1")"
|
||||
target_rank="$(layer_rank "$2")"
|
||||
(( source_rank < target_rank ))
|
||||
}
|
||||
|
||||
assert_dependency_direction() {
|
||||
local expected="$1"
|
||||
local source="$2"
|
||||
local target="$3"
|
||||
local actual='allowed'
|
||||
|
||||
if is_reverse_dependency "$source" "$target"; then
|
||||
actual='reverse'
|
||||
fi
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s -> %s (expected %s, got %s)\n' \
|
||||
"$source" "$target" "$expected" "$actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_layer_model_self_tests() {
|
||||
local server_source app_source storage_source server_target admin_target app_target
|
||||
|
||||
server_source="$(classify_source_layer rustfs/src/server/http.rs)"
|
||||
app_source="$(classify_source_layer rustfs/src/app/bucket_usecase.rs)"
|
||||
storage_source="$(classify_source_layer rustfs/src/storage/rpc/node_service.rs)"
|
||||
server_target="$(classify_target_layer server::http)"
|
||||
admin_target="$(classify_target_layer admin::router)"
|
||||
app_target="$(classify_target_layer app::bucket_usecase)"
|
||||
|
||||
assert_dependency_direction 'allowed' "$server_source" "$admin_target"
|
||||
assert_dependency_direction 'allowed' "$server_source" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$server_target"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$server_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$admin_target"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$admin_target"
|
||||
}
|
||||
|
||||
normalize_import_group_item() {
|
||||
local prefix="$1"
|
||||
local item="$2"
|
||||
@@ -182,6 +242,11 @@ normalize_import_path() {
|
||||
|
||||
emit_crate_use_statements() {
|
||||
(cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while
|
||||
# inline #[cfg(test)] imports remain subject to the source file's layer.
|
||||
if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then
|
||||
continue
|
||||
fi
|
||||
perl -0777 -ne '
|
||||
while (/\buse\s+crate::.*?;/sg) {
|
||||
my $statement = $&;
|
||||
@@ -194,6 +259,26 @@ emit_crate_use_statements() {
|
||||
done)
|
||||
}
|
||||
|
||||
write_baseline_file() {
|
||||
local entries="$1"
|
||||
|
||||
cat >"$BASELINE_FILE" <<'EOF'
|
||||
# Layer dependency baseline for the rustfs binary crate.
|
||||
#
|
||||
# The guard models production imports as:
|
||||
# composition -> interface -> app -> infra
|
||||
#
|
||||
# Canonical dependency entry:
|
||||
# dep|source_file|source_layer->target_layer|crate::imported_symbol
|
||||
#
|
||||
# Canonical conceptual cycle entry:
|
||||
# cycle|left_layer<->right_layer
|
||||
EOF
|
||||
cat "$entries" >>"$BASELINE_FILE"
|
||||
}
|
||||
|
||||
run_layer_model_self_tests
|
||||
|
||||
normalize_baseline_file() {
|
||||
local input="$1"
|
||||
local output="$2"
|
||||
@@ -265,10 +350,7 @@ while IFS= read -r line; do
|
||||
printf '%s->%s\n' "$source_layer" "$target_layer" >>"$EDGES_RAW"
|
||||
fi
|
||||
|
||||
source_rank="$(layer_rank "$source_layer")"
|
||||
target_rank="$(layer_rank "$target_layer")"
|
||||
|
||||
if (( source_rank < target_rank )); then
|
||||
if is_reverse_dependency "$source_layer" "$target_layer"; then
|
||||
printf 'dep|%s|%s->%s|crate::%s\n' "$file" "$source_layer" "$target_layer" "$import_path" >>"$VIOLATIONS_RAW"
|
||||
fi
|
||||
done < <(normalize_import_path "$text")
|
||||
@@ -292,7 +374,7 @@ done <"${TMP_DIR}/edges_sorted.txt" | sort -u >"${TMP_DIR}/cycles_sorted.txt"
|
||||
cat "${TMP_DIR}/violations_sorted.txt" "${TMP_DIR}/cycles_sorted.txt" | sort -u >"$CURRENT_BASELINE"
|
||||
|
||||
if [[ "$MODE" == "update" ]]; then
|
||||
cp "$CURRENT_BASELINE" "$BASELINE_FILE"
|
||||
write_baseline_file "$CURRENT_BASELINE"
|
||||
echo "Updated baseline: $BASELINE_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -1,35 +1,44 @@
|
||||
# Layer dependency baseline for the rustfs binary crate.
|
||||
#
|
||||
# These are intra-crate module references within rustfs/ that cross the
|
||||
# conceptual layer boundaries (app, infra/server, interface/admin).
|
||||
# Since they live inside one Cargo crate, Rust doesn't enforce separation.
|
||||
# The list is maintained for architectural awareness during code review.
|
||||
# The guard models production imports as:
|
||||
# composition -> interface -> app -> infra
|
||||
#
|
||||
# Format for dependency entries:
|
||||
# status|source_file|direction|imported_symbol|reason
|
||||
# Canonical dependency entry:
|
||||
# dep|source_file|source_layer->target_layer|crate::imported_symbol
|
||||
#
|
||||
# Format for conceptual cycles:
|
||||
# status|cycle|direction_pair|reason
|
||||
#
|
||||
# Status:
|
||||
# accepted - reviewed and intentionally allowed
|
||||
# todo - should be resolved in a future refactor
|
||||
|
||||
# Canonical conceptual cycle entry:
|
||||
# cycle|left_layer<->right_layer
|
||||
cycle|app<->infra
|
||||
cycle|app<->interface
|
||||
cycle|composition<->infra
|
||||
cycle|composition<->interface
|
||||
cycle|infra<->interface
|
||||
dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::ENV_SCANNER_ENABLED
|
||||
dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::scanner_enabled_from_env
|
||||
dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::DependencyReadiness
|
||||
dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::collect_dependency_readiness_report
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook
|
||||
dep|rustfs/src/init.rs|infra->interface|crate::admin
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::server::RemoteAddr
|
||||
dep|rustfs/src/app/object_usecase.rs|app->interface|crate::server::convert_ecstore_object_info
|
||||
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::DependencyReadiness
|
||||
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::DependencyReadinessReport
|
||||
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::ReadinessDegradedReason
|
||||
dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::snapshot_dependency_readiness_report
|
||||
dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context
|
||||
dep|rustfs/src/server/http.rs|infra->interface|crate::admin
|
||||
dep|rustfs/src/server/layer.rs|infra->interface|crate::admin::console::is_console_path
|
||||
dep|rustfs/src/storage/access.rs|infra->interface|crate::server::RemoteAddr
|
||||
dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::server::cors
|
||||
dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::storage::ecfs::ListObjectUnorderedQuery
|
||||
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::ecfs::FS
|
||||
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::ecfs::validate_object_lock_configuration_input
|
||||
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::s3_api::common::rustfs_initiator
|
||||
dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::s3_api::common::rustfs_owner
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::convert_ecstore_object_info
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_audit_module_enabled
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_notify_module_enabled
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_audit_module_enabled
|
||||
dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_notify_module_enabled
|
||||
dep|rustfs/src/storage/rpc/http_service.rs|infra->interface|crate::server::RPC_PREFIX
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_dynamic_config_runtime_state
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_runtime_config_snapshot
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::site_replication::reload_site_replication_runtime_state
|
||||
dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM
|
||||
dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::heal_enabled_from_env
|
||||
dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::scanner_enabled_from_env
|
||||
|
||||
@@ -25,9 +25,13 @@ COLD_IMAGE="${COLD_IMAGE:-${NEW_IMAGE}}"
|
||||
BASE_PORT="${BASE_PORT:-19400}"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/manual-transition-1508-docker/$(date +%Y%m%dT%H%M%S)}"
|
||||
KEEP_UP=false
|
||||
ROLLBACK_NEW2_TO_OLD=true
|
||||
ROLLBACK_PHASE="${ROLLBACK_PHASE:-after-terminal}"
|
||||
OLD_NODE_PHASE="${OLD_NODE_PHASE:-initial}"
|
||||
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
|
||||
POLL_SECONDS="${POLL_SECONDS:-180}"
|
||||
TRANSITION_WORKERS="${TRANSITION_WORKERS:-2}"
|
||||
TRANSITION_QUEUE_CAPACITY="${TRANSITION_QUEUE_CAPACITY:-64}"
|
||||
FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT:-false}"
|
||||
|
||||
HOT_ACCESS_KEY="${HOT_ACCESS_KEY:-mvadmin}"
|
||||
HOT_SECRET_KEY="${HOT_SECRET_KEY:-mvsecret}"
|
||||
@@ -58,18 +62,31 @@ Options:
|
||||
--object-count <n> Non-empty probe object count
|
||||
--tier <name> Remote tier name
|
||||
--keep-up Leave Docker services running
|
||||
--no-rollback Do not replace node2 with old image after job admission
|
||||
--rollback-phase <phase> Rollback node2 timing: after-terminal, in-flight, none
|
||||
--old-node-phase <phase> Old node1 timing: initial, before-job
|
||||
--no-rollback Alias for --rollback-phase none
|
||||
-h, --help Show help
|
||||
|
||||
Environment:
|
||||
PROJECT_NAME OLD_IMAGE NEW_IMAGE COLD_IMAGE BASE_PORT OUT_DIR KEEP_UP
|
||||
HOT_ACCESS_KEY HOT_SECRET_KEY COLD_ACCESS_KEY COLD_SECRET_KEY
|
||||
TIER_NAME TIER_BUCKET TIER_PREFIX JOB_BUCKET JOB_PREFIX OBJECT_COUNT
|
||||
WAIT_TIMEOUT_SECS POLL_SECONDS AWS_SIGV4_SCOPE
|
||||
ROLLBACK_PHASE WAIT_TIMEOUT_SECS POLL_SECONDS TRANSITION_WORKERS
|
||||
TRANSITION_QUEUE_CAPACITY OLD_NODE_PHASE FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT AWS_SIGV4_SCOPE
|
||||
|
||||
Artifacts:
|
||||
compose.yml, image inspect files, health/readiness logs, API responses,
|
||||
terminal status, old-node readback, container logs, summary.env.
|
||||
|
||||
Result classifications:
|
||||
strict_mixed_rollout_pass Real old/new images, non-empty completed transition, zero failures
|
||||
baseline_tiered_storage_pass Same old/new image completed transition; useful baseline, not #1508 closure
|
||||
blocked_manual_api_not_implemented Manual transition API returned 501 before job admission
|
||||
blocked_manual_api_unavailable Manual transition API did not return a usable job_id
|
||||
blocked_cluster_readiness_failed Docker cluster did not reach health/readiness before admission
|
||||
blocked_empty_scan_or_lifecycle Job completed without lifecycle-matching transition work
|
||||
blocked_manual_job_preempted_by_lifecycle_queue Lifecycle/immediate transition queued work before the job
|
||||
strict_mixed_rollout_fail Mixed rollout ran but did not satisfy the strict #1508 gate
|
||||
USAGE
|
||||
}
|
||||
|
||||
@@ -111,6 +128,28 @@ parse_positive_int() {
|
||||
fi
|
||||
}
|
||||
|
||||
validate_rollback_phase() {
|
||||
case "$ROLLBACK_PHASE" in
|
||||
after-terminal|in-flight|none)
|
||||
;;
|
||||
*)
|
||||
log_error "--rollback-phase must be one of: after-terminal, in-flight, none"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
validate_old_node_phase() {
|
||||
case "$OLD_NODE_PHASE" in
|
||||
initial|before-job)
|
||||
;;
|
||||
*)
|
||||
log_error "--old-node-phase must be one of: initial, before-job"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -150,8 +189,16 @@ parse_args() {
|
||||
KEEP_UP=true
|
||||
shift
|
||||
;;
|
||||
--rollback-phase)
|
||||
ROLLBACK_PHASE="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--old-node-phase)
|
||||
OLD_NODE_PHASE="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--no-rollback)
|
||||
ROLLBACK_NEW2_TO_OLD=false
|
||||
ROLLBACK_PHASE=none
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
@@ -198,12 +245,18 @@ cleanup() {
|
||||
}
|
||||
|
||||
write_compose_file() {
|
||||
local node1_image
|
||||
COMPOSE_FILE="${OUT_DIR}/compose.yml"
|
||||
node1_image="$OLD_IMAGE"
|
||||
if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then
|
||||
node1_image="$NEW_IMAGE"
|
||||
fi
|
||||
cat >"$COMPOSE_FILE" <<EOF
|
||||
services:
|
||||
cold:
|
||||
image: ${COLD_IMAGE}
|
||||
hostname: cold
|
||||
user: "0:0"
|
||||
environment:
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_ACCESS_KEY=${COLD_ACCESS_KEY}
|
||||
@@ -222,14 +275,20 @@ services:
|
||||
- rustfs-1508-net
|
||||
|
||||
node1:
|
||||
image: ${OLD_IMAGE}
|
||||
image: ${node1_image}
|
||||
hostname: node1
|
||||
user: "0:0"
|
||||
environment: &hot-env
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_ACCESS_KEY=${HOT_ACCESS_KEY}
|
||||
- RUSTFS_SECRET_KEY=${HOT_SECRET_KEY}
|
||||
- RUSTFS_VOLUMES=$(hot_volumes)
|
||||
- RUSTFS_SCANNER_ENABLED=false
|
||||
- RUSTFS_SCANNER_CYCLE=3600
|
||||
- RUSTFS_SCANNER_START_DELAY_SECS=3600
|
||||
- RUSTFS_MAX_TRANSITION_WORKERS=${TRANSITION_WORKERS}
|
||||
- RUSTFS_TRANSITION_QUEUE_CAPACITY=${TRANSITION_QUEUE_CAPACITY}
|
||||
- RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT=${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=warn
|
||||
volumes:
|
||||
@@ -245,6 +304,7 @@ services:
|
||||
node2:
|
||||
image: ${NEW_IMAGE}
|
||||
hostname: node2
|
||||
user: "0:0"
|
||||
environment: *hot-env
|
||||
volumes:
|
||||
- node2_data_0:/data/rustfs0
|
||||
@@ -259,6 +319,7 @@ services:
|
||||
node3:
|
||||
image: ${NEW_IMAGE}
|
||||
hostname: node3
|
||||
user: "0:0"
|
||||
environment: *hot-env
|
||||
volumes:
|
||||
- node3_data_0:/data/rustfs0
|
||||
@@ -273,6 +334,7 @@ services:
|
||||
node4:
|
||||
image: ${NEW_IMAGE}
|
||||
hostname: node4
|
||||
user: "0:0"
|
||||
environment: *hot-env
|
||||
volumes:
|
||||
- node4_data_0:/data/rustfs0
|
||||
@@ -433,16 +495,19 @@ seed_objects() {
|
||||
}
|
||||
|
||||
start_transition_job() {
|
||||
local query response job_id
|
||||
local query response job_id run_http_code
|
||||
query="bucket=$(url_encode "$JOB_BUCKET")"
|
||||
query="${query}&prefix=$(url_encode "${JOB_PREFIX}/")"
|
||||
query="${query}&tier=$(url_encode "$TIER_NAME")"
|
||||
query="${query}&dryRun=false&maxObjects=${OBJECT_COUNT}&mode=async"
|
||||
response="$(curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}")"
|
||||
printf '%s\n' "$response" >"${OUT_DIR}/run-response.json"
|
||||
curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}" \
|
||||
-o "${OUT_DIR}/run-response.json" \
|
||||
-w "%{http_code}\n" >"${OUT_DIR}/run-response.http_code" || true
|
||||
response="$(cat "${OUT_DIR}/run-response.json" 2>/dev/null || true)"
|
||||
run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)"
|
||||
job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
|
||||
if [[ -z "$job_id" ]]; then
|
||||
log_error "manual transition response omitted job_id"
|
||||
log_warn "manual transition response omitted job_id, http_code=${run_http_code:-unknown}"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$job_id" >"${OUT_DIR}/job-id.txt"
|
||||
@@ -451,19 +516,25 @@ start_transition_job() {
|
||||
replace_node2_with_old_image() {
|
||||
local network
|
||||
network="$(network_name)"
|
||||
log_info "Replacing node2 with old image ${OLD_IMAGE} for in-flight rollback readback"
|
||||
log_info "Replacing node2 with old image ${OLD_IMAGE} for ${ROLLBACK_PHASE} rollback readback"
|
||||
compose stop node2 >/dev/null
|
||||
compose rm -f node2 >/dev/null
|
||||
docker run -d \
|
||||
--name "${PROJECT_NAME}-node2-rollback-old" \
|
||||
--network "$network" \
|
||||
--network-alias node2 \
|
||||
--user 0:0 \
|
||||
-p "$((BASE_PORT + 2)):9000" \
|
||||
-e RUSTFS_ADDRESS=:9000 \
|
||||
-e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
|
||||
-e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
|
||||
-e RUSTFS_VOLUMES="$(hot_volumes)" \
|
||||
-e RUSTFS_SCANNER_ENABLED=false \
|
||||
-e RUSTFS_SCANNER_CYCLE=3600 \
|
||||
-e RUSTFS_SCANNER_START_DELAY_SECS=3600 \
|
||||
-e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \
|
||||
-e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \
|
||||
-e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_OBS_LOGGER_LEVEL=warn \
|
||||
-v "${PROJECT_NAME}_node2_data_0:/data/rustfs0" \
|
||||
@@ -473,6 +544,37 @@ replace_node2_with_old_image() {
|
||||
"$OLD_IMAGE" >/dev/null
|
||||
}
|
||||
|
||||
replace_node1_with_old_image() {
|
||||
local network
|
||||
network="$(network_name)"
|
||||
log_info "Replacing node1 with old image ${OLD_IMAGE} before manual transition job"
|
||||
compose stop node1 >/dev/null
|
||||
compose rm -f node1 >/dev/null
|
||||
docker run -d \
|
||||
--name "${PROJECT_NAME}-node1-before-job-old" \
|
||||
--network "$network" \
|
||||
--network-alias node1 \
|
||||
--user 0:0 \
|
||||
-p "$((BASE_PORT + 1)):9000" \
|
||||
-e RUSTFS_ADDRESS=:9000 \
|
||||
-e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
|
||||
-e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
|
||||
-e RUSTFS_VOLUMES="$(hot_volumes)" \
|
||||
-e RUSTFS_SCANNER_ENABLED=false \
|
||||
-e RUSTFS_SCANNER_CYCLE=3600 \
|
||||
-e RUSTFS_SCANNER_START_DELAY_SECS=3600 \
|
||||
-e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \
|
||||
-e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \
|
||||
-e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_OBS_LOGGER_LEVEL=warn \
|
||||
-v "${PROJECT_NAME}_node1_data_0:/data/rustfs0" \
|
||||
-v "${PROJECT_NAME}_node1_data_1:/data/rustfs1" \
|
||||
-v "${PROJECT_NAME}_node1_data_2:/data/rustfs2" \
|
||||
-v "${PROJECT_NAME}_node1_data_3:/data/rustfs3" \
|
||||
"$OLD_IMAGE" >/dev/null
|
||||
}
|
||||
|
||||
poll_terminal_status() {
|
||||
local job_id="$1"
|
||||
local status_url status_json terminal_state
|
||||
@@ -512,6 +614,85 @@ head_probe() {
|
||||
-w "%{http_code}\n" >"${OUT_DIR}/head-object.http_code" || true
|
||||
}
|
||||
|
||||
image_id() {
|
||||
local file="$1"
|
||||
jq -r '.[0].Id // ""' "$file" 2>/dev/null || true
|
||||
}
|
||||
|
||||
is_compat_readback_code() {
|
||||
case "$1" in
|
||||
200|501)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
classify_result() {
|
||||
local terminal_state="$1"
|
||||
local transition_completed="$2"
|
||||
local transition_failed="$3"
|
||||
local tier_failure="$4"
|
||||
local old_code="$5"
|
||||
local rollback_code="$6"
|
||||
local run_http_code="$7"
|
||||
local lifecycle_config_found="$8"
|
||||
local scanned="$9"
|
||||
local eligible="${10}"
|
||||
local skipped_already_transitioned="${11}"
|
||||
local skipped_already_in_flight="${12}"
|
||||
local old_image_id new_image_id images_are_mixed readback_ok
|
||||
|
||||
if [[ -f "${OUT_DIR}/readiness-failed" ]]; then
|
||||
printf 'blocked_cluster_readiness_failed\n'
|
||||
return
|
||||
fi
|
||||
|
||||
old_image_id="$(image_id "${OUT_DIR}/old-image.inspect.json")"
|
||||
new_image_id="$(image_id "${OUT_DIR}/new-image.inspect.json")"
|
||||
images_are_mixed=false
|
||||
if [[ "$OLD_IMAGE" != "$NEW_IMAGE" && -n "$old_image_id" && -n "$new_image_id" && "$old_image_id" != "$new_image_id" ]]; then
|
||||
images_are_mixed=true
|
||||
fi
|
||||
|
||||
readback_ok=false
|
||||
if is_compat_readback_code "$old_code"; then
|
||||
if [[ "$ROLLBACK_PHASE" == "none" ]] || is_compat_readback_code "$rollback_code"; then
|
||||
readback_ok=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$run_http_code" == "501" ]]; then
|
||||
printf 'blocked_manual_api_not_implemented\n'
|
||||
return
|
||||
fi
|
||||
if [[ -z "$(cat "${OUT_DIR}/job-id.txt" 2>/dev/null || true)" ]]; then
|
||||
printf 'blocked_manual_api_unavailable\n'
|
||||
return
|
||||
fi
|
||||
if [[ "$terminal_state" == "completed" && ( "$lifecycle_config_found" != "true" || "$scanned" == "0" || "$eligible" == "0" || "$transition_completed" == "0" ) ]]; then
|
||||
printf 'blocked_empty_scan_or_lifecycle\n'
|
||||
return
|
||||
fi
|
||||
if [[ "$transition_completed" == "0" && "$eligible" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then
|
||||
if [[ "$skipped_already_transitioned" != "0" || "$skipped_already_in_flight" != "0" ]]; then
|
||||
printf 'blocked_manual_job_preempted_by_lifecycle_queue\n'
|
||||
return
|
||||
fi
|
||||
fi
|
||||
if [[ "$terminal_state" == "completed" && "$transition_completed" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then
|
||||
if [[ "$images_are_mixed" == "true" && "$readback_ok" == "true" ]]; then
|
||||
printf 'strict_mixed_rollout_pass\n'
|
||||
return
|
||||
fi
|
||||
printf 'baseline_tiered_storage_pass\n'
|
||||
return
|
||||
fi
|
||||
printf 'strict_mixed_rollout_fail\n'
|
||||
}
|
||||
|
||||
collect_logs() {
|
||||
local service
|
||||
if [[ -z "$COMPOSE_FILE" || ! -f "$COMPOSE_FILE" ]]; then
|
||||
@@ -520,37 +701,63 @@ collect_logs() {
|
||||
for service in cold node1 node2 node3 node4; do
|
||||
compose logs --no-color "$service" >"${OUT_DIR}/${service}.log" 2>/dev/null || true
|
||||
done
|
||||
docker logs "${PROJECT_NAME}-node1-before-job-old" >"${OUT_DIR}/node1-before-job-old.log" 2>/dev/null || true
|
||||
docker logs "${PROJECT_NAME}-node2-rollback-old" >"${OUT_DIR}/node2-rollback-old.log" 2>/dev/null || true
|
||||
}
|
||||
|
||||
summarize() {
|
||||
local terminal_state transition_completed tier_failure transition_failed old_code rollback_code
|
||||
local terminal_state transition_completed tier_failure transition_failed old_code rollback_code run_http_code lifecycle_config_found scanned eligible skipped_already_transitioned skipped_already_in_flight queue_queued queue_active result_classification
|
||||
terminal_state="$(cat "${OUT_DIR}/terminal-state.txt" 2>/dev/null || true)"
|
||||
transition_completed="$(jq -r '.report.transition_completed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
tier_failure="$(jq -r '.report.tier_failure // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
transition_failed="$(jq -r '.report.transition_failed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
old_code="$(cat "${OUT_DIR}/old-node-status.http_code" 2>/dev/null || true)"
|
||||
rollback_code="$(cat "${OUT_DIR}/rollback-node2-status.http_code" 2>/dev/null || true)"
|
||||
run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)"
|
||||
lifecycle_config_found="$(jq -r '.report.lifecycle_config_found // false' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf 'false')"
|
||||
scanned="$(jq -r '.report.scanned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
eligible="$(jq -r '.report.eligible // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
skipped_already_transitioned="$(jq -r '.report.skipped_already_transitioned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
skipped_already_in_flight="$(jq -r '.report.skipped_already_in_flight // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
queue_queued="$(jq -r '.queue_snapshot.queued // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
queue_active="$(jq -r '.queue_snapshot.active // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
result_classification="$(classify_result "$terminal_state" "$transition_completed" "$transition_failed" "$tier_failure" "$old_code" "$rollback_code" "$run_http_code" "$lifecycle_config_found" "$scanned" "$eligible" "$skipped_already_transitioned" "$skipped_already_in_flight")"
|
||||
cat >"${OUT_DIR}/summary.env" <<EOF
|
||||
project_name=${PROJECT_NAME}
|
||||
old_image=${OLD_IMAGE}
|
||||
new_image=${NEW_IMAGE}
|
||||
cold_image=${COLD_IMAGE}
|
||||
old_image_id=$(image_id "${OUT_DIR}/old-image.inspect.json")
|
||||
new_image_id=$(image_id "${OUT_DIR}/new-image.inspect.json")
|
||||
base_port=${BASE_PORT}
|
||||
tier=${TIER_NAME}
|
||||
job_bucket=${JOB_BUCKET}
|
||||
job_prefix=${JOB_PREFIX}
|
||||
object_count=${OBJECT_COUNT}
|
||||
transition_workers=${TRANSITION_WORKERS}
|
||||
transition_queue_capacity=${TRANSITION_QUEUE_CAPACITY}
|
||||
force_immediate_transition_enqueue_timeout=${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}
|
||||
run_http_code=${run_http_code}
|
||||
terminal_state=${terminal_state}
|
||||
lifecycle_config_found=${lifecycle_config_found}
|
||||
scanned=${scanned}
|
||||
eligible=${eligible}
|
||||
skipped_already_transitioned=${skipped_already_transitioned}
|
||||
skipped_already_in_flight=${skipped_already_in_flight}
|
||||
transition_completed=${transition_completed}
|
||||
transition_failed=${transition_failed}
|
||||
tier_failure=${tier_failure}
|
||||
queue_queued=${queue_queued}
|
||||
queue_active=${queue_active}
|
||||
old_node_status_http_code=${old_code}
|
||||
rollback_node2_status_http_code=${rollback_code}
|
||||
rollback_new2_to_old=${ROLLBACK_NEW2_TO_OLD}
|
||||
rollback_phase=${ROLLBACK_PHASE}
|
||||
old_node_phase=${OLD_NODE_PHASE}
|
||||
rollback_new2_to_old=$([[ "$ROLLBACK_PHASE" == "none" ]] && printf 'false' || printf 'true')
|
||||
result_classification=${result_classification}
|
||||
EOF
|
||||
cat "${OUT_DIR}/summary.env"
|
||||
if [[ "$terminal_state" != "completed" || "$transition_completed" == "0" || "$transition_failed" != "0" || "$tier_failure" != "0" ]]; then
|
||||
if [[ "$result_classification" != "strict_mixed_rollout_pass" ]]; then
|
||||
log_error "strict #1508 mixed-version transition gate failed; see ${OUT_DIR}"
|
||||
return 1
|
||||
fi
|
||||
@@ -560,6 +767,8 @@ main() {
|
||||
parse_args "$@"
|
||||
parse_positive_int "--base-port" "$BASE_PORT"
|
||||
parse_positive_int "--object-count" "$OBJECT_COUNT"
|
||||
validate_rollback_phase
|
||||
validate_old_node_phase
|
||||
require_cmd docker
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
@@ -574,20 +783,37 @@ main() {
|
||||
docker image inspect "$COLD_IMAGE" >"${OUT_DIR}/cold-image.inspect.json"
|
||||
|
||||
compose up -d
|
||||
wait_cluster_ready
|
||||
if ! wait_cluster_ready; then
|
||||
printf 'cluster readiness failed before manual transition admission\n' >"${OUT_DIR}/readiness-failed"
|
||||
collect_logs
|
||||
summarize
|
||||
return 1
|
||||
fi
|
||||
|
||||
curl_cold PUT "$(cold_endpoint)/${TIER_BUCKET}" -o "${OUT_DIR}/create-cold-bucket.response" -w "%{http_code}\n" >"${OUT_DIR}/create-cold-bucket.http_code"
|
||||
create_bucket "$(hot_endpoint 2)" "$JOB_BUCKET" "${OUT_DIR}/create-hot-bucket"
|
||||
add_tier
|
||||
put_lifecycle
|
||||
seed_objects
|
||||
start_transition_job
|
||||
if [[ "$ROLLBACK_NEW2_TO_OLD" == "true" ]]; then
|
||||
replace_node2_with_old_image
|
||||
put_lifecycle
|
||||
if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then
|
||||
replace_node1_with_old_image
|
||||
wait_http_ok "$(hot_endpoint 1)/health" "node1-old-live" || true
|
||||
wait_http_ok "$(hot_endpoint 1)/health/ready" "node1-old-ready" || true
|
||||
fi
|
||||
if start_transition_job; then
|
||||
if [[ "$ROLLBACK_PHASE" == "in-flight" ]]; then
|
||||
replace_node2_with_old_image
|
||||
fi
|
||||
poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")" || true
|
||||
if [[ "$ROLLBACK_PHASE" == "after-terminal" ]]; then
|
||||
replace_node2_with_old_image
|
||||
wait_http_ok "$(hot_endpoint 2)/health" "rollback-node2-live" || true
|
||||
fi
|
||||
capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
|
||||
head_probe
|
||||
else
|
||||
log_warn "Skipping terminal polling because no manual transition job was admitted"
|
||||
fi
|
||||
poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")"
|
||||
capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
|
||||
head_probe
|
||||
collect_logs
|
||||
summarize
|
||||
}
|
||||
|
||||
@@ -208,7 +208,16 @@ bash "$MIXED_DOCKER_HARNESS" --help >/tmp/manual_transition_mixed_version_docker
|
||||
rg -q "mixed_version_docker_harness" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--old-image" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--new-image" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--rollback-phase" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--old-node-phase" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--no-rollback" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "strict_mixed_rollout_pass" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "baseline_tiered_storage_pass" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "blocked_manual_api_not_implemented" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "blocked_cluster_readiness_failed" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "blocked_manual_job_preempted_by_lifecycle_queue" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "OLD_NODE_PHASE" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then
|
||||
echo "failure samples script should fail when --sample has no value" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user