mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-07 11:10:47 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0f125bb4ec | |||
| bda712df56 | |||
| 00102f5737 | |||
| 55246fa932 |
@@ -71,7 +71,6 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Copyright 2026 RustFS Team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
|
||||
# A compatible native runtime with Python 3.11+, getent and stat is required by
|
||||
# the isolated fixture's staging/provenance checks. This is a test image.
|
||||
ARG RUNTIME_BASE=python:3.13-slim-trixie
|
||||
FROM ${RUNTIME_BASE}
|
||||
|
||||
ARG SOURCE_REVISION
|
||||
ARG BINARY_SHA256
|
||||
ARG BUILD_PROFILE=dev
|
||||
COPY rustfs /usr/bin/rustfs
|
||||
RUN python3 - "$SOURCE_REVISION" "$BINARY_SHA256" "$BUILD_PROFILE" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
source, digest, profile = sys.argv[1:]
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", source):
|
||||
raise SystemExit("invalid source revision")
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise SystemExit("invalid binary digest")
|
||||
if profile not in ("dev", "release"):
|
||||
raise SystemExit("invalid build profile")
|
||||
with open("/usr/bin/rustfs", "rb") as binary:
|
||||
if hashlib.file_digest(binary, "sha256").hexdigest() != digest:
|
||||
raise SystemExit("binary digest mismatch")
|
||||
for tool in ("getent", "stat"):
|
||||
if shutil.which(tool) is None:
|
||||
raise SystemExit(f"missing runtime tool: {tool}")
|
||||
os.chmod("/usr/bin/rustfs", 0o755)
|
||||
PY
|
||||
LABEL io.rustfs.repro.source_revision="${SOURCE_REVISION}" \
|
||||
io.rustfs.repro.binary_sha256="${BINARY_SHA256}" \
|
||||
io.rustfs.repro.build_profile="${BUILD_PROFILE}"
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/bin/rustfs"]
|
||||
@@ -107,32 +107,6 @@ jobs:
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
|
||||
|
||||
# Native multipart completion publishes its payload from a different
|
||||
# parent than its metadata, and abort renames under a precondition; both
|
||||
# need their own Windows handle discipline.
|
||||
- name: Test native multipart complete and abort publication
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
|
||||
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
|
||||
--nocapture
|
||||
|
||||
# Owned-undo reclaim removes a directory it pinned, copy heal persists
|
||||
# fault bytes through a handle, and metadata reads meet a directory.
|
||||
- name: Test native multipart reclaim, copy heal, and directory reads
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_multipart_owned_undo
|
||||
native_multipart_copy_heal
|
||||
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
|
||||
--nocapture
|
||||
|
||||
- name: Test directory removal fallback
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
|
||||
|
||||
- name: Test Windows path conversion
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
Generated
-5
@@ -9612,7 +9612,6 @@ dependencies = [
|
||||
"rustfs-log-analyzer",
|
||||
"rustfs-madmin",
|
||||
"rustfs-mimalloc",
|
||||
"rustfs-mimalloc-sys",
|
||||
"rustfs-notify",
|
||||
"rustfs-object-capacity",
|
||||
"rustfs-object-data-cache",
|
||||
@@ -9863,7 +9862,6 @@ dependencies = [
|
||||
"path-absolutize",
|
||||
"pin-project-lite",
|
||||
"proptest",
|
||||
"quick-xml",
|
||||
"rand 0.10.3",
|
||||
"ratelimit",
|
||||
"rcgen",
|
||||
@@ -9896,9 +9894,7 @@ dependencies = [
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-signer",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-tls-runtime",
|
||||
"rustfs-uring",
|
||||
"rustfs-utils",
|
||||
"rustix",
|
||||
@@ -10900,7 +10896,6 @@ dependencies = [
|
||||
"time",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -675,10 +675,6 @@ impl DataUsageInfo {
|
||||
/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
|
||||
/// predate those fields.
|
||||
pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
|
||||
data_usage_identity_is_newer(candidate.snapshot_identity(), baseline.snapshot_identity())
|
||||
}
|
||||
|
||||
fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline: DataUsageSnapshotIdentity) -> bool {
|
||||
match (
|
||||
candidate.scanner_epoch.zip(candidate.scanner_cycle),
|
||||
baseline.scanner_epoch.zip(baseline.scanner_cycle),
|
||||
@@ -701,18 +697,10 @@ fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline:
|
||||
/// already advance the authoritative timestamp when deleting a bucket, so a
|
||||
/// rollback delete/recreate fences the previous bucket incarnation too.
|
||||
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
|
||||
observed_data_usage_is_newer_than_identity(observed, authoritative.snapshot_identity())
|
||||
}
|
||||
|
||||
/// Return whether a nonconverged observation may supersede `authoritative`.
|
||||
///
|
||||
/// This applies the structural, exact-baseline, and generation checks of
|
||||
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
|
||||
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
|
||||
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
|
||||
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
|
||||
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
|
||||
&& data_usage_snapshot_is_newer(observed, authoritative)
|
||||
}
|
||||
|
||||
/// Metadata describing the status of a disk-level data usage snapshot.
|
||||
@@ -3152,17 +3140,12 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(observed_data_usage_is_newer(&observed, &authoritative));
|
||||
assert!(observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()));
|
||||
|
||||
authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
|
||||
assert!(
|
||||
!observed_data_usage_is_newer(&observed, &authoritative),
|
||||
"an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
|
||||
);
|
||||
assert!(
|
||||
!observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
|
||||
"a captured identity must preserve the exact namespace fence"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3189,18 +3172,6 @@ mod tests {
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
|
||||
for (observed, expected) in [
|
||||
(candidate(2, 11, Some(false), true), true),
|
||||
(candidate(2, 9, Some(false), true), false),
|
||||
(candidate(2, 11, Some(true), true), false),
|
||||
(candidate(2, 11, Some(false), false), false),
|
||||
] {
|
||||
assert_eq!(
|
||||
observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
|
||||
expected,
|
||||
"observation identity selection must preserve convergence, completeness, and generation gates"
|
||||
);
|
||||
}
|
||||
|
||||
let mut partial = candidate(2, 11, Some(false), false);
|
||||
partial.usage_snapshot_partial = true;
|
||||
@@ -3215,30 +3186,6 @@ mod tests {
|
||||
segment_invalidation_proof: None,
|
||||
}];
|
||||
assert!(observed_data_usage_is_newer(&partial, &authoritative));
|
||||
assert!(observed_data_usage_is_newer_than_identity(&partial, authoritative.snapshot_identity()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_identity_selection_keeps_legacy_timestamp_ordering() {
|
||||
let baseline = DataUsageSnapshotIdentity {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
// An epoch without a cycle is not a coordinated position.
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: None,
|
||||
};
|
||||
let mut observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(11)),
|
||||
scanner_epoch: Some(1),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(baseline),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(observed_data_usage_is_newer_than_identity(&observed, baseline));
|
||||
observed.last_update = baseline.last_update;
|
||||
assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
|
||||
observed.last_update = None;
|
||||
assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -260,7 +260,7 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
let key = "guarded.txt";
|
||||
|
||||
admin_create_user(env, &user, user_secret).await?;
|
||||
let mut policy = serde_json::json!({
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
@@ -268,8 +268,9 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/*")],
|
||||
"Condition": { "Null": { "s3:if-match": "false" } }
|
||||
}]
|
||||
});
|
||||
admin_add_and_attach_policy(env, &policy_name, &policy.to_string(), &user).await?;
|
||||
})
|
||||
.to_string();
|
||||
admin_add_and_attach_policy(env, &policy_name, &policy, &user).await?;
|
||||
|
||||
admin.create_bucket().bucket(&bucket).send().await?;
|
||||
let original = admin
|
||||
@@ -292,46 +293,6 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
.expect_err("PutObject without If-Match must be denied");
|
||||
assert_error(&denied, "AccessDenied", 403, "unconditional PutObject");
|
||||
|
||||
let denied = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.if_match(&etag)
|
||||
.body(ByteStream::from_static(b"missing-read-permission"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match without GetObject must be denied");
|
||||
assert_error(&denied, "AccessDenied", 403, "conditional PutObject without GetObject");
|
||||
|
||||
// S3 requires GetObject as well as PutObject for an ETag-based write.
|
||||
policy["Statement"]
|
||||
.as_array_mut()
|
||||
.expect("policy statements")
|
||||
.push(serde_json::json!({
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/{key}")]
|
||||
}));
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
&policy_name,
|
||||
&policy.to_string(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let denied = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"blind-overwrite"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GetObject permission must not allow an unconditional PutObject");
|
||||
assert_error(&denied, "AccessDenied", 403, "unconditional PutObject with GetObject");
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
@@ -340,8 +301,6 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
.body(ByteStream::from_static(b"v2"))
|
||||
.send()
|
||||
.await?;
|
||||
let written = client.get_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(written.body.collect().await?.into_bytes().as_ref(), b"v2");
|
||||
|
||||
let _ = admin.delete_object().bucket(&bucket).key(key).send().await;
|
||||
let _ = admin.delete_bucket().bucket(&bucket).send().await;
|
||||
|
||||
@@ -412,38 +412,6 @@ mod integration_tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn upload_after_quota_reconciliation(
|
||||
env: &QuotaTestEnv,
|
||||
key: &str,
|
||||
size_bytes: usize,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let upload = async {
|
||||
loop {
|
||||
let result = env
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&env.bucket_name)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes]))
|
||||
.send()
|
||||
.await;
|
||||
match result {
|
||||
Ok(_) => break,
|
||||
Err(error) => assert_quota_rejection(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
error.as_service_error(),
|
||||
&error,
|
||||
),
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
// The durable V1 ledger can retain the old charge for its 30-second reconciliation cooldown.
|
||||
timeout(Duration::from_secs(45), upload)
|
||||
.await
|
||||
.map_err(|_| format!("PUT {key} did not succeed after quota reconciliation within 45 seconds").into())
|
||||
}
|
||||
|
||||
async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) {
|
||||
let error = env
|
||||
.client
|
||||
@@ -605,7 +573,8 @@ mod integration_tests {
|
||||
|
||||
assert!(!env.object_exists("file1.txt").await?);
|
||||
|
||||
upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
|
||||
// Now we should be able to upload again (quota freed up)
|
||||
env.upload_object("file3.txt", 256 * 1024).await?;
|
||||
assert!(env.object_exists("file3.txt").await?);
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
@@ -1012,7 +981,8 @@ mod integration_tests {
|
||||
);
|
||||
assert_eq!(delete_result.deleted().len(), 2);
|
||||
|
||||
upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
|
||||
// Now should be able to upload again (quota freed up)
|
||||
env.upload_object("file3.txt", 256 * 1024).await?;
|
||||
assert!(env.object_exists("file3.txt").await?);
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
|
||||
@@ -31,7 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips", "rustfs-tls-runtime/fips"]
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
@@ -203,9 +203,6 @@ libc.workspace = true
|
||||
# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
|
||||
rustix = { workspace = true, features = ["process", "fs", "thread"] }
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-signer.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
quick-xml.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
urlencoding = { workspace = true }
|
||||
|
||||
@@ -191,11 +191,8 @@ pub mod bucket {
|
||||
OBJECT_LOCK_CONFIG, UnreadableBucketConfig, is_unreadable_config_error, load_bucket_metadata,
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{
|
||||
BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
|
||||
SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
|
||||
};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -213,20 +210,19 @@ pub mod bucket {
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
|
||||
ObjectLockConfigState, ScannerBucketIncarnationReadGuard, acquire_bucket_metadata_transaction_lock,
|
||||
ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
|
||||
delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
|
||||
delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
remove_bucket_metadata_if_absent, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
|
||||
update_config_with, update_if_incarnation, update_if_incarnation_at, update_quota_if_incarnation,
|
||||
update_quota_if_incarnation_at, update_s3_tagging_config, update_under_transaction_lock,
|
||||
update_under_transaction_lock_at,
|
||||
update_quota_if_incarnation_at, update_under_transaction_lock, update_under_transaction_lock_at,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
|
||||
@@ -454,7 +450,6 @@ pub mod data_usage {
|
||||
pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,32 +256,8 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Validate tag entries supplied to the S3 API.
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
validate_tag_entries(tag_set, validate_s3_tag_entry)
|
||||
}
|
||||
|
||||
fn validate_tag_entries(
|
||||
tag_set: &[s3s::dto::Tag],
|
||||
mut validate_entry: impl FnMut(&str, &str) -> std::result::Result<(), &'static str>,
|
||||
) -> std::result::Result<(), &'static str> {
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
validate_entry(key, value)?;
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_s3_tag_entry(key: &str, value: &str) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
@@ -289,95 +265,29 @@ fn validate_s3_tag_entry(key: &str, value: &str) -> std::result::Result<(), &'st
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
/// Maximum number of metadata headers on a Swift resource.
|
||||
pub const MAX_SWIFT_METADATA_COUNT: usize = 90;
|
||||
/// Maximum byte length of a Swift metadata value.
|
||||
pub const MAX_SWIFT_METADATA_VALUE_SIZE: usize = 256;
|
||||
|
||||
/// Prefix for Swift account metadata persisted in tagging.xml.
|
||||
pub const SWIFT_ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
|
||||
/// Prefix for Swift container metadata persisted in tagging.xml.
|
||||
pub const SWIFT_CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
|
||||
|
||||
/// Marks a bucket whose shared tag config contains Swift-owned metadata.
|
||||
/// The key cannot be supplied through S3's tag alphabet.
|
||||
pub const SWIFT_TAGGING_MARKER: &str = "!rustfs-swift-metadata";
|
||||
|
||||
/// Whether a persisted tag belongs to Swift's internal metadata namespace.
|
||||
pub fn is_swift_bucket_metadata_tag_key(key: &str) -> bool {
|
||||
matches!(
|
||||
key,
|
||||
"swift-acl-read" | "swift-acl-write" | "swift-versions-location" | SWIFT_TAGGING_MARKER
|
||||
) || key.starts_with(SWIFT_ACCOUNT_META_TAG_PREFIX)
|
||||
|| key.starts_with(SWIFT_CONTAINER_META_TAG_PREFIX)
|
||||
}
|
||||
|
||||
fn validate_persisted_bucket_tags(tags: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
const SWIFT_ACL_MAX_BYTES: usize = 8192;
|
||||
const SWIFT_META_NAME_MAX_BYTES: usize = 128;
|
||||
const MAX_PERSISTED_TAGS: usize = MAX_BUCKET_TAGS + 2 * MAX_SWIFT_METADATA_COUNT + 4;
|
||||
if tags.len() > MAX_PERSISTED_TAGS {
|
||||
return Err("Too many persisted bucket tags");
|
||||
}
|
||||
|
||||
// Swift stores ACLs, versioning and metadata in the same XML as S3 tags.
|
||||
// Only those known keys may carry values outside the S3 tag alphabet.
|
||||
let mut account_metadata = 0;
|
||||
let mut container_metadata = 0;
|
||||
let mut s3_tags = 0;
|
||||
validate_tag_entries(tags, |key, value| {
|
||||
let swift_metadata = key
|
||||
.strip_prefix(SWIFT_ACCOUNT_META_TAG_PREFIX)
|
||||
.map(|name| (name, &mut account_metadata))
|
||||
.or_else(|| {
|
||||
key.strip_prefix(SWIFT_CONTAINER_META_TAG_PREFIX)
|
||||
.map(|name| (name, &mut container_metadata))
|
||||
});
|
||||
if let Some((name, count)) = swift_metadata
|
||||
&& name.len() <= SWIFT_META_NAME_MAX_BYTES
|
||||
&& http::HeaderName::from_bytes(name.as_bytes()).is_ok()
|
||||
&& value.len() <= MAX_SWIFT_METADATA_VALUE_SIZE
|
||||
&& http::HeaderValue::from_str(value).is_ok()
|
||||
{
|
||||
*count += 1;
|
||||
if *count > MAX_SWIFT_METADATA_COUNT {
|
||||
return Err("Too many Swift metadata tags");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if key == SWIFT_TAGGING_MARKER && value == "1" {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let valid_swift_header = match key {
|
||||
"swift-acl-read" | "swift-acl-write" => value.len() <= SWIFT_ACL_MAX_BYTES,
|
||||
"swift-versions-location" => value.len() <= 256 && !value.is_empty() && !value.contains('/'),
|
||||
_ => false,
|
||||
};
|
||||
if valid_swift_header && http::HeaderValue::from_str(value).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
s3_tags += 1;
|
||||
if s3_tags > MAX_BUCKET_TAGS {
|
||||
return Err("Cannot have more than 50 tags per bucket");
|
||||
}
|
||||
validate_s3_tag_entry(key, value)
|
||||
})
|
||||
}
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -1116,7 +1026,14 @@ impl BucketMetadata {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
validate_persisted_bucket_tags(&tagging.tag_set)
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
@@ -1910,40 +1827,6 @@ mod test {
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_bucket_tags_keep_separate_s3_and_swift_limits() {
|
||||
use s3s::dto::Tag;
|
||||
|
||||
let tag = |key: String, value: &str| Tag {
|
||||
key: Some(key),
|
||||
value: Some(value.to_owned()),
|
||||
};
|
||||
let acl = tag("swift-acl-read".to_owned(), ".r:*,AUTH_other");
|
||||
assert_eq!(
|
||||
validate_tag_set_entries(std::slice::from_ref(&acl)),
|
||||
Err("The TagValue you have provided is invalid")
|
||||
);
|
||||
|
||||
let mut tags = vec![acl.clone()];
|
||||
tags.extend((0..MAX_BUCKET_TAGS).map(|i| tag(format!("team-{i}"), "blue")));
|
||||
tags.extend((0..MAX_SWIFT_METADATA_COUNT).map(|i| tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}item!{i}"), "a,b")));
|
||||
assert_eq!(validate_persisted_bucket_tags(&tags), Ok(()));
|
||||
|
||||
tags.push(tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}overflow"), "a,b"));
|
||||
assert_eq!(validate_persisted_bucket_tags(&tags), Err("Too many Swift metadata tags"));
|
||||
tags.pop();
|
||||
|
||||
tags.push(acl);
|
||||
assert_eq!(
|
||||
validate_persisted_bucket_tags(&tags),
|
||||
Err("Cannot provide multiple Tags with the same key")
|
||||
);
|
||||
tags.pop();
|
||||
|
||||
tags[0].value = Some(".r:*,".repeat(1700));
|
||||
assert_eq!(validate_persisted_bucket_tags(&tags), Err("The TagValue you have provided is invalid"));
|
||||
}
|
||||
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
|
||||
@@ -22,7 +22,6 @@ use super::target::BucketTargets;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::diagnostics::object_lock::ObjectLockAttempt;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -600,9 +599,7 @@ pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
}
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let guard =
|
||||
acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts, "bucket_incarnation_options_read")
|
||||
.await?;
|
||||
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
|
||||
}
|
||||
|
||||
@@ -869,30 +866,6 @@ impl BucketMetadataMutationGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only scanner authority for one bucket incarnation.
|
||||
///
|
||||
/// Lock order is bucket lifecycle (read), then metadata transaction (read).
|
||||
/// This guard cannot authorize a configuration mutation or scanner ACK write.
|
||||
pub struct ScannerBucketIncarnationReadGuard {
|
||||
bucket: String,
|
||||
incarnation_id: Uuid,
|
||||
lifecycle_guard: rustfs_lock::NamespaceLockGuard,
|
||||
transaction_guard: rustfs_lock::NamespaceLockGuard,
|
||||
}
|
||||
|
||||
impl ScannerBucketIncarnationReadGuard {
|
||||
/// Returns the verified identity while both read fences remain valid.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if either lock has lost its lease.
|
||||
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
|
||||
if self.lifecycle_guard.is_lock_lost() || self.transaction_guard.is_lock_lost() {
|
||||
return Err(Error::other("scanner bucket incarnation read lock was lost"));
|
||||
}
|
||||
Ok((&self.bucket, self.incarnation_id))
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_config_write_guard(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str) -> Result<BucketMetadataMutationGuard> {
|
||||
acquire_config_write_guard_for_incarnation(sys, bucket, None).await
|
||||
}
|
||||
@@ -919,68 +892,6 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
|
||||
}
|
||||
|
||||
/// Validate scanner read authority without serializing compatible metadata readers.
|
||||
///
|
||||
/// The returned guard retains both read locks. It never migrates metadata or
|
||||
/// creates an incarnation; mutation callers must use the exclusive scanner fence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for invalid bucket/owner identities, unavailable locks or
|
||||
/// storage, absent incarnation metadata, or a deleted/recreated bucket.
|
||||
pub async fn acquire_scanner_bucket_incarnation_read_fence(
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
expected_owner_id: Uuid,
|
||||
) -> Result<ScannerBucketIncarnationReadGuard> {
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(
|
||||
get_bucket_metadata_sys()?,
|
||||
bucket,
|
||||
expected_incarnation_id,
|
||||
expected_owner_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_scanner_bucket_incarnation_read_fence_with_sys(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
expected_owner_id: Uuid,
|
||||
) -> Result<ScannerBucketIncarnationReadGuard> {
|
||||
super::utils::check_valid_bucket_name(bucket)?;
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
if expected_owner_id.is_nil() || metadata_sys.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
return Err(Error::other("scanner bucket incarnation owner does not match"));
|
||||
}
|
||||
let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let transaction_guard = await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"scanner incarnation read transaction lock acquisition",
|
||||
acquire_metadata_transaction_guard(
|
||||
metadata_sys.object_store(),
|
||||
bucket,
|
||||
"scanner_incarnation_probe",
|
||||
rustfs_lock::LockType::Shared,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let incarnation_id = validate_bucket_metadata_guard_incarnation(
|
||||
&metadata_sys,
|
||||
bucket,
|
||||
Some(expected_incarnation_id),
|
||||
&lifecycle_guard,
|
||||
&transaction_guard,
|
||||
)
|
||||
.await?;
|
||||
Ok(ScannerBucketIncarnationReadGuard {
|
||||
bucket: bucket.to_owned(),
|
||||
incarnation_id,
|
||||
lifecycle_guard,
|
||||
transaction_guard,
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire_config_write_guard_with_migration(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
@@ -1006,47 +917,15 @@ async fn acquire_config_write_guard_with_migration(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config transaction lock acquisition",
|
||||
acquire_transaction_lock_with_sys_for_purpose(
|
||||
&sys,
|
||||
bucket,
|
||||
if migrate {
|
||||
"bucket_config_update"
|
||||
} else {
|
||||
"scanner_incarnation_validation"
|
||||
},
|
||||
),
|
||||
acquire_transaction_lock_with_sys(&sys, bucket),
|
||||
)
|
||||
.await?;
|
||||
let current_incarnation_id = validate_bucket_metadata_guard_incarnation(
|
||||
&metadata_sys,
|
||||
bucket,
|
||||
expected_incarnation_id,
|
||||
&lifecycle_guard,
|
||||
&transaction_guard,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(BucketMetadataMutationGuard {
|
||||
bucket: bucket.to_string(),
|
||||
incarnation_id: current_incarnation_id,
|
||||
lifecycle_guard,
|
||||
transaction_guard,
|
||||
})
|
||||
}
|
||||
|
||||
async fn validate_bucket_metadata_guard_incarnation(
|
||||
metadata_sys: &BucketMetadataSys,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
transaction_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<Uuid> {
|
||||
await_bucket_namespace_operation(
|
||||
Some(lifecycle_guard),
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config existence validation",
|
||||
await_bucket_namespace_operation(
|
||||
Some(transaction_guard),
|
||||
Some(&transaction_guard),
|
||||
bucket,
|
||||
"bucket config existence transaction validation",
|
||||
async {
|
||||
@@ -1064,11 +943,11 @@ async fn validate_bucket_metadata_guard_incarnation(
|
||||
)
|
||||
.await?;
|
||||
let current_incarnation_id = await_bucket_namespace_operation(
|
||||
Some(lifecycle_guard),
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config incarnation validation",
|
||||
await_bucket_namespace_operation(
|
||||
Some(transaction_guard),
|
||||
Some(&transaction_guard),
|
||||
bucket,
|
||||
"bucket config incarnation transaction validation",
|
||||
load_bucket_incarnation(metadata_sys.object_store(), bucket),
|
||||
@@ -1081,7 +960,12 @@ async fn validate_bucket_metadata_guard_incarnation(
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
|
||||
Ok(current_incarnation_id)
|
||||
Ok(BucketMetadataMutationGuard {
|
||||
bucket: bucket.to_string(),
|
||||
incarnation_id: current_incarnation_id,
|
||||
lifecycle_guard,
|
||||
transaction_guard,
|
||||
})
|
||||
}
|
||||
|
||||
/// Rewrite one config file while the caller already holds this bucket's
|
||||
@@ -1290,23 +1174,11 @@ async fn delete_under_config_write_guard(
|
||||
/// targets one, so a writer on another node cannot land a whole-file save in
|
||||
/// the middle of this read-modify-write.
|
||||
pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
update_config_with_expected(bucket, config_file, None, mutate).await
|
||||
}
|
||||
|
||||
async fn update_config_with_expected<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
mutate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let guard = Box::pin(acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id)).await?;
|
||||
let guard = Box::pin(acquire_config_write_guard(sys.clone(), bucket)).await?;
|
||||
guard.ensure_valid(bucket)?;
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
let updated = await_bucket_namespace_operation(
|
||||
@@ -1325,70 +1197,6 @@ where
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
fn swift_manages_bucket_tags(tagging: &Tagging) -> bool {
|
||||
tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|tag| tag.key.as_deref() == Some(super::metadata::SWIFT_TAGGING_MARKER) && tag.value.as_deref() == Some("1"))
|
||||
}
|
||||
|
||||
/// Read the S3 tag set while keeping Swift's internal tags private.
|
||||
pub async fn get_s3_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
|
||||
let (mut tagging, updated) = get_tagging_config(bucket).await?;
|
||||
if swift_manages_bucket_tags(&tagging) {
|
||||
tagging.tag_set.retain(|tag| {
|
||||
!tag.key
|
||||
.as_deref()
|
||||
.is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
|
||||
});
|
||||
}
|
||||
Ok((tagging, updated))
|
||||
}
|
||||
|
||||
/// Replace or clear S3 tags atomically while retaining Swift metadata.
|
||||
///
|
||||
/// The expected incarnation, when present, is checked under the bucket
|
||||
/// lifecycle and metadata guards. A failed write leaves the previous config
|
||||
/// intact. Repeated writes replace the same S3 set.
|
||||
pub async fn update_s3_tagging_config(
|
||||
bucket: &str,
|
||||
tagging: Option<Tagging>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let tags = tagging.map(|tagging| tagging.tag_set).unwrap_or_default();
|
||||
let invalid = |message: &str| Error::InvalidArgument(bucket.to_owned(), String::new(), message.to_owned());
|
||||
if tags.len() > super::metadata::MAX_BUCKET_TAGS {
|
||||
return Err(invalid("Cannot have more than 50 tags per bucket"));
|
||||
}
|
||||
super::metadata::validate_tag_set_entries(&tags).map_err(invalid)?;
|
||||
update_config_with_expected(bucket, BUCKET_TAGGING_CONFIG, expected_incarnation_id, |metadata| {
|
||||
let mut merged = metadata.tagging_config.clone().unwrap_or_default();
|
||||
if swift_manages_bucket_tags(&merged) {
|
||||
if tags.iter().any(|tag| {
|
||||
tag.key
|
||||
.as_deref()
|
||||
.is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
|
||||
}) {
|
||||
return Err(invalid("Tag key is reserved for Swift metadata on this bucket"));
|
||||
}
|
||||
merged.tag_set.retain(|tag| {
|
||||
tag.key
|
||||
.as_deref()
|
||||
.is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
|
||||
});
|
||||
} else {
|
||||
merged.tag_set.clear();
|
||||
}
|
||||
merged.tag_set.extend(tags);
|
||||
if merged.tag_set.is_empty() {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
super::utils::serialize(&merged).map_err(Error::other)
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Acquire a bucket's metadata transaction lock, held across a whole
|
||||
/// read-modify-write of its metadata file.
|
||||
///
|
||||
@@ -1427,18 +1235,13 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, "bucket_metadata_read").await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
op: &'static str,
|
||||
) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let api = sys.read().await.object_store();
|
||||
acquire_metadata_transaction_guard(api, bucket, op, rustfs_lock::LockType::Shared).await
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
}
|
||||
|
||||
/// Readers already holding an Object Lock snapshot must share its guard:
|
||||
@@ -1447,7 +1250,6 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
op: &'static str,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
|
||||
let store = object_store_in(ctx).await?;
|
||||
@@ -1457,22 +1259,12 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
|
||||
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
|
||||
});
|
||||
}
|
||||
Ok(Arc::new(
|
||||
acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, op).await?,
|
||||
))
|
||||
Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
|
||||
}
|
||||
|
||||
async fn acquire_transaction_lock_with_sys(
|
||||
sys: &Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
acquire_transaction_lock_with_sys_for_purpose(sys, bucket, "bucket_metadata_transaction_write").await
|
||||
}
|
||||
|
||||
async fn acquire_transaction_lock_with_sys_for_purpose(
|
||||
sys: &Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
op: &'static str,
|
||||
) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
// Resolve the store under a short-lived read guard: this runs before the
|
||||
// write guard in `acquire_config_write_guards`, and must not still hold a
|
||||
@@ -1481,22 +1273,12 @@ async fn acquire_transaction_lock_with_sys_for_purpose(
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let timeout = crate::set_disk::get_lock_acquire_timeout();
|
||||
let mut attempt = ObjectLockAttempt::start(
|
||||
op,
|
||||
RUSTFS_META_BUCKET,
|
||||
&bucket_metadata_transaction_lock_key(bucket),
|
||||
None,
|
||||
&lock,
|
||||
"write",
|
||||
timeout,
|
||||
);
|
||||
let acquire = lock.get_write_lock(timeout);
|
||||
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
tokio::pin!(acquire);
|
||||
let mut notified = false;
|
||||
let result = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if !notified {
|
||||
notify_config_write_lock_attempt(bucket);
|
||||
@@ -1506,38 +1288,11 @@ async fn acquire_transaction_lock_with_sys_for_purpose(
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await;
|
||||
attempt.observe_acquisition(&result);
|
||||
Ok(result?)
|
||||
.await?;
|
||||
Ok(guard)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
{
|
||||
let result = acquire.await;
|
||||
attempt.observe_acquisition(&result);
|
||||
Ok(result?)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_transaction_guard(
|
||||
api: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
op: &'static str,
|
||||
lock_type: rustfs_lock::LockType,
|
||||
) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let object = bucket_metadata_transaction_lock_key(bucket);
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let timeout = crate::set_disk::get_lock_acquire_timeout();
|
||||
let mode = match lock_type {
|
||||
rustfs_lock::LockType::Shared => "read",
|
||||
rustfs_lock::LockType::Exclusive => "write",
|
||||
};
|
||||
let mut attempt = ObjectLockAttempt::start(op, RUSTFS_META_BUCKET, &object, None, &lock, mode, timeout);
|
||||
let result = match lock_type {
|
||||
rustfs_lock::LockType::Shared => lock.get_read_lock(timeout).await,
|
||||
rustfs_lock::LockType::Exclusive => lock.get_write_lock(timeout).await,
|
||||
};
|
||||
attempt.observe_acquisition(&result);
|
||||
Ok(result?)
|
||||
Ok(acquire.await?)
|
||||
}
|
||||
|
||||
/// The lock resource name is deliberately still the `bucket-targets` one it
|
||||
@@ -3039,13 +2794,13 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn find_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
let _transaction_guard = acquire_metadata_transaction_guard(
|
||||
self.object_store(),
|
||||
bucket,
|
||||
"bucket_incarnation_disk_read",
|
||||
rustfs_lock::LockType::Shared,
|
||||
)
|
||||
.await?;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.find_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
|
||||
.await
|
||||
}
|
||||
@@ -3121,13 +2876,13 @@ impl BucketMetadataSys {
|
||||
/// revalidated at write quorum, so a stale snapshot can neither revert an
|
||||
/// acknowledged configuration update nor outlive a delete/recreate.
|
||||
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
let transaction_guard = acquire_metadata_transaction_guard(
|
||||
self.object_store(),
|
||||
bucket,
|
||||
"bucket_creation_commit_migration",
|
||||
rustfs_lock::LockType::Exclusive,
|
||||
)
|
||||
.await?;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let transaction_guard = transaction_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let namespace_guard = namespace_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
@@ -3177,13 +2932,13 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
let _transaction_guard = acquire_metadata_transaction_guard(
|
||||
self.object_store(),
|
||||
bucket,
|
||||
"bucket_metadata_legacy_migration",
|
||||
rustfs_lock::LockType::Exclusive,
|
||||
)
|
||||
.await?;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
|
||||
let authority = self
|
||||
.load_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||
@@ -4222,12 +3977,6 @@ mod tests {
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
|
||||
.await
|
||||
.is_err(),
|
||||
"read-only scanner validation must not migrate an absent incarnation"
|
||||
);
|
||||
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
|
||||
assert!(
|
||||
sys.read()
|
||||
@@ -4284,128 +4033,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn scanner_incarnation_read_fence_shares_reads_and_blocks_mutations() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
|
||||
let bucket = "scanner-read-fence";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
|
||||
.await
|
||||
.expect("read-only scanner fence");
|
||||
assert_eq!(guard.checked_bucket_incarnation().expect("valid read fences"), (bucket, incarnation));
|
||||
|
||||
let transaction_lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await
|
||||
.expect("metadata transaction namespace");
|
||||
assert!(
|
||||
transaction_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
|
||||
"scanner read authority must exclude metadata writers"
|
||||
);
|
||||
let lifecycle_lock = store
|
||||
.new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("bucket lifecycle namespace");
|
||||
assert!(
|
||||
lifecycle_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
|
||||
"scanner read authority must exclude delete and recreation"
|
||||
);
|
||||
let another = timeout(
|
||||
Duration::from_secs(1),
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id),
|
||||
)
|
||||
.await
|
||||
.expect("scanner readers must not serialize")
|
||||
.expect("second read-only scanner fence");
|
||||
let compatible = timeout(
|
||||
Duration::from_secs(1),
|
||||
acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket),
|
||||
)
|
||||
.await
|
||||
.expect("metadata readers must remain compatible")
|
||||
.expect("metadata transaction reader");
|
||||
drop(compatible);
|
||||
drop(another);
|
||||
|
||||
let mut mutation = Box::pin(acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(incarnation), false));
|
||||
assert!(matches!(futures::poll!(&mut mutation), std::task::Poll::Pending));
|
||||
let mut deletion = Box::pin(store.acquire_bucket_lifecycle_write_lock(bucket));
|
||||
assert!(matches!(futures::poll!(&mut deletion), std::task::Poll::Pending));
|
||||
drop(deletion);
|
||||
drop(guard);
|
||||
let mutation = timeout(Duration::from_secs(1), mutation)
|
||||
.await
|
||||
.expect("metadata mutation must progress after scanner read release")
|
||||
.expect("exclusive mutation fence");
|
||||
drop(mutation);
|
||||
let lifecycle = timeout(Duration::from_secs(1), store.acquire_bucket_lifecycle_write_lock(bucket))
|
||||
.await
|
||||
.expect("bucket delete fence must progress after readers release")
|
||||
.expect("exclusive lifecycle fence");
|
||||
drop(lifecycle);
|
||||
|
||||
for (owner, expected) in [
|
||||
(Uuid::new_v4(), incarnation),
|
||||
(Uuid::nil(), incarnation),
|
||||
(store.id, Uuid::nil()),
|
||||
] {
|
||||
assert!(
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, expected, owner)
|
||||
.await
|
||||
.is_err(),
|
||||
"invalid owner or nil incarnation must not receive authority"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale incarnation must not receive authority"
|
||||
);
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete bucket");
|
||||
assert!(
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
|
||||
.await
|
||||
.is_err(),
|
||||
"deleted bucket must not receive authority"
|
||||
);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
let replacement = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("replacement incarnation");
|
||||
assert_ne!(replacement, incarnation);
|
||||
assert!(
|
||||
acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
|
||||
.await
|
||||
.is_err(),
|
||||
"recreation must reject the previous scanner identity"
|
||||
);
|
||||
let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys, bucket, replacement, store.id)
|
||||
.await
|
||||
.expect("replacement read-only scanner fence");
|
||||
assert_eq!(
|
||||
guard.checked_bucket_incarnation().expect("replacement remains fenced"),
|
||||
(bucket, replacement)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -886,13 +886,7 @@ pub(crate) async fn begin(
|
||||
}
|
||||
|
||||
let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(
|
||||
ctx,
|
||||
bucket,
|
||||
opts,
|
||||
"quota_context_metadata_read",
|
||||
)
|
||||
.await?;
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
|
||||
metadata_lock_started,
|
||||
|
||||
@@ -366,84 +366,10 @@ fn resolve_replay_cache_capacity() -> usize {
|
||||
decision.capacity
|
||||
}
|
||||
|
||||
const RPC_NONCE_EXPIRATIONS_PER_PAGE: usize = 1_024;
|
||||
|
||||
struct RpcNonceExpiration {
|
||||
nonce: Uuid,
|
||||
expiry_offset_ns: u32,
|
||||
valid_until_offset_secs: i32,
|
||||
}
|
||||
|
||||
struct RpcNonceExpirationPage {
|
||||
expires_at: Instant,
|
||||
valid_until: i64,
|
||||
entries: VecDeque<RpcNonceExpiration>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceExpirations {
|
||||
pages: VecDeque<RpcNonceExpirationPage>,
|
||||
}
|
||||
|
||||
impl RpcNonceExpirations {
|
||||
fn push_back(&mut self, expires_at: Instant, valid_until: i64, nonce: Uuid) {
|
||||
// Use exact offsets and start another page if either offset is too large.
|
||||
// Fully drained pages can be freed while newer nonces are still valid.
|
||||
if let Some(page) = self.pages.back_mut()
|
||||
&& page.entries.len() < RPC_NONCE_EXPIRATIONS_PER_PAGE
|
||||
&& let Some(offset) = expires_at.checked_duration_since(page.expires_at)
|
||||
&& let Ok(expiry_offset_ns) = u32::try_from(offset.as_nanos())
|
||||
&& let Some(offset) = valid_until.checked_sub(page.valid_until)
|
||||
&& let Ok(valid_until_offset_secs) = i32::try_from(offset)
|
||||
{
|
||||
page.entries.push_back(RpcNonceExpiration {
|
||||
nonce,
|
||||
expiry_offset_ns,
|
||||
valid_until_offset_secs,
|
||||
});
|
||||
} else {
|
||||
self.pages.push_back(RpcNonceExpirationPage {
|
||||
expires_at,
|
||||
valid_until,
|
||||
entries: VecDeque::from([RpcNonceExpiration {
|
||||
nonce,
|
||||
expiry_offset_ns: 0,
|
||||
valid_until_offset_secs: 0,
|
||||
}]),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn pop_expired(&mut self, now: Instant, wall_time: i64) -> Option<Uuid> {
|
||||
let page = self.pages.front_mut()?;
|
||||
let entry = page.entries.front()?;
|
||||
let elapsed = now.checked_duration_since(page.expires_at)?;
|
||||
let valid_until = page.valid_until + i64::from(entry.valid_until_offset_secs);
|
||||
if elapsed <= Duration::from_nanos(u64::from(entry.expiry_offset_ns)) || wall_time <= valid_until {
|
||||
return None;
|
||||
}
|
||||
|
||||
let nonce = entry.nonce;
|
||||
page.entries.pop_front();
|
||||
if page.entries.is_empty() {
|
||||
self.pages.pop_front();
|
||||
if self.pages.is_empty() {
|
||||
self.pages.shrink_to_fit();
|
||||
}
|
||||
}
|
||||
Some(nonce)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn capacity(&self) -> usize {
|
||||
self.pages.iter().map(|page| page.entries.capacity()).sum()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
nonces: HashSet<Uuid>,
|
||||
expirations: RpcNonceExpirations,
|
||||
expirations: VecDeque<(Instant, i64, Uuid)>,
|
||||
max_wall_time: i64,
|
||||
}
|
||||
|
||||
@@ -499,13 +425,16 @@ fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
|
||||
impl RpcNonceCache {
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
|
||||
let mut removed = 0;
|
||||
while let Some(nonce) = self.expirations.pop_expired(now, wall_time) {
|
||||
while matches!(
|
||||
self.expirations.front(),
|
||||
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
|
||||
) {
|
||||
let Some((_, _, nonce)) = self.expirations.pop_front() else {
|
||||
break;
|
||||
};
|
||||
self.nonces.remove(&nonce);
|
||||
removed += 1;
|
||||
}
|
||||
if removed > 0 && self.nonces.is_empty() {
|
||||
self.nonces.shrink_to_fit();
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
@@ -539,7 +468,7 @@ impl RpcNonceCache {
|
||||
}
|
||||
self.nonces.insert(record.nonce);
|
||||
self.expirations
|
||||
.push_back(record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce);
|
||||
.push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
|
||||
(
|
||||
Ok(()),
|
||||
Some(RpcNonceCacheMetrics {
|
||||
@@ -3249,51 +3178,6 @@ mod tests {
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_releases_expiration_storage_after_a_burst() {
|
||||
let start = Instant::now();
|
||||
let expiry = start + REPLAY_CACHE_RETENTION;
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
for nonce in 1..=4_096 {
|
||||
let record = test_nonce_record(Uuid::from_u128(nonce), 100, start, 100, expiry, 4_096);
|
||||
cache.check_and_record(record).0.expect("first use of nonce");
|
||||
}
|
||||
assert!(cache.expirations.capacity() >= 4_096);
|
||||
assert_eq!(cache.remove_expired(expiry, 701), 0);
|
||||
assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 400), 0);
|
||||
assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 701), 4_096);
|
||||
assert!(cache.nonces.is_empty());
|
||||
assert_eq!(cache.nonces.capacity(), 0);
|
||||
assert_eq!(cache.expirations.capacity(), 0, "expired entries must release their expiration storage");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_expirations_preserve_exact_deadlines_across_pages() {
|
||||
let expiry = Instant::now();
|
||||
let first = Uuid::from_u128(1);
|
||||
let second = Uuid::from_u128(2);
|
||||
let third = Uuid::from_u128(3);
|
||||
let fourth = Uuid::from_u128(4);
|
||||
let mut queue = super::RpcNonceExpirations::default();
|
||||
let next = expiry + Duration::from_nanos(1);
|
||||
let later = expiry + Duration::from_secs(5);
|
||||
|
||||
queue.push_back(expiry, 501, first);
|
||||
queue.push_back(next, 500, second);
|
||||
queue.push_back(later, 505, third);
|
||||
queue.push_back(later, i64::MAX, fourth);
|
||||
assert_eq!(queue.pages.len(), 3);
|
||||
assert_eq!(queue.pop_expired(next, 501), None);
|
||||
assert_eq!(queue.pop_expired(next, 502), Some(first));
|
||||
assert_eq!(queue.pop_expired(next, 502), None);
|
||||
assert_eq!(queue.pop_expired(later, 502), Some(second));
|
||||
assert_eq!(queue.pages.len(), 2);
|
||||
assert_eq!(queue.pop_expired(later, 506), None);
|
||||
assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), 506), Some(third));
|
||||
assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), i64::MAX), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_metrics_mark_successful_records_only() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -27,13 +27,13 @@ use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, MultipartSourceResponse,
|
||||
};
|
||||
use crate::disk::{
|
||||
BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo,
|
||||
DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload,
|
||||
GuardedFileInfoVersions, PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo,
|
||||
WalkDirOptions, batch_read_version_one_by_one,
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection,
|
||||
DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload, GuardedFileInfoVersions,
|
||||
PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -1032,7 +1032,6 @@ impl RemoteDisk {
|
||||
volume: &str,
|
||||
payload: Vec<u8>,
|
||||
probe_only: bool,
|
||||
bounded_prepare: bool,
|
||||
) -> Result<DeleteVersionsResponse> {
|
||||
if payload.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
@@ -1057,8 +1056,6 @@ impl RemoteDisk {
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::GuardedDeleteUnsupported
|
||||
} else if bounded_prepare && status.code() == Code::InvalidArgument {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
@@ -1092,14 +1089,6 @@ impl RemoteDisk {
|
||||
volume: &str,
|
||||
operation: DeleteRecoveryRpcOperation,
|
||||
) -> Result<DeleteRecoveryRpcReply> {
|
||||
let protocol_version = match &operation {
|
||||
DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. }
|
||||
if opts.requires_bounded_delete_recovery =>
|
||||
{
|
||||
BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION
|
||||
}
|
||||
_ => DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
};
|
||||
let payload = rmp_serde::to_vec_named(&operation)?;
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
@@ -1130,7 +1119,7 @@ impl RemoteDisk {
|
||||
let mut request = Request::new(RecoverDeleteTransactionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_owned(),
|
||||
protocol_version,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
operation: payload.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref())
|
||||
@@ -1148,7 +1137,7 @@ impl RemoteDisk {
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if response.protocol_version != protocol_version {
|
||||
if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
if !response.success {
|
||||
@@ -2790,7 +2779,7 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true, false).await?;
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -2803,12 +2792,7 @@ impl DiskAPI for RemoteDisk {
|
||||
opts: DeleteOptions,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
let count = versions.len();
|
||||
let bounded_prepare = opts.requires_bounded_delete_recovery;
|
||||
let payload = GuardedDeleteVersionsPayload {
|
||||
versions,
|
||||
opts,
|
||||
bounded_recovery_protocol: bounded_prepare.then_some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION),
|
||||
};
|
||||
let payload = GuardedDeleteVersionsPayload { versions, opts };
|
||||
if let Err(error) = payload.validate() {
|
||||
return vec![Some(error); count];
|
||||
}
|
||||
@@ -2816,10 +2800,7 @@ impl DiskAPI for RemoteDisk {
|
||||
Ok(encoded) => encoded,
|
||||
Err(error) => return vec![Some(error.into()); count],
|
||||
};
|
||||
match self
|
||||
.guarded_delete_versions_rpc(volume, encoded, false, bounded_prepare)
|
||||
.await
|
||||
{
|
||||
match self.guarded_delete_versions_rpc(volume, encoded, false).await {
|
||||
Ok(response) => {
|
||||
// New guarded peers must return typed errors, never lossy strings.
|
||||
if response.item_errors.len() != count {
|
||||
|
||||
@@ -2344,31 +2344,33 @@ fn grow_decommission_target_reservation(
|
||||
if required_peak <= existing_remaining {
|
||||
return Ok(false);
|
||||
}
|
||||
// Scanner estimates can predate source writes. Grow against current
|
||||
// target capacity, then validate the aggregate reservation below.
|
||||
if reservation.source_data_equivalent_bytes > 0 {
|
||||
return Err(decommission_target_no_longer_fits_error(
|
||||
target_pool_index,
|
||||
format!("source pool {source_pool_index} reservation estimate is exhausted"),
|
||||
));
|
||||
}
|
||||
if let Some(target) = existing_target
|
||||
&& target_capacity.physical_free <= target.physical_free_at_reservation
|
||||
{
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} target pool {target_pool_index} reservation is exhausted"
|
||||
)));
|
||||
}
|
||||
let current_peak = reservation.remaining_peak_physical_bytes();
|
||||
let additional_peak = required_peak.saturating_sub(existing_remaining);
|
||||
let additional_predicted = additional_peak.div_ceil(1usize.saturating_add(reservation.temporary_copies));
|
||||
let additional_source_data = capacity_source_data_equivalent(additional_predicted, target_layout)?;
|
||||
reservation.source_data_equivalent_bytes = reservation
|
||||
.source_data_equivalent_bytes
|
||||
.saturating_add(additional_source_data);
|
||||
reservation.source_physical_bytes = reservation
|
||||
.source_physical_bytes
|
||||
.saturating_add(capacity_target_physical_bytes(additional_source_data, reservation.source_layout)?);
|
||||
// Recompute from the physical total so EC rounding matches the
|
||||
// persisted reader after every growth, including repeated growth.
|
||||
reservation.source_data_equivalent_bytes =
|
||||
capacity_source_data_equivalent(reservation.source_physical_bytes, reservation.source_layout)?;
|
||||
reservation.predicted_physical_bytes = reservation
|
||||
.targets
|
||||
.iter()
|
||||
.map(|target| target.layout)
|
||||
.chain(std::iter::once(target_layout))
|
||||
.try_fold(
|
||||
reservation.predicted_physical_bytes.saturating_add(additional_predicted),
|
||||
|predicted, layout| {
|
||||
capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, layout)
|
||||
.map(|required| predicted.max(required))
|
||||
},
|
||||
)?;
|
||||
.predicted_physical_bytes
|
||||
.saturating_add(additional_predicted)
|
||||
.max(capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target_layout)?);
|
||||
reservation.temporary_physical_bytes = reservation
|
||||
.predicted_physical_bytes
|
||||
.saturating_mul(reservation.temporary_copies);
|
||||
@@ -2400,7 +2402,6 @@ fn grow_decommission_target_reservation(
|
||||
});
|
||||
}
|
||||
renew_decommission_capacity_reservation(reservation, now, true);
|
||||
validate_decommission_capacity_reservation(Some(reservation))?;
|
||||
pool.last_update = pool.last_update.max(now);
|
||||
Ok(true)
|
||||
})();
|
||||
@@ -8291,18 +8292,16 @@ where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
// Bound the CAS state retained by callers, including ordinary metadata
|
||||
// transactions that never enter owner repair. Large inline retry states
|
||||
// transactions that never enter identity repair. Large inline retry states
|
||||
// otherwise propagate through startup and cross-crate storage futures.
|
||||
Box::pin(async move {
|
||||
let native_owner_repair = object == POOL_META_NAME && matches!(phase, "native_heal" | "prepare_cas");
|
||||
if object != POOL_META_IDENTITY_NAME && !native_owner_repair {
|
||||
if object != POOL_META_IDENTITY_NAME {
|
||||
return execute_pool_meta_object_cas_once(pool, object, data, opts, fence, phase).await;
|
||||
}
|
||||
// Repair only the exact payload already published by this owner. A
|
||||
// quorum read cannot discharge responsibility for a missing replica.
|
||||
let committed_payload = data;
|
||||
let mut outcome =
|
||||
execute_pool_meta_object_cas_once(pool.clone(), object, committed_payload.clone(), opts, fence, phase).await;
|
||||
// Keep the exact identity owned by this transaction. A quorum read alone
|
||||
// cannot discharge repair responsibility for a missing disk replica.
|
||||
let identity = data;
|
||||
let mut outcome = execute_pool_meta_object_cas_once(pool.clone(), object, identity.clone(), opts, fence, phase).await;
|
||||
for delay_secs in [1, 2, 4] {
|
||||
if !outcome.source.as_deref().is_some_and(|err| {
|
||||
matches!(err, Error::Io(inner) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataRepairRequired>()))
|
||||
@@ -8315,28 +8314,19 @@ where
|
||||
let repair_options = async {
|
||||
fence.ensure_held()?;
|
||||
let (current, info) = read_config_no_lock_preserve_empty_with_metadata(pool.clone(), object).await?;
|
||||
if current != committed_payload {
|
||||
if current != identity {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let token = if native_owner_repair {
|
||||
match pool_meta_cas_token_from_info(info) {
|
||||
token @ PoolMetaCasToken::ExistingNative { .. } => token,
|
||||
_ => return Err(Error::PreconditionFailed),
|
||||
}
|
||||
} else {
|
||||
info.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.map(PoolMetaCasToken::Existing)
|
||||
.unwrap_or(PoolMetaCasToken::Unsafe)
|
||||
};
|
||||
let token = info
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.map(PoolMetaCasToken::Existing)
|
||||
.unwrap_or(PoolMetaCasToken::Unsafe);
|
||||
pool_meta_cas_options(object, &token, fence)
|
||||
}
|
||||
.await;
|
||||
outcome = match repair_options {
|
||||
Ok(opts) => {
|
||||
execute_pool_meta_object_cas_once(pool.clone(), object, committed_payload.clone(), opts, fence, phase)
|
||||
.await
|
||||
}
|
||||
Ok(opts) => execute_pool_meta_object_cas_once(pool.clone(), object, identity.clone(), opts, fence, phase).await,
|
||||
Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err),
|
||||
};
|
||||
// A rejected repair cannot turn the preceding committed write into a
|
||||
@@ -12144,13 +12134,8 @@ async fn pause_decommission_start_for_test(store_id: uuid::Uuid, point: Decommis
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
struct DecommissionCapacityInfoOverride {
|
||||
snapshots: std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>,
|
||||
owned_snapshots: Option<std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
type DecommissionCapacityInfoOverrides = std::sync::Mutex<HashMap<uuid::Uuid, DecommissionCapacityInfoOverride>>;
|
||||
type DecommissionCapacityInfoOverrides =
|
||||
std::sync::Mutex<HashMap<uuid::Uuid, std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>>;
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock<DecommissionCapacityInfoOverrides> = std::sync::OnceLock::new();
|
||||
@@ -12167,30 +12152,16 @@ pub(crate) fn set_decommission_capacity_info_overrides_for_test(
|
||||
.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.expect("decommission capacity info override should not be poisoned")
|
||||
.insert(
|
||||
store_id,
|
||||
DecommissionCapacityInfoOverride {
|
||||
snapshots: snapshots.into(),
|
||||
owned_snapshots: None,
|
||||
},
|
||||
);
|
||||
.insert(store_id, snapshots.into());
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn take_decommission_capacity_info_override_for_test(
|
||||
store_id: uuid::Uuid,
|
||||
use_owned_usage: bool,
|
||||
) -> Option<Vec<DecommissionPoolCapacityInfo>> {
|
||||
fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<DecommissionPoolCapacityInfo>> {
|
||||
let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES
|
||||
.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.expect("decommission capacity info override should not be poisoned");
|
||||
let entry = overrides.get_mut(&store_id)?;
|
||||
let queue = if use_owned_usage {
|
||||
entry.owned_snapshots.as_mut().unwrap_or(&mut entry.snapshots)
|
||||
} else {
|
||||
&mut entry.snapshots
|
||||
};
|
||||
let queue = overrides.get_mut(&store_id)?;
|
||||
if queue.len() > 1 {
|
||||
queue.pop_front()
|
||||
} else {
|
||||
@@ -15260,7 +15231,7 @@ impl ECStore {
|
||||
ensure_decommission_start_pool_states(&latest_pool_meta, indices)?;
|
||||
ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?;
|
||||
ensure_decommission_capacity_writer_supported(&latest_pool_meta)?;
|
||||
let capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
|
||||
let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
|
||||
activation_fence.ensure_held()?;
|
||||
let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
|
||||
|
||||
@@ -15622,7 +15593,7 @@ impl ECStore {
|
||||
|
||||
async fn get_decommission_prewrite_free_capacity(&self, target_pool_index: usize) -> Result<usize> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id, false) {
|
||||
if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
|
||||
return capacity_infos
|
||||
.iter()
|
||||
.find(|capacity| capacity.pool_index == target_pool_index)
|
||||
@@ -15645,7 +15616,7 @@ impl ECStore {
|
||||
|
||||
async fn get_decommission_pool_capacity_infos(&self, use_owned_usage: bool) -> Result<Vec<DecommissionPoolCapacityInfo>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id, use_owned_usage) {
|
||||
if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
|
||||
return Ok(capacity_infos);
|
||||
}
|
||||
|
||||
@@ -15863,8 +15834,14 @@ impl ECStore {
|
||||
log_decommission_adopted_target_choice(reservation, owner, pool_index, &over_budget);
|
||||
return Ok(pool_index);
|
||||
}
|
||||
// A stale nonzero estimate can exhaust its allocation too. Selection
|
||||
// is provisional until fenced growth persists the new reservation.
|
||||
if reservation.source_data_equivalent_bytes > 0 {
|
||||
let mut skipped = over_budget.iter().copied().collect::<Vec<_>>();
|
||||
skipped.sort_unstable();
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {} has no target allocation for {expected_data_bytes} data bytes; target pools over their temporary-copy budget: {skipped:?}",
|
||||
owner.source_pool_index
|
||||
)));
|
||||
}
|
||||
let capacities = self.get_decommission_all_pool_capacity_infos().await?;
|
||||
let active_sources = active_decommission_source_indices(&pool_meta);
|
||||
capacities
|
||||
@@ -25796,55 +25773,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_activation_reserves_owned_usage_on_a_shared_filesystem() {
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let target = DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0);
|
||||
let filesystem_source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 400, 400);
|
||||
let owned_source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 400, 30);
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, vec![vec![filesystem_source, target]]);
|
||||
{
|
||||
let mut overrides = super::DECOMMISSION_CAPACITY_INFO_OVERRIDES
|
||||
.get()
|
||||
.expect("capacity overrides initialized")
|
||||
.lock()
|
||||
.expect("capacity overrides lock");
|
||||
overrides
|
||||
.get_mut(&store.id)
|
||||
.expect("this store has an override")
|
||||
.owned_snapshots = Some(vec![vec![owned_source, target]].into());
|
||||
}
|
||||
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("activation should reserve the owned bytes rather than unrelated mount usage");
|
||||
|
||||
let local = store.pool_meta.read().await;
|
||||
let reservation = local.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|state| state.capacity_reservation.as_ref())
|
||||
.expect("activation must retain its capacity reservation");
|
||||
assert_eq!(reservation.source_physical_bytes, 30);
|
||||
assert_eq!(reservation.peak_physical_bytes, 60);
|
||||
drop(local);
|
||||
let mut persisted = PoolMeta::default();
|
||||
persisted
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("activation should be durable");
|
||||
let reservation = persisted.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|state| state.capacity_reservation.as_ref())
|
||||
.expect("persisted activation must retain its capacity reservation");
|
||||
assert_eq!(reservation.source_physical_bytes, 30);
|
||||
assert_eq!(reservation.peak_physical_bytes, 60);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_activation_lock_recheck_rejects_sudden_space_drop_without_persisting() {
|
||||
@@ -25879,173 +25807,6 @@ mod tests {
|
||||
assert!(persisted.pools[0].decommission.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_copies_native_object_after_nonzero_usage_estimate_is_exhausted() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
let bucket = format!("stale-owned-{}", uuid::Uuid::new_v4());
|
||||
let object = "new-after-scan.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create source bucket");
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let capacities = vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1 << 30, 1),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 1 << 30, 1 << 30, 0),
|
||||
];
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities]);
|
||||
let expected = vec![0x57; 16 * 1024];
|
||||
let mut body = crate::object_api::PutObjReader::from_vec(expected.clone());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, object, &mut body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write an object larger than the earlier scanner estimate");
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(
|
||||
&[0],
|
||||
vec![DecomBucketInfo {
|
||||
name: bucket.clone(),
|
||||
prefix: String::new(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("activate the stale nonzero reservation");
|
||||
let canceler = DecommissionCanceler::new(CancellationToken::new());
|
||||
store.decommission_cancelers.write().await[0] = Some(canceler.clone());
|
||||
store
|
||||
.do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
|
||||
.await
|
||||
.expect("run the real native-disk mover");
|
||||
|
||||
let mut reader = store.pools[1]
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the newer object must reach the target despite the stale estimate");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read migrated payload");
|
||||
assert_eq!(actual, expected);
|
||||
assert!(
|
||||
store.pools[0]
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"source cleanup follows the verified target copy"
|
||||
);
|
||||
let meta = store.pool_meta.read().await;
|
||||
let state = meta.pools[0].decommission.as_ref().expect("retirement state is retained");
|
||||
assert!(state.complete && !state.failed);
|
||||
assert_eq!(state.items_decommission_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg(feature = "test-util")]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_native_ec_growth_keeps_all_metadata_replicas_readable() {
|
||||
use rustfs_data_usage::{DataUsageCache, DataUsageEntry};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, store, _other_store) =
|
||||
crate::services::rebalance::test_three_pool_stores_with_four_disk_sets_with_isolated_node_contexts(None).await;
|
||||
let bucket = format!("stale-ec-{}", uuid::Uuid::new_v4());
|
||||
let object = "new-after-scan.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for set in &store.pools[0].disk_set {
|
||||
let encoded = rmp_serde::to_vec_named(&serde_json::json!({
|
||||
"info": {
|
||||
"name": DATA_USAGE_ROOT,
|
||||
"next_cycle": 1,
|
||||
"last_update": std::time::SystemTime::now(),
|
||||
"skip_healing": false,
|
||||
"snapshot_complete": true
|
||||
},
|
||||
"cache": {
|
||||
DATA_USAGE_ROOT: DataUsageEntry { size: 3, objects: 1, ..Default::default() }
|
||||
}
|
||||
}))
|
||||
.expect("encode completed scanner fixture");
|
||||
let cache = DataUsageCache::unmarshal(&encoded).expect("read scanner fixture");
|
||||
assert!(cache.dui(DATA_USAGE_ROOT, &[]).is_complete_bucket_usage_snapshot());
|
||||
set.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("{}/{}", crate::disk::BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME),
|
||||
&mut crate::object_api::PutObjReader::from_vec(encoded),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("persist the earlier complete cache");
|
||||
}
|
||||
let expected = vec![0x37; 16 * 1024];
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut crate::object_api::PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("write after the scan");
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(
|
||||
&[0],
|
||||
vec![DecomBucketInfo {
|
||||
name: bucket.clone(),
|
||||
prefix: String::new(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("activate from the earlier owned estimate");
|
||||
{
|
||||
let meta = store.pool_meta.read().await;
|
||||
let reservation = meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.capacity_reservation
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
assert_eq!(reservation.source_layout, DecommissionErasureLayout { data: 3, parity: 1 });
|
||||
assert_eq!(reservation.source_physical_bytes, 8);
|
||||
}
|
||||
let canceler = DecommissionCanceler::new(CancellationToken::new());
|
||||
store.decommission_cancelers.write().await[0] = Some(canceler.clone());
|
||||
store
|
||||
.do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
|
||||
.await
|
||||
.expect("run native EC migration");
|
||||
for pool in &store.pools {
|
||||
let mut reloaded = PoolMeta::default();
|
||||
reloaded
|
||||
.load_no_lock_from_replicas(vec![pool.clone()])
|
||||
.await
|
||||
.expect("each saved metadata replica must reload");
|
||||
let state = reloaded.pools[0].decommission.as_ref().expect("saved retirement state");
|
||||
assert!(state.complete && !state.failed);
|
||||
}
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("target object must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read migrated EC payload");
|
||||
assert_eq!(actual, expected);
|
||||
assert!(
|
||||
store.pools[0]
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_worker_pauses_on_runtime_capacity_shortage_without_source_side_effect() {
|
||||
@@ -31288,10 +31049,6 @@ mod pools_tests {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
let mut reader = self.inner.get_object_reader(bucket, object, range, headers, opts).await?;
|
||||
if object == POOL_META_NAME {
|
||||
reader.object_info.mod_time = Some(time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
|
||||
reader.object_info.data_dir = Some(uuid::Uuid::from_u128(7));
|
||||
}
|
||||
match self.fault {
|
||||
IdentityRepairFault::Changed => {
|
||||
reader.stream = Box::new(Cursor::new(b"other-identity".to_vec()));
|
||||
@@ -31321,30 +31078,16 @@ mod pools_tests {
|
||||
.and_then(HTTPPreconditions::if_none_match_value),
|
||||
None
|
||||
);
|
||||
let written = if object == POOL_META_NAME {
|
||||
&self.inner.stored
|
||||
} else {
|
||||
&self.inner.identity
|
||||
};
|
||||
let current_etag = written.lock().expect("metadata lock").as_ref().map(|(_, etag)| etag.clone());
|
||||
assert_eq!(
|
||||
opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value),
|
||||
current_etag.as_deref(),
|
||||
self.inner
|
||||
.identity
|
||||
.lock()
|
||||
.expect("identity lock")
|
||||
.as_ref()
|
||||
.map(|(_, etag)| etag.as_str()),
|
||||
"repair must use the observed post-commit CAS token",
|
||||
);
|
||||
if object == POOL_META_NAME {
|
||||
let condition = rustfs_utils::http::metadata_compat::get_consistent_str(
|
||||
&opts.user_defined,
|
||||
crate::disk::POOL_META_NATIVE_CAS_METADATA_SUFFIX,
|
||||
)
|
||||
.expect("native repair requires a physical predecessor condition");
|
||||
let condition: crate::disk::PoolMetaNativeCasCondition =
|
||||
serde_json::from_str(condition).expect("parse native CAS condition");
|
||||
let parent = condition.parent.expect("native CAS uses the actual committed head");
|
||||
assert_eq!(Some(parent.etag), current_etag);
|
||||
assert_eq!(parent.mod_time_unix_nanos, 1_000_000_000);
|
||||
assert_eq!(parent.data_dir, Some(uuid::Uuid::from_u128(7)));
|
||||
}
|
||||
if self.fault == IdentityRepairFault::Reject {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
@@ -31449,46 +31192,6 @@ mod pools_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn pool_meta_prepare_owner_repair_uses_exact_native_head() {
|
||||
let payload = b"owned-pending-pool-meta";
|
||||
for (fault, writes, reads, succeeds) in [
|
||||
(IdentityRepairFault::Once, 2, 1, true),
|
||||
(IdentityRepairFault::Changed, 1, 1, false),
|
||||
(IdentityRepairFault::MissingEtag, 1, 1, false),
|
||||
(IdentityRepairFault::Unknown, 1, 0, false),
|
||||
] {
|
||||
let pool = IdentityRepairStorage::new(fault);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let mut arm = state.arm_transaction();
|
||||
let result = save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_NAME,
|
||||
payload.to_vec(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"prepare_cas",
|
||||
&mut arm,
|
||||
)
|
||||
.await;
|
||||
|
||||
if succeeds {
|
||||
result.expect("prepare must repair its own partially replicated payload");
|
||||
arm.disarm();
|
||||
assert_eq!(pool.inner.stored.lock().expect("metadata lock").as_ref().expect("written").0, payload);
|
||||
} else {
|
||||
let error = result.expect_err("uncertain or changed authority must remain blocked");
|
||||
let failure = error.pool_metadata_failure().expect("typed transaction uncertainty");
|
||||
assert_eq!(failure.kind, crate::error::PoolMetadataFailure::TransactionUnknown);
|
||||
assert_eq!(failure.phase, "prepare_cas");
|
||||
}
|
||||
drop(arm);
|
||||
assert_eq!(state.ensure_write_safe("after prepare owner repair").is_ok(), succeeds);
|
||||
assert_eq!(pool.writes.load(Ordering::SeqCst), writes, "{fault:?}");
|
||||
assert_eq!(pool.reads.load(Ordering::SeqCst), reads, "{fault:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn pool_identity_owner_repair_cancellation_keeps_the_transaction_armed() {
|
||||
for fault in [
|
||||
@@ -37853,89 +37556,10 @@ mod pools_tests {
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.unwrap();
|
||||
assert_eq!(reservation.source_data_equivalent_bytes, 3);
|
||||
assert_eq!(reservation.predicted_physical_bytes, 4);
|
||||
assert_eq!(reservation.source_data_equivalent_bytes, 2);
|
||||
assert_eq!(reservation.predicted_physical_bytes, 3);
|
||||
assert_eq!(reservation.targets.len(), 1);
|
||||
assert_eq!(reservation.targets[0].reserved_physical_bytes, 8);
|
||||
validate_decommission_capacity_reservation(Some(reservation)).expect("empty-snapshot growth must remain readable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_capacity_grows_a_stale_nonzero_estimate_and_rolls_back_if_space_is_short() {
|
||||
let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
|
||||
let layout = DecommissionErasureLayout { data: 3, parity: 1 };
|
||||
let initial = vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 64, 2),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 64, 64, 0),
|
||||
];
|
||||
let mut meta = PoolMeta {
|
||||
version: POOL_META_VERSION,
|
||||
pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
|
||||
..Default::default()
|
||||
};
|
||||
meta.decommission(0, initial[0].space).expect("activate source");
|
||||
reserve_decommission_start_target_capacity(
|
||||
&mut meta,
|
||||
&[0],
|
||||
&initial,
|
||||
uuid::Uuid::new_v4(),
|
||||
1,
|
||||
now,
|
||||
DECOMMISSION_CAPACITY_MODEL_VERSION,
|
||||
)
|
||||
.expect("reserve the stale estimate");
|
||||
assert!(
|
||||
grow_decommission_target_reservation(&mut meta, &initial, 0, 1, 16, uuid::Uuid::new_v4(), now)
|
||||
.expect("a nonzero stale estimate may grow against real free space")
|
||||
);
|
||||
let before = serde_json::to_value(&meta).expect("snapshot reservation before rejected growth");
|
||||
let shortage = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 43, 64, 21)];
|
||||
assert!(
|
||||
grow_decommission_target_reservation(&mut meta, &shortage, 0, 1, 32, uuid::Uuid::new_v4(), now).is_err(),
|
||||
"real free-space shortage still rejects growth"
|
||||
);
|
||||
assert_eq!(serde_json::to_value(&meta).expect("snapshot after rejected growth"), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_capacity_growth_keeps_rounded_and_mixed_layout_estimates_loadable() {
|
||||
let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
|
||||
for source_layout in [
|
||||
DecommissionErasureLayout { data: 3, parity: 1 },
|
||||
DecommissionErasureLayout { data: 2, parity: 2 },
|
||||
] {
|
||||
let capacities = vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, source_layout, 0, 8, 8),
|
||||
DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 2, parity: 2 }, 16, 16, 0),
|
||||
DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 3, parity: 1 }, 1024, 1024, 0),
|
||||
];
|
||||
let mut meta = PoolMeta {
|
||||
version: POOL_META_VERSION,
|
||||
pools: (0..3).map(|index| decommission_test_pool_status(index, None)).collect(),
|
||||
..Default::default()
|
||||
};
|
||||
meta.decommission(0, capacities[0].space).unwrap();
|
||||
reserve_decommission_start_target_capacity(
|
||||
&mut meta,
|
||||
&[0],
|
||||
&capacities,
|
||||
uuid::Uuid::new_v4(),
|
||||
1,
|
||||
now,
|
||||
DECOMMISSION_CAPACITY_MODEL_VERSION,
|
||||
)
|
||||
.expect("initial reservation");
|
||||
grow_decommission_target_reservation(&mut meta, &capacities, 0, 2, 16, uuid::Uuid::new_v4(), now)
|
||||
.expect("grow against the target's free space");
|
||||
let reservation = meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.capacity_reservation
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
validate_decommission_capacity_reservation(Some(reservation)).expect("growth must satisfy the persisted reader");
|
||||
}
|
||||
assert_eq!(reservation.targets[0].reserved_physical_bytes, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -34,7 +34,7 @@ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapsho
|
||||
use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram,
|
||||
observed_data_usage_is_newer, observed_data_usage_is_newer_than_identity,
|
||||
observed_data_usage_is_newer,
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
@@ -227,10 +227,7 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
|
||||
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
|
||||
#[derive(Debug, Clone)]
|
||||
struct CachedDataUsageSnapshot {
|
||||
/// Last successful load, retained through failures for lineage comparison.
|
||||
info: Option<DataUsageInfo>,
|
||||
/// Controls serving independently from the retained ordering evidence.
|
||||
last_load_failed: bool,
|
||||
loaded_at: tokio::time::Instant,
|
||||
/// Pre-discard per-bucket sizes from the same load, retained even when the
|
||||
/// snapshot is incomplete and its bucket data is discarded. Consumed only
|
||||
@@ -243,7 +240,6 @@ impl CachedDataUsageSnapshot {
|
||||
fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
|
||||
self.info
|
||||
.as_ref()
|
||||
.filter(|_| !self.last_load_failed)
|
||||
.map(project)
|
||||
.ok_or_else(|| Error::other("data usage snapshot load recently failed"))
|
||||
}
|
||||
@@ -279,17 +275,6 @@ fn cache_data_usage_snapshot_result<T>(
|
||||
mut degraded_baseline,
|
||||
observed_unavailable,
|
||||
}) => {
|
||||
// A primary read can fall back to an older backup. Refresh the
|
||||
// retry age without replacing a newer snapshot this process has
|
||||
// already read; snapshot size itself is not an ordering signal.
|
||||
if let Some(previous) = cache.as_mut()
|
||||
&& let Some(cached_info) = previous.info.as_ref()
|
||||
&& snapshot_predates(&info, cached_info)
|
||||
{
|
||||
previous.loaded_at = loaded_at;
|
||||
previous.last_load_failed = false;
|
||||
return Some(Ok(project(cached_info)));
|
||||
}
|
||||
// A flaky observed read must not shrink quota coverage for a TTL
|
||||
// window: carry the previous refresh's baseline entries forward,
|
||||
// letting the fresh (authoritative) values win where they exist.
|
||||
@@ -301,23 +286,21 @@ fn cache_data_usage_snapshot_result<T>(
|
||||
let projected = project(&info);
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
Ok(projected)
|
||||
}
|
||||
Err(e) => {
|
||||
// A failed load is negatively cached, but cannot erase the last
|
||||
// good snapshot's ordering proof or grounded quota baseline.
|
||||
let cached = cache.get_or_insert_with(|| CachedDataUsageSnapshot {
|
||||
// Keep the previous baseline through a failed refresh: quota
|
||||
// admission must not lose its last grounded values because one
|
||||
// backend read errored.
|
||||
let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: None,
|
||||
last_load_failed: true,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
degraded_baseline,
|
||||
});
|
||||
cached.loaded_at = loaded_at;
|
||||
cached.last_load_failed = true;
|
||||
Err(e)
|
||||
}
|
||||
})
|
||||
@@ -1262,7 +1245,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
|
||||
let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
// The observed-newness gate below compares against the snapshot as
|
||||
// persisted, before normalization demotes or discards anything.
|
||||
let authoritative_as_persisted = loaded_snapshot.snapshot_identity();
|
||||
let authoritative_as_persisted = loaded_snapshot.clone();
|
||||
let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
|
||||
|
||||
// A bucket without a converged scanner cycle behind it — a freshly joined
|
||||
@@ -1280,7 +1263,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
|
||||
// read per snapshot refresh.
|
||||
let mut observed_unavailable = false;
|
||||
match load_observed_data_usage_snapshot(store).await {
|
||||
Ok(Some(observed)) if observed_data_usage_is_newer_than_identity(&observed, authoritative_as_persisted) => {
|
||||
Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
|
||||
backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
|
||||
}
|
||||
Ok(_) => {}
|
||||
@@ -2384,9 +2367,6 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
|
||||
let Some(cached) = cache.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
if cached.last_load_failed {
|
||||
return false;
|
||||
}
|
||||
let Some(info) = cached.info.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
@@ -2402,35 +2382,11 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
|
||||
.is_none_or(|attempted_at| now.duration_since(attempted_at) >= refresh_age)
|
||||
}
|
||||
|
||||
/// Compare authoritative lineage by the writer's epoch, cycle, then timestamp
|
||||
/// order. Within one lineage, compare the observation's coordinated position
|
||||
/// before its timestamp; a later observation cannot revive an older lineage.
|
||||
fn snapshot_predates(loaded: &DataUsageInfo, cached: &DataUsageInfo) -> bool {
|
||||
let loaded_baseline = loaded
|
||||
.usage_snapshot_authoritative_baseline
|
||||
.unwrap_or_else(|| loaded.snapshot_identity());
|
||||
let cached_baseline = cached
|
||||
.usage_snapshot_authoritative_baseline
|
||||
.unwrap_or_else(|| cached.snapshot_identity());
|
||||
let loaded_position = (loaded_baseline.scanner_epoch, loaded_baseline.scanner_cycle);
|
||||
let cached_position = (cached_baseline.scanner_epoch, cached_baseline.scanner_cycle);
|
||||
if loaded_position != cached_position {
|
||||
return loaded_position < cached_position;
|
||||
}
|
||||
let (loaded, cached) = if loaded_baseline == cached_baseline {
|
||||
let loaded_observation_position = loaded.scanner_epoch.zip(loaded.scanner_cycle);
|
||||
let cached_observation_position = cached.scanner_epoch.zip(cached.scanner_cycle);
|
||||
if loaded_observation_position != cached_observation_position {
|
||||
return loaded_observation_position < cached_observation_position;
|
||||
}
|
||||
(loaded.last_update, cached.last_update)
|
||||
} else {
|
||||
(loaded_baseline.last_update, cached_baseline.last_update)
|
||||
};
|
||||
/// Whether a freshly loaded snapshot is older than the cached one, as when a
|
||||
/// primary read fails and the loader falls back to the previous `.bkp` copy.
|
||||
fn snapshot_predates(loaded: Option<SystemTime>, cached: Option<SystemTime>) -> bool {
|
||||
match (loaded, cached) {
|
||||
(Some(loaded), Some(cached)) => {
|
||||
loaded < cached && !rustfs_data_usage::usage_last_update_is_untrusted_future(cached, SystemTime::now())
|
||||
}
|
||||
(Some(loaded), Some(cached)) => loaded < cached,
|
||||
(None, Some(_)) => true,
|
||||
_ => false,
|
||||
}
|
||||
@@ -2471,15 +2427,22 @@ where
|
||||
return false;
|
||||
};
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
let cached_last_update = cache
|
||||
.as_ref()
|
||||
.and_then(|cached| cached.info.as_ref())
|
||||
.and_then(|info| info.last_update);
|
||||
if snapshot_predates(loaded.info.last_update, cached_last_update) {
|
||||
return false;
|
||||
}
|
||||
cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded),
|
||||
tokio::time::Instant::now(),
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
|info| info.buckets_usage.contains_key(bucket),
|
||||
|_| (),
|
||||
)
|
||||
.is_some_and(|result| matches!(result, Ok(true)))
|
||||
.is_some()
|
||||
}
|
||||
|
||||
// Calls of `reload_unknown_bucket_usage` per bucket. Unit tests do not run in
|
||||
@@ -4261,7 +4224,6 @@ mod tests {
|
||||
let degraded_baseline = info.bucket_sizes.clone();
|
||||
*data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info),
|
||||
last_load_failed: false,
|
||||
loaded_at: tokio::time::Instant::now()
|
||||
.checked_sub(age)
|
||||
.expect("the monotonic clock must reach back by the requested age"),
|
||||
@@ -4738,355 +4700,6 @@ mod tests {
|
||||
assert_snapshot_bucket(&cached, "bucket");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_usage_snapshot_refresh_keeps_newer_counts_over_an_older_backup() {
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::config::com::save_config;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let mut current = usage_snapshot_with_new_bucket(SystemTime::UNIX_EPOCH + Duration::from_secs(2), 100);
|
||||
current.scanner_epoch = Some(4);
|
||||
current.scanner_cycle = Some(11);
|
||||
current.usage_snapshot_converged = Some(true);
|
||||
let mut backup = data_usage_info_for_test("existing", 1, 5, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
|
||||
backup.scanner_epoch = Some(4);
|
||||
backup.scanner_cycle = Some(10);
|
||||
backup.usage_snapshot_converged = Some(true);
|
||||
for (path, info) in [
|
||||
(DATA_USAGE_OBJ_NAME_PATH.as_str(), ¤t),
|
||||
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
|
||||
] {
|
||||
save_config(store.clone(), path, serde_json::to_vec(info).expect("encode usage snapshot"))
|
||||
.await
|
||||
.expect("persist initial usage snapshot");
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
for loaded in [
|
||||
load_data_usage_from_backend_cached(store.clone()).await,
|
||||
load_admin_data_usage_from_backend_cached(store.clone()).await,
|
||||
] {
|
||||
assert_eq!(loaded.expect("prime usage cache"), current);
|
||||
}
|
||||
|
||||
// Bucket cleanup can reduce usage without advancing the scanner
|
||||
// position. Only its publication timestamp advances.
|
||||
let mut after_delete = data_usage_info_for_test("existing", 1, 3, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
|
||||
after_delete.scanner_epoch = current.scanner_epoch;
|
||||
after_delete.scanner_cycle = current.scanner_cycle;
|
||||
after_delete.usage_snapshot_converged = Some(true);
|
||||
let mut stale_observation =
|
||||
data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
|
||||
stale_observation.scanner_epoch = Some(4);
|
||||
stale_observation.scanner_cycle = Some(12);
|
||||
stale_observation.usage_snapshot_converged = Some(false);
|
||||
stale_observation.usage_snapshot_authoritative_baseline = Some(backup.snapshot_identity());
|
||||
assert!(observed_data_usage_is_newer(&stale_observation, &backup));
|
||||
assert!(!observed_data_usage_is_newer(&stale_observation, ¤t));
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&stale_observation).expect("encode observation of an older namespace"),
|
||||
)
|
||||
.await
|
||||
.expect("persist an observation anchored to the old backup");
|
||||
for (payload, expected) in [
|
||||
(b"not a usage snapshot".to_vec(), ¤t),
|
||||
(serde_json::to_vec(&after_delete).expect("encode post-delete usage"), &after_delete),
|
||||
] {
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), payload)
|
||||
.await
|
||||
.expect("replace primary usage snapshot");
|
||||
for cache in [data_usage_snapshot_cache(), admin_data_usage_snapshot_cache()] {
|
||||
cache
|
||||
.write()
|
||||
.await
|
||||
.as_mut()
|
||||
.expect("usage cache should be populated")
|
||||
.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
}
|
||||
for loaded in [
|
||||
load_data_usage_from_backend_cached(store.clone()).await,
|
||||
load_admin_data_usage_from_backend_cached(store.clone()).await,
|
||||
] {
|
||||
assert_eq!(loaded.expect("refresh an expired usage cache"), *expected);
|
||||
}
|
||||
assert!(
|
||||
!data_usage_snapshot_cache()
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("quota cache should remain populated")
|
||||
.degraded_baseline
|
||||
.contains_key("retired-incarnation"),
|
||||
"a newer observation anchored to an older namespace must not backfill ghost usage"
|
||||
);
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_admin_usage_refresh_orders_observations_by_generation_before_time() {
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::config::com::save_config;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let mut authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
|
||||
authoritative.scanner_epoch = Some(7);
|
||||
authoritative.scanner_cycle = Some(10);
|
||||
authoritative.usage_snapshot_converged = Some(true);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&authoritative).expect("encode authoritative usage"),
|
||||
)
|
||||
.await
|
||||
.expect("persist the common authoritative baseline");
|
||||
let observation = |epoch, cycle, seconds, objects, size| {
|
||||
let mut info =
|
||||
data_usage_info_for_test("bucket", objects, size, SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
|
||||
info.scanner_epoch = Some(epoch);
|
||||
info.scanner_cycle = Some(cycle);
|
||||
info.usage_snapshot_converged = Some(false);
|
||||
info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
info
|
||||
};
|
||||
let first = observation(7, 11, 200, 11, 1_100);
|
||||
let newer_cycle = observation(7, 12, 150, 12, 1_200);
|
||||
let older_cycle = observation(7, 11, 300, 99, 9_900);
|
||||
let newer_epoch = observation(8, 1, 50, 1, 100);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
for (observed, expected) in [
|
||||
(&first, &first),
|
||||
(&newer_cycle, &newer_cycle),
|
||||
(&older_cycle, &newer_cycle),
|
||||
(&newer_epoch, &newer_epoch),
|
||||
] {
|
||||
assert!(
|
||||
observed_data_usage_is_newer(observed, &authoritative),
|
||||
"the persisted observation must pass the real loader's selection gate"
|
||||
);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(observed).expect("encode scanner observation"),
|
||||
)
|
||||
.await
|
||||
.expect("persist scanner observation");
|
||||
if let Some(cached) = admin_data_usage_snapshot_cache().write().await.as_mut() {
|
||||
cached.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
}
|
||||
let loaded = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("refresh admin usage through the real backend loader");
|
||||
assert_eq!(
|
||||
&loaded,
|
||||
expected,
|
||||
"same-baseline observation ordering must follow epoch/cycle before wall clock: incoming={:?}",
|
||||
observed.snapshot_identity()
|
||||
);
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_usage_snapshot_refresh_accepts_new_authority_over_a_later_old_observation() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let baseline = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
|
||||
let mut observed = data_usage_info_for_test("bucket", 5, 60, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
|
||||
observed.usage_snapshot_converged = Some(false);
|
||||
observed.usage_snapshot_authoritative_baseline = Some(baseline.snapshot_identity());
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(observed),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
let after_delete = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
|
||||
let refreshed = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded_usage_for_test(after_delete.clone())),
|
||||
loaded_at,
|
||||
1,
|
||||
1,
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("matching refresh generation")
|
||||
.expect("a newer authority must retire observations of its predecessor");
|
||||
assert_eq!(refreshed, after_delete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_usage_snapshot_refresh_orders_lineage_by_epoch_cycle_then_time() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
for (cached_position, loaded_position, loaded_time, accept) in [
|
||||
((Some(2), Some(10)), (Some(1), Some(99)), 110, false),
|
||||
((Some(2), Some(10)), (Some(2), Some(9)), 110, false),
|
||||
((Some(2), Some(10)), (Some(3), Some(1)), 90, true),
|
||||
((Some(2), Some(10)), (Some(2), Some(11)), 90, true),
|
||||
((Some(2), Some(10)), (Some(2), Some(10)), 110, true),
|
||||
((Some(3), None), (Some(2), Some(10)), 110, false),
|
||||
((Some(2), Some(10)), (Some(3), None), 90, true),
|
||||
((None, None), (None, None), 110, true),
|
||||
((None, None), (None, None), 90, false),
|
||||
] {
|
||||
let mut previous = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
|
||||
(previous.scanner_epoch, previous.scanner_cycle) = cached_position;
|
||||
let mut loaded = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(loaded_time));
|
||||
(loaded.scanner_epoch, loaded.scanner_cycle) = loaded_position;
|
||||
let expected = if accept { &loaded } else { &previous };
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(previous.clone()),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
let refreshed = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded_usage_for_test(loaded.clone())),
|
||||
loaded_at,
|
||||
1,
|
||||
1,
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("matching refresh generation")
|
||||
.expect("a valid usage snapshot remains available");
|
||||
assert_eq!(
|
||||
&refreshed, expected,
|
||||
"lineage ordering must match scanner publication: cached={cached_position:?}, loaded={loaded_position:?}, loaded_time={loaded_time}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_usage_snapshot_refresh_preserves_rejection_backoff_and_baseline() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(expected.clone()),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::from([("bucket".to_string(), 42)]),
|
||||
});
|
||||
let refreshed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
let refreshed = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
|
||||
refreshed_at,
|
||||
1,
|
||||
1,
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("matching refresh generation")
|
||||
.expect("the newer cached snapshot remains available");
|
||||
assert_eq!(refreshed, expected);
|
||||
let retained = cache.as_ref().expect("cache should remain populated");
|
||||
assert_eq!(retained.degraded_baseline.get("bucket"), Some(&42));
|
||||
assert_eq!(retained.loaded_at, refreshed_at, "an older fallback must retain the TTL backoff");
|
||||
|
||||
let result =
|
||||
cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), refreshed_at, 1, 1, DataUsageInfo::clone)
|
||||
.expect("matching refresh generation");
|
||||
assert!(matches!(result, Err(Error::ErasureReadQuorum)));
|
||||
let failed = cache.as_ref().expect("negative cache should be populated");
|
||||
assert!(failed.last_load_failed);
|
||||
assert_eq!(failed.info.as_ref(), Some(&expected), "failure must retain the last good ordering proof");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_usage_snapshot_refresh_accepts_clock_recovery() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let now = SystemTime::now();
|
||||
let future = now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(60);
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("bucket", 3, 42, future)),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
let recovered = data_usage_info_for_test("bucket", 1, 7, now);
|
||||
let refreshed = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded_usage_for_test(recovered.clone())),
|
||||
loaded_at,
|
||||
1,
|
||||
1,
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("matching refresh generation")
|
||||
.expect("a clock recovery snapshot must not be frozen behind an untrusted timestamp");
|
||||
assert_eq!(refreshed, recovered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_usage_snapshot_refresh_keeps_last_good_lineage_across_a_failed_load() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let current = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
|
||||
let mut cache = None;
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded_usage_for_test(current.clone())),
|
||||
loaded_at,
|
||||
1,
|
||||
1,
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("matching refresh generation")
|
||||
.expect("prime the last successful snapshot");
|
||||
assert_eq!(first, current);
|
||||
|
||||
let failed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
let failure =
|
||||
cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), failed_at, 1, 1, DataUsageInfo::clone)
|
||||
.expect("matching refresh generation");
|
||||
assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
|
||||
assert!(
|
||||
fresh_cached_data_usage_snapshot(
|
||||
&cache,
|
||||
failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
|
||||
Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS),
|
||||
|_| panic!("a failed load must not serve its retained snapshot as a success"),
|
||||
)
|
||||
.expect("the failed load remains inside its negative TTL")
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
!unknown_bucket_reload_due(
|
||||
&cache,
|
||||
"unknown",
|
||||
failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
|
||||
),
|
||||
"an unknown bucket must not bypass the failed load's TTL"
|
||||
);
|
||||
|
||||
let recovered_at = failed_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
let recovered = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
|
||||
recovered_at,
|
||||
1,
|
||||
1,
|
||||
DataUsageInfo::clone,
|
||||
)
|
||||
.expect("matching refresh generation")
|
||||
.expect("a readable older backup must preserve the last good lineage");
|
||||
assert_eq!(recovered, current);
|
||||
assert_eq!(
|
||||
cache.as_ref().expect("recovered cache").degraded_baseline.get("bucket"),
|
||||
Some(&42),
|
||||
"a failed load must not erase the baseline's ordering proof"
|
||||
);
|
||||
assert_eq!(
|
||||
fresh_cached_data_usage_snapshot(&cache, recovered_at, Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS), |info| info
|
||||
.objects_total_size,)
|
||||
.expect("the recovered load starts a fresh TTL")
|
||||
.expect("the last successful snapshot is available again"),
|
||||
42
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn unavailable_observed_read_keeps_previous_baseline_coverage() {
|
||||
@@ -5094,7 +4707,6 @@ mod tests {
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
|
||||
});
|
||||
@@ -5128,7 +4740,6 @@ mod tests {
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
@@ -6317,7 +5928,6 @@ mod tests {
|
||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||
*snapshot_cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(successor),
|
||||
last_load_failed: false,
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
@@ -6389,7 +5999,6 @@ mod tests {
|
||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||
*snapshot_cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(successor),
|
||||
last_load_failed: false,
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
@@ -6442,7 +6051,6 @@ mod tests {
|
||||
let stale = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
|
||||
*data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
|
||||
info: Some(stale),
|
||||
last_load_failed: false,
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
|
||||
@@ -21,7 +21,6 @@ fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
let cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
@@ -67,7 +66,6 @@ fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
|
||||
|
||||
cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
last_load_failed: false,
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
@@ -129,33 +127,6 @@ fn projected_publication_moves_bucket_maps_into_cache() {
|
||||
cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
|
||||
original_key
|
||||
);
|
||||
|
||||
let failure = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Err(Error::ErasureReadQuorum),
|
||||
tokio::time::Instant::now(),
|
||||
1,
|
||||
1,
|
||||
|_: &DataUsageInfo| -> u64 { panic!("a failed load must not project its retained snapshot") },
|
||||
)
|
||||
.expect("matching refresh generation");
|
||||
assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
|
||||
let retained = cache
|
||||
.as_ref()
|
||||
.expect("negative cache")
|
||||
.info
|
||||
.as_ref()
|
||||
.expect("last good ordering proof");
|
||||
assert_eq!(
|
||||
retained
|
||||
.buckets_usage
|
||||
.get_key_value("known")
|
||||
.expect("retained bucket")
|
||||
.0
|
||||
.as_ptr(),
|
||||
original_key,
|
||||
"a failed load must retain the snapshot without copying its bucket maps"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -167,16 +167,6 @@ impl ObjectLockAttempt {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Correlate a raw guard's acquisition with its peer identities without
|
||||
/// claiming ownership of the guard's lifetime. Peer release events retain
|
||||
/// the identities needed to reconstruct when the actual owner releases it.
|
||||
pub(crate) fn observe_acquisition(&mut self, result: &Result<NamespaceLockGuard, LockError>) {
|
||||
self.observe(result);
|
||||
if let Some(context) = self.0.as_mut() {
|
||||
context.finished = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn duration_ms(duration: Duration) -> u64 {
|
||||
@@ -510,45 +500,6 @@ pub(crate) mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn raw_guard_acquisition_diagnostic_does_not_claim_guard_lifetime() {
|
||||
temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
|
||||
let capture = CapturedLockEvents::default();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("raw guard diagnostic runtime");
|
||||
tracing::subscriber::with_default(capture.subscriber(), || {
|
||||
runtime.block_on(async {
|
||||
let lock = namespace();
|
||||
let mut attempt = ObjectLockAttempt::start(
|
||||
"bucket_incarnation_disk_read",
|
||||
"bucket",
|
||||
"transaction.lock",
|
||||
None,
|
||||
&lock,
|
||||
"read",
|
||||
Duration::from_secs(1),
|
||||
);
|
||||
let result = lock.get_read_lock(Duration::from_secs(1)).await;
|
||||
attempt.observe_acquisition(&result);
|
||||
drop(attempt);
|
||||
let guard = result.expect("retain the raw guard after diagnostic completion");
|
||||
assert!(!guard.is_lock_lost());
|
||||
let states = capture
|
||||
.rows()
|
||||
.iter()
|
||||
.map(|row| row["fields"]["state"].as_str().expect("diagnostic state string").to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(states, ["acquiring", "acquired"]);
|
||||
drop(guard);
|
||||
assert_eq!(capture.rows().len(), 2, "the acquisition observer must not invent a guard release");
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn object_lock_diagnostics_disabled_does_not_allocate_context_or_emit() {
|
||||
|
||||
@@ -148,58 +148,47 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
native_io(move || create_dir_all_std(&path)).await
|
||||
}
|
||||
|
||||
fn is_dir_error(path: &Path, e: &io::Error) -> bool {
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
// macOS: remove_file on a directory returns EPERM
|
||||
|| (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
|
||||
|| is_windows_dir_access_denied(path, e)
|
||||
}
|
||||
|
||||
/// Windows reports `remove_file` on a directory (or directory symlink) as
|
||||
/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
|
||||
/// failure without inspecting the entry itself.
|
||||
#[cfg(windows)]
|
||||
fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
|
||||
use std::os::windows::fs::FileTypeExt;
|
||||
|
||||
e.kind() == io::ErrorKind::PermissionDenied
|
||||
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| {
|
||||
let file_type = metadata.file_type();
|
||||
file_type.is_dir() || file_type.is_symlink_dir()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || remove_std(path)).await
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || match std::fs::remove_file(&path) {
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir_all(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
let path = path.as_ref();
|
||||
match std::fs::remove_file(path) {
|
||||
match std::fs::remove_file(path.as_ref()) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
|
||||
Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9142,15 +9142,11 @@ impl LocalDisk {
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
let payload = super::GuardedDeleteVersionsPayload {
|
||||
versions,
|
||||
opts,
|
||||
bounded_recovery_protocol: None,
|
||||
};
|
||||
let payload = super::GuardedDeleteVersionsPayload { versions, opts };
|
||||
if let Err(error) = payload.validate() {
|
||||
return vec![Some(error); payload.versions.len()];
|
||||
}
|
||||
let super::GuardedDeleteVersionsPayload { versions, mut opts, .. } = payload;
|
||||
let super::GuardedDeleteVersionsPayload { versions, mut opts } = payload;
|
||||
opts.recoverable_delete = true;
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for version in &versions {
|
||||
@@ -9993,7 +9989,7 @@ impl LocalDisk {
|
||||
if !std::fs::symlink_metadata(&receipt_path)?.is_file() {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidData, "delete receipt is not a regular file"));
|
||||
}
|
||||
os::open_file_for_sync_std(&receipt_path)?
|
||||
std::fs::File::open(receipt_path)?
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
@@ -14514,7 +14510,7 @@ impl DiskAPI for LocalDisk {
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let verify_and_sync = (|| -> std::io::Result<bool> {
|
||||
let mut file = os::open_file_for_sync_std(&source)?;
|
||||
let mut file = std::fs::File::open(&source)?;
|
||||
let mut header = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
|
||||
std::io::Read::read_exact(&mut file, &mut header)?;
|
||||
if header != integrity.index_header() {
|
||||
@@ -19755,9 +19751,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
// Only the Unix grouped dst dir fsync worker consults the injected
|
||||
// failure; the non-Unix worker syncs by path without the hook.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn rename_data_non_inline_dst_dir_fsync_group_commit_failure_rolls_back_fresh_put() {
|
||||
@@ -19846,9 +19839,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
// Only the Unix grouped dst dir fsync worker consults the injected
|
||||
// failure; the non-Unix worker syncs by path without the hook.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn rename_data_inline_dst_dir_fsync_group_commit_failure_rolls_back_fresh_put() {
|
||||
@@ -30454,7 +30444,6 @@ mod test {
|
||||
expected_delete_marker: None,
|
||||
delete_recovery_bucket_incarnation: None,
|
||||
recoverable_delete: false,
|
||||
requires_bounded_delete_recovery: false,
|
||||
orphan_data_dir: false,
|
||||
};
|
||||
disk.delete("test-volume", "test-file.txt", delete_opts)
|
||||
|
||||
@@ -200,7 +200,7 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
os::open_file_for_sync_std(&backup_path)?.sync_data()?;
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -1658,13 +1658,12 @@ impl LocalDisk {
|
||||
);
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
#[cfg(target_os = "linux")]
|
||||
let payload_guard = if multipart_publication.is_some() {
|
||||
Some(
|
||||
os::multipart_payload_commit_guard(
|
||||
&commit.directory_guard,
|
||||
src_data_path.parent().ok_or(DiskError::InvalidPath)?,
|
||||
commit.publication_root,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1673,9 +1672,9 @@ impl LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
#[cfg(target_os = "linux")]
|
||||
let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let directory_guard = &commit.directory_guard;
|
||||
if let Err(err) = os::rename_all_with_commit_guard(
|
||||
src_data_path,
|
||||
|
||||
@@ -1517,7 +1517,7 @@ impl LocalDisk {
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
if !parts.is_empty() {
|
||||
let anchor_path = directory.join(RETIREMENT_ANCHOR);
|
||||
let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
|
||||
@@ -1550,13 +1550,7 @@ impl LocalDisk {
|
||||
}
|
||||
// An anchor-less empty DD is only rmdir'ed. A new child makes
|
||||
// this fail closed; no recursive removal can consume its bytes.
|
||||
// A Windows pin refuses DELETE sharing, so the leaf pin is
|
||||
// released first; the object pin still holds the parent.
|
||||
#[cfg(windows)]
|
||||
drop(directory_pin);
|
||||
std::fs::remove_dir(&directory)?;
|
||||
#[cfg(not(windows))]
|
||||
drop(directory_pin);
|
||||
if sync {
|
||||
os::fsync_dir_std(&object_path)?;
|
||||
}
|
||||
|
||||
@@ -2125,19 +2125,10 @@ pub struct GuardedFileInfoVersions {
|
||||
pub struct GuardedDeleteVersionsPayload {
|
||||
pub versions: Vec<GuardedFileInfoVersions>,
|
||||
pub opts: DeleteOptions,
|
||||
// The strict legacy decoder rejects this field before applying any change.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bounded_recovery_protocol: Option<u32>,
|
||||
}
|
||||
|
||||
impl GuardedDeleteVersionsPayload {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self
|
||||
.bounded_recovery_protocol
|
||||
.is_some_and(|version| version != BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION)
|
||||
{
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
validate_delete_recovery_options(&self.opts)?;
|
||||
if self.versions.is_empty()
|
||||
|| self.versions.len() > 1000
|
||||
@@ -2315,9 +2306,6 @@ pub struct DeleteOptions {
|
||||
pub delete_recovery_bucket_incarnation: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub recoverable_delete: bool,
|
||||
/// Transport requirement for preparing a delete that will use bounded settlement.
|
||||
#[serde(skip)]
|
||||
pub requires_bounded_delete_recovery: bool,
|
||||
/// Reclaim only plain erasure-part leaves, preserving every other entry.
|
||||
/// Keep `recursive` false so an older peer cannot widen this operation.
|
||||
#[serde(default)]
|
||||
@@ -2325,8 +2313,6 @@ pub struct DeleteOptions {
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1;
|
||||
/// Accepted only for prepare RPCs. Legacy peers must refuse before staging a delete.
|
||||
pub const BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION: u32 = 2;
|
||||
pub(crate) fn get_delete_recovery_timeout() -> Result<std::time::Duration> {
|
||||
let value = std::env::var("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS");
|
||||
match value {
|
||||
@@ -2923,7 +2909,6 @@ mod tests {
|
||||
insert_str(&mut fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32));
|
||||
let generation = CurrentDeleteGeneration::from_file_info(&fi).expect("capture complete remote identity");
|
||||
let payload = GuardedDeleteVersionsPayload {
|
||||
bounded_recovery_protocol: None,
|
||||
versions: vec![GuardedFileInfoVersions {
|
||||
group: FileInfoVersions {
|
||||
name: "object".into(),
|
||||
@@ -2978,7 +2963,6 @@ mod tests {
|
||||
expected_current: CurrentDeleteGeneration::Absent,
|
||||
};
|
||||
let mut payload = GuardedDeleteVersionsPayload {
|
||||
bounded_recovery_protocol: None,
|
||||
versions: vec![group],
|
||||
opts: DeleteOptions {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
@@ -2990,25 +2974,6 @@ mod tests {
|
||||
let decoded: GuardedDeleteVersionsPayload =
|
||||
rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode guarded batch");
|
||||
decoded.validate().expect("round-trip guarded batch remains valid");
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LegacyGuardedDeletePayload {
|
||||
#[serde(rename = "versions")]
|
||||
_versions: Vec<GuardedFileInfoVersions>,
|
||||
#[serde(rename = "opts")]
|
||||
_opts: DeleteOptions,
|
||||
}
|
||||
rustfs_protos::decode_delete_recovery_payload::<LegacyGuardedDeletePayload>(&encoded)
|
||||
.expect("legacy reader accepts legacy prepare");
|
||||
payload.bounded_recovery_protocol = Some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION);
|
||||
let bounded = rmp_serde::to_vec_named(&payload).expect("serialize bounded guarded prepare");
|
||||
assert!(rustfs_protos::decode_delete_recovery_payload::<LegacyGuardedDeletePayload>(&bounded).is_err());
|
||||
let bounded: GuardedDeleteVersionsPayload =
|
||||
rustfs_protos::decode_delete_recovery_payload(&bounded).expect("current reader accepts bounded prepare");
|
||||
bounded.validate().expect("valid bounded guarded prepare");
|
||||
payload.bounded_recovery_protocol = Some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION + 1);
|
||||
assert_eq!(payload.validate(), Err(DiskError::UnsupportedDisk));
|
||||
payload.bounded_recovery_protocol = None;
|
||||
let mut trailing = encoded;
|
||||
trailing.push(0xc0);
|
||||
assert!(rustfs_protos::decode_delete_recovery_payload::<GuardedDeleteVersionsPayload>(&trailing).is_err());
|
||||
@@ -3203,15 +3168,9 @@ mod tests {
|
||||
expected_delete_marker: None,
|
||||
delete_recovery_bucket_incarnation: None,
|
||||
recoverable_delete: false,
|
||||
requires_bounded_delete_recovery: true,
|
||||
orphan_data_dir: false,
|
||||
};
|
||||
|
||||
let wire = serde_json::to_value(&opts).expect("encode legacy wire options");
|
||||
assert!(wire.get("requires_bounded_delete_recovery").is_none());
|
||||
let decoded: DeleteOptions = serde_json::from_value(wire).expect("decode legacy wire options");
|
||||
assert!(!decoded.requires_bounded_delete_recovery);
|
||||
|
||||
assert!(opts.recursive);
|
||||
assert!(!opts.immediate);
|
||||
assert!(opts.undo_write);
|
||||
|
||||
@@ -1149,8 +1149,6 @@ fn normalized_info(fi: &FileInfo, completion: bool) -> FileInfo {
|
||||
part.checksums = None;
|
||||
part.index = None;
|
||||
part.integrity = None;
|
||||
// The completed object carries the aggregate proof in metadata.
|
||||
part.encrypted_etag = false;
|
||||
}
|
||||
}
|
||||
fi
|
||||
@@ -1741,12 +1739,6 @@ mod multipart_proof_authority_tests {
|
||||
completed.mod_time = Some(original.mod_time.unwrap() + time::Duration::seconds(1));
|
||||
completed.size = 15;
|
||||
completed.add_object_part(1, "etag".into(), 15, completed.mod_time, 15, None, None);
|
||||
completed.parts[0].encrypted_etag = true;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut completed.metadata,
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
|
||||
"v1".to_owned(),
|
||||
);
|
||||
let publication = condition.attach(&mut completed).unwrap();
|
||||
assert_eq!(publication.version_id, None);
|
||||
assert_eq!(
|
||||
@@ -1756,11 +1748,6 @@ mod multipart_proof_authority_tests {
|
||||
);
|
||||
let mut read_completed = persist_null_fixture(&completed);
|
||||
assert_eq!(read_completed.version_id, None);
|
||||
assert!(!read_completed.parts[0].encrypted_etag, "xl.meta stores the aggregate proof");
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(&read_completed.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG),
|
||||
Some("v1")
|
||||
);
|
||||
let reread_publication = reread_condition.attach(&mut read_completed).unwrap();
|
||||
assert_eq!(
|
||||
publication, reread_publication,
|
||||
|
||||
+135
-466
@@ -610,26 +610,6 @@ pub(crate) mod prepare_rename_test_hooks {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
/// Hooks keyed by a directory component name, run before a Windows
|
||||
/// directory pin retries a transient sharing conflict on it.
|
||||
#[cfg(windows)]
|
||||
static BEFORE_COMPONENT_RETRY: LazyLock<Mutex<HashMap<std::ffi::OsString, Hook>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn install_before_component_retry(component: &std::ffi::OsStr, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_COMPONENT_RETRY.lock().insert(component.to_os_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn run_before_component_retry(component: &std::ffi::OsStr) {
|
||||
let hook = BEFORE_COMPONENT_RETRY.lock().remove(component);
|
||||
if let Some(hook) = hook {
|
||||
let _pause = super::enter_blocking_test_hook();
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fsync a directory so recently created or renamed entries survive power loss.
|
||||
@@ -664,16 +644,6 @@ pub(crate) fn fsync_directory_handle_std(dir: &Path, opened: Option<&std::fs::Fi
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open an existing file to read it and flush it. Windows flushes only a
|
||||
/// handle with write access, so the file is opened read-write there; other
|
||||
/// platforms keep the read-only open that `sync_all` already accepts.
|
||||
pub(crate) fn open_file_for_sync_std(path: &Path) -> io::Result<std::fs::File> {
|
||||
#[cfg(windows)]
|
||||
return std::fs::OpenOptions::new().read(true).write(true).open(path);
|
||||
#[cfg(not(windows))]
|
||||
std::fs::File::open(path)
|
||||
}
|
||||
|
||||
/// Async wrapper around [`fsync_dir_std`]; runs the blocking Unix fsync off the runtime.
|
||||
pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dir_with_owner(dir.as_ref(), None).await
|
||||
@@ -4614,7 +4584,7 @@ pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &
|
||||
let Some(source_parent) = src_file_path.parent() else {
|
||||
return false;
|
||||
};
|
||||
let source_parent_guard = match lock_windows_directory_tree(source_parent, publication_root) {
|
||||
let source_parent_guard = match lock_windows_directory_tree(source_parent, None, publication_root) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return err.kind() == io::ErrorKind::NotFound,
|
||||
};
|
||||
@@ -5053,7 +5023,7 @@ fn prepare_rename_with_retry(
|
||||
open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
|
||||
(source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity)
|
||||
} else {
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, publication_root)?;
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, destination_parent, publication_root)?;
|
||||
let (source_identity_anchor, expected_source_identity) =
|
||||
open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
|
||||
let parent_guard = prepare_destination_parent()?;
|
||||
@@ -5474,9 +5444,7 @@ impl PublicationRoot {
|
||||
pub(crate) fn open_readonly(&self, path: &Path) -> io::Result<std::fs::File> {
|
||||
#[cfg(target_os = "linux")]
|
||||
return self.open_relative(path, rustix::fs::OFlags::RDONLY, rustix::fs::Mode::empty());
|
||||
#[cfg(windows)]
|
||||
return open_windows_readonly(path);
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
std::fs::File::open(path)
|
||||
}
|
||||
|
||||
@@ -5940,11 +5908,14 @@ impl RenameCommitGuard {
|
||||
}
|
||||
|
||||
/// Reopen a destination tree for handle-relative child publication.
|
||||
/// Every component is pinned by `open_windows_directory_component`.
|
||||
/// Ancestors remain write-exclusive while the final parent shares writes
|
||||
/// required by the kernel's relative rename. Delete sharing stays omitted
|
||||
/// throughout, so every retained directory identity remains pinned.
|
||||
fn destination_directory_guard(&self, directory: &Path, create_missing: bool) -> io::Result<RenameDestinationPathGuard> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Wdk::Storage::FileSystem::{FILE_OPEN, FILE_OPEN_IF};
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
|
||||
let relative = directory.strip_prefix(&self.destination_parent).map_err(|_| {
|
||||
io::Error::new(
|
||||
@@ -5977,17 +5948,35 @@ impl RenameCommitGuard {
|
||||
let parent = handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows destination guard lost its parent handle"))?;
|
||||
handles.push(open_windows_directory_component(parent, component, FILE_OPEN)?);
|
||||
let relative_components = relative.components().filter_map(|component| match component {
|
||||
Component::Normal(component) => Some(component),
|
||||
_ => None,
|
||||
});
|
||||
for component in relative_components {
|
||||
let mut relative_components = relative
|
||||
.components()
|
||||
.filter_map(|component| match component {
|
||||
Component::Normal(component) => Some(component),
|
||||
_ => None,
|
||||
})
|
||||
.peekable();
|
||||
let destination_parent_share = if relative_components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
handles.push(open_windows_relative_directory_component(
|
||||
parent,
|
||||
component,
|
||||
FILE_OPEN,
|
||||
destination_parent_share,
|
||||
)?);
|
||||
while let Some(component) = relative_components.next() {
|
||||
let parent = handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows destination path guard lost its parent handle"))?;
|
||||
let disposition = if create_missing { FILE_OPEN_IF } else { FILE_OPEN };
|
||||
handles.push(open_windows_directory_component(parent, component, disposition)?);
|
||||
let share_access = if relative_components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
handles.push(open_windows_relative_directory_component(parent, component, disposition, share_access)?);
|
||||
}
|
||||
Ok(RenameDestinationPathGuard {
|
||||
directory: directory.to_path_buf(),
|
||||
@@ -6031,47 +6020,18 @@ pub(crate) fn multipart_payload_commit_guard_std(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn multipart_payload_commit_guard_std(
|
||||
metadata_guard: &RenameCommitGuard,
|
||||
source_parent: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
// Pin the payload staging tree like a distinct-tree metadata source; the
|
||||
// target stays the exact parent handle the metadata transaction holds.
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, publication_root)?;
|
||||
Ok(RenameCommitGuard {
|
||||
source_parent: source_parent.to_path_buf(),
|
||||
destination_parent: metadata_guard.destination_parent.clone(),
|
||||
source_parent_guard,
|
||||
destination_parent_guard: metadata_guard.destination_parent_guard.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Native MPU metadata and payload have distinct source parents. Reuse the
|
||||
/// exact target identity already held by the metadata transaction, while
|
||||
/// independently pinning the private payload source under the same root.
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) async fn multipart_payload_commit_guard(
|
||||
metadata_guard: &RenameCommitGuard,
|
||||
source_parent: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let metadata_guard = metadata_guard.clone();
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
#[cfg(target_os = "linux")]
|
||||
let operation = {
|
||||
// The Linux guard already carries its publication root.
|
||||
let _ = publication_root;
|
||||
move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)
|
||||
};
|
||||
#[cfg(windows)]
|
||||
let operation = {
|
||||
let publication_root = publication_root.clone();
|
||||
move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent, &publication_root)
|
||||
};
|
||||
run_blocking_namespace_operation(lease, operation).await
|
||||
run_blocking_namespace_operation(lease, move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)).await
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_rename_commit_guard(
|
||||
@@ -6113,16 +6073,20 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
// A same-directory rename reuses one guard for both sides. Every pin
|
||||
// shares writes and excludes delete; see
|
||||
// `open_windows_directory_component`.
|
||||
// A same-directory rename must use one shared-write parent handle:
|
||||
// retaining a second read-only-share handle would block the kernel's
|
||||
// relative target open. Delete sharing stays excluded, so identity is
|
||||
// still pinned. Distinct source trees remain strict.
|
||||
let same_parent = publication_root.relative_path(source_parent)? == publication_root.relative_path(destination_parent)?;
|
||||
let (source_parent_guard, destination_parent_guard) = if same_parent {
|
||||
let destination_parent_guard =
|
||||
mkdir_all_below_existing_base_std(destination_parent, destination_base, publication_root)?;
|
||||
(destination_parent_guard.clone(), destination_parent_guard)
|
||||
} else {
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, publication_root)?;
|
||||
// The source parent also hosts private rollback staging files.
|
||||
// Their handle-relative publication needs write sharing on this
|
||||
// final directory while delete sharing remains excluded.
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, Some(source_parent), publication_root)?;
|
||||
let destination_parent_guard =
|
||||
mkdir_all_below_existing_base_std(destination_parent, destination_base, publication_root)?;
|
||||
(source_parent_guard, destination_parent_guard)
|
||||
@@ -6142,28 +6106,6 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
}
|
||||
}
|
||||
|
||||
/// Unix opens a directory read-only and lets the reader classify it, while
|
||||
/// Windows refuses the plain open with ACCESS_DENIED. Reopen only an actual
|
||||
/// directory with backup semantics so the same `is_dir` checks apply.
|
||||
#[cfg(windows)]
|
||||
fn open_windows_readonly(path: &Path) -> io::Result<std::fs::File> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS;
|
||||
|
||||
match std::fs::File::open(path) {
|
||||
Err(error)
|
||||
if error.kind() == io::ErrorKind::PermissionDenied
|
||||
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir()) =>
|
||||
{
|
||||
std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
|
||||
.open(path)
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn open_windows_publication_root(path: &Path) -> io::Result<(PathBuf, WindowsDirectoryHandle)> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
@@ -6350,12 +6292,22 @@ fn windows_file_attribute_tag(
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn lock_windows_directory_tree(path: &Path, publication_root: &PublicationRoot) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
fn lock_windows_directory_tree(
|
||||
path: &Path,
|
||||
shared_write_ancestor: Option<&Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
use windows_sys::Wdk::Storage::FileSystem::FILE_OPEN;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
|
||||
let relative = publication_root.relative_path(path)?;
|
||||
let shared_write_relative = shared_write_ancestor
|
||||
.map(|ancestor| publication_root.relative_path(ancestor))
|
||||
.transpose()?
|
||||
.filter(|ancestor| relative.starts_with(*ancestor));
|
||||
let mut handles = Vec::with_capacity(relative.components().count().saturating_add(1));
|
||||
handles.push(publication_root.directory.clone());
|
||||
let mut opened_relative = PathBuf::new();
|
||||
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
@@ -6370,13 +6322,26 @@ fn lock_windows_directory_tree(path: &Path, publication_root: &PublicationRoot)
|
||||
let parent = handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows directory guard lost its root handle"))?;
|
||||
handles.push(open_windows_directory_component(parent, component, FILE_OPEN)?);
|
||||
opened_relative.push(component);
|
||||
let child = if shared_write_relative.is_some_and(|ancestor| opened_relative.as_path() == ancestor) {
|
||||
// A handle-relative rename opens the target parent for write. When
|
||||
// that parent is also a source ancestor, this source-side handle
|
||||
// must share write access or the transaction blocks itself. Delete
|
||||
// sharing remains omitted, so the directory identity stays pinned.
|
||||
open_windows_relative_directory_component(parent, component, FILE_OPEN, FILE_SHARE_READ | FILE_SHARE_WRITE)?
|
||||
} else {
|
||||
open_windows_directory_component(parent, component, FILE_OPEN)?
|
||||
};
|
||||
handles.push(child);
|
||||
}
|
||||
|
||||
Ok(ExistingBaseDirectoryGuard::new(handles))
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
// SAFETY: the object attributes borrow a checked UTF-16 component and live
|
||||
// parent handle for the duration of the synchronous NtCreateFile call.
|
||||
#[allow(unsafe_code)]
|
||||
fn open_windows_relative(
|
||||
parent: &winapi_util::Handle,
|
||||
component: &std::ffi::OsStr,
|
||||
@@ -6387,64 +6352,6 @@ fn open_windows_relative(
|
||||
file_attributes: u32,
|
||||
dont_reparse: bool,
|
||||
) -> io::Result<winapi_util::Handle> {
|
||||
open_windows_relative_with_status(
|
||||
parent,
|
||||
component,
|
||||
&WindowsRelativeOpen {
|
||||
desired_access,
|
||||
share_access,
|
||||
create_disposition,
|
||||
create_options,
|
||||
file_attributes,
|
||||
dont_reparse,
|
||||
},
|
||||
)
|
||||
.map_err(|failure| failure.error)
|
||||
}
|
||||
|
||||
/// `NtCreateFile` arguments for one handle-relative open.
|
||||
#[cfg(windows)]
|
||||
struct WindowsRelativeOpen {
|
||||
desired_access: u32,
|
||||
share_access: u32,
|
||||
create_disposition: u32,
|
||||
create_options: u32,
|
||||
file_attributes: u32,
|
||||
dont_reparse: bool,
|
||||
}
|
||||
|
||||
/// A failed relative open that keeps the NTSTATUS, which tells transient
|
||||
/// states such as a pending delete apart from errors that share its Win32 code.
|
||||
#[cfg(windows)]
|
||||
struct WindowsRelativeOpenError {
|
||||
status: Option<windows_sys::Win32::Foundation::NTSTATUS>,
|
||||
error: io::Error,
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
impl From<io::Error> for WindowsRelativeOpenError {
|
||||
fn from(error: io::Error) -> Self {
|
||||
Self { status: None, error }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
// SAFETY: the object attributes borrow a checked UTF-16 component and live
|
||||
// parent handle for the duration of the synchronous NtCreateFile call.
|
||||
#[allow(unsafe_code)]
|
||||
fn open_windows_relative_with_status(
|
||||
parent: &winapi_util::Handle,
|
||||
component: &std::ffi::OsStr,
|
||||
request: &WindowsRelativeOpen,
|
||||
) -> std::result::Result<winapi_util::Handle, WindowsRelativeOpenError> {
|
||||
let &WindowsRelativeOpen {
|
||||
desired_access,
|
||||
share_access,
|
||||
create_disposition,
|
||||
create_options,
|
||||
file_attributes,
|
||||
dont_reparse,
|
||||
} = request;
|
||||
use std::{
|
||||
mem::size_of,
|
||||
os::windows::{ffi::OsStrExt, io::AsRawHandle, io::FromRawHandle},
|
||||
@@ -6459,7 +6366,10 @@ fn open_windows_relative_with_status(
|
||||
|
||||
let mut name = component.encode_wide().collect::<Vec<_>>();
|
||||
if name.is_empty() || name.contains(&0) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "relative Windows file name is empty or contains a NUL").into());
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"relative Windows file name is empty or contains a NUL",
|
||||
));
|
||||
}
|
||||
let name_bytes = name
|
||||
.len()
|
||||
@@ -6499,17 +6409,13 @@ fn open_windows_relative_with_status(
|
||||
)
|
||||
};
|
||||
if status < 0 {
|
||||
let error = match i32::try_from(unsafe { RtlNtStatusToDosError(status) }) {
|
||||
Ok(code) => io::Error::from_raw_os_error(code),
|
||||
Err(_) => io::Error::other(format!("Windows relative open failed with NTSTATUS {status:#x}")),
|
||||
return match i32::try_from(unsafe { RtlNtStatusToDosError(status) }) {
|
||||
Ok(code) => Err(io::Error::from_raw_os_error(code)),
|
||||
Err(_) => Err(io::Error::other(format!("Windows relative open failed with NTSTATUS {status:#x}"))),
|
||||
};
|
||||
return Err(WindowsRelativeOpenError {
|
||||
status: Some(status),
|
||||
error,
|
||||
});
|
||||
}
|
||||
if handle.is_null() {
|
||||
return Err(io::Error::other("Windows relative open returned an invalid handle").into());
|
||||
return Err(io::Error::other("Windows relative open returned an invalid handle"));
|
||||
}
|
||||
|
||||
Ok(unsafe { winapi_util::Handle::from_raw_handle(handle) })
|
||||
@@ -6744,7 +6650,7 @@ pub(crate) fn compare_and_update_control_file(
|
||||
let parent = file_path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
|
||||
let parent_guard = lock_windows_directory_tree(parent, publication_root)?;
|
||||
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
|
||||
let lock = open_windows_relative(
|
||||
parent_guard.last_handle()?,
|
||||
std::ffi::OsStr::new(".rustfs-cas.lock"),
|
||||
@@ -6808,78 +6714,41 @@ pub(crate) fn compare_and_update_control_file(
|
||||
Ok(ConditionalFileUpdate::Updated)
|
||||
}
|
||||
|
||||
/// How often a directory pin retries a transient delete-access conflict.
|
||||
/// Six doubling delays from 1 ms wait about 63 ms in total, which outlasts a
|
||||
/// prune whose thread is preempted while it holds delete access.
|
||||
#[cfg(windows)]
|
||||
const WINDOWS_DIRECTORY_PIN_RETRIES: u32 = 6;
|
||||
#[cfg(windows)]
|
||||
const WINDOWS_DIRECTORY_PIN_FIRST_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(1);
|
||||
|
||||
/// Pin one directory component of a guarded path.
|
||||
///
|
||||
/// The handle shares reads and writes but never delete: renaming or removing
|
||||
/// a directory needs delete access to it, so omitting delete sharing pins its
|
||||
/// identity. Write sharing must stay granted on every component, ancestors
|
||||
/// included, because the kernel opens the target parent for write during a
|
||||
/// rename. A directory pinned here as one object's ancestor is also the
|
||||
/// publication parent of a nested object (`a/b` holds `a/b/c`), and a
|
||||
/// read-only share would make that unrelated rename fail with
|
||||
/// ERROR_SHARING_VIOLATION. Write sharing does not unpin an ancestor: a
|
||||
/// pinned ancestor always contains its pinned child, and NTFS sets a
|
||||
/// name-surrogate reparse point only on an empty directory.
|
||||
///
|
||||
/// Excluding delete also means the open fails while another handle holds
|
||||
/// delete access. Pruning an empty prefix directory takes that access only
|
||||
/// briefly: `RemoveDirectoryW` opens the directory for delete and closes it
|
||||
/// again, whether or not the directory turns out to be empty. A sharing
|
||||
/// violation or a pending delete is therefore retried a bounded number of
|
||||
/// times. After a removal finishes, the retry sees the directory as missing
|
||||
/// and follows `create_disposition`, so `FILE_OPEN` still reports it missing
|
||||
/// and `FILE_OPEN_IF` creates it again.
|
||||
#[cfg(windows)]
|
||||
fn open_windows_directory_component(
|
||||
parent: &WindowsDirectoryHandle,
|
||||
component: &std::ffi::OsStr,
|
||||
create_disposition: u32,
|
||||
) -> io::Result<WindowsDirectoryHandle> {
|
||||
use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ;
|
||||
|
||||
open_windows_relative_directory_component(parent, component, create_disposition, FILE_SHARE_READ)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn open_windows_relative_directory_component(
|
||||
parent: &WindowsDirectoryHandle,
|
||||
component: &std::ffi::OsStr,
|
||||
create_disposition: u32,
|
||||
share_access: u32,
|
||||
) -> io::Result<WindowsDirectoryHandle> {
|
||||
use windows_sys::{
|
||||
Wdk::Storage::FileSystem::{FILE_DIRECTORY_FILE, FILE_OPEN_REPARSE_POINT},
|
||||
Win32::{
|
||||
Foundation::{STATUS_DELETE_PENDING, STATUS_SHARING_VIOLATION},
|
||||
Storage::FileSystem::{
|
||||
FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE,
|
||||
FILE_TRAVERSE,
|
||||
},
|
||||
Win32::Storage::FileSystem::{
|
||||
FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_TRAVERSE,
|
||||
},
|
||||
};
|
||||
|
||||
let request = WindowsRelativeOpen {
|
||||
desired_access: FILE_TRAVERSE | FILE_READ_ATTRIBUTES,
|
||||
share_access: FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
let anchor = open_windows_relative(
|
||||
&parent.handle,
|
||||
component,
|
||||
FILE_TRAVERSE | FILE_READ_ATTRIBUTES,
|
||||
share_access,
|
||||
create_disposition,
|
||||
create_options: FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT,
|
||||
file_attributes: FILE_ATTRIBUTE_DIRECTORY,
|
||||
dont_reparse: true,
|
||||
};
|
||||
let mut retry_delay = WINDOWS_DIRECTORY_PIN_FIRST_RETRY_DELAY;
|
||||
let mut retries = 0;
|
||||
let anchor = loop {
|
||||
match open_windows_relative_with_status(&parent.handle, component, &request) {
|
||||
Ok(anchor) => break anchor,
|
||||
Err(WindowsRelativeOpenError {
|
||||
status: Some(STATUS_SHARING_VIOLATION | STATUS_DELETE_PENDING),
|
||||
..
|
||||
}) if retries < WINDOWS_DIRECTORY_PIN_RETRIES => {
|
||||
#[cfg(test)]
|
||||
prepare_rename_test_hooks::run_before_component_retry(component);
|
||||
std::thread::sleep(retry_delay);
|
||||
retry_delay = retry_delay.saturating_mul(2);
|
||||
retries += 1;
|
||||
}
|
||||
Err(failure) => return Err(failure.error),
|
||||
}
|
||||
};
|
||||
FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT,
|
||||
FILE_ATTRIBUTE_DIRECTORY,
|
||||
true,
|
||||
)?;
|
||||
let info = windows_file_attribute_tag(&anchor)?;
|
||||
if info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY == 0 {
|
||||
return Err(io::Error::from(io::ErrorKind::NotADirectory));
|
||||
@@ -7183,7 +7052,7 @@ pub(crate) fn pin_multipart_source_directory(
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let _ = base;
|
||||
lock_windows_directory_tree(directory, publication_root)
|
||||
lock_windows_directory_tree(directory, Some(directory), publication_root)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
@@ -7273,31 +7142,12 @@ pub(crate) fn rename_multipart_source_owned_std(
|
||||
{
|
||||
let _ = &guard;
|
||||
let source_guard = pin_multipart_source_directory(source_parent, base, root)?;
|
||||
// The prepared Windows source holds DELETE access and shares only
|
||||
// read, so the precondition's own opens of that entry (or, for a
|
||||
// directory, of its guarded tree) would fail with a sharing violation.
|
||||
// Check it first under a non-exclusive identity anchor, then require
|
||||
// the exclusive handle to name the same file.
|
||||
#[cfg(windows)]
|
||||
let (_source_identity_anchor, expected_source_identity) = {
|
||||
let anchor = open_windows_rename_source_identity(source, &source_guard)?;
|
||||
precondition()?;
|
||||
anchor
|
||||
};
|
||||
let preparation = prepare_rename_with_retry(source, destination, base, root)?;
|
||||
#[cfg(windows)]
|
||||
if windows_file_identity(&preparation.source)? != expected_source_identity {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"rename source identity changed after its precondition",
|
||||
));
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
precondition()?;
|
||||
#[cfg(unix)]
|
||||
validate_prepared_rename_source(&pinned_source, source)?;
|
||||
@@ -7703,15 +7553,12 @@ fn read_multipart_metadata(
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
|
||||
// A read must not recreate a retired upload: pin the existing parent
|
||||
// instead of taking the write-side guard, which creates missing parents.
|
||||
existing_base_directory_suffix(parent, base)?;
|
||||
let guard = pin_multipart_source_directory(parent, base, publication_root)?;
|
||||
let guard = prepare_rename_commit_guard(parent, parent, base, publication_root)?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
|
||||
let mut file = open_windows_relative(
|
||||
guard.last_handle()?,
|
||||
guard.source_parent_guard.last_handle()?,
|
||||
name,
|
||||
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_READ_DATA,
|
||||
FILE_SHARE_READ,
|
||||
@@ -7887,7 +7734,7 @@ mod multipart_source_preservation_tests {
|
||||
let metadata_guard =
|
||||
prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
|
||||
let lease = acquire_namespace_mutation_lease(&target).await;
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, lease.clone())
|
||||
.await
|
||||
.expect("distinct pinned payload source");
|
||||
assert!(
|
||||
@@ -7922,61 +7769,6 @@ mod multipart_source_preservation_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_multipart_payload_guard_publishes_distinct_source_into_metadata_target() {
|
||||
// Build paths from the configured root: the resolved handle path may
|
||||
// be verbatim, where `/` in a joined key is not a separator.
|
||||
let (root, publication, _source, _destination) = fixture();
|
||||
let root = root.path();
|
||||
let canonical = root.join("multipart/upload");
|
||||
let temporary = root.join("tmp/attempt");
|
||||
let bucket = root.join("bucket");
|
||||
let target = bucket.join("object");
|
||||
for directory in [&canonical, &temporary, &target] {
|
||||
std::fs::create_dir_all(directory).expect("two-source guard directories");
|
||||
}
|
||||
let payload = temporary.join("payload");
|
||||
std::fs::create_dir(&payload).expect("private payload DD");
|
||||
std::fs::write(payload.join("part.1"), b"native selected shard").expect("private payload shard");
|
||||
let head = canonical.join("xl.meta");
|
||||
let destination = target.join("xl.meta");
|
||||
let metadata_guard =
|
||||
prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
|
||||
let lease = acquire_namespace_mutation_lease(&target).await;
|
||||
assert!(
|
||||
rename_all_with_commit_guard(&payload, target.join("payload"), &bucket, &publication, &metadata_guard, lease.clone())
|
||||
.await
|
||||
.is_err(),
|
||||
"the metadata guard does not pin the private payload parent"
|
||||
);
|
||||
assert!(
|
||||
payload.join("part.1").exists(),
|
||||
"a rejected payload rename must leave its source in place"
|
||||
);
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
|
||||
.await
|
||||
.expect("distinct pinned payload source");
|
||||
let mut prepared = create_prepared_rename_source_with_commit_guard(&head, &destination, &metadata_guard)
|
||||
.expect("canonical prepared metadata");
|
||||
prepared
|
||||
.write_all(b"prepared native metadata", true)
|
||||
.expect("durable canonical metadata");
|
||||
rename_all_with_commit_guard(&payload, target.join("payload"), &bucket, &publication, &payload_guard, lease.clone())
|
||||
.await
|
||||
.expect("payload rename uses its own pinned TMP parent");
|
||||
assert!(head.exists(), "payload publication must not consume the canonical head");
|
||||
rename_all_with_prepared_source(prepared, &head, &destination, &bucket, &publication, &metadata_guard, lease)
|
||||
.await
|
||||
.expect("metadata rename keeps its canonical parent");
|
||||
assert!(!head.exists());
|
||||
assert_eq!(std::fs::read(&destination).expect("published metadata"), b"prepared native metadata");
|
||||
assert_eq!(
|
||||
std::fs::read(target.join("payload/part.1")).expect("published selected shard"),
|
||||
b"native selected shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn multipart_payload_guard_rejects_replaced_private_source_or_shared_destination() {
|
||||
@@ -7998,7 +7790,7 @@ mod multipart_source_preservation_tests {
|
||||
let metadata_guard =
|
||||
prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
|
||||
let lease = acquire_namespace_mutation_lease(&target).await;
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, lease.clone())
|
||||
.await
|
||||
.expect("pin payload and shared target");
|
||||
let replaced = if replace_source { &temporary } else { &target };
|
||||
@@ -8255,6 +8047,7 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Wdk::Storage::FileSystem::{FILE_OPEN, FILE_OPEN_IF};
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
|
||||
let base_relative = publication_root.relative_path(base_dir)?;
|
||||
let capacity = base_relative
|
||||
@@ -8265,12 +8058,13 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
let mut handles = Vec::with_capacity(capacity);
|
||||
handles.push(publication_root.directory.clone());
|
||||
let mut guard = ExistingBaseDirectoryGuard::new(handles);
|
||||
let components = base_relative
|
||||
let mut components = base_relative
|
||||
.components()
|
||||
.map(|component| (component, FILE_OPEN))
|
||||
.chain(relative.components().map(|component| (component, FILE_OPEN_IF)))
|
||||
.filter(|(component, _)| !matches!(component, Component::CurDir));
|
||||
for (component, disposition) in components {
|
||||
.filter(|(component, _)| !matches!(component, Component::CurDir))
|
||||
.peekable();
|
||||
while let Some((component, disposition)) = components.next() {
|
||||
let Component::Normal(component) = component else {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
@@ -8281,16 +8075,22 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
.handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows publication root guard is empty"))?;
|
||||
// Every component, including the final parent the kernel opens
|
||||
// for write during a relative rename, shares writes from its first
|
||||
// open; see `open_windows_directory_component`.
|
||||
let child = open_windows_directory_component(parent, component, disposition)?;
|
||||
// The kernel opens the final parent for write during a relative
|
||||
// rename. Share writes from its first open: a temporary read-only
|
||||
// share would block another rename into the same trash directory.
|
||||
// Ancestors stay strict and no handle shares delete access, keeping
|
||||
// every retained directory identity pinned.
|
||||
let share_access = if components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
let child = open_windows_relative_directory_component(parent, component, disposition, share_access)?;
|
||||
guard.handles.push(child);
|
||||
}
|
||||
// The hook observes the window right after the final component opened.
|
||||
#[cfg(test)]
|
||||
if guard.handles.len() > 1 {
|
||||
prepare_rename_test_hooks::run_after_component_opened(dir_path);
|
||||
#[cfg(test)]
|
||||
if components.peek().is_none() {
|
||||
prepare_rename_test_hooks::run_after_component_opened(dir_path);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(guard)
|
||||
@@ -9696,7 +9496,7 @@ mod tests {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use windows_sys::Win32::{
|
||||
Foundation::ERROR_SHARING_VIOLATION,
|
||||
Foundation::{ERROR_SHARING_VIOLATION, GENERIC_WRITE},
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
@@ -9738,13 +9538,13 @@ mod tests {
|
||||
let second_preparation = prepare_rename_with_retry(&sources[1], &destinations[1], base, &publication_root)
|
||||
.expect("prepare the second trash rename");
|
||||
assert!(interleaved.load(Ordering::Acquire), "the competing parent-open window must be exercised");
|
||||
for path in [&tmp, &trash] {
|
||||
for (path, access) in [(&tmp, GENERIC_WRITE), (&trash, DELETE)] {
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.access_mode(access)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)
|
||||
.expect_err("pinned ancestors and final parents must exclude deletion");
|
||||
.expect_err("ancestor writes and final-parent deletion must remain excluded");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
|
||||
@@ -9761,115 +9561,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_rename_into_a_pinned_ancestor_succeeds_while_deletion_stays_excluded() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::ERROR_SHARING_VIOLATION,
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
},
|
||||
};
|
||||
|
||||
// `bucket/parent` is an object directory and also the prefix of the
|
||||
// nested object `bucket/parent/child`. A guard on the nested object
|
||||
// pins `parent` as an ancestor while the parent object publishes a new
|
||||
// data directory into it by pathname.
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let parent = base.join("parent");
|
||||
let child = parent.join("child");
|
||||
std::fs::create_dir_all(&child).expect("create nested object directory");
|
||||
let staged = temp_dir.path().join("staged-data-dir");
|
||||
std::fs::create_dir(&staged).expect("create staged data directory");
|
||||
std::fs::write(staged.join("part.1"), b"payload").expect("write staged part");
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
|
||||
let child_guard =
|
||||
super::mkdir_all_below_existing_base_std(&child, &base, &publication_root).expect("pin the nested object tree");
|
||||
|
||||
let destination = parent.join(uuid::Uuid::new_v4().to_string());
|
||||
std::fs::rename(&staged, &destination).expect("a pinned ancestor must accept a rename into it");
|
||||
assert_eq!(std::fs::read(destination.join("part.1")).expect("read published part"), b"payload");
|
||||
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&parent)
|
||||
.expect_err("the pinned ancestor must still refuse deletion");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
|
||||
);
|
||||
drop(child_guard);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_directory_pin_retries_a_transient_delete_holder_and_fails_closed_on_a_lasting_one() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::ERROR_SHARING_VIOLATION,
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE,
|
||||
FILE_SHARE_READ, FILE_SHARE_WRITE, SYNCHRONIZE,
|
||||
},
|
||||
};
|
||||
|
||||
// Open the prefix the way `RemoveDirectoryW` does while it prunes an
|
||||
// object prefix: delete access, every share mode granted.
|
||||
let open_like_a_pruner = |path: &Path| {
|
||||
std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)
|
||||
.expect("open the prefix for delete")
|
||||
};
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
|
||||
|
||||
let transient_name = uuid::Uuid::new_v4().to_string();
|
||||
let transient_prefix = base.join(&transient_name);
|
||||
std::fs::create_dir_all(&transient_prefix).expect("create transient prefix");
|
||||
let holder = Arc::new(std::sync::Mutex::new(Some(open_like_a_pruner(&transient_prefix))));
|
||||
let holder_for_hook = Arc::clone(&holder);
|
||||
let released = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let released_for_hook = Arc::clone(&released);
|
||||
prepare_rename_test_hooks::install_before_component_retry(std::ffi::OsStr::new(&transient_name), move || {
|
||||
holder_for_hook.lock().expect("holder mutex poisoned").take();
|
||||
released_for_hook.store(true, std::sync::atomic::Ordering::Release);
|
||||
});
|
||||
let child = transient_prefix.join("child");
|
||||
let guard = super::mkdir_all_below_existing_base_std(&child, &base, &publication_root)
|
||||
.expect("a pin must outlast a transient delete holder");
|
||||
assert!(
|
||||
released.load(std::sync::atomic::Ordering::Acquire),
|
||||
"the pin must have met the delete holder"
|
||||
);
|
||||
assert!(child.is_dir(), "the retried walk must still create the child");
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&transient_prefix)
|
||||
.expect_err("the retried pin must still exclude deletion");
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_SHARING_VIOLATION).ok());
|
||||
drop(guard);
|
||||
|
||||
let lasting_prefix = base.join(uuid::Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&lasting_prefix).expect("create lasting prefix");
|
||||
let lasting_holder = open_like_a_pruner(&lasting_prefix);
|
||||
let Err(err) = super::mkdir_all_below_existing_base_std(&lasting_prefix.join("child"), &base, &publication_root) else {
|
||||
panic!("a pin must not wait forever on a lasting delete holder");
|
||||
};
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_SHARING_VIOLATION).ok());
|
||||
drop(lasting_holder);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_rename_all_supports_same_parent_publication() {
|
||||
@@ -10567,7 +10258,7 @@ mod tests {
|
||||
|
||||
let publication_root = test_publication_root(&[&src, &dst, &base]);
|
||||
let source_parent_guard =
|
||||
lock_windows_directory_tree(src.parent().expect("source reparse entry must have a parent"), &publication_root)
|
||||
lock_windows_directory_tree(src.parent().expect("source reparse entry must have a parent"), None, &publication_root)
|
||||
.expect("pin the source parent tree");
|
||||
let identity_error = match open_windows_rename_source_identity(&src, &source_parent_guard) {
|
||||
Ok(_) => panic!("a non-dedup source reparse entry must be rejected by its tag"),
|
||||
@@ -10601,51 +10292,30 @@ mod tests {
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_rename_all_keeps_the_source_parent_from_becoming_a_reparse_point() {
|
||||
async fn windows_rename_all_excludes_source_parent_reparse_writers() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::{ERROR_DIR_NOT_EMPTY, ERROR_SHARING_VIOLATION, GENERIC_WRITE},
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
},
|
||||
Foundation::GENERIC_WRITE,
|
||||
Storage::FileSystem::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, FILE_SHARE_WRITE},
|
||||
};
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let source_parent = temp_dir.path().join("staging");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
let src = source_parent.join("staged-object");
|
||||
let dst = base.join("published-object");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir(&source_parent).expect("create source parent");
|
||||
std::fs::create_dir(&outside).expect("create outside target");
|
||||
std::fs::write(&src, b"original").expect("write original staged object");
|
||||
|
||||
let source_parent_for_hook = source_parent.clone();
|
||||
let outside_for_hook = outside.clone();
|
||||
windows_rename_test_hooks::install_before_publication(&dst, move || {
|
||||
// The pin shares writes, so a writer can open the source parent,
|
||||
// but the parent still holds the staged entry and NTFS refuses to
|
||||
// turn a non-empty directory into a mount point.
|
||||
let writable_parent = std::fs::OpenOptions::new()
|
||||
std::fs::OpenOptions::new()
|
||||
.access_mode(GENERIC_WRITE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&source_parent_for_hook)
|
||||
.map(winapi_util::Handle::from_file)
|
||||
.expect("a write-sharing pin admits a concurrent writer");
|
||||
let err = try_set_windows_mount_point(&writable_parent, &outside_for_hook)
|
||||
.expect_err("the pinned source parent must not become a mount point");
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_DIR_NOT_EMPTY).ok());
|
||||
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&source_parent_for_hook)
|
||||
.expect_err("the source-parent guard must exclude deletion and rename");
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_SHARING_VIOLATION).ok());
|
||||
.expect_err("the source-parent guard must exclude a reparse writer");
|
||||
});
|
||||
|
||||
rename_all(&src, &dst, &base)
|
||||
@@ -10654,7 +10324,6 @@ mod tests {
|
||||
|
||||
assert!(!src.exists(), "the original staged entry must be moved");
|
||||
assert_eq!(std::fs::read(&dst).expect("read published original object"), b"original");
|
||||
assert!(!outside.join("published-object").exists(), "publication must not escape the base");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
@@ -10734,7 +10403,7 @@ mod tests {
|
||||
let guard = mkdir_all_below_existing_base_std(&base, &base).expect("guard destination base");
|
||||
let publication_root = test_publication_root(&[&src, &dst, &base]);
|
||||
let source_parent_guard =
|
||||
lock_windows_directory_tree(src.parent().expect("source path must have a parent"), &publication_root)
|
||||
lock_windows_directory_tree(src.parent().expect("source path must have a parent"), None, &publication_root)
|
||||
.expect("anchor source parent");
|
||||
let source = open_windows_rename_source(&src, &source_parent_guard).expect("anchor source entry");
|
||||
|
||||
|
||||
@@ -3636,73 +3636,4 @@ mod tests {
|
||||
assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
|
||||
assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_full_buf_or_eof_preserves_invalid_data_and_wraps_truncated_io_errors() {
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
struct StepReader {
|
||||
prefix: Vec<u8>,
|
||||
offset: usize,
|
||||
err_kind: Option<std::io::ErrorKind>,
|
||||
err_msg: &'static str,
|
||||
}
|
||||
|
||||
impl AsyncRead for StepReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.offset < self.prefix.len() {
|
||||
let take = (self.prefix.len() - self.offset).min(buf.remaining());
|
||||
buf.put_slice(&self.prefix[self.offset..self.offset + take]);
|
||||
self.offset += take;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
if let Some(kind) = self.err_kind.take() {
|
||||
return Poll::Ready(Err(std::io::Error::new(kind, self.err_msg)));
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Error on the very first read (total == 0) preserves the original ErrorKind.
|
||||
let mut r1 = StepReader {
|
||||
prefix: Vec::new(),
|
||||
offset: 0,
|
||||
err_kind: Some(std::io::ErrorKind::BrokenPipe),
|
||||
err_msg: "immediate broken pipe",
|
||||
};
|
||||
let mut buf = BytesMut::with_capacity(16);
|
||||
let err = read_full_buf_or_eof(&mut r1, &mut buf, 16)
|
||||
.await
|
||||
.expect_err("immediate read error must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::BrokenPipe);
|
||||
|
||||
// 2. InvalidData (e.g. HashReader checksum/trailer mismatch) after partial fill preserves InvalidData.
|
||||
let mut r2 = StepReader {
|
||||
prefix: b"partial".to_vec(),
|
||||
offset: 0,
|
||||
err_kind: Some(std::io::ErrorKind::InvalidData),
|
||||
err_msg: "Checksum mismatch",
|
||||
};
|
||||
let mut buf2 = BytesMut::with_capacity(16);
|
||||
let err = read_full_buf_or_eof(&mut r2, &mut buf2, 16)
|
||||
.await
|
||||
.expect_err("InvalidData after partial fill must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("Checksum mismatch"));
|
||||
|
||||
// 3. Non-InvalidData error after partial fill is normalized to UnexpectedEof.
|
||||
let mut r3 = StepReader {
|
||||
prefix: b"partial".to_vec(),
|
||||
offset: 0,
|
||||
err_kind: Some(std::io::ErrorKind::BrokenPipe),
|
||||
err_msg: "mid-stream reset",
|
||||
};
|
||||
let mut buf3 = BytesMut::with_capacity(16);
|
||||
let err = read_full_buf_or_eof(&mut r3, &mut buf3, 16)
|
||||
.await
|
||||
.expect_err("truncated stream error must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -934,124 +934,4 @@ mod tests {
|
||||
"cached record must still reject corrupt or mismatched shard payloads"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shard_verifier_rejects_out_of_bounds_index_and_empty_proof_sources_fail_closed() {
|
||||
let mut builder = IntegrityBuilder::new(IntegrityLayout::new(2, 2, 8, false).expect("layout"), 1).expect("builder");
|
||||
builder
|
||||
.push([b"abcd".as_slice(), b"efgh", b"ijkl", b"mnop"].into_iter())
|
||||
.await
|
||||
.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let proof = Arc::new(PartProofReader {
|
||||
part: prepared.part,
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
|
||||
// coding_index >= layout.shards() is rejected at construction time.
|
||||
let err = ShardVerifier::new(proof.clone(), 4, 0, None)
|
||||
.err()
|
||||
.expect("coding_index out of bounds must fail");
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("shard integrity verification failed"));
|
||||
|
||||
// Valid coding_index with neither inline bytes nor disk sources fails closed when reading stripe record.
|
||||
let mut verifier = ShardVerifier::new(proof, 0, 0, None).expect("verifier");
|
||||
let err = verifier
|
||||
.verify(b"abcd")
|
||||
.await
|
||||
.expect_err("empty proof sources with no inline index must fail closed");
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("shard integrity verification failed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spilled_index_write_verify_and_restore_proof_replicas_round_trip() {
|
||||
let (_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(Default::default()).await;
|
||||
let mut disk_slots = store.pools[0].disk_set[0].get_local_disks().await;
|
||||
let disks: Vec<_> = disk_slots.iter().flatten().cloned().collect();
|
||||
assert_eq!(disks.len(), 2);
|
||||
let volume = "spilled-integrity-bucket";
|
||||
for disk in &disks {
|
||||
disk.make_volume(volume).await.expect("make volume");
|
||||
}
|
||||
|
||||
// 2 shards (1 data + 1 parity) matches the 2-disk fixture; 2350 stripes * 448B > 1 MiB spills to Spool::File(_).
|
||||
let layout = IntegrityLayout::new(1, 1, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
let shards = [b"abcdefgh".as_slice(); 2];
|
||||
for _ in 0..2350 {
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
}
|
||||
let part_size = 2350 * 8;
|
||||
let prepared = builder.finish(part_size).await.expect("spilled index");
|
||||
assert!(matches!(prepared.index, Spool::File(_)));
|
||||
|
||||
let data_dir = uuid::Uuid::new_v4();
|
||||
let object = "object";
|
||||
let directory = format!("{object}/{data_dir}");
|
||||
let part = prepared
|
||||
.write(&mut disk_slots, volume, volume, &directory)
|
||||
.await
|
||||
.expect("write spilled index");
|
||||
|
||||
let rel_path = format!("{directory}/{}", part.file_name());
|
||||
for disk in &disks {
|
||||
verify_index_file(disk, volume, &rel_path, &part)
|
||||
.await
|
||||
.expect("verify spilled index on disk");
|
||||
}
|
||||
|
||||
// Corrupt replica 0; restore_proof_replicas should repair it from healthy replica 1.
|
||||
disks[0]
|
||||
.write_all(volume, &rel_path, bytes::Bytes::from_static(b"corrupted-index"))
|
||||
.await
|
||||
.expect("corrupt disk 0");
|
||||
assert!(verify_index_file(&disks[0], volume, &rel_path, &part).await.is_err());
|
||||
|
||||
let mut fi = rustfs_filemeta::FileInfo {
|
||||
volume: volume.to_owned(),
|
||||
name: object.to_owned(),
|
||||
data_dir: Some(data_dir),
|
||||
erasure: rustfs_filemeta::ErasureInfo {
|
||||
data_blocks: 1,
|
||||
parity_blocks: 1,
|
||||
block_size: 8,
|
||||
..Default::default()
|
||||
},
|
||||
parts: vec![rustfs_filemeta::ObjectPartInfo {
|
||||
number: 1,
|
||||
size: part_size,
|
||||
actual_size: part_size as i64,
|
||||
integrity: Some(part.clone()),
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
fi.persist_shard_integrity().expect("persist shard integrity");
|
||||
let repaired = restore_proof_replicas(&fi, &disk_slots, volume, object)
|
||||
.await
|
||||
.expect("restore proof replica 0");
|
||||
assert_eq!(repaired, 1);
|
||||
verify_index_file(&disks[0], volume, &rel_path, &part)
|
||||
.await
|
||||
.expect("repaired replica 0 must verify");
|
||||
|
||||
// Negative durability boundary: when all replicas are corrupted, restore_proof_replicas must fail closed.
|
||||
for disk in &disks {
|
||||
disk.write_all(volume, &rel_path, bytes::Bytes::from_static(b"all-corrupted"))
|
||||
.await
|
||||
.expect("corrupt all replicas");
|
||||
}
|
||||
let err = restore_proof_replicas(&fi, &disk_slots, volume, object)
|
||||
.await
|
||||
.expect_err("all corrupted proof replicas must fail closed");
|
||||
assert!(
|
||||
err.to_string().contains("shard integrity verification failed"),
|
||||
"unexpected error when all proof replicas are corrupt: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,7 +478,6 @@ struct ScannerPublicationCommitScopeInner {
|
||||
#[derive(Clone)]
|
||||
pub struct ScannerPublicationCommitScope {
|
||||
inner: Arc<ScannerPublicationCommitScopeInner>,
|
||||
owns_completion: bool,
|
||||
}
|
||||
|
||||
/// RAII fallback for storage paths that return before their commit closure
|
||||
@@ -570,16 +569,6 @@ impl ScannerPublicationCommitScope {
|
||||
movement_permit: Mutex::new(Some(movement_permit)),
|
||||
lease_release_safe,
|
||||
}),
|
||||
owns_completion: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// A pool may use the shared fence, but only the enclosing operation may
|
||||
/// report completion and authorize release of its leases.
|
||||
pub(crate) fn pool_delete_participant(&self) -> Self {
|
||||
Self {
|
||||
inner: Arc::clone(&self.inner),
|
||||
owns_completion: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -618,9 +607,7 @@ impl ScannerPublicationCommitScope {
|
||||
/// storage mutation owner. Once set, dropping a scanner waiter leaves the
|
||||
/// scope in-flight until that owner reports committed or indeterminate.
|
||||
pub fn attach_mutation_owner(&self) {
|
||||
if self.owns_completion {
|
||||
self.inner.owner_attached.store(true, Ordering::Release);
|
||||
}
|
||||
self.inner.owner_attached.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn owner_attached(&self) -> bool {
|
||||
@@ -667,17 +654,16 @@ impl ScannerPublicationCommitScope {
|
||||
}
|
||||
|
||||
pub fn mark_aborted_before_commit(&self) -> bool {
|
||||
if self.owns_completion
|
||||
&& self
|
||||
.inner
|
||||
.state
|
||||
.compare_exchange(
|
||||
SCANNER_PUBLICATION_SCOPE_ADMITTED,
|
||||
SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
)
|
||||
.is_ok()
|
||||
if self
|
||||
.inner
|
||||
.state
|
||||
.compare_exchange(
|
||||
SCANNER_PUBLICATION_SCOPE_ADMITTED,
|
||||
SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
self.inner.lease_release_safe.store(true, Ordering::Release);
|
||||
self.inner.completed.notify_waiters();
|
||||
@@ -691,9 +677,6 @@ impl ScannerPublicationCommitScope {
|
||||
}
|
||||
|
||||
fn mark_terminal(&self, terminal: ScannerPublicationCommitState) -> bool {
|
||||
if !self.owns_completion {
|
||||
return false;
|
||||
}
|
||||
self.inner
|
||||
.state
|
||||
.compare_exchange(SCANNER_PUBLICATION_SCOPE_IN_FLIGHT, terminal.as_u8(), Ordering::AcqRel, Ordering::Acquire)
|
||||
@@ -726,7 +709,7 @@ impl ScannerPublicationCommitScope {
|
||||
/// Release the storage-owned movement permit only after a known-safe
|
||||
/// terminal result. Returns `false` for in-flight or indeterminate work.
|
||||
pub async fn release_movement_permit(&self) -> bool {
|
||||
if !self.owns_completion || !self.state().permits_lease_release() {
|
||||
if !self.state().permits_lease_release() {
|
||||
return false;
|
||||
}
|
||||
self.inner.movement_permit.lock().await.take().is_some()
|
||||
|
||||
@@ -385,165 +385,3 @@ async fn pause_and_cancel_are_durable_during_content_read() {
|
||||
JobState::Cancelled
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_request_validation_and_protection_classification_cover_all_negative_branches() {
|
||||
use crate::object_api::ObjectInfo;
|
||||
|
||||
let valid_digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"payload")
|
||||
.expect("sha256")
|
||||
.encoded;
|
||||
|
||||
// 1. Empty items and > MAX_ITEMS (64).
|
||||
let mut empty_req = request(JobMode::Audit, valid_digest.clone());
|
||||
empty_req.items.clear();
|
||||
let err = empty_req.validate().expect_err("empty items must fail");
|
||||
assert!(
|
||||
err.to_string().contains("a job requires 1..=64 explicit source versions"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
let mut too_many = request(JobMode::Audit, valid_digest.clone());
|
||||
too_many.items = (0..65)
|
||||
.map(|i| ItemRequest {
|
||||
key: format!("key-{i}"),
|
||||
version_id: None,
|
||||
expected_sha256: None,
|
||||
target_key: None,
|
||||
})
|
||||
.collect();
|
||||
let err = too_many.validate().expect_err(">64 items must fail");
|
||||
assert!(
|
||||
err.to_string().contains("a job requires 1..=64 explicit source versions"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
// 2. bytes_per_second and max_object_bytes bounds.
|
||||
let mut low_bps = request(JobMode::Audit, valid_digest.clone());
|
||||
low_bps.bytes_per_second = 64 * 1024 - 1;
|
||||
let err = low_bps.validate().expect_err("low bytes_per_second must fail");
|
||||
assert!(err.to_string().contains("invalid byte rate or staging limit"), "unexpected error: {err}");
|
||||
|
||||
let mut high_bps = request(JobMode::Audit, valid_digest.clone());
|
||||
high_bps.bytes_per_second = 1024 * 1024 * 1024 + 1;
|
||||
let err = high_bps.validate().expect_err("high bytes_per_second must fail");
|
||||
assert!(err.to_string().contains("invalid byte rate or staging limit"), "unexpected error: {err}");
|
||||
|
||||
let mut zero_max = request(JobMode::Audit, valid_digest.clone());
|
||||
zero_max.max_object_bytes = 0;
|
||||
let err = zero_max.validate().expect_err("zero max_object_bytes must fail");
|
||||
assert!(err.to_string().contains("invalid byte rate or staging limit"), "unexpected error: {err}");
|
||||
|
||||
let mut huge_max = request(JobMode::Audit, valid_digest.clone());
|
||||
huge_max.max_object_bytes = model::MAX_OBJECT_BYTES + 1;
|
||||
let err = huge_max.validate().expect_err("huge max_object_bytes must fail");
|
||||
assert!(err.to_string().contains("invalid byte rate or staging limit"), "unexpected error: {err}");
|
||||
|
||||
// 3. Invalid keys (empty, trailing '/', embedded '\0', > 1024 chars).
|
||||
for bad_key in ["", "trailing/", "null\0byte", &"x".repeat(1025)] {
|
||||
let mut req = request(JobMode::Audit, valid_digest.clone());
|
||||
req.items[0].key = bad_key.to_string();
|
||||
let err = req.validate().expect_err("invalid key must fail");
|
||||
assert!(
|
||||
err.to_string().contains("invalid object key"),
|
||||
"expected 'invalid object key' for {bad_key:?}, got {err}"
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Invalid version_id (nil UUID, malformed UUID, empty string) and valid UUID.
|
||||
for bad_ver in ["", "not-a-uuid", &Uuid::nil().to_string()] {
|
||||
let mut req = request(JobMode::Audit, valid_digest.clone());
|
||||
req.items[0].version_id = Some(bad_ver.to_string());
|
||||
let err = req.validate().expect_err("invalid version_id must fail");
|
||||
assert!(
|
||||
err.to_string().contains("invalid source version"),
|
||||
"expected 'invalid source version' for {bad_ver:?}, got {err}"
|
||||
);
|
||||
}
|
||||
let mut valid_uuid_ver = request(JobMode::Audit, valid_digest.clone());
|
||||
valid_uuid_ver.items[0].version_id = Some(Uuid::new_v4().to_string());
|
||||
assert!(valid_uuid_ver.validate().is_ok());
|
||||
|
||||
// 5. Invalid expected_sha256 (bad base64, wrong byte length).
|
||||
let err = request(JobMode::Audit, "!!!not_base64!!!".to_string())
|
||||
.validate()
|
||||
.expect_err("non-base64 sha256 must fail");
|
||||
assert!(err.to_string().contains("SHA256 must be canonical Base64"), "unexpected error: {err}");
|
||||
let err = request(JobMode::Audit, "AAAA".to_string())
|
||||
.validate()
|
||||
.expect_err("wrong-length sha256 must fail");
|
||||
assert!(
|
||||
err.to_string().contains("SHA256 must be canonical Base64 of 32 bytes"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
// 6. Mode-specific target_key rules (Audit forbids target_key; Migrate requires unique target_key != key).
|
||||
let mut audit_with_target = request(JobMode::Audit, valid_digest.clone());
|
||||
audit_with_target.items[0].target_key = Some("target".to_string());
|
||||
let err = audit_with_target.validate().expect_err("audit with target_key must fail");
|
||||
assert!(
|
||||
err.to_string().contains("only migrations require an explicit target key"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
let mut migrate_no_target = request(JobMode::Migrate, valid_digest.clone());
|
||||
migrate_no_target.items[0].target_key = None;
|
||||
let err = migrate_no_target
|
||||
.validate()
|
||||
.expect_err("migrate without target_key must fail");
|
||||
assert!(
|
||||
err.to_string().contains("only migrations require an explicit target key"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
let mut migrate_same_target = request(JobMode::Migrate, valid_digest.clone());
|
||||
migrate_same_target.items[0].target_key = Some("source".to_string());
|
||||
let err = migrate_same_target
|
||||
.validate()
|
||||
.expect_err("migrate target == source must fail");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("targets must be distinct from every source and target"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
let mut migrate_dup_target = request(JobMode::Migrate, valid_digest);
|
||||
migrate_dup_target.items.push(ItemRequest {
|
||||
key: "source-2".into(),
|
||||
version_id: None,
|
||||
expected_sha256: None,
|
||||
target_key: Some("target".into()),
|
||||
});
|
||||
let err = migrate_dup_target.validate().expect_err("duplicate migrate target must fail");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("targets must be distinct from every source and target"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
// 7. model::unsupported and model::protection classifications.
|
||||
let mut delete_marker = ObjectInfo {
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(model::unsupported(&delete_marker), Some("delete_marker"));
|
||||
assert_eq!(model::protection(&delete_marker), Protection::NoPayload);
|
||||
delete_marker.delete_marker = false;
|
||||
|
||||
let mut transitioned = ObjectInfo::default();
|
||||
transitioned.transitioned_object.status = "Complete".to_string();
|
||||
assert_eq!(model::unsupported(&transitioned), Some("transitioned_object"));
|
||||
|
||||
let mut encrypted = ObjectInfo::default();
|
||||
Arc::make_mut(&mut encrypted.user_defined).insert("x-amz-server-side-encryption".to_string(), "AES256".to_string());
|
||||
assert_eq!(model::unsupported(&encrypted), Some("transformed_object"));
|
||||
|
||||
let mut invalid_decl = ObjectInfo::default();
|
||||
rustfs_utils::http::insert_str(
|
||||
Arc::make_mut(&mut invalid_decl.user_defined),
|
||||
rustfs_filemeta::shard_integrity::SUFFIX_SHARD_INTEGRITY,
|
||||
"not-json".to_string(),
|
||||
);
|
||||
assert_eq!(model::unsupported(&invalid_decl), Some("invalid_protection_declaration"));
|
||||
assert_eq!(model::protection(&invalid_decl), Protection::InvalidDeclaration);
|
||||
}
|
||||
|
||||
@@ -1910,10 +1910,6 @@ mod tests {
|
||||
&ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
mod_time: earlier(version.mod_time),
|
||||
// Without the object lock the rename waits for every
|
||||
// drive, so no early-ACK tail is still writing the copies
|
||||
// removed below.
|
||||
no_lock: true,
|
||||
..versioned
|
||||
},
|
||||
))
|
||||
@@ -2254,195 +2250,6 @@ mod tests {
|
||||
.expect("large-stack rebalance test thread should complete");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn real_rebalance_preserves_encrypted_multipart_etag_proof() {
|
||||
run_large_stack_async_test("rebalance-encrypted-multipart-etag", || async {
|
||||
use crate::io_support::rio::{HashReader, WriteEncryption, WritePlan};
|
||||
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SUFFIX_OPAQUE_ENCRYPTED_ETAG, etag_can_be_listed};
|
||||
|
||||
const REBALANCE_ID: &str = "rebalance-encrypted-multipart-etag";
|
||||
for (object, versioned, protected) in [
|
||||
("protected-version", true, true),
|
||||
("protected-null", false, true),
|
||||
("legacy-version", true, false),
|
||||
("legacy-null", false, false),
|
||||
] {
|
||||
// Each case needs a fresh progress counter: the fixture's small
|
||||
// capacity can reach its rebalance goal after one encrypted object.
|
||||
let (_temp_dirs, store, _peer) =
|
||||
crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = "rebalance-encrypted-etag";
|
||||
Box::pin(store.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: versioned,
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("create the migration bucket");
|
||||
prepare_rebalance_test_volumes(&store).await;
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let rebalance_meta = active_rebalance_meta(REBALANCE_ID);
|
||||
rebalance_meta
|
||||
.save(Arc::clone(&store.pools[0]))
|
||||
.await
|
||||
.expect("persist active rebalance");
|
||||
*store.rebalance_meta.write().await = Some(rebalance_meta);
|
||||
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let write_opts = ObjectOptions {
|
||||
versioned,
|
||||
user_defined: std::collections::HashMap::from([(SSEC_ALGORITHM_HEADER.to_owned(), "AES256".to_owned())]),
|
||||
..Default::default()
|
||||
};
|
||||
let upload = Box::pin(source_set.new_multipart_upload(bucket, object, &write_opts))
|
||||
.await
|
||||
.expect("create source encrypted multipart upload");
|
||||
let plaintext = b"encrypted multipart bytes must survive pool migration";
|
||||
let size = i64::try_from(plaintext.len()).expect("small fixture length");
|
||||
let source = HashReader::from_stream(std::io::Cursor::new(plaintext.to_vec()), size, size, None, None, false)
|
||||
.expect("plaintext source reader");
|
||||
let encrypted = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::multipart([7; 32], [9; 12], 1))
|
||||
.apply(source, size)
|
||||
.expect("encrypt the source part through the production writer");
|
||||
let mut part_opts = write_opts.clone();
|
||||
if !protected {
|
||||
// Model a retained legacy fingerprint without producer proof.
|
||||
part_opts.preserve_etag = Some("0123456789abcdef0123456789abcdef".to_owned());
|
||||
}
|
||||
let part = Box::pin(source_set.put_object_part(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::new(encrypted),
|
||||
&part_opts,
|
||||
))
|
||||
.await
|
||||
.expect("persist source encrypted part");
|
||||
let written = Box::pin(Arc::clone(&source_set).complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&write_opts,
|
||||
))
|
||||
.await
|
||||
.expect("complete source encrypted multipart upload");
|
||||
assert_eq!(written.version_id.is_some(), versioned);
|
||||
assert!(written.is_multipart(), "the actual migration must use its multipart path");
|
||||
assert_eq!(etag_can_be_listed(&written.user_defined), protected);
|
||||
let read_opts = ObjectOptions {
|
||||
version_id: Some(data_movement::data_movement_exact_version_id(written.version_id)),
|
||||
data_movement: true,
|
||||
raw_data_movement_read: true,
|
||||
..Default::default()
|
||||
};
|
||||
let source_bytes = {
|
||||
let mut reader = Box::pin(source_set.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts))
|
||||
.await
|
||||
.expect("read source ciphertext");
|
||||
let mut bytes = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.expect("collect source ciphertext");
|
||||
bytes
|
||||
};
|
||||
assert_ne!(source_bytes.as_slice(), plaintext, "the fixture must contain encrypted bytes");
|
||||
let entry = metacache_entry_from_source(&source_set, bucket, object).await;
|
||||
let outcome = tokio::time::timeout(
|
||||
StdDuration::from_secs(60),
|
||||
Box::pin(Arc::clone(&store).rebalance_entry(
|
||||
RebalanceEntryTarget {
|
||||
bucket: bucket.to_owned(),
|
||||
pool_index: 0,
|
||||
},
|
||||
entry,
|
||||
Arc::clone(&source_set),
|
||||
Arc::new(RebalanceBucketConfigs {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
}),
|
||||
Arc::from(REBALANCE_ID),
|
||||
CancellationToken::new(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("rebalance must finish within the test deadline")
|
||||
.expect("rebalance source entry");
|
||||
let target = match target_set.get_object_info(bucket, object, &read_opts).await {
|
||||
Ok(target) => target,
|
||||
Err(err) => {
|
||||
let target_versions = target_set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("inspect target after metadata read failure")
|
||||
.map(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.into_iter()
|
||||
.map(|version| version.version_id)
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let source_versions = source_set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("inspect source after metadata read failure")
|
||||
.map(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.into_iter()
|
||||
.map(|version| version.version_id)
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
panic!(
|
||||
"read migrated {object}: {err}; outcome={outcome:?}, expected_version={:?}, target_versions={target_versions:?}, source_versions={source_versions:?}",
|
||||
written.version_id
|
||||
);
|
||||
}
|
||||
};
|
||||
assert_eq!(target.version_id, written.version_id, "{object}");
|
||||
assert_eq!(target.etag, written.etag, "{object}");
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_OPAQUE_ENCRYPTED_ETAG),
|
||||
protected.then_some("v1"),
|
||||
"aggregate ETag proof must follow the exact source version for {object}"
|
||||
);
|
||||
assert_eq!(etag_can_be_listed(&target.user_defined), protected, "{object}");
|
||||
assert_eq!(outcome, RebalanceEntryOutcome::Completed, "{object}");
|
||||
let mut target_reader =
|
||||
Box::pin(target_set.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts))
|
||||
.await
|
||||
.expect("read target ciphertext");
|
||||
let mut target_bytes = Vec::new();
|
||||
target_reader
|
||||
.stream
|
||||
.read_to_end(&mut target_bytes)
|
||||
.await
|
||||
.expect("collect target ciphertext");
|
||||
assert_eq!(target_bytes, source_bytes, "rebalance preserves ciphertext bytes for {object}");
|
||||
assert!(
|
||||
source_set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("inspect source after migration")
|
||||
.is_none(),
|
||||
"source cleanup follows a verified target for {object}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const TRANSITIONED_REBALANCE_ID: &str = "rebalance-transitioned-source-cleanup";
|
||||
|
||||
/// One transitioned source version on pool 0 of a two-pool store with an
|
||||
|
||||
@@ -26,7 +26,6 @@ pub(crate) mod tier_probe_intent;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
pub mod warm_backend_azure_blob;
|
||||
#[cfg(feature = "gcs")]
|
||||
pub mod warm_backend_gcs;
|
||||
pub mod warm_backend_huaweicloud;
|
||||
|
||||
@@ -327,7 +327,6 @@ pub const TIER_CONFIG_VERSION: u16 = 2;
|
||||
const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
|
||||
const EXTERNAL_TIER_TYPE_S3: i32 = 1;
|
||||
const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
|
||||
const EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL: &str = "rustfs-azure-blob-v1:";
|
||||
const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
|
||||
const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
|
||||
const EXTERNAL_TIER_VERSION_WASABI_V1: &str = "rustfs-wasabi-v1";
|
||||
@@ -2148,7 +2147,6 @@ fn build_coordinator_tier_mutation_intent(
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(TierMutationIntent {
|
||||
requires_native_azure: candidate.tiers.values().any(TierConfig::is_native_azure),
|
||||
mutation_id: uuid::Uuid::new_v4(),
|
||||
revision: 1,
|
||||
kind,
|
||||
@@ -2517,11 +2515,11 @@ fn tier_backend_identity(config: &TierConfig) -> io::Result<TierDestinationId> {
|
||||
}),
|
||||
TierType::Azure => config.azure.as_ref().map(|value| {
|
||||
(
|
||||
if config.is_native_azure() { "azure-blob-v1" } else { "azure" },
|
||||
"azure",
|
||||
value.endpoint.as_str(),
|
||||
value.bucket.as_str(),
|
||||
value.prefix.as_str(),
|
||||
if config.is_native_azure() { "" } else { value.region.as_str() },
|
||||
value.region.as_str(),
|
||||
value.access_key.as_str(),
|
||||
)
|
||||
}),
|
||||
@@ -2594,7 +2592,7 @@ fn normalized_tier_prefix<'a>(tier_type: &TierType, prefix: &'a str) -> &'a str
|
||||
}
|
||||
|
||||
fn tier_exact_get_delete(config: &TierConfig) -> bool {
|
||||
config.is_native_azure() || ProviderVersionCapabilities::for_tier_type(&config.tier_type.as_lowercase()).exact_get_delete
|
||||
ProviderVersionCapabilities::for_tier_type(&config.tier_type.as_lowercase()).exact_get_delete
|
||||
}
|
||||
|
||||
struct TierDriverGeneration {
|
||||
@@ -3132,15 +3130,7 @@ fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<External
|
||||
.ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
|
||||
out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
|
||||
out.azure = Some(ExternalTierAzure {
|
||||
// Keep the v2 Azure payload readable and rewriteable by older
|
||||
// servers, but make their S3 constructor reject its endpoint.
|
||||
// An unknown numeric type would discard the Azure payload on
|
||||
// an unrelated configuration rewrite by an older server.
|
||||
endpoint: if tier.is_native_azure() {
|
||||
format!("{EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL}{}", az.endpoint)
|
||||
} else {
|
||||
az.endpoint.clone()
|
||||
},
|
||||
endpoint: az.endpoint.clone(),
|
||||
account_name: az.access_key.clone(),
|
||||
account_key: az.secret_key.clone(),
|
||||
bucket: az.bucket.clone(),
|
||||
@@ -3299,15 +3289,6 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let azure_blob = ext.version == super::tier_config::AZURE_BLOB_TIER_VERSION;
|
||||
let native_endpoint = ext
|
||||
.azure
|
||||
.as_ref()
|
||||
.is_some_and(|azure| azure.endpoint.starts_with(EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL));
|
||||
if azure_blob != native_endpoint || azure_blob && (ext.tier_type != EXTERNAL_TIER_TYPE_AZURE || ext.tier_type_hint.is_some())
|
||||
{
|
||||
return Err(io::Error::other("native Azure Blob tier has inconsistent protocol discriminators"));
|
||||
}
|
||||
let wasabi_hint = ext.tier_type_hint.as_deref() == Some("wasabi");
|
||||
let wasabi_version = ext.version == EXTERNAL_TIER_VERSION_WASABI_V1;
|
||||
if wasabi_hint && !wasabi_version {
|
||||
@@ -3328,9 +3309,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
|
||||
cfg.name
|
||||
)));
|
||||
}
|
||||
let tier_type = if azure_blob {
|
||||
TierType::Azure
|
||||
} else if wasabi_version {
|
||||
let tier_type = if wasabi_version {
|
||||
TierType::Wasabi
|
||||
} else {
|
||||
tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or(match ext.tier_type {
|
||||
@@ -3408,15 +3387,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
|
||||
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
|
||||
cfg.azure = Some(crate::services::tier::tier_config::TierAzure {
|
||||
name: cfg.name.clone(),
|
||||
endpoint: if azure_blob {
|
||||
az.endpoint
|
||||
.strip_prefix(EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL)
|
||||
.filter(|endpoint| !endpoint.is_empty())
|
||||
.ok_or_else(|| io::Error::other("native Azure Blob endpoint marker is missing"))?
|
||||
.to_string()
|
||||
} else {
|
||||
az.endpoint.clone()
|
||||
},
|
||||
endpoint: az.endpoint.clone(),
|
||||
access_key: az.account_name.clone(),
|
||||
secret_key: az.account_key.clone(),
|
||||
bucket: az.bucket.clone(),
|
||||
@@ -3587,9 +3558,6 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
|
||||
if data.starts_with(&TIER_CONFIG_FORMAT.to_le_bytes()) {
|
||||
return decode_external_tiering_config_blob(data);
|
||||
}
|
||||
let mut cfg: TierConfigMgr = serde_json::from_slice(data)?;
|
||||
for (name, tier) in &mut cfg.tiers {
|
||||
if matches!(&tier.tier_type, TierType::Wasabi) {
|
||||
@@ -3607,11 +3575,6 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
|
||||
// The legacy JSON shape omits the driver protocol version. Native
|
||||
// Azure must retain the binary discriminator even through this helper.
|
||||
if self.tiers.values().any(TierConfig::is_native_azure) {
|
||||
return encode_external_tiering_config_blob(self);
|
||||
}
|
||||
let data = serde_json::to_vec(&self)?;
|
||||
let mut data = Bytes::from(data);
|
||||
|
||||
@@ -3645,17 +3608,6 @@ impl TierConfigMgr {
|
||||
deadline: Option<Instant>,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
normalize_s3_gcs_add_tier_name(&mut tier_config)?;
|
||||
if tier_config.tier_type == TierType::Azure {
|
||||
tier_config.version = super::tier_config::AZURE_BLOB_TIER_VERSION.to_string();
|
||||
if let Some(azure) = tier_config.azure.as_mut() {
|
||||
if azure.storage_class.eq_ignore_ascii_case("STANDARD") {
|
||||
azure.storage_class.clear();
|
||||
}
|
||||
if azure.endpoint.is_empty() {
|
||||
azure.endpoint = format!("https://{}.blob.core.windows.net", azure.access_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.ensure_generation_is_idle(&tier_config.name)?;
|
||||
let tier_name = tier_config.name.clone();
|
||||
if tier_name != tier_name.to_uppercase() {
|
||||
@@ -9584,98 +9536,6 @@ mod tests {
|
||||
assert_eq!(az.sp_auth.client_secret, "secret");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_azure_add_preserves_protocol_through_restart_and_requires_peer_support() {
|
||||
let mut manager = empty_mgr();
|
||||
let mut tier = build_azure_tier("account");
|
||||
let name = tier.name.clone();
|
||||
let azure = tier.azure.as_mut().expect("Azure payload should exist");
|
||||
azure.endpoint.clear();
|
||||
azure.storage_class = "STANDARD".to_string();
|
||||
azure.secret_key = base64_simd::STANDARD.encode_to_string([7; 32]);
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(healthy_driver_factory(), manager.add(tier, true))
|
||||
.await
|
||||
.expect("current Console Azure payload should be admitted");
|
||||
let created = &manager.tiers[&name];
|
||||
assert!(created.is_native_azure());
|
||||
assert!(tier_exact_get_delete(created));
|
||||
let azure = created.azure.as_ref().expect("Azure payload should survive");
|
||||
assert_eq!(azure.endpoint, "https://account.blob.core.windows.net");
|
||||
assert!(azure.storage_class.is_empty());
|
||||
|
||||
let bytes = manager
|
||||
.marshal()
|
||||
.expect("native configuration should serialize without losing its protocol");
|
||||
let external: ExternalTierConfigMgr =
|
||||
rmp_serde::from_slice(&bytes[4..]).expect("native config should use the binary shape");
|
||||
assert_eq!(external.tiers[&name].tier_type, EXTERNAL_TIER_TYPE_AZURE);
|
||||
let stored = external.tiers[&name]
|
||||
.azure
|
||||
.as_ref()
|
||||
.expect("native storage keeps the existing Azure payload");
|
||||
assert_eq!(stored.endpoint, "rustfs-azure-blob-v1:https://account.blob.core.windows.net");
|
||||
let legacy_backend_config = crate::services::tier::tier_config::TierAzure {
|
||||
endpoint: stored.endpoint.clone(),
|
||||
access_key: stored.account_name.clone(),
|
||||
secret_key: stored.account_key.clone(),
|
||||
bucket: stored.bucket.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
crate::services::tier::warm_backend_azure::WarmBackendAzure::new(&legacy_backend_config, &name)
|
||||
.await
|
||||
.is_err(),
|
||||
"the unchanged legacy S3 constructor must reject the native endpoint before any request"
|
||||
);
|
||||
assert!(
|
||||
external.tiers[&name].tier_type_hint.is_none(),
|
||||
"the protocol marker must not be overridden by an extended provider hint"
|
||||
);
|
||||
let restarted = TierConfigMgr::unmarshal(&bytes).expect("native configuration should reload");
|
||||
assert!(restarted.tiers[&name].is_native_azure());
|
||||
assert_eq!(
|
||||
tier_backend_identity(created).expect("created identity"),
|
||||
tier_backend_identity(&restarted.tiers[&name]).expect("reloaded identity")
|
||||
);
|
||||
let targets = vec![TierMutationIntentTarget {
|
||||
tier_name: name,
|
||||
old_backend_identity: None,
|
||||
new_backend_identity: Some(tier_backend_identity(created).expect("native destination identity")),
|
||||
}];
|
||||
let intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &manager, targets)
|
||||
.expect("native intent should build")
|
||||
.expect("native Add must use peer prepare");
|
||||
assert!(intent.requires_native_azure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_azure_protocol_is_distinct_from_legacy_and_rejects_corrupt_discriminators() {
|
||||
let legacy = build_azure_tier("account");
|
||||
let mut native = legacy.clone_with_credentials();
|
||||
native.version = super::super::tier_config::AZURE_BLOB_TIER_VERSION.to_string();
|
||||
assert_ne!(
|
||||
tier_backend_identity(&legacy).expect("legacy identity"),
|
||||
tier_backend_identity(&native).expect("native identity"),
|
||||
"native Blob must never acquire a legacy S3 destination's cleanup authority"
|
||||
);
|
||||
let stored = to_external_tier_config(&legacy.name, &legacy).expect("legacy encoding");
|
||||
assert_eq!(stored.tier_type, EXTERNAL_TIER_TYPE_AZURE);
|
||||
let decoded = from_external_tier_config(legacy.name, stored).expect("legacy decoding");
|
||||
assert!(!decoded.is_native_azure());
|
||||
assert!(!tier_exact_get_delete(&decoded));
|
||||
let stored = to_external_tier_config(&native.name, &native).expect("native encoding");
|
||||
for mutate in [
|
||||
|config: &mut ExternalTierConfig| config.version = "v1".to_string(),
|
||||
|config: &mut ExternalTierConfig| config.tier_type = EXTERNAL_TIER_TYPE_S3,
|
||||
|config: &mut ExternalTierConfig| config.tier_type_hint = Some("azure".to_string()),
|
||||
] {
|
||||
let mut corrupt = stored.clone();
|
||||
mutate(&mut corrupt);
|
||||
assert!(from_external_tier_config(native.name.clone(), corrupt).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_external_blob_gcs_writer_stays_raw_for_v2_old_readers() {
|
||||
const RAW_JSON: &str = r#"{"type":"service_account","project_id":"tier-"}"#;
|
||||
@@ -10084,7 +9944,6 @@ mod tests {
|
||||
|
||||
fn prepared_remove_intent(tier_name: &str, mutation_id: uuid::Uuid) -> TierMutationIntent {
|
||||
TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Remove,
|
||||
|
||||
@@ -139,7 +139,6 @@ impl TierType {
|
||||
}
|
||||
|
||||
pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
|
||||
pub(crate) const AZURE_BLOB_TIER_VERSION: &str = "rustfs-azure-blob-v1";
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
@@ -179,10 +178,6 @@ impl Clone for TierConfig {
|
||||
}
|
||||
|
||||
impl TierConfig {
|
||||
pub(crate) fn is_native_azure(&self) -> bool {
|
||||
self.tier_type == TierType::Azure && self.version == AZURE_BLOB_TIER_VERSION
|
||||
}
|
||||
|
||||
pub(crate) fn redacted(&self) -> Self {
|
||||
let mut redacted = Self {
|
||||
version: self.version.clone(),
|
||||
|
||||
@@ -127,10 +127,6 @@ impl TierMutationIntentTarget {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierMutationIntent {
|
||||
// Old peers deny unknown intent fields before installing a prepare block.
|
||||
// Omit this extension for existing drivers to retain their wire checksum.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub requires_native_azure: bool,
|
||||
pub mutation_id: Uuid,
|
||||
pub revision: u64,
|
||||
pub kind: TierMutationIntentKind,
|
||||
@@ -286,8 +282,7 @@ impl TierMutationIntent {
|
||||
}
|
||||
|
||||
pub(crate) fn same_identity_as(&self, other: &Self) -> bool {
|
||||
self.requires_native_azure == other.requires_native_azure
|
||||
&& self.mutation_id == other.mutation_id
|
||||
self.mutation_id == other.mutation_id
|
||||
&& self.kind == other.kind
|
||||
&& self.old_config_etag == other.old_config_etag
|
||||
&& self.candidate_digest == other.candidate_digest
|
||||
@@ -830,10 +825,6 @@ fn identity_is_empty(identity: &TierDestinationId) -> bool {
|
||||
identity.iter().all(|byte| *byte == 0)
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!value
|
||||
}
|
||||
|
||||
fn digest_is_empty(digest: &TierMutationDigest) -> bool {
|
||||
digest.iter().all(|byte| *byte == 0)
|
||||
}
|
||||
@@ -849,7 +840,6 @@ mod tests {
|
||||
|
||||
fn prepared_intent() -> TierMutationIntent {
|
||||
TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id: Uuid::new_v4(),
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
@@ -877,49 +867,6 @@ mod tests {
|
||||
assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_azure_requirement_is_identity_bound_and_rejected_by_the_previous_wire_reader() {
|
||||
// Preserve the previous strict wire shape as an independent reader.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LegacyIntent {
|
||||
mutation_id: Uuid,
|
||||
revision: u64,
|
||||
kind: TierMutationIntentKind,
|
||||
state: TierMutationIntentState,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
old_config_etag: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
committed_config_etag: Option<String>,
|
||||
candidate_digest: TierMutationDigest,
|
||||
affected_targets: Vec<TierMutationIntentTarget>,
|
||||
expires_at_unix_nanos: i64,
|
||||
}
|
||||
let legacy = prepared_intent();
|
||||
let legacy_bytes = serde_json::to_vec(&legacy).expect("legacy intent should encode");
|
||||
let old_reader: LegacyIntent =
|
||||
serde_json::from_slice(&legacy_bytes).expect("ordinary mutations remain readable by old peers");
|
||||
assert_eq!(
|
||||
serde_json::to_vec(&old_reader).expect("old encoding"),
|
||||
legacy_bytes,
|
||||
"ordinary mutation checksums must stay unchanged"
|
||||
);
|
||||
|
||||
let mut native = legacy.clone();
|
||||
native.requires_native_azure = true;
|
||||
let native_bytes = serde_json::to_vec(&native).expect("native intent should encode");
|
||||
assert!(
|
||||
serde_json::from_slice::<LegacyIntent>(&native_bytes).is_err(),
|
||||
"an old peer must reject native Azure before prepare"
|
||||
);
|
||||
assert!(!legacy.same_identity_as(&native), "an abort cannot drop the native-driver requirement");
|
||||
let encoded = native.encode().expect("native durable intent should encode");
|
||||
assert_eq!(
|
||||
TierMutationIntent::decode(native.mutation_id, &encoded).expect("new peer should decode"),
|
||||
native
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutation_mutex_serializes_the_same_id() {
|
||||
let mutation_id = Uuid::new_v4();
|
||||
|
||||
@@ -25,7 +25,6 @@ use crate::services::tier::{
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||
warm_backend_aliyun::WarmBackendAliyun,
|
||||
warm_backend_azure::WarmBackendAzure,
|
||||
warm_backend_azure_blob::{AzureBlobError, WarmBackendAzureBlob},
|
||||
warm_backend_huaweicloud::WarmBackendHuaweicloud,
|
||||
warm_backend_minio::WarmBackendMinIO,
|
||||
warm_backend_r2::WarmBackendR2,
|
||||
@@ -310,10 +309,10 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
|
||||
/// Connection parameters every S3-compatible warm backend provider supplies.
|
||||
///
|
||||
/// The Aliyun, legacy Azure, Huaweicloud, Tencent, MinIO, R2, and RustFS backends all
|
||||
/// The Aliyun, Azure, Huaweicloud, Tencent, MinIO, R2, and RustFS backends all
|
||||
/// wrap [`WarmBackendS3`] around a statically-credentialed [`TransitionClient`]
|
||||
/// built from exactly these values. `bucket_lookup` is a parameter rather than a
|
||||
/// constant because the providers split into two families: Aliyun, legacy Azure,
|
||||
/// constant because the providers split into two families: Aliyun, Azure,
|
||||
/// Huaweicloud, and Tencent pin [`BucketLookupType::BucketLookupDNS`], while
|
||||
/// MinIO, R2, and RustFS leave it at [`BucketLookupType::BucketLookupAuto`].
|
||||
pub(crate) struct S3CompatibleWarmBackendParams<'a> {
|
||||
@@ -674,9 +673,6 @@ fn probe_cleanup_incomplete_error() -> AdminError {
|
||||
}
|
||||
|
||||
fn is_remote_tier_auth_rejection(err: &std::io::Error) -> bool {
|
||||
if AzureBlobError::from_io(err).is_some() && err.kind() == std::io::ErrorKind::PermissionDenied {
|
||||
return true;
|
||||
}
|
||||
let Some(response) = err.get_ref().and_then(|err| err.downcast_ref::<ErrorResponse>()) else {
|
||||
return false;
|
||||
};
|
||||
@@ -710,27 +706,10 @@ async fn check_warm_backend_with_deadlines(
|
||||
Ok(Err(err)) if is_remote_tier_auth_rejection(&err) => {
|
||||
return Err(ERR_TIER_INVALID_CREDENTIALS.clone());
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
if let Some(rejection) = AzureBlobError::from_io(&err).filter(|error| error.rejects_write()) {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = rejection.to_string();
|
||||
return Err(admin_err);
|
||||
}
|
||||
Ok(Err(_)) => {
|
||||
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
|
||||
Ok(()) => {
|
||||
let mut admin_err = ERR_TIER_PERM_ERR.clone();
|
||||
if let Some(original) = AzureBlobError::from_io(&err) {
|
||||
admin_err.message = original.to_string();
|
||||
}
|
||||
admin_err
|
||||
}
|
||||
Err(_) => {
|
||||
let mut admin_err = probe_cleanup_incomplete_error();
|
||||
if let Some(original) = AzureBlobError::from_io(&err) {
|
||||
admin_err.message.push_str(&format!(": {original}"));
|
||||
}
|
||||
admin_err
|
||||
}
|
||||
Ok(()) => ERR_TIER_PERM_ERR.clone(),
|
||||
Err(_) => probe_cleanup_incomplete_error(),
|
||||
});
|
||||
}
|
||||
Err(_) => {
|
||||
@@ -972,23 +951,24 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
}
|
||||
}
|
||||
TierType::Azure => {
|
||||
let azure = tier.azure.as_ref().ok_or_else(|| AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Azure tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
})?;
|
||||
let driver = if tier.is_native_azure() {
|
||||
WarmBackendAzureBlob::new(azure).map(|driver| -> WarmBackendImpl { Box::new(driver) })
|
||||
if let Some(azure_config) = tier.azure.as_ref() {
|
||||
let dd = WarmBackendAzure::new(azure_config, &tier.name).await;
|
||||
if let Err(err) = dd {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
d = Some(Box::new(dd.expect("Failed to create Azure backend")));
|
||||
} else {
|
||||
WarmBackendAzure::new(azure, &tier.name)
|
||||
.await
|
||||
.map(|driver| -> WarmBackendImpl { Box::new(driver) })
|
||||
};
|
||||
d = Some(driver.map_err(|err| AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
})?);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Azure tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
TierType::GCS => {
|
||||
@@ -1062,13 +1042,6 @@ pub(crate) async fn new_transition_candidate_reconciler(
|
||||
tier: &TierConfig,
|
||||
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
|
||||
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
|
||||
TierType::Azure if tier.is_native_azure() => Box::new(
|
||||
WarmBackendAzureBlob::new(tier.azure.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?).map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::S3 => Box::new(
|
||||
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
|
||||
@@ -88,7 +88,7 @@ impl WarmBackend for WarmBackendAzure {
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
// Legacy Azure uses the shared S3/SigV4 transport, but its normal
|
||||
// Azure currently uses the shared S3/SigV4 transport, but its normal
|
||||
// object path cannot persist exact remote versions across mixed
|
||||
// RustFS releases. The mutation probe may still detect and precisely
|
||||
// remove a versioned test object before rejecting that configuration.
|
||||
|
||||
@@ -1,881 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Native Azure Blob tier transport, including Azure Stack Hub 2301 and later.
|
||||
//!
|
||||
//! Writes use one conditional commit and never replay a failed HTTP request.
|
||||
//! Larger bodies are staged in bounded blocks with a unique upload namespace;
|
||||
//! an interrupted staging operation leaves only Azure's expiring uncommitted
|
||||
//! blocks. Lifecycle uploads use the existing transaction journal after a lost
|
||||
//! commit acknowledgement; validation probes retain in-process compensation.
|
||||
//! Versioned GET/DELETE address the persisted version. Current-root cleanup
|
||||
//! may demote that matching version under a strong ETag before exact deletion.
|
||||
//! Unversioned DELETE also requires a strong ETag to refuse replacement.
|
||||
|
||||
use super::tier_config::TierAzure;
|
||||
use super::warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options, transition_client_timeouts_from_env,
|
||||
};
|
||||
use super::warm_backend_s3::transition_candidate_metadata_matches;
|
||||
use crate::bucket::lifecycle::transition_transaction::canonical_transition_remote_object;
|
||||
use bytes::Bytes;
|
||||
use futures_util::stream;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode};
|
||||
use quick_xml::{Reader, events::Event};
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
use rustfs_signer::azure_shared_key_signature;
|
||||
use rustfs_utils::crypto::hex;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::hash::FastMd5;
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, internal_key_rustfs,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Cursor, ErrorKind};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use url::Url;
|
||||
|
||||
const API_VERSION: &str = "2020-10-02";
|
||||
const MIN_BLOCK_BYTES: u64 = 16 * 1024 * 1024;
|
||||
const MAX_BLOCK_BYTES: u64 = 100 * 1024 * 1024;
|
||||
const MAX_BLOCKS: u64 = 50_000;
|
||||
const MAX_CONTROL_BYTES: usize = 64 * 1024;
|
||||
const MAX_METADATA_BYTES: usize = 8 * 1024;
|
||||
const METADATA_PREFIX: &str = "x-ms-meta-rustfs_";
|
||||
const HTTP_DATE: &[time::format_description::BorrowedFormatItem<'static>] =
|
||||
time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
|
||||
|
||||
/// A bounded diagnostic that never contains credentials, URLs or response bodies.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("Azure Blob {operation} failed (HTTP {status}, code {code}, request ID {request_id})")]
|
||||
pub(crate) struct AzureBlobError {
|
||||
operation: &'static str,
|
||||
status: StatusCode,
|
||||
code: String,
|
||||
request_id: String,
|
||||
}
|
||||
|
||||
impl AzureBlobError {
|
||||
pub(super) fn from_io(error: &io::Error) -> Option<&Self> {
|
||||
error.get_ref()?.downcast_ref()
|
||||
}
|
||||
|
||||
pub(super) fn rejects_write(&self) -> bool {
|
||||
matches!(
|
||||
self.status,
|
||||
StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND | StatusCode::CONFLICT | StatusCode::PRECONDITION_FAILED
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WarmBackendAzureBlob {
|
||||
client: reqwest::Client,
|
||||
endpoint: Url,
|
||||
account: String,
|
||||
key: Vec<u8>,
|
||||
container: String,
|
||||
prefix: String,
|
||||
request_timeout: Duration,
|
||||
}
|
||||
|
||||
impl WarmBackendAzureBlob {
|
||||
/// Build a native Blob backend without performing remote I/O.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Rejects unsupported credentials or storage classes, an invalid account,
|
||||
/// container, prefix, endpoint or account key, and TLS/client setup failures.
|
||||
pub fn new(conf: &TierAzure) -> io::Result<Self> {
|
||||
validate_config(conf)?;
|
||||
let endpoint = if conf.endpoint.is_empty() {
|
||||
format!("https://{}.blob.core.windows.net", conf.access_key)
|
||||
} else {
|
||||
conf.endpoint.clone()
|
||||
};
|
||||
let endpoint = Url::parse(&endpoint).map_err(|_| invalid_input("Azure Blob endpoint is not a valid URL"))?;
|
||||
if !matches!(endpoint.scheme(), "https" | "http")
|
||||
|| endpoint.host_str().is_none()
|
||||
|| !endpoint.username().is_empty()
|
||||
|| endpoint.password().is_some()
|
||||
|| !matches!(endpoint.path(), "" | "/")
|
||||
|| endpoint.query().is_some()
|
||||
|| endpoint.fragment().is_some()
|
||||
{
|
||||
return Err(invalid_input(
|
||||
"Azure Blob endpoint must be an HTTP(S) origin without credentials, path or query",
|
||||
));
|
||||
}
|
||||
validate_outbound_url(&endpoint).map_err(|_| invalid_input("Azure Blob endpoint is not allowed"))?;
|
||||
let key = decode_key(&conf.secret_key)?;
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(io::Error::other)?;
|
||||
let timeouts = transition_client_timeouts_from_env();
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(timeouts.connect_timeout)
|
||||
.read_timeout(timeouts.response_body_idle_timeout)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.retry(reqwest::retry::never())
|
||||
.no_gzip()
|
||||
.no_brotli()
|
||||
.no_zstd()
|
||||
.no_deflate()
|
||||
.build()
|
||||
.map_err(transport_error)?;
|
||||
Ok(Self {
|
||||
client,
|
||||
endpoint,
|
||||
account: conf.access_key.clone(),
|
||||
key,
|
||||
container: conf.bucket.clone(),
|
||||
prefix: conf.prefix.strip_suffix('/').unwrap_or(&conf.prefix).to_string(),
|
||||
request_timeout: timeouts.request_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
fn object_url(&self, object: &str, version: &str) -> io::Result<Url> {
|
||||
validate_version(version)?;
|
||||
let name = if self.prefix.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{}/{object}", self.prefix)
|
||||
};
|
||||
validate_object_name(&name)?;
|
||||
let mut url = self.endpoint.clone();
|
||||
{
|
||||
let mut segments = url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| invalid_input("Azure Blob endpoint has no path"))?;
|
||||
segments.clear().push(&self.container);
|
||||
for segment in name.split('/') {
|
||||
segments.push(segment);
|
||||
}
|
||||
}
|
||||
if !version.is_empty() {
|
||||
url.query_pairs_mut().append_pair("versionid", version);
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn container_url(&self) -> io::Result<Url> {
|
||||
let mut url = self.endpoint.clone();
|
||||
url.path_segments_mut()
|
||||
.map_err(|_| invalid_input("Azure Blob endpoint has no path"))?
|
||||
.clear()
|
||||
.push(&self.container);
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
async fn send(&self, method: Method, url: Url, mut headers: HeaderMap, body: Option<Bytes>) -> io::Result<reqwest::Response> {
|
||||
if let Some(body) = &body {
|
||||
insert_header(&mut headers, "content-length", &body.len().to_string())?;
|
||||
let bytes = body.clone();
|
||||
let digest = tokio::task::spawn_blocking(move || FastMd5::digest(&bytes))
|
||||
.await
|
||||
.map_err(|_| io::Error::other("Azure Blob content checksum task failed"))?;
|
||||
insert_header(&mut headers, "content-md5", &base64_simd::STANDARD.encode_to_string(digest))?;
|
||||
}
|
||||
headers.insert("x-ms-version", HeaderValue::from_static(API_VERSION));
|
||||
headers.insert("accept-encoding", HeaderValue::from_static("identity"));
|
||||
let date = OffsetDateTime::now_utc()
|
||||
.format(HTTP_DATE)
|
||||
.map_err(|_| io::Error::other("Azure Blob request date is not representable"))?;
|
||||
insert_header(&mut headers, "x-ms-date", &date)?;
|
||||
let signature = azure_shared_key_signature(&self.key, &self.account, method.as_str(), &url, &headers)
|
||||
.map_err(|err| io::Error::new(ErrorKind::InvalidInput, err))?;
|
||||
insert_header(&mut headers, "authorization", &format!("SharedKey {}:{signature}", self.account))?;
|
||||
let mut request = reqwest::Request::new(method, url);
|
||||
*request.headers_mut() = headers;
|
||||
*request.body_mut() = body.map(reqwest::Body::from);
|
||||
tokio::time::timeout(self.request_timeout, self.client.execute(request))
|
||||
.await
|
||||
.map_err(|_| io::Error::new(ErrorKind::TimedOut, "Azure Blob request timed out"))?
|
||||
.map_err(transport_error)
|
||||
}
|
||||
|
||||
async fn expect_status(
|
||||
&self,
|
||||
response: reqwest::Response,
|
||||
expected: StatusCode,
|
||||
operation: &'static str,
|
||||
) -> io::Result<reqwest::Response> {
|
||||
if response.status() == expected {
|
||||
return Ok(response);
|
||||
}
|
||||
Err(service_error(response, operation, self.request_timeout).await)
|
||||
}
|
||||
|
||||
async fn head(&self, object: &str, version: &str) -> io::Result<Option<HeaderMap>> {
|
||||
let response = self
|
||||
.send(Method::HEAD, self.object_url(object, version)?, HeaderMap::new(), None)
|
||||
.await?;
|
||||
if response.status() == StatusCode::OK {
|
||||
if !version.is_empty() {
|
||||
validate_response_version(response.headers(), version)?;
|
||||
}
|
||||
return Ok(Some(response.headers().clone()));
|
||||
}
|
||||
let err = service_error(response, "HEAD", self.request_timeout).await;
|
||||
if err.kind() == ErrorKind::NotFound {
|
||||
return Ok(None);
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
|
||||
async fn delete(&self, object: &str, version: &str, headers: HeaderMap) -> io::Result<()> {
|
||||
let response = self
|
||||
.send(Method::DELETE, self.object_url(object, version)?, headers, None)
|
||||
.await?;
|
||||
if response.status() == StatusCode::ACCEPTED {
|
||||
return Ok(());
|
||||
}
|
||||
let error = service_error(response, "DELETE", self.request_timeout).await;
|
||||
if error.kind() == ErrorKind::NotFound {
|
||||
return Ok(());
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
|
||||
async fn write(
|
||||
&self,
|
||||
object: &str,
|
||||
reader: ReaderImpl,
|
||||
length: i64,
|
||||
metadata: HashMap<String, String>,
|
||||
) -> io::Result<String> {
|
||||
let size = u64::try_from(length).map_err(|_| invalid_input("Azure Blob upload requires a known, non-negative length"))?;
|
||||
let block_size = block_size(size)?;
|
||||
let url = self.object_url(object, "")?;
|
||||
let mut headers = metadata_headers(metadata)?;
|
||||
headers.insert("if-none-match", HeaderValue::from_static("*"));
|
||||
let mut reader: Box<dyn AsyncRead + Send + Sync + Unpin> = match reader {
|
||||
ReaderImpl::Body(bytes) => Box::new(Cursor::new(bytes)),
|
||||
ReaderImpl::ObjectBody(reader) => Box::new(reader),
|
||||
};
|
||||
if size <= block_size {
|
||||
let body = read_part(&mut reader, size).await?;
|
||||
ensure_eof(&mut reader).await?;
|
||||
headers.insert("x-ms-blob-type", HeaderValue::from_static("BlockBlob"));
|
||||
let response = self.send(Method::PUT, url, headers, Some(body)).await?;
|
||||
let response = self.expect_status(response, StatusCode::CREATED, "PUT").await?;
|
||||
return response_version(response.headers());
|
||||
}
|
||||
|
||||
let upload = uuid::Uuid::new_v4().simple().to_string();
|
||||
let mut blocks = Vec::new();
|
||||
let mut remaining = size;
|
||||
while remaining > 0 {
|
||||
let count = remaining.min(block_size);
|
||||
let body = read_part(&mut reader, count).await?;
|
||||
let block_id = base64_simd::STANDARD.encode_to_string(format!("{upload}-{:05}", blocks.len()));
|
||||
let mut block_url = url.clone();
|
||||
block_url
|
||||
.query_pairs_mut()
|
||||
.append_pair("comp", "block")
|
||||
.append_pair("blockid", &block_id);
|
||||
let response = self.send(Method::PUT, block_url, HeaderMap::new(), Some(body)).await?;
|
||||
self.expect_status(response, StatusCode::CREATED, "PUT block").await?;
|
||||
blocks.push(block_id);
|
||||
remaining -= count;
|
||||
}
|
||||
ensure_eof(&mut reader).await?;
|
||||
let mut xml = String::from("<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList>");
|
||||
for block in blocks {
|
||||
// IDs are generated base64, so they contain no XML delimiters.
|
||||
xml.push_str("<Uncommitted>");
|
||||
xml.push_str(&block);
|
||||
xml.push_str("</Uncommitted>");
|
||||
}
|
||||
xml.push_str("</BlockList>");
|
||||
let mut commit_url = url;
|
||||
commit_url.query_pairs_mut().append_pair("comp", "blocklist");
|
||||
headers.insert("content-type", HeaderValue::from_static("application/xml"));
|
||||
let response = self.send(Method::PUT, commit_url, headers, Some(Bytes::from(xml))).await?;
|
||||
let response = self.expect_status(response, StatusCode::CREATED, "PUT block list").await?;
|
||||
response_version(response.headers())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendAzureBlob {
|
||||
fn validate_remote_version_id(&self, version: &str) -> io::Result<()> {
|
||||
validate_version(version)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, reader: ReaderImpl, length: i64) -> io::Result<String> {
|
||||
self.write(object, reader, length, HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
reader: ReaderImpl,
|
||||
length: i64,
|
||||
metadata: HashMap<String, String>,
|
||||
) -> io::Result<String> {
|
||||
self.write(object, reader, length, metadata).await
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, version: &str, opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
let range = requested_range(&opts)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some((start, end)) = range {
|
||||
let value = end.map_or_else(|| format!("bytes={start}-"), |end| format!("bytes={start}-{end}"));
|
||||
insert_header(&mut headers, "range", &value)?;
|
||||
}
|
||||
let response = self
|
||||
.send(Method::GET, self.object_url(object, version)?, headers, None)
|
||||
.await?;
|
||||
let expected = if range.is_some() {
|
||||
StatusCode::PARTIAL_CONTENT
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
let response = self.expect_status(response, expected, "GET").await?;
|
||||
validate_response_version(response.headers(), version)?;
|
||||
let length = response
|
||||
.headers()
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.ok_or_else(|| invalid_data("Azure Blob GET omitted a valid Content-Length"))?;
|
||||
if let Some(range) = range {
|
||||
validate_content_range(response.headers(), range, length)?;
|
||||
}
|
||||
let body = stream::try_unfold((response, length), |(mut response, remaining)| async move {
|
||||
loop {
|
||||
match response.chunk().await.map_err(transport_error)? {
|
||||
Some(chunk) if chunk.is_empty() => continue,
|
||||
Some(chunk) => {
|
||||
let count =
|
||||
u64::try_from(chunk.len()).map_err(|_| invalid_data("Azure Blob response length overflow"))?;
|
||||
let remaining = remaining
|
||||
.checked_sub(count)
|
||||
.ok_or_else(|| invalid_data("Azure Blob returned more bytes than declared"))?;
|
||||
return Ok::<_, io::Error>(Some((chunk, (response, remaining))));
|
||||
}
|
||||
None if remaining == 0 => return Ok(None),
|
||||
None => {
|
||||
return Err(io::Error::new(
|
||||
ErrorKind::UnexpectedEof,
|
||||
"Azure Blob response ended before its declared length",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(ReadCloser::new(tokio_util::io::StreamReader::new(Box::pin(body))))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, version: &str) -> io::Result<()> {
|
||||
let mut headers = HeaderMap::new();
|
||||
if version.is_empty() {
|
||||
let Some(current) = self.head(object, "").await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !response_version(¤t)?.is_empty() {
|
||||
return Err(invalid_data("Azure Blob became versioned; refusing an unversioned delete"));
|
||||
}
|
||||
headers = delete_preconditions(¤t)?;
|
||||
}
|
||||
let Err(error) = self.delete(object, version, headers).await else {
|
||||
return Ok(());
|
||||
};
|
||||
if version.is_empty()
|
||||
|| !AzureBlobError::from_io(&error).is_some_and(|azure| {
|
||||
matches!(azure.status, StatusCode::FORBIDDEN | StatusCode::CONFLICT)
|
||||
&& azure.code == "OperationNotAllowedOnRootBlob"
|
||||
})
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
// Azure may require the current version to become a previous version
|
||||
// before exact deletion. Only its matching generation can be demoted.
|
||||
if let Some(current) = self.head(object, "").await? {
|
||||
validate_response_version(¤t, version)?;
|
||||
self.delete(object, "", delete_preconditions(¤t)?).await?;
|
||||
}
|
||||
// A successful base delete alone does not complete version cleanup.
|
||||
// An error or lost acknowledgement remains retryable by the caller.
|
||||
self.delete(object, version, HeaderMap::new()).await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
|
||||
Ok(match self.head(object, "").await? {
|
||||
None => TransitionCandidateProbe::Missing,
|
||||
Some(headers) => classify_head(&headers)?,
|
||||
})
|
||||
}
|
||||
|
||||
async fn probe_transition_version(&self, object: &str, version: &str) -> io::Result<TransitionCandidateProbe> {
|
||||
if version.is_empty() {
|
||||
return Err(invalid_input("Azure Blob exact probe requires a version ID"));
|
||||
}
|
||||
Ok(match self.head(object, version).await? {
|
||||
None => TransitionCandidateProbe::Missing,
|
||||
Some(_) => TransitionCandidateProbe::VersionedPresent(version.to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> io::Result<bool> {
|
||||
let mut url = self.container_url()?;
|
||||
let prefix = if self.prefix.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}/", self.prefix)
|
||||
};
|
||||
url.query_pairs_mut()
|
||||
.append_pair("restype", "container")
|
||||
.append_pair("comp", "list")
|
||||
.append_pair("prefix", &prefix)
|
||||
.append_pair("maxresults", "1");
|
||||
let response = self.send(Method::GET, url, HeaderMap::new(), None).await?;
|
||||
let response = self.expect_status(response, StatusCode::OK, "LIST").await?;
|
||||
list_has_objects(&read_control_body(response, self.request_timeout).await?)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for WarmBackendAzureBlob {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
let Some(headers) = self.head(object, "").await? else {
|
||||
return Ok(TransitionCandidateProbe::Missing);
|
||||
};
|
||||
let mut metadata = HashMap::new();
|
||||
for suffix in [SUFFIX_TRANSITION_TRANSACTION_ID, SUFFIX_TRANSITION_TIER_DESTINATION_ID] {
|
||||
for key in [internal_key_rustfs(suffix), format!("{MINIO_INTERNAL_PREFIX}{suffix}")] {
|
||||
if let Some(value) = headers.get(metadata_header_name(&key)) {
|
||||
let decoded = base64_simd::STANDARD
|
||||
.decode_to_vec(value.as_bytes())
|
||||
.map_err(|_| invalid_data("Azure Blob candidate metadata is not valid base64"))?;
|
||||
let value =
|
||||
String::from_utf8(decoded).map_err(|_| invalid_data("Azure Blob candidate metadata is not UTF-8"))?;
|
||||
metadata.insert(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
classify_head(&headers)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_config(conf: &TierAzure) -> io::Result<()> {
|
||||
if !(3..=24).contains(&conf.access_key.len())
|
||||
|| !conf
|
||||
.access_key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
|
||||
{
|
||||
return Err(invalid_input("Azure Blob account name must contain 3 to 24 lowercase letters or digits"));
|
||||
}
|
||||
let container = conf.bucket.as_bytes();
|
||||
if !(3..=63).contains(&container.len())
|
||||
|| !container
|
||||
.iter()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-')
|
||||
|| container.first() == Some(&b'-')
|
||||
|| container.last() == Some(&b'-')
|
||||
|| conf.bucket.contains("--")
|
||||
{
|
||||
return Err(invalid_input("Azure Blob container name is invalid"));
|
||||
}
|
||||
if !(conf.storage_class.is_empty() || conf.storage_class == "STANDARD")
|
||||
|| !conf.sp_auth.tenant_id.is_empty()
|
||||
|| !conf.sp_auth.client_id.is_empty()
|
||||
|| !conf.sp_auth.client_secret.is_empty()
|
||||
{
|
||||
return Err(invalid_input(
|
||||
"Azure Blob tiers currently require an account key and the default storage class",
|
||||
));
|
||||
}
|
||||
let prefix = conf.prefix.strip_suffix('/').unwrap_or(&conf.prefix);
|
||||
if !prefix.is_empty() {
|
||||
validate_object_name(prefix)?;
|
||||
// Canonical suffix length is independent of the bucket and UUID values.
|
||||
let object = canonical_transition_remote_object(
|
||||
uuid::Uuid::from_u128(1),
|
||||
&conf.bucket,
|
||||
uuid::Uuid::from_u128(2),
|
||||
uuid::Uuid::from_u128(3),
|
||||
)
|
||||
.map_err(io::Error::other)?;
|
||||
validate_object_name(&format!("{prefix}/{object}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_key(key: &str) -> io::Result<Vec<u8>> {
|
||||
let key = base64_simd::STANDARD
|
||||
.decode_to_vec(key.trim().as_bytes())
|
||||
.map_err(|_| invalid_input("Azure Blob account key must be base64"))?;
|
||||
if key.is_empty() {
|
||||
return Err(invalid_input("Azure Blob account key must not be empty"));
|
||||
}
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn validate_object_name(name: &str) -> io::Result<()> {
|
||||
if name.is_empty()
|
||||
|| name.len() > 1760
|
||||
|| name.chars().count() > 880
|
||||
|| name.split('/').any(|segment| matches!(segment, "." | ".."))
|
||||
{
|
||||
return Err(invalid_input(
|
||||
"Azure Blob tier name exceeds Stack Hub limits or contains a standalone dot segment",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_version(version: &str) -> io::Result<()> {
|
||||
if version.len() > 1024 || version.bytes().any(|byte| !byte.is_ascii_graphic()) {
|
||||
return Err(invalid_input("Azure Blob version ID is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn response_version(headers: &HeaderMap) -> io::Result<String> {
|
||||
let Some(version) = headers.get("x-ms-version-id") else {
|
||||
return Ok(String::new());
|
||||
};
|
||||
let version = version
|
||||
.to_str()
|
||||
.map_err(|_| invalid_data("Azure Blob version ID is not ASCII"))?;
|
||||
validate_version(version)?;
|
||||
Ok(version.to_string())
|
||||
}
|
||||
|
||||
fn validate_response_version(headers: &HeaderMap, requested: &str) -> io::Result<()> {
|
||||
if response_version(headers)? != requested {
|
||||
return Err(invalid_data("Azure Blob did not return the requested exact version"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete_preconditions(current: &HeaderMap) -> io::Result<HeaderMap> {
|
||||
let etag = current
|
||||
.get("etag")
|
||||
.ok_or_else(|| invalid_data("Azure Blob HEAD omitted the delete precondition"))?;
|
||||
let bytes = etag.as_bytes();
|
||||
// A wildcard, weak tag or list cannot identify a single matching generation.
|
||||
if bytes.len() < 3
|
||||
|| bytes.first() != Some(&b'"')
|
||||
|| bytes.last() != Some(&b'"')
|
||||
|| !bytes[1..bytes.len() - 1]
|
||||
.iter()
|
||||
.all(|byte| *byte == b'!' || (b'#'..=b'~').contains(byte))
|
||||
{
|
||||
return Err(invalid_data("Azure Blob HEAD omitted a single strong ETag"));
|
||||
}
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("if-match", etag.clone());
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
fn classify_head(headers: &HeaderMap) -> io::Result<TransitionCandidateProbe> {
|
||||
let version = response_version(headers)?;
|
||||
Ok(if version.is_empty() {
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
} else {
|
||||
TransitionCandidateProbe::VersionedPresent(version)
|
||||
})
|
||||
}
|
||||
|
||||
fn block_size(size: u64) -> io::Result<u64> {
|
||||
if size > MAX_BLOCK_BYTES * MAX_BLOCKS {
|
||||
return Err(invalid_input("Azure Blob exceeds the Stack Hub block count or object size limit"));
|
||||
}
|
||||
Ok(size.div_ceil(MAX_BLOCKS).max(MIN_BLOCK_BYTES))
|
||||
}
|
||||
|
||||
async fn read_part(reader: &mut (dyn AsyncRead + Send + Sync + Unpin), length: u64) -> io::Result<Bytes> {
|
||||
let length = usize::try_from(length).map_err(|_| invalid_input("Azure Blob block length is not representable"))?;
|
||||
let mut bytes = vec![0; length];
|
||||
reader.read_exact(&mut bytes).await?;
|
||||
Ok(Bytes::from(bytes))
|
||||
}
|
||||
|
||||
async fn ensure_eof(reader: &mut (dyn AsyncRead + Send + Sync + Unpin)) -> io::Result<()> {
|
||||
if reader.read(&mut [0_u8; 1]).await? != 0 {
|
||||
return Err(invalid_input("Azure Blob upload body exceeds its declared length"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn metadata_header_name(key: &str) -> String {
|
||||
format!("{METADATA_PREFIX}{}", hex(key.as_bytes()))
|
||||
}
|
||||
|
||||
fn metadata_headers(metadata: HashMap<String, String>) -> io::Result<HeaderMap> {
|
||||
let opts = build_transition_put_options(String::new(), metadata);
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("x-ms-blob-content-type", opts.content_type),
|
||||
("x-ms-blob-content-encoding", opts.content_encoding),
|
||||
("x-ms-blob-content-language", opts.content_language),
|
||||
("x-ms-blob-content-disposition", opts.content_disposition),
|
||||
("x-ms-blob-cache-control", opts.cache_control),
|
||||
] {
|
||||
if !value.is_empty() {
|
||||
insert_header(&mut headers, name, &value)?;
|
||||
}
|
||||
}
|
||||
let mut size = 0_usize;
|
||||
for (key, value) in opts.user_metadata {
|
||||
let key = key.to_ascii_lowercase();
|
||||
let key = key.strip_prefix("x-amz-meta-").unwrap_or(&key);
|
||||
let name = metadata_header_name(key);
|
||||
let value = base64_simd::STANDARD.encode_to_string(value.as_bytes());
|
||||
size = size.saturating_add(name.len()).saturating_add(value.len());
|
||||
if size > MAX_METADATA_BYTES {
|
||||
return Err(invalid_input("Azure Blob encoded metadata exceeds 8 KiB"));
|
||||
}
|
||||
if headers.contains_key(&name) {
|
||||
return Err(invalid_input("Azure Blob metadata contains conflicting names"));
|
||||
}
|
||||
insert_header(&mut headers, &name, &value)?;
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
fn requested_range(opts: &WarmBackendGetOpts) -> io::Result<Option<(u64, Option<u64>)>> {
|
||||
let start = u64::try_from(opts.start_offset).map_err(|_| invalid_input("Azure Blob range starts before zero"))?;
|
||||
let length = u64::try_from(opts.length).map_err(|_| invalid_input("Azure Blob range length is negative"))?;
|
||||
if length > 0 {
|
||||
let end = start
|
||||
.checked_add(length - 1)
|
||||
.ok_or_else(|| invalid_input("Azure Blob range overflows"))?;
|
||||
return Ok(Some((start, Some(end))));
|
||||
}
|
||||
Ok((start > 0).then_some((start, None)))
|
||||
}
|
||||
|
||||
fn validate_content_range(headers: &HeaderMap, requested: (u64, Option<u64>), length: u64) -> io::Result<()> {
|
||||
let value = headers
|
||||
.get("content-range")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
let range = value
|
||||
.strip_prefix("bytes ")
|
||||
.and_then(|value| value.split_once('/'))
|
||||
.and_then(|(range, total)| {
|
||||
let (start, end) = range.split_once('-')?;
|
||||
Some((start.parse::<u64>().ok()?, end.parse::<u64>().ok()?, total.parse::<u64>().ok()?))
|
||||
});
|
||||
let Some((start, end, total)) = range else {
|
||||
return Err(invalid_data("Azure Blob returned an invalid Content-Range"));
|
||||
};
|
||||
if start != requested.0
|
||||
|| end < start
|
||||
|| end >= total
|
||||
|| end != requested.1.unwrap_or(total - 1).min(total - 1)
|
||||
|| end.checked_sub(start).and_then(|size| size.checked_add(1)) != Some(length)
|
||||
{
|
||||
return Err(invalid_data("Azure Blob returned a different range from the one requested"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_control_body(mut response: reqwest::Response, timeout: Duration) -> io::Result<Vec<u8>> {
|
||||
tokio::time::timeout(timeout, async move {
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await.map_err(transport_error)? {
|
||||
if body.len().saturating_add(chunk.len()) > MAX_CONTROL_BYTES {
|
||||
return Err(invalid_data("Azure Blob control response exceeds 64 KiB"));
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(body)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(ErrorKind::TimedOut, "Azure Blob control response timed out"))?
|
||||
}
|
||||
|
||||
async fn service_error(response: reqwest::Response, operation: &'static str, timeout: Duration) -> io::Error {
|
||||
let status = response.status();
|
||||
let request_id = diagnostic_token(
|
||||
response
|
||||
.headers()
|
||||
.get("x-ms-request-id")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
);
|
||||
let mut code = diagnostic_token(
|
||||
response
|
||||
.headers()
|
||||
.get("x-ms-error-code")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
);
|
||||
if code == "unknown"
|
||||
&& let Ok(body) = read_control_body(response, timeout).await
|
||||
&& let Some(xml_code) = xml_error_code(&body)
|
||||
{
|
||||
code = diagnostic_token(Some(&xml_code));
|
||||
}
|
||||
let kind = if status == StatusCode::NOT_FOUND && code == "BlobNotFound" {
|
||||
ErrorKind::NotFound
|
||||
} else if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
|
||||
ErrorKind::PermissionDenied
|
||||
} else if status == StatusCode::PRECONDITION_FAILED {
|
||||
ErrorKind::AlreadyExists
|
||||
} else {
|
||||
ErrorKind::Other
|
||||
};
|
||||
io::Error::new(
|
||||
kind,
|
||||
AzureBlobError {
|
||||
operation,
|
||||
status,
|
||||
code,
|
||||
request_id,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn diagnostic_token(value: Option<&str>) -> String {
|
||||
value
|
||||
.filter(|value| {
|
||||
!value.is_empty() && value.len() <= 128 && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
})
|
||||
.unwrap_or("unknown")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn xml_error_code(body: &[u8]) -> Option<String> {
|
||||
let mut reader = Reader::from_reader(body);
|
||||
let mut depth = 0_u32;
|
||||
let mut root = false;
|
||||
let mut code = None;
|
||||
loop {
|
||||
match reader.read_event().ok()? {
|
||||
Event::Start(start) => {
|
||||
if depth == 0 {
|
||||
if root || start.local_name().as_ref() != "Error" {
|
||||
return None;
|
||||
}
|
||||
root = true;
|
||||
}
|
||||
if depth == 1 && start.local_name().as_ref() == "Code" {
|
||||
if code.is_some() {
|
||||
return None;
|
||||
}
|
||||
let text = reader.read_text(start.name()).ok()?;
|
||||
code = Some(quick_xml::escape::unescape(text.as_ref()).ok()?.into_owned());
|
||||
continue;
|
||||
}
|
||||
depth += 1;
|
||||
if depth > 16 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Event::End(_) => depth = depth.checked_sub(1)?,
|
||||
Event::Text(text) if depth == 0 && !text.as_ref().trim().is_empty() => return None,
|
||||
Event::CData(_) if depth == 0 => return None,
|
||||
Event::Eof => return (root && depth == 0).then_some(code).flatten(),
|
||||
Event::DocType(_) => return None,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn list_has_objects(body: &[u8]) -> io::Result<bool> {
|
||||
let mut reader = Reader::from_reader(body);
|
||||
let mut depth = 0_u32;
|
||||
let mut root = false;
|
||||
let mut blobs = false;
|
||||
let mut found = false;
|
||||
loop {
|
||||
let event = reader
|
||||
.read_event()
|
||||
.map_err(|_| invalid_data("Azure Blob listing is not valid XML"))?;
|
||||
let empty = matches!(event, Event::Empty(_));
|
||||
match event {
|
||||
Event::Start(start) | Event::Empty(start) => {
|
||||
if depth == 0 && (root || start.local_name().as_ref() != "EnumerationResults") {
|
||||
return Err(invalid_data("Azure Blob listing has an unexpected root"));
|
||||
}
|
||||
root |= start.local_name().as_ref() == "EnumerationResults";
|
||||
if start.local_name().as_ref() == "Blobs" {
|
||||
if blobs || depth != 1 {
|
||||
return Err(invalid_data("Azure Blob listing has an invalid Blobs element"));
|
||||
}
|
||||
blobs = true;
|
||||
}
|
||||
found |= matches!(start.local_name().as_ref(), "Blob" | "BlobPrefix");
|
||||
if !empty && start.local_name().as_ref() == "NextMarker" {
|
||||
let marker = reader
|
||||
.read_text(start.name())
|
||||
.map_err(|_| invalid_data("Azure Blob listing marker is invalid"))?;
|
||||
found |= !marker.is_empty();
|
||||
continue;
|
||||
}
|
||||
if !empty {
|
||||
depth += 1;
|
||||
if depth > 16 {
|
||||
return Err(invalid_data("Azure Blob listing exceeds the XML depth limit"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Event::End(_) => {
|
||||
depth = depth
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| invalid_data("Azure Blob listing has an invalid closing tag"))?;
|
||||
}
|
||||
Event::DocType(_) => return Err(invalid_data("Azure Blob listing must not contain a document type")),
|
||||
Event::Text(text) if depth == 0 && !text.as_ref().trim().is_empty() => {
|
||||
return Err(invalid_data("Azure Blob listing contains text outside its root"));
|
||||
}
|
||||
Event::CData(_) if depth == 0 => return Err(invalid_data("Azure Blob listing contains text outside its root")),
|
||||
Event::Eof if root && blobs && depth == 0 => return Ok(found),
|
||||
Event::Eof => return Err(invalid_data("Azure Blob listing is incomplete")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> io::Result<()> {
|
||||
let name =
|
||||
HeaderName::from_bytes(name.as_bytes()).map_err(|_| invalid_input("Azure Blob metadata header name is invalid"))?;
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_input("Azure Blob metadata header value is invalid"))?;
|
||||
headers.insert(name, value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn invalid_input(message: &'static str) -> io::Error {
|
||||
io::Error::new(ErrorKind::InvalidInput, message)
|
||||
}
|
||||
|
||||
fn invalid_data(message: &'static str) -> io::Error {
|
||||
io::Error::new(ErrorKind::InvalidData, message)
|
||||
}
|
||||
|
||||
fn transport_error(err: reqwest::Error) -> io::Error {
|
||||
let kind = if err.is_timeout() {
|
||||
ErrorKind::TimedOut
|
||||
} else {
|
||||
ErrorKind::Other
|
||||
};
|
||||
io::Error::new(kind, err.without_url())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "warm_backend_azure_blob_tests.rs"]
|
||||
mod tests;
|
||||
@@ -1,929 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::services::tier::warm_backend::{WarmBackendImpl, check_warm_backend};
|
||||
use rustfs_utils::http::metadata_compat::insert_str;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
struct Reply {
|
||||
status: StatusCode,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
disconnect: bool,
|
||||
}
|
||||
|
||||
fn reply(status: StatusCode, headers: &[(&str, &str)], body: &[u8]) -> Reply {
|
||||
let mut map = HeaderMap::new();
|
||||
for (name, value) in headers {
|
||||
insert_header(&mut map, name, value).expect("fixture headers should be valid");
|
||||
}
|
||||
Reply {
|
||||
status,
|
||||
headers: map,
|
||||
body: Bytes::copy_from_slice(body),
|
||||
disconnect: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn present(version: Option<&str>) -> Reply {
|
||||
let mut response = reply(StatusCode::OK, &[("content-length", "6"), ("etag", "\"original\"")], b"");
|
||||
if let Some(version) = version {
|
||||
insert_header(&mut response.headers, "x-ms-version-id", version).expect("fixture version should be valid");
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn missing() -> Reply {
|
||||
reply(StatusCode::NOT_FOUND, &[("x-ms-error-code", "BlobNotFound")], b"")
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Captured {
|
||||
method: String,
|
||||
target: String,
|
||||
headers: HeaderMap,
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
async fn capture(stream: &mut TcpStream) -> Captured {
|
||||
let mut bytes = Vec::new();
|
||||
let boundary = loop {
|
||||
let mut buffer = [0_u8; 4096];
|
||||
let count = stream.read(&mut buffer).await.expect("fixture request should be readable");
|
||||
assert_ne!(count, 0, "request must include a complete HTTP head");
|
||||
bytes.extend_from_slice(&buffer[..count]);
|
||||
if let Some(index) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break index + 4;
|
||||
}
|
||||
assert!(bytes.len() <= 32 * 1024, "request headers must remain bounded");
|
||||
};
|
||||
let text = std::str::from_utf8(&bytes[..boundary]).expect("request headers should be ASCII");
|
||||
let mut lines = text.split("\r\n");
|
||||
let mut request_line = lines.next().expect("request line").split_whitespace();
|
||||
let method = request_line.next().expect("request method").to_string();
|
||||
let target = request_line.next().expect("request target").to_string();
|
||||
let mut headers = HeaderMap::new();
|
||||
for line in lines.filter(|line| !line.is_empty()) {
|
||||
let (name, value) = line.split_once(':').expect("request header should include a colon");
|
||||
insert_header(&mut headers, name, value.trim()).expect("request headers should parse");
|
||||
}
|
||||
let length = headers
|
||||
.get("content-length")
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.expect("length should be ASCII")
|
||||
.parse::<usize>()
|
||||
.expect("length should be numeric")
|
||||
})
|
||||
.unwrap_or_default();
|
||||
assert!(length <= 17 * 1024 * 1024, "fixture only accepts its bounded test bodies");
|
||||
let mut body = bytes[boundary..].to_vec();
|
||||
let received = body.len();
|
||||
body.resize(length, 0);
|
||||
stream
|
||||
.read_exact(&mut body[received..])
|
||||
.await
|
||||
.expect("request body should match its declared length");
|
||||
Captured {
|
||||
method,
|
||||
target,
|
||||
headers,
|
||||
body,
|
||||
}
|
||||
}
|
||||
|
||||
async fn fixture(responses: Vec<Reply>) -> (WarmBackendAzureBlob, JoinHandle<Vec<Captured>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind");
|
||||
let address = listener.local_addr().expect("fixture address");
|
||||
let task = tokio::spawn(async move {
|
||||
let mut requests = Vec::new();
|
||||
for mut response in responses {
|
||||
let (mut stream, _) = tokio::time::timeout(Duration::from_secs(10), listener.accept())
|
||||
.await
|
||||
.expect("expected request must arrive")
|
||||
.expect("fixture should accept");
|
||||
requests.push(capture(&mut stream).await);
|
||||
if response.disconnect {
|
||||
continue;
|
||||
}
|
||||
if !response.headers.contains_key("content-length") {
|
||||
insert_header(&mut response.headers, "content-length", &response.body.len().to_string())
|
||||
.expect("fixture length should be valid");
|
||||
}
|
||||
let mut wire = format!(
|
||||
"HTTP/1.1 {} {}\r\nConnection: close\r\n",
|
||||
response.status.as_u16(),
|
||||
response.status.canonical_reason().unwrap_or("")
|
||||
);
|
||||
for (name, value) in &response.headers {
|
||||
wire.push_str(&format!("{name}: {}\r\n", value.to_str().expect("fixture header is ASCII")));
|
||||
}
|
||||
wire.push_str("\r\n");
|
||||
stream.write_all(wire.as_bytes()).await.expect("fixture should send headers");
|
||||
stream.write_all(&response.body).await.expect("fixture should send body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
rustfs_tls_runtime::ensure_crypto_provider().expect("fixture TLS provider should initialize");
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.retry(reqwest::retry::never())
|
||||
.read_timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.expect("fixture client should build");
|
||||
(
|
||||
WarmBackendAzureBlob {
|
||||
client,
|
||||
endpoint: Url::parse(&format!("http://{address}")).expect("fixture endpoint"),
|
||||
account: "account".to_string(),
|
||||
key: vec![7; 32],
|
||||
container: "container".to_string(),
|
||||
prefix: "cold".to_string(),
|
||||
request_timeout: Duration::from_secs(5),
|
||||
},
|
||||
task,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutation_probe_uses_blob_shared_key_and_cleans_up_with_an_etag() {
|
||||
let responses = vec![
|
||||
reply(StatusCode::CREATED, &[], b""),
|
||||
present(None),
|
||||
reply(StatusCode::PARTIAL_CONTENT, &[("content-range", "bytes 0-5/6")], b"RustFS"),
|
||||
present(None),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
missing(),
|
||||
];
|
||||
let (backend, task) = fixture(responses).await;
|
||||
let backend: WarmBackendImpl = Box::new(backend);
|
||||
check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect("native Blob mutation probe should succeed");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(
|
||||
requests.iter().map(|request| request.method.as_str()).collect::<Vec<_>>(),
|
||||
["PUT", "HEAD", "GET", "HEAD", "DELETE", "HEAD"]
|
||||
);
|
||||
assert!(requests[0].target.starts_with("/container/cold/rustfs-tier-probe-"));
|
||||
assert_eq!(requests[0].body, b"RustFS");
|
||||
assert_eq!(requests[0].headers["x-ms-blob-type"], "BlockBlob");
|
||||
assert_eq!(requests[0].headers["if-none-match"], "*");
|
||||
assert_eq!(requests[4].headers["if-match"], "\"original\"");
|
||||
for request in &requests {
|
||||
assert_eq!(request.target, requests[0].target, "the whole probe must address the same blob");
|
||||
assert_eq!(request.headers["x-ms-version"], API_VERSION);
|
||||
assert!(
|
||||
request.headers["authorization"]
|
||||
.to_str()
|
||||
.expect("authorization is ASCII")
|
||||
.starts_with("SharedKey account:")
|
||||
);
|
||||
assert!(!request.headers.contains_key("x-amz-date"));
|
||||
assert!(!request.target.contains("location"));
|
||||
assert!(!request.target.contains("versionid"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn versioned_mutation_probe_cleans_up_after_forbidden_current_version_delete() {
|
||||
let version = "2026-11-06T01:02:03.0000000Z";
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(StatusCode::CREATED, &[("x-ms-version-id", version)], b""),
|
||||
present(Some(version)),
|
||||
reply(
|
||||
StatusCode::PARTIAL_CONTENT,
|
||||
&[("content-range", "bytes 0-5/6"), ("x-ms-version-id", version)],
|
||||
b"RustFS",
|
||||
),
|
||||
reply(StatusCode::FORBIDDEN, &[], b"<Error><Code>OperationNotAllowedOnRootBlob</Code></Error>"),
|
||||
present(Some(version)),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
missing(),
|
||||
])
|
||||
.await;
|
||||
let backend: WarmBackendImpl = Box::new(backend);
|
||||
check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect("a versioned Add/Verify probe must finish cleanup after the specific 403");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(
|
||||
requests.iter().map(|request| request.method.as_str()).collect::<Vec<_>>(),
|
||||
["PUT", "HEAD", "GET", "DELETE", "HEAD", "DELETE", "DELETE", "HEAD"]
|
||||
);
|
||||
let base = &requests[0].target;
|
||||
assert!(base.starts_with("/container/cold/rustfs-tier-probe-"));
|
||||
for index in [1, 4, 5, 7] {
|
||||
assert_eq!(&requests[index].target, base, "base operations must retain the probe identity");
|
||||
}
|
||||
for index in [2, 3, 6] {
|
||||
let url = Url::parse(&format!("http://fixture{}", requests[index].target)).expect("captured URL");
|
||||
assert_eq!(url.path(), base);
|
||||
assert_eq!(
|
||||
url.query_pairs().collect::<Vec<_>>(),
|
||||
[("versionid".into(), version.into())],
|
||||
"exact reads and deletes must retain the timestamp version"
|
||||
);
|
||||
}
|
||||
assert_eq!(requests[5].headers["if-match"], "\"original\"");
|
||||
for request in &requests {
|
||||
assert_eq!(request.headers["x-ms-version"], "2020-10-02");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_put_preserves_azure_error_and_never_deletes_an_existing_blob() {
|
||||
for (status, code) in [
|
||||
(StatusCode::BAD_REQUEST, "InvalidHeaderValue"),
|
||||
(StatusCode::PRECONDITION_FAILED, "ConditionNotMet"),
|
||||
] {
|
||||
let (backend, task) = fixture(vec![reply(
|
||||
status,
|
||||
&[("x-ms-error-code", code), ("x-ms-request-id", "abc-123")],
|
||||
b"<Error><Code>PRIVATE-RESPONSE-BODY</Code></Error>",
|
||||
)])
|
||||
.await;
|
||||
let backend: WarmBackendImpl = Box::new(backend);
|
||||
let err = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("deterministic rejected write must fail");
|
||||
assert!(err.message.contains(code));
|
||||
assert!(err.message.contains("abc-123"));
|
||||
assert!(!err.message.contains("PRIVATE-RESPONSE-BODY"));
|
||||
assert!(!err.message.contains("cleanup is incomplete"));
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.len(), 1, "a rejected conditional PUT must not trigger cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_version_headers_use_the_unversioned_read_and_delete_paths() {
|
||||
let mut head = present(Some(""));
|
||||
head.headers.insert("content-length", HeaderValue::from_static("0"));
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(StatusCode::CREATED, &[("x-ms-version-id", "")], b""),
|
||||
reply(StatusCode::OK, &[("x-ms-version-id", "")], b""),
|
||||
head,
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
])
|
||||
.await;
|
||||
let version = backend
|
||||
.put("empty", ReaderImpl::Body(Bytes::new()), 0)
|
||||
.await
|
||||
.expect("empty version is unversioned");
|
||||
assert!(version.is_empty());
|
||||
let mut reader = backend
|
||||
.get("empty", &version, WarmBackendGetOpts::default())
|
||||
.await
|
||||
.expect("unversioned read");
|
||||
let mut body = Vec::new();
|
||||
reader.read_to_end(&mut body).await.expect("empty blob should read");
|
||||
assert!(body.is_empty());
|
||||
backend
|
||||
.remove("empty", &version)
|
||||
.await
|
||||
.expect("unversioned conditional delete");
|
||||
for request in task.await.expect("fixture should join") {
|
||||
assert_eq!(request.target, "/container/cold/empty");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_credentials_are_reported_without_an_uncertain_cleanup() {
|
||||
let (backend, task) = fixture(vec![reply(
|
||||
StatusCode::FORBIDDEN,
|
||||
&[("x-ms-error-code", "AuthenticationFailed")],
|
||||
b"",
|
||||
)])
|
||||
.await;
|
||||
let backend: WarmBackendImpl = Box::new(backend);
|
||||
let err = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("invalid credentials must fail");
|
||||
assert_eq!(err.code, crate::services::tier::tier_handlers::ERR_TIER_INVALID_CREDENTIALS.code);
|
||||
assert_eq!(task.await.expect("fixture should join").len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_put_response_is_not_replayed_and_its_candidate_is_cleaned() {
|
||||
let mut lost = reply(StatusCode::CREATED, &[], b"");
|
||||
lost.disconnect = true;
|
||||
let (backend, task) = fixture(vec![
|
||||
lost,
|
||||
present(None),
|
||||
present(None),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
missing(),
|
||||
missing(),
|
||||
])
|
||||
.await;
|
||||
let backend: WarmBackendImpl = Box::new(backend);
|
||||
let err = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("a lost write acknowledgement must remain a failure");
|
||||
assert!(!err.message.contains("cleanup is incomplete"));
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.iter().filter(|request| request.method == "PUT").count(), 1);
|
||||
assert_eq!(requests.iter().filter(|request| request.method == "DELETE").count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn block_upload_commits_only_the_complete_declared_body() {
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(StatusCode::CREATED, &[], b""),
|
||||
reply(StatusCode::CREATED, &[], b""),
|
||||
reply(StatusCode::CREATED, &[], b""),
|
||||
])
|
||||
.await;
|
||||
let length = usize::try_from(MIN_BLOCK_BYTES + 5).expect("fixture size");
|
||||
let bytes = Bytes::from(vec![42; length]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.put(
|
||||
"large",
|
||||
ReaderImpl::ObjectBody(ReadCloser::new(Cursor::new(bytes.clone()))),
|
||||
i64::try_from(length).expect("fixture length")
|
||||
)
|
||||
.await
|
||||
.expect("block upload"),
|
||||
""
|
||||
);
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.len(), 3);
|
||||
let mut block_ids = Vec::new();
|
||||
let mut body = Vec::new();
|
||||
for request in &requests[..2] {
|
||||
let url = Url::parse(&format!("http://fixture{}", request.target)).expect("captured URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query["comp"], "block");
|
||||
block_ids.push(query["blockid"].clone());
|
||||
assert!(request.headers.contains_key("content-md5"));
|
||||
body.extend_from_slice(&request.body);
|
||||
}
|
||||
assert_eq!(body, bytes);
|
||||
assert_ne!(block_ids[0], block_ids[1]);
|
||||
assert!(requests[2].target.ends_with("?comp=blocklist"));
|
||||
assert_eq!(requests[2].headers["if-none-match"], "*");
|
||||
let expected = format!(
|
||||
"<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList><Uncommitted>{}</Uncommitted><Uncommitted>{}</Uncommitted></BlockList>",
|
||||
block_ids[0], block_ids[1],
|
||||
);
|
||||
assert_eq!(requests[2].body, expected.as_bytes());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_error_remains_visible_after_successful_probe_compensation() {
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&[("x-ms-error-code", "InternalError"), ("x-ms-request-id", "error-123")],
|
||||
b"",
|
||||
),
|
||||
present(None),
|
||||
present(None),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
missing(),
|
||||
missing(),
|
||||
])
|
||||
.await;
|
||||
let backend: WarmBackendImpl = Box::new(backend);
|
||||
let error = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("failed upload remains an error after cleanup");
|
||||
assert!(error.message.contains("500"));
|
||||
assert!(error.message.contains("InternalError"));
|
||||
assert!(error.message.contains("error-123"));
|
||||
assert!(!error.message.contains("cleanup is incomplete"));
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.iter().filter(|request| request.method == "PUT").count(), 1);
|
||||
assert_eq!(requests.iter().filter(|request| request.method == "DELETE").count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_blob_and_declared_length_failures_are_handled_before_commit() {
|
||||
let (backend, task) = fixture(vec![reply(StatusCode::CREATED, &[], b"")]).await;
|
||||
backend
|
||||
.put("empty", ReaderImpl::Body(Bytes::new()), 0)
|
||||
.await
|
||||
.expect("empty upload");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests[0].headers["content-length"], "0");
|
||||
assert_eq!(requests[0].headers["content-md5"], "1B2M2Y8AsgTpgAmY7PhCfg==");
|
||||
for (body, kind) in [
|
||||
(b"short".as_slice(), ErrorKind::UnexpectedEof),
|
||||
(b"too-long".as_slice(), ErrorKind::InvalidInput),
|
||||
] {
|
||||
let (backend, task) = fixture(vec![]).await;
|
||||
let err = backend
|
||||
.put("bad", ReaderImpl::Body(Bytes::copy_from_slice(body)), 6)
|
||||
.await
|
||||
.expect_err("length mismatch");
|
||||
assert_eq!(err.kind(), kind);
|
||||
assert!(task.await.expect("fixture should join").is_empty());
|
||||
}
|
||||
let (backend, task) = fixture(vec![reply(StatusCode::CREATED, &[], b"")]).await;
|
||||
let body = Bytes::from(vec![0; usize::try_from(MIN_BLOCK_BYTES + 1).expect("fixture size")]);
|
||||
let err = backend
|
||||
.put(
|
||||
"short-block",
|
||||
ReaderImpl::Body(body),
|
||||
i64::try_from(MIN_BLOCK_BYTES + 2).expect("fixture length"),
|
||||
)
|
||||
.await
|
||||
.expect_err("short final block");
|
||||
assert_eq!(err.kind(), ErrorKind::UnexpectedEof);
|
||||
assert_eq!(
|
||||
task.await.expect("fixture should join").len(),
|
||||
1,
|
||||
"short input must never commit its staged blocks"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn range_get_preserves_key_encoding_and_exact_version_for_delete() {
|
||||
let version = "2026-11-06T01:02:03.0000000Z";
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(
|
||||
StatusCode::PARTIAL_CONTENT,
|
||||
&[("content-range", "bytes 1-3/6"), ("x-ms-version-id", version)],
|
||||
b"ust",
|
||||
),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
])
|
||||
.await;
|
||||
let object = "dir/a b+?#%2F.bin";
|
||||
let mut reader = backend
|
||||
.get(
|
||||
object,
|
||||
version,
|
||||
WarmBackendGetOpts {
|
||||
start_offset: 1,
|
||||
length: 3,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned range read");
|
||||
let mut body = Vec::new();
|
||||
reader.read_to_end(&mut body).await.expect("range body");
|
||||
assert_eq!(body, b"ust");
|
||||
backend.remove_exact(object, version).await.expect("exact delete");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.len(), 2, "exact delete needs no current-version HEAD");
|
||||
for request in requests {
|
||||
let url = Url::parse(&format!("http://fixture{}", request.target)).expect("captured URL");
|
||||
assert_eq!(
|
||||
urlencoding::decode(url.path()).expect("decoded path"),
|
||||
format!("/container/cold/{object}")
|
||||
);
|
||||
let query: Vec<_> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query, [("versionid".to_string(), version.to_string())]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_version_delete_demotes_only_the_matching_root_before_exact_cleanup() {
|
||||
for status in [StatusCode::FORBIDDEN, StatusCode::CONFLICT] {
|
||||
let version = "target-version";
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(status, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
|
||||
present(Some(version)),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
])
|
||||
.await;
|
||||
backend
|
||||
.remove_exact("current", version)
|
||||
.await
|
||||
.expect("current version should be removable");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.len(), 4);
|
||||
assert_eq!(requests[0].target, "/container/cold/current?versionid=target-version");
|
||||
assert_eq!(requests[1].method, "HEAD");
|
||||
assert_eq!(requests[1].target, "/container/cold/current");
|
||||
assert_eq!(requests[2].method, "DELETE");
|
||||
assert_eq!(requests[2].target, "/container/cold/current");
|
||||
assert_eq!(requests[2].headers["if-match"], "\"original\"");
|
||||
assert_eq!(requests[3].method, "DELETE");
|
||||
assert_eq!(requests[3].target, requests[0].target);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_version_delete_refuses_replacements_before_and_after_head() {
|
||||
for replaced_after_head in [false, true] {
|
||||
let mut responses = vec![
|
||||
reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
|
||||
present(Some(if replaced_after_head {
|
||||
"target-version"
|
||||
} else {
|
||||
"replacement-version"
|
||||
})),
|
||||
];
|
||||
if replaced_after_head {
|
||||
responses.push(reply(StatusCode::PRECONDITION_FAILED, &[("x-ms-error-code", "ConditionNotMet")], b""));
|
||||
}
|
||||
let (backend, task) = fixture(responses).await;
|
||||
assert!(backend.remove_exact("current", "target-version").await.is_err());
|
||||
let requests = task.await.expect("fixture should join");
|
||||
if replaced_after_head {
|
||||
assert_eq!(requests.len(), 3, "a failed condition must stop exact cleanup");
|
||||
assert_eq!(requests[2].target, "/container/cold/current");
|
||||
assert_eq!(requests[2].headers["if-match"], "\"original\"");
|
||||
} else {
|
||||
assert_eq!(requests.len(), 2, "a different current version must never authorize a base delete");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_version_delete_retries_the_exact_version_when_the_root_is_already_absent() {
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
|
||||
missing(),
|
||||
reply(StatusCode::ACCEPTED, &[], b""),
|
||||
])
|
||||
.await;
|
||||
backend
|
||||
.remove_exact("current", "target-version")
|
||||
.await
|
||||
.expect("previous version cleanup");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert_eq!(requests[2].target, "/container/cold/current?versionid=target-version");
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.all(|request| request.method != "DELETE" || request.target.contains("versionid="))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_version_delete_keeps_partial_or_uncertain_cleanup_retryable() {
|
||||
for fails_during_exact_cleanup in [false, true] {
|
||||
for lost_response in [false, true] {
|
||||
let mut responses = vec![
|
||||
reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
|
||||
present(Some("target-version")),
|
||||
];
|
||||
if fails_during_exact_cleanup {
|
||||
responses.push(reply(StatusCode::ACCEPTED, &[], b""));
|
||||
}
|
||||
let mut failure = reply(StatusCode::INTERNAL_SERVER_ERROR, &[("x-ms-error-code", "InternalError")], b"");
|
||||
failure.disconnect = lost_response;
|
||||
responses.push(failure);
|
||||
responses.push(reply(StatusCode::ACCEPTED, &[], b""));
|
||||
let (backend, task) = fixture(responses).await;
|
||||
backend
|
||||
.remove_exact("current", "target-version")
|
||||
.await
|
||||
.expect_err("base removal alone or an uncertain response must not complete cleanup");
|
||||
backend
|
||||
.remove_exact("current", "target-version")
|
||||
.await
|
||||
.expect("retry exact cleanup");
|
||||
let requests = task.await.expect("fixture should join");
|
||||
let expected = if fails_during_exact_cleanup { 5 } else { 4 };
|
||||
assert_eq!(requests.len(), expected, "uncertain destructive requests must not be replayed internally");
|
||||
assert_eq!(requests[2].headers["if-match"], "\"original\"");
|
||||
assert_eq!(requests[expected - 1].target, "/container/cold/current?versionid=target-version");
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.method == "DELETE" && !request.target.contains('?'))
|
||||
.count(),
|
||||
1,
|
||||
"a retry must not delete the root again"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_requires_a_single_strong_etag_and_an_exact_current_version() {
|
||||
for versioned in [false, true] {
|
||||
for etag in [
|
||||
None,
|
||||
Some(""),
|
||||
Some("*"),
|
||||
Some("W/\"weak\""),
|
||||
Some("\"first\", \"second\""),
|
||||
Some("\"\""),
|
||||
] {
|
||||
let mut head = present(if versioned { Some("target-version") } else { None });
|
||||
head.headers.remove("etag");
|
||||
if let Some(etag) = etag {
|
||||
insert_header(&mut head.headers, "etag", etag).expect("fixture etag should be representable");
|
||||
}
|
||||
let mut responses = Vec::new();
|
||||
if versioned {
|
||||
responses.push(reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""));
|
||||
}
|
||||
responses.push(head);
|
||||
let (backend, task) = fixture(responses).await;
|
||||
assert!(
|
||||
backend
|
||||
.remove("current", if versioned { "target-version" } else { "" })
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.all(|request| request.method != "DELETE" || request.target.contains("versionid=")),
|
||||
"a missing, wildcard, weak or multi-value ETag cannot authorize root removal"
|
||||
);
|
||||
}
|
||||
}
|
||||
for current_version in [None, Some("")] {
|
||||
let (backend, task) = fixture(vec![
|
||||
reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
|
||||
present(current_version),
|
||||
])
|
||||
.await;
|
||||
assert!(backend.remove_exact("current", "target-version").await.is_err());
|
||||
assert_eq!(task.await.expect("fixture should join").len(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_delete_does_not_demote_roots_for_other_service_errors() {
|
||||
for (status, code) in [
|
||||
(StatusCode::CONFLICT, "BlobImmutableDueToPolicy"),
|
||||
(StatusCode::FORBIDDEN, "BlobImmutableDueToPolicy"),
|
||||
(StatusCode::FORBIDDEN, "AuthenticationFailed"),
|
||||
(StatusCode::FORBIDDEN, "AuthorizationFailure"),
|
||||
(StatusCode::FORBIDDEN, "AuthorizationPermissionMismatch"),
|
||||
(StatusCode::FORBIDDEN, "OperationNotAllowedOnRootBlobOther"),
|
||||
(StatusCode::UNAUTHORIZED, "OperationNotAllowedOnRootBlob"),
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "OperationNotAllowedOnRootBlob"),
|
||||
] {
|
||||
let (backend, task) = fixture(vec![reply(status, &[("x-ms-error-code", code)], b"")]).await;
|
||||
let error = backend
|
||||
.remove_exact("current", "target-version")
|
||||
.await
|
||||
.expect_err("service rejection");
|
||||
assert_eq!(AzureBlobError::from_io(&error).expect("typed service error").code, code);
|
||||
assert_eq!(
|
||||
task.await.expect("fixture should join").len(),
|
||||
1,
|
||||
"unrelated errors cannot trigger root removal"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_reserves_the_complete_canonical_lifecycle_name_before_backend_creation() {
|
||||
use crate::bucket::lifecycle::transition_transaction::canonical_transition_remote_object;
|
||||
let mut config = TierAzure {
|
||||
access_key: "account".to_string(),
|
||||
secret_key: base64_simd::STANDARD.encode_to_string([7; 32]),
|
||||
bucket: "container".to_string(),
|
||||
prefix: "a".repeat(800),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
WarmBackendAzureBlob::new(&config).is_err(),
|
||||
"a prefix that fits only the probe is unusable for lifecycle"
|
||||
);
|
||||
let object = canonical_transition_remote_object(
|
||||
uuid::Uuid::from_u128(1),
|
||||
"source-bucket",
|
||||
uuid::Uuid::from_u128(2),
|
||||
uuid::Uuid::from_u128(3),
|
||||
)
|
||||
.expect("canonical lifecycle key");
|
||||
let max_ascii_prefix = 880 - object.chars().count() - 1;
|
||||
config.prefix = "a".repeat(max_ascii_prefix);
|
||||
let backend = WarmBackendAzureBlob::new(&config).expect("complete name at the character limit");
|
||||
assert!(backend.object_url(&object, "").is_ok());
|
||||
config.prefix.push('/');
|
||||
let backend = WarmBackendAzureBlob::new(&config).expect("one trailing slash is normalized");
|
||||
assert!(backend.object_url(&object, "").is_ok());
|
||||
config.prefix = "a".repeat(max_ascii_prefix + 1);
|
||||
assert!(WarmBackendAzureBlob::new(&config).is_err(), "reject one extra character before I/O");
|
||||
let max_unicode_prefix = (1760 - object.len() - 1) / '界'.len_utf8();
|
||||
config.prefix = "界".repeat(max_unicode_prefix);
|
||||
let backend = WarmBackendAzureBlob::new(&config).expect("complete Unicode name within the byte limit");
|
||||
assert!(backend.object_url(&object, "").is_ok());
|
||||
config.prefix.push('界');
|
||||
assert!(
|
||||
WarmBackendAzureBlob::new(&config).is_err(),
|
||||
"the byte limit must include the canonical suffix"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_drift_and_incorrect_ranges_fail_closed() {
|
||||
let (backend, task) = fixture(vec![present(Some("new-version"))]).await;
|
||||
let err = backend
|
||||
.remove("changed", "")
|
||||
.await
|
||||
.expect_err("unversioned delete must reject version drift");
|
||||
assert!(err.to_string().contains("became versioned"));
|
||||
assert_eq!(task.await.expect("fixture should join").len(), 1);
|
||||
for headers in [
|
||||
vec![("content-range", "bytes 0-2/6")],
|
||||
vec![("content-range", "bytes 1-3/6"), ("x-ms-version-id", "unexpected-version")],
|
||||
] {
|
||||
let (backend, task) = fixture(vec![reply(StatusCode::PARTIAL_CONTENT, &headers, b"ust")]).await;
|
||||
let result = backend
|
||||
.get(
|
||||
"object",
|
||||
"",
|
||||
WarmBackendGetOpts {
|
||||
start_offset: 1,
|
||||
length: 3,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "wrong offset or an unexpected version cannot satisfy the read");
|
||||
task.await.expect("fixture should join");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unversioned_delete_keeps_the_head_precondition_when_the_blob_changes() {
|
||||
let (backend, task) = fixture(vec![
|
||||
present(None),
|
||||
reply(StatusCode::PRECONDITION_FAILED, &[("x-ms-error-code", "ConditionNotMet")], b""),
|
||||
])
|
||||
.await;
|
||||
assert!(backend.remove("changed", "").await.is_err());
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert_eq!(requests[1].headers["if-match"], "\"original\"");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn absence_requires_blob_not_found_and_container_listing_is_bounded() {
|
||||
for (response, missing_expected) in [
|
||||
(missing(), true),
|
||||
(reply(StatusCode::NOT_FOUND, &[("x-ms-error-code", "ContainerNotFound")], b""), false),
|
||||
(reply(StatusCode::NOT_FOUND, &[], b""), false),
|
||||
] {
|
||||
let (backend, task) = fixture(vec![response]).await;
|
||||
let result = backend.probe_transition_candidate("object").await;
|
||||
if missing_expected {
|
||||
assert_eq!(result.expect("explicit blob absence"), TransitionCandidateProbe::Missing);
|
||||
} else {
|
||||
assert!(result.is_err(), "an absent container or an ambiguous HEAD is not blob absence");
|
||||
}
|
||||
task.await.expect("fixture should join");
|
||||
}
|
||||
for (body, in_use) in [
|
||||
(b"<EnumerationResults><Blobs/><NextMarker/></EnumerationResults>".as_slice(), false),
|
||||
(
|
||||
b"<EnumerationResults><Blobs><Blob><Name>cold/object</Name></Blob></Blobs><NextMarker/></EnumerationResults>"
|
||||
.as_slice(),
|
||||
true,
|
||||
),
|
||||
(
|
||||
b"<EnumerationResults><Blobs/><NextMarker>next</NextMarker></EnumerationResults>".as_slice(),
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let (backend, task) = fixture(vec![reply(StatusCode::OK, &[], body)]).await;
|
||||
assert_eq!(backend.in_use().await.expect("native container listing"), in_use);
|
||||
let requests = task.await.expect("fixture should join");
|
||||
assert!(
|
||||
requests[0]
|
||||
.target
|
||||
.contains("restype=container&comp=list&prefix=cold%2F&maxresults=1")
|
||||
);
|
||||
}
|
||||
let oversized = vec![b' '; MAX_CONTROL_BYTES + 1];
|
||||
let (backend, task) = fixture(vec![reply(StatusCode::OK, &[], &oversized)]).await;
|
||||
assert!(backend.in_use().await.is_err(), "oversized listing cannot establish an empty destination");
|
||||
task.await.expect("fixture should join");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_requires_the_uploaded_transaction_and_destination_identity() {
|
||||
let identity = TransitionCandidateIdentity {
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
destination_id: [3; 32],
|
||||
};
|
||||
let mut metadata = HashMap::new();
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITION_TRANSACTION_ID, identity.transaction_id.to_string());
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, hex(identity.destination_id));
|
||||
let headers = metadata_headers(metadata).expect("identity metadata should encode for Azure");
|
||||
assert_eq!(
|
||||
headers
|
||||
.keys()
|
||||
.filter(|name| name.as_str().starts_with(METADATA_PREFIX))
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
let mut owned = present(Some("owned-version"));
|
||||
owned.headers.extend(headers);
|
||||
let (backend, task) = fixture(vec![owned, present(None)]).await;
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_for("object", identity)
|
||||
.await
|
||||
.expect("owned candidate"),
|
||||
TransitionCandidateProbe::VersionedPresent("owned-version".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_for("object", identity)
|
||||
.await
|
||||
.expect("foreign candidate classification"),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
task.await.expect("fixture should join");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configuration_xml_and_size_limits_fail_before_network_io() {
|
||||
let mut config = TierAzure {
|
||||
access_key: "account".to_string(),
|
||||
secret_key: base64_simd::STANDARD.encode_to_string([7; 32]),
|
||||
bucket: "container".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
WarmBackendAzureBlob::new(&config).is_ok(),
|
||||
"public endpoint should be derived from the account"
|
||||
);
|
||||
config.endpoint = "http://127.0.0.1:10000".to_string();
|
||||
assert!(
|
||||
WarmBackendAzureBlob::new(&config).is_err(),
|
||||
"production constructor must apply the outbound guard"
|
||||
);
|
||||
config.endpoint = "https://account.blob.example/path".to_string();
|
||||
assert!(
|
||||
WarmBackendAzureBlob::new(&config).is_err(),
|
||||
"an endpoint path cannot rewrite object addressing"
|
||||
);
|
||||
for name in [".", "..", "a/../b", "a/./b"] {
|
||||
assert!(validate_object_name(name).is_err());
|
||||
}
|
||||
assert_eq!(
|
||||
block_size(MAX_BLOCK_BYTES * MAX_BLOCKS).expect("largest supported object"),
|
||||
MAX_BLOCK_BYTES
|
||||
);
|
||||
assert!(block_size(MAX_BLOCK_BYTES * MAX_BLOCKS + 1).is_err());
|
||||
for xml in [
|
||||
b"<html>not a container</html>".as_slice(),
|
||||
b"<EnumerationResults><Blobs>".as_slice(),
|
||||
b"<!DOCTYPE x><EnumerationResults><Blobs/></EnumerationResults>".as_slice(),
|
||||
b"<EnumerationResults><Other><Blobs/></Other></EnumerationResults>".as_slice(),
|
||||
b"garbage<EnumerationResults><Blobs/></EnumerationResults>".as_slice(),
|
||||
] {
|
||||
assert!(list_has_objects(xml).is_err());
|
||||
}
|
||||
assert_eq!(
|
||||
xml_error_code(b"<Error><Code>BlobNotFound</Code></Error>").as_deref(),
|
||||
Some("BlobNotFound")
|
||||
);
|
||||
assert!(xml_error_code(b"<Error><Code>BlobNotFound</Code>").is_none());
|
||||
assert!(xml_error_code(b"<Error><Code>BlobNotFound</Code></Error>garbage").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_response_has_a_total_deadline_after_headers_arrive() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("control fixture should bind");
|
||||
let address = listener.local_addr().expect("control fixture address");
|
||||
let (release, wait) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("control fixture should accept");
|
||||
capture(&mut stream).await;
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 16\r\n\r\n")
|
||||
.await
|
||||
.expect("control response headers");
|
||||
let _ = wait.await;
|
||||
});
|
||||
let (mut backend, empty) = fixture(Vec::new()).await;
|
||||
empty.await.expect("unused fixture should close");
|
||||
backend.endpoint = Url::parse(&format!("http://{address}")).expect("control endpoint should parse");
|
||||
let response = backend
|
||||
.send(Method::GET, backend.container_url().expect("container URL"), HeaderMap::new(), None)
|
||||
.await
|
||||
.expect("headers should arrive before testing the body deadline");
|
||||
let err = tokio::time::timeout(Duration::from_secs(1), read_control_body(response, Duration::from_millis(20)))
|
||||
.await
|
||||
.expect("body deadline must be shorter than the transport idle timeout")
|
||||
.expect_err("an unfinished control response must time out");
|
||||
assert_eq!(err.kind(), ErrorKind::TimedOut);
|
||||
let _ = release.send(());
|
||||
server.await.expect("control fixture should join");
|
||||
}
|
||||
@@ -351,7 +351,7 @@ fn transition_candidate_metadata(info: ObjectInfo) -> HashMap<String, String> {
|
||||
metadata
|
||||
}
|
||||
|
||||
pub(super) fn transition_candidate_metadata_matches(
|
||||
fn transition_candidate_metadata_matches(
|
||||
metadata: &HashMap<String, String>,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<bool, std::io::Error> {
|
||||
|
||||
@@ -284,9 +284,8 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
|
||||
present_count += 1;
|
||||
let part = &parts[part_idx];
|
||||
let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
|
||||
let (count, protected_count, _) = part_quorum.entry(key).or_insert((0, 0, part));
|
||||
let (count, _) = part_quorum.entry(key).or_insert((0, part));
|
||||
*count += 1;
|
||||
*protected_count += usize::from(part.encrypted_etag);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -297,16 +296,12 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
|
||||
}
|
||||
}
|
||||
|
||||
let max_part = part_quorum.values().max_by_key(|(count, _, _)| count);
|
||||
let max_quorum = max_part.map_or(0, |(count, _, _)| *count).max(missing_count);
|
||||
if let Some((count, protected_count, part)) = max_part
|
||||
let max_part = part_quorum.values().max_by_key(|(count, _)| count);
|
||||
let max_quorum = max_part.map_or(0, |(count, _)| *count).max(missing_count);
|
||||
if let Some((count, part)) = max_part
|
||||
&& *count >= read_quorum
|
||||
{
|
||||
let mut part = (*part).clone();
|
||||
// Older peers may omit the extension while agreeing on the part itself.
|
||||
// Require independent proof quorum without reducing part availability.
|
||||
part.encrypted_etag = *protected_count >= read_quorum;
|
||||
return Ok(part);
|
||||
return Ok((*part).clone());
|
||||
}
|
||||
|
||||
if missing_count >= read_quorum {
|
||||
@@ -13395,53 +13390,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_read_part_requires_matching_quorum_for_encrypted_etag_proof() {
|
||||
let protected = ObjectPartInfo {
|
||||
encrypted_etag: true,
|
||||
..read_part_test_part(1, "protected-etag")
|
||||
};
|
||||
let legacy = ObjectPartInfo {
|
||||
encrypted_etag: false,
|
||||
..protected.clone()
|
||||
};
|
||||
for (parts, quorum, expected) in [
|
||||
(vec![protected.clone(), legacy.clone(), legacy.clone()], 2, false),
|
||||
(vec![legacy.clone(), protected.clone(), protected.clone()], 2, true),
|
||||
(vec![protected.clone(), protected.clone(), legacy.clone(), legacy.clone()], 3, false),
|
||||
(
|
||||
vec![
|
||||
protected.clone(),
|
||||
ObjectPartInfo {
|
||||
etag: "other-generation".to_owned(),
|
||||
..protected.clone()
|
||||
},
|
||||
legacy.clone(),
|
||||
],
|
||||
2,
|
||||
false,
|
||||
),
|
||||
(
|
||||
vec![
|
||||
protected.clone(),
|
||||
ObjectPartInfo {
|
||||
error: Some("read failed".to_owned()),
|
||||
..protected.clone()
|
||||
},
|
||||
legacy,
|
||||
],
|
||||
2,
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let responses = parts.into_iter().map(|part| Some(vec![part])).collect::<Vec<_>>();
|
||||
let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, quorum)
|
||||
.expect("matching metadata remains readable across producer versions");
|
||||
assert_eq!(part.etag, protected.etag);
|
||||
assert_eq!(part.encrypted_etag, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_read_part_diagnostic_branch_keeps_read_quorum_error() {
|
||||
temp_env::with_var("RUSTFS_ISSUE3031_DIAG_ENABLE", Some("true"), || {
|
||||
|
||||
@@ -7980,23 +7980,21 @@ fn is_min_allowed_part_size(size: i64) -> bool {
|
||||
}
|
||||
|
||||
fn get_complete_multipart_md5(parts: &[CompletePart]) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
for etag in parts.iter().filter_map(|part| part.etag.as_deref()) {
|
||||
// Native part ETags decode to one fixed-size digest. Preserve the
|
||||
// historical hex-or-raw interpretation for imported part ETags.
|
||||
let mut digest = [0u8; 16];
|
||||
if etag.len() == digest.len() * 2 {
|
||||
match hex_simd::decode(etag.as_bytes(), hex_simd::Out::from_slice(&mut digest)) {
|
||||
Ok(decoded) => hasher.update(decoded),
|
||||
Err(_) => hasher.update(etag.as_bytes()),
|
||||
let mut buf = Vec::new();
|
||||
|
||||
for part in parts.iter() {
|
||||
if let Some(etag) = &part.etag {
|
||||
if let Ok(etag_bytes) = hex_simd::decode_to_vec(etag.as_bytes()) {
|
||||
buf.extend(etag_bytes);
|
||||
} else {
|
||||
buf.extend(etag.bytes());
|
||||
}
|
||||
} else if let Ok(decoded) = hex_simd::decode_to_vec(etag.as_bytes()) {
|
||||
hasher.update(decoded);
|
||||
} else {
|
||||
hasher.update(etag.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(&buf);
|
||||
|
||||
let digest = hasher.finalize();
|
||||
let etag_hex = faster_hex::hex_string(digest.as_slice());
|
||||
format!("{}-{}", etag_hex, parts.len())
|
||||
@@ -9684,59 +9682,50 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_multipart_md5_preserves_legacy_etag_encodings() {
|
||||
fn part(etag: Option<&str>, part_num: usize) -> CompletePart {
|
||||
fn test_get_complete_multipart_md5() {
|
||||
// Test MD5 calculation for multipart upload
|
||||
let parts = vec![
|
||||
CompletePart {
|
||||
part_num,
|
||||
etag: etag.map(str::to_owned),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn buffered_digest(parts: &[CompletePart]) -> String {
|
||||
let mut bytes = Vec::new();
|
||||
for etag in parts.iter().filter_map(|part| part.etag.as_deref()) {
|
||||
match hex_simd::decode_to_vec(etag.as_bytes()) {
|
||||
Ok(decoded) => bytes.extend(decoded),
|
||||
Err(_) => bytes.extend_from_slice(etag.as_bytes()),
|
||||
}
|
||||
}
|
||||
format!("{}-{}", faster_hex::hex_string(&Md5::digest(bytes)), parts.len())
|
||||
}
|
||||
|
||||
assert_eq!(get_complete_multipart_md5(&[]), "d41d8cd98f00b204e9800998ecf8427e-0");
|
||||
assert_eq!(get_complete_multipart_md5(&[part(None, 1)]), "d41d8cd98f00b204e9800998ecf8427e-1");
|
||||
assert_eq!(get_complete_multipart_md5(&[part(Some("00"), 1)]), "93b885adfe0da089cdf634904fd59f71-1");
|
||||
assert_eq!(
|
||||
get_complete_multipart_md5(&[
|
||||
part(Some("d41d8cd98f00b204e9800998ecf8427e"), 1),
|
||||
part(Some("098f6bcd4621d373cade4e832627b4f6"), 2),
|
||||
]),
|
||||
"f666c301a43456b66b9f3a47f696d391-2"
|
||||
);
|
||||
|
||||
let encodings = [
|
||||
None,
|
||||
Some(""),
|
||||
Some("00"),
|
||||
Some("0aBcFf"),
|
||||
Some("0123456789abcdef0123456789abcdef"),
|
||||
Some("0123456789ABCDEF0123456789ABCDEF"),
|
||||
Some("0123456789abcdef0123456789abcdef00"),
|
||||
Some("0123456789abcdef0123456789abcdeg"),
|
||||
Some("0123456789abcdef0123456789abcdeg00"),
|
||||
Some("f"),
|
||||
Some("not-hex"),
|
||||
Some("\"0123456789abcdef0123456789abcdef\""),
|
||||
Some("0123456789abcdef0123456789abcdef-2"),
|
||||
Some("café"),
|
||||
part_num: 1,
|
||||
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
},
|
||||
CompletePart {
|
||||
part_num: 2,
|
||||
etag: Some("098f6bcd4621d373cade4e832627b4f6".to_string()),
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
},
|
||||
];
|
||||
for first in encodings {
|
||||
for second in encodings {
|
||||
let parts = [part(first, 1), part(second, 2)];
|
||||
assert_eq!(get_complete_multipart_md5(&parts), buffered_digest(&parts), "{first:?}, {second:?}");
|
||||
}
|
||||
}
|
||||
|
||||
let md5 = get_complete_multipart_md5(&parts);
|
||||
assert!(md5.ends_with("-2")); // Should end with part count
|
||||
assert!(md5.len() > 10); // Should have reasonable length
|
||||
|
||||
// Test with empty parts
|
||||
let empty_parts = vec![];
|
||||
let empty_result = get_complete_multipart_md5(&empty_parts);
|
||||
assert!(empty_result.ends_with("-0"));
|
||||
|
||||
// Test with single part
|
||||
let single_part = vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
}];
|
||||
let single_result = get_complete_multipart_md5(&single_part);
|
||||
assert!(single_result.ends_with("-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -311,85 +311,4 @@ mod tests {
|
||||
.await
|
||||
.expect("single-disk no-parity PUT should pass self-verify");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_parity_self_verify_rejects_missing_inline_data_missing_disk_shard_and_under_quorum() {
|
||||
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
|
||||
let shard_size = 16usize;
|
||||
let payload = b"self verify negative path";
|
||||
let encoded = encode_streaming_shard(payload, shard_size).await;
|
||||
let disks_opt = disks.iter().cloned().map(Some).collect::<Vec<_>>();
|
||||
|
||||
// 1. Inline shard with `data: None` fails closed.
|
||||
let empty_parts = [FileInfo {
|
||||
data: None,
|
||||
..Default::default()
|
||||
}];
|
||||
let err = verify_written_bitrot_shards(
|
||||
&disks_opt,
|
||||
Some(&empty_parts),
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "missing-inline-data",
|
||||
part_number: None,
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path: "unused",
|
||||
logical_shard_size: payload.len(),
|
||||
shard_size,
|
||||
write_quorum: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("inline shard missing data must fail");
|
||||
assert!(err.to_string().contains("missing bitrot data"));
|
||||
|
||||
// 2. Disk shard missing on disk (FileNotFound) fails closed with shard error context.
|
||||
let err = verify_written_bitrot_shards(
|
||||
&disks_opt,
|
||||
None,
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "missing-disk-file",
|
||||
part_number: Some(3),
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path: "nonexistent-temp-shard",
|
||||
logical_shard_size: payload.len(),
|
||||
shard_size,
|
||||
write_quorum: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("missing disk shard must fail");
|
||||
assert!(err.to_string().contains("no-parity bitrot self-verify failed for shard 0"));
|
||||
|
||||
// 3. Verified shard count below write_quorum fails closed even when available shard is valid.
|
||||
let valid_parts = [
|
||||
FileInfo {
|
||||
data: Some(encoded),
|
||||
..Default::default()
|
||||
},
|
||||
FileInfo::default(),
|
||||
];
|
||||
let two_slots = vec![disks_opt[0].clone(), None];
|
||||
let err = verify_written_bitrot_shards(
|
||||
&two_slots,
|
||||
Some(&valid_parts),
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "under-quorum",
|
||||
part_number: None,
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path: "unused",
|
||||
logical_shard_size: payload.len(),
|
||||
shard_size,
|
||||
write_quorum: 2,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("under-quorum verification must fail");
|
||||
assert!(err.to_string().contains("quorum unavailable: 1 shard(s) verified, need 2"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,8 +46,8 @@ use super::super::{
|
||||
log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
|
||||
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, should_persist_encryption_original_size,
|
||||
strip_internal_multipart_metadata, to_object_err, warn,
|
||||
restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
|
||||
to_object_err, warn,
|
||||
};
|
||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||
#[cfg(test)]
|
||||
@@ -2728,9 +2728,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.map(crate::io_support::rio::compression_index_storage_bytes);
|
||||
|
||||
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
|
||||
let encrypted_etag = data.stream.encrypted_etag
|
||||
&& !etag.is_empty()
|
||||
&& opts.preserve_etag.as_ref().is_none_or(|preserved| preserved == &etag);
|
||||
|
||||
if let Some(ref tag) = opts.preserve_etag {
|
||||
etag = tag.clone();
|
||||
@@ -2751,7 +2748,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_info = ObjectPartInfo {
|
||||
etag: etag.clone(),
|
||||
encrypted_etag,
|
||||
number: part_id,
|
||||
size: w_size,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
@@ -3651,7 +3647,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
}
|
||||
let source_condition = capture_source_condition(bucket, object, &upload_id_path, &fi, Uuid::new_v4())?;
|
||||
let preserved_encrypted_etag = preserves_staged_encrypted_etag_proof(&fi, opts, expected_restore_operation_id);
|
||||
let upload_data_dir = source_condition.lineage.original_data_dir.to_string();
|
||||
// The source generation remains in the captured precondition. Its
|
||||
// ACTIVE/recovery controls are not final user-object metadata.
|
||||
@@ -4122,17 +4117,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
// etag
|
||||
let (etag, encrypted_etag) = resolve_complete_etag(opts, &uploaded_parts, &fi.parts);
|
||||
let encrypted_etag = encrypted_etag
|
||||
|| preserved_encrypted_etag
|
||||
&& fi.version_id.filter(|version| !version.is_nil()) == source_condition.lineage.original_version_id;
|
||||
let etag = resolve_complete_etag(opts, &uploaded_parts);
|
||||
|
||||
// New writes attest their selected parts. Same-version internal moves
|
||||
// and restores can retain the proof from quorum-validated source metadata.
|
||||
// An upload can contain parts from an older writer or a peer. Until
|
||||
// every part's format is attested, keep its ETag out of keyless lists.
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
if encrypted_etag {
|
||||
insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
}
|
||||
|
||||
fi.metadata.insert("etag".to_owned(), etag);
|
||||
|
||||
@@ -4959,57 +4948,19 @@ fn completed_multipart_upload_matches(
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
// Capture only the original staged proof, before completion applies metadata
|
||||
// updates. Publication must also retain the source lineage's version.
|
||||
fn preserves_staged_encrypted_etag_proof(fi: &FileInfo, opts: &ObjectOptions, restore_operation_id: Option<Uuid>) -> bool {
|
||||
if opts.replication_request {
|
||||
return false;
|
||||
}
|
||||
let movement = opts.data_movement
|
||||
&& opts.version_id.is_some()
|
||||
&& get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD)
|
||||
.is_some_and(|identity| !identity.is_empty());
|
||||
// Commit rechecks this operation against the live object under its lock.
|
||||
let restore = restore_operation_id
|
||||
.is_some_and(|expected| restore_operation_id_from_metadata(&fi.metadata).is_ok_and(|id| id == Some(expected)));
|
||||
if !movement && !restore {
|
||||
return false;
|
||||
}
|
||||
let Ok(version) = opts.version_id.as_deref().map(Uuid::parse_str).transpose() else {
|
||||
return false;
|
||||
};
|
||||
let Some(etag) = opts.preserve_etag.as_deref().filter(|etag| !etag.is_empty()) else {
|
||||
return false;
|
||||
};
|
||||
version.filter(|version| !version.is_nil()) == fi.version_id.filter(|version| !version.is_nil())
|
||||
&& fi.metadata.get("etag").map(String::as_str) == Some(etag)
|
||||
&& get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG) == Some("v1")
|
||||
}
|
||||
|
||||
/// Final ETag for a completed multipart object. An authorized replication
|
||||
/// request preserves the source ETag so the replication HEAD comparison
|
||||
/// converges even when the source ETag is not derivable from the uploaded
|
||||
/// parts (foreign-origin objects, ciphertext-derived ETags); the internal
|
||||
/// metadata override comes next; otherwise the ETag is computed from parts.
|
||||
fn resolve_complete_etag(
|
||||
opts: &ObjectOptions,
|
||||
uploaded_parts: &[CompletePart],
|
||||
stored_parts: &[ObjectPartInfo],
|
||||
) -> (String, bool) {
|
||||
let preserved = opts
|
||||
.preserve_etag
|
||||
.as_ref()
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.or_else(|| opts.user_defined.get("etag"));
|
||||
let protected_parts = !stored_parts.is_empty() && stored_parts.iter().all(|part| part.encrypted_etag);
|
||||
if let Some(etag) = preserved
|
||||
&& !protected_parts
|
||||
{
|
||||
return (etag.clone(), false);
|
||||
fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart]) -> String {
|
||||
if let Some(etag) = opts.preserve_etag.as_ref().filter(|etag| !etag.is_empty()) {
|
||||
return etag.clone();
|
||||
}
|
||||
let computed = get_complete_multipart_md5(uploaded_parts);
|
||||
let encrypted_etag = protected_parts && preserved.is_none_or(|etag| etag == &computed);
|
||||
(preserved.cloned().unwrap_or(computed), encrypted_etag)
|
||||
if let Some(etag) = opts.user_defined.get("etag") {
|
||||
return etag.clone();
|
||||
}
|
||||
get_complete_multipart_md5(uploaded_parts)
|
||||
}
|
||||
|
||||
/// Serialize the object-level checksum record that `complete_multipart_upload`
|
||||
@@ -9142,26 +9093,6 @@ mod tests {
|
||||
receipts.push(part.etag);
|
||||
}
|
||||
}
|
||||
let (staged, _) = set
|
||||
.check_upload_id_exists(bucket, "object", &upload.upload_id, false)
|
||||
.await
|
||||
.expect("read upload generation");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, "object", &upload.upload_id);
|
||||
let data_dir = staged.data_dir.expect("staging data directory");
|
||||
for disk in &disks {
|
||||
let metadata = disk
|
||||
.read_parts(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&[
|
||||
format!("{upload_path}/{data_dir}/part.1.meta"),
|
||||
format!("{upload_path}/{data_dir}/part.3.meta"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("read persisted part proofs");
|
||||
assert!(metadata[0].encrypted_etag, "rewritten protected part is attested");
|
||||
assert!(!metadata[1].encrypted_etag, "an overridden part ETag has no producer proof");
|
||||
}
|
||||
assert_ne!(receipts[0], receipts[1], "spare parts must not identify another part's plaintext");
|
||||
assert_ne!(receipts[0], receipts[2], "same-number retries get a new receipt");
|
||||
for _ in 0..2 {
|
||||
@@ -9198,230 +9129,6 @@ mod tests {
|
||||
.await
|
||||
.expect("committed completion replay is stable");
|
||||
assert_eq!(replay.etag, committed.etag);
|
||||
for completed in [committed, replay] {
|
||||
assert!(
|
||||
rustfs_utils::http::etag_can_be_listed(&completed.user_defined),
|
||||
"the selected protected part retains its ETag after completion and replay"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(&completed.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,),
|
||||
Some("v1")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_encrypted_multipart_clears_unproven_upload_etag_marker() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "legacy-encrypted-etag";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let mut opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_owned(), "AES256".to_owned())]),
|
||||
..Default::default()
|
||||
};
|
||||
insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
// Model an older writer whose part metadata has no protected-format proof.
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set, bucket, "object", b"legacy part", &opts).await;
|
||||
let completed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy part can still complete");
|
||||
let reread = set
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read completed metadata");
|
||||
assert_eq!(completed.etag, reread.etag);
|
||||
for info in [completed, reread] {
|
||||
assert!(!rustfs_utils::http::etag_can_be_listed(&info.user_defined));
|
||||
assert!(!contains_key_str(&info.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_etag_proof_requires_verified_source_identity() {
|
||||
let version = Uuid::new_v4();
|
||||
let mut source = FileInfo {
|
||||
version_id: Some(version),
|
||||
metadata: HashMap::from([("etag".to_owned(), "protected-source-etag".to_owned())]),
|
||||
..Default::default()
|
||||
};
|
||||
insert_str(
|
||||
&mut source.metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
"source-generation".to_owned(),
|
||||
);
|
||||
insert_str(&mut source.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
let opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
version_id: Some(version.to_string()),
|
||||
preserve_etag: source.metadata.get("etag").cloned(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(preserves_staged_encrypted_etag_proof(&source, &opts, None));
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let null_source = FileInfo {
|
||||
version_id,
|
||||
..source.clone()
|
||||
};
|
||||
let null_opts = ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..opts.clone()
|
||||
};
|
||||
assert!(preserves_staged_encrypted_etag_proof(&null_source, &null_opts, None));
|
||||
}
|
||||
for rejected in [
|
||||
ObjectOptions {
|
||||
data_movement: false,
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
replication_request: true,
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: None,
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some("invalid".to_owned()),
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_etag: None,
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_etag: Some(String::new()),
|
||||
..opts.clone()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_etag: Some("replacement-etag".to_owned()),
|
||||
..opts.clone()
|
||||
},
|
||||
] {
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&source, &rejected, None));
|
||||
}
|
||||
for suffix in [
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
|
||||
] {
|
||||
let mut missing = source.clone();
|
||||
rustfs_utils::http::remove_str(&mut missing.metadata, suffix);
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&missing, &opts, None));
|
||||
insert_str(&mut missing.metadata, suffix, String::new());
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&missing, &opts, None));
|
||||
let mut conflicting = source.clone();
|
||||
conflicting
|
||||
.metadata
|
||||
.insert(format!("x-minio-internal-{suffix}"), "conflict".to_owned());
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&conflicting, &opts, None));
|
||||
}
|
||||
for value in [None, Some("unknown")] {
|
||||
let mut unproven = source.clone();
|
||||
rustfs_utils::http::remove_str(&mut unproven.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
if let Some(value) = value {
|
||||
insert_str(&mut unproven.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, value.to_owned());
|
||||
}
|
||||
let request_claim = ObjectOptions {
|
||||
user_defined: source.metadata.clone(),
|
||||
..opts.clone()
|
||||
};
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&unproven, &request_claim, None));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_etag_proof_requires_matching_staged_operation() {
|
||||
let operation_id = Uuid::new_v4();
|
||||
for version_id in [None, Some(Uuid::new_v4())] {
|
||||
let mut staged = FileInfo {
|
||||
version_id,
|
||||
metadata: HashMap::from([("etag".to_owned(), "source-etag".to_owned())]),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
version_id: version_id.map(|id| id.to_string()),
|
||||
preserve_etag: Some("source-etag".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
insert_str(&mut staged.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&staged, &opts, Some(operation_id)));
|
||||
insert_str(&mut staged.metadata, SUFFIX_RESTORE_OPERATION_ID, operation_id.to_string());
|
||||
assert!(preserves_staged_encrypted_etag_proof(&staged, &opts, Some(operation_id)));
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&staged, &opts, Some(Uuid::new_v4())));
|
||||
rustfs_utils::http::remove_str(&mut staged.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
|
||||
assert!(!preserves_staged_encrypted_etag_proof(&staged, &opts, Some(operation_id)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_data_movement_etag_proof_rejects_version_rewrites_and_late_claims() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "movement-etag-proof";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
for (object, staged_proof, publish_versioned, late_claim, expected_proof) in [
|
||||
("same-version", true, true, false, true),
|
||||
("rewritten-version", true, false, false, false),
|
||||
("late-claim", false, true, true, false),
|
||||
] {
|
||||
let version = Uuid::new_v4();
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
versioned: true,
|
||||
version_id: Some(version.to_string()),
|
||||
preserve_etag: Some("protected-source-etag".to_owned()),
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_owned(), "AES256".to_owned())]),
|
||||
..Default::default()
|
||||
};
|
||||
insert_str(
|
||||
&mut opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
"source-generation".to_owned(),
|
||||
);
|
||||
if staged_proof {
|
||||
insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
}
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set, bucket, object, b"raw source bytes", &opts).await;
|
||||
opts.versioned = publish_versioned;
|
||||
if late_claim {
|
||||
let mut metadata = HashMap::new();
|
||||
insert_str(&mut metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
|
||||
opts.eval_metadata = Some(metadata);
|
||||
}
|
||||
let completed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &opts)
|
||||
.await
|
||||
.expect("complete raw internal movement");
|
||||
assert_eq!(completed.version_id, publish_versioned.then_some(version), "{object}");
|
||||
assert_eq!(completed.etag, opts.preserve_etag, "{object}");
|
||||
let persisted = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(crate::data_movement::data_movement_exact_version_id(completed.version_id)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read completed proof from disk");
|
||||
assert_eq!(
|
||||
rustfs_utils::http::etag_can_be_listed(&persisted.user_defined),
|
||||
expected_proof,
|
||||
"{object}"
|
||||
);
|
||||
assert_eq!(
|
||||
get_consistent_str(&persisted.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG),
|
||||
expected_proof.then_some("v1"),
|
||||
"{object}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -16498,7 +16205,7 @@ mod tests {
|
||||
preserve_etag: Some(foreign_etag.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(resolve_complete_etag(&opts, &[], &[]), (foreign_etag, false));
|
||||
assert_eq!(resolve_complete_etag(&opts, &[]), foreign_etag);
|
||||
|
||||
// Empty preserve value degrades to the next source.
|
||||
let opts_empty = ObjectOptions {
|
||||
@@ -16506,67 +16213,11 @@ mod tests {
|
||||
user_defined: std::collections::HashMap::from([("etag".to_string(), "override-etag".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(resolve_complete_etag(&opts_empty, &[], &[]), ("override-etag".to_string(), false));
|
||||
assert_eq!(resolve_complete_etag(&opts_empty, &[]), "override-etag");
|
||||
|
||||
// Without either source the ETag is computed from the parts.
|
||||
let computed = resolve_complete_etag(&ObjectOptions::default(), &[], &[]);
|
||||
assert_eq!(computed, (get_complete_multipart_md5(&[]), false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_etag_proof_requires_all_selected_parts_and_matching_overrides() {
|
||||
for proofs in [&[][..], &[true], &[true, true], &[true, false], &[false]] {
|
||||
let stored: Vec<_> = proofs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, &encrypted_etag)| ObjectPartInfo {
|
||||
number: index + 1,
|
||||
etag: "0123456789abcdef0123456789abcdef".to_owned(),
|
||||
encrypted_etag,
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let uploaded: Vec<_> = stored
|
||||
.iter()
|
||||
.map(|part| CompletePart {
|
||||
part_num: part.number,
|
||||
etag: Some(part.etag.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let computed = get_complete_multipart_md5(&uploaded);
|
||||
let protected = !proofs.is_empty() && proofs.iter().all(|&proof| proof);
|
||||
assert_eq!(
|
||||
resolve_complete_etag(&ObjectOptions::default(), &uploaded, &stored),
|
||||
(computed.clone(), protected)
|
||||
);
|
||||
|
||||
for preserved in ["foreign-etag", computed.as_str(), ""] {
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
preserve_etag: Some(preserved.to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectOptions {
|
||||
user_defined: HashMap::from([("etag".to_owned(), preserved.to_owned())]),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectOptions {
|
||||
preserve_etag: Some(preserved.to_owned()),
|
||||
user_defined: HashMap::from([("etag".to_owned(), "metadata-etag".to_owned())]),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let expected_etag = if !preserved.is_empty() || opts.preserve_etag.is_none() {
|
||||
preserved
|
||||
} else {
|
||||
opts.user_defined.get("etag").map_or(computed.as_str(), String::as_str)
|
||||
};
|
||||
let result = resolve_complete_etag(&opts, &uploaded, &stored);
|
||||
assert_eq!(result, (expected_etag.to_owned(), protected && expected_etag == computed));
|
||||
}
|
||||
}
|
||||
}
|
||||
let computed = resolve_complete_etag(&ObjectOptions::default(), &[]);
|
||||
assert_eq!(computed, get_complete_multipart_md5(&[]));
|
||||
}
|
||||
|
||||
/// Accumulate the per-part digests and the running full-object checksum the
|
||||
|
||||
@@ -9103,9 +9103,6 @@ impl SetDisks {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let requires_bounded_delete_recovery = commit_context
|
||||
.options
|
||||
.is_some_and(|options| options.delete_recovery_work_budget.is_some());
|
||||
for disk in disks.iter() {
|
||||
let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
|
||||
let transaction_guard = commit_context.transaction_guard.as_ref();
|
||||
@@ -9117,7 +9114,6 @@ impl SetDisks {
|
||||
let options = DeleteOptions {
|
||||
old_data_dir: Some(rollback_dir),
|
||||
delete_recovery_bucket_incarnation: source_bucket_incarnation_id,
|
||||
requires_bounded_delete_recovery,
|
||||
..Default::default()
|
||||
};
|
||||
let result = if is_meta_bucketname(bucket) {
|
||||
@@ -10561,7 +10557,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.collect();
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let lock_lost_during_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let requires_bounded_delete_recovery = opts.delete_recovery_work_budget.is_some();
|
||||
|
||||
// let mut errors = Vec::with_capacity(disks.len());
|
||||
|
||||
@@ -10579,7 +10574,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let options = DeleteOptions {
|
||||
old_data_dir: Some(rollback_dir),
|
||||
delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id,
|
||||
requires_bounded_delete_recovery,
|
||||
..Default::default()
|
||||
};
|
||||
if current_generations
|
||||
@@ -17179,9 +17173,6 @@ mod transition_commit_failure_tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn multipart_restore_copies_the_stored_representation_back_verbatim() {
|
||||
use crate::io_support::rio::{WriteEncryption, WritePlan};
|
||||
use rustfs_utils::http::{SUFFIX_OPAQUE_ENCRYPTED_ETAG, etag_can_be_listed};
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-multipart-stored-coordinates-bucket";
|
||||
let object = "object.bin";
|
||||
@@ -17189,8 +17180,9 @@ mod transition_commit_failure_tests {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
// The first plaintext part clears the non-final 5 MiB threshold.
|
||||
let part_sizes = [6 * 1024 * 1024_usize, 256 * 1024_usize];
|
||||
// The minimum-part-size gate reads the PLAINTEXT length, so part one
|
||||
// clears 5 MiB there while its stored form carries encoding overhead.
|
||||
let part_shapes = [(6 * 1024 * 1024_usize, 9_216_usize), (256 * 1024_usize, 512_usize)];
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(rustfs_utils::http::INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "AES256".to_string());
|
||||
user_defined.insert(
|
||||
@@ -17211,22 +17203,20 @@ mod transition_commit_failure_tests {
|
||||
.expect("multipart upload should be created");
|
||||
|
||||
let mut uploaded_parts = Vec::new();
|
||||
for (index, &part_size) in part_sizes.iter().enumerate() {
|
||||
let plaintext = stored_representation_bytes(index as u32 * 7 + 1, part_size);
|
||||
let actual_size = part_size as i64;
|
||||
let plaintext_reader =
|
||||
HashReader::from_stream(std::io::Cursor::new(plaintext), actual_size, actual_size, None, None, false)
|
||||
.expect("plaintext part reader");
|
||||
let encrypted_reader = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::multipart([7; 32], [9; 12], index + 1))
|
||||
.apply(plaintext_reader, actual_size)
|
||||
.expect("native protected encrypted writer");
|
||||
let mut reader = PutObjReader::new(encrypted_reader);
|
||||
let mut expected_stored = Vec::new();
|
||||
for (index, (plaintext_len, overhead)) in part_shapes.iter().enumerate() {
|
||||
let stored = stored_representation_bytes(index as u32 * 7 + 1, plaintext_len + overhead);
|
||||
expected_stored.extend_from_slice(&stored);
|
||||
let stored_len = stored.len() as i64;
|
||||
let hash_reader =
|
||||
HashReader::from_stream(std::io::Cursor::new(stored), stored_len, *plaintext_len as i64, None, None, false)
|
||||
.expect("hash reader over the stored representation");
|
||||
let mut reader = PutObjReader::new(hash_reader);
|
||||
let info = set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, index + 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stored part should be staged");
|
||||
assert!(info.size as i64 > actual_size, "part contains actual encryption overhead");
|
||||
assert_eq!(info.size as i64, stored_len, "a part is stored in its encoded length");
|
||||
uploaded_parts.push(CompletePart {
|
||||
part_num: info.part_num,
|
||||
etag: info.etag,
|
||||
@@ -17250,12 +17240,8 @@ mod transition_commit_failure_tests {
|
||||
"the fixture must keep the two coordinate systems apart: stored {size} vs plaintext {actual_size}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(&original.user_defined, SUFFIX_OPAQUE_ENCRYPTED_ETAG),
|
||||
Some("v1")
|
||||
);
|
||||
assert!(etag_can_be_listed(&original.user_defined), "the source has a real producer-attested ETag");
|
||||
let expected_stored = read_stored_representation(&set_disks, bucket, object).await;
|
||||
let stored_before = read_stored_representation(&set_disks, bucket, object).await;
|
||||
assert_stored_representation_eq(&stored_before, &expected_stored, "the fixture must store its encoded bytes verbatim");
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
@@ -17279,16 +17265,6 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("multipart source should transition before restore");
|
||||
|
||||
let transitioned = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read metadata after actual tier transition");
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(&transitioned.user_defined, SUFFIX_OPAQUE_ENCRYPTED_ETAG),
|
||||
Some("v1")
|
||||
);
|
||||
assert!(etag_can_be_listed(&transitioned.user_defined), "transition retains the source proof");
|
||||
|
||||
let operation_id = Uuid::new_v4();
|
||||
set_disks
|
||||
.put_object_metadata(
|
||||
@@ -17334,11 +17310,6 @@ mod transition_commit_failure_tests {
|
||||
// backlog#2369 P7.1: the copy-back digests stored bytes, so the object's
|
||||
// public ETag has to be carried over rather than recomputed.
|
||||
assert_eq!(restored.etag, original.etag, "restore must preserve the object ETag");
|
||||
assert_eq!(restored.version_id, original.version_id, "restore keeps the same version");
|
||||
assert!(
|
||||
etag_can_be_listed(&restored.user_defined),
|
||||
"the restored version retains its protected ETag proof"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -20461,7 +20432,7 @@ mod transition_upload_integrity_tests {
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial_test::serial]
|
||||
async fn commit_lock_acquire_failure_cleans_remote_candidate_and_preserves_source() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
@@ -20470,16 +20441,7 @@ mod transition_upload_integrity_tests {
|
||||
let bucket = "transition-lock-acquire-failure-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"transition commit lock failure must clean the remote candidate".repeat(1024);
|
||||
let original = temp_env::async_with_vars(
|
||||
[(
|
||||
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
|
||||
Some("false"),
|
||||
)],
|
||||
async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await },
|
||||
)
|
||||
.await;
|
||||
// Complete source I/O before controlling the quorum retry deadline.
|
||||
tokio::time::pause();
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
@@ -20491,8 +20453,13 @@ mod transition_upload_integrity_tests {
|
||||
.await
|
||||
.expect_err("transition must fail when the commit namespace write lock cannot reach quorum");
|
||||
assert!(
|
||||
matches!(error, StorageError::NamespaceLockQuorumUnavailable { required: 2, .. }),
|
||||
"the injected permanent quorum failure must reach the transition commit boundary: {error:?}"
|
||||
matches!(
|
||||
error,
|
||||
StorageError::NamespaceLockQuorumUnavailable { .. }
|
||||
| StorageError::Lock(LockError::QuorumNotReached { .. })
|
||||
| StorageError::Lock(LockError::Internal { .. })
|
||||
),
|
||||
"unexpected transition lock-acquire error: {error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.put_count().await,
|
||||
|
||||
@@ -24035,7 +24035,6 @@ mod tests {
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-record", &[4])).await;
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let intent = TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
@@ -24221,7 +24220,6 @@ mod tests {
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan", &[4])).await;
|
||||
let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
@@ -24283,7 +24281,6 @@ mod tests {
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-advance", &[4])).await;
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let intent = TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
@@ -24358,7 +24355,6 @@ mod tests {
|
||||
|
||||
let abort_id = uuid::Uuid::new_v4();
|
||||
let abort_intent = TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id: abort_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
@@ -24401,7 +24397,6 @@ mod tests {
|
||||
candidate_digest: [u8; 32],
|
||||
) -> TierMutationIntent {
|
||||
TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
@@ -25478,7 +25473,6 @@ mod tests {
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan-page", &[4])).await;
|
||||
let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
|
||||
requires_native_azure: false,
|
||||
mutation_id,
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
|
||||
@@ -318,10 +318,7 @@ mod multipart_copy_heal_tests {
|
||||
let mut corrupt = original.clone();
|
||||
corrupt[0] ^= 0x80;
|
||||
std::fs::write(&shard, &corrupt).expect("actual bitrot corruption in shared physical shard");
|
||||
// Windows only flushes a handle opened for writing.
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&shard)
|
||||
std::fs::File::open(&shard)
|
||||
.expect("open corrupted actual shard")
|
||||
.sync_all()
|
||||
.expect("persist actual fault bytes");
|
||||
|
||||
@@ -3218,12 +3218,7 @@ impl ECStore {
|
||||
pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
check_valid_bucket_name(bucket)?;
|
||||
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata_guard = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
"object_lock_admission_metadata_read",
|
||||
)
|
||||
.await?;
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let (state, bucket_incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
|
||||
if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
|
||||
@@ -3243,12 +3238,7 @@ impl ECStore {
|
||||
pub(crate) async fn object_lock_config_snapshot_for_single_put(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
check_valid_bucket_name(bucket)?;
|
||||
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata_guard = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
"object_lock_snapshot_metadata_read",
|
||||
)
|
||||
.await?;
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let (state, bucket_incarnation_id, config_revision, quota_res) =
|
||||
get_object_lock_and_quota_config_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
|
||||
if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
|
||||
@@ -3277,12 +3267,7 @@ impl ECStore {
|
||||
if lifecycle_fence.is_lock_lost() {
|
||||
return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
|
||||
}
|
||||
let metadata_guard = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
"object_lock_completion_metadata_read",
|
||||
)
|
||||
.await?;
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let (state, bucket_incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&metadata_guard)).await?;
|
||||
if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
|
||||
|
||||
@@ -445,13 +445,6 @@ impl ECStore {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let scoped_cleanup = opts
|
||||
.scanner_publication_commit_scope
|
||||
.as_ref()
|
||||
.zip(opts.http_preconditions.as_ref())
|
||||
.filter(|_| bucket == RUSTFS_META_BUCKET && opts.delete_prefix_object);
|
||||
let _cleanup_outcome =
|
||||
scoped_cleanup.map(|(scope, _)| crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone()));
|
||||
let mut first_error = None;
|
||||
let mut first_volume_error = None;
|
||||
let mut has_success = false;
|
||||
@@ -464,30 +457,9 @@ impl ECStore {
|
||||
}
|
||||
let mut opts = opts.clone();
|
||||
opts.delete_prefix = true;
|
||||
if let Some((scope, preconditions)) = scoped_cleanup {
|
||||
// The exact-object delete retains its namespace writer across
|
||||
// this lookup and dispatch. Never remove an unobserved revision.
|
||||
let lookup = ObjectOptions {
|
||||
no_lock: true,
|
||||
suppress_read_repair: true,
|
||||
preserve_delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
let disks = pool.get_disks_by_key(object);
|
||||
let current = disks.get_object_info(bucket, object, &lookup).await;
|
||||
if let Some(error) =
|
||||
crate::set_disk::SetDisks::evaluate_write_precondition(current, preconditions, bucket, object, &lookup)
|
||||
{
|
||||
if matches!(error, StorageError::PreconditionFailed) || crate::error::is_err_object_not_found(&error) {
|
||||
continue;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
opts.scanner_publication_commit_scope = Some(scope.pool_delete_participant());
|
||||
}
|
||||
match pool.delete_object(bucket, object, opts).await {
|
||||
Ok(_) => has_success = true,
|
||||
Err(err) if scoped_cleanup.is_none() && is_err_strict_volume_not_found(&err) => {
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => {
|
||||
if first_volume_error.is_none() {
|
||||
first_volume_error = Some(err);
|
||||
}
|
||||
@@ -500,21 +472,14 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
let result = match first_error {
|
||||
match first_error {
|
||||
Some(err) => Err(err),
|
||||
None if has_success => Ok(()),
|
||||
None => match first_volume_error {
|
||||
Some(err) => Err(err),
|
||||
None if scoped_cleanup.is_some() => Err(StorageError::PreconditionFailed),
|
||||
None => Ok(()),
|
||||
},
|
||||
};
|
||||
if let Some((scope, _)) = scoped_cleanup
|
||||
&& result.is_ok()
|
||||
{
|
||||
let _ = scope.mark_committed();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option<usize> {
|
||||
@@ -1414,129 +1379,6 @@ mod tests {
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
async fn run_scanner_observed_cleanup(
|
||||
if_match_override: Option<&str>,
|
||||
second_payload: Option<&[u8]>,
|
||||
lose_second_quorum: bool,
|
||||
) -> (
|
||||
Result<crate::object_api::ObjectInfo>,
|
||||
crate::object_api::ScannerPublicationCommitState,
|
||||
Vec<bool>,
|
||||
) {
|
||||
use crate::config::com::save_config_with_opts_and_metadata;
|
||||
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||
use std::time::Duration;
|
||||
|
||||
let (_directories, store, shutdown) = setup_multi_pool_test_store("scanner-observed-cleanup", &[4, 4]).await;
|
||||
let object = "buckets/.usage.observed.json";
|
||||
let mut tags = Vec::new();
|
||||
let old_payload = b"old observed snapshot";
|
||||
for (pool, payload) in store.pools.iter().zip([Some(old_payload.as_slice()), second_payload]) {
|
||||
let Some(payload) = payload else { continue };
|
||||
let info = save_config_with_opts_and_metadata(pool.clone(), object, payload.to_vec(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("seed an observed snapshot on each pool");
|
||||
tags.push(info.etag.expect("seed has CAS tag"));
|
||||
}
|
||||
if let Some(second_payload) = second_payload {
|
||||
assert_eq!(tags[0] == tags[1], old_payload == second_payload);
|
||||
}
|
||||
let epoch = store.scanner_data_usage_publication_epoch().await.expect("publication epoch");
|
||||
let scope = store
|
||||
.scanner_data_usage_publication_commit_scope(epoch, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
|
||||
.await
|
||||
.expect("admit observed cleanup");
|
||||
let disconnected = if lose_second_quorum {
|
||||
let pool = &store.pools[1];
|
||||
tokio::time::timeout(Duration::from_secs(10), pool.stop_endpoint_monitor_for_test())
|
||||
.await
|
||||
.expect("fixture must own the disk slots");
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let original = {
|
||||
let mut disks = set.disks.write().await;
|
||||
let offline = vec![None; disks.len()];
|
||||
std::mem::replace(&mut *disks, offline)
|
||||
};
|
||||
Some((set, original))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = store
|
||||
.delete_object(
|
||||
crate::disk::RUSTFS_META_BUCKET,
|
||||
object,
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(if_match_override.map_or_else(|| tags.remove(0), str::to_owned)),
|
||||
..Default::default()
|
||||
}),
|
||||
scanner_publication_commit_scope: Some(scope.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if let Some((set, disks)) = disconnected {
|
||||
*set.disks.write().await = disks;
|
||||
assert!(!scope.release_movement_permit().await, "partial cleanup cannot release its permit");
|
||||
}
|
||||
let mut still_present = Vec::new();
|
||||
for pool in &store.pools {
|
||||
still_present.push(
|
||||
read_config_no_lock_preserve_empty_with_metadata(pool.clone(), object)
|
||||
.await
|
||||
.is_ok(),
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
(result, scope.state(), still_present)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_observed_cleanup_must_keep_scope_live_across_both_pools() {
|
||||
let (result, state, still_present) = run_scanner_observed_cleanup(None, Some(b"old observed snapshot"), false).await;
|
||||
assert!(result.is_ok(), "result {result:?}, scope {state:?}, pool copies {still_present:?}");
|
||||
assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Committed);
|
||||
assert_eq!(still_present, vec![false, false]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_observed_cleanup_must_preserve_both_pools_if_cas_changed() {
|
||||
let (result, state, still_present) =
|
||||
run_scanner_observed_cleanup(Some("stale-cas-token"), Some(b"old observed snapshot"), false).await;
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::PreconditionFailed))
|
||||
&& state == crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
|
||||
&& still_present == [true, true],
|
||||
"result {result:?}, scope {state:?}, pool copies {still_present:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_observed_cleanup_must_preserve_a_nonmatching_pool_copy() {
|
||||
let (result, state, still_present) = run_scanner_observed_cleanup(None, Some(b"newer observed snapshot"), false).await;
|
||||
assert!(result.is_ok(), "result {result:?}, scope {state:?}, pool copies {still_present:?}");
|
||||
assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Committed);
|
||||
assert_eq!(still_present, vec![false, true]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_observed_cleanup_must_succeed_when_another_pool_has_no_copy() {
|
||||
let (result, state, still_present) = run_scanner_observed_cleanup(None, None, false).await;
|
||||
assert!(result.is_ok(), "result {result:?}, scope {state:?}, pool copies {still_present:?}");
|
||||
assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Committed);
|
||||
assert_eq!(still_present, vec![false, false]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_observed_cleanup_must_retain_ownership_after_partial_failure() {
|
||||
let (result, state, still_present) = run_scanner_observed_cleanup(None, Some(b"old observed snapshot"), true).await;
|
||||
assert!(result.is_err(), "partial cleanup must fail");
|
||||
assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Indeterminate);
|
||||
assert_eq!(still_present, vec![false, true]);
|
||||
}
|
||||
|
||||
struct LifecycleDeleteAllFailureGuard;
|
||||
|
||||
impl Drop for LifecycleDeleteAllFailureGuard {
|
||||
|
||||
@@ -70,17 +70,13 @@ pub struct ObjectPartInfo {
|
||||
pub error: Option<String>,
|
||||
#[serde(default)]
|
||||
pub integrity: Option<crate::shard_integrity::PartIntegrity>,
|
||||
/// The stored ETag was generated by the protected encrypted writer.
|
||||
#[serde(default)]
|
||||
pub encrypted_etag: bool,
|
||||
}
|
||||
|
||||
impl Serialize for ObjectPartInfo {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
|
||||
// Named fields let release readers ignore optional extensions. Appending
|
||||
// a ninth element to the old positional array makes those readers fail.
|
||||
let mut map =
|
||||
serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some()) + usize::from(self.encrypted_etag)))?;
|
||||
let mut map = serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some())))?;
|
||||
map.serialize_entry("etag", &self.etag)?;
|
||||
map.serialize_entry("number", &self.number)?;
|
||||
map.serialize_entry("size", &self.size)?;
|
||||
@@ -92,9 +88,6 @@ impl Serialize for ObjectPartInfo {
|
||||
if let Some(integrity) = &self.integrity {
|
||||
map.serialize_entry("integrity", integrity)?;
|
||||
}
|
||||
if self.encrypted_etag {
|
||||
map.serialize_entry("encrypted_etag", &true)?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
@@ -1150,7 +1143,6 @@ impl FileInfo {
|
||||
checksums,
|
||||
error: None,
|
||||
integrity: None,
|
||||
encrypted_etag: false,
|
||||
};
|
||||
|
||||
for p in self.parts.iter_mut() {
|
||||
@@ -2239,59 +2231,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_part_encrypted_etag_round_trips_with_legacy_compatibility() {
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct LegacyPart {
|
||||
etag: String,
|
||||
number: usize,
|
||||
size: usize,
|
||||
actual_size: i64,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
index: Option<Bytes>,
|
||||
checksums: Option<HashMap<String, String>>,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
let legacy = LegacyPart {
|
||||
etag: "0123456789abcdef0123456789abcdef".to_owned(),
|
||||
number: 1,
|
||||
size: 42,
|
||||
actual_size: 42,
|
||||
mod_time: None,
|
||||
index: None,
|
||||
checksums: None,
|
||||
error: None,
|
||||
};
|
||||
for encoded in [
|
||||
rmp_serde::to_vec(&legacy).expect("legacy positional part encodes"),
|
||||
rmp_serde::to_vec_named(&legacy).expect("legacy named part encodes"),
|
||||
] {
|
||||
let decoded = ObjectPartInfo::unmarshal(&encoded).expect("legacy part remains readable");
|
||||
assert_eq!(decoded.etag, legacy.etag);
|
||||
assert!(!decoded.encrypted_etag, "an older writer cannot attest a protected ETag");
|
||||
}
|
||||
|
||||
for encrypted_etag in [false, true] {
|
||||
let current = ObjectPartInfo {
|
||||
etag: legacy.etag.clone(),
|
||||
number: legacy.number,
|
||||
size: legacy.size,
|
||||
actual_size: legacy.actual_size,
|
||||
encrypted_etag,
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = current.marshal_msg().expect("current part encodes as a map");
|
||||
let field_count = rmp::decode::read_map_len(&mut encoded.as_slice()).expect("part format is a map");
|
||||
assert_eq!(field_count, if encrypted_etag { 9 } else { 8 });
|
||||
assert_eq!(ObjectPartInfo::unmarshal(&encoded).expect("proof round trips"), current);
|
||||
let old_reader: LegacyPart = rmp_serde::from_slice(&encoded).expect("old readers ignore optional proof");
|
||||
assert_eq!(old_reader.etag, current.etag);
|
||||
assert_eq!(old_reader.number, current.number);
|
||||
assert_eq!(old_reader.size, current.size);
|
||||
}
|
||||
}
|
||||
|
||||
fn object_part_info_strategy() -> impl Strategy<Value = ObjectPartInfo> {
|
||||
(
|
||||
small_string_strategy(),
|
||||
@@ -2313,7 +2252,6 @@ mod tests {
|
||||
checksums,
|
||||
error,
|
||||
integrity: None,
|
||||
encrypted_etag: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2523,7 +2461,6 @@ mod tests {
|
||||
),
|
||||
error: Some("part-error".to_string()),
|
||||
integrity: None,
|
||||
encrypted_etag: false,
|
||||
}],
|
||||
erasure: ErasureInfo {
|
||||
algorithm: "erasure-algorithm".to_string(),
|
||||
|
||||
@@ -1346,7 +1346,6 @@ impl<'a> BorrowedMetaObject<'a> {
|
||||
checksums: None,
|
||||
error: None,
|
||||
integrity: None,
|
||||
encrypted_etag: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -2626,7 +2625,6 @@ impl From<MetaObjectV1Part> for ObjectPartInfo {
|
||||
checksums: value.checksums,
|
||||
error: value.error,
|
||||
integrity: None,
|
||||
encrypted_etag: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ workspace = true
|
||||
[dependencies]
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics", "serde"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
|
||||
@@ -429,7 +429,6 @@ const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network
|
||||
const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
|
||||
const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
|
||||
const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
|
||||
const INTERNODE_OPERATION_SERVER_DURATION_MS: &str = "rustfs_system_network_internode_operation_server_duration_ms";
|
||||
const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
|
||||
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
|
||||
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
|
||||
@@ -563,10 +562,6 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_OPERATION_SERVER_DURATION_MS,
|
||||
labels: SERVER_OPERATION_BACKEND_LABELS,
|
||||
},
|
||||
];
|
||||
|
||||
static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
|
||||
@@ -692,78 +687,6 @@ impl GrpcReadVersionMetricHandles {
|
||||
static GRPC_READ_VERSION_METRIC_HANDLES: LazyLock<GrpcReadVersionMetricHandles> =
|
||||
LazyLock::new(|| GrpcReadVersionMetricHandles::new(current_server_label()));
|
||||
|
||||
#[cfg(not(test))]
|
||||
static GRPC_READ_MULTIPLE_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
|
||||
operation_duration_histogram(
|
||||
INTERNODE_OPERATION_DURATION_MS,
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
)
|
||||
});
|
||||
|
||||
#[cfg(not(test))]
|
||||
static GRPC_BATCH_READ_VERSION_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
|
||||
operation_duration_histogram(
|
||||
INTERNODE_OPERATION_DURATION_MS,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
)
|
||||
});
|
||||
|
||||
#[cfg(not(test))]
|
||||
static GRPC_READ_MULTIPLE_SERVER_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
|
||||
operation_duration_histogram(
|
||||
INTERNODE_OPERATION_SERVER_DURATION_MS,
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
)
|
||||
});
|
||||
|
||||
#[cfg(not(test))]
|
||||
static GRPC_BATCH_READ_VERSION_SERVER_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
|
||||
operation_duration_histogram(
|
||||
INTERNODE_OPERATION_SERVER_DURATION_MS,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
)
|
||||
});
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn grpc_operation_duration_if_ready(operation: &'static str, backend: &'static str) -> Option<&'static metrics::Histogram> {
|
||||
STABLE_SERVER_LABEL.get()?;
|
||||
if backend != INTERNODE_TRANSPORT_BACKEND_GRPC {
|
||||
return None;
|
||||
}
|
||||
match operation {
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION => Some(&GRPC_READ_VERSION_METRIC_HANDLES.duration),
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE => Some(&GRPC_READ_MULTIPLE_DURATION),
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION => Some(&GRPC_BATCH_READ_VERSION_DURATION),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn grpc_server_duration_if_ready(operation: &'static str, backend: &'static str) -> Option<&'static metrics::Histogram> {
|
||||
STABLE_SERVER_LABEL.get()?;
|
||||
if backend != INTERNODE_TRANSPORT_BACKEND_GRPC {
|
||||
return None;
|
||||
}
|
||||
match operation {
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE => Some(&GRPC_READ_MULTIPLE_SERVER_DURATION),
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION => Some(&GRPC_BATCH_READ_VERSION_SERVER_DURATION),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn operation_duration_histogram(name: &'static str, operation: &'static str, backend: &'static str) -> metrics::Histogram {
|
||||
metrics::histogram!(
|
||||
name,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn server_metric_handles_if_ready() -> Option<&'static InternodeServerMetricHandles> {
|
||||
STABLE_SERVER_LABEL.get()?;
|
||||
@@ -1021,28 +944,17 @@ impl InternodeMetrics {
|
||||
pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) {
|
||||
let duration_ms = duration.as_secs_f64() * 1000.0;
|
||||
#[cfg(not(test))]
|
||||
if let Some(histogram) = grpc_operation_duration_if_ready(operation, backend) {
|
||||
histogram.record(duration_ms);
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.duration.record(duration_ms);
|
||||
return;
|
||||
}
|
||||
operation_duration_histogram(INTERNODE_OPERATION_DURATION_MS, operation, backend).record(duration_ms);
|
||||
}
|
||||
|
||||
/// Records server-side handler latency in a separate histogram from client RPC duration.
|
||||
/// The caller supplies the handler lifetime, including any error or cancellation path.
|
||||
pub fn record_server_duration_for_operation_and_backend(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
duration: Duration,
|
||||
) {
|
||||
let duration_ms = duration.as_secs_f64() * 1000.0;
|
||||
#[cfg(not(test))]
|
||||
if let Some(histogram) = grpc_server_duration_if_ready(operation, backend) {
|
||||
histogram.record(duration_ms);
|
||||
return;
|
||||
}
|
||||
operation_duration_histogram(INTERNODE_OPERATION_SERVER_DURATION_MS, operation, backend).record(duration_ms);
|
||||
metrics::histogram!(
|
||||
INTERNODE_OPERATION_DURATION_MS,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend
|
||||
)
|
||||
.record(duration_ms);
|
||||
}
|
||||
|
||||
pub fn record_stage_duration_for_operation_and_backend(
|
||||
@@ -1787,7 +1699,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 23);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 22);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
@@ -1825,8 +1737,6 @@ mod tests {
|
||||
// Payload histogram + large-payload counter carry operation+backend labels.
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[20].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[21].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[22].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[22].name, INTERNODE_OPERATION_SERVER_DURATION_MS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit, with_local_recorder};
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_UNKNOWN,
|
||||
InternodeMetrics, set_internode_server_label,
|
||||
};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
const DURATION_METRIC: &str = "rustfs_system_network_internode_operation_duration_ms";
|
||||
const SERVER_DURATION_METRIC: &str = "rustfs_system_network_internode_operation_server_duration_ms";
|
||||
const SERVER_READ_OPERATIONS: [&str; 2] = [
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
];
|
||||
const READ_OPERATIONS: [&str; 3] = [
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
];
|
||||
|
||||
#[derive(Default)]
|
||||
struct RegistrationCountingRecorder {
|
||||
inner: DebuggingRecorder,
|
||||
duration_registrations: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Recorder for RegistrationCountingRecorder {
|
||||
fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.inner.describe_counter(key, unit, description);
|
||||
}
|
||||
|
||||
fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.inner.describe_gauge(key, unit, description);
|
||||
}
|
||||
|
||||
fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.inner.describe_histogram(key, unit, description);
|
||||
}
|
||||
|
||||
fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter {
|
||||
self.inner.register_counter(key, metadata)
|
||||
}
|
||||
|
||||
fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge {
|
||||
self.inner.register_gauge(key, metadata)
|
||||
}
|
||||
|
||||
fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram {
|
||||
if key.name() == DURATION_METRIC || key.name() == SERVER_DURATION_METRIC {
|
||||
self.duration_registrations.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.inner.register_histogram(key, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Integration tests exercise the production cache, which is disabled in unit tests
|
||||
// so individual unit tests can install independent local recorders.
|
||||
#[test]
|
||||
fn unary_read_duration_handles_are_cached_only_after_server_label_is_ready() {
|
||||
let recorder = RegistrationCountingRecorder::default();
|
||||
let snapshotter = recorder.inner.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
with_local_recorder(&recorder, || {
|
||||
for operation in READ_OPERATIONS {
|
||||
metrics.record_duration_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_GRPC, Duration::ZERO);
|
||||
}
|
||||
for operation in SERVER_READ_OPERATIONS {
|
||||
metrics.record_server_duration_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_GRPC, Duration::ZERO);
|
||||
}
|
||||
set_internode_server_label("cached-duration-test");
|
||||
|
||||
for operation in READ_OPERATIONS {
|
||||
metrics.record_duration_for_operation_and_backend(
|
||||
operation,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
Duration::from_micros(125),
|
||||
);
|
||||
let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
|
||||
metrics.record_duration_for_operation_and_backend(
|
||||
operation,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
Duration::from_micros(250),
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.duration_registrations.load(Ordering::Relaxed),
|
||||
registrations,
|
||||
"{operation} should reuse its duration handle after warm-up"
|
||||
);
|
||||
}
|
||||
|
||||
for operation in SERVER_READ_OPERATIONS {
|
||||
metrics.record_server_duration_for_operation_and_backend(
|
||||
operation,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
Duration::from_micros(1500),
|
||||
);
|
||||
let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
|
||||
metrics.record_server_duration_for_operation_and_backend(
|
||||
operation,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
Duration::from_micros(2500),
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.duration_registrations.load(Ordering::Relaxed),
|
||||
registrations,
|
||||
"{operation} should reuse its separate server duration handle after warm-up"
|
||||
);
|
||||
}
|
||||
|
||||
// Other backends and operations must retain their own labels and fallback.
|
||||
for (operation, backend) in READ_OPERATIONS
|
||||
.map(|operation| (operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN))
|
||||
.into_iter()
|
||||
.chain([(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC)])
|
||||
{
|
||||
let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
|
||||
metrics.record_duration_for_operation_and_backend(operation, backend, Duration::from_millis(1));
|
||||
assert_eq!(recorder.duration_registrations.load(Ordering::Relaxed), registrations + 1);
|
||||
}
|
||||
for (operation, backend) in SERVER_READ_OPERATIONS
|
||||
.map(|operation| (operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN))
|
||||
.into_iter()
|
||||
.chain([(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC)])
|
||||
{
|
||||
let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
|
||||
metrics.record_server_duration_for_operation_and_backend(operation, backend, Duration::from_millis(3));
|
||||
assert_eq!(recorder.duration_registrations.load(Ordering::Relaxed), registrations + 1);
|
||||
}
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
let durations: Vec<_> = rows
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == DURATION_METRIC || key.key().name() == SERVER_DURATION_METRIC)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
durations
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == DURATION_METRIC)
|
||||
.count(),
|
||||
10
|
||||
);
|
||||
assert_eq!(
|
||||
durations
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == SERVER_DURATION_METRIC)
|
||||
.count(),
|
||||
7
|
||||
);
|
||||
for (key, _, _, value) in durations {
|
||||
let server_duration = key.key().name() == SERVER_DURATION_METRIC;
|
||||
let operations = if server_duration {
|
||||
SERVER_READ_OPERATIONS.as_slice()
|
||||
} else {
|
||||
READ_OPERATIONS.as_slice()
|
||||
};
|
||||
let labels: Vec<_> = key.key().labels().map(|label| (label.key(), label.value())).collect();
|
||||
assert_eq!(labels.len(), 3);
|
||||
let server = labels.iter().find(|(name, _)| *name == "server").expect("server label").1;
|
||||
let operation = labels
|
||||
.iter()
|
||||
.find(|(name, _)| *name == "operation")
|
||||
.expect("operation label")
|
||||
.1;
|
||||
let backend = labels.iter().find(|(name, _)| *name == "backend").expect("backend label").1;
|
||||
let expected: &[f64] = if server == "unset" {
|
||||
assert!(operations.contains(&operation));
|
||||
assert_eq!(backend, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
&[0.0]
|
||||
} else {
|
||||
assert_eq!(server, "cached-duration-test");
|
||||
if backend == INTERNODE_TRANSPORT_BACKEND_GRPC && operations.contains(&operation) {
|
||||
if server_duration { &[1.5, 2.5] } else { &[0.125, 0.25] }
|
||||
} else {
|
||||
assert!(
|
||||
(backend == INTERNODE_TRANSPORT_BACKEND_UNKNOWN && operations.contains(&operation))
|
||||
|| (backend == INTERNODE_TRANSPORT_BACKEND_GRPC && operation == INTERNODE_OPERATION_GRPC_READ_ALL)
|
||||
);
|
||||
if server_duration { &[3.0] } else { &[1.0] }
|
||||
}
|
||||
};
|
||||
match value {
|
||||
DebugValue::Histogram(samples) => {
|
||||
assert_eq!(samples.iter().map(|sample| sample.0).collect::<Vec<_>>(), expected);
|
||||
}
|
||||
other => panic!("duration metric must be a histogram, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
-447
@@ -12,14 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
use std::collections::HashMap;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::{Instant, timeout, timeout_at};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
use crate::{
|
||||
FastLockGuard, GlobalLockManager, LockClient, LockId, LockInfo, LockManager, LockMetadata, LockPriority, LockRequest,
|
||||
@@ -28,98 +26,13 @@ use crate::{
|
||||
|
||||
/// Default shard count for guard storage (must be power of 2)
|
||||
const DEFAULT_GUARD_SHARD_COUNT: usize = 64;
|
||||
const EVENT_NAMESPACE_LOCK_ACQUIRE: &str = "namespace_lock_acquire";
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_LOCK: &str = "lock";
|
||||
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
|
||||
|
||||
type GuardMap = HashMap<LockId, LocalGuardEntry>;
|
||||
type GuardShard = Arc<GuardRegistry>;
|
||||
type GuardShard = Arc<RwLock<HashMap<LockId, LocalGuardEntry>>>;
|
||||
type GuardStorage = Arc<Vec<GuardShard>>;
|
||||
|
||||
/// Registry access never reserves the table for a task that has not resumed.
|
||||
/// Only short synchronous map operations hold the native lock; contention
|
||||
/// waits on a notification without blocking an executor thread.
|
||||
#[derive(Debug, Default)]
|
||||
struct GuardRegistry {
|
||||
entries: RwLock<GuardMap>,
|
||||
changed: Notify,
|
||||
}
|
||||
|
||||
struct GuardRegistryWake<'a>(&'a Notify);
|
||||
|
||||
impl Drop for GuardRegistryWake<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
struct GuardRegistryAccess<'a, G> {
|
||||
// Field drop order unlocks the table before waking its waiters, including
|
||||
// when a map operation unwinds. Physical guards are removed before Drop.
|
||||
guard: G,
|
||||
_wake: GuardRegistryWake<'a>,
|
||||
}
|
||||
|
||||
impl<G: Deref> Deref for GuardRegistryAccess<'_, G> {
|
||||
type Target = G::Target;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.guard
|
||||
}
|
||||
}
|
||||
|
||||
impl<G: DerefMut> DerefMut for GuardRegistryAccess<'_, G> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.guard
|
||||
}
|
||||
}
|
||||
|
||||
impl GuardRegistry {
|
||||
fn access<G>(&self, guard: G) -> GuardRegistryAccess<'_, G> {
|
||||
GuardRegistryAccess {
|
||||
guard,
|
||||
_wake: GuardRegistryWake(&self.changed),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read(&self) -> GuardRegistryAccess<'_, RwLockReadGuard<'_, GuardMap>> {
|
||||
tokio::task::consume_budget().await;
|
||||
if let Some(guard) = self.entries.try_read() {
|
||||
return self.access(guard);
|
||||
}
|
||||
loop {
|
||||
tokio::task::consume_budget().await;
|
||||
let notified = self.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if let Some(guard) = self.entries.try_read() {
|
||||
return self.access(guard);
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn write(&self) -> GuardRegistryAccess<'_, RwLockWriteGuard<'_, GuardMap>> {
|
||||
tokio::task::consume_budget().await;
|
||||
if let Some(guard) = self.entries.try_write() {
|
||||
return self.access(guard);
|
||||
}
|
||||
loop {
|
||||
tokio::task::consume_budget().await;
|
||||
// Register before rechecking so an unlock between the first
|
||||
// failed try and this wait cannot leave the task asleep.
|
||||
let notified = self.changed.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if let Some(guard) = self.entries.try_write() {
|
||||
return self.access(guard);
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Local lock client using FastLock with sharded guard storage for better concurrency
|
||||
#[derive(Debug)]
|
||||
pub struct LocalClient {
|
||||
@@ -181,7 +94,7 @@ impl LocalClient {
|
||||
pub fn with_shard_count(shard_count: usize) -> Self {
|
||||
assert!(shard_count.is_power_of_two(), "Shard count must be power of 2");
|
||||
|
||||
let guard_storage: Vec<GuardShard> = (0..shard_count).map(|_| Arc::new(GuardRegistry::default())).collect();
|
||||
let guard_storage: Vec<GuardShard> = (0..shard_count).map(|_| Arc::new(RwLock::new(HashMap::new()))).collect();
|
||||
|
||||
Self::with_storage(Arc::new(guard_storage), None, crate::fast_lock::CLEANUP_INTERVAL)
|
||||
}
|
||||
@@ -202,7 +115,7 @@ impl LocalClient {
|
||||
/// This allows simulating multi-node environments where each node has its own lock backend
|
||||
pub fn with_manager(manager: Arc<GlobalLockManager>) -> Self {
|
||||
let guard_storage = (0..DEFAULT_GUARD_SHARD_COUNT)
|
||||
.map(|_| Arc::new(GuardRegistry::default()))
|
||||
.map(|_| Arc::new(RwLock::new(HashMap::new())))
|
||||
.collect();
|
||||
Self::with_storage(Arc::new(guard_storage), Some(manager), crate::fast_lock::CLEANUP_INTERVAL)
|
||||
}
|
||||
@@ -210,7 +123,7 @@ impl LocalClient {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_manager_and_reaper_interval(manager: Arc<GlobalLockManager>, reaper_interval: Duration) -> Self {
|
||||
let guard_storage = (0..DEFAULT_GUARD_SHARD_COUNT)
|
||||
.map(|_| Arc::new(GuardRegistry::default()))
|
||||
.map(|_| Arc::new(RwLock::new(HashMap::new())))
|
||||
.collect();
|
||||
Self::with_storage(Arc::new(guard_storage), Some(manager), reaper_interval)
|
||||
}
|
||||
@@ -227,7 +140,7 @@ impl LocalClient {
|
||||
}
|
||||
|
||||
/// Get the shard for a given lock ID
|
||||
fn get_shard(&self, lock_id: &LockId) -> &GuardShard {
|
||||
fn get_shard(&self, lock_id: &LockId) -> &Arc<RwLock<HashMap<LockId, LocalGuardEntry>>> {
|
||||
let index = self.get_resource_shard_index(&lock_id.resource);
|
||||
&self.guard_storage[index]
|
||||
}
|
||||
@@ -319,7 +232,6 @@ impl Default for LocalClient {
|
||||
#[async_trait::async_trait]
|
||||
impl LockClient for LocalClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let started = Instant::now();
|
||||
self.ensure_reaper();
|
||||
if request.lock_id.resource != request.resource {
|
||||
return Ok(LockResponse::failure(
|
||||
@@ -339,7 +251,6 @@ impl LockClient for LocalClient {
|
||||
.await
|
||||
.unwrap_or(0)
|
||||
};
|
||||
let reclaim_elapsed = started.elapsed();
|
||||
|
||||
let build_lock_request = |acquire_timeout| match request.lock_type {
|
||||
LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
|
||||
@@ -354,65 +265,17 @@ impl LockClient for LocalClient {
|
||||
if remaining.is_zero() {
|
||||
return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
|
||||
}
|
||||
let manager_started = Instant::now();
|
||||
match lock_manager.acquire_lock(build_lock_request(remaining)).await {
|
||||
Ok(guard) => {
|
||||
let manager_elapsed = manager_started.elapsed();
|
||||
let registration_started = Instant::now();
|
||||
let physical_guard_id = guard.guard_id();
|
||||
let lock_id = request.lock_id.clone();
|
||||
let entry = LocalGuardEntry::new(guard, request.ttl);
|
||||
let acquired_at = entry.acquired_at;
|
||||
let expires_at = entry.expires_at;
|
||||
|
||||
let (replaced_entry, registration_wait, registration_mutation, registry_entries, registry_capacity) = {
|
||||
{
|
||||
let shard = self.get_shard(&lock_id);
|
||||
let Ok(mut guards) = timeout_at(acquire_deadline, shard.write()).await else {
|
||||
return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
|
||||
};
|
||||
// A ready table cannot publish a grant after the peer
|
||||
// budget expired while its acquisition task was idle.
|
||||
if Instant::now() >= acquire_deadline {
|
||||
drop(guards);
|
||||
return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
|
||||
}
|
||||
let registration_wait = registration_started.elapsed();
|
||||
let mutation_started = Instant::now();
|
||||
let replaced_entry = guards.insert(lock_id.clone(), entry);
|
||||
(
|
||||
replaced_entry,
|
||||
registration_wait,
|
||||
mutation_started.elapsed(),
|
||||
guards.len(),
|
||||
guards.capacity(),
|
||||
)
|
||||
};
|
||||
// Repeated shared requests can replace an entry. Releasing
|
||||
// its physical guard must not reenter locks under the table.
|
||||
drop(replaced_entry);
|
||||
|
||||
let registration_elapsed = registration_started.elapsed();
|
||||
if started.elapsed() >= Duration::from_millis(100) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_lock::namespace_owner",
|
||||
event = EVENT_NAMESPACE_LOCK_ACQUIRE,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "local_completed",
|
||||
resource = %request.resource,
|
||||
lock_id = %lock_id,
|
||||
mode = ?request.lock_type,
|
||||
physical_guard_id,
|
||||
reclaim_ms = reclaim_elapsed.as_millis(),
|
||||
manager_ms = manager_elapsed.as_millis(),
|
||||
registration_ms = registration_elapsed.as_millis(),
|
||||
registration_wait_ms = registration_wait.as_millis(),
|
||||
registration_mutation_ms = registration_mutation.as_millis(),
|
||||
registry_entries,
|
||||
registry_capacity,
|
||||
acquire_ms = started.elapsed().as_millis(),
|
||||
"Namespace local lock acquisition completed slowly"
|
||||
);
|
||||
let mut guards = shard.write().await;
|
||||
guards.insert(lock_id.clone(), entry);
|
||||
}
|
||||
|
||||
let lock_info = LockInfo {
|
||||
@@ -606,8 +469,6 @@ impl LockClient for LocalClient {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{GlobalLockManager, LockClient, LockRequest, LockType};
|
||||
use futures::FutureExt;
|
||||
use std::task::Poll;
|
||||
|
||||
fn request(resource: crate::ObjectKey, owner: &str, ttl: Duration) -> LockRequest {
|
||||
LockRequest::new(resource, LockType::Exclusive, owner)
|
||||
@@ -615,295 +476,6 @@ mod tests {
|
||||
.with_acquire_timeout(Duration::from_millis(80))
|
||||
}
|
||||
|
||||
fn hold_guard_shard(shard: GuardShard) -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) {
|
||||
let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(0);
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build the independent guard-table owner runtime");
|
||||
let guard = runtime.block_on(shard.write());
|
||||
entered_tx.send(()).expect("signal that the guard table is occupied");
|
||||
// A dropped controller also releases the table on test failure.
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(3));
|
||||
drop(guard);
|
||||
});
|
||||
entered_rx
|
||||
.recv_timeout(Duration::from_secs(3))
|
||||
.expect("guard-table owner must enter before the tested operation");
|
||||
(release_tx, thread)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unpolled_registry_writer_does_not_block_compatible_acquisition() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager(manager.clone());
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let first = LockRequest::new(crate::ObjectKey::new("bucket", ""), LockType::Shared, "first");
|
||||
assert!(client.acquire_lock(&first).await.expect("first shared lock").success);
|
||||
let (release_table, table_owner) = hold_guard_shard(client.get_shard(&first.lock_id).clone());
|
||||
let mut pending_release = Box::pin(client.release(&first.lock_id));
|
||||
assert!(matches!(futures::poll!(&mut pending_release), Poll::Pending));
|
||||
release_table.send(()).expect("release the occupied table");
|
||||
table_owner.join().expect("guard-table owner must finish");
|
||||
|
||||
// The earlier writer is deliberately not polled: an unoccupied
|
||||
// bookkeeping table must remain usable by a currently running peer.
|
||||
let second = LockRequest::new(first.resource.clone(), LockType::Shared, "second");
|
||||
let response = client
|
||||
.acquire_lock(&second)
|
||||
.now_or_never()
|
||||
.expect("a sleeping registry writer must not reserve the available table")
|
||||
.expect("compatible shared acquisition should return a response");
|
||||
assert!(response.success);
|
||||
assert_eq!(
|
||||
manager
|
||||
.as_fast_lock_manager()
|
||||
.expect("enabled physical lock backend")
|
||||
.get_shard(&first.resource)
|
||||
.active_guard_count(),
|
||||
2
|
||||
);
|
||||
assert!(
|
||||
pending_release
|
||||
.await
|
||||
.expect("the earlier writer still releases its exact grant")
|
||||
);
|
||||
assert!(client.release(&second.lock_id).await.expect("release the second grant"));
|
||||
assert_eq!(
|
||||
manager
|
||||
.as_fast_lock_manager()
|
||||
.expect("enabled physical lock backend")
|
||||
.get_shard(&first.resource)
|
||||
.active_guard_count(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acquisition_deadline_rejects_registration_after_physical_grant() {
|
||||
for unlock_before_poll in [false, true] {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager(manager.clone());
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "registration-deadline");
|
||||
let blocker = manager
|
||||
.acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "blocker"))
|
||||
.await
|
||||
.expect("hold the physical lock until guard-table preflight finishes");
|
||||
let request =
|
||||
request(resource.clone(), "owner", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(20));
|
||||
let mut acquire = Box::pin(tokio::task::unconstrained(client.acquire_lock(&request)));
|
||||
assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
|
||||
let (release_table, table_owner) = hold_guard_shard(client.get_shard(&request.lock_id).clone());
|
||||
drop(blocker);
|
||||
assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
|
||||
assert_eq!(
|
||||
manager
|
||||
.as_fast_lock_manager()
|
||||
.expect("enabled physical lock backend")
|
||||
.get_shard(&resource)
|
||||
.active_guard_count(),
|
||||
1,
|
||||
"the tested future owns a physical grant"
|
||||
);
|
||||
tokio::time::advance(Duration::from_millis(21)).await;
|
||||
if unlock_before_poll {
|
||||
release_table.send(()).expect("make registration ready after its deadline");
|
||||
table_owner.join().expect("guard-table owner must finish");
|
||||
assert!(!acquire.await.expect("expired registration response").success);
|
||||
} else {
|
||||
let response = timeout(Duration::from_millis(100), &mut acquire)
|
||||
.await
|
||||
.expect("registration must honor its deadline while the table remains occupied")
|
||||
.expect("expired registration response");
|
||||
assert!(!response.success);
|
||||
release_table.send(()).expect("release the occupied table");
|
||||
table_owner.join().expect("guard-table owner must finish");
|
||||
}
|
||||
assert!(
|
||||
client
|
||||
.check_status(&request.lock_id)
|
||||
.await
|
||||
.expect("check the expired identity")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.as_fast_lock_manager()
|
||||
.expect("enabled physical lock backend")
|
||||
.get_shard(&resource)
|
||||
.active_guard_count(),
|
||||
0,
|
||||
"expired registration must release its grant"
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.acquire_lock(crate::ObjectLockRequest::new_write(resource, "next-owner"))
|
||||
.await
|
||||
.is_ok(),
|
||||
"the next owner must not inherit an abandoned physical grant"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancelling_registration_releases_the_unpublished_physical_grant() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager(manager.clone());
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "cancel-registration");
|
||||
let blocker = manager
|
||||
.acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "blocker"))
|
||||
.await
|
||||
.expect("hold the physical lock through registry preflight");
|
||||
let request = request(resource.clone(), "cancelled", Duration::from_secs(30));
|
||||
let mut acquire = Box::pin(tokio::task::unconstrained(client.acquire_lock(&request)));
|
||||
assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
|
||||
let (release_table, table_owner) = hold_guard_shard(client.get_shard(&request.lock_id).clone());
|
||||
drop(blocker);
|
||||
assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
|
||||
let backend = manager.as_fast_lock_manager().expect("enabled physical lock backend");
|
||||
assert_eq!(backend.get_shard(&resource).active_guard_count(), 1);
|
||||
drop(acquire);
|
||||
assert_eq!(backend.get_shard(&resource).active_guard_count(), 0);
|
||||
assert!(
|
||||
manager
|
||||
.acquire_lock(crate::ObjectLockRequest::new_write(resource, "next-owner"))
|
||||
.await
|
||||
.is_ok(),
|
||||
"cancellation must release the physical lock without waiting for the table"
|
||||
);
|
||||
release_table.send(()).expect("release the occupied table");
|
||||
table_owner.join().expect("guard-table owner must finish");
|
||||
assert!(
|
||||
client
|
||||
.check_status(&request.lock_id)
|
||||
.await
|
||||
.expect("check cancelled identity")
|
||||
.is_none()
|
||||
);
|
||||
assert!(client.list_lock_leases().await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_a_registry_waiter_keeps_other_waiters_live() {
|
||||
let shard = Arc::new(GuardRegistry::default());
|
||||
let (release_table, table_owner) = hold_guard_shard(shard.clone());
|
||||
let mut cancelled = Box::pin(shard.write());
|
||||
let mut survivor = Box::pin(shard.read());
|
||||
assert!(matches!(futures::poll!(&mut cancelled), Poll::Pending));
|
||||
assert!(matches!(futures::poll!(&mut survivor), Poll::Pending));
|
||||
drop(cancelled);
|
||||
release_table.send(()).expect("wake registry waiters");
|
||||
table_owner.join().expect("guard-table owner must finish");
|
||||
let access = timeout(Duration::from_millis(100), survivor)
|
||||
.await
|
||||
.expect("a cancelled waiter must not consume another waiter's unlock notification");
|
||||
assert!(access.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn uncontended_registry_access_retains_cooperative_scheduling() {
|
||||
let shard = GuardRegistry::default();
|
||||
let mut yielded = false;
|
||||
for _ in 0..1024 {
|
||||
match shard.read().now_or_never() {
|
||||
Some(access) => drop(access),
|
||||
None => {
|
||||
yielded = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(yielded, "a loop of ready registry operations must still yield to other executor tasks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_shared_identity_replaces_only_its_previous_physical_guard() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager(manager.clone());
|
||||
let request = LockRequest::new(crate::ObjectKey::new("bucket", ""), LockType::Shared, "owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("first shared grant").success);
|
||||
assert!(client.acquire_lock(&request).await.expect("repeated shared identity").success);
|
||||
let backend = manager.as_fast_lock_manager().expect("enabled physical lock backend");
|
||||
assert_eq!(backend.get_shard(&request.resource).active_guard_count(), 1);
|
||||
assert_eq!(client.list_lock_leases().await.len(), 1);
|
||||
assert!(client.release(&request.lock_id).await.expect("release the replacement grant"));
|
||||
assert_eq!(backend.get_shard(&request.resource).active_guard_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn hot_shared_registry_keeps_refresh_reaper_and_snapshots_live() {
|
||||
for width in [64, 128] {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = Arc::new(LocalClient::with_manager_and_reaper_interval(manager.clone(), Duration::from_millis(1)));
|
||||
let resource = crate::ObjectKey::new("bucket", "");
|
||||
let expired = LockRequest::new(resource.clone(), LockType::Shared, "expired").with_ttl(Duration::ZERO);
|
||||
assert!(client.acquire_lock(&expired).await.expect("expired seed grant").success);
|
||||
let acquired = Arc::new(tokio::sync::Barrier::new(width + 1));
|
||||
let observed = Arc::new(Notify::new());
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for index in 0..width {
|
||||
let client = client.clone();
|
||||
let resource = resource.clone();
|
||||
let acquired = acquired.clone();
|
||||
let observed = observed.clone();
|
||||
tasks.spawn(async move {
|
||||
for iteration in 0..8 {
|
||||
let request = LockRequest::new(resource.clone(), LockType::Shared, format!("load-{index}"));
|
||||
assert!(client.acquire_lock(&request).await.expect("concurrent shared grant").success);
|
||||
if iteration == 0 {
|
||||
let notification = observed.notified();
|
||||
tokio::pin!(notification);
|
||||
notification.as_mut().enable();
|
||||
acquired.wait().await;
|
||||
notification.await;
|
||||
} else {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(client.refresh(&request.lock_id).await.expect("refresh concurrent grant"));
|
||||
assert!(
|
||||
client
|
||||
.check_status(&request.lock_id)
|
||||
.await
|
||||
.expect("read concurrent grant")
|
||||
.is_some()
|
||||
);
|
||||
assert!(client.release(&request.lock_id).await.expect("release concurrent grant"));
|
||||
}
|
||||
});
|
||||
}
|
||||
timeout(Duration::from_secs(5), acquired.wait())
|
||||
.await
|
||||
.expect("all shared grants must reach the snapshot gate");
|
||||
let snapshot = client.list_lock_leases().await;
|
||||
assert_eq!(snapshot.iter().filter(|lease| lease.owner.starts_with("load-")).count(), width);
|
||||
observed.notify_waiters();
|
||||
let mut completed = 0;
|
||||
timeout(Duration::from_secs(5), async {
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
result.expect("all shared grant workers must finish");
|
||||
completed += 1;
|
||||
let _ = client.list_lock_leases().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("refresh, cleanup, and concurrent snapshots must make progress");
|
||||
assert_eq!(completed, width);
|
||||
wait_until_reaped(&client, &expired.lock_id).await;
|
||||
assert!(client.list_lock_leases().await.is_empty());
|
||||
assert!(
|
||||
manager
|
||||
.as_fast_lock_manager()
|
||||
.expect("enabled physical lock backend")
|
||||
.list_locks()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_until_reaped(client: &LocalClient, lock_id: &LockId) {
|
||||
for _ in 0..80 {
|
||||
if client.check_status(lock_id).await.unwrap().is_none() {
|
||||
@@ -930,9 +502,8 @@ mod tests {
|
||||
}
|
||||
|
||||
let lock_request = request(resource, "owner", Duration::from_secs(30));
|
||||
let response = client
|
||||
.acquire_lock(&lock_request)
|
||||
.now_or_never()
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
|
||||
.await
|
||||
.expect("an acquire must not sweep unrelated guard shards")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(response.success);
|
||||
@@ -959,9 +530,8 @@ mod tests {
|
||||
}
|
||||
|
||||
let replacement = request(resource, "replacement-owner", Duration::from_secs(30));
|
||||
let response = client
|
||||
.acquire_lock(&replacement)
|
||||
.now_or_never()
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&replacement))
|
||||
.await
|
||||
.expect("expired guard reclaim must only write the resource shard")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(response.success);
|
||||
@@ -977,7 +547,7 @@ mod tests {
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-deadline");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
let (release_table, table_owner) = hold_guard_shard(client.guard_storage[target_shard].clone());
|
||||
let _target_write = client.guard_storage[target_shard].write().await;
|
||||
let lock_request = request(resource, "owner", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
|
||||
@@ -985,8 +555,6 @@ mod tests {
|
||||
.expect("acquire should return after its reclaim budget expires")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(!response.success);
|
||||
release_table.send(()).expect("release the occupied resource table");
|
||||
table_owner.join().expect("guard-table owner must finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+126
-901
File diff suppressed because it is too large
Load Diff
@@ -43,9 +43,14 @@ const CONTAINER_META_HEADER_PREFIX: &str = "x-container-meta-";
|
||||
/// Request header prefix removing a container metadata item.
|
||||
const CONTAINER_META_REMOVE_HEADER_PREFIX: &str = "x-remove-container-meta-";
|
||||
|
||||
pub(crate) use super::storage_api::tagging::{
|
||||
SWIFT_ACCOUNT_META_TAG_PREFIX as ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX as CONTAINER_META_TAG_PREFIX,
|
||||
};
|
||||
/// Bucket-tag namespace holding account metadata items.
|
||||
pub(crate) const ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
|
||||
/// Bucket-tag namespace holding container metadata items.
|
||||
///
|
||||
/// Deliberately narrower than the `swift-` tags around it: the container ACL
|
||||
/// (`swift-acl-*`) and versioning (`swift-versions-location`) tags share this
|
||||
/// tag set and must survive a metadata POST.
|
||||
pub(crate) const CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
|
||||
|
||||
/// The metadata changes carried by one account or container POST.
|
||||
///
|
||||
|
||||
@@ -61,9 +61,11 @@ pub use errors::{SwiftError, SwiftResult};
|
||||
pub use metadata_update::MetadataUpdate;
|
||||
pub use router::{SwiftRoute, SwiftRouter};
|
||||
|
||||
pub(crate) use storage_api::tagging::{
|
||||
MAX_SWIFT_METADATA_COUNT as MAX_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE as MAX_METADATA_VALUE_SIZE,
|
||||
};
|
||||
/// Maximum number of metadata headers allowed per resource (Swift standard)
|
||||
pub(crate) const MAX_METADATA_COUNT: usize = 90;
|
||||
|
||||
/// Maximum size in bytes for a single metadata value (Swift standard)
|
||||
pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
|
||||
|
||||
/// Validate metadata against Swift limits
|
||||
///
|
||||
|
||||
@@ -16,7 +16,6 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
|
||||
use rustfs_ecstore::api::bucket::metadata::SWIFT_TAGGING_MARKER;
|
||||
use rustfs_ecstore::api::bucket::metadata::{BUCKET_TAGGING_CONFIG, is_unreadable_config_error};
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{get as get_swift_bucket_metadata_from_backend, update_config_with};
|
||||
use rustfs_ecstore::api::bucket::utils::serialize as serialize_bucket_config;
|
||||
@@ -26,7 +25,7 @@ use rustfs_ecstore::api::notification::get_global_notification_sys;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::object_store_handle as resolve_swift_object_store_handle;
|
||||
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use s3s::dto::Tagging;
|
||||
|
||||
use super::{SwiftError, SwiftResult};
|
||||
|
||||
@@ -55,12 +54,6 @@ pub(crate) mod public_api {
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod tagging {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::{
|
||||
MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE, SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod versioning {
|
||||
pub(crate) use super::storage_contracts::{ListOperations, ObjectOperations};
|
||||
}
|
||||
@@ -112,7 +105,7 @@ where
|
||||
// including the container ACL and versioning tags — because the rewrite
|
||||
// closures treat "no parsed tags" as "no tags".
|
||||
let result = update_config_with(&bucket, BUCKET_TAGGING_CONFIG, |bm| {
|
||||
let mut tagging = match rewrite(bm.tagging_config.as_ref()) {
|
||||
let tagging = match rewrite(bm.tagging_config.as_ref()) {
|
||||
Ok(tagging) => tagging,
|
||||
Err(err) => {
|
||||
rejected = Some(err);
|
||||
@@ -123,16 +116,6 @@ where
|
||||
if tagging.tag_set.is_empty() {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
if !tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.any(|tag| tag.key.as_deref() == Some(SWIFT_TAGGING_MARKER))
|
||||
{
|
||||
tagging.tag_set.push(Tag {
|
||||
key: Some(SWIFT_TAGGING_MARKER.to_owned()),
|
||||
value: Some("1".to_owned()),
|
||||
});
|
||||
}
|
||||
// The S3 XML serializer, not quick_xml: the metadata loader's
|
||||
// parse step must be able to round-trip what we persist.
|
||||
serialize_bucket_config(&tagging)
|
||||
|
||||
@@ -82,22 +82,6 @@ Configure WebDAV via environment variables:
|
||||
| `RUSTFS_WEBDAV_MAX_BODY_SIZE` | Max upload size (bytes) | 5GB |
|
||||
| `RUSTFS_WEBDAV_REQUEST_TIMEOUT` | Request timeout (seconds) | 300 |
|
||||
| `RUSTFS_WEBDAV_MAX_CONNECTIONS` | Max concurrently served connections | 1024 |
|
||||
| `RUSTFS_CORS_ALLOWED_ORIGINS` | Browser origins allowed by the S3 and WebDAV listeners | empty |
|
||||
|
||||
## Browser Access and CORS
|
||||
|
||||
WebDAV reads `RUSTFS_CORS_ALLOWED_ORIGINS` at server startup. Set a comma-separated list of exact frontend origins to allow browser clients, for example `https://dav-console.example,https://files.example`. Explicit origins allow credentialed requests. The default empty value disables cross-origin access. A standalone `*` permits any origin with credentials mode `omit` and emits no `Access-Control-Allow-Credentials` header.
|
||||
|
||||
Valid `OPTIONS` preflight requests from an allowed origin receive `204 No Content` without Basic authentication. Actual WebDAV requests and ordinary `OPTIONS` requests still use Basic authentication and the existing IAM checks. Allowed origins receive CORS headers on success and error responses. Preflights allow WebDAV methods and explicitly echo valid requested header names, including `Authorization`, `Depth`, `Destination`, and `If-Match`. Responses expose `ETag`, `DAV`, `Allow`, `Lock-Token`, and range headers to browser code.
|
||||
|
||||
```bash
|
||||
curl -i -X OPTIONS http://localhost:8080/ \
|
||||
-H 'Origin: https://dav-console.example' \
|
||||
-H 'Access-Control-Request-Method: PROPFIND' \
|
||||
-H 'Access-Control-Request-Headers: authorization,depth,content-type'
|
||||
```
|
||||
|
||||
Configure an explicit HTTPS frontend origin for credentialed browser access and use HTTPS on the WebDAV listener when sending access keys and secret keys. This listener-wide policy applies across WebDAV paths. S3 bucket CORS rules remain scoped to the S3 API.
|
||||
|
||||
## Quick Start
|
||||
|
||||
|
||||
@@ -23,10 +23,6 @@ pub enum WebDavInitError {
|
||||
Bind(#[from] std::io::Error),
|
||||
#[error("server error: {0}")]
|
||||
Server(String),
|
||||
#[error("WebDAV background task failed: {0}")]
|
||||
TaskJoin(#[from] tokio::task::JoinError),
|
||||
#[error("WebDAV connections did not drain before the shutdown deadline")]
|
||||
ShutdownDeadline,
|
||||
#[error("invalid WebDAV configuration: {0}")]
|
||||
InvalidConfig(String),
|
||||
#[error("TLS error: {0}")]
|
||||
|
||||
@@ -19,12 +19,8 @@ use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, is_tem
|
||||
use bytes::Bytes;
|
||||
use dav_server::DavHandler;
|
||||
use dav_server::fakels::FakeLs;
|
||||
use http::header::{
|
||||
ACCESS_CONTROL_ALLOW_CREDENTIALS, ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN,
|
||||
ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, AUTHORIZATION, ORIGIN, REFERER,
|
||||
USER_AGENT, VARY,
|
||||
};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use http::header::{AUTHORIZATION, REFERER, USER_AGENT};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
|
||||
use hyper::body::Body as HttpBody;
|
||||
use hyper::server::conn::http1;
|
||||
@@ -35,23 +31,18 @@ use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_
|
||||
use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
|
||||
use rustfs_utils::MaskedAccessKey;
|
||||
use std::convert::Infallible;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::net::IpAddr;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::{Semaphore, broadcast, watch};
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
use tracing::{Instrument, debug, error, info, info_span, warn};
|
||||
|
||||
const CONNECTION_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
||||
const LOG_SUBSYSTEM_WEBDAV_SERVER: &str = "webdav_server";
|
||||
const LOG_SUBSYSTEM_WEBDAV_AUTH: &str = "webdav_auth";
|
||||
@@ -69,177 +60,6 @@ const EVENT_WEBDAV_CONNECTION_CAP_STATE: &str = "webdav_connection_cap_state";
|
||||
/// materialise a whole object in memory for every GET.
|
||||
type WebDavBody = Pin<Box<dyn HttpBody<Data = Bytes, Error = io::Error> + Send>>;
|
||||
|
||||
/// Listener-wide browser policy, parsed once and shared across connections.
|
||||
enum WebDavCors {
|
||||
Disabled,
|
||||
Wildcard,
|
||||
Origins(Vec<HeaderValue>),
|
||||
}
|
||||
|
||||
impl WebDavCors {
|
||||
const METHODS: &str = "OPTIONS, GET, HEAD, PUT, DELETE, PATCH, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK";
|
||||
|
||||
fn new(origins: &str) -> Self {
|
||||
match origins.trim() {
|
||||
"" => Self::Disabled,
|
||||
"*" => Self::Wildcard,
|
||||
origins => Self::Origins(
|
||||
origins
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|origin| !origin.is_empty() && *origin != "*")
|
||||
.filter_map(|origin| HeaderValue::from_str(origin).ok())
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn allowed_origin(&self, headers: &HeaderMap) -> Option<HeaderValue> {
|
||||
if matches!(self, Self::Disabled) {
|
||||
return None;
|
||||
}
|
||||
let mut origins = headers.get_all(ORIGIN).iter();
|
||||
let origin = origins.next()?;
|
||||
let value = origin.to_str().ok()?;
|
||||
if origins.next().is_some() || value.is_empty() || value.bytes().any(|byte| byte.is_ascii_whitespace() || byte == b',') {
|
||||
return None;
|
||||
}
|
||||
match self {
|
||||
Self::Origins(allowed) => allowed.iter().find(|allowed| *allowed == origin).cloned(),
|
||||
Self::Wildcard => Some(HeaderValue::from_static("*")),
|
||||
Self::Disabled => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep the CORS boundary outside authentication and DAV dispatch so every
|
||||
/// application response, including early errors, follows the same policy.
|
||||
async fn handle<B, F, Fut>(&self, req: Request<B>, next: F) -> Result<Response<WebDavBody>, Infallible>
|
||||
where
|
||||
F: FnOnce(Request<B>) -> Fut,
|
||||
Fut: Future<Output = Result<Response<WebDavBody>, Infallible>>,
|
||||
{
|
||||
let allow_origin = self.allowed_origin(req.headers());
|
||||
let is_options = req.method() == Method::OPTIONS;
|
||||
let is_preflight =
|
||||
is_options && req.headers().contains_key(ORIGIN) && req.headers().contains_key(ACCESS_CONTROL_REQUEST_METHOD);
|
||||
|
||||
let mut response = if is_preflight {
|
||||
self.preflight(req.headers(), allow_origin.is_some())
|
||||
} else {
|
||||
next(req).await?
|
||||
};
|
||||
|
||||
// Absent and rejected Origin values also select cache variants. Append
|
||||
// rather than overwrite the DAV handler's existing Vary fields.
|
||||
response.headers_mut().append(VARY, HeaderValue::from_static("Origin"));
|
||||
if is_options {
|
||||
response.headers_mut().append(
|
||||
VARY,
|
||||
HeaderValue::from_static("Access-Control-Request-Method, Access-Control-Request-Headers"),
|
||||
);
|
||||
}
|
||||
if let Some(origin) = allow_origin {
|
||||
response.headers_mut().insert(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
|
||||
if matches!(self, Self::Origins(_)) {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(ACCESS_CONTROL_ALLOW_CREDENTIALS, HeaderValue::from_static("true"));
|
||||
}
|
||||
if !is_preflight {
|
||||
response.headers_mut().insert(
|
||||
ACCESS_CONTROL_EXPOSE_HEADERS,
|
||||
HeaderValue::from_static("ETag, DAV, Allow, Lock-Token, Content-Range, Accept-Ranges, WWW-Authenticate"),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn preflight(&self, headers: &HeaderMap, origin_allowed: bool) -> Response<WebDavBody> {
|
||||
let mut methods = headers.get_all(ACCESS_CONTROL_REQUEST_METHOD).iter();
|
||||
let method = methods.next().and_then(|value| value.to_str().ok());
|
||||
let method_allowed = method.is_some_and(|method| Self::METHODS.split(", ").any(|allowed| allowed == method));
|
||||
if !origin_allowed || !method_allowed || methods.next().is_some() {
|
||||
return error_response(StatusCode::FORBIDDEN, "CORS preflight denied");
|
||||
}
|
||||
let Ok(allowed_headers) = cors_requested_headers(headers) else {
|
||||
return error_response(StatusCode::FORBIDDEN, "CORS preflight denied");
|
||||
};
|
||||
|
||||
let mut response = Response::new(fixed_body(Bytes::new()));
|
||||
*response.status_mut() = StatusCode::NO_CONTENT;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(ACCESS_CONTROL_ALLOW_METHODS, HeaderValue::from_static(Self::METHODS));
|
||||
// A wildcard does not cover Authorization, and is not a wildcard for
|
||||
// credentialed requests. Echo the validated names explicitly instead.
|
||||
if let Some(value) = allowed_headers {
|
||||
response.headers_mut().insert(ACCESS_CONTROL_ALLOW_HEADERS, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate borrowed field names, reusing an ordinary single header value.
|
||||
/// RFC 9110 section 5.6.1.2 requires recipients to ignore empty list elements;
|
||||
/// normalize those and repeated fields into one nonempty response list.
|
||||
fn cors_requested_headers(headers: &HeaderMap) -> Result<Option<HeaderValue>, ()> {
|
||||
let values = headers.get_all(ACCESS_CONTROL_REQUEST_HEADERS);
|
||||
let mut fields = values.iter();
|
||||
let Some(first) = fields.next() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut normalize = fields.next().is_some();
|
||||
let mut has_name = false;
|
||||
for value in values.iter() {
|
||||
for name in value.to_str().map_err(|_| ())?.split(',') {
|
||||
let name = name.trim_matches([' ', '\t']);
|
||||
if name.is_empty() {
|
||||
normalize = true;
|
||||
continue;
|
||||
}
|
||||
if !is_cors_header_name(name) {
|
||||
return Err(());
|
||||
}
|
||||
has_name = true;
|
||||
}
|
||||
}
|
||||
if !has_name {
|
||||
return Err(());
|
||||
}
|
||||
if !normalize {
|
||||
return Ok(Some(first.clone()));
|
||||
}
|
||||
|
||||
let capacity = values.iter().map(|value| value.as_bytes().len() + 1).sum();
|
||||
let mut normalized = String::with_capacity(capacity);
|
||||
for value in values {
|
||||
for name in value.to_str().map_err(|_| ())?.split(',') {
|
||||
let name = name.trim_matches([' ', '\t']);
|
||||
if name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !normalized.is_empty() {
|
||||
normalized.push(',');
|
||||
}
|
||||
normalized.push_str(name);
|
||||
}
|
||||
}
|
||||
HeaderValue::try_from(normalized).map(Some).map_err(|_| ())
|
||||
}
|
||||
|
||||
/// RFC 9110 field-name is a nonempty sequence of ASCII tchar bytes.
|
||||
fn is_cors_header_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#' | b'$' | b'%' | b'&' | b'\'' | b'*' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~'
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
|
||||
let mut policy_headers = HeaderMap::new();
|
||||
for name in [USER_AGENT, REFERER] {
|
||||
@@ -254,11 +74,6 @@ fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
|
||||
policy_headers
|
||||
}
|
||||
|
||||
struct ConnectionLimits {
|
||||
max_body_size: u64,
|
||||
request_timeout: Duration,
|
||||
}
|
||||
|
||||
/// WebDAV server implementation
|
||||
pub struct WebDavServer<S>
|
||||
where
|
||||
@@ -268,7 +83,6 @@ where
|
||||
config: WebDavConfig,
|
||||
/// S3 storage backend
|
||||
storage: S,
|
||||
cors: Arc<WebDavCors>,
|
||||
}
|
||||
|
||||
impl<S> WebDavServer<S>
|
||||
@@ -286,13 +100,7 @@ where
|
||||
/// Create a new WebDAV server
|
||||
pub async fn new(config: WebDavConfig, storage: S) -> Result<Self, WebDavInitError> {
|
||||
config.validate().await?;
|
||||
let origins =
|
||||
rustfs_utils::get_env_str(rustfs_config::ENV_CORS_ALLOWED_ORIGINS, rustfs_config::DEFAULT_CORS_ALLOWED_ORIGINS);
|
||||
Ok(Self {
|
||||
config,
|
||||
storage,
|
||||
cors: Arc::new(WebDavCors::new(&origins)),
|
||||
})
|
||||
Ok(Self { config, storage })
|
||||
}
|
||||
|
||||
/// Start the WebDAV server
|
||||
@@ -327,8 +135,7 @@ where
|
||||
bind_addr = %self.config.bind_addr,
|
||||
"WebDAV server listening"
|
||||
);
|
||||
let (shutdown_tx, shutdown) = watch::channel(false);
|
||||
let mut reload_task = None;
|
||||
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
|
||||
|
||||
// Setup TLS if enabled
|
||||
let tls_acceptor = if self.config.tls_enabled {
|
||||
@@ -344,9 +151,15 @@ where
|
||||
|
||||
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
|
||||
.map_err(|e| WebDavInitError::Tls(format!("Failed to create certificate resolver: {}", e)))?;
|
||||
let server_config = crate::common::tls::server_config(resolver.clone(), self.config.ca_file.as_deref())
|
||||
let _reload_task = spawn_server_cert_reload_loop(
|
||||
"webdav",
|
||||
resolver.clone(),
|
||||
Self::tls_reload_options(),
|
||||
reload_shutdown_rx.clone(),
|
||||
);
|
||||
|
||||
let server_config = crate::common::tls::server_config(resolver, self.config.ca_file.as_deref())
|
||||
.map_err(|error| WebDavInitError::Tls(format!("Invalid client CA configuration: {error}")))?;
|
||||
reload_task = spawn_server_cert_reload_loop("webdav", resolver, Self::tls_reload_options(), shutdown.clone());
|
||||
|
||||
Some(TlsAcceptor::from(Arc::new(server_config)))
|
||||
} else {
|
||||
@@ -360,15 +173,7 @@ where
|
||||
let request_timeout = Duration::from_secs(self.config.request_timeout_secs);
|
||||
let connection_limiter = Arc::new(Semaphore::new(self.config.max_connections.min(Semaphore::MAX_PERMITS)));
|
||||
|
||||
let mut connections = JoinSet::new();
|
||||
let mut failure = None;
|
||||
'accept: loop {
|
||||
while let Some(result) = connections.try_join_next() {
|
||||
if let Err(error) = result {
|
||||
failure = Some(WebDavInitError::TaskJoin(error));
|
||||
break 'accept;
|
||||
}
|
||||
}
|
||||
loop {
|
||||
// Admission control: hold a permit before accepting, so at
|
||||
// saturation the kernel backlog absorbs the burst instead of the
|
||||
// process spawning an unbounded number of connection tasks. The
|
||||
@@ -399,6 +204,7 @@ where
|
||||
state = "shutdown_requested",
|
||||
"WebDAV shutdown requested"
|
||||
);
|
||||
let _ = reload_shutdown_tx.send(true);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -410,35 +216,25 @@ where
|
||||
match accept_result {
|
||||
Ok((stream, addr)) => {
|
||||
let storage = storage.clone();
|
||||
let cors = self.cors.clone();
|
||||
let tls_acceptor = tls_acceptor.clone();
|
||||
|
||||
let limits = ConnectionLimits {
|
||||
max_body_size: self.config.max_body_size,
|
||||
request_timeout,
|
||||
};
|
||||
let max_body_size = self.config.max_body_size;
|
||||
let source_ip: IpAddr = addr.ip();
|
||||
let span = info_span!(
|
||||
"webdav-connection",
|
||||
peer = %source_ip,
|
||||
transport = if tls_acceptor.is_some() { "tls" } else { "tcp" },
|
||||
);
|
||||
let mut shutdown = shutdown.clone();
|
||||
connections.spawn(
|
||||
tokio::spawn(
|
||||
async move {
|
||||
let _permit = permit;
|
||||
if let Some(acceptor) = tls_acceptor {
|
||||
// A handshake that never completes would otherwise
|
||||
// hold its connection permit forever.
|
||||
let handshake = tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.changed() => return,
|
||||
result = timeout(request_timeout, acceptor.accept(stream)) => result,
|
||||
};
|
||||
match handshake {
|
||||
match timeout(request_timeout, acceptor.accept(stream)).await {
|
||||
Ok(Ok(tls_stream)) => {
|
||||
let io = TokioIo::new(tls_stream);
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, limits, cors, shutdown).await {
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, max_body_size, request_timeout).await {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_CONNECTION_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
@@ -476,7 +272,7 @@ where
|
||||
}
|
||||
} else {
|
||||
let io = TokioIo::new(stream);
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, limits, cors, shutdown).await {
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, max_body_size, request_timeout).await {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_CONNECTION_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
@@ -513,29 +309,13 @@ where
|
||||
state = "shutdown_requested",
|
||||
"WebDAV shutdown requested"
|
||||
);
|
||||
let _ = reload_shutdown_tx.send(true);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drop(listener);
|
||||
shutdown_tx.send_replace(true);
|
||||
let drain = async {
|
||||
while let Some(result) = connections.join_next().await {
|
||||
if let Err(error) = result {
|
||||
failure.get_or_insert(WebDavInitError::TaskJoin(error));
|
||||
}
|
||||
}
|
||||
};
|
||||
if timeout(CONNECTION_DRAIN_TIMEOUT, drain).await.is_err() {
|
||||
connections.shutdown().await;
|
||||
failure.get_or_insert(WebDavInitError::ShutdownDeadline);
|
||||
}
|
||||
if let Some(task) = reload_task
|
||||
&& let Err(error) = task.await
|
||||
{
|
||||
failure.get_or_insert(WebDavInitError::TaskJoin(error));
|
||||
}
|
||||
let _ = reload_shutdown_tx.send(true);
|
||||
info!(
|
||||
event = EVENT_WEBDAV_SERVER_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
@@ -543,7 +323,7 @@ where
|
||||
state = "stopped",
|
||||
"WebDAV server stopped"
|
||||
);
|
||||
failure.map_or(Ok(()), Err)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handle a single connection with hyper-util TokioIo wrapper
|
||||
@@ -552,49 +332,25 @@ where
|
||||
storage: S,
|
||||
source_ip: IpAddr,
|
||||
secure_transport: bool,
|
||||
limits: ConnectionLimits,
|
||||
cors: Arc<WebDavCors>,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
max_body_size: u64,
|
||||
request_timeout: Duration,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let ConnectionLimits {
|
||||
max_body_size,
|
||||
request_timeout,
|
||||
} = limits;
|
||||
// These borrows live for the connection. Preflights complete before
|
||||
// invoking the closure that clones the backend for an actual request.
|
||||
let cors = cors.as_ref();
|
||||
let storage = &storage;
|
||||
let received_request = AtomicBool::new(false);
|
||||
let service = service_fn(|req: Request<hyper::body::Incoming>| {
|
||||
received_request.store(true, Ordering::Relaxed);
|
||||
cors.handle(req, move |req| {
|
||||
Self::handle_request(req, storage.clone(), source_ip, secure_transport, max_body_size, request_timeout)
|
||||
})
|
||||
let service = service_fn(move |req: Request<hyper::body::Incoming>| {
|
||||
let storage = storage.clone();
|
||||
async move { Self::handle_request(req, storage, source_ip, secure_transport, max_body_size, request_timeout).await }
|
||||
});
|
||||
|
||||
// A peer that opens a connection and dribbles (or never finishes)
|
||||
// request headers is disconnected once the configured request
|
||||
// timeout elapses. The timer is required for the deadline to apply.
|
||||
let connection = http1::Builder::new()
|
||||
http1::Builder::new()
|
||||
.timer(TokioTimer::new())
|
||||
.header_read_timeout(request_timeout)
|
||||
.serve_connection(io, service);
|
||||
tokio::pin!(connection);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.changed() => {
|
||||
// Hyper starts in a busy state even before parsing the first
|
||||
// request. An incomplete header has no application work to drain.
|
||||
if received_request.load(Ordering::Relaxed) {
|
||||
connection.as_mut().graceful_shutdown();
|
||||
connection.await?;
|
||||
}
|
||||
}
|
||||
result = &mut connection => result?,
|
||||
}
|
||||
.serve_connection(io, service)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -937,7 +693,6 @@ mod tests {
|
||||
use s3s::dto::*;
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
@@ -946,19 +701,8 @@ mod tests {
|
||||
|
||||
/// Storage double for the connection-level tests. Those requests are
|
||||
/// rejected before authentication succeeds, so storage is never reached.
|
||||
#[derive(Default)]
|
||||
struct StubStorage {
|
||||
clone_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Clone for StubStorage {
|
||||
fn clone(&self) -> Self {
|
||||
self.clone_count.fetch_add(1, Ordering::Relaxed);
|
||||
Self {
|
||||
clone_count: self.clone_count.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct StubStorage;
|
||||
|
||||
impl Debug for StubStorage {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
@@ -1129,362 +873,6 @@ mod tests {
|
||||
.expect("build get request")
|
||||
}
|
||||
|
||||
const BROWSER_ORIGIN: &str = "https://dav-console.example";
|
||||
|
||||
async fn cors_http_request(origins: &str, method: &str, headers: &str) -> String {
|
||||
cors_http_exchange(origins, method, headers).await.0
|
||||
}
|
||||
|
||||
async fn cors_http_exchange(origins: &str, method: &str, headers: &str) -> (String, usize) {
|
||||
let storage = StubStorage::default();
|
||||
let clone_count = storage.clone_count.clone();
|
||||
let (mut client, server) = tokio::io::duplex(8192);
|
||||
let (_shutdown_tx, shutdown) = tokio::sync::watch::channel(false);
|
||||
let connection = tokio::spawn(WebDavServer::<StubStorage>::handle_connection_impl(
|
||||
TokioIo::new(server),
|
||||
storage,
|
||||
TEST_IP,
|
||||
false,
|
||||
ConnectionLimits {
|
||||
max_body_size: 16,
|
||||
request_timeout: Duration::from_secs(30),
|
||||
},
|
||||
Arc::new(WebDavCors::new(origins)),
|
||||
shutdown,
|
||||
));
|
||||
let request = format!("{method} / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n{headers}\r\n");
|
||||
client.write_all(request.as_bytes()).await.expect("write browser request");
|
||||
let mut response = Vec::new();
|
||||
timeout(Duration::from_secs(5), client.read_to_end(&mut response))
|
||||
.await
|
||||
.expect("browser response timeout")
|
||||
.expect("read browser response");
|
||||
timeout(Duration::from_secs(5), connection)
|
||||
.await
|
||||
.expect("connection shutdown timeout")
|
||||
.expect("connection task")
|
||||
.expect("serve browser request");
|
||||
(
|
||||
String::from_utf8(response).expect("ASCII HTTP response"),
|
||||
clone_count.load(Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
fn browser_request(method: &str, path: &str) -> Request<Full<Bytes>> {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(path)
|
||||
.header(ORIGIN, BROWSER_ORIGIN)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("build browser request")
|
||||
}
|
||||
|
||||
fn assert_browser_headers(response: &Response<WebDavBody>) {
|
||||
assert_eq!(response.headers()[ACCESS_CONTROL_ALLOW_ORIGIN], BROWSER_ORIGIN);
|
||||
assert_eq!(response.headers()[ACCESS_CONTROL_ALLOW_CREDENTIALS], "true");
|
||||
assert!(response.headers().get_all(VARY).iter().any(|value| value == "Origin"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_preflight_bypasses_authentication() {
|
||||
for method in [
|
||||
"OPTIONS",
|
||||
"GET",
|
||||
"HEAD",
|
||||
"PUT",
|
||||
"DELETE",
|
||||
"PATCH",
|
||||
"PROPFIND",
|
||||
"PROPPATCH",
|
||||
"MKCOL",
|
||||
"COPY",
|
||||
"MOVE",
|
||||
"LOCK",
|
||||
"UNLOCK",
|
||||
] {
|
||||
let headers = format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: {method}\r\nAccess-Control-Request-Headers: authorization,depth,content-type,destination,if-match\r\n"
|
||||
);
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 204"), "{method}: {response}");
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
assert!(response.contains("access-control-allow-credentials: true\r\n"));
|
||||
let allowed_methods = response
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("access-control-allow-methods: "))
|
||||
.expect("preflight must advertise allowed methods");
|
||||
assert!(allowed_methods.split(',').map(str::trim).any(|allowed| allowed == method));
|
||||
assert!(response.contains("access-control-allow-headers: authorization,depth,content-type,destination,if-match\r\n"));
|
||||
assert!(response.contains("vary: Access-Control-Request-Method, Access-Control-Request-Headers\r\n"));
|
||||
assert!(!response.contains("www-authenticate:"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_actual_requests_and_plain_options_require_authentication() {
|
||||
for method in ["GET", "PUT", "DELETE", "PROPFIND", "MOVE", "OPTIONS"] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, method, &format!("Origin: {BROWSER_ORIGIN}\r\n")).await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"), "{method}: {response}");
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
assert!(response.contains("www-authenticate: Basic"));
|
||||
}
|
||||
for method in ["GET", "PROPFIND"] {
|
||||
for headers in ["", "Origin: https://other.example\r\n"] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, method, headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"), "{method}: {response}");
|
||||
assert!(!response.contains("access-control-allow-origin:"));
|
||||
assert!(response.contains("vary: Origin\r\n"));
|
||||
}
|
||||
}
|
||||
for headers in [
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAuthorization: Basic !!!!\r\n"),
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PUT\r\n"),
|
||||
] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "PUT", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"));
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
}
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", "Access-Control-Request-Method: PROPFIND\r\n").await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"));
|
||||
assert!(!response.contains("access-control-allow-origin:"));
|
||||
assert!(response.contains("vary: Origin\r\n"));
|
||||
assert!(response.contains("vary: Access-Control-Request-Method, Access-Control-Request-Headers\r\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_keeps_early_body_limit_errors_visible() {
|
||||
let response =
|
||||
cors_http_request(BROWSER_ORIGIN, "PUT", &format!("Origin: {BROWSER_ORIGIN}\r\nContent-Length: 17\r\n")).await;
|
||||
assert!(response.starts_with("HTTP/1.1 413"));
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_rejects_unconfigured_and_unmatched_origins() {
|
||||
for (configuration, origin) in [
|
||||
("", BROWSER_ORIGIN),
|
||||
(" ", BROWSER_ORIGIN),
|
||||
(BROWSER_ORIGIN, "https://dav-console.example.attacker.test"),
|
||||
(BROWSER_ORIGIN, "null"),
|
||||
(BROWSER_ORIGIN, "https://other.example"),
|
||||
("*, https://other.example", BROWSER_ORIGIN),
|
||||
] {
|
||||
let headers = format!("Origin: {origin}\r\nAccess-Control-Request-Method: PROPFIND\r\n");
|
||||
let response = cors_http_request(configuration, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 403"), "{configuration:?}: {response}");
|
||||
assert!(!response.contains("access-control-allow-origin:"));
|
||||
assert!(!response.contains("access-control-allow-credentials:"));
|
||||
assert!(response.contains("vary: Origin\r\n"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_rejects_malformed_preflights() {
|
||||
for headers in [
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: TRACE\r\n"),
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: propfind\r\n"),
|
||||
format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Method: GET\r\n"
|
||||
),
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nOrigin: https://other.example\r\nAccess-Control-Request-Method: PROPFIND\r\n"),
|
||||
format!("Origin: {BROWSER_ORIGIN},https://other.example\r\nAccess-Control-Request-Method: PROPFIND\r\n"),
|
||||
format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: invalid header\r\n"
|
||||
),
|
||||
format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: , \t,\r\n"
|
||||
),
|
||||
] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 403"), "{headers}: {response}");
|
||||
assert!(!response.contains("access-control-allow-methods:"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_normalizes_header_lists() {
|
||||
for requested in [
|
||||
"authorization, ,depth,",
|
||||
" , authorization,\tdepth , ",
|
||||
"authorization,\r\nAccess-Control-Request-Headers: , depth,",
|
||||
] {
|
||||
let headers = format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: {requested}\r\n"
|
||||
);
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 204"), "{response}");
|
||||
assert!(response.contains("access-control-allow-headers: authorization,depth\r\n"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webdav_cors_header_name_validation_matches_http_tokens() {
|
||||
for byte in u8::MIN..=u8::MAX {
|
||||
let bytes = [b'x', byte, b'y'];
|
||||
let actual = std::str::from_utf8(&bytes).is_ok_and(is_cors_header_name);
|
||||
assert_eq!(actual, http::HeaderName::from_bytes(&bytes).is_ok(), "byte {byte}");
|
||||
}
|
||||
assert!(!is_cors_header_name(""));
|
||||
assert!(is_cors_header_name("Authorization"));
|
||||
assert!(is_cors_header_name("!#$%&'*+-.^_`|~"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_only_clones_backend_for_actual_requests() {
|
||||
for (method, requested_method, expected_clones, expected_status) in [
|
||||
("OPTIONS", Some("PROPFIND"), 0, "204"),
|
||||
("OPTIONS", Some("TRACE"), 0, "403"),
|
||||
("OPTIONS", None, 1, "401"),
|
||||
("PUT", Some("PUT"), 1, "401"),
|
||||
] {
|
||||
let mut headers = format!("Origin: {BROWSER_ORIGIN}\r\n");
|
||||
if let Some(requested_method) = requested_method {
|
||||
headers.push_str(&format!("Access-Control-Request-Method: {requested_method}\r\n"));
|
||||
}
|
||||
let (response, clones) = cors_http_exchange(BROWSER_ORIGIN, method, &headers).await;
|
||||
assert!(response.starts_with(&format!("HTTP/1.1 {expected_status}")), "{response}");
|
||||
assert_eq!(clones, expected_clones, "{method} with {requested_method:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_wildcard_never_allows_credentials() {
|
||||
let headers = format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: Authorization\r\n"
|
||||
);
|
||||
let preflight = cors_http_request(" * ", "OPTIONS", &headers).await;
|
||||
assert!(preflight.starts_with("HTTP/1.1 204"));
|
||||
assert!(preflight.contains("access-control-allow-origin: *\r\n"));
|
||||
assert!(preflight.contains("access-control-allow-headers: Authorization\r\n"));
|
||||
assert!(!preflight.contains("access-control-allow-credentials:"));
|
||||
let actual = cors_http_request("*", "PROPFIND", &format!("Origin: {BROWSER_ORIGIN}\r\n")).await;
|
||||
assert!(actual.starts_with("HTTP/1.1 401"));
|
||||
assert!(actual.contains("access-control-allow-origin: *\r\n"));
|
||||
assert!(!actual.contains("access-control-allow-credentials:"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_allowlist_trims_entries_and_matches_exactly() {
|
||||
let configuration = format!(" https://other.example , {BROWSER_ORIGIN} ");
|
||||
let headers = format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\n");
|
||||
let response = cors_http_request(&configuration, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 204"));
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_wraps_dav_multistatus_etag_and_streamed_body() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
let handler = memfs_handler();
|
||||
let created = dispatch_dav(chunked_request(2, 4), handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await;
|
||||
assert_eq!(created.status(), StatusCode::CREATED);
|
||||
|
||||
let propfind = cors
|
||||
.handle(browser_request("PROPFIND", "/"), |req| async {
|
||||
Ok(dispatch_dav(req, handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("PROPFIND response");
|
||||
assert_eq!(propfind.status(), StatusCode::MULTI_STATUS);
|
||||
assert_browser_headers(&propfind);
|
||||
|
||||
let get = cors
|
||||
.handle(browser_request("GET", "/upload.bin"), |req| async {
|
||||
Ok(dispatch_dav(req, handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("GET response");
|
||||
assert_eq!(get.status(), StatusCode::OK);
|
||||
assert_browser_headers(&get);
|
||||
assert!(get.headers().contains_key("etag"));
|
||||
assert!(
|
||||
get.headers()[ACCESS_CONTROL_EXPOSE_HEADERS]
|
||||
.to_str()
|
||||
.expect("exposed headers")
|
||||
.contains("ETag")
|
||||
);
|
||||
assert!(get.body().size_hint().upper().is_none(), "CORS must preserve streaming");
|
||||
let data = get.into_body().collect().await.expect("stream GET body").to_bytes();
|
||||
assert_eq!(data.as_ref(), b"aaaaaaaa");
|
||||
|
||||
let mut put = browser_request("PUT", "/upload.bin");
|
||||
put.headers_mut()
|
||||
.insert("if-match", HeaderValue::from_static("\"stale-etag\""));
|
||||
let conditional = cors
|
||||
.handle(put, |req| async {
|
||||
Ok(dispatch_dav(req, handler, TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("conditional PUT response");
|
||||
assert_eq!(conditional.status(), StatusCode::PRECONDITION_FAILED);
|
||||
assert_browser_headers(&conditional);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_wraps_dispatch_body_limit_errors() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
let mut request = chunked_request(8, 8);
|
||||
request.headers_mut().insert(ORIGIN, HeaderValue::from_static(BROWSER_ORIGIN));
|
||||
let oversized = cors
|
||||
.handle(request, |req| async {
|
||||
Ok(dispatch_dav(req, memfs_handler(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("oversized response");
|
||||
assert_eq!(oversized.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
assert_browser_headers(&oversized);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn webdav_cors_wraps_dispatch_timeout_errors() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
let request = Request::builder()
|
||||
.method("PUT")
|
||||
.uri("/stalled.bin")
|
||||
.header(ORIGIN, BROWSER_ORIGIN)
|
||||
.body(StalledBody)
|
||||
.expect("stalled browser request");
|
||||
let response = cors
|
||||
.handle(request, |req| async {
|
||||
Ok(dispatch_dav(req, memfs_handler(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("timeout response");
|
||||
assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);
|
||||
assert_browser_headers(&response);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_preserves_existing_vary_on_success_and_errors() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
for status in [
|
||||
StatusCode::OK,
|
||||
StatusCode::FORBIDDEN,
|
||||
StatusCode::NOT_FOUND,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
] {
|
||||
let response = cors
|
||||
.handle(browser_request("GET", "/"), |_| async {
|
||||
let mut response = error_response(status, "test response");
|
||||
response
|
||||
.headers_mut()
|
||||
.append(VARY, HeaderValue::from_static("Accept-Encoding"));
|
||||
Ok(response)
|
||||
})
|
||||
.await
|
||||
.expect("wrapped response");
|
||||
assert_eq!(response.status(), status);
|
||||
assert_browser_headers(&response);
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.get_all(VARY)
|
||||
.iter()
|
||||
.any(|value| value == "Accept-Encoding")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_headers_drop_credentials_and_s3_auth_spoofing() {
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -1581,18 +969,13 @@ mod tests {
|
||||
async fn slow_request_headers_hit_request_timeout() {
|
||||
let (mut client, server) = tokio::io::duplex(1024);
|
||||
|
||||
let (_shutdown_tx, shutdown) = tokio::sync::watch::channel(false);
|
||||
let conn = tokio::spawn(WebDavServer::<StubStorage>::handle_connection_impl(
|
||||
TokioIo::new(server),
|
||||
StubStorage::default(),
|
||||
StubStorage,
|
||||
TEST_IP,
|
||||
false,
|
||||
ConnectionLimits {
|
||||
max_body_size: 1024,
|
||||
request_timeout: Duration::from_secs(30),
|
||||
},
|
||||
Arc::new(WebDavCors::new("")),
|
||||
shutdown,
|
||||
1024,
|
||||
Duration::from_secs(30),
|
||||
));
|
||||
|
||||
client
|
||||
@@ -1604,40 +987,6 @@ mod tests {
|
||||
assert!(outcome.is_ok(), "connection outlived the configured request timeout");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_closes_connection_with_incomplete_first_headers() {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
|
||||
let (mut client, server) = tokio::io::duplex(1024);
|
||||
let (shutdown_tx, shutdown) = watch::channel(false);
|
||||
let connection = WebDavServer::<StubStorage>::handle_connection_impl(
|
||||
TokioIo::new(server),
|
||||
StubStorage::default(),
|
||||
TEST_IP,
|
||||
false,
|
||||
ConnectionLimits {
|
||||
max_body_size: 1024,
|
||||
request_timeout: Duration::from_secs(30),
|
||||
},
|
||||
Arc::new(WebDavCors::new("")),
|
||||
shutdown,
|
||||
);
|
||||
tokio::pin!(connection);
|
||||
client
|
||||
.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\n")
|
||||
.await
|
||||
.expect("write partial headers");
|
||||
let pending = poll_fn(|cx| Poll::Ready(connection.as_mut().poll(cx).is_pending())).await;
|
||||
assert!(pending, "connection should be waiting for the rest of the first header");
|
||||
shutdown_tx.send_replace(true);
|
||||
|
||||
timeout(Duration::from_secs(1), connection.as_mut())
|
||||
.await
|
||||
.expect("shutdown should not wait for an incomplete header")
|
||||
.expect("no request reached the service");
|
||||
}
|
||||
|
||||
/// R05-CAN-097: the accept loop must not serve more connections at once
|
||||
/// than the configured cap.
|
||||
#[tokio::test]
|
||||
@@ -1651,7 +1000,7 @@ mod tests {
|
||||
request_timeout_secs: 30,
|
||||
max_connections: 1,
|
||||
};
|
||||
let server = WebDavServer::new(config, StubStorage::default()).await.expect("build server");
|
||||
let server = WebDavServer::new(config, StubStorage).await.expect("build server");
|
||||
|
||||
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.expect("bind listener");
|
||||
let addr = listener.local_addr().expect("listener addr");
|
||||
@@ -1686,16 +1035,8 @@ mod tests {
|
||||
.expect("read second");
|
||||
assert!(served > 0, "second connection returned no data");
|
||||
|
||||
shutdown_tx.send(()).expect("signal server");
|
||||
timeout(Duration::from_secs(5), serving)
|
||||
.await
|
||||
.expect("server did not retire the keep-alive connection")
|
||||
.expect("server task panicked")
|
||||
.expect("server did not shut down cleanly");
|
||||
let remaining = timeout(Duration::from_secs(5), second.read_to_end(&mut Vec::new()))
|
||||
.await
|
||||
.expect("connection remained open after server exit");
|
||||
remaining.expect("read remaining response and EOF");
|
||||
let _ = shutdown_tx.send(());
|
||||
let _ = timeout(Duration::from_secs(10), serving).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -21,7 +21,5 @@
|
||||
//! Swift implementation.
|
||||
|
||||
pub use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get as get_bucket_metadata, get_s3_tagging_config, set_bucket_metadata, update_s3_tagging_config,
|
||||
};
|
||||
pub use rustfs_ecstore::api::bucket::metadata_sys::{get as get_bucket_metadata, set_bucket_metadata};
|
||||
pub use rustfs_ecstore::api::runtime::object_store_handle;
|
||||
|
||||
@@ -103,110 +103,11 @@ async fn swift_metadata_writes_are_durable() {
|
||||
|
||||
posts_survive_disk_truth_reload(&env).await;
|
||||
tag_writers_preserve_each_others_state(&env).await;
|
||||
s3_tag_writes_preserve_swift_metadata(&env).await;
|
||||
unrelated_account_posts_preserve_the_tempurl_key().await;
|
||||
acl_only_posts_leave_container_metadata_alone(&env).await;
|
||||
versioning_writes_reject_missing_containers().await;
|
||||
}
|
||||
|
||||
async fn s3_tag_writes_preserve_swift_metadata(env: &TestECStoreEnv) {
|
||||
use ecstore_test_compat::{get_s3_tagging_config, update_s3_tagging_config};
|
||||
|
||||
let s3_only = "s3-only-swift-like-keys";
|
||||
env.make_bucket(s3_only, false).await;
|
||||
let mut ordinary_tags = get_bucket_metadata(s3_only)
|
||||
.await
|
||||
.expect("S3-only bucket metadata")
|
||||
.tagging_config
|
||||
.clone()
|
||||
.unwrap_or_default();
|
||||
ordinary_tags.tag_set =
|
||||
serde_json::from_value(json!([{"key": "swift-meta-owner", "value": "blue"}])).expect("ordinary tag fixture");
|
||||
update_s3_tagging_config(s3_only, Some(ordinary_tags.clone()), None)
|
||||
.await
|
||||
.expect("S3-only buckets accept Swift-like names");
|
||||
reload_bucket_metadata_from_disk(s3_only).await;
|
||||
assert_eq!(
|
||||
get_s3_tagging_config(s3_only).await.expect("S3-only tag read").0.tag_set,
|
||||
ordinary_tags.tag_set
|
||||
);
|
||||
|
||||
let project_id = "swifts3tagproj";
|
||||
let account = format!("AUTH_{project_id}");
|
||||
let container = "shared-s3-view";
|
||||
let bucket = ContainerMapper::default().swift_to_s3_bucket(container, project_id);
|
||||
let credentials = keystone_credentials(project_id);
|
||||
env.make_bucket(&bucket, false).await;
|
||||
let mut metadata = MetadataUpdate::default();
|
||||
for index in 0..51 {
|
||||
metadata = metadata.set(&format!("item-{index}"), "a,b");
|
||||
}
|
||||
update_container_metadata(&account, container, &credentials, metadata)
|
||||
.await
|
||||
.expect("51 Swift metadata values should persist");
|
||||
container::set_container_acl(&account, container, Some(".r:*"), None, &credentials)
|
||||
.await
|
||||
.expect("public ACL should persist");
|
||||
let mut s3_tags = get_s3_tagging_config(&bucket).await.expect("initial S3 view").0;
|
||||
assert!(s3_tags.tag_set.is_empty(), "Swift fields must be absent from the initial S3 view");
|
||||
s3_tags.tag_set = (0..50)
|
||||
.map(|index| serde_json::from_value(json!({"key": format!("team-{index}"), "value": "blue"})).expect("S3 tag fixture"))
|
||||
.collect();
|
||||
update_s3_tagging_config(&bucket, Some(s3_tags.clone()), None)
|
||||
.await
|
||||
.expect("S3 tags must coexist with Swift metadata");
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
let (read_back, _) = get_s3_tagging_config(&bucket).await.expect("S3 view should read");
|
||||
assert_eq!(read_back.tag_set, s3_tags.tag_set, "the S3 view must contain only the 50 S3 tags");
|
||||
for forbidden_key in ["swift-meta-item-0", "!rustfs-swift-metadata"] {
|
||||
let mut forbidden = s3_tags.clone();
|
||||
forbidden.tag_set.truncate(1);
|
||||
forbidden.tag_set[0].key = Some(forbidden_key.to_owned());
|
||||
forbidden.tag_set[0].value = Some("1".to_owned());
|
||||
assert!(
|
||||
update_s3_tagging_config(&bucket, Some(forbidden), None).await.is_err(),
|
||||
"S3 must reject overwriting Swift metadata or forging its marker"
|
||||
);
|
||||
assert_eq!(
|
||||
get_s3_tagging_config(&bucket)
|
||||
.await
|
||||
.expect("rejected writes preserve S3 tags")
|
||||
.0
|
||||
.tag_set,
|
||||
read_back.tag_set
|
||||
);
|
||||
}
|
||||
update_s3_tagging_config(&bucket, Some(read_back), None)
|
||||
.await
|
||||
.expect("S3 read-back must be writable");
|
||||
assert!(
|
||||
update_s3_tagging_config(&bucket, Some(s3_tags), Some(uuid::Uuid::new_v4()))
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale bucket incarnation must reject the mutation"
|
||||
);
|
||||
update_s3_tagging_config(&bucket, None, None)
|
||||
.await
|
||||
.expect("S3 delete should preserve Swift metadata");
|
||||
reload_bucket_metadata_from_disk(&bucket).await;
|
||||
assert!(
|
||||
get_s3_tagging_config(&bucket)
|
||||
.await
|
||||
.expect("Swift-only config has an empty S3 view")
|
||||
.0
|
||||
.tag_set
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(persisted_container_metadata(&bucket).await.len(), 51);
|
||||
assert!(
|
||||
container::get_container_acl(&account, container, &credentials)
|
||||
.await
|
||||
.expect("Swift ACL should load")
|
||||
.check_read_access(None, None, None),
|
||||
"S3 tag mutations must preserve the public Swift ACL"
|
||||
);
|
||||
}
|
||||
|
||||
async fn posts_survive_disk_truth_reload(env: &TestECStoreEnv) {
|
||||
// --- Container metadata POST (X-Container-Meta-*) ---
|
||||
let project_id = "swiftpersistproj";
|
||||
@@ -333,18 +234,13 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
|
||||
env.make_bucket(&ContainerMapper::default().swift_to_s3_bucket(archive, project_id), false)
|
||||
.await;
|
||||
|
||||
update_container_metadata(
|
||||
&swift_account,
|
||||
container,
|
||||
&credentials,
|
||||
MetadataUpdate::default().set("color", "blue,green"),
|
||||
)
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default().set("color", "blue"))
|
||||
.await
|
||||
.expect("container metadata POST should succeed");
|
||||
container::enable_versioning(&swift_account, container, archive, &credentials)
|
||||
.await
|
||||
.expect("enable versioning should succeed");
|
||||
container::set_container_acl(&swift_account, container, Some(".r:*,AUTH_other"), Some("AUTH_other"), &credentials)
|
||||
container::set_container_acl(&swift_account, container, Some(".r:*"), Some("AUTH_other"), &credentials)
|
||||
.await
|
||||
.expect("set container ACL should succeed");
|
||||
|
||||
@@ -352,7 +248,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
|
||||
|
||||
// All three writers' state coexists after a disk-truth reload.
|
||||
let meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(meta.get("color").map(String::as_str), Some("blue,green"));
|
||||
assert_eq!(meta.get("color").map(String::as_str), Some("blue"));
|
||||
assert_eq!(
|
||||
container::get_versions_location(&swift_account, container, &credentials)
|
||||
.await
|
||||
@@ -363,7 +259,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
.await
|
||||
.expect("container ACL should load");
|
||||
assert!(acl.check_read_access(None, None, None), "public read ACL must survive the reload");
|
||||
assert!(!acl.read.is_empty(), "read ACL must survive the reload");
|
||||
assert!(!acl.write.is_empty(), "write ACL must survive the reload");
|
||||
|
||||
// Disabling versioning drops only the versioning tag.
|
||||
@@ -383,7 +279,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
|
||||
let meta = persisted_container_metadata(&bucket).await;
|
||||
assert_eq!(
|
||||
meta.get("color").map(String::as_str),
|
||||
Some("blue,green"),
|
||||
Some("blue"),
|
||||
"disable_versioning must not disturb container metadata"
|
||||
);
|
||||
let acl = container::get_container_acl(&swift_account, container, &credentials)
|
||||
|
||||
@@ -1728,149 +1728,4 @@ mod tests {
|
||||
let xxh3_multipart = ChecksumType(ChecksumType::XXHASH3.0 | ChecksumType::MULTIPART.0);
|
||||
assert!(!xxh3_multipart.is_s3s_typed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_matches_covers_positive_and_negative_branches() {
|
||||
let payload = b"durability check payload";
|
||||
let valid = Checksum::new_from_data(ChecksumType::CRC32C, payload).expect("valid crc32c");
|
||||
|
||||
// Positive match with no multipart count.
|
||||
assert!(valid.matches(payload, 0).is_ok());
|
||||
|
||||
// Empty encoded checksum is a no-op success.
|
||||
let mut empty_encoded = valid.clone();
|
||||
empty_encoded.encoded.clear();
|
||||
assert!(empty_encoded.matches(b"anything", 0).is_ok());
|
||||
|
||||
// Non-empty encoded checksum with unset/no-hasher type fails closed.
|
||||
let no_hasher = Checksum {
|
||||
checksum_type: ChecksumType::NONE,
|
||||
encoded: valid.encoded.clone(),
|
||||
raw: valid.raw.clone(),
|
||||
want_parts: 0,
|
||||
};
|
||||
let err = no_hasher.matches(payload, 0).expect_err("NONE hasher must fail");
|
||||
let err_msg = err.to_string();
|
||||
assert!(err_msg.contains("no hasher available"), "unexpected error: {err_msg}");
|
||||
|
||||
// Multipart want_parts mismatch fails with formatted "<encoded>-<parts>" want/got.
|
||||
let mp = Checksum::new_with_type(ChecksumType::CRC32C, &format!("{}-3", valid.encoded)).expect("multipart checksum");
|
||||
assert_eq!(mp.want_parts, 3);
|
||||
let err = mp.matches(payload, 2).expect_err("parts count mismatch must fail");
|
||||
let err_msg = err.to_string();
|
||||
assert!(err_msg.contains(&format!("{}-3", valid.encoded)), "missing want parts: {err_msg}");
|
||||
assert!(err_msg.contains(&format!("{}-2", valid.encoded)), "missing got parts: {err_msg}");
|
||||
assert!(mp.matches(payload, 3).is_ok(), "matching parts count should succeed");
|
||||
|
||||
// Payload digest mismatch fails closed.
|
||||
let err = valid
|
||||
.matches(b"corrupted payload", 0)
|
||||
.expect_err("corrupted payload must fail");
|
||||
assert_eq!(err.want, valid.encoded);
|
||||
assert_ne!(err.got, valid.encoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_constructors_add_part_and_header_negative_branches() {
|
||||
use super::get_content_checksum;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
|
||||
// new_with_type negative branches
|
||||
assert!(Checksum::new_with_type(ChecksumType::NONE, "AAAA").is_none());
|
||||
assert!(Checksum::new_with_type(ChecksumType::CRC32, "AAAA-1-2").is_none());
|
||||
assert!(Checksum::new_with_type(ChecksumType::CRC32, "AAAA-notanint").is_none());
|
||||
assert!(Checksum::new_with_type(ChecksumType::CRC32, "!!!not_base64!!!").is_none());
|
||||
let short_b64 = base64_simd::STANDARD.encode_to_string([1u8, 2u8]);
|
||||
assert!(Checksum::new_with_type(ChecksumType::CRC32, &short_b64).is_none());
|
||||
|
||||
// as_map returns None when raw length does not match raw_byte_len()
|
||||
let mut invalid_raw = Checksum::new_from_data(ChecksumType::CRC32, b"hello").expect("crc32");
|
||||
assert!(invalid_raw.as_map().is_some());
|
||||
invalid_raw.raw.pop();
|
||||
assert!(invalid_raw.as_map().is_none());
|
||||
|
||||
// obj_type and from_header coverage
|
||||
assert_eq!(ChecksumType::CRC64_NVME.obj_type(), "FULL_OBJECT");
|
||||
assert_eq!(ChecksumType(ChecksumType::SHA256.0 | ChecksumType::MULTIPART.0).obj_type(), "COMPOSITE");
|
||||
assert_eq!(ChecksumType::NONE.obj_type(), "");
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
assert_eq!(ChecksumType::from_header(&headers), ChecksumType::NONE);
|
||||
headers.insert("x-amz-checksum-algorithm", HeaderValue::from_static("CRC32C"));
|
||||
headers.insert("x-amz-checksum-type", HeaderValue::from_static("FULL_OBJECT"));
|
||||
let parsed = ChecksumType::from_header(&headers);
|
||||
assert!(parsed.is(ChecksumType::CRC32C) && parsed.full_object_requested());
|
||||
|
||||
// add_part positive and negative branches
|
||||
let part_a = b"hello ";
|
||||
let part_b = b"world!";
|
||||
let part_b_len = part_b.len() as i64;
|
||||
for t in [ChecksumType::CRC32, ChecksumType::CRC32C, ChecksumType::CRC64_NVME] {
|
||||
let mut acc = Checksum::new_from_data(t, part_a).expect("part a");
|
||||
let b_sum = Checksum::new_from_data(t, part_b).expect("part b");
|
||||
let full = Checksum::new_from_data(t, b"hello world!").expect("full");
|
||||
// size == 0 is a no-op
|
||||
acc.add_part(&b_sum, 0).expect("zero size no-op");
|
||||
assert_eq!(acc.encoded, Checksum::new_from_data(t, part_a).unwrap().encoded);
|
||||
// size > 0 combines CRCs into full-object digest
|
||||
acc.add_part(&b_sum, part_b_len).expect("combine");
|
||||
assert_eq!(acc.encoded, full.encoded, "combined CRC mismatch for {t:?}");
|
||||
}
|
||||
|
||||
// add_part rejects unmergeable algorithm, mismatched types, and invalid raw lengths
|
||||
let mut sha_a = Checksum::new_from_data(ChecksumType::SHA256, part_a).expect("sha256");
|
||||
let sha_b = Checksum::new_from_data(ChecksumType::SHA256, part_b).expect("sha256");
|
||||
let err = sha_a
|
||||
.add_part(&sha_b, part_b_len)
|
||||
.expect_err("unmergeable algorithm must fail");
|
||||
assert!(err.contains("checksum type cannot be merged"), "unexpected error: {err}");
|
||||
|
||||
let mut crc32_a = Checksum::new_from_data(ChecksumType::CRC32, part_a).expect("crc32");
|
||||
let crc32c_b = Checksum::new_from_data(ChecksumType::CRC32C, part_b).expect("crc32c");
|
||||
let err = crc32_a
|
||||
.add_part(&crc32c_b, part_b_len)
|
||||
.expect_err("mismatched types must fail");
|
||||
assert!(err.contains("checksum type does not match"), "unexpected error: {err}");
|
||||
|
||||
let mut bad_base = crc32_a.clone();
|
||||
bad_base.raw.truncate(1);
|
||||
let err = bad_base
|
||||
.add_part(&crc32_a, part_b_len)
|
||||
.expect_err("invalid base raw len must fail");
|
||||
assert!(err.contains("invalid base checksum"), "unexpected error: {err}");
|
||||
|
||||
let mut bad_part = crc32_a.clone();
|
||||
bad_part.raw.truncate(1);
|
||||
let err = crc32_a
|
||||
.add_part(&bad_part, part_b_len)
|
||||
.expect_err("invalid part raw len must fail");
|
||||
assert!(err.contains("invalid part checksum"), "unexpected error: {err}");
|
||||
|
||||
// get_content_checksum negative branches
|
||||
let mut dup_trailer = HeaderMap::new();
|
||||
dup_trailer.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-crc32, x-amz-checksum-crc32c"));
|
||||
let err = get_content_checksum(&dup_trailer).expect_err("duplicate trailer checksums must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("Invalid checksum"), "unexpected error: {err}");
|
||||
|
||||
let mut unmergeable_full_trailer = HeaderMap::new();
|
||||
unmergeable_full_trailer.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-sha256"));
|
||||
unmergeable_full_trailer.insert("x-amz-checksum-type", HeaderValue::from_static("FULL_OBJECT"));
|
||||
let err = get_content_checksum(&unmergeable_full_trailer).expect_err("FULL_OBJECT SHA256 trailer must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("Invalid checksum"), "unexpected error: {err}");
|
||||
|
||||
let mut invalid_type_header = HeaderMap::new();
|
||||
invalid_type_header.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-crc32"));
|
||||
invalid_type_header.insert("x-amz-checksum-type", HeaderValue::from_static("BOGUS_TYPE"));
|
||||
let err = get_content_checksum(&invalid_type_header).expect_err("BOGUS_TYPE checksum-type must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("Invalid checksum"), "unexpected error: {err}");
|
||||
|
||||
let mut malformed_direct = HeaderMap::new();
|
||||
malformed_direct.insert("x-amz-checksum-crc32", HeaderValue::from_static("not_valid_base64!"));
|
||||
let err = get_content_checksum(&malformed_direct).expect_err("malformed base64 header must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("invalid checksum value"), "unexpected error: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1269,104 +1269,4 @@ mod trailer_tests {
|
||||
let err = read_all(&mut outer).await.expect_err("mismatch through a wrapper must fail");
|
||||
assert_checksum_mismatch(&err);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wrapping_hash_reader_rejects_already_read_conflicting_checksum_and_size() {
|
||||
let data = b"wrapping validation payload";
|
||||
let size = data.len() as i64;
|
||||
|
||||
// 1. Wrapping a HashReader that has already been partially read must fail.
|
||||
let mut inner = HashReader::from_stream(Cursor::new(data.to_vec()), size, size, None, None, false).expect("inner");
|
||||
let mut one = [0u8; 1];
|
||||
inner.read_exact(&mut one).await.expect("read one byte");
|
||||
assert_eq!(inner.bytes_read(), 1);
|
||||
let err = HashReader::new(Box::new(inner), size, size, None, None, false)
|
||||
.err()
|
||||
.expect("wrapping after read must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("already read"));
|
||||
|
||||
// 2. Wrapping a HashReader with a conflicting MD5 checksum must fail.
|
||||
let inner = HashReader::from_stream(
|
||||
Cursor::new(data.to_vec()),
|
||||
size,
|
||||
size,
|
||||
Some("11111111111111111111111111111111".to_string()),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("inner with md5");
|
||||
let err = HashReader::new(
|
||||
Box::new(inner),
|
||||
size,
|
||||
size,
|
||||
Some("22222222222222222222222222222222".to_string()),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.err()
|
||||
.expect("conflicting md5 checksum must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("checksum mismatch"));
|
||||
|
||||
// 3. Wrapping a HashReader with a conflicting positive size must fail.
|
||||
let inner = HashReader::from_stream(Cursor::new(data.to_vec()), size, size, None, None, false).expect("inner");
|
||||
let err = HashReader::new(Box::new(inner), size + 10, size, None, None, false)
|
||||
.err()
|
||||
.expect("conflicting positive size must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("size mismatch"));
|
||||
|
||||
// 4. Wrapping an unbounded (-1) HashReader propagates newly discovered positive size and actual_size.
|
||||
let inner = HashReader::from_stream(Cursor::new(data.to_vec()), -1, -1, None, None, false).expect("unbounded inner");
|
||||
let outer = HashReader::new(Box::new(inner), size, size, Some("abcd".to_string()), None, false).expect("outer");
|
||||
assert_eq!(outer.size(), size);
|
||||
assert_eq!(outer.actual_size(), size);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hash_reader_checksum_helpers_reject_invalid_inputs() {
|
||||
let data = b"checksum helper validation";
|
||||
let size = data.len() as i64;
|
||||
let mut reader = HashReader::from_stream(Cursor::new(data.to_vec()), size, size, None, None, false).expect("reader");
|
||||
|
||||
// add_calculated_checksum rejects both NONE and INVALID checksum types.
|
||||
let err = reader
|
||||
.add_calculated_checksum(ChecksumType::NONE)
|
||||
.expect_err("NONE checksum type must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
let err = reader
|
||||
.add_calculated_checksum(ChecksumType::INVALID)
|
||||
.expect_err("INVALID checksum type must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
|
||||
// add_non_trailing_checksum handles None as a no-op and rejects a Checksum with no hasher.
|
||||
reader.add_non_trailing_checksum(None, false).expect("None is no-op");
|
||||
let invalid_checksum = Checksum {
|
||||
checksum_type: ChecksumType::INVALID,
|
||||
encoded: "AAAA".to_string(),
|
||||
raw: vec![1, 2, 3, 4],
|
||||
want_parts: 0,
|
||||
};
|
||||
let err = reader
|
||||
.add_non_trailing_checksum(Some(invalid_checksum), false)
|
||||
.expect_err("invalid checksum_type must fail");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
|
||||
// add_checksum_no_trailer attaches a direct header checksum without trailer reader.
|
||||
let mut reader =
|
||||
HashReader::from_stream(Cursor::new(data.to_vec()), size, size, None, None, false).expect("fresh reader");
|
||||
assert!(reader.checksum().is_none(), "HashReader::checksum() is None when no content_hash is set");
|
||||
let crc32_val = encoded(ChecksumType::CRC32C, data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-checksum-crc32c", HeaderValue::from_str(&crc32_val).expect("header"));
|
||||
reader
|
||||
.add_checksum_no_trailer(&headers, false)
|
||||
.expect("direct header checksum");
|
||||
assert!(
|
||||
reader.checksum().is_some(),
|
||||
"HashReader::checksum() returns Some after add_checksum_no_trailer"
|
||||
);
|
||||
assert_eq!(read_all(&mut reader).await.expect("verified"), data);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1823,12 +1823,7 @@ impl DataUsageCache {
|
||||
return;
|
||||
};
|
||||
|
||||
// Wide bucket roots previously only flattened above a hardcoded 250k
|
||||
// fanout, so caches with 10k–250k direct children (common with many
|
||||
// top-level prefixes) never shrank and retained one entry per prefix
|
||||
// after #8971's non-root compaction. Honor `limit` here so completed
|
||||
// sweeps and emergency compaction actually bound root fanout (#8961).
|
||||
if top_e.children.len() > limit {
|
||||
if top_e.children.len() > 250_000 {
|
||||
self.reduce_children_of(&hash_path(&self.info.name), limit, true);
|
||||
}
|
||||
if self.cache.len() <= limit {
|
||||
|
||||
@@ -3223,200 +3223,6 @@ fn test_reduce_children_of_usize_underflow_saturates() {
|
||||
assert!(cache.find("bucket/large/b").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_force_compact_flattens_wide_root_above_limit() {
|
||||
// Regression for #8961: a bucket root with fanout between the ordinary
|
||||
// compact limit and the old hardcoded 250_000 ceiling must shrink.
|
||||
// Before the fix, force_compact kept one entry per prefix forever.
|
||||
const PREFIXES: usize = 64;
|
||||
const LIMIT: usize = 16;
|
||||
const {
|
||||
assert!(PREFIXES > LIMIT);
|
||||
}
|
||||
|
||||
let root = hash_path("bucket");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
for index in 0..PREFIXES {
|
||||
let child = hash_path(&format!("bucket/prefix-{index:04}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
assert_eq!(cache.find("bucket").map(|entry| entry.children.len()), Some(PREFIXES));
|
||||
assert_eq!(cache.cache.len(), PREFIXES + 1);
|
||||
|
||||
cache.force_compact(LIMIT);
|
||||
|
||||
let root_entry = cache.find("bucket").expect("root retained");
|
||||
assert!(root_entry.compacted, "wide root must flatten under force_compact");
|
||||
assert!(root_entry.children.is_empty(), "flattened root must drop per-prefix children");
|
||||
assert_eq!((root_entry.objects, root_entry.size), (PREFIXES, PREFIXES));
|
||||
assert_eq!(cache.cache.len(), 1, "prefix leaves must be released after root flatten");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_force_compact_keeps_narrow_root_children() {
|
||||
const PREFIXES: usize = 4;
|
||||
const LIMIT: usize = 16;
|
||||
const {
|
||||
assert!(PREFIXES < LIMIT);
|
||||
}
|
||||
|
||||
let root = hash_path("bucket");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace_hashed(
|
||||
&root,
|
||||
&None,
|
||||
&DataUsageEntry {
|
||||
objects: PREFIXES,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
for index in 0..PREFIXES {
|
||||
let child = hash_path(&format!("bucket/prefix-{index:04}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
objects: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cache.force_compact(LIMIT);
|
||||
|
||||
let root_entry = cache.find("bucket").expect("root retained");
|
||||
assert!(!root_entry.compacted);
|
||||
assert_eq!(root_entry.children.len(), PREFIXES);
|
||||
assert_eq!(cache.cache.len(), PREFIXES + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_force_compact_interrupted_ceiling_retains_wide_root_children() {
|
||||
// Interrupted sweeps pass the emergency FORCE ceiling into force_compact so
|
||||
// a resume frontier under a 10k–250k-wide root stays addressable.
|
||||
const PREFIXES: usize = 64;
|
||||
const ORDINARY_LIMIT: usize = 16;
|
||||
const INTERRUPTED_CEILING: usize = 250_000;
|
||||
const {
|
||||
assert!(PREFIXES > ORDINARY_LIMIT);
|
||||
assert!(PREFIXES < INTERRUPTED_CEILING);
|
||||
}
|
||||
|
||||
let root = hash_path("bucket");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
for index in 0..PREFIXES {
|
||||
let child = hash_path(&format!("bucket/prefix-{index:04}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cache.force_compact(INTERRUPTED_CEILING);
|
||||
|
||||
let root_entry = cache.find("bucket").expect("root retained");
|
||||
assert!(!root_entry.compacted, "interrupted ceiling must not flatten a 64-wide root");
|
||||
assert_eq!(root_entry.children.len(), PREFIXES);
|
||||
assert_eq!(cache.cache.len(), PREFIXES + 1);
|
||||
// Sanity: the ordinary completion limit would have flattened this layout.
|
||||
let mut completed = cache.clone();
|
||||
completed.force_compact(ORDINARY_LIMIT);
|
||||
assert!(
|
||||
completed.find("bucket").is_some_and(|entry| entry.compacted),
|
||||
"ordinary limit still flattens the same wide root after completion"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_force_compact_interrupted_ceiling_seals_reloaded_frontier() {
|
||||
// The interrupted call runs before seal_scan_frontier. Flattening at the
|
||||
// ordinary limit deletes the child that frontier names, so a saved cache
|
||||
// reloads with no resume position and the next cycle replays finished prefixes.
|
||||
const PREFIXES: usize = 64;
|
||||
const ORDINARY_LIMIT: usize = 16;
|
||||
const INTERRUPTED_CEILING: usize = 250_000;
|
||||
const {
|
||||
assert!(PREFIXES > ORDINARY_LIMIT);
|
||||
assert!(PREFIXES < INTERRUPTED_CEILING);
|
||||
}
|
||||
let frontier = format!("bucket/prefix-{:04}", PREFIXES / 2);
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_bucket_checkpoint("bucket", 8, 1, DataUsageCacheSource::new(0, 0), TEST_PLAN_DIGEST, valid_scan_identity());
|
||||
let root = hash_path("bucket");
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
for index in 0..PREFIXES {
|
||||
let child = hash_path(&format!("bucket/prefix-{index:04}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let mut interrupted = cache.clone();
|
||||
interrupted.force_compact(INTERRUPTED_CEILING);
|
||||
assert!(interrupted.find(&frontier).is_some(), "interrupted ceiling must keep the frontier child");
|
||||
interrupted
|
||||
.seal_scan_frontier(Some(&frontier))
|
||||
.expect("seal frontier after interrupted compaction");
|
||||
let encoded = rmp_serde::to_vec(&interrupted).expect("partial cache should encode");
|
||||
let reloaded: DataUsageCache = rmp_serde::from_slice(&encoded).expect("partial cache should reload");
|
||||
assert_eq!(reloaded.validated_scan_frontier(), Some(frontier.as_str()));
|
||||
assert!(reloaded.find(&frontier).is_some(), "reloaded frontier child must stay addressable");
|
||||
|
||||
cache.force_compact(ORDINARY_LIMIT);
|
||||
assert!(cache.find(&frontier).is_none(), "ordinary limit removes the frontier child");
|
||||
cache
|
||||
.seal_scan_frontier(Some(&frontier))
|
||||
.expect("seal after a completed flatten");
|
||||
assert!(
|
||||
cache.validated_scan_frontier().is_none(),
|
||||
"a flattened root cannot seal the removed frontier child"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_save_op_times_out_and_returns_error_after_retries() {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -109,10 +109,10 @@ pub use scanner_io::{
|
||||
ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
|
||||
acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
|
||||
probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
|
||||
record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
|
||||
scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
|
||||
record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
|
||||
record_status_class_mutation, replay_durable_dirty_usage_producer_record, scanner_activity_epoch,
|
||||
scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
|
||||
@@ -523,18 +523,6 @@ fn data_usage_entry_has_progress(entry: &DataUsageEntry) -> bool {
|
||||
data_usage_root_has_progress(entry)
|
||||
}
|
||||
|
||||
/// Root-fanout limit for `DataUsageCache::force_compact` at the end of a sweep.
|
||||
///
|
||||
/// An interrupted sweep still has to seal its resume frontier, so it keeps the
|
||||
/// emergency ceiling. A finished walk applies the ordinary child limit.
|
||||
fn force_compact_limit_for_sweep(interrupted: bool) -> usize {
|
||||
if interrupted {
|
||||
DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS
|
||||
} else {
|
||||
DATA_SCANNER_COMPACT_AT_CHILDREN
|
||||
}
|
||||
}
|
||||
|
||||
fn set_scan_checkpoint(cache: &mut DataUsageCache, reason: DataUsageScanCheckpointReason) {
|
||||
let resume_after = clone_cache_value(&cache.info.scan_resume_after).or_else(|| {
|
||||
cache
|
||||
@@ -1427,7 +1415,7 @@ impl FolderScanner {
|
||||
mode: FolderScanMode,
|
||||
) -> Result<FolderScanOutcome, ScannerError> {
|
||||
if self.checkpoint_tx.is_none() || self.old_cache.info.scan_progress.is_none() {
|
||||
return Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode, false)).await;
|
||||
return Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode)).await;
|
||||
}
|
||||
let depth = self.checkpoint_depth;
|
||||
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
@@ -1441,7 +1429,7 @@ impl FolderScanner {
|
||||
std::mem::swap(parent_hash, &mut self.checkpoint_ancestors[depth].hash);
|
||||
std::mem::swap(parent_entry, &mut self.checkpoint_ancestors[depth].entry);
|
||||
self.checkpoint_depth += 1;
|
||||
let result = Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode, false)).await;
|
||||
let result = Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode)).await;
|
||||
self.checkpoint_depth -= 1;
|
||||
// Restore the parent before cancellation/error handling stitches partial work.
|
||||
std::mem::swap(parent_hash, &mut self.checkpoint_ancestors[depth].hash);
|
||||
@@ -1862,7 +1850,7 @@ impl FolderScanner {
|
||||
folder: CachedFolder,
|
||||
into: &mut DataUsageEntry,
|
||||
) -> Result<(), ScannerError> {
|
||||
self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace, false)
|
||||
self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
@@ -1875,7 +1863,6 @@ impl FolderScanner {
|
||||
folder: CachedFolder,
|
||||
into: &mut DataUsageEntry,
|
||||
mode: FolderScanMode,
|
||||
isolate_compacted_namespace_errors: bool,
|
||||
) -> Result<FolderScanOutcome, ScannerError> {
|
||||
let done_folder = Metrics::time(Metric::ScanFolder);
|
||||
let trace_started_at = trace_start_instant();
|
||||
@@ -2546,23 +2533,13 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
// Checkpointed sweeps need child entries to validate their resume
|
||||
// frontier (#8292). Outside that, also compact a wide bucket root:
|
||||
// excluding it left #8961's ~170k top-level prefixes unbounded until
|
||||
// the emergency 250k ceiling. Keep FORCE as the absolute backstop.
|
||||
let is_scan_root = folder.name == self.old_cache.info.name;
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
let total_folders = queued_folders.len();
|
||||
let force_compact = total_folders >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
// Reused root children still have links in the new cache.
|
||||
let should_compact = force_compact
|
||||
|| (!forward_sweep
|
||||
&& total_folders >= DATA_SCANNER_COMPACT_AT_FOLDERS
|
||||
&& (!is_scan_root || into.children.is_empty()));
|
||||
// Preserve the ordinary namespace walk's child isolation when the
|
||||
// root starts sharing one accumulator across its descendants.
|
||||
let isolate_compacted_children =
|
||||
isolate_compacted_namespace_errors || (is_scan_root && !into.compacted && should_compact && !force_compact);
|
||||
// frontier. Keep the emergency compaction limit for large trees.
|
||||
let should_compact = (self.new_cache.info.name != folder.name
|
||||
&& self.old_cache.info.scan_progress.is_none()
|
||||
&& queued_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|
||||
|| queued_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
|
||||
let total_folders = queued_folders.len();
|
||||
self.alert_excessive_folders(&folder.name, total_folders);
|
||||
|
||||
if !into.compacted && should_compact {
|
||||
@@ -2585,6 +2562,7 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
let is_scan_root = folder.name == self.old_cache.info.name;
|
||||
let scan_checkpoint = self.old_cache.info.scan_checkpoint.as_ref();
|
||||
let checkpoint_resume_after = scan_checkpoint.and_then(|checkpoint| {
|
||||
if is_scan_root {
|
||||
@@ -2608,6 +2586,7 @@ impl FolderScanner {
|
||||
.is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
|
||||
let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
|
||||
let has_queued_folders = !queued_folders.is_empty();
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
let resume_order = if forward_sweep {
|
||||
order_queued_folders_for_resume(&mut queued_folders, None)
|
||||
} else {
|
||||
@@ -2710,52 +2689,19 @@ impl FolderScanner {
|
||||
|
||||
(self.update_current_path)(&folder_item.name).await;
|
||||
|
||||
// Keep a compacted namespace child's accounting local until it
|
||||
// succeeds. Ordinary child errors can then discard it without
|
||||
// rolling back totals already committed by healthy siblings.
|
||||
let isolate_compacted_child =
|
||||
into.compacted && isolate_compacted_children && child_mode == FolderScanMode::Namespace;
|
||||
let outcome = if into.compacted && !isolate_compacted_child {
|
||||
// Keep the shared-accumulator path free of child summaries.
|
||||
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), into, child_mode, false)).await?
|
||||
let outcome = if into.compacted {
|
||||
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), into, child_mode)).await?
|
||||
} else {
|
||||
let mut dst = DataUsageEntry {
|
||||
compacted: isolate_compacted_child,
|
||||
..Default::default()
|
||||
};
|
||||
// Provisional child usage must not escape through progress
|
||||
// updates before this accumulator is committed to its parent.
|
||||
let suspended_updates = if isolate_compacted_child {
|
||||
self.usage_updates.take()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = if isolate_compacted_child {
|
||||
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), &mut dst, child_mode, true)).await
|
||||
} else {
|
||||
self.scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst, child_mode)
|
||||
.await
|
||||
};
|
||||
if isolate_compacted_child {
|
||||
self.usage_updates = suspended_updates;
|
||||
}
|
||||
match result {
|
||||
Ok(outcome) => {
|
||||
if isolate_compacted_child {
|
||||
into.merge(&dst);
|
||||
}
|
||||
outcome
|
||||
}
|
||||
let mut dst = DataUsageEntry::default();
|
||||
match self
|
||||
.scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst, child_mode)
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => outcome,
|
||||
Err(e) => {
|
||||
if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
|
||||
if isolate_compacted_child {
|
||||
// Cancellation and resource deferral keep completed
|
||||
// work even though the child has not finished.
|
||||
into.merge(&dst);
|
||||
} else {
|
||||
self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
|
||||
.await;
|
||||
}
|
||||
self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
|
||||
.await;
|
||||
return Err(e);
|
||||
}
|
||||
if child_mode != FolderScanMode::Namespace {
|
||||
@@ -2766,15 +2712,13 @@ impl FolderScanner {
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
state = "child_scan_failed",
|
||||
folder = %folder.name,
|
||||
child_folder = %folder_item.name,
|
||||
state = "child_scan_failed",
|
||||
error = %e,
|
||||
"Scanner child folder scan failed"
|
||||
);
|
||||
// A skipped namespace is a coverage gap even when no
|
||||
// checkpointed forward sweep is active.
|
||||
self.coverage_gap = true;
|
||||
self.coverage_gap |= forward_sweep;
|
||||
self.finish_raw_enumeration_parent(&folder_item.name);
|
||||
continue;
|
||||
}
|
||||
@@ -3260,24 +3204,13 @@ impl FolderScanner {
|
||||
self.compact_completed_folder(&folder, &this_hash, into);
|
||||
|
||||
// Compact if too many children...
|
||||
// Allow compact_self on the bucket root when this is not a checkpointed
|
||||
// forward sweep so wide roots can flatten instead of retaining one
|
||||
// entry per top-level prefix (#8961). Checkpointed sweeps keep the
|
||||
// previous root exclusion so resume frontiers stay addressable (#8292).
|
||||
if !into.compacted {
|
||||
let done_compact = Metrics::time(Metric::CompactFolder);
|
||||
let compact_self = self.new_cache.info.name != folder.name || self.old_cache.info.scan_progress.is_none();
|
||||
self.new_cache
|
||||
.reduce_children_of(&this_hash, DATA_SCANNER_COMPACT_AT_CHILDREN, compact_self);
|
||||
// The root accumulator is also used by interruption handling.
|
||||
// Reduction may have removed all its old child links, so publish
|
||||
// the aggregate there before any cancellation-prone flush wait.
|
||||
if folder.parent.is_none()
|
||||
&& let Some(root) = self.new_cache.cache.get(&this_hash.0)
|
||||
&& root.compacted
|
||||
{
|
||||
into.clone_from(root);
|
||||
}
|
||||
self.new_cache.reduce_children_of(
|
||||
&this_hash,
|
||||
DATA_SCANNER_COMPACT_AT_CHILDREN,
|
||||
self.new_cache.info.name != folder.name,
|
||||
);
|
||||
done_compact();
|
||||
}
|
||||
|
||||
@@ -3291,8 +3224,6 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
if folder.parent.is_none() {
|
||||
#[cfg(test)]
|
||||
tests::checkpoint_fixture::deep_compacted::cancel_at_root_flush(&self.root, &self.new_cache, &ctx);
|
||||
wait_for_scanner_io(&ctx, self.sleeper.flush_pacing(&mut self.pacing_debt)).await?;
|
||||
}
|
||||
self.finish_size_reconciliation_batch();
|
||||
@@ -3484,7 +3415,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
let coverage_gap = scanner.coverage_gap;
|
||||
scanner.reconcile_failed_objects_after_full_scan();
|
||||
let new_cache = scanner.as_mut_new_cache();
|
||||
new_cache.force_compact(force_compact_limit_for_sweep(false));
|
||||
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
new_cache.info.last_update = Some(SystemTime::now());
|
||||
new_cache.info.next_cycle = next_cycle;
|
||||
let unresolved_objects = coverage_gap
|
||||
@@ -3506,9 +3437,6 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
progress.started_plan = progress.requested_plan;
|
||||
new_cache.info.scan_plan_digest = None;
|
||||
}
|
||||
} else if coverage_gap {
|
||||
// Legacy sampling must not certify an incomplete namespace.
|
||||
new_cache.info.scan_plan_digest = None;
|
||||
}
|
||||
let had_scan_checkpoint = had_scan_checkpoint || new_cache.info.scan_checkpoint.is_some();
|
||||
new_cache.info.scan_resume_after = None;
|
||||
@@ -3554,7 +3482,6 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
let root_hash = hash_path(&bucket_name);
|
||||
let root_has_progress = data_usage_root_has_progress(&root);
|
||||
let pending_heals_changed = scanner.pending_heals_changed;
|
||||
let legacy_coverage_gap = scanner.coverage_gap && scanner.old_cache.info.scan_progress.is_none();
|
||||
let raw_enumeration_path_page_indexes = scanner.raw_enumeration_path_page_indexes();
|
||||
let (raw_enumeration_cursor, raw_enumeration_page_index) = scanner.take_raw_enumeration_resume_state();
|
||||
let carry_forward_cache = ((raw_enumeration_cursor.is_some() || raw_enumeration_page_index.is_some())
|
||||
@@ -3589,20 +3516,11 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
|| !new_cache.info.size_reconciliation.is_empty()
|
||||
{
|
||||
if new_cache.root().is_some() {
|
||||
// Interrupted sweeps seal a resume frontier after this.
|
||||
// Flattening the root at the ordinary 10k limit would
|
||||
// drop frontier-named children and force the next cycle
|
||||
// to replay completed prefixes (#8292 / review on #9193).
|
||||
// Keep only the emergency FORCE ceiling here; successful
|
||||
// completions above apply DATA_SCANNER_COMPACT_AT_CHILDREN.
|
||||
new_cache.force_compact(force_compact_limit_for_sweep(true));
|
||||
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
}
|
||||
new_cache.info.last_update = Some(SystemTime::now());
|
||||
new_cache.info.next_cycle = next_cycle;
|
||||
new_cache.info.snapshot_complete = false;
|
||||
if legacy_coverage_gap {
|
||||
new_cache.info.scan_plan_digest = None;
|
||||
}
|
||||
if root_has_progress {
|
||||
set_scan_checkpoint(new_cache, checkpoint_reason_from_budget(budget.reason()));
|
||||
}
|
||||
|
||||
@@ -4475,110 +4475,3 @@ fn raw_enumeration_progress_retains_resume_index_until_unordered_entries_reappea
|
||||
"same source identity should keep the resume index even when read_dir order changes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_children_of_wide_root_keeps_into_aligned_before_cancel_wait() {
|
||||
// Non-checkpointed scans can reduce a reused wide root (>10k clean children)
|
||||
// into a compacted aggregate. The live `into` accumulator must be refreshed
|
||||
// from that cache entry before any await (flush_pacing): otherwise a cancel
|
||||
// handler that replace_hashed's `into` reintroduces deleted child links.
|
||||
const PREFIXES: usize = DATA_SCANNER_COMPACT_AT_CHILDREN + 1;
|
||||
const {
|
||||
assert!(PREFIXES > DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
}
|
||||
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let root = hash_path("bucket");
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
for index in 0..PREFIXES {
|
||||
let child = hash_path(&format!("bucket/prefix-{index:05}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let mut into = cache.find("bucket").expect("root").clone();
|
||||
assert!(!into.compacted);
|
||||
assert_eq!(into.children.len(), PREFIXES);
|
||||
|
||||
cache.reduce_children_of(&root, DATA_SCANNER_COMPACT_AT_CHILDREN, true);
|
||||
let cached = cache.find("bucket").expect("compacted root").clone();
|
||||
assert!(cached.compacted);
|
||||
assert!(cached.children.is_empty());
|
||||
assert_eq!(cached.objects, PREFIXES);
|
||||
|
||||
// Pre-fix shape: into still holds the deleted child links.
|
||||
assert_eq!(into.children.len(), PREFIXES);
|
||||
assert!(!into.compacted);
|
||||
|
||||
// Production fix: copy the compacted aggregate back into the accumulator
|
||||
// before any cancel-sensitive wait.
|
||||
if cached.compacted {
|
||||
into = cached;
|
||||
}
|
||||
|
||||
// Simulate the interruption handler overwriting the cache from `into`.
|
||||
cache.replace_hashed(&root, &None, &into);
|
||||
let after_cancel = cache.find("bucket").expect("root after cancel overwrite");
|
||||
assert!(after_cancel.compacted, "cancel must not resurrect a non-compacted root");
|
||||
assert!(after_cancel.children.is_empty(), "cancel must not reintroduce dangling child links");
|
||||
assert_eq!(after_cancel.objects, PREFIXES);
|
||||
assert_eq!(cache.cache.len(), 1, "deleted children must stay deleted after cancel overwrite");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn force_compact_limit_for_interrupted_sweep_keeps_frontier_above_ordinary_limit() {
|
||||
const PREFIXES: usize = DATA_SCANNER_COMPACT_AT_CHILDREN + 1;
|
||||
const {
|
||||
assert!(PREFIXES > DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
assert!(PREFIXES < DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS);
|
||||
}
|
||||
|
||||
let frontier = format!("bucket/prefix-{:05}", PREFIXES / 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let root = hash_path("bucket");
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
for index in 0..PREFIXES {
|
||||
let child = hash_path(&format!("bucket/prefix-{index:05}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let mut completed = cache.clone();
|
||||
completed.force_compact(force_compact_limit_for_sweep(false));
|
||||
assert!(completed.find("bucket").is_some_and(|entry| entry.compacted));
|
||||
assert!(completed.find(&frontier).is_none());
|
||||
|
||||
cache.force_compact(force_compact_limit_for_sweep(true));
|
||||
assert!(cache.find(&frontier).is_some(), "interrupted ceiling must keep the frontier child");
|
||||
let root_entry = cache.find("bucket").expect("root retained");
|
||||
assert!(!root_entry.compacted);
|
||||
assert_eq!(root_entry.children.len(), PREFIXES);
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
mod completed_compaction;
|
||||
pub(in crate::scanner_folder) mod deep_compacted;
|
||||
mod deep_compacted;
|
||||
mod memory_budget;
|
||||
mod periodic_snapshot;
|
||||
mod segment_observation;
|
||||
|
||||
@@ -293,507 +293,3 @@ async fn check_large_prefix_resumption(mode: HealScanMode) {
|
||||
}
|
||||
panic!("large prefix must finish within repeated bounded scans");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_legacy_permission_failure_isolated_to_child() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
assert!(!crate::scanner_is_erasure().await, "single-drive Namespace scan");
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
write_checkpoint_object(&root, "healthy/object", &[(None, 7)]).await;
|
||||
let bucket = root.join("bucket");
|
||||
let denied = bucket.join("denied");
|
||||
std::fs::create_dir(&denied).expect("create denied root child");
|
||||
for idx in 0..DATA_SCANNER_COMPACT_AT_FOLDERS - 2 {
|
||||
std::fs::create_dir(bucket.join(format!("empty-{idx:05}"))).expect("create wide directory");
|
||||
}
|
||||
assert_eq!(std::fs::read_dir(&bucket).expect("list root").count(), DATA_SCANNER_COMPACT_AT_FOLDERS);
|
||||
std::fs::set_permissions(&denied, std::fs::Permissions::from_mode(0o000)).expect("remove child read access");
|
||||
|
||||
match std::fs::read_dir(&denied) {
|
||||
Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied),
|
||||
Ok(_) => {
|
||||
std::fs::set_permissions(denied, std::fs::Permissions::from_mode(0o700)).expect("restore child access");
|
||||
eprintln!("skipping permission-denied fixture because this process can read mode-000 directories");
|
||||
return;
|
||||
}
|
||||
}
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
assert!(cache.info.scan_progress.is_none());
|
||||
cache.info.skip_healing = true;
|
||||
let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
|
||||
let wide = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
super::scan_options(HealScanMode::Normal),
|
||||
)
|
||||
.await;
|
||||
std::fs::set_permissions(denied, std::fs::Permissions::from_mode(0o700)).expect("restore child access");
|
||||
let ScannerDiskScanOutcome::Partial(wide) = wide.expect("wide legacy scanner must keep the same child-error isolation")
|
||||
else {
|
||||
panic!("an unreadable prefix must prevent complete snapshot publication");
|
||||
};
|
||||
assert!(!wide.info.snapshot_complete);
|
||||
let summary = wide.checked_flatten("bucket").expect("aggregate healthy child");
|
||||
assert!(summary.compacted);
|
||||
assert_eq!((summary.objects, summary.size), (1, 7));
|
||||
assert_eq!(wide.cache.len(), 1);
|
||||
}
|
||||
|
||||
// Select one fixture disk so unrelated concurrent scans cannot trigger this
|
||||
// cancellation. The hook runs at the root flush boundary, after compaction.
|
||||
static CANCEL_ROOT_FLUSH: Mutex<Option<std::path::PathBuf>> = Mutex::new(None);
|
||||
|
||||
pub(in crate::scanner_folder) fn cancel_at_root_flush(root: &str, cache: &DataUsageCache, ctx: &CancellationToken) {
|
||||
let mut selected = CANCEL_ROOT_FLUSH.lock().expect("root flush cancellation lock");
|
||||
if selected.as_deref() == Some(std::path::Path::new(root)) {
|
||||
let cached = cache.root().expect("root cached before flush");
|
||||
assert!(cached.compacted, "fixture must reach post-reduction compaction");
|
||||
assert!(cached.children.is_empty());
|
||||
*selected = None;
|
||||
ctx.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
struct RootFlushCancellationGuard;
|
||||
|
||||
impl Drop for RootFlushCancellationGuard {
|
||||
fn drop(&mut self) {
|
||||
*CANCEL_ROOT_FLUSH.lock().expect("clear root flush cancellation") = None;
|
||||
}
|
||||
}
|
||||
|
||||
async fn wide_legacy_root() -> (FolderScanner, std::path::PathBuf, TestGuard, DataUsageCache) {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
write_checkpoint_object(&root, "a-healthy/object", &[(Some(Uuid::from_u128(101)), 7)]).await;
|
||||
write_checkpoint_object(&root, "z-healthy/object", &[(Some(Uuid::from_u128(102)), 11)]).await;
|
||||
for index in 0..DATA_SCANNER_COMPACT_AT_FOLDERS - 2 {
|
||||
std::fs::create_dir(root.join("bucket").join(format!("empty-{index:05}"))).expect("create wide root child");
|
||||
}
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
cache.info.skip_healing = true;
|
||||
assert!(cache.info.scan_progress.is_none());
|
||||
(scanner, root, cleanup, cache)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_legacy_namespace_error_isolated_to_child() {
|
||||
assert!(!crate::scanner_is_erasure().await, "single-drive Namespace scan");
|
||||
let (scanner, root, _cleanup, cache) = wide_legacy_root().await;
|
||||
// The error occurs while resolving a symlink inside the child, after its
|
||||
// directory opened successfully. An open-only permission check misses it.
|
||||
let regular = root.join("regular-file");
|
||||
std::fs::write(®ular, b"payload").expect("write non-directory symlink target");
|
||||
for broken in ["m-broken", "nested/broken"] {
|
||||
let directory = root.join("bucket").join(broken);
|
||||
std::fs::create_dir_all(&directory).expect("create failing namespace child");
|
||||
let link = directory.join("bad-link");
|
||||
symlink(regular.join("child"), &link).expect("create NotADirectory symlink");
|
||||
assert_eq!(
|
||||
std::fs::metadata(link).expect_err("invalid symlink target").kind(),
|
||||
ErrorKind::NotADirectory
|
||||
);
|
||||
}
|
||||
write_checkpoint_object(&root, "nested/healthy/object", &[(Some(Uuid::from_u128(103)), 13)]).await;
|
||||
write_checkpoint_object(&root, "m-partial", &[(Some(Uuid::from_u128(104)), 101)]).await;
|
||||
// Fail enumeration after this child's metadata contributed usage. Simply
|
||||
// catching the error with a shared accumulator would retain its 101 bytes.
|
||||
let _partial =
|
||||
erasure_probe::OrderedDirectory::failing(&scanner.local_disk.path().join("bucket/m-partial"), 1, ErrorKind::Other);
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 5).await;
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("namespace errors must prevent complete snapshot publication without aborting healthy siblings")
|
||||
};
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
assert_eq!(budget.progress().0, 4, "healthy siblings and the failing child must read metadata");
|
||||
let total = cache.checked_flatten("bucket").expect("valid compacted root");
|
||||
assert!(total.compacted);
|
||||
assert_eq!((total.objects, total.versions, total.size), (3, 3, 31));
|
||||
assert_eq!(cache.cache.len(), 1, "child isolation must retain aggregate-only storage");
|
||||
let mut reloaded = save_reload(&FixtureStore::new(), &cache).await;
|
||||
assert_eq!(reloaded.checked_flatten("bucket").expect("persisted aggregate").objects, 3);
|
||||
drop(_partial);
|
||||
for broken in ["m-broken", "nested/broken"] {
|
||||
std::fs::remove_dir_all(root.join("bucket").join(broken)).expect("remove failed namespace fixture");
|
||||
}
|
||||
assert_eq!(
|
||||
reloaded.prepare_for_scan("bucket", 12, 7, SOURCE, PLAN, false),
|
||||
crate::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
reloaded.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, reloaded, HealScanMode::Normal, 5).await;
|
||||
let ScannerDiskScanOutcome::Complete(verified) = outcome else {
|
||||
panic!("a clean retry must rebuild complete coverage")
|
||||
};
|
||||
assert!(verified.info.snapshot_complete);
|
||||
assert_eq!(
|
||||
budget.progress().0,
|
||||
4,
|
||||
"retry must revisit every object including the previously failed child"
|
||||
);
|
||||
let total = verified.checked_flatten("bucket").expect("verified complete aggregate");
|
||||
assert_eq!((total.objects, total.versions, total.size), (4, 4, 132));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_legacy_child_cancellation_preserves_partial_accounting() {
|
||||
let (scanner, _root, _cleanup, cache) = wide_legacy_root().await;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 1).await;
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
assert_eq!(budget.progress().0, 1);
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("cancellation inside a compacted namespace child must propagate")
|
||||
};
|
||||
let total = cache.checked_flatten("bucket").expect("partial compacted accounting");
|
||||
assert!(total.compacted);
|
||||
assert_eq!((total.objects, total.versions), (1, 1));
|
||||
assert!([7, 11].contains(&total.size), "completed child-local bytes must survive cancellation");
|
||||
let reloaded = save_reload(&FixtureStore::new(), &cache).await;
|
||||
assert_eq!(reloaded.checked_flatten("bucket").expect("persisted partial accounting").size, total.size);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_reused_children_cancellation_preserves_compacted_accounting() {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
cache.info.skip_healing = true;
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let children = DATA_SCANNER_COMPACT_AT_CHILDREN + 1;
|
||||
for index in 0..children {
|
||||
let name = format!("bucket/clean-{index:05}");
|
||||
std::fs::create_dir_all(root.join(&name)).expect("create reusable root child");
|
||||
cache.replace(
|
||||
&name,
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
compacted: true,
|
||||
objects: 1,
|
||||
versions: 2,
|
||||
size: 7,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
assert_eq!(cache.root().expect("wide baseline root").children.len(), children);
|
||||
*CANCEL_ROOT_FLUSH.lock().expect("install root flush cancellation") = Some(scanner.local_disk.path());
|
||||
let _cancel_guard = RootFlushCancellationGuard;
|
||||
let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
|
||||
let mut options = super::scan_options(HealScanMode::Normal);
|
||||
options.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".into()]));
|
||||
let outcome = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(budget.token(), budget.clone(), vec![scanner.local_disk.clone()], cache, None, options)
|
||||
.await
|
||||
.expect("cancelled root flush must return a partial cache");
|
||||
assert!(CANCEL_ROOT_FLUSH.lock().expect("flush hook observed").is_none());
|
||||
assert_eq!(budget.progress().0, 0, "clean children must be reused without metadata reads");
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("flush cancellation must propagate")
|
||||
};
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
let total = cache
|
||||
.checked_flatten("bucket")
|
||||
.expect("cancellation must not leave dangling root links");
|
||||
assert!(total.compacted);
|
||||
assert!(total.children.is_empty());
|
||||
assert_eq!((total.objects, total.versions, total.size), (children, children * 2, children * 7));
|
||||
assert_eq!(cache.cache.len(), 1);
|
||||
let reloaded = save_reload(&FixtureStore::new(), &cache).await;
|
||||
assert_eq!(reloaded.checked_flatten("bucket").expect("persisted root accounting").objects, children);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_legacy_child_resource_deferral_preserves_partial_accounting() {
|
||||
use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let (mut scanner, _root, _cleanup, cache) = wide_legacy_root().await;
|
||||
scanner.old_cache = cache.clone();
|
||||
scanner.new_cache.info = cache.info;
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
|
||||
let admission = NodeAdmission::new(NonZeroU16::MIN);
|
||||
admission.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
scanner.node_admission = Some(admission.clone());
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".into()));
|
||||
let deferred = Arc::new(AtomicBool::new(false));
|
||||
let observed = deferred.clone();
|
||||
scanner.update_current_path = Arc::new(move |path| {
|
||||
if path == "bucket/a-healthy/object" {
|
||||
// Defer the next directory after this object's metadata completed.
|
||||
observed.store(true, Ordering::SeqCst);
|
||||
} else if observed.load(Ordering::SeqCst) {
|
||||
admission.update(&mut FeedbackController::new(), None);
|
||||
}
|
||||
Box::pin(async {})
|
||||
});
|
||||
let mut total = DataUsageEntry::default();
|
||||
let result = scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".into(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut total,
|
||||
)
|
||||
.await;
|
||||
assert!(deferred.load(Ordering::SeqCst), "first metadata object must be reached");
|
||||
assert!(
|
||||
matches!(result, Err(ScannerError::ResourceDeferred)),
|
||||
"resource pressure must propagate: {result:?}"
|
||||
);
|
||||
assert!(total.compacted);
|
||||
assert_eq!((total.objects, total.versions, total.size), (1, 1, 7));
|
||||
assert!(scanner.usage_updates.is_some(), "resource deferral must restore the progress owner");
|
||||
let committed = receiver.try_recv().expect("completed sibling progress");
|
||||
assert_eq!((committed.objects, committed.versions, committed.size), (1, 1, 7));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_legacy_usage_updates_exclude_uncommitted_child_accounting() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("1"))], async {
|
||||
let (mut scanner, root, _cleanup, cache) = wide_legacy_root().await;
|
||||
write_checkpoint_object(&root, "000-partial", &[(Some(Uuid::from_u128(105)), 101)]).await;
|
||||
let _partial = erasure_probe::OrderedDirectory::failing(&root.join("bucket/000-partial"), 1, ErrorKind::Other);
|
||||
scanner.old_cache = cache.clone();
|
||||
scanner.new_cache.info = cache.info;
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".into()));
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".into(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut DataUsageEntry::default(),
|
||||
)
|
||||
.await
|
||||
.expect("failed namespace must not abort healthy siblings");
|
||||
let committed = scanner
|
||||
.new_cache
|
||||
.checked_flatten("bucket")
|
||||
.expect("canonical committed aggregate");
|
||||
assert_eq!((committed.objects, committed.versions, committed.size), (2, 2, 18));
|
||||
let first = receiver.try_recv().expect("committed usage progress");
|
||||
assert_eq!(
|
||||
(first.objects, first.versions, first.size),
|
||||
(1, 1, 7),
|
||||
"provisional 101-byte child must never be published"
|
||||
);
|
||||
assert!(receiver.try_recv().is_err(), "minute interval still bounds updates");
|
||||
let updates = scanner
|
||||
.usage_updates
|
||||
.as_mut()
|
||||
.expect("requested usage owner must be restored");
|
||||
let total = updates.cache.checked_flatten("bucket").expect("connected committed progress");
|
||||
assert_eq!((total.objects, total.versions, total.size), (2, 2, 18));
|
||||
assert_eq!(
|
||||
updates.cache.cache.len(),
|
||||
1,
|
||||
"compacted progress must not retain provisional child entries"
|
||||
);
|
||||
updates.last_update = SystemTime::UNIX_EPOCH;
|
||||
scanner.send_update().await;
|
||||
let last = receiver.try_recv().expect("final committed aggregate");
|
||||
assert_eq!((last.objects, last.versions, last.size), (2, 2, 18));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn legacy_namespace_gap_requires_fresh_verification_before_sampling() {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for (object, version, size) in [("prefix/000-partial", 111, 101), ("prefix/healthy", 112, 7)] {
|
||||
write_checkpoint_object(&root, object, &[(Some(Uuid::from_u128(version)), size)]).await;
|
||||
}
|
||||
let partial = erasure_probe::OrderedDirectory::failing(
|
||||
&scanner.local_disk.path().join("bucket/prefix/000-partial"),
|
||||
1,
|
||||
ErrorKind::Other,
|
||||
);
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 3).await;
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("a narrow root must also refuse to certify skipped namespace coverage")
|
||||
};
|
||||
assert_eq!(budget.progress().0, 2, "failed child must reach provisional metadata accounting");
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
assert!(cache.find(PREFIX).expect("incomplete aggregate").compacted);
|
||||
assert_eq!(
|
||||
cache.info.scan_plan_digest, None,
|
||||
"incomplete coverage must invalidate sampled baseline reuse"
|
||||
);
|
||||
let mut reloaded = save_reload(&FixtureStore::new(), &cache).await;
|
||||
drop(partial);
|
||||
assert_eq!(
|
||||
reloaded.prepare_for_scan("bucket", 12, 7, SOURCE, PLAN, false),
|
||||
crate::DataUsageCachePrepareOutcome::Reset,
|
||||
"a legacy retry must not sample an incomplete compacted subtree"
|
||||
);
|
||||
reloaded.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, reloaded, HealScanMode::Normal, 3).await;
|
||||
let ScannerDiskScanOutcome::Complete(cache) = outcome else {
|
||||
panic!("clean retry must restore complete coverage")
|
||||
};
|
||||
assert!(cache.info.snapshot_complete);
|
||||
assert_eq!(budget.progress().0, 2, "both metadata objects must be rechecked");
|
||||
let total = cache.checked_flatten("bucket").expect("verified retry accounting");
|
||||
assert_eq!((total.objects, total.versions, total.size), (2, 2, 108));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_root_legacy_symlink_lookup_failure_isolated_to_child() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
assert!(!crate::scanner_is_erasure().await, "single-drive Namespace scan");
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
write_checkpoint_object(&root, "healthy/object", &[(None, 7)]).await;
|
||||
let bucket = root.join("bucket");
|
||||
let bad = bucket.join("bad");
|
||||
std::fs::create_dir(&bad).expect("create bad root child");
|
||||
let regular_file = root.join("regular-file");
|
||||
std::fs::write(®ular_file, b"not a directory").expect("create regular symlink ancestor");
|
||||
let broken_link = bad.join("broken-link");
|
||||
symlink(regular_file.join("child"), &broken_link).expect("create symlink through non-directory");
|
||||
assert_eq!(
|
||||
std::fs::metadata(broken_link).expect_err("target ancestor is a file").kind(),
|
||||
std::io::ErrorKind::NotADirectory
|
||||
);
|
||||
for idx in 0..DATA_SCANNER_COMPACT_AT_FOLDERS - 2 {
|
||||
std::fs::create_dir(bucket.join(format!("empty-{idx:05}"))).expect("create wide directory");
|
||||
}
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, _) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 100).await;
|
||||
let ScannerDiskScanOutcome::Partial(wide) = outcome else {
|
||||
panic!("a failed child must preserve sibling progress without certifying complete coverage");
|
||||
};
|
||||
assert!(!wide.info.snapshot_complete);
|
||||
let summary = wide.checked_flatten("bucket").expect("aggregate healthy child");
|
||||
assert!(summary.compacted);
|
||||
assert_eq!((summary.objects, summary.size), (1, 7));
|
||||
assert_eq!(wide.cache.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn legacy_failed_object_keeps_retry_history_after_partial_scan() {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
write_test_object_metadata_bytes(&root, "bucket", "broken/object", &[]).await;
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, _) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 100).await;
|
||||
let ScannerDiskScanOutcome::Partial(partial) = outcome else {
|
||||
panic!("a failed metadata read must keep the sweep partial");
|
||||
};
|
||||
assert_eq!(partial.info.failed_objects.len(), 1, "failed metadata should enter the retry cache");
|
||||
let mut reloaded = save_reload(&FixtureStore::new(), &partial).await;
|
||||
reloaded.prepare_for_scan("bucket", 12, 7, SOURCE, PLAN, false);
|
||||
assert_eq!(
|
||||
reloaded.info.failed_objects, partial.info.failed_objects,
|
||||
"same-plan retry must retain the timestamp that suppresses repeated repair attempts"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn interrupted_legacy_gap_requires_fresh_verification_before_sampling() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for (object, size) in [("prefix/a-bad/object", 101), ("prefix/z-good", 7), ("z-tail/object", 11)] {
|
||||
write_checkpoint_object(&root, object, &[(None, size)]).await;
|
||||
}
|
||||
let regular_file = root.join("regular-file");
|
||||
std::fs::write(®ular_file, b"not a directory").expect("create regular symlink ancestor");
|
||||
let broken_link = root.join("bucket/prefix/a-bad/bad-link");
|
||||
symlink(regular_file.join("child"), &broken_link).expect("create symlink through non-directory");
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 2).await;
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
let ScannerDiskScanOutcome::Partial(partial) = outcome else {
|
||||
panic!("the tail must interrupt the scan after the prefix was compacted");
|
||||
};
|
||||
let prefix = partial.find(PREFIX).expect("incomplete compacted prefix");
|
||||
assert!(prefix.compacted);
|
||||
assert_eq!((prefix.objects, prefix.size), (1, 7));
|
||||
|
||||
let mut reloaded = save_reload(&FixtureStore::new(), &partial).await;
|
||||
std::fs::remove_file(broken_link).expect("restore namespace traversal");
|
||||
let next_cycle = (12_u32..28)
|
||||
.find(|cycle| !hash_path(PREFIX).mod_(*cycle, data_usage_update_dir_cycles()))
|
||||
.expect("choose a cycle that would otherwise sample the incomplete prefix");
|
||||
reloaded.prepare_for_scan("bucket", u64::from(next_cycle), 7, SOURCE, PLAN, false);
|
||||
reloaded.info.skip_healing = true;
|
||||
let (outcome, _) = scan(&scanner.local_disk, reloaded, HealScanMode::Normal, 4).await;
|
||||
let ScannerDiskScanOutcome::Complete(verified) = outcome else {
|
||||
panic!("retry must complete after the namespace is repaired");
|
||||
};
|
||||
let total = verified.checked_flatten("bucket").expect("verified retry accounting");
|
||||
assert_eq!((total.objects, total.size), (3, 119));
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ enum DirectoryOrder {
|
||||
pub(in crate::scanner_folder) type OrderedEntries = VecDeque<std::io::Result<native_io::DirEntry>>;
|
||||
static ORDERED_DIRECTORIES: LazyLock<Mutex<HashMap<String, DirectoryOrder>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(in crate::scanner_folder) struct OrderedDirectory(String);
|
||||
struct OrderedDirectory(String);
|
||||
|
||||
impl OrderedDirectory {
|
||||
fn new(path: &std::path::Path) -> Self {
|
||||
@@ -26,7 +26,7 @@ impl OrderedDirectory {
|
||||
.insert(path.clone(), DirectoryOrder::PartLast);
|
||||
Self(path)
|
||||
}
|
||||
pub(in crate::scanner_folder) fn failing(path: &std::path::Path, entries: usize, kind: std::io::ErrorKind) -> Self {
|
||||
fn failing(path: &std::path::Path, entries: usize, kind: std::io::ErrorKind) -> Self {
|
||||
let guard = Self::new(path);
|
||||
ORDERED_DIRECTORIES
|
||||
.lock()
|
||||
@@ -205,7 +205,6 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("classify full UUID namespace");
|
||||
@@ -272,7 +271,6 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
task.await.expect("open-wait cancellation");
|
||||
@@ -312,7 +310,6 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(ctx.is_cancelled());
|
||||
@@ -382,7 +379,7 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
|
||||
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
let mut into = DataUsageEntry::default();
|
||||
let outcome = scanner
|
||||
.scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
|
||||
.scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false })
|
||||
.await
|
||||
.expect("classify candidate");
|
||||
assert_eq!(outcome, FolderScanOutcome::ErasureData);
|
||||
@@ -595,7 +592,6 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("classify symlink namespace");
|
||||
|
||||
@@ -2024,10 +2024,10 @@ pub use dirty_usage::{
|
||||
ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
|
||||
ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
|
||||
acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
|
||||
probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
|
||||
record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
|
||||
scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
|
||||
record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
|
||||
record_status_class_mutation, replay_durable_dirty_usage_producer_record, scanner_activity_epoch,
|
||||
scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -446,38 +446,6 @@ fn validate_durable_dirty_usage_replay_scope(
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate scoped ACK capability without clearing dirty work or taking a
|
||||
/// metadata mutation fence. Storage owns the read-only incarnation guards.
|
||||
/// A successful probe returns zero cleared records and retains no guards.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects lost incarnation fences, a changed process epoch, or zero,
|
||||
/// saturated, or future generations.
|
||||
pub fn probe_scoped_dirty_usage(
|
||||
instance_id: &str,
|
||||
entries: &[(&crate::storage_api::EcstoreScannerBucketIncarnationReadGuard, u64)],
|
||||
) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
|
||||
// Keep the generation observation ordered with dirty mutations. The probe
|
||||
// neither changes this map nor acquires its scope/producer companion maps.
|
||||
let _dirty = dirty_usage_buckets();
|
||||
let checked = entries
|
||||
.iter()
|
||||
.map(|(guard, generation)| {
|
||||
guard
|
||||
.checked_bucket_incarnation()
|
||||
.map(|(bucket, _)| (bucket, *generation))
|
||||
.map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
|
||||
})
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
validate_scoped_dirty_usage_generation(
|
||||
instance_id,
|
||||
scanner_activity_epoch(),
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
|
||||
&checked,
|
||||
)?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
/// A scoped ACK requires storage-owned lifecycle and incarnation fences.
|
||||
/// Callers must only send ACKs backed by durable per-bucket publication.
|
||||
pub fn acknowledge_scoped_dirty_usage(
|
||||
@@ -544,7 +512,16 @@ fn apply_scoped_dirty_usage_ack(
|
||||
entries: &[(&str, u64)],
|
||||
probe_only: bool,
|
||||
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
|
||||
validate_scoped_dirty_usage_generation(instance_id, current_instance, current_generation, entries)?;
|
||||
if instance_id != current_instance {
|
||||
return Err(ScannerDirtyUsageAckError::ProcessChanged);
|
||||
}
|
||||
if current_generation == u64::MAX
|
||||
|| entries
|
||||
.iter()
|
||||
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
|
||||
{
|
||||
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
|
||||
}
|
||||
let mut cleared = 0;
|
||||
if !probe_only {
|
||||
for (bucket, generation) in entries {
|
||||
@@ -558,25 +535,6 @@ fn apply_scoped_dirty_usage_ack(
|
||||
Ok(cleared)
|
||||
}
|
||||
|
||||
fn validate_scoped_dirty_usage_generation(
|
||||
instance_id: &str,
|
||||
current_instance: &str,
|
||||
current_generation: u64,
|
||||
entries: &[(&str, u64)],
|
||||
) -> std::result::Result<(), ScannerDirtyUsageAckError> {
|
||||
if instance_id != current_instance {
|
||||
return Err(ScannerDirtyUsageAckError::ProcessChanged);
|
||||
}
|
||||
if current_generation == u64::MAX
|
||||
|| entries
|
||||
.iter()
|
||||
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
|
||||
{
|
||||
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod status_class_mutation_tests {
|
||||
use super::*;
|
||||
@@ -665,25 +623,6 @@ mod scoped_dirty_usage_tests {
|
||||
use crate::segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
use serial_test::serial;
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_probe_rejects_invalid_epoch_and_generation() {
|
||||
assert_eq!(validate_scoped_dirty_usage_generation("p", "p", 8, &[("cold", 8)]), Ok(()));
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_generation("old", "new", 8, &[("cold", 8)]),
|
||||
Err(ScannerDirtyUsageAckError::ProcessChanged)
|
||||
);
|
||||
for generation in [0, 9, u64::MAX] {
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_generation("p", "p", 8, &[("cold", 8), ("hot", generation)]),
|
||||
Err(ScannerDirtyUsageAckError::InvalidGeneration)
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_generation("p", "p", u64::MAX, &[("cold", 8)]),
|
||||
Err(ScannerDirtyUsageAckError::InvalidGeneration)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
|
||||
@@ -53,10 +53,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard,
|
||||
ScannerBucketIncarnationReadGuard as EcstoreScannerBucketIncarnationReadGuard,
|
||||
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
|
||||
get_replication_config as ecstore_get_replication_config,
|
||||
BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
|
||||
get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::replication::{
|
||||
ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,
|
||||
|
||||
@@ -43,7 +43,6 @@ rustfs-utils = { workspace = true, features = ["full"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
base64-simd.workspace = true
|
||||
thiserror.workspace = true
|
||||
url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = { workspace = true, features = ["fmt"] }
|
||||
|
||||
@@ -13,14 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod constants;
|
||||
pub mod request_signature_azure;
|
||||
pub mod request_signature_streaming;
|
||||
pub mod request_signature_streaming_unsigned_trailer;
|
||||
pub mod request_signature_v2;
|
||||
pub mod request_signature_v4;
|
||||
pub mod utils;
|
||||
|
||||
pub use request_signature_azure::{AzureSignError, azure_shared_key_signature};
|
||||
pub use request_signature_streaming::streaming_sign_v4;
|
||||
pub use request_signature_streaming::try_streaming_sign_v4;
|
||||
pub use request_signature_v2::SignV2Error;
|
||||
|
||||
@@ -1,258 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared Key signing for Azure Blob REST requests.
|
||||
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::crypto::hmac_sha256;
|
||||
use std::collections::BTreeMap;
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum AzureSignError {
|
||||
#[error("Azure Shared Key signing requires a non-empty account key")]
|
||||
EmptyKey,
|
||||
#[error("Azure Shared Key header {0} is not valid ASCII")]
|
||||
InvalidHeader(String),
|
||||
#[error("Azure Shared Key header {0} has multiple values")]
|
||||
DuplicateHeader(&'static str),
|
||||
}
|
||||
|
||||
/// Sign an Azure Blob request using a decoded storage-account key.
|
||||
///
|
||||
/// The caller supplies the final URL and headers, including x-ms-date and
|
||||
/// x-ms-version. Content headers and conditions are signed for writes as well
|
||||
/// as reads. This implements the Shared Key format for service versions
|
||||
/// 2015-02-21 and newer, where a zero Content-Length is signed as an empty
|
||||
/// field. The returned base64 signature does not include the account name.
|
||||
/// Service-header whitespace is folded only outside quoted strings; quoted
|
||||
/// whitespace and escaped characters retain their original bytes.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Rejects an empty key, non-ASCII signed headers and repeated standard
|
||||
/// headers instead of signing a different request from the one on the wire.
|
||||
pub fn azure_shared_key_signature(
|
||||
key: &[u8],
|
||||
account: &str,
|
||||
method: &str,
|
||||
url: &Url,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<String, AzureSignError> {
|
||||
if key.is_empty() {
|
||||
return Err(AzureSignError::EmptyKey);
|
||||
}
|
||||
let mut canonical = String::with_capacity(512);
|
||||
canonical.push_str(method);
|
||||
canonical.push('\n');
|
||||
for name in [
|
||||
"content-encoding",
|
||||
"content-language",
|
||||
"content-length",
|
||||
"content-md5",
|
||||
"content-type",
|
||||
"date",
|
||||
"if-modified-since",
|
||||
"if-match",
|
||||
"if-none-match",
|
||||
"if-unmodified-since",
|
||||
"range",
|
||||
] {
|
||||
let mut values = headers.get_all(name).iter();
|
||||
let value = values.next();
|
||||
if values.next().is_some() {
|
||||
return Err(AzureSignError::DuplicateHeader(name));
|
||||
}
|
||||
let value = value
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map(str::trim)
|
||||
.map_err(|_| AzureSignError::InvalidHeader(name.to_string()))
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
if !(name == "content-length" && value == "0" || name == "date" && headers.contains_key("x-ms-date")) {
|
||||
canonical.push_str(value);
|
||||
}
|
||||
canonical.push('\n');
|
||||
}
|
||||
|
||||
let mut service_headers: BTreeMap<&str, Vec<String>> = BTreeMap::new();
|
||||
for (name, value) in headers {
|
||||
if name.as_str().starts_with("x-ms-") {
|
||||
let value = value.to_str().map_err(|_| AzureSignError::InvalidHeader(name.to_string()))?;
|
||||
service_headers
|
||||
.entry(name.as_str())
|
||||
.or_default()
|
||||
.push(normalize_service_header(value));
|
||||
}
|
||||
}
|
||||
for (name, values) in service_headers {
|
||||
canonical.push_str(name);
|
||||
canonical.push(':');
|
||||
canonical.push_str(&values.join(","));
|
||||
canonical.push('\n');
|
||||
}
|
||||
canonical.push('/');
|
||||
canonical.push_str(account);
|
||||
canonical.push_str(url.path());
|
||||
let mut query: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
for (name, value) in url.query_pairs() {
|
||||
query.entry(name.to_ascii_lowercase()).or_default().push(value.into_owned());
|
||||
}
|
||||
for (name, mut values) in query {
|
||||
values.sort();
|
||||
canonical.push('\n');
|
||||
canonical.push_str(&name);
|
||||
canonical.push(':');
|
||||
canonical.push_str(&values.join(","));
|
||||
}
|
||||
Ok(base64_simd::STANDARD.encode_to_string(hmac_sha256(key, canonical.as_bytes())))
|
||||
}
|
||||
|
||||
fn normalize_service_header(value: &str) -> String {
|
||||
// HeaderValue::to_str already excludes non-ASCII bytes and CR/LF.
|
||||
let mut normalized = String::with_capacity(value.len());
|
||||
let mut quoted = false;
|
||||
let mut escaped = false;
|
||||
let mut pending_space = false;
|
||||
for byte in value.bytes() {
|
||||
if !quoted && matches!(byte, b' ' | b'\t') {
|
||||
pending_space = !normalized.is_empty();
|
||||
continue;
|
||||
}
|
||||
if pending_space {
|
||||
normalized.push(' ');
|
||||
pending_space = false;
|
||||
}
|
||||
normalized.push(char::from(byte));
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if quoted && byte == b'\\' {
|
||||
escaped = true;
|
||||
} else if byte == b'"' {
|
||||
quoted = !quoted;
|
||||
}
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::HeaderValue;
|
||||
|
||||
#[test]
|
||||
fn signs_write_content_and_conditions_with_independent_hmac_vectors() {
|
||||
let url = Url::parse("https://account.blob.core.windows.net/container/blob?comp=block&blockid=AAAA")
|
||||
.expect("fixture URL should parse");
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("content-length", "3"),
|
||||
("content-md5", "YWJj"),
|
||||
("content-type", "application/octet-stream"),
|
||||
("if-none-match", "*"),
|
||||
("x-ms-date", "Fri, 06 Nov 2026 01:00:00 GMT"),
|
||||
("x-ms-version", "2020-10-02"),
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
// Fixed vectors generated with Python's hmac/hashlib from the Azure
|
||||
// documented canonical fields, independently of this implementation.
|
||||
for (length, expected) in [
|
||||
("3", "TMvdha8PzCSf0SB3v9EQSv0Se1gpdu1HusHBamwkyws="),
|
||||
("0", "kxBwUHkI/MK9beMd85Mij0MvaAlBtQQVYo9Qvl9rams="),
|
||||
] {
|
||||
headers.insert("content-length", HeaderValue::from_static(length));
|
||||
let signature = azure_shared_key_signature(b"0123456789abcdef0123456789abcdef", "account", "PUT", &url, &headers)
|
||||
.expect("valid write should sign");
|
||||
assert_eq!(signature, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signs_quoted_content_disposition_without_collapsing_filename_whitespace() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
("content-length", "3"),
|
||||
("content-md5", "YWJj"),
|
||||
("content-type", "application/octet-stream"),
|
||||
("if-none-match", "*"),
|
||||
("x-ms-date", "Fri, 06 Nov 2026 01:00:00 GMT"),
|
||||
("x-ms-version", "2020-10-02"),
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
// Independent Python hmac/hashlib vectors use the documented canonical
|
||||
// strings: only whitespace outside quoted strings is folded.
|
||||
for (value, signatures) in [
|
||||
(
|
||||
"attachment; filename=\"a b.txt\"",
|
||||
[
|
||||
"QO88BLMyaCEREeo/zP7esZKpkx9N/bwRQDwAkrjotB8=",
|
||||
"jZKYLpdUVYulDaVaNmpjy46IBxuuHyDmZ26ecnEs8wo=",
|
||||
],
|
||||
),
|
||||
(
|
||||
"\tattachment; \tfilename=\"a\\\" b.txt\" \t",
|
||||
[
|
||||
"dJI4cH77ju0YzM37HT66vck9p9obz9jdp5k04nhLkA0=",
|
||||
"13VDC1M4IIl11zPnxjBQhOd/CnzrxRj59DJ+wgDNQrQ=",
|
||||
],
|
||||
),
|
||||
(
|
||||
"attachment; filename=\"a\t\tb.txt\"",
|
||||
[
|
||||
"ONDPQmh7ulrnziZyCiHOm241GhEJ4iTZa/ZWmgdE4sY=",
|
||||
"pHkeDR6QaUwuQTJsP/RFTlfPqzOA86DG6HSpdm4Fh2U=",
|
||||
],
|
||||
),
|
||||
] {
|
||||
headers.insert("x-ms-blob-content-disposition", HeaderValue::from_static(value));
|
||||
for (address, expected) in [
|
||||
"https://account.blob.core.windows.net/container/blob",
|
||||
"https://account.blob.core.windows.net/container/blob?comp=blocklist",
|
||||
]
|
||||
.into_iter()
|
||||
.zip(signatures)
|
||||
{
|
||||
let url = Url::parse(address).expect("fixture URL should parse");
|
||||
let signature = azure_shared_key_signature(b"0123456789abcdef0123456789abcdef", "account", "PUT", &url, &headers)
|
||||
.expect("quoted content disposition should sign");
|
||||
assert_eq!(signature, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_ambiguous_headers_without_exposing_their_values() {
|
||||
let url = Url::parse("https://account.blob.core.windows.net/container/blob").expect("fixture URL should parse");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.append("content-length", HeaderValue::from_static("1"));
|
||||
headers.append("content-length", HeaderValue::from_static("2"));
|
||||
assert!(matches!(
|
||||
azure_shared_key_signature(b"key", "account", "PUT", &url, &headers),
|
||||
Err(AzureSignError::DuplicateHeader("content-length"))
|
||||
));
|
||||
headers.clear();
|
||||
headers.insert(
|
||||
"x-ms-meta-value",
|
||||
HeaderValue::from_bytes(b"\xff-private").expect("opaque header is legal HTTP"),
|
||||
);
|
||||
let err =
|
||||
azure_shared_key_signature(b"key", "account", "PUT", &url, &headers).expect_err("opaque signed header must fail");
|
||||
assert_eq!(err.to_string(), "Azure Shared Key header x-ms-meta-value is not valid ASCII");
|
||||
}
|
||||
}
|
||||
@@ -187,8 +187,6 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
|
||||
|
||||
`TierMutationIntent` models Add/Edit/Remove/Clear with `Prepared`, `Committed`, and `Aborted`. The identity includes the old tier-config ETag, candidate digest, canonical sorted affected targets and their old/new backend identities, and expiry. Revision and committed config ETag are the only mutable generation data.
|
||||
|
||||
A candidate configuration containing a native Azure Blob tier adds the immutable `requires_native_azure: true` intent extension. The flag participates in intent identity and checksum validation. Older strict readers reject it before installing a peer Prepare block, so all-node Prepare gates publication on native-driver support. The flag is omitted for ordinary mutations, preserving their prior wire bytes and checksums. Native Azure's persisted type and upgrade restrictions are documented in [Azure Blob remote tiers](../operations/azure-blob-tiering.md#upgrade-and-rollback).
|
||||
|
||||
New intents use a 15-minute expiry. A peer-only terminal tombstone is retained until that expiry plus five minutes of clock-skew allowance and until no coordinator record remains. Expiry bounds replay protection; it is not config commit/abort evidence.
|
||||
|
||||
The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. Prepare, Commit, and Abort use all-node fanout rather than quorum: independent peer calls use a work-conserving concurrency limit of four, a 30-second per-peer deadline, and a 30-second fanout-wide deadline; Prepare is additionally capped by the intent expiry. The coordinator collects every completed outcome. A timed-out or otherwise ambiguous started Prepare is included in compensating Abort because cancellation does not prove the peer failed to persist its fence; peers not started before the fanout deadline make Prepare fail but do not require Abort. The coordinator then conditionally writes tier config, durably commits the coordinator intent, releases its exclusive guards, requires every prepared peer to commit, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
|
||||
|
||||
@@ -67,7 +67,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. Object-backed recovery reports a staged request as SUPERSEDED only when a reachable finalized same-base winner proves it lost CAS and its idempotency binding is intact. The original staged receipt is retained; its old request still conflicts. The winning history must be retained with the losing receipt; missing or contradictory history requires manual review. |
|
||||
| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
|
||||
| Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. |
|
||||
| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; vended storage sessions cannot authenticate Catalog requests; responses carry `Cache-Control: no-store, private`. |
|
||||
| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; responses carry `Cache-Control: no-store, private`. |
|
||||
| Catalog diagnostics and export | Supported | Object-backed and durable-strong stores expose recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, and scale validation matrix. Durable-strong responses also identify the snapshot path, ETag, and format version; diagnostics re-check the snapshot observation before returning and use zero-file retention until strong maintenance configuration is persisted. |
|
||||
| Durable catalog backup and restore | Preview / controlled | Durable-strong only. `POST /iceberg/v1/{warehouse}/catalog/backup` seals a table-bucket snapshot behind the publication and migration fences, records catalog/object watermarks, and stores an immutable backup artifact. `POST /iceberg/v1/{warehouse}/catalog/restore` verifies those watermarks, requires an explicit replacement decision for an existing target, applies the catalog snapshot with ETag-CAS, and leaves a recoverable restore intent if maintenance-state finalization fails. It does not copy data files or claim automatic cross-region failover. |
|
||||
| Catalog import and rollback | Supported | Import/register and online rollback go through validation and commit paths. Online rollback accepts only forward-safe targets; restoring an older target that lowers watermarks is an offline disaster-recovery operation with all writers stopped. |
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
# Azure Blob Remote Tiers
|
||||
|
||||
New Azure tiers use the native Azure Blob REST API with storage-account Shared Key authentication. Azure public cloud and Azure Stack Hub use the same driver; Stack Hub requires its own account Blob endpoint. The driver uses service version 2020-10-02, which Microsoft lists for Stack Hub 2301 and later. See [Microsoft's storage compatibility matrix](https://learn.microsoft.com/en-us/azure-stack/user/azure-stack-acs-differences).
|
||||
|
||||
The implementation is in [warm_backend_azure_blob.rs](../../crates/ecstore/src/services/tier/warm_backend_azure_blob.rs). Account-key signing is shared with the on-demand migration reader through [request_signature_azure.rs](../../crates/signer/src/request_signature_azure.rs); the two features have different storage-operation and service-version contracts.
|
||||
|
||||
## Configuration
|
||||
|
||||
Create the destination container first. Reserve its selected prefix for RustFS tier data, and do not modify the generated blobs with another writer.
|
||||
|
||||
| Console/API field | Azure meaning |
|
||||
|---|---|
|
||||
| Azure Blob | Select the Azure tier provider |
|
||||
| Access Key / accessKey | Storage account name |
|
||||
| Secret Key / secretKey | Base64 storage-account key |
|
||||
| Bucket / bucket | Existing container name |
|
||||
| Endpoint / endpoint | Account Blob service origin; an empty value derives the public-cloud endpoint |
|
||||
| Prefix / prefix | Optional prefix reserved for this tier |
|
||||
| Region / region | Not used by native Blob signing or routing |
|
||||
| Storage Class / storageClass | Leave empty; the Console placeholder STANDARD is normalized to the default |
|
||||
|
||||
For Stack Hub, use the complete account endpoint supplied by its portal, such as https://account.blob.region.example. Do not supply a container path or a SAS query string as part of the endpoint. HTTPS uses the server's trusted CA roots, including the deployment's installed private CA bundle.
|
||||
|
||||
The existing Add Tier API accepts this shape:
|
||||
|
||||
~~~json
|
||||
{
|
||||
"type": "azure",
|
||||
"azure": {
|
||||
"name": "AZURE_COLD",
|
||||
"endpoint": "https://account.blob.region.example",
|
||||
"accessKey": "account",
|
||||
"secretKey": "<base64-account-key>",
|
||||
"bucket": "archive",
|
||||
"prefix": "rustfs"
|
||||
}
|
||||
}
|
||||
~~~
|
||||
|
||||
Only storage-account keys are supported by this tier driver. Service-principal credentials, SAS tokens and explicit Hot/Cool/Archive access-tier selection are not enabled. Stack Hub's own support matrix also differs from public Azure for authentication and access tiers; selecting Azure Blob does not promise every Azure feature.
|
||||
|
||||
## Upload, Read, and Cleanup
|
||||
|
||||
The driver writes Block Blobs. Small objects use Put Blob; larger objects use bounded Put Block requests followed by one conditional Put Block List. Each block has Content-MD5 and a unique upload-specific ID. The complete declared input must be consumed before commit, and the driver does not automatically replay failed requests. The Stack Hub limit of 50,000 blocks with at most 100 MiB per block bounds the accepted object size.
|
||||
|
||||
For a container with Blob versioning enabled, Azure returns timestamp version IDs rather than UUIDs. Before transitioning objects into such a container, upgrade every participating reader, writer and recovery worker, enable both `RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE=true` and `RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED=true` on the serving fleet, and ensure its live remote-version-state capability proof is current. The flags are operator attestation and a write request; they do not replace the live proof. With either flag disabled or the proof absent/stale, RustFS refuses to commit a new transition carrying an opaque Azure version and retains opaque-version recovery cleanup. See [opaque remote-version metadata requirements](tier-ilm-debugging.md#metadata-key-conventions).
|
||||
|
||||
The native Azure tier's all-peer Prepare check does not enable this separate gate. A successful Add/Verify probe checks the Blob transport and cleanup, so it does not by itself establish that versioned lifecycle writes can commit.
|
||||
|
||||
RustFS retains an opaque Azure version ID when the service returns one. Reads and ordinary deletes of that object explicitly address that version. If Azure rejects deleting the current version with OperationNotAllowedOnRootBlob, cleanup verifies the base blob still has the target version, conditionally deletes that base with its strong ETag, then retries the exact-version delete. A missing base also permits only an exact-version retry. A replacement version or a failed condition stops this fallback. A partial completion or lost acknowledgement remains an error until a later exact cleanup succeeds. An empty remote version sends no versionid parameter; unversioned cleanup also obtains a strong ETag and uses If-Match. Snapshots and immutability policies may prevent deletion and leave the normal tier cleanup record pending.
|
||||
|
||||
Metadata is encoded into legal Azure metadata names and values. The transaction and destination identities retain both RustFS and MinIO aliases. Recovery checks those identities before recognizing an uncertain upload as its candidate. Encoded metadata is limited to 8 KiB. The complete destination blob name, including the configured prefix, must fit the Stack Hub limit of 880 characters and 1,760 UTF-8 bytes; standalone dot path segments are rejected before I/O. Configuration validation normalizes one trailing prefix slash and reserves the entire canonical lifecycle object-name suffix before running a remote probe, so a prefix that fits a short probe but cannot fit actual transitions is rejected when constructing the tier.
|
||||
|
||||
Interrupted block staging can leave uncommitted blocks that Azure expires according to its own policy. Lifecycle uploads retain the existing transaction-journal recovery contract for a lost commit acknowledgement. Add/Edit/Verify probes retain the existing bounded in-process compensation: killing the process during a validation probe is not covered by a durable probe recovery worker. See [validation probe recovery status](tier-ilm-debugging.md#validation-probe-crash-recovery-status).
|
||||
|
||||
## Upgrade and Rollback
|
||||
|
||||
Existing Azure configurations retain their legacy S3-compatible transport. New Add operations select native Blob and persist a distinct protocol version and an encoded endpoint marker inside the existing binary configuration shape. A legacy destination and a native destination have different backend identities even if their account, endpoint, container and prefix are identical. Credential edits do not change the stored protocol.
|
||||
|
||||
Upgrade every topology member before adding a native Azure tier. A mutation whose candidate configuration contains a native Azure tier carries the immutable requires_native_azure intent extension. Earlier peers reject that extension during Prepare, before the new configuration can be committed. Ordinary tier mutations without native Azure keep their previous wire shape and checksums.
|
||||
|
||||
A rejected attempt during a mixed-version upgrade may retain a coordinator recovery record until every peer can read the intent and complete its normal abort/recovery exchange. Finish upgrading the remaining nodes before retrying; do not manually remove mutation records.
|
||||
|
||||
Do not downgrade a cluster while a native Azure tier or its recovery records remain in use. An older server can preserve and rewrite the existing Azure payload, but its S3 driver rejects the encoded native endpoint before making a request. Restore or migrate the referenced data and complete normal tier removal and recovery before rollback. Do not manually remove the native endpoint marker or protocol version, or delete tier metadata to bypass reference checks.
|
||||
|
||||
## Diagnostics and Verification
|
||||
|
||||
Native service failures retain a bounded HTTP status, Azure service error code and x-ms-request-id. Credential failures and deterministic rejected writes are reported directly; they do not cause a cleanup attempt against an object the write did not create. A transport failure or lost acknowledgement still requires candidate reconciliation, and an unconfirmed cleanup remains an error. Credentials, signed authorization values and raw response bodies are excluded from these diagnostics.
|
||||
|
||||
Local regression tests cover the signed Blob wire requests, block commit boundaries, empty and truncated input, ranged and versioned reads, conditional cleanup, identity-bound recovery, service errors, bounded XML parsing, persisted protocol selection and rejection by the previous peer reader. Before certifying a Stack Hub release, run Add/Verify, lifecycle transition, read-back/restore, expiry and restart/failure tests against that actual Stack Hub version. Mock protocol tests and Azure public-cloud results alone do not establish Stack Hub interoperability.
|
||||
@@ -11,11 +11,18 @@ to the original plaintext. Encryption and decryption formats do not change.
|
||||
|
||||
## Listings and older objects
|
||||
|
||||
ListObjects, ListObjectsV2, and ListObjectVersions omit the optional ETag field for encrypted objects whose metadata does not attest to the protected format. This includes older encrypted objects and preserved source ETags without that proof. New single-part encrypted writes carry the internal format marker and expose their protected ETag in listings. Completed encrypted multipart uploads also carry that marker when every selected part has quorum-backed proof that its stored ETag came from the encryption reader and the final ETag is the aggregate of those parts. Unencrypted listings retain their existing ETags.
|
||||
ListObjects, ListObjectsV2, and ListObjectVersions omit the optional ETag field
|
||||
for encrypted objects whose metadata does not attest to the protected format.
|
||||
This includes encrypted objects written before the upgrade, preserved source
|
||||
ETags without that proof, and completed encrypted multipart uploads. New
|
||||
single-part encrypted writes carry the internal format marker and expose their
|
||||
protected ETag in listings. Unencrypted listings retain their existing ETags.
|
||||
Existing persistent listing snapshots are invalidated; the provider falls
|
||||
back to verified storage until a new snapshot is built.
|
||||
|
||||
Each new encrypted part records its producer proof in optional part metadata. Older part metadata defaults to unproven. Mixing proven and unproven selected parts, or preserving a different source ETag, keeps the completed object unproven. The proof is checked during completion using the existing part metadata reads, so listings need no additional object reads or KMS requests. Internal same-version data movement, including rebalance and decommission, retains an existing source proof when quorum-validated staging metadata and the completed object agree on the version and preserved ETag. Tier restore also retains the proof for the same version and ETag when it commits under the matching restore operation. Sources without proof continue to require authorized HEAD access or a credentialed rewrite.
|
||||
|
||||
Clients that require an ETag for unproven objects must issue an authorized HeadObject request, including the SSE-C key when required. GET, HEAD, and conditional requests use the object's stored ETag. Already-completed encrypted multipart versions keep their original metadata and can still have no listing ETag after upgrading. Clients that enumerate historical versions, including backup software, may therefore require migration or an application change even after new uploads work. See the existing-data guidance below.
|
||||
Clients that require an ETag for these older or multipart objects must issue an
|
||||
authorized HeadObject request, including the SSE-C key when required. GET,
|
||||
HEAD, and conditional requests continue to use the object's stored ETag.
|
||||
|
||||
## Replication
|
||||
|
||||
|
||||
@@ -58,7 +58,6 @@ All names are defined in `crates/io-metrics/src/internode_metrics.rs`.
|
||||
| Metric | Stage signal |
|
||||
| --- | --- |
|
||||
| `rustfs_system_network_internode_operation_duration_ms{operation,backend}` | control-plane RTT (P0), lock/bulk latency (P1), first-op latency (P3) |
|
||||
| `rustfs_system_network_internode_operation_server_duration_ms{operation,backend}` | server-side `ReadMultiple` / `BatchReadVersion` handler lifetime, including decode, disk work, encode, errors, and cancellation; separate from client RTT |
|
||||
| `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) |
|
||||
| `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing a channel (P1 target) |
|
||||
| `rustfs_system_network_internode_dial_avg_time_nanos`, `rustfs_system_network_internode_dial_errors_total` | connect cost and failures (P3) |
|
||||
|
||||
@@ -17,20 +17,12 @@ A timed-out request is no longer cancelled. Cancelling sends `RST_STREAM`, and e
|
||||
|
||||
Three consecutive lock-acquisition timeouts open a request-level per-peer breaker. While open, new `lock` and `lock_batch` calls fail fast as retryable contention; after the backoff window one half-open probe is admitted. The breaker does not block release or refresh traffic, and it is separate from channel eviction. A per-peer in-flight admission limit also bounds requests that were already admitted before the breaker opened.
|
||||
|
||||
Single-resource distributed acquisition bounds each peer's contention wait to one second, but keeps pending responses and partial grants in the same quorum round until a response proves that round cannot reach quorum or the caller's remaining acquisition budget expires. A delayed response can still complete a healthy quorum; it does not restart acquisition with a fresh lock ID merely because delivery took longer than the peer's contention wait. Explicit contention failures retain rollback and retry behavior. Grants whose conservative lease deadline has expired before quorum are rolled back. Cancelling acquisition transfers both confirmed grants and pending replies to background cleanup. The caller's total deadline, read/write quorums, and fast quorum return remain unchanged.
|
||||
|
||||
Unlock cleanup that remains unconfirmed after three quick retries continues in the background with a deferred schedule (1s, 2s, 4s, 8s, 16s). Exhausting that schedule means cleanup was not confirmed; it does not prove that the peer still holds the lock. A server-side lease bounds any grant that remains.
|
||||
|
||||
Cleanup of a previously confirmed grant is idempotent for its exact `LockId`: `Released` and authoritative `AlreadyAbsent` both complete cleanup. The unlock RPCs append a structured `release_status` to each response or batch entry while preserving the existing `success` and `error_info` fields. Ordinary release and force-release still report a missing lock as failure. A new client keeps retrying unknown transport/backend results and absence responses from older peers that omit the structured status; it never infers absence from diagnostic text. Older clients ignore the added field and retain their existing behavior during upgrades or rollback.
|
||||
|
||||
An absence response cannot settle an acquisition that may still grant the same ID later. Pending acquisition completion retains its own cleanup owner, and uncertain remote acquisition failures keep their conservative late-release retries. Only a completed, confirmed grant can use authoritative absence to stop retrying.
|
||||
|
||||
## Local grant registration
|
||||
|
||||
The local guard registry uses nonblocking native-lock attempts for short map operations and async notifications when the table is occupied. An unpolled metadata waiter does not reserve the table, so currently running peers can register compatible grants without inheriting that waiter's scheduling delay. Notification registration precedes the retry, and unlocking precedes waking waiters. Every registry access and contention retry retains cooperative scheduling points; no executor thread waits on a blocking table-lock acquisition.
|
||||
|
||||
Physical acquisition and registry contention share the peer's existing acquisition deadline. An expired registration releases its unpublished physical guard and returns retryable contention, including when the table becomes available after the deadline. Replaced, removed, and reclaimed physical guards are dropped outside the registry lock. This does not change object-lock fairness, quorum, lease duration, or caller cancellation handling.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Environment variable | Default | Behavior |
|
||||
|
||||
@@ -31,10 +31,6 @@ A previously published gauge can remain visible after a statistic becomes unavai
|
||||
|
||||
When live requested bytes are unavailable, capture a supported heap profile or allocation/free trace. Preserve attribution to allocation stacks and lifetime; committed bytes and RSS alone cannot establish a leak. Other raw allocator statistics, including page accounting, depend on the allocator build and are not interchangeable with live application bytes.
|
||||
|
||||
### Linux transparent huge pages
|
||||
|
||||
Linux RustFS binaries disable transparent huge pages for mimalloc by default. This reduces resident memory when reclaimable allocations occupy only part of a huge page. Check `THP_enabled` in `/proc/<pid>/status` and `AnonHugePages` in `/proc/<pid>/smaps_rollup` to verify the running process. Both values should be zero with the default. The Linux binary also defaults to immediately purging free arena ranges. Set `MIMALLOC_PURGE_DELAY` before startup to override the delay in milliseconds.
|
||||
|
||||
### Idle reclaim
|
||||
|
||||
The reclaim loop waits for request, delete-tail, scanner, heal, EC and GET-buffer activity to become idle. Continuous traffic can skip reclaim. Check the activity gauges, skipped reasons and idle streak rather than assuming that waiting a fixed period guarantees collection.
|
||||
|
||||
@@ -22,7 +22,7 @@ RustFS validates every operator-configured outbound destination to close a serve
|
||||
| On-demand migration sources | Literal check, relaxed | Same guard and same escape hatch as replication targets (see below) |
|
||||
| Site replication peers | Literal check | `rustfs/src/site_replication/mod.rs` |
|
||||
| Self-hosted tiering warm backends (S3, MinIO, RustFS) | Literal check, relaxed | `validate_outbound_url_allow_private` allows private storage addresses except known IPv6 metadata addresses; RustFS retains its own debug-only loopback exception |
|
||||
| Cloud tiering warm backends (Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`, or the direct literal check in `warm_backend_azure_blob.rs` for native Azure; Wasabi additionally requires its canonical provider endpoint before using the S3 constructor |
|
||||
| Cloud tiering warm backends (Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`; Wasabi additionally requires its canonical provider endpoint before using the S3 constructor |
|
||||
| Keystone `auth_url` | Literal check | `crates/keystone/src/config.rs` |
|
||||
|
||||
The allowlist affects only the "Full policy" rows. A literal-check subsystem rejects a hostname that is itself a restricted IP literal, does not re-check what a hostname resolves to, and cannot be widened by `RUSTFS_OUTBOUND_ALLOW_ORIGINS`.
|
||||
|
||||
@@ -65,7 +65,3 @@ Typed pool metadata failures additionally carry `pool_metadata_reason`, `pool_me
|
||||
Storage inventory emits a WARN event on the first failed probe and an INFO event on recovery, using `event="storage_info_probe"`. A recovery event confirms the RPC succeeded, not that every reported disk is healthy. Bucket metadata load/retry errors include the bucket and a bounded error code, so one failing bucket can be identified without dumping its metadata.
|
||||
|
||||
No new environment variable, admin authorization action, or recovery command is required.
|
||||
|
||||
## Delete recovery during rolling upgrades
|
||||
|
||||
S3 deletes that require bounded recovery need peers that support the bounded prepare protocol. An older peer rejects that prepare before staging a mutation. If too few upgraded drives can satisfy the write quorum, the delete can fail with a retryable error until more peers are upgraded. Ordinary recovery inspection and settlement continue to use the original protocol.
|
||||
|
||||
@@ -24,8 +24,6 @@
|
||||
|
||||
## Lifecycle rule limits and evaluation
|
||||
|
||||
For native Azure configuration, protocol compatibility and the supported Stack Hub baseline, see [Azure Blob remote tiers](azure-blob-tiering.md).
|
||||
|
||||
Each lifecycle rule supports at most one `Transition` and one `NoncurrentVersionTransition`. A version can make one initial transition; chaining additional tiers after it reaches `complete` is not supported. Splitting stages across overlapping rules does not enable a transition chain. `PutBucketLifecycleConfiguration` rejects multiple entries in either transition array with `InvalidArgument`, including in disabled rules. Existing stored multi-entry arrays are not executed; replace each with a single intended destination. Independent expiration actions in the rule remain eligible.
|
||||
|
||||
`Expiration.Days` and `Expiration.Date` are mutually exclusive. A request containing both is rejected instead of silently selecting the date. When expiration and transition are both eligible, expiration takes precedence; a failed earlier transition does not keep an expired object indefinitely. Deadlines select the earliest action within the same action class.
|
||||
|
||||
@@ -41,8 +41,6 @@ The [immutable validation candidate reference](validation-candidates.md) describ
|
||||
|
||||
The [distributed multipart RPC delay gates](multipart-rpc-delay.md) define authenticated method-entry barriers, latency and peer-loss oracles, and the boundary between hook evidence and production incident attribution.
|
||||
|
||||
The [Docker rejoin readiness reproduction](docker-rejoin-readiness.md) compares node and cluster admission gates with fixed-source HTTPS containers and preserves the first object failure across recovery.
|
||||
|
||||
## Naming conventions
|
||||
|
||||
### Reserved test-name substrings (migration gate)
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# Docker rejoin readiness reproduction
|
||||
|
||||
**Use this when:** distinguishing restarted-node admission failures from acknowledged-overwrite peer GET failures in an HTTPS erasure set, or validating a change against fixed-source Docker binaries.
|
||||
|
||||
**Source of truth:** `scripts/repro_rejoin_readiness_docker.py` (`Experiment`, `GATE_PATHS`, `cleanup_resources`), `scripts/test_repro_rejoin_readiness_docker.py`, and `.docker/rejoin-readiness.Dockerfile`.
|
||||
|
||||
## Scope and prerequisites
|
||||
|
||||
The harness operates one isolated project on the selected Docker daemon. It records the client host, daemon architecture, kernel/storage driver, runtime distribution, data-volume filesystem, resource limits and binary identity. Matching Linux AMD64 and ARM64 daemon/image/binary combinations are accepted; a mismatch is refused before server startup. A successful run on a virtualized ARM64 Docker host is functional evidence for that environment, not native AMD64, independent-host durability, or production throughput evidence.
|
||||
|
||||
Use Docker, host curl with built-in `--aws-sigv4`, OpenSSL and Python. Run the client on the daemon host or a local Docker VM that forwards published loopback ports to it; selecting a remote Docker context alone does not forward those ports. The runtime image must provide `/bin/sh`, Python 3.11+, `getent`, `stat`, and a compatible `/usr/bin/rustfs`. The fixture keeps the root filesystem read-only, mounts owned Linux data/configuration volumes, verifies TLS, and publishes only loopback S3 ports. It explicitly creates and records a non-internal bridge network; include this difference when comparing it with an internal-network fixture. It does not use production credentials or data. See [probe semantics](../architecture/readiness-matrix.md#probe-semantics) and [recovered-backend routing](../operations/rolling-restart.md#returning-a-recovered-node-to-a-load-balancer).
|
||||
|
||||
## Prepare an immutable image
|
||||
|
||||
Build the selected source in an isolated checkout with a recorded compiler, target, features and profile. Reusing dependency caches is safe only when package identities and inputs remain correct; use a separate target directory for each source revision. If seeding it from a compiled cache, run `cargo clean --workspace` in the selected source before building; cached local path-package APIs must not cross revisions. Distinct container source paths alone do not establish freshness when checkout mtimes predate cached outputs. Do not label a dirty runtime build as an unmodified commit. Copy the completed binary into a private image context named `rustfs`, and record its actual SHA256. Use a libc-compatible native base with no inherited `RUSTFS_*`, `MINIO_*` compatibility aliases or `RUST_LOG` configuration; preflight rejects those overrides so each recorded control uses the fixture's explicit environment. The wrapper's default is an example GNU runtime, not a production-image recommendation.
|
||||
|
||||
```bash
|
||||
REJOIN_SOURCE_SHA="$(git rev-parse HEAD)"
|
||||
REJOIN_BINARY_SHA="$(openssl dgst -sha256 /path/to/context/rustfs | awk '{print $NF}')"
|
||||
docker build --pull=false -f .docker/rejoin-readiness.Dockerfile \
|
||||
--build-arg RUNTIME_BASE='<compatible-native-base>@sha256:<digest>' \
|
||||
--build-arg SOURCE_REVISION="$REJOIN_SOURCE_SHA" \
|
||||
--build-arg BINARY_SHA256="$REJOIN_BINARY_SHA" \
|
||||
--build-arg BUILD_PROFILE=release \
|
||||
-t rustfs-rejoin:fixed /path/to/context
|
||||
```
|
||||
|
||||
The wrapper validates its source/profile/digest arguments. The runner resolves the image to its immutable ID, validates labels, and independently verifies the binary digest and ELF architecture in the isolated staging container before executing it. Its evidence manifest records what was tested; an image name or `--version` string alone is insufficient provenance. An existing historical reproducer's revision checks must be updated together with its new binary and image labels; preserve the original fixture as the control.
|
||||
|
||||
## Run controlled cases
|
||||
|
||||
```bash
|
||||
python3 scripts/repro_rejoin_readiness_docker.py run \
|
||||
--image rustfs-rejoin:fixed --expected-source "$REJOIN_SOURCE_SHA" \
|
||||
--build-profile release --out-dir /path/to/new-private-evidence \
|
||||
--gate node-write --payload-mode distinct --cleanup-on-success
|
||||
```
|
||||
|
||||
Use a fresh output directory and project for each case. `--gate cluster` preserves the cluster-only recovery condition; `--gate node-write` additionally requires node readiness. Compare the gates on the same binary before attributing a change to code. Compare source revisions with the same architecture, runtime, profile, quotas, payload mode and workload. The default outage duration matches the historical reduced fixture; it is a fault interval, not an extra delay after the recovery gate. `--workers 2` is a separate worker-count experiment and must not silently change the default control.
|
||||
|
||||
`--payload-mode original` preserves identical overwrite bytes. The distinct mode retains the last acknowledged payload per key so an old matching version cannot masquerade as the latest overwrite. `--stress-iterations` increases same-key overwrite/peer-read pairs during each outage. Bucket creation may retry startup 503; object operations have no automatic retries. A first failed object response remains a failed result even if recovery or later observations succeed. Cluster-only health may pass before S3 admission; that observation is distinct from a survivor's namespace-lock timeout.
|
||||
|
||||
## Failure evidence and cleanup
|
||||
|
||||
The runner saves request timestamps, status, response bodies/headers, acknowledged-content identities, container generations, phase snapshots, Docker output and runtime logs. The warn/file profile archives live `/logs` through in-container Python with a 32 MiB/256-file budget; stopped tmpfs logs are explicitly unavailable. Docker copy can omit live tmpfs mounts and is not used for log capture. On a namespace-lock timeout it samples the read-only signed `/rustfs/admin/v3/top/locks` endpoint on the surviving nodes before restarting stopped peers. A missing/unsupported diagnostic endpoint is recorded, not interpreted as an empty lock table. No force-unlock is performed. See [distributed lock diagnostics](distributed-e2e.md) for the limits of guard-drop and physical-release observations.
|
||||
|
||||
Use `--diagnostics` only as a separately identified diagnostic run; trace overhead is not performance evidence. This profile uses stdout only and explicitly bounded Docker JSON logs (20 MiB per file, five retained files); capture includes the retained history across restart. `summary.log_captures` records sources and bytes, and final `diagnostic_coverage` requires acquired, guard-dropped and physical-release JSON event categories across the pool. Missing any category fails diagnostic coverage while retaining the independent HTTP results. Category coverage does not prove release of every owner or resolve a historical timeout. Keep initial failure, later convergence, and restarted-node checks separate. A retained run stops its owned nodes after capturing final evidence; its volumes and private files remain available for investigation. Successful runs can remove resources with `--cleanup-on-success`.
|
||||
|
||||
```bash
|
||||
python3 scripts/repro_rejoin_readiness_docker.py cleanup \
|
||||
--out-dir /path/to/private-evidence
|
||||
```
|
||||
|
||||
Every data/configuration volume is verified before staging mounts or changes it; mounts disable image-content copying. Cleanup verifies every extant resource against the manifest's project, source and ownership labels before deleting any resource. During a run, Ctrl-C and SIGTERM interrupt capture as failures and continue through shutdown and summary writing; interrupted retention is explicitly reported as unconfirmed. The CLI restores the previous SIGTERM handler when the run returns or raises an exception. It refuses foreign resources and does not prune the daemon. Output directories contain generated credentials, curl configurations and private CA/node keys: do not commit or upload them wholesale. Publish only a reviewed, redacted result summary with source/build identity and explicit environment limits.
|
||||
@@ -388,8 +388,6 @@ rustfs-mimalloc = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libsystemd.workspace = true
|
||||
rustfs-mimalloc = { workspace = true, features = ["no_thp"] }
|
||||
rustfs-mimalloc-sys = "0.7.3"
|
||||
|
||||
[target.'cfg(any(all(target_os = "macos", any(target_arch = "x86_64", target_arch = "aarch64")), all(target_os = "linux", target_env = "gnu", any(target_arch = "x86_64", target_arch = "aarch64"))))'.dependencies]
|
||||
pyroscope = { workspace = true, optional = true, features = ["backend-pprof-rs"] }
|
||||
|
||||
@@ -1326,16 +1326,6 @@ async fn table_catalog_request_principal(req: &S3Request<Body>) -> S3Result<Tabl
|
||||
Some(context.as_ref()),
|
||||
)
|
||||
.await?;
|
||||
// Storage credentials are a data-plane delegation, never a catalog identity.
|
||||
if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
|
||||
.iter()
|
||||
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::AccessDenied,
|
||||
"table-scoped storage credentials cannot access the catalog control plane",
|
||||
));
|
||||
}
|
||||
let iam = context.iam();
|
||||
if !iam.is_ready() {
|
||||
return Err(table_catalog_internal_error("iam not init"));
|
||||
|
||||
@@ -323,119 +323,6 @@ async fn table_catalog_authentication_and_credentials_use_the_request_context()
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn table_storage_credentials_are_rejected_by_catalog_authentication() {
|
||||
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
rustfs_iam::store::object::ObjectStore::new(store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.expect("seed request IAM format");
|
||||
let iam = rustfs_iam::build_iam_sys(store.clone())
|
||||
.await
|
||||
.expect("initialize request IAM");
|
||||
let parent_key = "storage-session-parent";
|
||||
let parent_secret = "storage-session-parent-secret";
|
||||
iam.create_user(
|
||||
parent_key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: parent_secret.to_string(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create session parent");
|
||||
iam.set_policy(
|
||||
"storage-session-parent-policy",
|
||||
Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"admin:GetTable"}]}"#)
|
||||
.expect("parse catalog read policy"),
|
||||
)
|
||||
.await
|
||||
.expect("store parent policy");
|
||||
iam.policy_db_set(parent_key, UserType::Reg, false, "storage-session-parent-policy")
|
||||
.await
|
||||
.expect("attach parent policy");
|
||||
let root_key = "storage-session-root";
|
||||
let root_secret = "storage-session-root-secret";
|
||||
let context = Arc::new(AppContext::new(store, Arc::new(RequestIam { handle: iam.clone() }), Arc::new(RequestKms)));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: root_key.to_string(),
|
||||
secret_key: root_secret.to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
for (key, secret) in [(root_key, root_secret), (parent_key, parent_secret)] {
|
||||
authorize_table_catalog_request(
|
||||
&table_catalog_handler_request(context.clone(), key, secret),
|
||||
AdminAction::GetTableAction,
|
||||
)
|
||||
.await
|
||||
.expect("ordinary catalog identities remain authorized");
|
||||
}
|
||||
|
||||
for parent in [root_key, parent_key] {
|
||||
let mut cases = vec![(None, serde_json::Value::Null)];
|
||||
for marker in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
|
||||
for value in [
|
||||
serde_json::json!("table-scope"),
|
||||
serde_json::json!(""),
|
||||
serde_json::Value::Null,
|
||||
serde_json::json!(7),
|
||||
serde_json::json!([]),
|
||||
serde_json::json!({}),
|
||||
] {
|
||||
cases.push((Some(marker), value));
|
||||
}
|
||||
}
|
||||
for (marker, value) in cases {
|
||||
let expiration = OffsetDateTime::now_utc().saturating_add(Duration::minutes(15));
|
||||
let mut claims = HashMap::from([
|
||||
("exp".to_string(), serde_json::json!(expiration.unix_timestamp())),
|
||||
("parent".to_string(), serde_json::json!(parent)),
|
||||
]);
|
||||
if let Some(marker) = marker {
|
||||
claims.insert(marker.to_string(), value);
|
||||
}
|
||||
let mut credential = get_new_credentials_with_metadata(&claims, root_secret).expect("sign a temporary credential");
|
||||
credential.parent_user = parent.to_string();
|
||||
iam.set_temp_user(&credential.access_key, &credential, None)
|
||||
.await
|
||||
.expect("store temporary credential");
|
||||
let mut request = table_catalog_handler_request(context.clone(), &credential.access_key, &credential.secret_key);
|
||||
request.headers.insert(
|
||||
"x-amz-security-token",
|
||||
HeaderValue::from_str(&credential.session_token).expect("session token header"),
|
||||
);
|
||||
let unscoped = authorize_table_catalog_request(&request, AdminAction::GetTableAction).await;
|
||||
let scoped = authorize_table_catalog_resource_request(
|
||||
&request,
|
||||
&TableCatalogResource::warehouse("warehouse"),
|
||||
AdminAction::GetTableAction,
|
||||
)
|
||||
.await;
|
||||
if marker.is_some() {
|
||||
assert_eq!(
|
||||
unscoped.expect_err("any storage marker must deny catalog access").code(),
|
||||
&S3ErrorCode::AccessDenied
|
||||
);
|
||||
assert_eq!(
|
||||
scoped
|
||||
.err()
|
||||
.expect("partial or malformed storage claims must fail closed")
|
||||
.code(),
|
||||
&S3ErrorCode::AccessDenied
|
||||
);
|
||||
} else {
|
||||
unscoped.expect("ordinary temporary credentials retain catalog authorization");
|
||||
scoped.expect("ordinary temporary credentials retain resource authorization");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
@@ -1295,30 +1182,6 @@ async fn load_table_handler_negotiates_vended_credentials_without_breaking_metad
|
||||
);
|
||||
}
|
||||
}
|
||||
let delegated = &issued_json["storage-credentials"][0]["config"];
|
||||
let mut storage_request = table_catalog_handler_request(
|
||||
context.clone(),
|
||||
delegated[S3_ACCESS_KEY_ID_CONFIG_KEY].as_str().expect("vended access key"),
|
||||
delegated[S3_SECRET_ACCESS_KEY_CONFIG_KEY]
|
||||
.as_str()
|
||||
.expect("vended secret key"),
|
||||
);
|
||||
storage_request.headers.insert(
|
||||
"x-amz-security-token",
|
||||
HeaderValue::from_str(delegated[S3_SESSION_TOKEN_CONFIG_KEY].as_str().expect("vended session token"))
|
||||
.expect("session token header"),
|
||||
);
|
||||
let error = call_load_table_handler(storage_request)
|
||||
.await
|
||||
.expect_err("vended storage credentials must not enter the catalog handler");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
let reloaded = call_load_table_handler(table_catalog_handler_request(context, vended_access_key, vended_secret_key))
|
||||
.await
|
||||
.expect("the parent catalog identity remains authorized");
|
||||
let reloaded_json: serde_json::Value =
|
||||
serde_json::from_slice(&reloaded.output.1.bytes().expect("reloaded table body")).expect("parse reloaded table");
|
||||
assert_eq!(reloaded_json["metadata-location"], issued_json["metadata-location"]);
|
||||
assert_eq!(reloaded_json["metadata"], issued_json["metadata"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -32,8 +32,8 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
},
|
||||
metadata::{
|
||||
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TARGETS_FILE,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG,
|
||||
BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
|
||||
},
|
||||
metadata_sys,
|
||||
policy_sys::PolicySys,
|
||||
@@ -1989,7 +1989,7 @@ impl DefaultBucketUsecase {
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketTaggingInput { bucket, .. } = req.input;
|
||||
|
||||
metadata_sys::update_s3_tagging_config(&bucket, None, expected_incarnation_id)
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_TAGGING_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
@@ -2899,7 +2899,9 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
metadata_sys::update_s3_tagging_config(&bucket, Some(tagging.clone()), expected_incarnation_id)
|
||||
let data = serialize_config(&tagging)?;
|
||||
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_TAGGING_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
|
||||
@@ -3044,23 +3044,6 @@ mod tests {
|
||||
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
|
||||
let (_store, bucket) =
|
||||
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
s3s::S3::put_bucket_versioning(
|
||||
&fs,
|
||||
build_request(
|
||||
s3s::dto::PutBucketVersioningInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.versioning_configuration(s3s::dto::VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")),
|
||||
..Default::default()
|
||||
})
|
||||
.build()
|
||||
.expect("bucket versioning input"),
|
||||
Method::PUT,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning for encrypted multipart listing");
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let key = "customer-encrypted";
|
||||
let customer_key = "KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio=";
|
||||
@@ -3154,29 +3137,6 @@ mod tests {
|
||||
.expect("key MD5"),
|
||||
customer_md5
|
||||
);
|
||||
let etag = response.output.e_tag.expect("completed encrypted object ETag");
|
||||
let version_id = response.output.version_id.expect("completed object version");
|
||||
assert_ne!(version_id, "null");
|
||||
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
|
||||
let head = s3s::S3::head_object(
|
||||
&fs,
|
||||
build_request(
|
||||
s3s::dto::HeadObjectInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
version_id: Some(version_id.clone()),
|
||||
sse_customer_algorithm: Some("AES256".to_string()),
|
||||
sse_customer_key: Some(customer_key.to_string()),
|
||||
sse_customer_key_md5: Some(customer_md5.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::HEAD,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("authorized HEAD of completed encrypted version");
|
||||
assert_eq!(head.output.e_tag, Some(etag));
|
||||
assert_eq!(head.output.version_id, Some(version_id));
|
||||
} else {
|
||||
assert_eq!(
|
||||
result
|
||||
@@ -3189,232 +3149,6 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
async fn assert_multipart_etag_listings(
|
||||
fs: &crate::storage::ecfs::FS,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
etag: &ETag,
|
||||
version_id: &str,
|
||||
) {
|
||||
use s3s::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
|
||||
use s3s::xml::{Serialize, Serializer};
|
||||
|
||||
fn assert_xml_etag<T: Serialize>(output: &T, etag: &ETag) {
|
||||
let mut bytes = Vec::new();
|
||||
output
|
||||
.serialize(&mut Serializer::new(&mut bytes))
|
||||
.expect("serialize listing XML");
|
||||
let xml = String::from_utf8(bytes).expect("listing XML is UTF-8");
|
||||
assert!(
|
||||
xml.contains(&format!("<ETag>\"{}\"</ETag>", etag.value())),
|
||||
"listing must include the completed ETag: {xml}"
|
||||
);
|
||||
}
|
||||
|
||||
// Listings carry neither the SSE-C key nor a per-object decryption request.
|
||||
let versions = s3s::S3::list_object_versions(
|
||||
fs,
|
||||
build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("keyless version listing")
|
||||
.output;
|
||||
let entries = versions.versions.as_ref().expect("completed version must be listed");
|
||||
assert_eq!(entries.len(), 1, "completion replay must preserve a single version");
|
||||
assert_eq!(entries[0].key.as_deref(), Some(key));
|
||||
assert_eq!(entries[0].version_id.as_deref(), Some(version_id));
|
||||
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
|
||||
assert_xml_etag(&versions, etag);
|
||||
|
||||
let v1 = s3s::S3::list_objects(
|
||||
fs,
|
||||
build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("keyless V1 listing")
|
||||
.output;
|
||||
let v2 = s3s::S3::list_objects_v2(
|
||||
fs,
|
||||
build_request(
|
||||
ListObjectsV2Input {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("keyless V2 listing")
|
||||
.output;
|
||||
for contents in [&v1.contents, &v2.contents] {
|
||||
let entries = contents.as_ref().expect("completed object must be listed");
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].key.as_deref(), Some(key));
|
||||
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
|
||||
}
|
||||
assert_xml_etag(&v1, etag);
|
||||
assert_xml_etag(&v2, etag);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn kms_multipart_completion_and_replay_preserve_etag_in_versioned_listings() {
|
||||
crate::app::gating_test_env::run_large_stack_test("kms-multipart-listing-etag", || async {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_kms::types::{CreateKeyRequest, KeyUsage};
|
||||
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let key_dir = tempfile::TempDir::new().expect("local KMS key directory");
|
||||
let manager = ambient.kms().handle();
|
||||
manager
|
||||
.reconfigure(rustfs_kms::KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
|
||||
.await
|
||||
.expect("configure local KMS");
|
||||
let kms_key = manager
|
||||
.get_encryption_service()
|
||||
.await
|
||||
.expect("local KMS encryption service")
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some("multipart-listing-key".to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
policy: None,
|
||||
tags: HashMap::new(),
|
||||
origin: None,
|
||||
})
|
||||
.await
|
||||
.expect("create multipart KMS key");
|
||||
let bucket = format!("kms-mpu-etag-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned Object Lock bucket");
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
let key = "kms-encrypted";
|
||||
s3s::S3::put_bucket_encryption(
|
||||
&fs,
|
||||
build_request(
|
||||
s3s::dto::PutBucketEncryptionInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.server_side_encryption_configuration(s3s::dto::ServerSideEncryptionConfiguration {
|
||||
rules: vec![s3s::dto::ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(s3s::dto::ServerSideEncryptionByDefault {
|
||||
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
kms_master_key_id: Some(kms_key.key_id.clone()),
|
||||
}),
|
||||
..Default::default()
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.expect("bucket encryption input"),
|
||||
Method::PUT,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("configure bucket-default SSE-KMS");
|
||||
let upload_id = s3s::S3::create_multipart_upload(
|
||||
&fs,
|
||||
build_request(
|
||||
CreateMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::POST,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("create KMS multipart upload")
|
||||
.output
|
||||
.upload_id
|
||||
.expect("KMS upload id");
|
||||
let part = s3s::S3::upload_part(
|
||||
&fs,
|
||||
build_request(
|
||||
UploadPartInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
upload_id: upload_id.clone(),
|
||||
part_number: 1,
|
||||
content_length: Some(3),
|
||||
body: Some(StreamingBlob::from(Bytes::from_static(b"abc"))),
|
||||
..Default::default()
|
||||
},
|
||||
Method::PUT,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("upload encrypted KMS part")
|
||||
.output;
|
||||
let complete = CompleteMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
upload_id,
|
||||
multipart_upload: Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: part.e_tag,
|
||||
..Default::default()
|
||||
}]),
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
for _ in 0..2 {
|
||||
let response = s3s::S3::complete_multipart_upload(&fs, build_request(complete.clone(), Method::POST))
|
||||
.await
|
||||
.expect("complete and replay KMS multipart upload")
|
||||
.output;
|
||||
let etag = response.e_tag.expect("KMS completed ETag");
|
||||
let version_id = response.version_id.expect("KMS completed version");
|
||||
assert_ne!(version_id, "null");
|
||||
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
|
||||
let head = s3s::S3::head_object(
|
||||
&fs,
|
||||
build_request(
|
||||
s3s::dto::HeadObjectInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
version_id: Some(version_id.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::HEAD,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("HEAD of completed KMS version")
|
||||
.output;
|
||||
assert_eq!(head.e_tag, Some(etag));
|
||||
assert_eq!(head.version_id, Some(version_id));
|
||||
assert_eq!(head.server_side_encryption.as_ref().map(|value| value.as_str()), Some("aws:kms"));
|
||||
assert_eq!(head.ssekms_key_id.as_deref(), Some(kms_key.key_id.as_str()));
|
||||
}
|
||||
manager.stop().await.expect("stop local KMS");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists() {
|
||||
|
||||
@@ -486,6 +486,8 @@ pub(crate) mod bucket {
|
||||
pub(crate) const BUCKET_REPLICATION_CONFIG: &str =
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG;
|
||||
pub(crate) const BUCKET_SSECONFIG: &str = crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_SSECONFIG;
|
||||
pub(crate) const BUCKET_TAGGING_CONFIG: &str =
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
pub(crate) const BUCKET_TARGETS_FILE: &str = crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_TARGETS_FILE;
|
||||
pub(crate) const BUCKET_VERSIONING_CONFIG: &str =
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG;
|
||||
@@ -601,20 +603,7 @@ pub(crate) mod bucket {
|
||||
pub(crate) async fn get_tagging_config(
|
||||
bucket: &str,
|
||||
) -> Result<(Tagging, OffsetDateTime), crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_bucket::metadata_sys::get_s3_tagging_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn update_s3_tagging_config(
|
||||
bucket: &str,
|
||||
tagging: Option<Tagging>,
|
||||
expected_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> Result<OffsetDateTime, crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_bucket::metadata_sys::update_s3_tagging_config(
|
||||
bucket,
|
||||
tagging,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
crate::storage::storage_api::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn update(
|
||||
|
||||
+2
-4
@@ -1250,8 +1250,7 @@ pub async fn init_webdav_system(
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::broadcast::channel(1);
|
||||
|
||||
let task_handle = tokio::spawn(async move {
|
||||
let result = server.start(shutdown_rx).await;
|
||||
if let Err(e) = &result {
|
||||
if let Err(e) = server.start(shutdown_rx).await {
|
||||
error!(
|
||||
target: "rustfs::init",
|
||||
event = EVENT_PROTOCOL_SERVER_STATE,
|
||||
@@ -1272,7 +1271,6 @@ pub async fn init_webdav_system(
|
||||
state = "stopped",
|
||||
"Protocol server stopped"
|
||||
);
|
||||
result
|
||||
});
|
||||
|
||||
info!(
|
||||
@@ -1286,7 +1284,7 @@ pub async fn init_webdav_system(
|
||||
tls_enabled,
|
||||
"Protocol runtime started"
|
||||
);
|
||||
Ok(Some(ShutdownHandle::new_webdav(shutdown_tx, task_handle)))
|
||||
Ok(Some(ShutdownHandle::new(shutdown_tx, task_handle)))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -62,12 +62,6 @@ static GLOBAL: hotpath::CountingAllocator<std::alloc::System> = hotpath::Countin
|
||||
static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
|
||||
|
||||
fn main() -> std::process::ExitCode {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[allow(unsafe_code)]
|
||||
// SAFETY: The binary entrypoint runs this before RustFS creates any threads.
|
||||
unsafe {
|
||||
rustfs_mimalloc::MiMalloc::option_set_default(rustfs_mimalloc_sys::mi_option_t::mi_option_purge_delay, 0);
|
||||
}
|
||||
let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
|
||||
|
||||
rustfs::startup_entrypoint::run_process()
|
||||
|
||||
@@ -42,18 +42,15 @@ use super::source_client::{
|
||||
};
|
||||
use super::storage_api::HTTPRangeSpec;
|
||||
use super::storage_api::remote_s3_client::RemoteS3ClientError;
|
||||
#[cfg(test)]
|
||||
use hmac::{Hmac, Mac, digest::KeyInit};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use quick_xml::Reader;
|
||||
use quick_xml::events::{BytesStart, Event};
|
||||
#[cfg(test)]
|
||||
use sha2::Sha256;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use url::Url;
|
||||
|
||||
#[cfg(test)]
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
/// Blob REST version this backend pins. Every response field it reads exists
|
||||
@@ -304,8 +301,61 @@ impl SourceBackend for AzureSourceBackend {
|
||||
/// canonicalized resource. GET and HEAD carry no body, so every `Content-*`
|
||||
/// slot stays empty.
|
||||
fn shared_key_signature(key: &[u8], account: &str, method: &str, url: &Url, headers: &HeaderMap) -> Result<String, SourceError> {
|
||||
rustfs_signer::azure_shared_key_signature(key, account, method, url, headers)
|
||||
.map_err(|err| SourceError::Other(err.to_string()))
|
||||
let mut string_to_sign = String::with_capacity(256);
|
||||
string_to_sign.push_str(method);
|
||||
string_to_sign.push('\n');
|
||||
// Content-Encoding, Content-Language, Content-Length, Content-MD5,
|
||||
// Content-Type, Date, If-Modified-Since, If-Match, If-None-Match,
|
||||
// If-Unmodified-Since: all empty. `Date` stays empty because `x-ms-date`
|
||||
// carries the timestamp and Azure then ignores this slot.
|
||||
for _ in 0..10 {
|
||||
string_to_sign.push('\n');
|
||||
}
|
||||
string_to_sign.push_str(header(headers, "range").unwrap_or_default());
|
||||
string_to_sign.push('\n');
|
||||
|
||||
// Canonicalized headers: every `x-ms-*` header, lowercased and sorted.
|
||||
let mut canonical_headers = BTreeMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = name.as_str();
|
||||
if let Some(rest) = name.strip_prefix("x-ms-")
|
||||
&& !rest.is_empty()
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
canonical_headers.insert(name.to_string(), value.trim().to_string());
|
||||
}
|
||||
}
|
||||
for (name, value) in &canonical_headers {
|
||||
string_to_sign.push_str(name);
|
||||
string_to_sign.push(':');
|
||||
string_to_sign.push_str(value);
|
||||
string_to_sign.push('\n');
|
||||
}
|
||||
|
||||
// Canonicalized resource: the account, the encoded path, then every query
|
||||
// parameter lowercased and sorted, with repeated values joined by commas.
|
||||
string_to_sign.push('/');
|
||||
string_to_sign.push_str(account);
|
||||
string_to_sign.push_str(url.path());
|
||||
let mut canonical_query: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
for (name, value) in url.query_pairs() {
|
||||
canonical_query
|
||||
.entry(name.to_ascii_lowercase())
|
||||
.or_default()
|
||||
.push(value.into_owned());
|
||||
}
|
||||
for (name, mut values) in canonical_query {
|
||||
values.sort();
|
||||
string_to_sign.push('\n');
|
||||
string_to_sign.push_str(&name);
|
||||
string_to_sign.push(':');
|
||||
string_to_sign.push_str(&values.join(","));
|
||||
}
|
||||
|
||||
let mut mac = HmacSha256::new_from_slice(key)
|
||||
.map_err(|_| SourceError::Other("azure account key has an unusable length".to_string()))?;
|
||||
mac.update(string_to_sign.as_bytes());
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user