mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-06 18:52:15 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b61a127464 | |||
| d997f01d8d | |||
| ee173d469f | |||
| 7b499bbbd8 | |||
| 5b134dbe2d | |||
| 29fe1c6d46 |
@@ -28,7 +28,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_build_rustfs_options.sh
|
||||
./scripts/test_docker_runtime_timezone.sh
|
||||
./scripts/test_entrypoint_credentials.sh
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
|
||||
@@ -110,36 +110,6 @@ docker compose up -d
|
||||
|
||||
---
|
||||
|
||||
## Docker Compose environment values
|
||||
|
||||
Use map syntax when quoting environment values in YAML:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
rustfs:
|
||||
image: rustfs/rustfs:latest
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- ./logs:/var/log/rustfs
|
||||
environment:
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_CONSOLE_ADDRESS: ":9001"
|
||||
RUSTFS_CONSOLE_ENABLE: "true"
|
||||
RUSTFS_OBS_LOGGER_LEVEL: error
|
||||
RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
|
||||
RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
|
||||
RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
|
||||
```
|
||||
|
||||
Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
|
||||
|
||||
With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Common Operations
|
||||
|
||||
### Cleaning Up
|
||||
|
||||
@@ -28,15 +28,6 @@ processors:
|
||||
check_interval: 1s
|
||||
limit_mib: 1024
|
||||
spike_limit_mib: 256
|
||||
transform/metrics:
|
||||
error_mode: ignore
|
||||
metric_statements:
|
||||
# RustFS process metrics already expose process_pid as a metric label.
|
||||
# Remove only the duplicate datapoint label and keep the shared resource
|
||||
# process.pid available to sibling metrics and lock histograms.
|
||||
- context: datapoint
|
||||
statements:
|
||||
- delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
|
||||
transform/logs:
|
||||
error_mode: ignore
|
||||
log_statements:
|
||||
@@ -121,7 +112,7 @@ service:
|
||||
exporters: [ otlp/tempo, otlp/jaeger ]
|
||||
metrics:
|
||||
receivers: [ otlp ]
|
||||
processors: [ transform/metrics, memory_limiter, batch ]
|
||||
processors: [ batch ]
|
||||
exporters: [ prometheus ]
|
||||
logs:
|
||||
receivers: [ otlp ]
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Run CLA Bot
|
||||
if: github.event_name != 'merge_group'
|
||||
uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
|
||||
uses: overtrue/cla-bot@f054da99d5777e47e1af9933737b9c0c5584be8d # v0.1.0
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
registry-token: ${{ steps.registry-token.outputs.token }}
|
||||
|
||||
@@ -107,6 +107,32 @@ jobs:
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
|
||||
|
||||
# Native multipart completion publishes its payload from a different
|
||||
# parent than its metadata, and abort renames under a precondition; both
|
||||
# need their own Windows handle discipline.
|
||||
- name: Test native multipart complete and abort publication
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
|
||||
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
|
||||
--nocapture
|
||||
|
||||
# Owned-undo reclaim removes a directory it pinned, copy heal persists
|
||||
# fault bytes through a handle, and metadata reads meet a directory.
|
||||
- name: Test native multipart reclaim, copy heal, and directory reads
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_multipart_owned_undo
|
||||
native_multipart_copy_heal
|
||||
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
|
||||
--nocapture
|
||||
|
||||
- name: Test directory removal fallback
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
|
||||
|
||||
- name: Test Windows path conversion
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
@@ -77,7 +77,9 @@ core protocol surface plus cross-protocol consistency:
|
||||
existing path returns an error, bad-password authentication is rejected
|
||||
- Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
|
||||
S3 write then SFTP read with SHA256 match
|
||||
- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
|
||||
- Cross-API directory visibility: SFTP-created sub-directory visible via S3
|
||||
ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
|
||||
directory entry
|
||||
|
||||
#### sftp_compliance (`test_sftp_compliance_suite`)
|
||||
|
||||
|
||||
@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
|
||||
|
||||
// --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
|
||||
// SFTP mkdir writes a zero-byte object at the trailing-slash key.
|
||||
// Storage encodes that key to the __XLDIR__ marker, and the S3
|
||||
// listing path decodes the marker back to a trailing-slash key.
|
||||
// SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
|
||||
// decodes that marker back to a trailing-slash key, so the asserted
|
||||
// pattern is "subdir_sftp/".
|
||||
info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
|
||||
let sftp_subdir_name = "subdir_sftp";
|
||||
let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
|
||||
@@ -422,21 +422,20 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
);
|
||||
info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
|
||||
|
||||
// --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
|
||||
// aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
|
||||
// key. The __XLDIR__ suffix is reserved on the client-key boundary, so
|
||||
// a direct put of that suffix is rejected. Storage encodes the slash
|
||||
// key, and SFTP readdir must show a directory entry.
|
||||
info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
|
||||
// --- Cross-API directory visibility: S3 marker, SFTP readdir ---
|
||||
// aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
|
||||
// __XLDIR__ suffix. SFTP readdir must decode the marker back to a
|
||||
// bare directory entry whose file_type reports as a directory.
|
||||
info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
|
||||
let s3_subdir_name = "subdir_s3";
|
||||
let s3_subdir_key = format!("{s3_subdir_name}/");
|
||||
let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
|
||||
s3.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&s3_subdir_key)
|
||||
.key(&s3_subdir_marker_key)
|
||||
.body(ByteStream::from_static(b""))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
|
||||
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
|
||||
let bucket_entries: Vec<(String, bool)> = sftp
|
||||
.read_dir(&bucket_path)
|
||||
.await?
|
||||
@@ -445,9 +444,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
|
||||
assert!(
|
||||
visible_via_sftp,
|
||||
"S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
|
||||
"S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
|
||||
);
|
||||
info!("PASS: S3 directory object visible to SFTP readdir as a directory");
|
||||
info!("PASS: S3 marker visible to SFTP readdir as a directory");
|
||||
|
||||
// --- Pre-cleanup of cross-protocol fixtures ---
|
||||
// Removes the new files and sub-directories so the existing rmdir
|
||||
|
||||
@@ -42,7 +42,6 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +50,6 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
|
||||
success: false,
|
||||
error_info: Some(error.into()),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +59,6 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
lock_result_from_error(format!("lock not found for release: {lock_id}"))
|
||||
@@ -167,7 +164,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -179,7 +175,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -189,7 +184,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -203,7 +197,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -215,7 +208,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -225,7 +217,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -239,7 +230,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -251,7 +241,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -261,7 +250,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -275,7 +263,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -285,13 +272,11 @@ impl NodeService for MinimalLockNodeService {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("can not refresh, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -996,32 +996,21 @@ fn internode_idempotent_read_retries() -> usize {
|
||||
static PREWARMED_PEERS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashSet<String>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new()));
|
||||
|
||||
fn claim_control_channel_prewarm(addr: &str) -> bool {
|
||||
let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
|
||||
return false;
|
||||
};
|
||||
prewarmed.insert(addr.to_owned())
|
||||
}
|
||||
|
||||
fn release_control_channel_prewarm(addr: &str) {
|
||||
if let Ok(mut prewarmed) = PREWARMED_PEERS.lock() {
|
||||
prewarmed.remove(addr);
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort background prewarm of a peer's control channel (grpc-optimization P3-1). Deduped per
|
||||
/// peer; a dial failure releases the marker so a later disk construction can retry the prewarm.
|
||||
/// peer; a dial failure just falls through to the existing lazy connect and recovery monitor.
|
||||
fn spawn_control_channel_prewarm(addr: String) {
|
||||
if !claim_control_channel_prewarm(&addr) {
|
||||
return;
|
||||
{
|
||||
let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
|
||||
return;
|
||||
};
|
||||
if !prewarmed.insert(addr.clone()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
match node_service_time_out_client_no_auth(&addr).await {
|
||||
Ok(_) => debug!(addr = %addr, "internode control channel prewarmed"),
|
||||
Err(err) => {
|
||||
release_control_channel_prewarm(&addr);
|
||||
debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)");
|
||||
}
|
||||
Err(err) => debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)"),
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -3691,13 +3680,7 @@ impl DiskAPI for RemoteDisk {
|
||||
);
|
||||
}
|
||||
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
let response_result = client.batch_read_version(request).await;
|
||||
if let Some(rpc_started) = rpc_started {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
|
||||
rpc_started.elapsed(),
|
||||
);
|
||||
}
|
||||
let response = match response_result {
|
||||
let response = match client.batch_read_version(request).await {
|
||||
Ok(response) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
response.into_inner()
|
||||
@@ -4712,10 +4695,7 @@ impl DiskAPI for RemoteDisk {
|
||||
read_multiple_req_bin: read_multiple_req_bin.into(),
|
||||
});
|
||||
|
||||
let rpc_started = Instant::now();
|
||||
let response = client.read_multiple(request).await;
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
|
||||
let response = response?.into_inner();
|
||||
let response = client.read_multiple(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
@@ -4766,7 +4746,6 @@ impl DiskAPI for RemoteDisk {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
|
||||
let rpc_started = Instant::now();
|
||||
let response = match client.write_all(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(err) => {
|
||||
@@ -4775,7 +4754,6 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
|
||||
|
||||
if !response.success {
|
||||
@@ -4873,7 +4851,6 @@ impl DiskAPI for RemoteDisk {
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
|
||||
let rpc_started = Instant::now();
|
||||
let response = match client.read_all(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(err) => {
|
||||
@@ -4882,8 +4859,6 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
|
||||
|
||||
if !response.success {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
@@ -12000,15 +11975,4 @@ mod tests {
|
||||
|
||||
accept_task.abort();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prewarm_claim_is_deduplicated_and_released_after_failure() {
|
||||
let addr = "http://prewarm-retry-test.invalid:9000";
|
||||
release_control_channel_prewarm(addr);
|
||||
assert!(claim_control_channel_prewarm(addr));
|
||||
assert!(!claim_control_channel_prewarm(addr));
|
||||
release_control_channel_prewarm(addr);
|
||||
assert!(claim_control_channel_prewarm(addr));
|
||||
release_control_channel_prewarm(addr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,12 +20,12 @@ use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult,
|
||||
LockReleaseStatus, PingRequest,
|
||||
PingRequest,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder,
|
||||
@@ -281,31 +281,6 @@ fn acquired_lock_ids(lock_ids: &[LockId], results: &[GenerallyLockResult]) -> Ve
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn cleanup_outcome(success: bool, error_info: Option<&str>, release_status: i32) -> Result<LockCleanupOutcome> {
|
||||
let status =
|
||||
LockReleaseStatus::try_from(release_status).map_err(|_| LockError::internal("Unknown remote lock release status"))?;
|
||||
match status {
|
||||
LockReleaseStatus::AlreadyAbsent if !success => Ok(LockCleanupOutcome::AlreadyAbsent),
|
||||
LockReleaseStatus::Released if success && error_info.is_none() => Ok(LockCleanupOutcome::Released),
|
||||
LockReleaseStatus::Unspecified => {
|
||||
if let Some(error) = error_info {
|
||||
Err(LockError::internal(error))
|
||||
} else if success {
|
||||
Ok(LockCleanupOutcome::Released)
|
||||
} else {
|
||||
Ok(LockCleanupOutcome::Unconfirmed)
|
||||
}
|
||||
}
|
||||
_ => Err(LockError::internal("Inconsistent remote lock release status")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum AcquisitionCleanup {
|
||||
Confirmed,
|
||||
Uncertain,
|
||||
}
|
||||
|
||||
/// Remote lock client implementation
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemoteClient {
|
||||
@@ -315,18 +290,13 @@ pub struct RemoteClient {
|
||||
impl RemoteClient {
|
||||
const ONLINE_CHECK_RESOURCE: &'static str = "health-lock-online";
|
||||
|
||||
fn record_release_ack(&self, lock_id: &LockId, outcome: LockCleanupOutcome) {
|
||||
fn record_release_ack(&self, lock_id: &LockId) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "acknowledged",
|
||||
result = match outcome {
|
||||
LockCleanupOutcome::Released => "released",
|
||||
LockCleanupOutcome::AlreadyAbsent => "already_absent",
|
||||
LockCleanupOutcome::Unconfirmed => "unconfirmed",
|
||||
},
|
||||
peer_kind = "remote",
|
||||
lock_id = %lock_id,
|
||||
peer = %self.addr,
|
||||
@@ -833,9 +803,7 @@ impl RemoteClient {
|
||||
Some(Box::new(move |response: Response<GenerallyLockResponse>| {
|
||||
Box::pin(async move {
|
||||
if response.get_ref().success {
|
||||
client
|
||||
.release_late_acquisitions(vec![lock_id], AcquisitionCleanup::Confirmed)
|
||||
.await;
|
||||
client.release_late_acquisitions(vec![lock_id]).await;
|
||||
}
|
||||
})
|
||||
}))
|
||||
@@ -847,9 +815,7 @@ impl RemoteClient {
|
||||
Box::pin(async move {
|
||||
let acquired = acquired_lock_ids(&lock_ids, &response.get_ref().results);
|
||||
if !acquired.is_empty() {
|
||||
client
|
||||
.release_late_acquisitions(acquired, AcquisitionCleanup::Confirmed)
|
||||
.await;
|
||||
client.release_late_acquisitions(acquired).await;
|
||||
}
|
||||
})
|
||||
}))
|
||||
@@ -859,98 +825,19 @@ impl RemoteClient {
|
||||
let client = self.clone();
|
||||
Some(Box::new(move |_| {
|
||||
Box::pin(async move {
|
||||
// Absence cannot settle an acquisition whose grant is still
|
||||
// uncertain. Keep the conservative retry schedule.
|
||||
client
|
||||
.release_late_acquisitions(lock_ids, AcquisitionCleanup::Uncertain)
|
||||
.await;
|
||||
client.release_late_acquisitions(lock_ids).await;
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
async fn request_release(&self, lock_id: &LockId) -> Result<GenerallyLockResponse> {
|
||||
let unlock_request = Self::create_unlock_request(lock_id);
|
||||
let request_string = serde_json::to_string(&unlock_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = unlock_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
Ok(self
|
||||
.execute_rpc(
|
||||
"release",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner())
|
||||
}
|
||||
|
||||
async fn request_release_batch(&self, lock_ids: &[LockId]) -> Result<BatchGenerallyLockResponse> {
|
||||
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(&unlock_requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: unlock_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
|
||||
Ok(self
|
||||
.execute_rpc(
|
||||
"release_batch",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock_batch(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner())
|
||||
}
|
||||
|
||||
fn cleanup_batch_results(&self, lock_ids: &[LockId], results: &[GenerallyLockResult]) -> Vec<LockCleanupOutcome> {
|
||||
lock_ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, lock_id)| {
|
||||
let outcome = results
|
||||
.get(idx)
|
||||
.and_then(|result| cleanup_outcome(result.success, result.error_info.as_deref(), result.release_status).ok())
|
||||
.unwrap_or(LockCleanupOutcome::Unconfirmed);
|
||||
if matches!(outcome, LockCleanupOutcome::Released | LockCleanupOutcome::AlreadyAbsent) {
|
||||
self.record_release_ack(lock_id, outcome);
|
||||
}
|
||||
outcome
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn cleanup_grants_batch(&self, lock_ids: &[LockId]) -> Result<Vec<LockCleanupOutcome>> {
|
||||
if lock_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let response = self.request_release_batch(lock_ids).await?;
|
||||
Ok(self.cleanup_batch_results(lock_ids, &response.results))
|
||||
}
|
||||
|
||||
/// A confirmed late grant can settle on authoritative absence. An
|
||||
/// uncertain acquisition must retain retries because release may race it.
|
||||
async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>, acquisition: AcquisitionCleanup) {
|
||||
/// A lock granted after its caller stopped waiting is an orphan until its
|
||||
/// lease expires; hand it back right away, best effort.
|
||||
async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>) {
|
||||
if !self.admit_late_release() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release("failed");
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "budget_exhausted",
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome = "budget_exhausted",
|
||||
@@ -968,27 +855,12 @@ impl RemoteClient {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
let released = match acquisition {
|
||||
AcquisitionCleanup::Confirmed => self.cleanup_grants_batch(&lock_ids).await,
|
||||
AcquisitionCleanup::Uncertain => self.release_locks_batch(&lock_ids).await.map(|results| {
|
||||
results
|
||||
.into_iter()
|
||||
.map(|released| {
|
||||
if released {
|
||||
LockCleanupOutcome::Released
|
||||
} else {
|
||||
LockCleanupOutcome::Unconfirmed
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
};
|
||||
match released {
|
||||
match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) => {
|
||||
lock_ids = lock_ids
|
||||
.into_iter()
|
||||
.zip(released)
|
||||
.filter_map(|(lock_id, outcome)| matches!(outcome, LockCleanupOutcome::Unconfirmed).then_some(lock_id))
|
||||
.filter_map(|(lock_id, released)| (!released).then_some(lock_id))
|
||||
.collect();
|
||||
if lock_ids.is_empty() {
|
||||
outcome = "released";
|
||||
@@ -1012,10 +884,10 @@ impl RemoteClient {
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_cleanup_completed",
|
||||
state = "late_grants_released",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
"Confirmed cleanup of late remote lock acquisitions"
|
||||
"Released remote locks granted after their caller stopped waiting"
|
||||
);
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
@@ -1024,25 +896,15 @@ impl RemoteClient {
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_cleanup_unconfirmed",
|
||||
state = "late_grants_unreleased",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not confirm cleanup of every late remote lock acquisition"
|
||||
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "warning_suppressed",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated late remote lock cleanup warning"
|
||||
);
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1490,37 +1352,68 @@ impl LockClient for RemoteClient {
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let resp = self.request_release(lock_id).await?;
|
||||
let unlock_request = Self::create_unlock_request(lock_id);
|
||||
let request_string = serde_json::to_string(&unlock_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = unlock_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc(
|
||||
"release",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
if resp.success {
|
||||
self.record_release_ack(lock_id, LockCleanupOutcome::Released);
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn cleanup_grant(&self, lock_id: &LockId) -> Result<LockCleanupOutcome> {
|
||||
let resp = self.request_release(lock_id).await?;
|
||||
let outcome = cleanup_outcome(resp.success, resp.error_info.as_deref(), resp.release_status)?;
|
||||
if matches!(outcome, LockCleanupOutcome::Released | LockCleanupOutcome::AlreadyAbsent) {
|
||||
self.record_release_ack(lock_id, outcome);
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
|
||||
if lock_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let resp = self.request_release_batch(lock_ids).await?;
|
||||
|
||||
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(&unlock_requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: unlock_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = self
|
||||
.execute_rpc(
|
||||
"release_batch",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock_batch(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
|
||||
Ok(lock_ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, lock_id)| {
|
||||
let result = resp.results.get(idx);
|
||||
if result.is_some_and(|result| result.success && result.error_info.is_none()) {
|
||||
self.record_release_ack(lock_id, LockCleanupOutcome::Released);
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
result.map(|result| result.success).unwrap_or(false)
|
||||
})
|
||||
@@ -1722,84 +1615,6 @@ impl LockClient for RemoteClient {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cleanup_outcome_requires_authoritative_absence() {
|
||||
assert_eq!(
|
||||
cleanup_outcome(false, Some("diagnostic text is not a protocol"), LockReleaseStatus::AlreadyAbsent.into())
|
||||
.expect("structured absence is authoritative"),
|
||||
LockCleanupOutcome::AlreadyAbsent
|
||||
);
|
||||
assert!(cleanup_outcome(false, Some("lock not found for release"), LockReleaseStatus::Unspecified.into()).is_err());
|
||||
assert_eq!(
|
||||
cleanup_outcome(false, None, LockReleaseStatus::Unspecified.into()).expect("legacy result is uncertain"),
|
||||
LockCleanupOutcome::Unconfirmed
|
||||
);
|
||||
for status in [LockReleaseStatus::Unspecified, LockReleaseStatus::Released] {
|
||||
assert_eq!(
|
||||
cleanup_outcome(true, None, status.into()).expect("positive release ACK"),
|
||||
LockCleanupOutcome::Released
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_outcome_rejects_inconsistent_and_unknown_statuses() {
|
||||
for (success, error, status) in [
|
||||
(true, None, LockReleaseStatus::AlreadyAbsent.into()),
|
||||
(false, None, LockReleaseStatus::Released.into()),
|
||||
(true, Some("backend error"), LockReleaseStatus::Released.into()),
|
||||
(true, None, 99),
|
||||
(false, Some("backend error"), 99),
|
||||
] {
|
||||
assert!(
|
||||
cleanup_outcome(success, error, status).is_err(),
|
||||
"must retain cleanup ownership for status {status}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_batch_results_preserves_each_lock_id_and_unknown_entries() {
|
||||
let client = RemoteClient::new("http://cleanup-batch-results-test".to_string());
|
||||
let lock_ids = (0..5)
|
||||
.map(|idx| LockId::new_unique(&ObjectKey::new("bucket", format!("cleanup-{idx}"))))
|
||||
.collect::<Vec<_>>();
|
||||
let results = vec![
|
||||
GenerallyLockResult {
|
||||
success: true,
|
||||
release_status: LockReleaseStatus::Released.into(),
|
||||
..Default::default()
|
||||
},
|
||||
GenerallyLockResult {
|
||||
success: false,
|
||||
error_info: Some("already released".to_string()),
|
||||
release_status: LockReleaseStatus::AlreadyAbsent.into(),
|
||||
..Default::default()
|
||||
},
|
||||
GenerallyLockResult {
|
||||
success: false,
|
||||
error_info: Some("lock not found for release".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
GenerallyLockResult {
|
||||
success: true,
|
||||
release_status: 99,
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
assert_eq!(
|
||||
client.cleanup_batch_results(&lock_ids, &results),
|
||||
vec![
|
||||
LockCleanupOutcome::Released,
|
||||
LockCleanupOutcome::AlreadyAbsent,
|
||||
LockCleanupOutcome::Unconfirmed,
|
||||
LockCleanupOutcome::Unconfirmed,
|
||||
LockCleanupOutcome::Unconfirmed,
|
||||
],
|
||||
"missing or unrecognized entries must not hide confirmed cleanup of other IDs"
|
||||
);
|
||||
}
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use rustfs_lock::{ObjectKey, types::LockPriority};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
@@ -481,14 +481,6 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
|
||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
@@ -529,14 +521,6 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
|
||||
match (opts.read_data, opts.healing, opts.incl_free_versions) {
|
||||
@@ -614,17 +598,6 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
@@ -717,14 +690,6 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
|
||||
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
|
||||
global_internode_metrics().record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
/// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
|
||||
/// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
|
||||
fn internode_rpc_large_payload_warn_bytes() -> usize {
|
||||
|
||||
@@ -315,7 +315,7 @@ pub(super) async fn encode_inline_pool_meta_replicas(
|
||||
.encode_with_shard_integrity(
|
||||
Cursor::new(Bytes::copy_from_slice(canonical)),
|
||||
&mut writers,
|
||||
crate::erasure::coding::encode::ShardWritePolicy::object(raw.len()),
|
||||
raw.len(),
|
||||
1,
|
||||
IntegrityEncodeMode::Inline(canonical.len()),
|
||||
protected,
|
||||
|
||||
@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
native_io(move || create_dir_all_std(&path)).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
fn is_dir_error(path: &Path, e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
// macOS: remove_file on a directory returns EPERM
|
||||
|| (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
|
||||
|| is_windows_dir_access_denied(path, e)
|
||||
}
|
||||
|
||||
/// Windows reports `remove_file` on a directory (or directory symlink) as
|
||||
/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
|
||||
/// failure without inspecting the entry itself.
|
||||
#[cfg(windows)]
|
||||
fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
|
||||
use std::os::windows::fs::FileTypeExt;
|
||||
|
||||
e.kind() == io::ErrorKind::PermissionDenied
|
||||
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| {
|
||||
let file_type = metadata.file_type();
|
||||
file_type.is_dir() || file_type.is_symlink_dir()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || remove_std(path)).await
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || match std::fs::remove_file(&path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir_all(path)).await
|
||||
}
|
||||
Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
match std::fs::remove_file(path.as_ref()) {
|
||||
let path = path.as_ref();
|
||||
match std::fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
|
||||
Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3159,7 +3159,6 @@ pub(super) fn has_rename_data_test_failpoints(dst_path: &str) -> bool {
|
||||
== Some(dst_path)
|
||||
|| os::has_dir_fsync_test_failures()
|
||||
|| os::file_sync_probe::is_active()
|
||||
|| os::prepared_publication_test_hooks::has_any_hooks()
|
||||
}
|
||||
|
||||
#[cfg(all(not(test), feature = "test-util", target_os = "linux"))]
|
||||
@@ -3170,7 +3169,6 @@ pub(super) fn has_rename_data_test_failpoints(dst_path: &str) -> bool {
|
||||
.iter()
|
||||
.any(|target| target == dst_path)
|
||||
|| os::has_dir_fsync_test_failures()
|
||||
|| os::prepared_publication_test_hooks::has_any_hooks()
|
||||
}
|
||||
|
||||
#[cfg(any(not(any(test, feature = "test-util")), not(target_os = "linux")))]
|
||||
@@ -4200,43 +4198,48 @@ fn reclaim_std_file_cache(file: &std::fs::File, reclaim_len: usize, reclaim_on_s
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn reclaim_std_file_cache(_file: &std::fs::File, _reclaim_len: usize, _reclaim_on_shutdown: bool) {}
|
||||
|
||||
const PRESYNCED_TMP_DATA_DIRS_PER_SHARD: usize = 64;
|
||||
const PRESYNCED_TMP_FILES_PER_SHARD: usize = 128;
|
||||
static PRESYNCED_TMP_DATA_DIRS: std::sync::LazyLock<ParkingLotMutex<HashSet<PathBuf>>> =
|
||||
std::sync::LazyLock::new(|| ParkingLotMutex::new(HashSet::new()));
|
||||
|
||||
static PRESYNCED_TMP_DATA_DIRS: std::sync::LazyLock<os::ShardedPathCache<()>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PRESYNCED_TMP_DATA_DIRS_PER_SHARD));
|
||||
static PRESYNCED_TMP_FILES: std::sync::LazyLock<ParkingLotMutex<HashSet<PathBuf>>> =
|
||||
std::sync::LazyLock::new(|| ParkingLotMutex::new(HashSet::new()));
|
||||
|
||||
static PRESYNCED_TMP_FILES: std::sync::LazyLock<os::ShardedPathCache<()>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PRESYNCED_TMP_FILES_PER_SHARD));
|
||||
|
||||
fn mark_presynced_tmp_data_dir(dir: impl AsRef<Path>) {
|
||||
PRESYNCED_TMP_DATA_DIRS.insert(dir.as_ref(), ());
|
||||
fn mark_presynced_tmp_data_dir(dir: PathBuf) {
|
||||
let mut set = PRESYNCED_TMP_DATA_DIRS.lock();
|
||||
if set.len() >= 4096 {
|
||||
set.clear();
|
||||
}
|
||||
set.insert(dir);
|
||||
}
|
||||
|
||||
fn unmark_presynced_tmp_data_dir(dir: &Path) -> bool {
|
||||
PRESYNCED_TMP_DATA_DIRS.remove(dir).is_some()
|
||||
PRESYNCED_TMP_DATA_DIRS.lock().remove(dir)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(in crate::disk) fn is_presynced_tmp_data_dir(dir: &Path) -> bool {
|
||||
PRESYNCED_TMP_DATA_DIRS.with_get(dir, |()| ()).is_some()
|
||||
PRESYNCED_TMP_DATA_DIRS.lock().contains(dir)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(in crate::disk) fn take_presynced_tmp_data_dir(dir: &Path) -> bool {
|
||||
PRESYNCED_TMP_DATA_DIRS.remove(dir).is_some()
|
||||
PRESYNCED_TMP_DATA_DIRS.lock().remove(dir)
|
||||
}
|
||||
|
||||
fn mark_presynced_tmp_file(file: impl AsRef<Path>) {
|
||||
PRESYNCED_TMP_FILES.insert(file.as_ref(), ());
|
||||
fn mark_presynced_tmp_file(file: PathBuf) {
|
||||
let mut set = PRESYNCED_TMP_FILES.lock();
|
||||
if set.len() >= 8192 {
|
||||
set.clear();
|
||||
}
|
||||
set.insert(file);
|
||||
}
|
||||
|
||||
fn unmark_presynced_tmp_file(file: &Path) {
|
||||
let _ = PRESYNCED_TMP_FILES.remove(file);
|
||||
PRESYNCED_TMP_FILES.lock().remove(file);
|
||||
}
|
||||
|
||||
fn take_presynced_tmp_file(file: &Path) -> bool {
|
||||
PRESYNCED_TMP_FILES.remove(file).is_some()
|
||||
PRESYNCED_TMP_FILES.lock().remove(file)
|
||||
}
|
||||
|
||||
fn open_lazy_local_file_std(file_path: &Path, volume_dir: &Path, _reclaim_on_shutdown: bool) -> std::io::Result<std::fs::File> {
|
||||
@@ -4322,7 +4325,7 @@ impl LazyLocalFileWriter {
|
||||
drop(file);
|
||||
if sync_on_close {
|
||||
if mark_dir_presynced && let Some(parent) = file_path.parent() {
|
||||
mark_presynced_tmp_data_dir(parent);
|
||||
mark_presynced_tmp_data_dir(parent.to_path_buf());
|
||||
}
|
||||
mark_presynced_tmp_file(file_path);
|
||||
}
|
||||
@@ -4344,7 +4347,7 @@ impl LazyLocalFileWriter {
|
||||
drop(file);
|
||||
if sync_on_close {
|
||||
if mark_dir_presynced && let Some(parent) = file_path.parent() {
|
||||
mark_presynced_tmp_data_dir(parent);
|
||||
mark_presynced_tmp_data_dir(parent.to_path_buf());
|
||||
}
|
||||
mark_presynced_tmp_file(file_path);
|
||||
}
|
||||
@@ -4472,7 +4475,7 @@ impl AsyncWrite for LazyLocalFileWriter {
|
||||
drop(file);
|
||||
if sync_on_close {
|
||||
if mark_dir_presynced && let Some(parent) = file_path.parent() {
|
||||
mark_presynced_tmp_data_dir(parent);
|
||||
mark_presynced_tmp_data_dir(parent.to_path_buf());
|
||||
}
|
||||
mark_presynced_tmp_file(file_path);
|
||||
}
|
||||
@@ -9989,7 +9992,7 @@ impl LocalDisk {
|
||||
if !std::fs::symlink_metadata(&receipt_path)?.is_file() {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidData, "delete receipt is not a regular file"));
|
||||
}
|
||||
std::fs::File::open(receipt_path)?
|
||||
os::open_file_for_sync_std(&receipt_path)?
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
@@ -14510,7 +14513,7 @@ impl DiskAPI for LocalDisk {
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let verify_and_sync = (|| -> std::io::Result<bool> {
|
||||
let mut file = std::fs::File::open(&source)?;
|
||||
let mut file = os::open_file_for_sync_std(&source)?;
|
||||
let mut header = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
|
||||
std::io::Read::read_exact(&mut file, &mut header)?;
|
||||
if header != integrity.index_header() {
|
||||
@@ -19751,6 +19754,9 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
// Only the Unix grouped dst dir fsync worker consults the injected
|
||||
// failure; the non-Unix worker syncs by path without the hook.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn rename_data_non_inline_dst_dir_fsync_group_commit_failure_rolls_back_fresh_put() {
|
||||
@@ -19839,6 +19845,9 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
// Only the Unix grouped dst dir fsync worker consults the injected
|
||||
// failure; the non-Unix worker syncs by path without the hook.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn rename_data_inline_dst_dir_fsync_group_commit_failure_rolls_back_fresh_put() {
|
||||
@@ -22180,7 +22189,7 @@ mod test {
|
||||
.get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}"))
|
||||
.expect("staged shard directory should resolve");
|
||||
assert!(
|
||||
PRESYNCED_TMP_DATA_DIRS.with_get(&staged_dir, |()| ()).is_none(),
|
||||
!PRESYNCED_TMP_DATA_DIRS.lock().contains(&staged_dir),
|
||||
"an unknown destination must defer payload sync until commit"
|
||||
);
|
||||
|
||||
@@ -22233,10 +22242,7 @@ mod test {
|
||||
let staged_dir = disk
|
||||
.get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}"))
|
||||
.expect("staged shard directory should resolve");
|
||||
assert!(
|
||||
PRESYNCED_TMP_DATA_DIRS.with_get(&staged_dir, |()| ()).is_some(),
|
||||
"the writer must pre-sync the shard"
|
||||
);
|
||||
assert!(PRESYNCED_TMP_DATA_DIRS.lock().contains(&staged_dir), "the writer must pre-sync the shard");
|
||||
assert!(!os::fsync_dir_recorder::was_fsynced(&staged_dir));
|
||||
|
||||
let mut fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None);
|
||||
@@ -22247,7 +22253,7 @@ mod test {
|
||||
.await
|
||||
.expect("rename_data should commit");
|
||||
assert!(
|
||||
PRESYNCED_TMP_DATA_DIRS.with_get(&staged_dir, |()| ()).is_none(),
|
||||
!PRESYNCED_TMP_DATA_DIRS.lock().contains(&staged_dir),
|
||||
"the fused commit must consume the pre-sync marker"
|
||||
);
|
||||
assert!(
|
||||
@@ -22438,141 +22444,6 @@ mod test {
|
||||
assert_eq!(resp.old_current_size, Some(OldCurrentSize::Present(1)));
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_rename_data_fused_overwrite_and_rollback_double_failure() {
|
||||
const CHILD: &str = "RUSTFS_TEST_FUSED_OVERWRITE_CHILD";
|
||||
const TEST: &str = "disk::local::test::test_rename_data_fused_overwrite_and_rollback_double_failure";
|
||||
|
||||
// Other tests install process-wide hooks that deliberately force rename_data off the fused path.
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", TEST, "--nocapture", "--test-threads=1"])
|
||||
.env(CHILD, "1")
|
||||
.env("RUSTFS_EXPERIMENTAL_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE", "false")
|
||||
.env("RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE", "false")
|
||||
.output()
|
||||
.expect("fused overwrite subprocess");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(output.status.success(), "{stdout}{stderr}");
|
||||
assert!(stdout.contains("FUSED_OVERWRITE_OK"), "fused overwrite test did not run: {stdout}");
|
||||
return;
|
||||
}
|
||||
|
||||
assert!(!os::dst_dir_fsync_group_commit_enabled());
|
||||
assert!(!os::file_fdatasync_group_commit_enabled());
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("utf8")).expect("endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
|
||||
let (bucket, object, version_id) = ("fused-overwrite-bucket", "fused-object", Uuid::nil());
|
||||
let (data_dir_v1, data_dir_v2) = (Uuid::new_v4(), Uuid::new_v4());
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
for (tmp, data_dir, payload) in [("tmp-v1", data_dir_v1, b"v1-d"), ("tmp-v2", data_dir_v2, b"v2-d")] {
|
||||
let mut w = disk
|
||||
.create_file(bucket, RUSTFS_META_TMP_BUCKET, &format!("{tmp}/{data_dir}/part.1"), 4)
|
||||
.await
|
||||
.expect("create shard");
|
||||
w.write_all(payload).await.expect("write");
|
||||
w.shutdown().await.expect("close");
|
||||
}
|
||||
let mut fi_v1 = test_file_info(object, version_id, Some(data_dir_v1), None);
|
||||
(fi_v1.size, fi_v1.parts[0].size, fi_v1.parts[0].actual_size) = (4, 4, 4);
|
||||
let resp1 = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v1", fi_v1, bucket, object)
|
||||
.await
|
||||
.expect("commit v1");
|
||||
assert_eq!((resp1.old_data_dir, resp1.rollback_data_dir), (None, None));
|
||||
let dst_object_dir = disk.io_get_object_path(bucket, object).expect("dst object path");
|
||||
let v1_meta_bytes = fs::read(dst_object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read v1 xl.meta");
|
||||
|
||||
let mut fi_v2 = test_file_info(object, version_id, Some(data_dir_v2), None);
|
||||
(fi_v2.size, fi_v2.parts[0].size, fi_v2.parts[0].actual_size) = (4, 4, 4);
|
||||
assert!(!has_rename_data_test_failpoints(object));
|
||||
let resp2 = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v2", fi_v2, bucket, object)
|
||||
.await
|
||||
.expect("fused overwrite v2");
|
||||
assert_eq!(
|
||||
(resp2.old_data_dir, resp2.rollback_data_dir, resp2.cleanup_data_dir),
|
||||
(Some(data_dir_v1), Some(data_dir_v1), Some(data_dir_v1))
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(dst_object_dir.join(data_dir_v1.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))
|
||||
.await
|
||||
.expect("v1 backup"),
|
||||
v1_meta_bytes
|
||||
);
|
||||
|
||||
fs::create_dir_all(dir.path().join(RUSTFS_META_TMP_BUCKET).join("tmp-v3"))
|
||||
.await
|
||||
.expect("create tmp-v3");
|
||||
let fi_v3 = test_file_info(object, version_id, None, Some(Bytes::from_static(b"v3-inline")));
|
||||
let resp3 = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v3", fi_v3, bucket, object)
|
||||
.await
|
||||
.expect("fused inline overwrite v3");
|
||||
assert_eq!(
|
||||
(resp3.old_data_dir, resp3.rollback_data_dir, resp3.cleanup_data_dir),
|
||||
(Some(data_dir_v2), Some(data_dir_v2), Some(data_dir_v2))
|
||||
);
|
||||
assert!(
|
||||
dst_object_dir
|
||||
.join(data_dir_v2.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP)
|
||||
.is_file()
|
||||
);
|
||||
|
||||
let dst_meta_path = dst_object_dir.join(STORAGE_FORMAT_FILE);
|
||||
let dst_vol_dir = disk.io_get_bucket_path(bucket).expect("bucket path");
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&dst_meta_path, None).await;
|
||||
let guard =
|
||||
commit::lock_rename_commit_directories(&dst_object_dir, &dst_object_dir, &dst_vol_dir, &disk.publication_root, lease)
|
||||
.await
|
||||
.expect("lock commit guard");
|
||||
let replacement_dir = dst_object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&replacement_dir).await.expect("create replacement_dir");
|
||||
fs::write(replacement_dir.join("part.1"), b"live").await.expect("write part");
|
||||
|
||||
os::fsync_dir_recorder::set_failure(&dst_object_dir, std::io::ErrorKind::Other);
|
||||
assert!(
|
||||
commit::fsync_inline_commit_std(
|
||||
&dst_meta_path,
|
||||
&dst_vol_dir,
|
||||
false,
|
||||
Some(Uuid::new_v4()),
|
||||
None,
|
||||
Some(&replacement_dir),
|
||||
&guard,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(replacement_dir.join("part.1").is_file());
|
||||
|
||||
os::fsync_dir_recorder::set_failure(&dst_object_dir, std::io::ErrorKind::Other);
|
||||
assert!(
|
||||
commit::fsync_inline_commit_std(
|
||||
&dst_meta_path,
|
||||
&dst_vol_dir,
|
||||
false,
|
||||
Some(data_dir_v1),
|
||||
None,
|
||||
Some(&replacement_dir),
|
||||
&guard,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!replacement_dir.exists());
|
||||
assert_eq!(fs::read(&dst_meta_path).await.expect("restored"), v1_meta_bytes);
|
||||
println!("FUSED_OVERWRITE_OK");
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1009: `observe_old_current_size` must mirror the pre-PUT
|
||||
/// `get_object_info` semantics bit for bit — latest version's
|
||||
/// `ObjectInfo.size` (0 for a delete-marker latest, which that lookup
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
//! Single-disk object rename publication and rollback. The shared execution core
|
||||
//! retains instrumentation, mutation leases, and commit guards through the syscall.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use super::has_rename_data_test_failpoints;
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
@@ -98,7 +96,7 @@ struct RenameCommitContext<'a> {
|
||||
data_paths: Option<&'a (PathBuf, PathBuf)>,
|
||||
}
|
||||
|
||||
pub(super) struct StagedRollbackBackup(PathBuf);
|
||||
struct StagedRollbackBackup(PathBuf);
|
||||
|
||||
impl StagedRollbackBackup {
|
||||
fn path(&self) -> &Path {
|
||||
@@ -200,7 +198,7 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
os::open_file_for_sync_std(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -235,13 +233,11 @@ fn rollback_inline_metadata_std(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fsync_inline_commit_std(
|
||||
fn fsync_inline_commit_std(
|
||||
dst: &Path,
|
||||
dst_volume_dir: &Path,
|
||||
destination_was_absent: bool,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_backup: Option<&StagedRollbackBackup>,
|
||||
new_data_dir: Option<&Path>,
|
||||
commit_guard: &os::RenameCommitGuard,
|
||||
) -> std::io::Result<()> {
|
||||
if let Some(dst_parent) = dst.parent() {
|
||||
@@ -251,10 +247,7 @@ pub(super) fn fsync_inline_commit_std(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
|
||||
if let Some(new_data_path) = new_data_dir {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
rollback_inline_metadata_std(dst, None, local_backup)?;
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -275,10 +268,7 @@ pub(super) fn fsync_inline_commit_std(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
|
||||
if let Some(new_data_path) = new_data_dir {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
rollback_inline_metadata_std(dst, None, local_backup)?;
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -837,7 +827,7 @@ impl LocalDisk {
|
||||
None
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() && has_rename_data_test_failpoints(&dst_path_str) {
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -860,9 +850,6 @@ impl LocalDisk {
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
xlmeta.add_version(fi_moved)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
|
||||
@@ -882,9 +869,8 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if (!no_inline || rollback_data_dir.is_none())
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (rollback_data_dir.is_some() || sync_commit_metadata || cfg!(test))
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& (sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
@@ -985,52 +971,9 @@ impl LocalDisk {
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
if let Err(err) =
|
||||
os::mkdir_all_below_existing_base_std(&backup_parent, &skip_parent, &publication_root_owned)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
let backup_path_guard = match guard.create_destination_directory_for_path_access(&backup_parent) {
|
||||
Ok(g) => g,
|
||||
Err(err) => return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into())),
|
||||
};
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
dst_buf.as_ref(),
|
||||
sync_commit_metadata,
|
||||
sync_commit_metadata,
|
||||
) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
} else if let (Some(old_data_dir), Some(staged_backup)) =
|
||||
(rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
|
||||
{
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
None,
|
||||
sync_commit_metadata,
|
||||
) {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
staged_rollback_path = None;
|
||||
}
|
||||
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
@@ -1058,16 +1001,15 @@ impl LocalDisk {
|
||||
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())?;
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
let err =
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
@@ -1080,16 +1022,14 @@ impl LocalDisk {
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
destination_was_absent,
|
||||
rollback_data_dir,
|
||||
staged_rollback_path.as_ref(),
|
||||
has_data_dir_path_owned
|
||||
.as_ref()
|
||||
.filter(|_| no_inline)
|
||||
.map(|(_, new_data_path)| new_data_path.as_path()),
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline {
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
@@ -1105,9 +1045,9 @@ impl LocalDisk {
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: None,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
@@ -1596,9 +1536,8 @@ impl LocalDisk {
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
let sync = tmp_meta_sync != SyncMode::None;
|
||||
async move {
|
||||
os::run_blocking_namespace_operation_with_sync(owned_mutation_lease, sync, move || {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
@@ -1606,7 +1545,7 @@ impl LocalDisk {
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
@@ -1658,12 +1597,13 @@ impl LocalDisk {
|
||||
);
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
let payload_guard = if multipart_publication.is_some() {
|
||||
Some(
|
||||
os::multipart_payload_commit_guard(
|
||||
&commit.directory_guard,
|
||||
src_data_path.parent().ok_or(DiskError::InvalidPath)?,
|
||||
commit.publication_root,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1672,9 +1612,9 @@ impl LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
let directory_guard = &commit.directory_guard;
|
||||
if let Err(err) = os::rename_all_with_commit_guard(
|
||||
src_data_path,
|
||||
@@ -1769,15 +1709,13 @@ impl LocalDisk {
|
||||
// sync finish. A detached spawn_blocking writer could survive
|
||||
// cancellation and later truncate a newer transaction's
|
||||
// deterministic rollback backup.
|
||||
let lease = commit.mutation_lease.clone();
|
||||
let sync = backup_sync != SyncMode::None;
|
||||
let write_result = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
backup_bytes.as_ref(),
|
||||
sync,
|
||||
backup_sync != SyncMode::None,
|
||||
backup_sync == SyncMode::FileAndDir,
|
||||
)
|
||||
})
|
||||
@@ -2217,9 +2155,7 @@ impl LocalDisk {
|
||||
&dst,
|
||||
&dst_volume_dir_for_commit,
|
||||
destination_was_absent,
|
||||
rollback_data_dir,
|
||||
staged_rollback_path.as_ref(),
|
||||
None,
|
||||
&rename_commit_guard_for_preparation,
|
||||
)
|
||||
{
|
||||
|
||||
@@ -271,9 +271,6 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
|
||||
compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
] {
|
||||
if canonical.metadata.is_empty() {
|
||||
break;
|
||||
}
|
||||
if !compat::contains_key_str(&canonical.metadata, suffix) {
|
||||
continue;
|
||||
}
|
||||
@@ -304,24 +301,22 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
|
||||
}
|
||||
compat::insert_str(&mut canonical.metadata, suffix, value);
|
||||
}
|
||||
fn sort_json_in_place(value: &mut serde_json::Value) {
|
||||
fn ordered(value: serde_json::Value) -> serde_json::Value {
|
||||
match value {
|
||||
serde_json::Value::Object(map) => {
|
||||
map.sort_keys();
|
||||
map.values_mut().for_each(sort_json_in_place);
|
||||
let sorted: std::collections::BTreeMap<_, _> = map.into_iter().collect();
|
||||
serde_json::Value::Object(sorted.into_iter().map(|(key, value)| (key, ordered(value))).collect())
|
||||
}
|
||||
serde_json::Value::Array(values) => values.iter_mut().for_each(sort_json_in_place),
|
||||
_ => {}
|
||||
serde_json::Value::Array(values) => serde_json::Value::Array(values.into_iter().map(ordered).collect()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
let mut json_value = serde_json::to_value(&canonical)?;
|
||||
sort_json_in_place(&mut json_value);
|
||||
Ok(crate::disk::DeleteRecoveryVersionIdentity {
|
||||
version_id: canonical.version_id,
|
||||
mod_time: canonical.mod_time,
|
||||
data_dir: canonical.data_dir,
|
||||
deleted: canonical.deleted,
|
||||
metadata_digest: digest(&serde_json::to_vec(&json_value)?),
|
||||
metadata_digest: digest(&serde_json::to_vec(&ordered(serde_json::to_value(canonical)?))?),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -432,13 +427,6 @@ fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result<Vec<u8>> {
|
||||
if bytes.len() > usize::try_from(MAX_METADATA).map_err(|_| DiskError::FileCorrupt)? - 128 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// MessagePack encoding of `DeleteRecoveryRpcReply::Inspected` is more compact
|
||||
// than JSON; only construct and serialize `projected` when the JSON size plus
|
||||
// `current_versions` and fixed envelope could approach the RPC payload limit.
|
||||
let rpc_budget = rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES.saturating_sub(4096);
|
||||
if bytes.len().saturating_mul(2).saturating_add(4096) <= rpc_budget {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let projected = crate::disk::DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
id: DeleteTransactionId {
|
||||
@@ -455,7 +443,7 @@ fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result<Vec<u8>> {
|
||||
explicit_delete_safe: false,
|
||||
fingerprint: [0; 32],
|
||||
});
|
||||
if rmp_serde::to_vec_named(&projected)?.len() > rpc_budget {
|
||||
if rmp_serde::to_vec_named(&projected)?.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES - 4096 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(bytes)
|
||||
@@ -7180,86 +7168,6 @@ mod tests {
|
||||
assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
|
||||
assert!(!root.exists(), "empty object directory must be pruned");
|
||||
}
|
||||
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_receipt_formatting_and_post_finished_unlink_interruption() {
|
||||
let (_temp, disk, info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
let mut v1 = info.clone();
|
||||
v1.version_id = Some(Uuid::new_v4());
|
||||
let mut v2 = info.clone();
|
||||
v2.version_id = Some(Uuid::new_v4());
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(v1.clone()).expect("add v1");
|
||||
meta.add_version(v2.clone()).expect("add v2");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let mut rev_info = v1.clone();
|
||||
rev_info
|
||||
.metadata
|
||||
.extend([("x-amz-meta-z".into(), "1".into()), ("x-amz-meta-a".into(), "2".into())]);
|
||||
let mut fwd_info = v1.clone();
|
||||
fwd_info
|
||||
.metadata
|
||||
.extend([("x-amz-meta-a".into(), "2".into()), ("x-amz-meta-z".into(), "1".into())]);
|
||||
assert_eq!(
|
||||
generation(rev_info).expect("rev").metadata_digest,
|
||||
generation(fwd_info).expect("fwd").metadata_digest
|
||||
);
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
disk.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused versioned delete preparation"),
|
||||
Some(())
|
||||
);
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish")
|
||||
.expect("versioned target satisfied");
|
||||
assert!(sole_transaction && target_satisfied && journal.target_digest.is_some());
|
||||
|
||||
let rollback = root.join(transaction.to_string());
|
||||
// Real finalizer failure at the second fsync_dir_std(&rollback) (after [TARGET, PREPARED, FINISHED] unlink)
|
||||
// must leave a durable finished=true receipt that inspects as CommitFinished.
|
||||
let rollback_for_hook = rollback.clone();
|
||||
let _hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::Remove,
|
||||
&rollback.join(FINISHED),
|
||||
move || {
|
||||
os::fsync_dir_recorder::set_failure(&rollback_for_hook, std::io::ErrorKind::Other);
|
||||
},
|
||||
);
|
||||
assert!(
|
||||
disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!rollback.join(FINISHED).exists());
|
||||
assert_eq!(
|
||||
disk.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("post-FINISHED-unlink failure")
|
||||
.state,
|
||||
DeleteRecoveryState::CommitFinished
|
||||
);
|
||||
assert!(
|
||||
disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("finalize")
|
||||
);
|
||||
assert!(!rollback.exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1517,7 +1517,7 @@ impl LocalDisk {
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
if !parts.is_empty() {
|
||||
let anchor_path = directory.join(RETIREMENT_ANCHOR);
|
||||
let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
|
||||
@@ -1550,7 +1550,13 @@ impl LocalDisk {
|
||||
}
|
||||
// An anchor-less empty DD is only rmdir'ed. A new child makes
|
||||
// this fail closed; no recursive removal can consume its bytes.
|
||||
// A Windows pin refuses DELETE sharing, so the leaf pin is
|
||||
// released first; the object pin still holds the parent.
|
||||
#[cfg(windows)]
|
||||
drop(directory_pin);
|
||||
std::fs::remove_dir(&directory)?;
|
||||
#[cfg(not(windows))]
|
||||
drop(directory_pin);
|
||||
if sync {
|
||||
os::fsync_dir_std(&object_path)?;
|
||||
}
|
||||
|
||||
@@ -398,9 +398,7 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
let sync_lane = sync_directory && incoming.data.is_none() && !incoming.is_remote();
|
||||
let lease = Arc::clone(&staging.lease);
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation_with_sync(lease, sync_lane, move |cancelled| {
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
|
||||
+467
-153
@@ -610,6 +610,26 @@ pub(crate) mod prepare_rename_test_hooks {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
/// Hooks keyed by a directory component name, run before a Windows
|
||||
/// directory pin retries a transient sharing conflict on it.
|
||||
#[cfg(windows)]
|
||||
static BEFORE_COMPONENT_RETRY: LazyLock<Mutex<HashMap<std::ffi::OsString, Hook>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn install_before_component_retry(component: &std::ffi::OsStr, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_COMPONENT_RETRY.lock().insert(component.to_os_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn run_before_component_retry(component: &std::ffi::OsStr) {
|
||||
let hook = BEFORE_COMPONENT_RETRY.lock().remove(component);
|
||||
if let Some(hook) = hook {
|
||||
let _pause = super::enter_blocking_test_hook();
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fsync a directory so recently created or renamed entries survive power loss.
|
||||
@@ -644,6 +664,16 @@ pub(crate) fn fsync_directory_handle_std(dir: &Path, opened: Option<&std::fs::Fi
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open an existing file to read it and flush it. Windows flushes only a
|
||||
/// handle with write access, so the file is opened read-write there; other
|
||||
/// platforms keep the read-only open that `sync_all` already accepts.
|
||||
pub(crate) fn open_file_for_sync_std(path: &Path) -> io::Result<std::fs::File> {
|
||||
#[cfg(windows)]
|
||||
return std::fs::OpenOptions::new().read(true).write(true).open(path);
|
||||
#[cfg(not(windows))]
|
||||
std::fs::File::open(path)
|
||||
}
|
||||
|
||||
/// Async wrapper around [`fsync_dir_std`]; runs the blocking Unix fsync off the runtime.
|
||||
pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dir_with_owner(dir.as_ref(), None).await
|
||||
@@ -3808,7 +3838,7 @@ pub(crate) async fn rename_reconciled_metadata(
|
||||
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
|
||||
) -> Result<()> {
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
|
||||
@@ -4584,7 +4614,7 @@ pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &
|
||||
let Some(source_parent) = src_file_path.parent() else {
|
||||
return false;
|
||||
};
|
||||
let source_parent_guard = match lock_windows_directory_tree(source_parent, None, publication_root) {
|
||||
let source_parent_guard = match lock_windows_directory_tree(source_parent, publication_root) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return err.kind() == io::ErrorKind::NotFound,
|
||||
};
|
||||
@@ -4808,23 +4838,6 @@ pub(crate) async fn run_cancellable_blocking_namespace_operation<T: Send + 'stat
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn run_cancellable_blocking_namespace_operation_with_sync<T: Send + 'static>(
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
sync: bool,
|
||||
operation: impl FnOnce(&dyn Fn() -> bool) -> io::Result<T> + Send + 'static,
|
||||
) -> io::Result<T> {
|
||||
if !sync {
|
||||
return run_cancellable_blocking_namespace_operation(lease, operation).await;
|
||||
}
|
||||
let queue = Arc::clone(&lease.blocking_queue);
|
||||
queue
|
||||
.run_fsync_cancellable(move |is_cancelled| {
|
||||
let _lease = lease;
|
||||
operation(is_cancelled)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Admit one strict inline commit under the disk sync limit. The caller already
|
||||
/// owns the namespace lease, establishing namespace -> disk ordering. Holding
|
||||
/// admission across adjacent durability barriers prevents one transaction from
|
||||
@@ -5023,7 +5036,7 @@ fn prepare_rename_with_retry(
|
||||
open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
|
||||
(source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity)
|
||||
} else {
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, destination_parent, publication_root)?;
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, publication_root)?;
|
||||
let (source_identity_anchor, expected_source_identity) =
|
||||
open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
|
||||
let parent_guard = prepare_destination_parent()?;
|
||||
@@ -5444,7 +5457,9 @@ impl PublicationRoot {
|
||||
pub(crate) fn open_readonly(&self, path: &Path) -> io::Result<std::fs::File> {
|
||||
#[cfg(target_os = "linux")]
|
||||
return self.open_relative(path, rustix::fs::OFlags::RDONLY, rustix::fs::Mode::empty());
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
#[cfg(windows)]
|
||||
return open_windows_readonly(path);
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
std::fs::File::open(path)
|
||||
}
|
||||
|
||||
@@ -5908,14 +5923,11 @@ impl RenameCommitGuard {
|
||||
}
|
||||
|
||||
/// Reopen a destination tree for handle-relative child publication.
|
||||
/// Ancestors remain write-exclusive while the final parent shares writes
|
||||
/// required by the kernel's relative rename. Delete sharing stays omitted
|
||||
/// throughout, so every retained directory identity remains pinned.
|
||||
/// Every component is pinned by `open_windows_directory_component`.
|
||||
fn destination_directory_guard(&self, directory: &Path, create_missing: bool) -> io::Result<RenameDestinationPathGuard> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Wdk::Storage::FileSystem::{FILE_OPEN, FILE_OPEN_IF};
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
|
||||
let relative = directory.strip_prefix(&self.destination_parent).map_err(|_| {
|
||||
io::Error::new(
|
||||
@@ -5948,35 +5960,17 @@ impl RenameCommitGuard {
|
||||
let parent = handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows destination guard lost its parent handle"))?;
|
||||
let mut relative_components = relative
|
||||
.components()
|
||||
.filter_map(|component| match component {
|
||||
Component::Normal(component) => Some(component),
|
||||
_ => None,
|
||||
})
|
||||
.peekable();
|
||||
let destination_parent_share = if relative_components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
handles.push(open_windows_relative_directory_component(
|
||||
parent,
|
||||
component,
|
||||
FILE_OPEN,
|
||||
destination_parent_share,
|
||||
)?);
|
||||
while let Some(component) = relative_components.next() {
|
||||
handles.push(open_windows_directory_component(parent, component, FILE_OPEN)?);
|
||||
let relative_components = relative.components().filter_map(|component| match component {
|
||||
Component::Normal(component) => Some(component),
|
||||
_ => None,
|
||||
});
|
||||
for component in relative_components {
|
||||
let parent = handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows destination path guard lost its parent handle"))?;
|
||||
let disposition = if create_missing { FILE_OPEN_IF } else { FILE_OPEN };
|
||||
let share_access = if relative_components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
handles.push(open_windows_relative_directory_component(parent, component, disposition, share_access)?);
|
||||
handles.push(open_windows_directory_component(parent, component, disposition)?);
|
||||
}
|
||||
Ok(RenameDestinationPathGuard {
|
||||
directory: directory.to_path_buf(),
|
||||
@@ -6020,18 +6014,47 @@ pub(crate) fn multipart_payload_commit_guard_std(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn multipart_payload_commit_guard_std(
|
||||
metadata_guard: &RenameCommitGuard,
|
||||
source_parent: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
// Pin the payload staging tree like a distinct-tree metadata source; the
|
||||
// target stays the exact parent handle the metadata transaction holds.
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, publication_root)?;
|
||||
Ok(RenameCommitGuard {
|
||||
source_parent: source_parent.to_path_buf(),
|
||||
destination_parent: metadata_guard.destination_parent.clone(),
|
||||
source_parent_guard,
|
||||
destination_parent_guard: metadata_guard.destination_parent_guard.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Native MPU metadata and payload have distinct source parents. Reuse the
|
||||
/// exact target identity already held by the metadata transaction, while
|
||||
/// independently pinning the private payload source under the same root.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
pub(crate) async fn multipart_payload_commit_guard(
|
||||
metadata_guard: &RenameCommitGuard,
|
||||
source_parent: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let metadata_guard = metadata_guard.clone();
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
run_blocking_namespace_operation(lease, move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)).await
|
||||
#[cfg(target_os = "linux")]
|
||||
let operation = {
|
||||
// The Linux guard already carries its publication root.
|
||||
let _ = publication_root;
|
||||
move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)
|
||||
};
|
||||
#[cfg(windows)]
|
||||
let operation = {
|
||||
let publication_root = publication_root.clone();
|
||||
move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent, &publication_root)
|
||||
};
|
||||
run_blocking_namespace_operation(lease, operation).await
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_rename_commit_guard(
|
||||
@@ -6073,20 +6096,16 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
// A same-directory rename must use one shared-write parent handle:
|
||||
// retaining a second read-only-share handle would block the kernel's
|
||||
// relative target open. Delete sharing stays excluded, so identity is
|
||||
// still pinned. Distinct source trees remain strict.
|
||||
// A same-directory rename reuses one guard for both sides. Every pin
|
||||
// shares writes and excludes delete; see
|
||||
// `open_windows_directory_component`.
|
||||
let same_parent = publication_root.relative_path(source_parent)? == publication_root.relative_path(destination_parent)?;
|
||||
let (source_parent_guard, destination_parent_guard) = if same_parent {
|
||||
let destination_parent_guard =
|
||||
mkdir_all_below_existing_base_std(destination_parent, destination_base, publication_root)?;
|
||||
(destination_parent_guard.clone(), destination_parent_guard)
|
||||
} else {
|
||||
// The source parent also hosts private rollback staging files.
|
||||
// Their handle-relative publication needs write sharing on this
|
||||
// final directory while delete sharing remains excluded.
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, Some(source_parent), publication_root)?;
|
||||
let source_parent_guard = lock_windows_directory_tree(source_parent, publication_root)?;
|
||||
let destination_parent_guard =
|
||||
mkdir_all_below_existing_base_std(destination_parent, destination_base, publication_root)?;
|
||||
(source_parent_guard, destination_parent_guard)
|
||||
@@ -6106,6 +6125,28 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
}
|
||||
}
|
||||
|
||||
/// Unix opens a directory read-only and lets the reader classify it, while
|
||||
/// Windows refuses the plain open with ACCESS_DENIED. Reopen only an actual
|
||||
/// directory with backup semantics so the same `is_dir` checks apply.
|
||||
#[cfg(windows)]
|
||||
fn open_windows_readonly(path: &Path) -> io::Result<std::fs::File> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS;
|
||||
|
||||
match std::fs::File::open(path) {
|
||||
Err(error)
|
||||
if error.kind() == io::ErrorKind::PermissionDenied
|
||||
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir()) =>
|
||||
{
|
||||
std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
|
||||
.open(path)
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn open_windows_publication_root(path: &Path) -> io::Result<(PathBuf, WindowsDirectoryHandle)> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
@@ -6292,22 +6333,12 @@ fn windows_file_attribute_tag(
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn lock_windows_directory_tree(
|
||||
path: &Path,
|
||||
shared_write_ancestor: Option<&Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
fn lock_windows_directory_tree(path: &Path, publication_root: &PublicationRoot) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
use windows_sys::Wdk::Storage::FileSystem::FILE_OPEN;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
|
||||
let relative = publication_root.relative_path(path)?;
|
||||
let shared_write_relative = shared_write_ancestor
|
||||
.map(|ancestor| publication_root.relative_path(ancestor))
|
||||
.transpose()?
|
||||
.filter(|ancestor| relative.starts_with(*ancestor));
|
||||
let mut handles = Vec::with_capacity(relative.components().count().saturating_add(1));
|
||||
handles.push(publication_root.directory.clone());
|
||||
let mut opened_relative = PathBuf::new();
|
||||
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
@@ -6322,26 +6353,13 @@ fn lock_windows_directory_tree(
|
||||
let parent = handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows directory guard lost its root handle"))?;
|
||||
opened_relative.push(component);
|
||||
let child = if shared_write_relative.is_some_and(|ancestor| opened_relative.as_path() == ancestor) {
|
||||
// A handle-relative rename opens the target parent for write. When
|
||||
// that parent is also a source ancestor, this source-side handle
|
||||
// must share write access or the transaction blocks itself. Delete
|
||||
// sharing remains omitted, so the directory identity stays pinned.
|
||||
open_windows_relative_directory_component(parent, component, FILE_OPEN, FILE_SHARE_READ | FILE_SHARE_WRITE)?
|
||||
} else {
|
||||
open_windows_directory_component(parent, component, FILE_OPEN)?
|
||||
};
|
||||
handles.push(child);
|
||||
handles.push(open_windows_directory_component(parent, component, FILE_OPEN)?);
|
||||
}
|
||||
|
||||
Ok(ExistingBaseDirectoryGuard::new(handles))
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
// SAFETY: the object attributes borrow a checked UTF-16 component and live
|
||||
// parent handle for the duration of the synchronous NtCreateFile call.
|
||||
#[allow(unsafe_code)]
|
||||
fn open_windows_relative(
|
||||
parent: &winapi_util::Handle,
|
||||
component: &std::ffi::OsStr,
|
||||
@@ -6352,6 +6370,64 @@ fn open_windows_relative(
|
||||
file_attributes: u32,
|
||||
dont_reparse: bool,
|
||||
) -> io::Result<winapi_util::Handle> {
|
||||
open_windows_relative_with_status(
|
||||
parent,
|
||||
component,
|
||||
&WindowsRelativeOpen {
|
||||
desired_access,
|
||||
share_access,
|
||||
create_disposition,
|
||||
create_options,
|
||||
file_attributes,
|
||||
dont_reparse,
|
||||
},
|
||||
)
|
||||
.map_err(|failure| failure.error)
|
||||
}
|
||||
|
||||
/// `NtCreateFile` arguments for one handle-relative open.
|
||||
#[cfg(windows)]
|
||||
struct WindowsRelativeOpen {
|
||||
desired_access: u32,
|
||||
share_access: u32,
|
||||
create_disposition: u32,
|
||||
create_options: u32,
|
||||
file_attributes: u32,
|
||||
dont_reparse: bool,
|
||||
}
|
||||
|
||||
/// A failed relative open that keeps the NTSTATUS, which tells transient
|
||||
/// states such as a pending delete apart from errors that share its Win32 code.
|
||||
#[cfg(windows)]
|
||||
struct WindowsRelativeOpenError {
|
||||
status: Option<windows_sys::Win32::Foundation::NTSTATUS>,
|
||||
error: io::Error,
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
impl From<io::Error> for WindowsRelativeOpenError {
|
||||
fn from(error: io::Error) -> Self {
|
||||
Self { status: None, error }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
// SAFETY: the object attributes borrow a checked UTF-16 component and live
|
||||
// parent handle for the duration of the synchronous NtCreateFile call.
|
||||
#[allow(unsafe_code)]
|
||||
fn open_windows_relative_with_status(
|
||||
parent: &winapi_util::Handle,
|
||||
component: &std::ffi::OsStr,
|
||||
request: &WindowsRelativeOpen,
|
||||
) -> std::result::Result<winapi_util::Handle, WindowsRelativeOpenError> {
|
||||
let &WindowsRelativeOpen {
|
||||
desired_access,
|
||||
share_access,
|
||||
create_disposition,
|
||||
create_options,
|
||||
file_attributes,
|
||||
dont_reparse,
|
||||
} = request;
|
||||
use std::{
|
||||
mem::size_of,
|
||||
os::windows::{ffi::OsStrExt, io::AsRawHandle, io::FromRawHandle},
|
||||
@@ -6366,10 +6442,7 @@ fn open_windows_relative(
|
||||
|
||||
let mut name = component.encode_wide().collect::<Vec<_>>();
|
||||
if name.is_empty() || name.contains(&0) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"relative Windows file name is empty or contains a NUL",
|
||||
));
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "relative Windows file name is empty or contains a NUL").into());
|
||||
}
|
||||
let name_bytes = name
|
||||
.len()
|
||||
@@ -6409,13 +6482,17 @@ fn open_windows_relative(
|
||||
)
|
||||
};
|
||||
if status < 0 {
|
||||
return match i32::try_from(unsafe { RtlNtStatusToDosError(status) }) {
|
||||
Ok(code) => Err(io::Error::from_raw_os_error(code)),
|
||||
Err(_) => Err(io::Error::other(format!("Windows relative open failed with NTSTATUS {status:#x}"))),
|
||||
let error = match i32::try_from(unsafe { RtlNtStatusToDosError(status) }) {
|
||||
Ok(code) => io::Error::from_raw_os_error(code),
|
||||
Err(_) => io::Error::other(format!("Windows relative open failed with NTSTATUS {status:#x}")),
|
||||
};
|
||||
return Err(WindowsRelativeOpenError {
|
||||
status: Some(status),
|
||||
error,
|
||||
});
|
||||
}
|
||||
if handle.is_null() {
|
||||
return Err(io::Error::other("Windows relative open returned an invalid handle"));
|
||||
return Err(io::Error::other("Windows relative open returned an invalid handle").into());
|
||||
}
|
||||
|
||||
Ok(unsafe { winapi_util::Handle::from_raw_handle(handle) })
|
||||
@@ -6650,7 +6727,7 @@ pub(crate) fn compare_and_update_control_file(
|
||||
let parent = file_path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
|
||||
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
|
||||
let parent_guard = lock_windows_directory_tree(parent, publication_root)?;
|
||||
let lock = open_windows_relative(
|
||||
parent_guard.last_handle()?,
|
||||
std::ffi::OsStr::new(".rustfs-cas.lock"),
|
||||
@@ -6714,41 +6791,78 @@ pub(crate) fn compare_and_update_control_file(
|
||||
Ok(ConditionalFileUpdate::Updated)
|
||||
}
|
||||
|
||||
/// How often a directory pin retries a transient delete-access conflict.
|
||||
/// Six doubling delays from 1 ms wait about 63 ms in total, which outlasts a
|
||||
/// prune whose thread is preempted while it holds delete access.
|
||||
#[cfg(windows)]
|
||||
const WINDOWS_DIRECTORY_PIN_RETRIES: u32 = 6;
|
||||
#[cfg(windows)]
|
||||
const WINDOWS_DIRECTORY_PIN_FIRST_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(1);
|
||||
|
||||
/// Pin one directory component of a guarded path.
|
||||
///
|
||||
/// The handle shares reads and writes but never delete: renaming or removing
|
||||
/// a directory needs delete access to it, so omitting delete sharing pins its
|
||||
/// identity. Write sharing must stay granted on every component, ancestors
|
||||
/// included, because the kernel opens the target parent for write during a
|
||||
/// rename. A directory pinned here as one object's ancestor is also the
|
||||
/// publication parent of a nested object (`a/b` holds `a/b/c`), and a
|
||||
/// read-only share would make that unrelated rename fail with
|
||||
/// ERROR_SHARING_VIOLATION. Write sharing does not unpin an ancestor: a
|
||||
/// pinned ancestor always contains its pinned child, and NTFS sets a
|
||||
/// name-surrogate reparse point only on an empty directory.
|
||||
///
|
||||
/// Excluding delete also means the open fails while another handle holds
|
||||
/// delete access. Pruning an empty prefix directory takes that access only
|
||||
/// briefly: `RemoveDirectoryW` opens the directory for delete and closes it
|
||||
/// again, whether or not the directory turns out to be empty. A sharing
|
||||
/// violation or a pending delete is therefore retried a bounded number of
|
||||
/// times. After a removal finishes, the retry sees the directory as missing
|
||||
/// and follows `create_disposition`, so `FILE_OPEN` still reports it missing
|
||||
/// and `FILE_OPEN_IF` creates it again.
|
||||
#[cfg(windows)]
|
||||
fn open_windows_directory_component(
|
||||
parent: &WindowsDirectoryHandle,
|
||||
component: &std::ffi::OsStr,
|
||||
create_disposition: u32,
|
||||
) -> io::Result<WindowsDirectoryHandle> {
|
||||
use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ;
|
||||
|
||||
open_windows_relative_directory_component(parent, component, create_disposition, FILE_SHARE_READ)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn open_windows_relative_directory_component(
|
||||
parent: &WindowsDirectoryHandle,
|
||||
component: &std::ffi::OsStr,
|
||||
create_disposition: u32,
|
||||
share_access: u32,
|
||||
) -> io::Result<WindowsDirectoryHandle> {
|
||||
use windows_sys::{
|
||||
Wdk::Storage::FileSystem::{FILE_DIRECTORY_FILE, FILE_OPEN_REPARSE_POINT},
|
||||
Win32::Storage::FileSystem::{
|
||||
FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_TRAVERSE,
|
||||
Win32::{
|
||||
Foundation::{STATUS_DELETE_PENDING, STATUS_SHARING_VIOLATION},
|
||||
Storage::FileSystem::{
|
||||
FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE,
|
||||
FILE_TRAVERSE,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
let anchor = open_windows_relative(
|
||||
&parent.handle,
|
||||
component,
|
||||
FILE_TRAVERSE | FILE_READ_ATTRIBUTES,
|
||||
share_access,
|
||||
let request = WindowsRelativeOpen {
|
||||
desired_access: FILE_TRAVERSE | FILE_READ_ATTRIBUTES,
|
||||
share_access: FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
create_disposition,
|
||||
FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT,
|
||||
FILE_ATTRIBUTE_DIRECTORY,
|
||||
true,
|
||||
)?;
|
||||
create_options: FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT,
|
||||
file_attributes: FILE_ATTRIBUTE_DIRECTORY,
|
||||
dont_reparse: true,
|
||||
};
|
||||
let mut retry_delay = WINDOWS_DIRECTORY_PIN_FIRST_RETRY_DELAY;
|
||||
let mut retries = 0;
|
||||
let anchor = loop {
|
||||
match open_windows_relative_with_status(&parent.handle, component, &request) {
|
||||
Ok(anchor) => break anchor,
|
||||
Err(WindowsRelativeOpenError {
|
||||
status: Some(STATUS_SHARING_VIOLATION | STATUS_DELETE_PENDING),
|
||||
..
|
||||
}) if retries < WINDOWS_DIRECTORY_PIN_RETRIES => {
|
||||
#[cfg(test)]
|
||||
prepare_rename_test_hooks::run_before_component_retry(component);
|
||||
std::thread::sleep(retry_delay);
|
||||
retry_delay = retry_delay.saturating_mul(2);
|
||||
retries += 1;
|
||||
}
|
||||
Err(failure) => return Err(failure.error),
|
||||
}
|
||||
};
|
||||
let info = windows_file_attribute_tag(&anchor)?;
|
||||
if info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY == 0 {
|
||||
return Err(io::Error::from(io::ErrorKind::NotADirectory));
|
||||
@@ -7052,7 +7166,7 @@ pub(crate) fn pin_multipart_source_directory(
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let _ = base;
|
||||
lock_windows_directory_tree(directory, Some(directory), publication_root)
|
||||
lock_windows_directory_tree(directory, publication_root)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
@@ -7142,12 +7256,31 @@ pub(crate) fn rename_multipart_source_owned_std(
|
||||
{
|
||||
let _ = &guard;
|
||||
let source_guard = pin_multipart_source_directory(source_parent, base, root)?;
|
||||
// The prepared Windows source holds DELETE access and shares only
|
||||
// read, so the precondition's own opens of that entry (or, for a
|
||||
// directory, of its guarded tree) would fail with a sharing violation.
|
||||
// Check it first under a non-exclusive identity anchor, then require
|
||||
// the exclusive handle to name the same file.
|
||||
#[cfg(windows)]
|
||||
let (_source_identity_anchor, expected_source_identity) = {
|
||||
let anchor = open_windows_rename_source_identity(source, &source_guard)?;
|
||||
precondition()?;
|
||||
anchor
|
||||
};
|
||||
let preparation = prepare_rename_with_retry(source, destination, base, root)?;
|
||||
#[cfg(windows)]
|
||||
if windows_file_identity(&preparation.source)? != expected_source_identity {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"rename source identity changed after its precondition",
|
||||
));
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
precondition()?;
|
||||
#[cfg(unix)]
|
||||
validate_prepared_rename_source(&pinned_source, source)?;
|
||||
@@ -7553,12 +7686,15 @@ fn read_multipart_metadata(
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
|
||||
let guard = prepare_rename_commit_guard(parent, parent, base, publication_root)?;
|
||||
// A read must not recreate a retired upload: pin the existing parent
|
||||
// instead of taking the write-side guard, which creates missing parents.
|
||||
existing_base_directory_suffix(parent, base)?;
|
||||
let guard = pin_multipart_source_directory(parent, base, publication_root)?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
|
||||
let mut file = open_windows_relative(
|
||||
guard.source_parent_guard.last_handle()?,
|
||||
guard.last_handle()?,
|
||||
name,
|
||||
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_READ_DATA,
|
||||
FILE_SHARE_READ,
|
||||
@@ -7734,7 +7870,7 @@ mod multipart_source_preservation_tests {
|
||||
let metadata_guard =
|
||||
prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
|
||||
let lease = acquire_namespace_mutation_lease(&target).await;
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, lease.clone())
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
|
||||
.await
|
||||
.expect("distinct pinned payload source");
|
||||
assert!(
|
||||
@@ -7769,6 +7905,61 @@ mod multipart_source_preservation_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_multipart_payload_guard_publishes_distinct_source_into_metadata_target() {
|
||||
// Build paths from the configured root: the resolved handle path may
|
||||
// be verbatim, where `/` in a joined key is not a separator.
|
||||
let (root, publication, _source, _destination) = fixture();
|
||||
let root = root.path();
|
||||
let canonical = root.join("multipart/upload");
|
||||
let temporary = root.join("tmp/attempt");
|
||||
let bucket = root.join("bucket");
|
||||
let target = bucket.join("object");
|
||||
for directory in [&canonical, &temporary, &target] {
|
||||
std::fs::create_dir_all(directory).expect("two-source guard directories");
|
||||
}
|
||||
let payload = temporary.join("payload");
|
||||
std::fs::create_dir(&payload).expect("private payload DD");
|
||||
std::fs::write(payload.join("part.1"), b"native selected shard").expect("private payload shard");
|
||||
let head = canonical.join("xl.meta");
|
||||
let destination = target.join("xl.meta");
|
||||
let metadata_guard =
|
||||
prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
|
||||
let lease = acquire_namespace_mutation_lease(&target).await;
|
||||
assert!(
|
||||
rename_all_with_commit_guard(&payload, target.join("payload"), &bucket, &publication, &metadata_guard, lease.clone())
|
||||
.await
|
||||
.is_err(),
|
||||
"the metadata guard does not pin the private payload parent"
|
||||
);
|
||||
assert!(
|
||||
payload.join("part.1").exists(),
|
||||
"a rejected payload rename must leave its source in place"
|
||||
);
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
|
||||
.await
|
||||
.expect("distinct pinned payload source");
|
||||
let mut prepared = create_prepared_rename_source_with_commit_guard(&head, &destination, &metadata_guard)
|
||||
.expect("canonical prepared metadata");
|
||||
prepared
|
||||
.write_all(b"prepared native metadata", true)
|
||||
.expect("durable canonical metadata");
|
||||
rename_all_with_commit_guard(&payload, target.join("payload"), &bucket, &publication, &payload_guard, lease.clone())
|
||||
.await
|
||||
.expect("payload rename uses its own pinned TMP parent");
|
||||
assert!(head.exists(), "payload publication must not consume the canonical head");
|
||||
rename_all_with_prepared_source(prepared, &head, &destination, &bucket, &publication, &metadata_guard, lease)
|
||||
.await
|
||||
.expect("metadata rename keeps its canonical parent");
|
||||
assert!(!head.exists());
|
||||
assert_eq!(std::fs::read(&destination).expect("published metadata"), b"prepared native metadata");
|
||||
assert_eq!(
|
||||
std::fs::read(target.join("payload/part.1")).expect("published selected shard"),
|
||||
b"native selected shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn multipart_payload_guard_rejects_replaced_private_source_or_shared_destination() {
|
||||
@@ -7790,7 +7981,7 @@ mod multipart_source_preservation_tests {
|
||||
let metadata_guard =
|
||||
prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
|
||||
let lease = acquire_namespace_mutation_lease(&target).await;
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, lease.clone())
|
||||
let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
|
||||
.await
|
||||
.expect("pin payload and shared target");
|
||||
let replaced = if replace_source { &temporary } else { &target };
|
||||
@@ -8047,7 +8238,6 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Wdk::Storage::FileSystem::{FILE_OPEN, FILE_OPEN_IF};
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_READ, FILE_SHARE_WRITE};
|
||||
|
||||
let base_relative = publication_root.relative_path(base_dir)?;
|
||||
let capacity = base_relative
|
||||
@@ -8058,13 +8248,12 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
let mut handles = Vec::with_capacity(capacity);
|
||||
handles.push(publication_root.directory.clone());
|
||||
let mut guard = ExistingBaseDirectoryGuard::new(handles);
|
||||
let mut components = base_relative
|
||||
let components = base_relative
|
||||
.components()
|
||||
.map(|component| (component, FILE_OPEN))
|
||||
.chain(relative.components().map(|component| (component, FILE_OPEN_IF)))
|
||||
.filter(|(component, _)| !matches!(component, Component::CurDir))
|
||||
.peekable();
|
||||
while let Some((component, disposition)) = components.next() {
|
||||
.filter(|(component, _)| !matches!(component, Component::CurDir));
|
||||
for (component, disposition) in components {
|
||||
let Component::Normal(component) = component else {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
@@ -8075,22 +8264,16 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
.handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows publication root guard is empty"))?;
|
||||
// The kernel opens the final parent for write during a relative
|
||||
// rename. Share writes from its first open: a temporary read-only
|
||||
// share would block another rename into the same trash directory.
|
||||
// Ancestors stay strict and no handle shares delete access, keeping
|
||||
// every retained directory identity pinned.
|
||||
let share_access = if components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
let child = open_windows_relative_directory_component(parent, component, disposition, share_access)?;
|
||||
// Every component, including the final parent the kernel opens
|
||||
// for write during a relative rename, shares writes from its first
|
||||
// open; see `open_windows_directory_component`.
|
||||
let child = open_windows_directory_component(parent, component, disposition)?;
|
||||
guard.handles.push(child);
|
||||
#[cfg(test)]
|
||||
if components.peek().is_none() {
|
||||
prepare_rename_test_hooks::run_after_component_opened(dir_path);
|
||||
}
|
||||
}
|
||||
// The hook observes the window right after the final component opened.
|
||||
#[cfg(test)]
|
||||
if guard.handles.len() > 1 {
|
||||
prepare_rename_test_hooks::run_after_component_opened(dir_path);
|
||||
}
|
||||
|
||||
Ok(guard)
|
||||
@@ -9496,7 +9679,7 @@ mod tests {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use windows_sys::Win32::{
|
||||
Foundation::{ERROR_SHARING_VIOLATION, GENERIC_WRITE},
|
||||
Foundation::ERROR_SHARING_VIOLATION,
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
@@ -9538,13 +9721,13 @@ mod tests {
|
||||
let second_preparation = prepare_rename_with_retry(&sources[1], &destinations[1], base, &publication_root)
|
||||
.expect("prepare the second trash rename");
|
||||
assert!(interleaved.load(Ordering::Acquire), "the competing parent-open window must be exercised");
|
||||
for (path, access) in [(&tmp, GENERIC_WRITE), (&trash, DELETE)] {
|
||||
for path in [&tmp, &trash] {
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(access)
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)
|
||||
.expect_err("ancestor writes and final-parent deletion must remain excluded");
|
||||
.expect_err("pinned ancestors and final parents must exclude deletion");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
|
||||
@@ -9561,6 +9744,115 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_rename_into_a_pinned_ancestor_succeeds_while_deletion_stays_excluded() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::ERROR_SHARING_VIOLATION,
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
},
|
||||
};
|
||||
|
||||
// `bucket/parent` is an object directory and also the prefix of the
|
||||
// nested object `bucket/parent/child`. A guard on the nested object
|
||||
// pins `parent` as an ancestor while the parent object publishes a new
|
||||
// data directory into it by pathname.
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let parent = base.join("parent");
|
||||
let child = parent.join("child");
|
||||
std::fs::create_dir_all(&child).expect("create nested object directory");
|
||||
let staged = temp_dir.path().join("staged-data-dir");
|
||||
std::fs::create_dir(&staged).expect("create staged data directory");
|
||||
std::fs::write(staged.join("part.1"), b"payload").expect("write staged part");
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
|
||||
let child_guard =
|
||||
super::mkdir_all_below_existing_base_std(&child, &base, &publication_root).expect("pin the nested object tree");
|
||||
|
||||
let destination = parent.join(uuid::Uuid::new_v4().to_string());
|
||||
std::fs::rename(&staged, &destination).expect("a pinned ancestor must accept a rename into it");
|
||||
assert_eq!(std::fs::read(destination.join("part.1")).expect("read published part"), b"payload");
|
||||
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&parent)
|
||||
.expect_err("the pinned ancestor must still refuse deletion");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
|
||||
);
|
||||
drop(child_guard);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_directory_pin_retries_a_transient_delete_holder_and_fails_closed_on_a_lasting_one() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::ERROR_SHARING_VIOLATION,
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE,
|
||||
FILE_SHARE_READ, FILE_SHARE_WRITE, SYNCHRONIZE,
|
||||
},
|
||||
};
|
||||
|
||||
// Open the prefix the way `RemoveDirectoryW` does while it prunes an
|
||||
// object prefix: delete access, every share mode granted.
|
||||
let open_like_a_pruner = |path: &Path| {
|
||||
std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)
|
||||
.expect("open the prefix for delete")
|
||||
};
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
|
||||
|
||||
let transient_name = uuid::Uuid::new_v4().to_string();
|
||||
let transient_prefix = base.join(&transient_name);
|
||||
std::fs::create_dir_all(&transient_prefix).expect("create transient prefix");
|
||||
let holder = Arc::new(std::sync::Mutex::new(Some(open_like_a_pruner(&transient_prefix))));
|
||||
let holder_for_hook = Arc::clone(&holder);
|
||||
let released = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let released_for_hook = Arc::clone(&released);
|
||||
prepare_rename_test_hooks::install_before_component_retry(std::ffi::OsStr::new(&transient_name), move || {
|
||||
holder_for_hook.lock().expect("holder mutex poisoned").take();
|
||||
released_for_hook.store(true, std::sync::atomic::Ordering::Release);
|
||||
});
|
||||
let child = transient_prefix.join("child");
|
||||
let guard = super::mkdir_all_below_existing_base_std(&child, &base, &publication_root)
|
||||
.expect("a pin must outlast a transient delete holder");
|
||||
assert!(
|
||||
released.load(std::sync::atomic::Ordering::Acquire),
|
||||
"the pin must have met the delete holder"
|
||||
);
|
||||
assert!(child.is_dir(), "the retried walk must still create the child");
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&transient_prefix)
|
||||
.expect_err("the retried pin must still exclude deletion");
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_SHARING_VIOLATION).ok());
|
||||
drop(guard);
|
||||
|
||||
let lasting_prefix = base.join(uuid::Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&lasting_prefix).expect("create lasting prefix");
|
||||
let lasting_holder = open_like_a_pruner(&lasting_prefix);
|
||||
let Err(err) = super::mkdir_all_below_existing_base_std(&lasting_prefix.join("child"), &base, &publication_root) else {
|
||||
panic!("a pin must not wait forever on a lasting delete holder");
|
||||
};
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_SHARING_VIOLATION).ok());
|
||||
drop(lasting_holder);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_rename_all_supports_same_parent_publication() {
|
||||
@@ -10258,7 +10550,7 @@ mod tests {
|
||||
|
||||
let publication_root = test_publication_root(&[&src, &dst, &base]);
|
||||
let source_parent_guard =
|
||||
lock_windows_directory_tree(src.parent().expect("source reparse entry must have a parent"), None, &publication_root)
|
||||
lock_windows_directory_tree(src.parent().expect("source reparse entry must have a parent"), &publication_root)
|
||||
.expect("pin the source parent tree");
|
||||
let identity_error = match open_windows_rename_source_identity(&src, &source_parent_guard) {
|
||||
Ok(_) => panic!("a non-dedup source reparse entry must be rejected by its tag"),
|
||||
@@ -10292,30 +10584,51 @@ mod tests {
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_rename_all_excludes_source_parent_reparse_writers() {
|
||||
async fn windows_rename_all_keeps_the_source_parent_from_becoming_a_reparse_point() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::GENERIC_WRITE,
|
||||
Storage::FileSystem::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, FILE_SHARE_WRITE},
|
||||
Foundation::{ERROR_DIR_NOT_EMPTY, ERROR_SHARING_VIOLATION, GENERIC_WRITE},
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
},
|
||||
};
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let source_parent = temp_dir.path().join("staging");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
let src = source_parent.join("staged-object");
|
||||
let dst = base.join("published-object");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir(&source_parent).expect("create source parent");
|
||||
std::fs::create_dir(&outside).expect("create outside target");
|
||||
std::fs::write(&src, b"original").expect("write original staged object");
|
||||
|
||||
let source_parent_for_hook = source_parent.clone();
|
||||
let outside_for_hook = outside.clone();
|
||||
windows_rename_test_hooks::install_before_publication(&dst, move || {
|
||||
std::fs::OpenOptions::new()
|
||||
// The pin shares writes, so a writer can open the source parent,
|
||||
// but the parent still holds the staged entry and NTFS refuses to
|
||||
// turn a non-empty directory into a mount point.
|
||||
let writable_parent = std::fs::OpenOptions::new()
|
||||
.access_mode(GENERIC_WRITE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&source_parent_for_hook)
|
||||
.expect_err("the source-parent guard must exclude a reparse writer");
|
||||
.map(winapi_util::Handle::from_file)
|
||||
.expect("a write-sharing pin admits a concurrent writer");
|
||||
let err = try_set_windows_mount_point(&writable_parent, &outside_for_hook)
|
||||
.expect_err("the pinned source parent must not become a mount point");
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_DIR_NOT_EMPTY).ok());
|
||||
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(DELETE)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&source_parent_for_hook)
|
||||
.expect_err("the source-parent guard must exclude deletion and rename");
|
||||
assert_eq!(err.raw_os_error(), i32::try_from(ERROR_SHARING_VIOLATION).ok());
|
||||
});
|
||||
|
||||
rename_all(&src, &dst, &base)
|
||||
@@ -10324,6 +10637,7 @@ mod tests {
|
||||
|
||||
assert!(!src.exists(), "the original staged entry must be moved");
|
||||
assert_eq!(std::fs::read(&dst).expect("read published original object"), b"original");
|
||||
assert!(!outside.join("published-object").exists(), "publication must not escape the base");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
@@ -10403,7 +10717,7 @@ mod tests {
|
||||
let guard = mkdir_all_below_existing_base_std(&base, &base).expect("guard destination base");
|
||||
let publication_root = test_publication_root(&[&src, &dst, &base]);
|
||||
let source_parent_guard =
|
||||
lock_windows_directory_tree(src.parent().expect("source path must have a parent"), None, &publication_root)
|
||||
lock_windows_directory_tree(src.parent().expect("source path must have a parent"), &publication_root)
|
||||
.expect("anchor source parent");
|
||||
let source = open_windows_rename_source(&src, &source_parent_guard).expect("anchor source entry");
|
||||
|
||||
|
||||
@@ -326,40 +326,6 @@ impl Default for WriteProgressPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller-specific completion behavior for an erasure write.
|
||||
///
|
||||
/// A multipart upload must not discard a different healthy shard from each
|
||||
/// part merely because another quorum finished first: the final object needs
|
||||
/// a quorum of disks carrying every part. Pending writers still obey the
|
||||
/// existing progress and absolute deadlines and never count as completed.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct ShardWritePolicy {
|
||||
quorum: usize,
|
||||
post_quorum: PostQuorumPolicy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum PostQuorumPolicy {
|
||||
Grace,
|
||||
ExistingBudget,
|
||||
}
|
||||
|
||||
impl ShardWritePolicy {
|
||||
pub(crate) fn object(quorum: usize) -> Self {
|
||||
Self {
|
||||
quorum,
|
||||
post_quorum: PostQuorumPolicy::Grace,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn multipart(quorum: usize) -> Self {
|
||||
Self {
|
||||
quorum,
|
||||
post_quorum: PostQuorumPolicy::ExistingBudget,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// After write quorum is already in hand, how long a slower shard may still
|
||||
/// finish before it is dropped. The per-shard stall budget (30s by default)
|
||||
/// is what a black-hole peer is allowed to consume *before* quorum exists.
|
||||
@@ -372,15 +338,13 @@ fn post_quorum_straggler_grace(stall_timeout: Option<std::time::Duration>) -> Op
|
||||
stall_timeout.map(|stall| stall.min(WRITE_QUORUM_STRAGGLER_GRACE))
|
||||
}
|
||||
|
||||
/// Drive shard operations within their existing deadlines. Ordinary object
|
||||
/// writes additionally stop after the post-quorum grace; multipart writes keep
|
||||
/// polling all candidates within the same operation budget. A `false` slot
|
||||
/// never completed and must be excluded from commit.
|
||||
/// Drive shard ops until write quorum has succeeded and stragglers have had
|
||||
/// [`post_quorum_straggler_grace`] to finish. Returns `true` for each slot
|
||||
/// whose future completed; `false` means the caller must fail that slot.
|
||||
async fn await_write_quorum<F>(
|
||||
write_quorum: usize,
|
||||
stall_timeout: Option<std::time::Duration>,
|
||||
slot_count: usize,
|
||||
post_quorum: PostQuorumPolicy,
|
||||
mut futures: FuturesUnordered<F>,
|
||||
) -> Vec<bool>
|
||||
where
|
||||
@@ -412,7 +376,6 @@ where
|
||||
if succeeded {
|
||||
successes = successes.saturating_add(1);
|
||||
if successes >= write_quorum
|
||||
&& post_quorum == PostQuorumPolicy::Grace
|
||||
&& grace.is_none()
|
||||
&& let Some(delay) = post_quorum_straggler_grace(stall_timeout)
|
||||
{
|
||||
@@ -441,7 +404,6 @@ pub(crate) struct MultiWriter<'a> {
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
integrity: Option<&'a mut IntegrityBuilder>,
|
||||
write_quorum: usize,
|
||||
post_quorum: PostQuorumPolicy,
|
||||
errs: Vec<Option<Error>>,
|
||||
policy: WriteProgressPolicy,
|
||||
/// Absolute deadline for the whole object, armed lazily on the first
|
||||
@@ -450,21 +412,16 @@ pub(crate) struct MultiWriter<'a> {
|
||||
}
|
||||
|
||||
impl<'a> MultiWriter<'a> {
|
||||
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_policy: ShardWritePolicy) -> Self {
|
||||
Self::with_policy(writers, write_policy, WriteProgressPolicy::default())
|
||||
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
|
||||
Self::with_policy(writers, write_quorum, WriteProgressPolicy::default())
|
||||
}
|
||||
|
||||
pub fn with_policy(
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
write_policy: ShardWritePolicy,
|
||||
policy: WriteProgressPolicy,
|
||||
) -> Self {
|
||||
pub fn with_policy(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize, policy: WriteProgressPolicy) -> Self {
|
||||
let length = writers.len();
|
||||
MultiWriter {
|
||||
writers,
|
||||
integrity: None,
|
||||
write_quorum: write_policy.quorum,
|
||||
post_quorum: write_policy.post_quorum,
|
||||
write_quorum,
|
||||
errs: vec![None; length],
|
||||
policy,
|
||||
absolute_deadline: None,
|
||||
@@ -549,8 +506,9 @@ impl<'a> MultiWriter<'a> {
|
||||
// failed and its disk dropped, so a stalled peer cannot pin an
|
||||
// otherwise-healthy write quorum (rustfs/backlog#1319). `budget`
|
||||
// is recomputed per block, so it bounds a stall — not the total
|
||||
// transfer time. Only ordinary object writes add the shorter
|
||||
// post-quorum grace; multipart preserves this operation budget.
|
||||
// transfer time — and a slow-but-honest writer is never killed
|
||||
// while quorum is still open. Once quorum has succeeded, a
|
||||
// straggler only gets `WRITE_QUORUM_STRAGGLER_GRACE` more time.
|
||||
futures.push(async move {
|
||||
match budget {
|
||||
Some(budget) => match tokio::time::timeout(budget, Self::write_shard(writer_opt, err, shard)).await {
|
||||
@@ -566,7 +524,7 @@ impl<'a> MultiWriter<'a> {
|
||||
(index, succeeded)
|
||||
});
|
||||
}
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
|
||||
};
|
||||
abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
|
||||
|
||||
@@ -629,8 +587,8 @@ impl<'a> MultiWriter<'a> {
|
||||
// here forever for a small object whose bytes were fully buffered
|
||||
// (so `write` never blocked). Bound it with the same progress
|
||||
// budget and drop the stalled writer before the quorum check.
|
||||
// Ordinary object writes also apply their post-quorum grace;
|
||||
// multipart waits for all candidates within this same budget.
|
||||
// After quorum has shut down, do not keep waiting out the rest of
|
||||
// that budget for a peer that has stopped answering.
|
||||
futures.push(async move {
|
||||
match budget {
|
||||
Some(budget) => match tokio::time::timeout(budget, Self::shutdown_writer(writer_opt, err)).await {
|
||||
@@ -646,7 +604,7 @@ impl<'a> MultiWriter<'a> {
|
||||
(index, succeeded)
|
||||
});
|
||||
}
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
|
||||
await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
|
||||
};
|
||||
abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
|
||||
|
||||
@@ -681,7 +639,7 @@ impl Erasure {
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
write_policy: ShardWritePolicy,
|
||||
quorum: usize,
|
||||
part_number: usize,
|
||||
mode: IntegrityEncodeMode,
|
||||
protected: bool,
|
||||
@@ -710,19 +668,19 @@ impl Erasure {
|
||||
}
|
||||
IntegrityEncodeMode::SingleBlock(hint) => {
|
||||
let (reader, size) = self
|
||||
.encode_small_direct(reader, writers, write_policy, true, hint, integrity.as_mut())
|
||||
.encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
IntegrityEncodeMode::Streaming => {
|
||||
let (reader, size) = self
|
||||
.encode_with_ingest_mode(reader, writers, write_policy, use_bytesmut_ingest(), integrity.as_mut())
|
||||
.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
IntegrityEncodeMode::Batched => {
|
||||
let (reader, size) = self
|
||||
.encode_batched_with_integrity(reader, writers, write_policy, integrity.as_mut())
|
||||
.encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut())
|
||||
.await?;
|
||||
(reader, size, None)
|
||||
}
|
||||
@@ -787,7 +745,7 @@ impl Erasure {
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
write_policy: ShardWritePolicy,
|
||||
quorum: usize,
|
||||
require_single_block: bool,
|
||||
size_hint: usize,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
@@ -822,7 +780,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
let block = self.encode_data_owned_block(buf)?;
|
||||
let mut mw = MultiWriter::new(writers, write_policy);
|
||||
let mut mw = MultiWriter::new(writers, quorum);
|
||||
mw.integrity = integrity;
|
||||
mw.write_block(&block).await?;
|
||||
mw.shutdown().await?;
|
||||
@@ -889,7 +847,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let use_bytesmut_ingest = use_bytesmut_ingest();
|
||||
self.encode_with_ingest_mode(reader, writers, ShardWritePolicy::object(quorum), use_bytesmut_ingest, None)
|
||||
self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -901,7 +859,7 @@ impl Erasure {
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
write_policy: ShardWritePolicy,
|
||||
quorum: usize,
|
||||
use_bytesmut_ingest: bool,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
@@ -1003,7 +961,7 @@ impl Erasure {
|
||||
Ok((reader, total))
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, write_policy);
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
writers.integrity = integrity;
|
||||
|
||||
let mut write_err = None;
|
||||
@@ -1059,15 +1017,14 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
self.encode_batched_with_integrity(reader, writers, ShardWritePolicy::object(quorum), None)
|
||||
.await
|
||||
self.encode_batched_with_integrity(reader, writers, quorum, None).await
|
||||
}
|
||||
|
||||
async fn encode_batched_with_integrity<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
write_policy: ShardWritePolicy,
|
||||
quorum: usize,
|
||||
integrity: Option<&mut IntegrityBuilder>,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
@@ -1148,7 +1105,7 @@ impl Erasure {
|
||||
Ok((reader, total))
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, write_policy);
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
writers.integrity = integrity;
|
||||
let mut write_err = None;
|
||||
|
||||
@@ -1208,7 +1165,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1225,7 +1182,7 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1242,8 +1199,7 @@ impl Erasure {
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
|
||||
.await
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
|
||||
}
|
||||
|
||||
/// Size-aware single-block fast path. `size_hint` only controls the bounded
|
||||
@@ -1259,8 +1215,7 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
|
||||
.await
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1677,219 +1632,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum DelayedPhase {
|
||||
Write,
|
||||
Flush,
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
struct PhaseDelayedWriter {
|
||||
inner: DeferredCommitWriter,
|
||||
phase: DelayedPhase,
|
||||
delay: Duration,
|
||||
timer: Option<Pin<Box<tokio::time::Sleep>>>,
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
impl PhaseDelayedWriter {
|
||||
fn poll_delay(&mut self, cx: &mut Context<'_>) -> Poll<()> {
|
||||
if self.ready {
|
||||
return Poll::Ready(());
|
||||
}
|
||||
let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep(self.delay)));
|
||||
match timer.as_mut().poll(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(()) => {
|
||||
self.ready = true;
|
||||
Poll::Ready(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for PhaseDelayedWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
if matches!(self.phase, DelayedPhase::Write) && self.poll_delay(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_write(cx, bytes)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
if matches!(self.phase, DelayedPhase::Flush) && self.poll_delay(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
if matches!(self.phase, DelayedPhase::Shutdown) && self.poll_delay(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn multipart_budget_keeps_slow_writers_in_every_encode_mode() {
|
||||
const BLOCK: usize = 64;
|
||||
for make_mode in [
|
||||
|| IntegrityEncodeMode::SingleBlock(BLOCK),
|
||||
|| IntegrityEncodeMode::Streaming,
|
||||
|| IntegrityEncodeMode::Batched,
|
||||
] {
|
||||
for phase in [DelayedPhase::Write, DelayedPhase::Flush, DelayedPhase::Shutdown] {
|
||||
for delay in [Duration::from_secs(2), Duration::from_secs(5)] {
|
||||
let committed: Vec<_> = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
|
||||
let mut writers = committed
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(slot, data)| {
|
||||
if slot == 0 {
|
||||
Some(bitrot_writer(
|
||||
PhaseDelayedWriter {
|
||||
inner: DeferredCommitWriter::new(data.clone()),
|
||||
phase,
|
||||
delay,
|
||||
timer: None,
|
||||
ready: false,
|
||||
},
|
||||
BLOCK / 2,
|
||||
))
|
||||
} else {
|
||||
Some(bitrot_writer(DeferredCommitWriter::new(data.clone()), BLOCK / 2))
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let payload = vec![0x5a; BLOCK];
|
||||
let erasure = Arc::new(Erasure::new(2, 2, BLOCK));
|
||||
let started = tokio::time::Instant::now();
|
||||
erasure
|
||||
.encode_with_shard_integrity(
|
||||
Cursor::new(payload.clone()),
|
||||
&mut writers,
|
||||
ShardWritePolicy::multipart(3),
|
||||
1,
|
||||
make_mode(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("multipart must keep finite slow writers within the existing budget");
|
||||
assert!(started.elapsed() >= delay, "must await the slow phase: {phase:?}");
|
||||
assert!(writers.iter().all(Option::is_some), "all four completed shards must survive");
|
||||
for data in &committed {
|
||||
let bytes = data.lock().expect("committed bytes").clone();
|
||||
crate::erasure::coding::bitrot_verify(
|
||||
Cursor::new(bytes.clone()),
|
||||
bytes.len(),
|
||||
BLOCK / 2,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
BLOCK / 2,
|
||||
)
|
||||
.await
|
||||
.expect("each completed shard must retain valid bitrot framing");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn multipart_budget_expires_blackholes_without_restarting_deadlines() {
|
||||
for shutdown in [false, true] {
|
||||
for blackholes in [1, 2] {
|
||||
for absolute in [Duration::ZERO, Duration::from_secs(3)] {
|
||||
let mut writers = (0..4)
|
||||
.map(|slot| {
|
||||
if slot < blackholes {
|
||||
Some(bitrot_writer_plain(
|
||||
if shutdown {
|
||||
StallingWriter::stalls_on_shutdown()
|
||||
} else {
|
||||
StallingWriter::stalls_on_write(0)
|
||||
},
|
||||
64,
|
||||
))
|
||||
} else {
|
||||
Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = {
|
||||
let mut writer = MultiWriter::with_policy(
|
||||
&mut writers,
|
||||
ShardWritePolicy::multipart(3),
|
||||
WriteProgressPolicy::new(Duration::from_secs(7), absolute),
|
||||
);
|
||||
if shutdown {
|
||||
writer.write(four_shards()).await.expect("buffered writes should complete");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
writer.shutdown().await
|
||||
} else {
|
||||
writer.write(four_shards()).await
|
||||
}
|
||||
};
|
||||
assert_eq!(result.is_ok(), blackholes == 1, "only a confirmed quorum may succeed");
|
||||
let expected = if absolute.is_zero() {
|
||||
Duration::from_secs(7 + u64::from(shutdown))
|
||||
} else {
|
||||
absolute
|
||||
};
|
||||
assert_eq!(started.elapsed(), expected, "budget must not restart at quorum or shutdown");
|
||||
assert!(writers[..blackholes].iter().all(Option::is_none));
|
||||
assert!(writers[blackholes..].iter().all(Option::is_some));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn multipart_budget_cancellation_drops_pending_writers() {
|
||||
struct DropSignal(Option<oneshot::Sender<()>>);
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
if let Some(tx) = self.0.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
impl AsyncWrite for DropSignal {
|
||||
fn poll_write(self: Pin<&mut Self>, _: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
Poll::Ready(Ok(bytes.len()))
|
||||
}
|
||||
fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut writers = vec![Some(bitrot_writer_plain(DropSignal(Some(tx)), 64))];
|
||||
writers.extend(
|
||||
(0..3).map(|_| Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))),
|
||||
);
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::multipart(3));
|
||||
writer.write(four_shards()).await.expect("buffered writes should succeed");
|
||||
let _ = entered_tx.send(());
|
||||
writer.shutdown().await
|
||||
});
|
||||
entered.await.expect("write must reach shutdown");
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
task.abort();
|
||||
assert!(
|
||||
task.await
|
||||
.expect_err("caller cancellation should cancel the task")
|
||||
.is_cancelled()
|
||||
);
|
||||
rx.await.expect("pending writer must be dropped with its owning task");
|
||||
}
|
||||
|
||||
/// A writer that always makes forward progress but each `poll_write` first
|
||||
/// waits `delay` (on the runtime timer). Under a paused virtual clock this
|
||||
/// deterministically models a slow-but-honest peer: it must never be killed
|
||||
@@ -1962,16 +1704,12 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
|
||||
let encode = match pipeline {
|
||||
EncodePipeline::Vec => tokio::spawn(async move {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
}),
|
||||
EncodePipeline::BytesMut => tokio::spawn(async move {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await
|
||||
}),
|
||||
EncodePipeline::Vec => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await })
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await })
|
||||
}
|
||||
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
|
||||
};
|
||||
|
||||
@@ -2029,16 +1767,8 @@ mod tests {
|
||||
))];
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
@@ -2111,16 +1841,8 @@ mod tests {
|
||||
let mut writers = vec![Some(bitrot_writer(FailAfterReaderErrorWriter { reader_failed }, BLOCK_SIZE))];
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
@@ -2157,7 +1879,7 @@ mod tests {
|
||||
let erasure_for_task = erasure.clone();
|
||||
let encode = tokio::spawn(async move {
|
||||
erasure_for_task
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, 1, false, None)
|
||||
.await
|
||||
});
|
||||
|
||||
@@ -2326,16 +2048,8 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(2, 2, 64));
|
||||
let result = match path {
|
||||
"direct" => erasure.encode_single_block_non_inline(reader, &mut writers, 2).await,
|
||||
"vec" => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), false, None)
|
||||
.await
|
||||
}
|
||||
"bytesmut" => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), true, None)
|
||||
.await
|
||||
}
|
||||
"vec" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, false, None).await,
|
||||
"bytesmut" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, true, None).await,
|
||||
"batched" => erasure.encode_batched(reader, &mut writers, 2).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
@@ -2380,7 +2094,7 @@ mod tests {
|
||||
async fn multi_writer_short_write_fails_before_shutdown() {
|
||||
let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 32))];
|
||||
let err = {
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
writer
|
||||
.write(vec![Bytes::from_static(b"short-write payload")])
|
||||
.await
|
||||
@@ -2397,7 +2111,7 @@ mod tests {
|
||||
async fn multi_writer_reports_fallback_summary_when_only_offline_writers_remain() {
|
||||
let mut writers = vec![None, None];
|
||||
let (err, summary) = {
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
let err = writer
|
||||
.write(vec![Bytes::from_static(b"offline-a"), Bytes::from_static(b"offline-b")])
|
||||
.await
|
||||
@@ -2417,7 +2131,7 @@ mod tests {
|
||||
assert!(summary.contains("required=1"));
|
||||
|
||||
let (shutdown_err, summary) = {
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
|
||||
let mut writer = MultiWriter::new(&mut writers, 1);
|
||||
let err = writer
|
||||
.shutdown()
|
||||
.await
|
||||
@@ -2441,7 +2155,7 @@ mod tests {
|
||||
async fn multi_writer_reports_quorum_failure_when_quorum_exceeds_writer_count() {
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed), 32))];
|
||||
let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(2));
|
||||
let mut writer = MultiWriter::new(&mut writers, 2);
|
||||
|
||||
let err = writer
|
||||
.write(vec![Bytes::from_static(b"quorum impossible")])
|
||||
@@ -2530,7 +2244,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
mw.write(four_shards())
|
||||
.await
|
||||
.expect("one stalled writer must still satisfy a 3/4 write quorum without hanging");
|
||||
@@ -2557,7 +2271,7 @@ mod tests {
|
||||
let started = tokio::time::Instant::now();
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
mw.write(four_shards())
|
||||
.await
|
||||
.expect("quorum must succeed without waiting out the black-hole stall");
|
||||
@@ -2590,7 +2304,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
mw.write(four_shards())
|
||||
.await
|
||||
.expect("a straggler inside the grace window must still satisfy the write");
|
||||
@@ -2615,7 +2329,7 @@ mod tests {
|
||||
];
|
||||
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
let err = mw
|
||||
.write(four_shards())
|
||||
.await
|
||||
@@ -2644,7 +2358,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
mw.write(four_shards()).await.expect("all writers accept the buffered shard");
|
||||
mw.shutdown()
|
||||
.await
|
||||
@@ -2665,7 +2379,7 @@ mod tests {
|
||||
];
|
||||
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
mw.write(four_shards()).await.expect("all writers accept the buffered shard");
|
||||
let err = mw
|
||||
.shutdown()
|
||||
@@ -2694,7 +2408,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
for _ in 0..8 {
|
||||
mw.write(four_full_shards(64))
|
||||
.await
|
||||
@@ -2724,7 +2438,7 @@ mod tests {
|
||||
// stall (10s) never fires for an 8s-per-block writer, but the 15s
|
||||
// absolute cap does once its aggregate engagement crosses the cap.
|
||||
let policy = WriteProgressPolicy::new(Duration::from_secs(10), Duration::from_secs(15));
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
|
||||
let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
|
||||
for _ in 0..4 {
|
||||
mw.write(four_full_shards(64))
|
||||
.await
|
||||
@@ -2869,7 +2583,7 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
|
||||
let (_reader, written) = erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
|
||||
.await
|
||||
.expect("BytesMut ingest path should encode the streaming payload");
|
||||
|
||||
@@ -2898,9 +2612,7 @@ mod tests {
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
let result = erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
|
||||
.await;
|
||||
let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await;
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
|
||||
let (_reader, written) = result.expect("bytesmut streaming encode should complete");
|
||||
@@ -3518,12 +3230,12 @@ mod tests {
|
||||
let (_reader, total) = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), false, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
erasure
|
||||
.encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
|
||||
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
|
||||
.await
|
||||
}
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,
|
||||
|
||||
@@ -205,7 +205,7 @@ impl super::Erasure {
|
||||
// the survivors. Reconstruction correctness is still guaranteed by the
|
||||
// read-side quorum check and parity cross-checks below, which are unaffected.
|
||||
let write_quorum = 1;
|
||||
let mut writers = MultiWriter::new(writers, super::encode::ShardWritePolicy::object(write_quorum));
|
||||
let mut writers = MultiWriter::new(writers, write_quorum);
|
||||
let read_timeout = get_object_disk_read_timeout();
|
||||
let shard_file_size = self.shard_file_size(total_length as i64) as usize;
|
||||
|
||||
|
||||
@@ -1910,6 +1910,10 @@ mod tests {
|
||||
&ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
mod_time: earlier(version.mod_time),
|
||||
// Without the object lock the rename waits for every
|
||||
// drive, so no early-ACK tail is still writing the copies
|
||||
// removed below.
|
||||
no_lock: true,
|
||||
..versioned
|
||||
},
|
||||
))
|
||||
|
||||
@@ -112,10 +112,6 @@ use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "multipart_quorum_tests.rs"]
|
||||
mod multipart_writer_test_hook;
|
||||
|
||||
// The erasure-set width bounds fan-out. Await every opener so errors retain
|
||||
// their disk slots and every successful writer remains owned until quorum is checked.
|
||||
async fn create_part_writers(
|
||||
@@ -128,13 +124,6 @@ async fn create_part_writers(
|
||||
let Some(disk) = disk else {
|
||||
return (None, Some(DiskError::DiskNotFound));
|
||||
};
|
||||
#[cfg(test)]
|
||||
if let Some(result) = multipart_writer_test_hook::create_writer(disk, path, length, shard_size).await {
|
||||
return match result {
|
||||
Ok(writer) => (Some(writer), None),
|
||||
Err(error) => (None, Some(error)),
|
||||
};
|
||||
}
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
false,
|
||||
Some(disk),
|
||||
@@ -2655,19 +2644,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched,
|
||||
SmallWritePath::Inline | SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
|
||||
};
|
||||
let shard_write_policy = crate::erasure::coding::encode::ShardWritePolicy::multipart(write_quorum);
|
||||
#[cfg(test)]
|
||||
let shard_write_policy = multipart_writer_test_hook::write_policy(&shuffle_disks, write_quorum)
|
||||
.unwrap_or(shard_write_policy);
|
||||
let (reader, w_size, _, integrity) = Arc::clone(&erasure)
|
||||
.encode_with_shard_integrity(
|
||||
stream,
|
||||
&mut writers,
|
||||
shard_write_policy,
|
||||
part_id,
|
||||
mode,
|
||||
protected_upload,
|
||||
)
|
||||
.encode_with_shard_integrity(stream, &mut writers, write_quorum, part_id, mode, protected_upload)
|
||||
.await?;
|
||||
|
||||
if let Some(stage_start) = encode_stage_start {
|
||||
|
||||
@@ -1,303 +0,0 @@
|
||||
//! Exercise participant preservation through real multipart staging and commit.
|
||||
|
||||
use super::*;
|
||||
use crate::disk::FileWriter;
|
||||
use crate::erasure::coding::CustomWriter;
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Fault {
|
||||
DelayedShutdown,
|
||||
WriteFailure,
|
||||
}
|
||||
|
||||
struct Round {
|
||||
fault: Fault,
|
||||
legacy_policy: bool,
|
||||
failed_slot: usize,
|
||||
completed: AtomicUsize,
|
||||
delayed_observed: AtomicBool,
|
||||
delayed_waker: Mutex<Option<Waker>>,
|
||||
delayed_dropped: AtomicBool,
|
||||
}
|
||||
|
||||
type RegisteredWriter = (Arc<Round>, usize);
|
||||
static WRITER_ROUNDS: OnceLock<Mutex<HashMap<String, RegisteredWriter>>> = OnceLock::new();
|
||||
|
||||
struct WriterRound {
|
||||
endpoints: Vec<String>,
|
||||
round: Arc<Round>,
|
||||
}
|
||||
|
||||
impl WriterRound {
|
||||
fn install(disks: &[DiskStore], failed_slot: usize, fault: Fault, legacy_policy: bool) -> Self {
|
||||
let round = Arc::new(Round {
|
||||
fault,
|
||||
legacy_policy,
|
||||
failed_slot,
|
||||
completed: AtomicUsize::new(0),
|
||||
delayed_observed: AtomicBool::new(false),
|
||||
delayed_waker: Mutex::new(None),
|
||||
delayed_dropped: AtomicBool::new(false),
|
||||
});
|
||||
let endpoints = disks.iter().map(|disk| disk.endpoint().to_string()).collect::<Vec<_>>();
|
||||
let mut registry = WRITER_ROUNDS
|
||||
.get_or_init(Mutex::default)
|
||||
.lock()
|
||||
.expect("writer hook registry");
|
||||
for (slot, endpoint) in endpoints.iter().enumerate() {
|
||||
assert!(registry.insert(endpoint.clone(), (round.clone(), slot)).is_none());
|
||||
}
|
||||
Self { endpoints, round }
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn write_policy(
|
||||
disks: &[Option<DiskStore>],
|
||||
quorum: usize,
|
||||
) -> Option<crate::erasure::coding::encode::ShardWritePolicy> {
|
||||
let registry = WRITER_ROUNDS.get()?.lock().expect("writer hook registry");
|
||||
disks.iter().flatten().find_map(|disk| {
|
||||
registry.get(&disk.endpoint().to_string()).and_then(|(round, _)| {
|
||||
round
|
||||
.legacy_policy
|
||||
.then(|| crate::erasure::coding::encode::ShardWritePolicy::object(quorum))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
impl Drop for WriterRound {
|
||||
fn drop(&mut self) {
|
||||
let mut registry = WRITER_ROUNDS
|
||||
.get()
|
||||
.expect("installed writer registry")
|
||||
.lock()
|
||||
.expect("writer hook registry");
|
||||
for endpoint in &self.endpoints {
|
||||
registry.remove(endpoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn create_writer(
|
||||
disk: &DiskStore,
|
||||
path: &str,
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
) -> Option<disk::error::Result<BitrotWriterWrapper>> {
|
||||
let registered = WRITER_ROUNDS
|
||||
.get()?
|
||||
.lock()
|
||||
.expect("writer hook registry")
|
||||
.get(&disk.endpoint().to_string())
|
||||
.cloned();
|
||||
let (round, slot) = registered?;
|
||||
let length = usize::try_from(length).expect("the fixture writes known-size parts");
|
||||
let encoded_length = coding::bitrot_shard_file_size(length, shard_size, HashAlgorithm::HighwayHash256S);
|
||||
let inner = match disk
|
||||
.create_file(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, path, encoded_length as i64)
|
||||
.await
|
||||
{
|
||||
Ok(inner) => inner,
|
||||
Err(error) => return Some(Err(error)),
|
||||
};
|
||||
let writer = ControlledWriter {
|
||||
inner,
|
||||
round,
|
||||
slot,
|
||||
delay: None,
|
||||
completed: false,
|
||||
};
|
||||
Some(Ok(BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(Box::new(writer)),
|
||||
shard_size,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
)))
|
||||
}
|
||||
|
||||
struct ControlledWriter {
|
||||
inner: FileWriter,
|
||||
round: Arc<Round>,
|
||||
slot: usize,
|
||||
delay: Option<Pin<Box<tokio::time::Sleep>>>,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl Drop for ControlledWriter {
|
||||
fn drop(&mut self) {
|
||||
if self.slot == self.round.failed_slot && !self.completed {
|
||||
self.round.delayed_dropped.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for ControlledWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<io::Result<usize>> {
|
||||
if self.slot == self.round.failed_slot && matches!(self.round.fault, Fault::WriteFailure) {
|
||||
return Poll::Ready(Err(io::Error::new(io::ErrorKind::BrokenPipe, "injected shard write failure")));
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_write(cx, bytes)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
if self.slot == self.round.failed_slot && matches!(self.round.fault, Fault::DelayedShutdown) {
|
||||
self.round.delayed_observed.store(true, Ordering::Release);
|
||||
// Delay only after three real writers finish. This ensures the old
|
||||
// post-quorum grace expires regardless of filesystem scheduling.
|
||||
{
|
||||
let mut waker = self.round.delayed_waker.lock().expect("delayed writer waker");
|
||||
*waker = Some(cx.waker().clone());
|
||||
if self.round.completed.load(Ordering::Acquire) < 3 {
|
||||
return Poll::Pending;
|
||||
}
|
||||
}
|
||||
let delay = self
|
||||
.delay
|
||||
.get_or_insert_with(|| Box::pin(tokio::time::sleep(Duration::from_secs(2))));
|
||||
if delay.as_mut().poll(cx).is_pending() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
}
|
||||
let result = Pin::new(&mut self.inner).poll_shutdown(cx);
|
||||
if matches!(result, Poll::Ready(Ok(()))) && !self.completed {
|
||||
self.completed = true;
|
||||
self.round.completed.fetch_add(1, Ordering::AcqRel);
|
||||
if let Some(waker) = self.round.delayed_waker.lock().expect("delayed writer waker").take() {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
async fn upload_two_parts(
|
||||
set: &Arc<SetDisks>,
|
||||
disks: &[DiskStore],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fault: Fault,
|
||||
legacy_policy: bool,
|
||||
) -> (String, Vec<CompletePart>, Vec<u8>, FileInfo) {
|
||||
for disk in disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &opts)
|
||||
.await
|
||||
.expect("new multipart upload");
|
||||
let mut parts = Vec::new();
|
||||
let mut expected = Vec::new();
|
||||
for (number, size, slot) in [(1, 5 * 1024 * 1024, 2), (2, 4096, 3)] {
|
||||
let payload = vec![number as u8; size];
|
||||
expected.extend_from_slice(&payload);
|
||||
let hook = WriterRound::install(disks, slot, fault, legacy_policy);
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
let part = set
|
||||
.put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &opts)
|
||||
.await
|
||||
.expect("three completed shards must admit each part");
|
||||
match fault {
|
||||
Fault::DelayedShutdown if !legacy_policy => {
|
||||
assert!(hook.round.delayed_observed.load(Ordering::Acquire));
|
||||
assert!(!hook.round.delayed_dropped.load(Ordering::Acquire), "finite slow shard must finish");
|
||||
assert_eq!(hook.round.completed.load(Ordering::Acquire), 4, "all real shard writers finished");
|
||||
}
|
||||
Fault::DelayedShutdown | Fault::WriteFailure => {
|
||||
assert!(hook.round.delayed_dropped.load(Ordering::Acquire), "failed shard must be dropped");
|
||||
assert_eq!(hook.round.completed.load(Ordering::Acquire), 3);
|
||||
}
|
||||
}
|
||||
parts.push(CompletePart {
|
||||
part_num: number,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
let (metadata, _) = set
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, true)
|
||||
.await
|
||||
.expect("upload metadata");
|
||||
(upload.upload_id, parts, expected, metadata)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_budget_keeps_different_finite_shutdown_stragglers_and_completes() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "multipart-finite-shutdown";
|
||||
let object = "object";
|
||||
let (upload, parts, expected, metadata) = upload_two_parts(&set, &disks, bucket, object, Fault::DelayedShutdown, false).await;
|
||||
let data_dir = metadata.data_dir.expect("parts-backed data directory");
|
||||
let erasure =
|
||||
coding::Erasure::try_new(metadata.erasure.data_blocks, metadata.erasure.parity_blocks, metadata.erasure.block_size)
|
||||
.expect("upload erasure layout");
|
||||
let stage = SetDisks::get_upload_id_dir(bucket, object, &upload);
|
||||
for disk in &disks {
|
||||
for (number, size) in [(1, 5 * 1024 * 1024), (2, 4096)] {
|
||||
let path = format!("{stage}/{data_dir}/part.{number}");
|
||||
let bytes = disk
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
|
||||
.await
|
||||
.expect("every disk has every part");
|
||||
let shard_len = erasure.shard_file_size(size);
|
||||
let encoded_len =
|
||||
coding::bitrot_shard_file_size(shard_len as usize, erasure.shard_size(), HashAlgorithm::HighwayHash256S);
|
||||
assert_eq!(bytes.len(), encoded_len, "complete physical shard: {path}");
|
||||
}
|
||||
}
|
||||
set.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("all parts have a common completed quorum");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read committed object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("whole object readback");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_budget_distinct_failed_shards_do_not_bypass_complete_quorum() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "multipart-distinct-failures";
|
||||
let object = "object";
|
||||
let (upload, parts, _, _) = upload_two_parts(&set, &disks, bucket, object, Fault::WriteFailure, false).await;
|
||||
let error = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("two common complete disks cannot meet the write quorum of three");
|
||||
assert!(error.is_quorum_error(), "expected complete quorum refusal, got {error:?}");
|
||||
set.check_upload_id_exists(bucket, object, &upload, false)
|
||||
.await
|
||||
.expect("failed completion retains retryable upload state");
|
||||
assert!(
|
||||
set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
|
||||
"failed completion must not publish an object"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_budget_old_grace_loses_common_part_quorum() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "multipart-old-grace";
|
||||
let object = "object";
|
||||
let (upload, parts, _, _) = upload_two_parts(&set, &disks, bucket, object, Fault::DelayedShutdown, true).await;
|
||||
let error = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the old per-part grace loses the common completion quorum");
|
||||
assert!(error.is_quorum_error(), "expected complete quorum refusal, got {error:?}");
|
||||
}
|
||||
@@ -4562,14 +4562,7 @@ impl SetDisks {
|
||||
SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
|
||||
};
|
||||
let (reader, w_size, inline_shards, integrity) = Arc::clone(&erasure)
|
||||
.encode_with_shard_integrity(
|
||||
stream,
|
||||
&mut writers,
|
||||
coding::encode::ShardWritePolicy::object(write_quorum),
|
||||
1,
|
||||
mode,
|
||||
protect_write,
|
||||
)
|
||||
.encode_with_shard_integrity(stream, &mut writers, write_quorum, 1, mode, protect_write)
|
||||
.await?;
|
||||
let encode_elapsed = encode_stage_start.map(|stage_start| stage_start.elapsed());
|
||||
let encode_ms = encode_elapsed.map(|elapsed| elapsed.as_millis() as u64).unwrap_or_default();
|
||||
|
||||
@@ -318,7 +318,10 @@ mod multipart_copy_heal_tests {
|
||||
let mut corrupt = original.clone();
|
||||
corrupt[0] ^= 0x80;
|
||||
std::fs::write(&shard, &corrupt).expect("actual bitrot corruption in shared physical shard");
|
||||
std::fs::File::open(&shard)
|
||||
// Windows only flushes a handle opened for writing.
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&shard)
|
||||
.expect("open corrupted actual shard")
|
||||
.sync_all()
|
||||
.expect("persist actual fault bytes");
|
||||
|
||||
@@ -359,16 +359,6 @@ impl LockClient for LocalClient {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_grant(&self, lock_id: &LockId) -> Result<super::LockCleanupOutcome> {
|
||||
self.release(lock_id).await.map(|released| {
|
||||
if released {
|
||||
super::LockCleanupOutcome::Released
|
||||
} else {
|
||||
super::LockCleanupOutcome::AlreadyAbsent
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let expired_entry = {
|
||||
|
||||
@@ -20,17 +20,6 @@ use async_trait::async_trait;
|
||||
use futures::future::join_all;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Outcome of cleaning up a lock whose acquisition has already completed.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum LockCleanupOutcome {
|
||||
/// This request released the granted lock.
|
||||
Released,
|
||||
/// The authoritative backend no longer holds this exact lock ID.
|
||||
AlreadyAbsent,
|
||||
/// The backend did not confirm release or authoritative absence.
|
||||
Unconfirmed,
|
||||
}
|
||||
|
||||
/// Lock client trait
|
||||
#[async_trait]
|
||||
pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
@@ -48,29 +37,6 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
/// Release lock
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
/// Idempotently clean up a previously confirmed grant for this exact lock ID.
|
||||
///
|
||||
/// Call only after acquisition has completed successfully. An absent lock
|
||||
/// cannot settle an acquisition that may still grant it later; that
|
||||
/// acquisition must retain its own completion cleanup. This operation must
|
||||
/// never release another owner's lock on the same resource.
|
||||
///
|
||||
/// The default preserves conservative behavior for clients without an
|
||||
/// authoritative absence result. Cancellation or an error leaves cleanup
|
||||
/// unconfirmed, so the caller must retain its retry ownership.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns backend or transport errors when cleanup cannot be confirmed.
|
||||
async fn cleanup_grant(&self, lock_id: &LockId) -> Result<LockCleanupOutcome> {
|
||||
self.release(lock_id).await.map(|released| {
|
||||
if released {
|
||||
LockCleanupOutcome::Released
|
||||
} else {
|
||||
LockCleanupOutcome::Unconfirmed
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Release multiple locks. Default implementation fans out to single-lock releases.
|
||||
async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
|
||||
Ok(join_all(lock_ids.iter().map(|lock_id| self.release(lock_id)))
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::{
|
||||
ObjectKey,
|
||||
client::{LockCleanupOutcome, LockClient},
|
||||
client::LockClient,
|
||||
error::{LockError, Result},
|
||||
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
|
||||
};
|
||||
@@ -36,10 +36,6 @@ use tokio::time::Instant;
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const EVENT_DISTRIBUTED_LOCK_CLEANUP: &str = "distributed_lock_cleanup";
|
||||
const LOG_COMPONENT_LOCK: &str = "lock";
|
||||
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
|
||||
|
||||
const UNLOCK_RETRY_ATTEMPTS: usize = 3;
|
||||
const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
|
||||
/// Slow retry schedule for unlocks that survive the fast retry loop. Lock RPC
|
||||
@@ -867,15 +863,7 @@ impl DistributedLock {
|
||||
for attempt in 1..=UNLOCK_RETRY_ATTEMPTS {
|
||||
pending = Self::release_pending_once(pending, attempt, context).await;
|
||||
if pending.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_DISTRIBUTED_LOCK_CLEANUP,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "completed",
|
||||
attempt,
|
||||
context,
|
||||
"Distributed unlock cleanup completed"
|
||||
);
|
||||
debug!(attempt, context, "distributed unlock completed");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -885,14 +873,10 @@ impl DistributedLock {
|
||||
}
|
||||
|
||||
warn!(
|
||||
event = EVENT_DISTRIBUTED_LOCK_CLEANUP,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "deferred",
|
||||
remaining = pending.len(),
|
||||
attempts = UNLOCK_RETRY_ATTEMPTS,
|
||||
context,
|
||||
"Distributed unlock cleanup remains unconfirmed; continuing with deferred retries"
|
||||
"distributed unlock left unreleased entries after retry; continuing with deferred retries"
|
||||
);
|
||||
Self::release_entries_deferred(pending, context).await;
|
||||
}
|
||||
@@ -916,33 +900,14 @@ impl DistributedLock {
|
||||
context: &'static str,
|
||||
) -> Vec<(LockId, Arc<dyn LockClient>)> {
|
||||
let release_results = join_all(pending.into_iter().map(|(lock_id, client)| async move {
|
||||
match client.cleanup_grant(&lock_id).await {
|
||||
Ok(LockCleanupOutcome::Released | LockCleanupOutcome::AlreadyAbsent) => None,
|
||||
Ok(LockCleanupOutcome::Unconfirmed) => {
|
||||
warn!(
|
||||
event = EVENT_DISTRIBUTED_LOCK_CLEANUP,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "unconfirmed",
|
||||
%lock_id,
|
||||
attempt,
|
||||
context,
|
||||
"Distributed unlock cleanup was not confirmed by client"
|
||||
);
|
||||
match client.release(&lock_id).await {
|
||||
Ok(true) => None,
|
||||
Ok(false) => {
|
||||
warn!(%lock_id, attempt, context, "distributed unlock did not find lock on client");
|
||||
Some((lock_id, client))
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_DISTRIBUTED_LOCK_CLEANUP,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "failed",
|
||||
%lock_id,
|
||||
attempt,
|
||||
context,
|
||||
error = %err,
|
||||
"Distributed unlock cleanup failed on client"
|
||||
);
|
||||
warn!(%lock_id, attempt, context, "distributed unlock failed on client: {}", err);
|
||||
Some((lock_id, client))
|
||||
}
|
||||
}
|
||||
@@ -954,8 +919,7 @@ impl DistributedLock {
|
||||
|
||||
/// Bounded slow retries for entries the fast loop could not release. Every
|
||||
/// caller runs on a background task, so waiting here blocks nobody; after
|
||||
/// the schedule is exhausted cleanup remains unconfirmed. A backend lease
|
||||
/// bounds the lifetime of any grant that still exists.
|
||||
/// the schedule is exhausted the server lease reclaims the entry.
|
||||
async fn release_entries_deferred(mut pending: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
|
||||
let mut attempt = UNLOCK_RETRY_ATTEMPTS;
|
||||
for delay in DEFERRED_UNLOCK_BACKOFF {
|
||||
@@ -963,28 +927,16 @@ impl DistributedLock {
|
||||
attempt += 1;
|
||||
pending = Self::release_pending_once(pending, attempt, context).await;
|
||||
if pending.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_DISTRIBUTED_LOCK_CLEANUP,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "completed",
|
||||
attempt,
|
||||
context,
|
||||
"Deferred distributed unlock cleanup converged"
|
||||
);
|
||||
debug!(attempt, context, "deferred distributed unlock converged");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
warn!(
|
||||
event = EVENT_DISTRIBUTED_LOCK_CLEANUP,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "unconfirmed",
|
||||
remaining = pending.len(),
|
||||
attempts = attempt,
|
||||
context,
|
||||
"Distributed unlock cleanup remains unconfirmed after deferred retries"
|
||||
"distributed unlock abandoned entries after deferred retry; the server lease will expire them"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1473,17 +1425,14 @@ fn record_lock_held_release(lock_type: LockType) {
|
||||
mod tests {
|
||||
use super::{
|
||||
DEFERRED_UNLOCK_BACKOFF, DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF,
|
||||
LOCK_ACQUIRE_SPARE_HEDGES, LockAcquireFailureKind, LockLostSignal, UNLOCK_RETRY_ATTEMPTS, UNLOCK_RETRY_BACKOFF,
|
||||
is_remote_lock_rpc_failure, should_warn_lock_failure,
|
||||
};
|
||||
use crate::{
|
||||
GlobalLockManager, LocalClient, LockCleanupOutcome, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats,
|
||||
LockType, ObjectKey, client::LockClient,
|
||||
LOCK_ACQUIRE_SPARE_HEDGES, LockAcquireFailureKind, LockLostSignal, UNLOCK_RETRY_ATTEMPTS, is_remote_lock_rpc_failure,
|
||||
should_warn_lock_failure,
|
||||
};
|
||||
use crate::{LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, ObjectKey, client::LockClient};
|
||||
use rand::{SeedableRng as _, TryRng, rngs::StdRng};
|
||||
use std::assert_matches;
|
||||
use std::convert::Infallible;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::{Arc, Mutex},
|
||||
@@ -2000,7 +1949,6 @@ mod tests {
|
||||
struct FlakyReleaseClient {
|
||||
failures_left: AtomicUsize,
|
||||
release_calls: Arc<AtomicUsize>,
|
||||
release_success: bool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -2015,7 +1963,7 @@ mod tests {
|
||||
self.failures_left.fetch_sub(1, Ordering::SeqCst);
|
||||
return Err(LockError::internal("remote lock rpc timed out: release"));
|
||||
}
|
||||
Ok(self.release_success)
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
|
||||
@@ -2053,7 +2001,6 @@ mod tests {
|
||||
let client: Arc<dyn LockClient> = Arc::new(FlakyReleaseClient {
|
||||
failures_left: AtomicUsize::new(UNLOCK_RETRY_ATTEMPTS + 2),
|
||||
release_calls: release_calls.clone(),
|
||||
release_success: true,
|
||||
});
|
||||
let lock_id = LockId::new_unique(&ObjectKey::new("bucket", "object"));
|
||||
|
||||
@@ -2072,7 +2019,6 @@ mod tests {
|
||||
let client: Arc<dyn LockClient> = Arc::new(FlakyReleaseClient {
|
||||
failures_left: AtomicUsize::new(usize::MAX),
|
||||
release_calls: release_calls.clone(),
|
||||
release_success: true,
|
||||
});
|
||||
let lock_id = LockId::new_unique(&ObjectKey::new("bucket", "object"));
|
||||
|
||||
@@ -2081,209 +2027,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
release_calls.load(Ordering::SeqCst),
|
||||
UNLOCK_RETRY_ATTEMPTS + DEFERRED_UNLOCK_BACKOFF.len(),
|
||||
"the retry budget stays bounded when cleanup cannot be confirmed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn release_entries_keeps_retrying_unconfirmed_legacy_results() {
|
||||
let release_calls = Arc::new(AtomicUsize::new(0));
|
||||
let client: Arc<dyn LockClient> = Arc::new(FlakyReleaseClient {
|
||||
failures_left: AtomicUsize::new(0),
|
||||
release_calls: release_calls.clone(),
|
||||
release_success: false,
|
||||
});
|
||||
let lock_id = LockId::new_unique(&ObjectKey::new("bucket", "legacy-unlock"));
|
||||
|
||||
DistributedLock::release_entries(vec![(lock_id, client)], "test_legacy_unlock").await;
|
||||
|
||||
assert_eq!(
|
||||
release_calls.load(Ordering::SeqCst),
|
||||
UNLOCK_RETRY_ATTEMPTS + DEFERRED_UNLOCK_BACKOFF.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Uses a real local backend, with only the transport acknowledgement lost.
|
||||
#[derive(Debug)]
|
||||
struct AckLossClient {
|
||||
backend: Arc<LocalClient>,
|
||||
lose_ack: AtomicBool,
|
||||
release_calls: AtomicUsize,
|
||||
released: tokio::sync::Notify,
|
||||
resume: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl AckLossClient {
|
||||
fn new(lose_ack: bool) -> Self {
|
||||
Self {
|
||||
backend: Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))),
|
||||
lose_ack: AtomicBool::new(lose_ack),
|
||||
release_calls: AtomicUsize::new(0),
|
||||
released: tokio::sync::Notify::new(),
|
||||
resume: tokio::sync::Notify::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LockClient for AckLossClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
self.backend.acquire_lock(request).await
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.cleanup_grant(lock_id)
|
||||
.await
|
||||
.map(|outcome| matches!(outcome, LockCleanupOutcome::Released))
|
||||
}
|
||||
|
||||
async fn cleanup_grant(&self, lock_id: &LockId) -> crate::Result<LockCleanupOutcome> {
|
||||
self.release_calls.fetch_add(1, Ordering::SeqCst);
|
||||
let outcome = self.backend.cleanup_grant(lock_id).await?;
|
||||
if outcome == LockCleanupOutcome::Released {
|
||||
self.released.notify_one();
|
||||
if self.lose_ack.swap(false, Ordering::SeqCst) {
|
||||
self.resume.notified().await;
|
||||
return Err(LockError::internal("remote lock rpc timed out: release"));
|
||||
}
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.backend.refresh(lock_id).await
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.backend.force_release(lock_id).await
|
||||
}
|
||||
|
||||
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
self.backend.check_status(lock_id).await
|
||||
}
|
||||
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
self.backend.get_stats().await
|
||||
}
|
||||
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
self.backend.close().await
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn is_local(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn release_entries_converges_after_lost_ack_without_releasing_successor() {
|
||||
let client = Arc::new(AckLossClient::new(true));
|
||||
let request = LockRequest::new(ObjectKey::new("bucket", "lost-release-ack"), LockType::Exclusive, "first-owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("acquire predecessor").success);
|
||||
let cleanup_client = client.clone();
|
||||
let lock_id = request.lock_id.clone();
|
||||
let started = tokio::time::Instant::now();
|
||||
let cleanup = tokio::spawn(async move {
|
||||
DistributedLock::release_entries(vec![(lock_id, cleanup_client)], "test_lost_release_ack").await;
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), client.released.notified())
|
||||
.await
|
||||
.expect("predecessor was released before its ACK was lost");
|
||||
|
||||
let successor = LockRequest::new(request.resource.clone(), LockType::Exclusive, "successor");
|
||||
assert!(client.acquire_lock(&successor).await.expect("acquire successor").success);
|
||||
client.resume.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(60), cleanup)
|
||||
.await
|
||||
.expect("cleanup stays bounded")
|
||||
.expect("cleanup task completed");
|
||||
|
||||
assert_eq!(
|
||||
client.release_calls.load(Ordering::SeqCst),
|
||||
2,
|
||||
"stop at the first authoritative absence response"
|
||||
);
|
||||
assert_eq!(started.elapsed(), UNLOCK_RETRY_BACKOFF, "do not enter the deferred retry schedule");
|
||||
assert!(
|
||||
client
|
||||
.backend
|
||||
.check_status(&request.lock_id)
|
||||
.await
|
||||
.expect("check predecessor")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
client
|
||||
.backend
|
||||
.check_status(&successor.lock_id)
|
||||
.await
|
||||
.expect("check successor")
|
||||
.is_some()
|
||||
);
|
||||
// FastLock acquisition deadlines use the real clock. Resume Tokio time
|
||||
// so virtual-time retries cannot expire the successor's lease while
|
||||
// waiting for the contender's real-time deadline.
|
||||
tokio::time::resume();
|
||||
let contender =
|
||||
LockRequest::new(request.resource, LockType::Exclusive, "contender").with_acquire_timeout(Duration::from_millis(50));
|
||||
assert!(
|
||||
!client
|
||||
.acquire_lock(&contender)
|
||||
.await
|
||||
.expect("check successor exclusivity")
|
||||
.success
|
||||
);
|
||||
assert!(client.backend.release(&successor.lock_id).await.expect("release successor"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn release_before_acquire_keeps_the_pending_grant_cleanup_owner() {
|
||||
let client = Arc::new(AckLossClient::new(false));
|
||||
let blocker = LockRequest::new(ObjectKey::new("bucket", "release-before-grant"), LockType::Exclusive, "blocker");
|
||||
assert!(client.acquire_lock(&blocker).await.expect("acquire blocker").success);
|
||||
let late = LockRequest::new(blocker.resource.clone(), LockType::Exclusive, "late-owner");
|
||||
let late_id = late.lock_id.clone();
|
||||
let started = Arc::new(tokio::sync::Notify::new());
|
||||
let task_started = started.clone();
|
||||
let acquire_client = client.clone();
|
||||
let mut pending = tokio::task::JoinSet::new();
|
||||
pending.spawn(async move {
|
||||
task_started.notify_one();
|
||||
(0, acquire_client.acquire_lock(&late).await)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), started.notified())
|
||||
.await
|
||||
.expect("pending acquire task started");
|
||||
assert!(!client.release(&late_id).await.expect("release arrived before grant"));
|
||||
let clients: Arc<[Arc<dyn LockClient>]> = vec![client.clone() as Arc<dyn LockClient>].into();
|
||||
DistributedLock::spawn_pending_cleanup(pending, clients, late_id.clone(), "test_release_before_grant");
|
||||
assert!(
|
||||
client
|
||||
.backend
|
||||
.release(&blocker.lock_id)
|
||||
.await
|
||||
.expect("unblock pending acquire")
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(1), client.released.notified())
|
||||
.await
|
||||
.expect("late grant retained its independent cleanup owner");
|
||||
|
||||
assert!(
|
||||
client
|
||||
.backend
|
||||
.check_status(&late_id)
|
||||
.await
|
||||
.expect("check late grant")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
client.release_calls.load(Ordering::SeqCst),
|
||||
2,
|
||||
"clean up again after the late grant actually succeeds"
|
||||
"the retry budget is bounded; the server lease reclaims what remains"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ pub mod types;
|
||||
// Re-export main types for easy access
|
||||
pub use crate::{
|
||||
// Client interfaces
|
||||
client::{LockCleanupOutcome, LockClient, local::LocalClient},
|
||||
client::{LockClient, local::LocalClient},
|
||||
distributed_lock::DistributedLockGuard,
|
||||
// Error types
|
||||
error::{LockError, Result},
|
||||
|
||||
@@ -59,14 +59,13 @@ pub(super) fn rules() -> Vec<Rule> {
|
||||
)
|
||||
},
|
||||
Rule {
|
||||
anchors: strings(["Distributed unlock cleanup failed on client"]),
|
||||
anchors: strings(["distributed unlock failed on client"]),
|
||||
..base(
|
||||
"dist-unlock-failed",
|
||||
P2Degraded,
|
||||
"lock",
|
||||
"分布式加/解锁在部分节点失败",
|
||||
any([
|
||||
contains("Distributed unlock cleanup failed on client"),
|
||||
contains("distributed unlock failed on client"),
|
||||
contains("Failed to acquire lock on client"),
|
||||
]),
|
||||
|
||||
@@ -165,7 +165,7 @@ fn every_rule_has_a_positive_sample() {
|
||||
("lock-acquire-timeout", msg("Lock acquisition timeout for resource 'b/o' after 30s")),
|
||||
("lock-quorum-nodes", msg("Quorum not reached: required 3, achieved 1")),
|
||||
("lock-owner-mismatch", msg("Not the lock owner: lock_id abc, owner node1")),
|
||||
("dist-unlock-failed", msg("Distributed unlock cleanup failed on client")),
|
||||
("dist-unlock-failed", msg("distributed unlock failed on client: node2")),
|
||||
(
|
||||
"lock-state-inconsistent",
|
||||
msg("Atomic state inconsistency during exclusive lock release: owner=x, atomic_state=101"),
|
||||
@@ -305,32 +305,6 @@ fn every_rule_has_a_positive_sample() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distributed_unlock_rule_matches_current_and_legacy_failures_only() {
|
||||
let engine = engine();
|
||||
for (message, expected) in [
|
||||
("Distributed unlock cleanup failed on client", true),
|
||||
("distributed unlock failed on client: node2", true),
|
||||
("Failed to acquire lock on client node2", true),
|
||||
("Distributed unlock cleanup completed", false),
|
||||
("Deferred distributed unlock cleanup converged", false),
|
||||
("Distributed unlock cleanup was not confirmed by client", false),
|
||||
("Distributed unlock cleanup remains unconfirmed after deferred retries", false),
|
||||
] {
|
||||
let sample = Sample {
|
||||
message,
|
||||
level: Some(LogLevel::Warn),
|
||||
target: "rustfs_lock::distributed_lock",
|
||||
..Default::default()
|
||||
};
|
||||
let matched = engine
|
||||
.matches(&event(&sample))
|
||||
.iter()
|
||||
.any(|&index| engine.rules()[index].id == "dist-unlock-failed");
|
||||
assert_eq!(matched, expected, "unexpected match for {message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_writer_rule_requires_all_targets_and_retains_legacy_messages() {
|
||||
let engine = engine();
|
||||
|
||||
@@ -56,17 +56,6 @@ pub(super) fn dot_entries() -> Vec<File> {
|
||||
]
|
||||
}
|
||||
|
||||
/// Client-visible S3 key for an SFTP sub-directory.
|
||||
///
|
||||
/// `object_key` is the parsed prefix without a trailing slash. The
|
||||
/// returned key ends in `/` so the storage layer can encode it to the
|
||||
/// reserved directory marker. Callers must not encode the key first:
|
||||
/// `validate_object_key` rejects any client key that already contains
|
||||
/// the marker suffix.
|
||||
fn directory_client_key(object_key: &str) -> String {
|
||||
format!("{object_key}/")
|
||||
}
|
||||
|
||||
fn directory_listing_input(
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
@@ -311,19 +300,19 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// MKDIR for a sub-directory path: write a zero-byte object at the
|
||||
/// client-visible directory key (`foo/`). Storage encodes that key to
|
||||
/// the on-disk marker `foo__XLDIR__` after the S3 client-key boundary.
|
||||
/// The encoded suffix is reserved, so PutObject must not be given it.
|
||||
/// MKDIR for a sub-directory path: write a zero-byte object at
|
||||
/// encode_dir_object(prefix + "/"). The encoding maps "foo/" to
|
||||
/// "foo__XLDIR__", which matches the RustFS marker convention used
|
||||
/// by the S3, Swift, and WebDAV backends.
|
||||
pub(super) async fn mkdir_subdir_marker(&self, bucket: &str, object_key: &str) -> Result<(), SftpError> {
|
||||
let dir_key = directory_client_key(object_key);
|
||||
self.authorize(&S3Action::PutObject, bucket, Some(&dir_key)).await?;
|
||||
let marker_key = path::encode_dir_object(&format!("{object_key}/"));
|
||||
self.authorize(&S3Action::PutObject, bucket, Some(&marker_key)).await?;
|
||||
|
||||
let body = stream::once(async { Ok::<Bytes, std::io::Error>(Bytes::new()) });
|
||||
let streaming = StreamingBlob::wrap(body);
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(dir_key)
|
||||
.key(marker_key.clone())
|
||||
.content_length(Some(0))
|
||||
.body(Some(streaming))
|
||||
.build()
|
||||
@@ -344,14 +333,15 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
}
|
||||
|
||||
/// RMDIR for a sub-directory path: validate no objects under the
|
||||
/// prefix, then authorise and delete the trailing-slash directory key.
|
||||
/// Storage encodes that key to the on-disk marker.
|
||||
/// prefix, then authorise and delete the __XLDIR__ marker that
|
||||
/// represents the directory.
|
||||
pub(super) async fn rmdir_subdir_marker(&self, bucket: &str, object_key: &str) -> Result<(), SftpError> {
|
||||
let dir_key = directory_client_key(object_key);
|
||||
self.validate_directory_empty(bucket, &dir_key).await?;
|
||||
let prefix = format!("{object_key}/");
|
||||
self.validate_directory_empty(bucket, &prefix).await?;
|
||||
|
||||
self.authorize(&S3Action::DeleteObject, bucket, Some(&dir_key)).await?;
|
||||
self.run_backend("delete_object", self.storage.delete_object(bucket, &dir_key, self.credentials()))
|
||||
let marker_key = path::encode_dir_object(&prefix);
|
||||
self.authorize(&S3Action::DeleteObject, bucket, Some(&marker_key)).await?;
|
||||
self.run_backend("delete_object", self.storage.delete_object(bucket, &marker_key, self.credentials()))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -847,95 +837,4 @@ mod tests {
|
||||
"error-level event must carry the SFTP READDIR failed message, captured: {captured}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mkdir_subdir_puts_trailing_slash_client_key() {
|
||||
let backend = Arc::new(DummyBackend::new());
|
||||
let mut driver = build_driver(Arc::clone(&backend), TEST_PART_SIZE);
|
||||
let authorized = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let record = Arc::clone(&authorized);
|
||||
|
||||
let status = with_test_auth_override(
|
||||
move |action, bucket, key| {
|
||||
record
|
||||
.lock()
|
||||
.expect("auth log")
|
||||
.push((action.as_str(), bucket.to_string(), key.map(str::to_string)));
|
||||
true
|
||||
},
|
||||
driver.mkdir(1, "/bucket/photos/2024".to_string(), russh_sftp::protocol::FileAttributes::default()),
|
||||
)
|
||||
.await
|
||||
.expect("mkdir of an ordinary directory");
|
||||
|
||||
assert!(matches!(status.status_code, StatusCode::Ok));
|
||||
let calls = backend.put_object_calls();
|
||||
assert_eq!(calls.len(), 1, "mkdir must issue one PutObject");
|
||||
assert_eq!(calls[0].bucket, "bucket");
|
||||
assert_eq!(calls[0].key, "photos/2024/");
|
||||
assert!(
|
||||
!calls[0].key.contains(rustfs_utils::path::GLOBAL_DIR_SUFFIX),
|
||||
"the client key must not carry the reserved directory-marker suffix"
|
||||
);
|
||||
let auth = authorized.lock().expect("auth log");
|
||||
assert_eq!(
|
||||
auth.as_slice(),
|
||||
[("s3:PutObject", "bucket".to_string(), Some("photos/2024/".to_string()))]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmdir_subdir_deletes_trailing_slash_client_key() {
|
||||
let backend = Arc::new(DummyBackend::new());
|
||||
backend.queue_delete_object_ok();
|
||||
let mut driver = build_driver(Arc::clone(&backend), TEST_PART_SIZE);
|
||||
let authorized = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let record = Arc::clone(&authorized);
|
||||
|
||||
let status = with_test_auth_override(
|
||||
move |action, bucket, key| {
|
||||
record
|
||||
.lock()
|
||||
.expect("auth log")
|
||||
.push((action.as_str(), bucket.to_string(), key.map(str::to_string)));
|
||||
true
|
||||
},
|
||||
driver.rmdir(2, "/bucket/photos/2024".to_string()),
|
||||
)
|
||||
.await
|
||||
.expect("rmdir of an empty directory");
|
||||
|
||||
assert!(matches!(status.status_code, StatusCode::Ok));
|
||||
let calls = backend.delete_object_calls();
|
||||
assert_eq!(calls.len(), 1, "rmdir must issue one DeleteObject");
|
||||
assert_eq!(calls[0].bucket, "bucket");
|
||||
assert_eq!(calls[0].key, "photos/2024/");
|
||||
assert!(
|
||||
!calls[0].key.contains(rustfs_utils::path::GLOBAL_DIR_SUFFIX),
|
||||
"the client key must not carry the reserved directory-marker suffix"
|
||||
);
|
||||
let auth = authorized.lock().expect("auth log");
|
||||
assert_eq!(
|
||||
auth.as_slice(),
|
||||
[
|
||||
("s3:ListBucket", "bucket".to_string(), Some("photos/2024/".to_string())),
|
||||
("s3:DeleteObject", "bucket".to_string(), Some("photos/2024/".to_string())),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mkdir_rejects_reserved_directory_marker_before_put() {
|
||||
let backend = Arc::new(DummyBackend::new());
|
||||
let mut driver = build_driver(Arc::clone(&backend), TEST_PART_SIZE);
|
||||
let path = format!("/bucket/sub{}", rustfs_utils::path::GLOBAL_DIR_SUFFIX);
|
||||
|
||||
let err = driver
|
||||
.mkdir(3, path, russh_sftp::protocol::FileAttributes::default())
|
||||
.await
|
||||
.expect_err("a client path containing the reserved marker must be rejected");
|
||||
|
||||
assert_eq!(StatusCode::from(err), StatusCode::BadMessage);
|
||||
assert!(backend.put_object_calls().is_empty(), "rejection must happen before PutObject");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -831,10 +831,14 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
|
||||
/// SSH_FXP_MKDIR, SFTP Internet Draft section 6.6. Bucket-level path
|
||||
/// (only the bucket component is set) issues CreateBucket. Sub-bucket
|
||||
/// path issues PutObject of a zero-byte object at the logical prefix
|
||||
/// plus a trailing slash. Storage encodes that client key into the
|
||||
/// directory marker. MKDIR at the SFTP root returns Failure because
|
||||
/// there is no parent into which a new top-level entity could be added.
|
||||
/// path issues PutObject of a zero-byte object at the encoded
|
||||
/// directory marker key. MKDIR at the SFTP root returns Failure
|
||||
/// because there is no parent into which a new top-level entity could
|
||||
/// be added.
|
||||
///
|
||||
/// The directory-marker key is built with rustfs_utils::path::
|
||||
/// encode_dir_object so the key format matches the convention used
|
||||
/// by the rest of RustFS (S3, Swift, WebDAV).
|
||||
#[tracing::instrument(level = "info", skip(self, _attrs), fields(id, path = %sanitise_control_bytes(&path)), err(Debug))]
|
||||
async fn mkdir(&mut self, id: u32, path: String, _attrs: FileAttributes) -> Result<Status, Self::Error> {
|
||||
self.enforce_server_readonly()?;
|
||||
@@ -854,9 +858,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
/// SSH_FXP_RMDIR, SFTP Internet Draft section 6.6. Empty check then
|
||||
/// delete. Bucket-level path lists the bucket with max_keys=1 and,
|
||||
/// on an empty result, calls DeleteBucket. Sub-bucket path lists the
|
||||
/// prefix and, on an empty result, calls DeleteObject on the
|
||||
/// trailing-slash directory key. Storage encodes that key to the
|
||||
/// on-disk marker.
|
||||
/// prefix and, on an empty result, calls DeleteObject on the encoded
|
||||
/// directory marker.
|
||||
///
|
||||
/// validate_directory_empty propagates the list error rather than
|
||||
/// swallowing it. Without that, a transient backend error during
|
||||
|
||||
@@ -82,22 +82,6 @@ Configure WebDAV via environment variables:
|
||||
| `RUSTFS_WEBDAV_MAX_BODY_SIZE` | Max upload size (bytes) | 5GB |
|
||||
| `RUSTFS_WEBDAV_REQUEST_TIMEOUT` | Request timeout (seconds) | 300 |
|
||||
| `RUSTFS_WEBDAV_MAX_CONNECTIONS` | Max concurrently served connections | 1024 |
|
||||
| `RUSTFS_CORS_ALLOWED_ORIGINS` | Browser origins allowed by the S3 and WebDAV listeners | empty |
|
||||
|
||||
## Browser Access and CORS
|
||||
|
||||
WebDAV reads `RUSTFS_CORS_ALLOWED_ORIGINS` at server startup. Set a comma-separated list of exact frontend origins to allow browser clients, for example `https://dav-console.example,https://files.example`. Explicit origins allow credentialed requests. The default empty value disables cross-origin access. A standalone `*` permits any origin with credentials mode `omit` and emits no `Access-Control-Allow-Credentials` header.
|
||||
|
||||
Valid `OPTIONS` preflight requests from an allowed origin receive `204 No Content` without Basic authentication. Actual WebDAV requests and ordinary `OPTIONS` requests still use Basic authentication and the existing IAM checks. Allowed origins receive CORS headers on success and error responses. Preflights allow WebDAV methods and explicitly echo valid requested header names, including `Authorization`, `Depth`, `Destination`, and `If-Match`. Responses expose `ETag`, `DAV`, `Allow`, `Lock-Token`, and range headers to browser code.
|
||||
|
||||
```bash
|
||||
curl -i -X OPTIONS http://localhost:8080/ \
|
||||
-H 'Origin: https://dav-console.example' \
|
||||
-H 'Access-Control-Request-Method: PROPFIND' \
|
||||
-H 'Access-Control-Request-Headers: authorization,depth,content-type'
|
||||
```
|
||||
|
||||
Configure an explicit HTTPS frontend origin for credentialed browser access and use HTTPS on the WebDAV listener when sending access keys and secret keys. This listener-wide policy applies across WebDAV paths. S3 bucket CORS rules remain scoped to the S3 API.
|
||||
|
||||
## Quick Start
|
||||
|
||||
|
||||
@@ -19,12 +19,8 @@ use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, is_tem
|
||||
use bytes::Bytes;
|
||||
use dav_server::DavHandler;
|
||||
use dav_server::fakels::FakeLs;
|
||||
use http::header::{
|
||||
ACCESS_CONTROL_ALLOW_CREDENTIALS, ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN,
|
||||
ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, AUTHORIZATION, ORIGIN, REFERER,
|
||||
USER_AGENT, VARY,
|
||||
};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use http::header::{AUTHORIZATION, REFERER, USER_AGENT};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
|
||||
use hyper::body::Body as HttpBody;
|
||||
use hyper::server::conn::http1;
|
||||
@@ -35,7 +31,6 @@ use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_
|
||||
use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
|
||||
use rustfs_utils::MaskedAccessKey;
|
||||
use std::convert::Infallible;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::net::IpAddr;
|
||||
use std::pin::Pin;
|
||||
@@ -65,177 +60,6 @@ const EVENT_WEBDAV_CONNECTION_CAP_STATE: &str = "webdav_connection_cap_state";
|
||||
/// materialise a whole object in memory for every GET.
|
||||
type WebDavBody = Pin<Box<dyn HttpBody<Data = Bytes, Error = io::Error> + Send>>;
|
||||
|
||||
/// Listener-wide browser policy, parsed once and shared across connections.
|
||||
enum WebDavCors {
|
||||
Disabled,
|
||||
Wildcard,
|
||||
Origins(Vec<HeaderValue>),
|
||||
}
|
||||
|
||||
impl WebDavCors {
|
||||
const METHODS: &str = "OPTIONS, GET, HEAD, PUT, DELETE, PATCH, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK";
|
||||
|
||||
fn new(origins: &str) -> Self {
|
||||
match origins.trim() {
|
||||
"" => Self::Disabled,
|
||||
"*" => Self::Wildcard,
|
||||
origins => Self::Origins(
|
||||
origins
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|origin| !origin.is_empty() && *origin != "*")
|
||||
.filter_map(|origin| HeaderValue::from_str(origin).ok())
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn allowed_origin(&self, headers: &HeaderMap) -> Option<HeaderValue> {
|
||||
if matches!(self, Self::Disabled) {
|
||||
return None;
|
||||
}
|
||||
let mut origins = headers.get_all(ORIGIN).iter();
|
||||
let origin = origins.next()?;
|
||||
let value = origin.to_str().ok()?;
|
||||
if origins.next().is_some() || value.is_empty() || value.bytes().any(|byte| byte.is_ascii_whitespace() || byte == b',') {
|
||||
return None;
|
||||
}
|
||||
match self {
|
||||
Self::Origins(allowed) => allowed.iter().find(|allowed| *allowed == origin).cloned(),
|
||||
Self::Wildcard => Some(HeaderValue::from_static("*")),
|
||||
Self::Disabled => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep the CORS boundary outside authentication and DAV dispatch so every
|
||||
/// application response, including early errors, follows the same policy.
|
||||
async fn handle<B, F, Fut>(&self, req: Request<B>, next: F) -> Result<Response<WebDavBody>, Infallible>
|
||||
where
|
||||
F: FnOnce(Request<B>) -> Fut,
|
||||
Fut: Future<Output = Result<Response<WebDavBody>, Infallible>>,
|
||||
{
|
||||
let allow_origin = self.allowed_origin(req.headers());
|
||||
let is_options = req.method() == Method::OPTIONS;
|
||||
let is_preflight =
|
||||
is_options && req.headers().contains_key(ORIGIN) && req.headers().contains_key(ACCESS_CONTROL_REQUEST_METHOD);
|
||||
|
||||
let mut response = if is_preflight {
|
||||
self.preflight(req.headers(), allow_origin.is_some())
|
||||
} else {
|
||||
next(req).await?
|
||||
};
|
||||
|
||||
// Absent and rejected Origin values also select cache variants. Append
|
||||
// rather than overwrite the DAV handler's existing Vary fields.
|
||||
response.headers_mut().append(VARY, HeaderValue::from_static("Origin"));
|
||||
if is_options {
|
||||
response.headers_mut().append(
|
||||
VARY,
|
||||
HeaderValue::from_static("Access-Control-Request-Method, Access-Control-Request-Headers"),
|
||||
);
|
||||
}
|
||||
if let Some(origin) = allow_origin {
|
||||
response.headers_mut().insert(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
|
||||
if matches!(self, Self::Origins(_)) {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(ACCESS_CONTROL_ALLOW_CREDENTIALS, HeaderValue::from_static("true"));
|
||||
}
|
||||
if !is_preflight {
|
||||
response.headers_mut().insert(
|
||||
ACCESS_CONTROL_EXPOSE_HEADERS,
|
||||
HeaderValue::from_static("ETag, DAV, Allow, Lock-Token, Content-Range, Accept-Ranges, WWW-Authenticate"),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn preflight(&self, headers: &HeaderMap, origin_allowed: bool) -> Response<WebDavBody> {
|
||||
let mut methods = headers.get_all(ACCESS_CONTROL_REQUEST_METHOD).iter();
|
||||
let method = methods.next().and_then(|value| value.to_str().ok());
|
||||
let method_allowed = method.is_some_and(|method| Self::METHODS.split(", ").any(|allowed| allowed == method));
|
||||
if !origin_allowed || !method_allowed || methods.next().is_some() {
|
||||
return error_response(StatusCode::FORBIDDEN, "CORS preflight denied");
|
||||
}
|
||||
let Ok(allowed_headers) = cors_requested_headers(headers) else {
|
||||
return error_response(StatusCode::FORBIDDEN, "CORS preflight denied");
|
||||
};
|
||||
|
||||
let mut response = Response::new(fixed_body(Bytes::new()));
|
||||
*response.status_mut() = StatusCode::NO_CONTENT;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(ACCESS_CONTROL_ALLOW_METHODS, HeaderValue::from_static(Self::METHODS));
|
||||
// A wildcard does not cover Authorization, and is not a wildcard for
|
||||
// credentialed requests. Echo the validated names explicitly instead.
|
||||
if let Some(value) = allowed_headers {
|
||||
response.headers_mut().insert(ACCESS_CONTROL_ALLOW_HEADERS, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate borrowed field names, reusing an ordinary single header value.
|
||||
/// RFC 9110 section 5.6.1.2 requires recipients to ignore empty list elements;
|
||||
/// normalize those and repeated fields into one nonempty response list.
|
||||
fn cors_requested_headers(headers: &HeaderMap) -> Result<Option<HeaderValue>, ()> {
|
||||
let values = headers.get_all(ACCESS_CONTROL_REQUEST_HEADERS);
|
||||
let mut fields = values.iter();
|
||||
let Some(first) = fields.next() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut normalize = fields.next().is_some();
|
||||
let mut has_name = false;
|
||||
for value in values.iter() {
|
||||
for name in value.to_str().map_err(|_| ())?.split(',') {
|
||||
let name = name.trim_matches([' ', '\t']);
|
||||
if name.is_empty() {
|
||||
normalize = true;
|
||||
continue;
|
||||
}
|
||||
if !is_cors_header_name(name) {
|
||||
return Err(());
|
||||
}
|
||||
has_name = true;
|
||||
}
|
||||
}
|
||||
if !has_name {
|
||||
return Err(());
|
||||
}
|
||||
if !normalize {
|
||||
return Ok(Some(first.clone()));
|
||||
}
|
||||
|
||||
let capacity = values.iter().map(|value| value.as_bytes().len() + 1).sum();
|
||||
let mut normalized = String::with_capacity(capacity);
|
||||
for value in values {
|
||||
for name in value.to_str().map_err(|_| ())?.split(',') {
|
||||
let name = name.trim_matches([' ', '\t']);
|
||||
if name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !normalized.is_empty() {
|
||||
normalized.push(',');
|
||||
}
|
||||
normalized.push_str(name);
|
||||
}
|
||||
}
|
||||
HeaderValue::try_from(normalized).map(Some).map_err(|_| ())
|
||||
}
|
||||
|
||||
/// RFC 9110 field-name is a nonempty sequence of ASCII tchar bytes.
|
||||
fn is_cors_header_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#' | b'$' | b'%' | b'&' | b'\'' | b'*' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~'
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
|
||||
let mut policy_headers = HeaderMap::new();
|
||||
for name in [USER_AGENT, REFERER] {
|
||||
@@ -259,7 +83,6 @@ where
|
||||
config: WebDavConfig,
|
||||
/// S3 storage backend
|
||||
storage: S,
|
||||
cors: Arc<WebDavCors>,
|
||||
}
|
||||
|
||||
impl<S> WebDavServer<S>
|
||||
@@ -277,13 +100,7 @@ where
|
||||
/// Create a new WebDAV server
|
||||
pub async fn new(config: WebDavConfig, storage: S) -> Result<Self, WebDavInitError> {
|
||||
config.validate().await?;
|
||||
let origins =
|
||||
rustfs_utils::get_env_str(rustfs_config::ENV_CORS_ALLOWED_ORIGINS, rustfs_config::DEFAULT_CORS_ALLOWED_ORIGINS);
|
||||
Ok(Self {
|
||||
config,
|
||||
storage,
|
||||
cors: Arc::new(WebDavCors::new(&origins)),
|
||||
})
|
||||
Ok(Self { config, storage })
|
||||
}
|
||||
|
||||
/// Start the WebDAV server
|
||||
@@ -399,7 +216,6 @@ where
|
||||
match accept_result {
|
||||
Ok((stream, addr)) => {
|
||||
let storage = storage.clone();
|
||||
let cors = self.cors.clone();
|
||||
let tls_acceptor = tls_acceptor.clone();
|
||||
|
||||
let max_body_size = self.config.max_body_size;
|
||||
@@ -418,7 +234,7 @@ where
|
||||
match timeout(request_timeout, acceptor.accept(stream)).await {
|
||||
Ok(Ok(tls_stream)) => {
|
||||
let io = TokioIo::new(tls_stream);
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, max_body_size, request_timeout, cors).await {
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, max_body_size, request_timeout).await {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_CONNECTION_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
@@ -456,7 +272,7 @@ where
|
||||
}
|
||||
} else {
|
||||
let io = TokioIo::new(stream);
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, max_body_size, request_timeout, cors).await {
|
||||
if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, max_body_size, request_timeout).await {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_CONNECTION_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
@@ -518,19 +334,13 @@ where
|
||||
secure_transport: bool,
|
||||
max_body_size: u64,
|
||||
request_timeout: Duration,
|
||||
cors: Arc<WebDavCors>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
// These borrows live for the connection. Preflights complete before
|
||||
// invoking the closure that clones the backend for an actual request.
|
||||
let cors = cors.as_ref();
|
||||
let storage = &storage;
|
||||
let service = service_fn(move |req: Request<hyper::body::Incoming>| {
|
||||
cors.handle(req, move |req| {
|
||||
Self::handle_request(req, storage.clone(), source_ip, secure_transport, max_body_size, request_timeout)
|
||||
})
|
||||
let storage = storage.clone();
|
||||
async move { Self::handle_request(req, storage, source_ip, secure_transport, max_body_size, request_timeout).await }
|
||||
});
|
||||
|
||||
// A peer that opens a connection and dribbles (or never finishes)
|
||||
@@ -883,7 +693,6 @@ mod tests {
|
||||
use s3s::dto::*;
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
@@ -892,19 +701,8 @@ mod tests {
|
||||
|
||||
/// Storage double for the connection-level tests. Those requests are
|
||||
/// rejected before authentication succeeds, so storage is never reached.
|
||||
#[derive(Default)]
|
||||
struct StubStorage {
|
||||
clone_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Clone for StubStorage {
|
||||
fn clone(&self) -> Self {
|
||||
self.clone_count.fetch_add(1, Ordering::Relaxed);
|
||||
Self {
|
||||
clone_count: self.clone_count.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct StubStorage;
|
||||
|
||||
impl Debug for StubStorage {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
@@ -1075,358 +873,6 @@ mod tests {
|
||||
.expect("build get request")
|
||||
}
|
||||
|
||||
const BROWSER_ORIGIN: &str = "https://dav-console.example";
|
||||
|
||||
async fn cors_http_request(origins: &str, method: &str, headers: &str) -> String {
|
||||
cors_http_exchange(origins, method, headers).await.0
|
||||
}
|
||||
|
||||
async fn cors_http_exchange(origins: &str, method: &str, headers: &str) -> (String, usize) {
|
||||
let storage = StubStorage::default();
|
||||
let clone_count = storage.clone_count.clone();
|
||||
let (mut client, server) = tokio::io::duplex(8192);
|
||||
let connection = tokio::spawn(WebDavServer::<StubStorage>::handle_connection_impl(
|
||||
TokioIo::new(server),
|
||||
storage,
|
||||
TEST_IP,
|
||||
false,
|
||||
16,
|
||||
Duration::from_secs(30),
|
||||
Arc::new(WebDavCors::new(origins)),
|
||||
));
|
||||
let request = format!("{method} / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n{headers}\r\n");
|
||||
client.write_all(request.as_bytes()).await.expect("write browser request");
|
||||
let mut response = Vec::new();
|
||||
timeout(Duration::from_secs(5), client.read_to_end(&mut response))
|
||||
.await
|
||||
.expect("browser response timeout")
|
||||
.expect("read browser response");
|
||||
timeout(Duration::from_secs(5), connection)
|
||||
.await
|
||||
.expect("connection shutdown timeout")
|
||||
.expect("connection task")
|
||||
.expect("serve browser request");
|
||||
(
|
||||
String::from_utf8(response).expect("ASCII HTTP response"),
|
||||
clone_count.load(Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
fn browser_request(method: &str, path: &str) -> Request<Full<Bytes>> {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(path)
|
||||
.header(ORIGIN, BROWSER_ORIGIN)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("build browser request")
|
||||
}
|
||||
|
||||
fn assert_browser_headers(response: &Response<WebDavBody>) {
|
||||
assert_eq!(response.headers()[ACCESS_CONTROL_ALLOW_ORIGIN], BROWSER_ORIGIN);
|
||||
assert_eq!(response.headers()[ACCESS_CONTROL_ALLOW_CREDENTIALS], "true");
|
||||
assert!(response.headers().get_all(VARY).iter().any(|value| value == "Origin"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_preflight_bypasses_authentication() {
|
||||
for method in [
|
||||
"OPTIONS",
|
||||
"GET",
|
||||
"HEAD",
|
||||
"PUT",
|
||||
"DELETE",
|
||||
"PATCH",
|
||||
"PROPFIND",
|
||||
"PROPPATCH",
|
||||
"MKCOL",
|
||||
"COPY",
|
||||
"MOVE",
|
||||
"LOCK",
|
||||
"UNLOCK",
|
||||
] {
|
||||
let headers = format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: {method}\r\nAccess-Control-Request-Headers: authorization,depth,content-type,destination,if-match\r\n"
|
||||
);
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 204"), "{method}: {response}");
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
assert!(response.contains("access-control-allow-credentials: true\r\n"));
|
||||
let allowed_methods = response
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("access-control-allow-methods: "))
|
||||
.expect("preflight must advertise allowed methods");
|
||||
assert!(allowed_methods.split(',').map(str::trim).any(|allowed| allowed == method));
|
||||
assert!(response.contains("access-control-allow-headers: authorization,depth,content-type,destination,if-match\r\n"));
|
||||
assert!(response.contains("vary: Access-Control-Request-Method, Access-Control-Request-Headers\r\n"));
|
||||
assert!(!response.contains("www-authenticate:"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_actual_requests_and_plain_options_require_authentication() {
|
||||
for method in ["GET", "PUT", "DELETE", "PROPFIND", "MOVE", "OPTIONS"] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, method, &format!("Origin: {BROWSER_ORIGIN}\r\n")).await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"), "{method}: {response}");
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
assert!(response.contains("www-authenticate: Basic"));
|
||||
}
|
||||
for method in ["GET", "PROPFIND"] {
|
||||
for headers in ["", "Origin: https://other.example\r\n"] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, method, headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"), "{method}: {response}");
|
||||
assert!(!response.contains("access-control-allow-origin:"));
|
||||
assert!(response.contains("vary: Origin\r\n"));
|
||||
}
|
||||
}
|
||||
for headers in [
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAuthorization: Basic !!!!\r\n"),
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PUT\r\n"),
|
||||
] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "PUT", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"));
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
}
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", "Access-Control-Request-Method: PROPFIND\r\n").await;
|
||||
assert!(response.starts_with("HTTP/1.1 401"));
|
||||
assert!(!response.contains("access-control-allow-origin:"));
|
||||
assert!(response.contains("vary: Origin\r\n"));
|
||||
assert!(response.contains("vary: Access-Control-Request-Method, Access-Control-Request-Headers\r\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_keeps_early_body_limit_errors_visible() {
|
||||
let response =
|
||||
cors_http_request(BROWSER_ORIGIN, "PUT", &format!("Origin: {BROWSER_ORIGIN}\r\nContent-Length: 17\r\n")).await;
|
||||
assert!(response.starts_with("HTTP/1.1 413"));
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_rejects_unconfigured_and_unmatched_origins() {
|
||||
for (configuration, origin) in [
|
||||
("", BROWSER_ORIGIN),
|
||||
(" ", BROWSER_ORIGIN),
|
||||
(BROWSER_ORIGIN, "https://dav-console.example.attacker.test"),
|
||||
(BROWSER_ORIGIN, "null"),
|
||||
(BROWSER_ORIGIN, "https://other.example"),
|
||||
("*, https://other.example", BROWSER_ORIGIN),
|
||||
] {
|
||||
let headers = format!("Origin: {origin}\r\nAccess-Control-Request-Method: PROPFIND\r\n");
|
||||
let response = cors_http_request(configuration, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 403"), "{configuration:?}: {response}");
|
||||
assert!(!response.contains("access-control-allow-origin:"));
|
||||
assert!(!response.contains("access-control-allow-credentials:"));
|
||||
assert!(response.contains("vary: Origin\r\n"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_rejects_malformed_preflights() {
|
||||
for headers in [
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: TRACE\r\n"),
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: propfind\r\n"),
|
||||
format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Method: GET\r\n"
|
||||
),
|
||||
format!("Origin: {BROWSER_ORIGIN}\r\nOrigin: https://other.example\r\nAccess-Control-Request-Method: PROPFIND\r\n"),
|
||||
format!("Origin: {BROWSER_ORIGIN},https://other.example\r\nAccess-Control-Request-Method: PROPFIND\r\n"),
|
||||
format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: invalid header\r\n"
|
||||
),
|
||||
format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: , \t,\r\n"
|
||||
),
|
||||
] {
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 403"), "{headers}: {response}");
|
||||
assert!(!response.contains("access-control-allow-methods:"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_normalizes_header_lists() {
|
||||
for requested in [
|
||||
"authorization, ,depth,",
|
||||
" , authorization,\tdepth , ",
|
||||
"authorization,\r\nAccess-Control-Request-Headers: , depth,",
|
||||
] {
|
||||
let headers = format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: {requested}\r\n"
|
||||
);
|
||||
let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 204"), "{response}");
|
||||
assert!(response.contains("access-control-allow-headers: authorization,depth\r\n"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webdav_cors_header_name_validation_matches_http_tokens() {
|
||||
for byte in u8::MIN..=u8::MAX {
|
||||
let bytes = [b'x', byte, b'y'];
|
||||
let actual = std::str::from_utf8(&bytes).is_ok_and(is_cors_header_name);
|
||||
assert_eq!(actual, http::HeaderName::from_bytes(&bytes).is_ok(), "byte {byte}");
|
||||
}
|
||||
assert!(!is_cors_header_name(""));
|
||||
assert!(is_cors_header_name("Authorization"));
|
||||
assert!(is_cors_header_name("!#$%&'*+-.^_`|~"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_only_clones_backend_for_actual_requests() {
|
||||
for (method, requested_method, expected_clones, expected_status) in [
|
||||
("OPTIONS", Some("PROPFIND"), 0, "204"),
|
||||
("OPTIONS", Some("TRACE"), 0, "403"),
|
||||
("OPTIONS", None, 1, "401"),
|
||||
("PUT", Some("PUT"), 1, "401"),
|
||||
] {
|
||||
let mut headers = format!("Origin: {BROWSER_ORIGIN}\r\n");
|
||||
if let Some(requested_method) = requested_method {
|
||||
headers.push_str(&format!("Access-Control-Request-Method: {requested_method}\r\n"));
|
||||
}
|
||||
let (response, clones) = cors_http_exchange(BROWSER_ORIGIN, method, &headers).await;
|
||||
assert!(response.starts_with(&format!("HTTP/1.1 {expected_status}")), "{response}");
|
||||
assert_eq!(clones, expected_clones, "{method} with {requested_method:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_wildcard_never_allows_credentials() {
|
||||
let headers = format!(
|
||||
"Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: Authorization\r\n"
|
||||
);
|
||||
let preflight = cors_http_request(" * ", "OPTIONS", &headers).await;
|
||||
assert!(preflight.starts_with("HTTP/1.1 204"));
|
||||
assert!(preflight.contains("access-control-allow-origin: *\r\n"));
|
||||
assert!(preflight.contains("access-control-allow-headers: Authorization\r\n"));
|
||||
assert!(!preflight.contains("access-control-allow-credentials:"));
|
||||
let actual = cors_http_request("*", "PROPFIND", &format!("Origin: {BROWSER_ORIGIN}\r\n")).await;
|
||||
assert!(actual.starts_with("HTTP/1.1 401"));
|
||||
assert!(actual.contains("access-control-allow-origin: *\r\n"));
|
||||
assert!(!actual.contains("access-control-allow-credentials:"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_allowlist_trims_entries_and_matches_exactly() {
|
||||
let configuration = format!(" https://other.example , {BROWSER_ORIGIN} ");
|
||||
let headers = format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\n");
|
||||
let response = cors_http_request(&configuration, "OPTIONS", &headers).await;
|
||||
assert!(response.starts_with("HTTP/1.1 204"));
|
||||
assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_wraps_dav_multistatus_etag_and_streamed_body() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
let handler = memfs_handler();
|
||||
let created = dispatch_dav(chunked_request(2, 4), handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await;
|
||||
assert_eq!(created.status(), StatusCode::CREATED);
|
||||
|
||||
let propfind = cors
|
||||
.handle(browser_request("PROPFIND", "/"), |req| async {
|
||||
Ok(dispatch_dav(req, handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("PROPFIND response");
|
||||
assert_eq!(propfind.status(), StatusCode::MULTI_STATUS);
|
||||
assert_browser_headers(&propfind);
|
||||
|
||||
let get = cors
|
||||
.handle(browser_request("GET", "/upload.bin"), |req| async {
|
||||
Ok(dispatch_dav(req, handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("GET response");
|
||||
assert_eq!(get.status(), StatusCode::OK);
|
||||
assert_browser_headers(&get);
|
||||
assert!(get.headers().contains_key("etag"));
|
||||
assert!(
|
||||
get.headers()[ACCESS_CONTROL_EXPOSE_HEADERS]
|
||||
.to_str()
|
||||
.expect("exposed headers")
|
||||
.contains("ETag")
|
||||
);
|
||||
assert!(get.body().size_hint().upper().is_none(), "CORS must preserve streaming");
|
||||
let data = get.into_body().collect().await.expect("stream GET body").to_bytes();
|
||||
assert_eq!(data.as_ref(), b"aaaaaaaa");
|
||||
|
||||
let mut put = browser_request("PUT", "/upload.bin");
|
||||
put.headers_mut()
|
||||
.insert("if-match", HeaderValue::from_static("\"stale-etag\""));
|
||||
let conditional = cors
|
||||
.handle(put, |req| async {
|
||||
Ok(dispatch_dav(req, handler, TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("conditional PUT response");
|
||||
assert_eq!(conditional.status(), StatusCode::PRECONDITION_FAILED);
|
||||
assert_browser_headers(&conditional);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_wraps_dispatch_body_limit_errors() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
let mut request = chunked_request(8, 8);
|
||||
request.headers_mut().insert(ORIGIN, HeaderValue::from_static(BROWSER_ORIGIN));
|
||||
let oversized = cors
|
||||
.handle(request, |req| async {
|
||||
Ok(dispatch_dav(req, memfs_handler(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("oversized response");
|
||||
assert_eq!(oversized.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
assert_browser_headers(&oversized);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn webdav_cors_wraps_dispatch_timeout_errors() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
let request = Request::builder()
|
||||
.method("PUT")
|
||||
.uri("/stalled.bin")
|
||||
.header(ORIGIN, BROWSER_ORIGIN)
|
||||
.body(StalledBody)
|
||||
.expect("stalled browser request");
|
||||
let response = cors
|
||||
.handle(request, |req| async {
|
||||
Ok(dispatch_dav(req, memfs_handler(), TEST_IP, 16, Duration::from_secs(30)).await)
|
||||
})
|
||||
.await
|
||||
.expect("timeout response");
|
||||
assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);
|
||||
assert_browser_headers(&response);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webdav_cors_preserves_existing_vary_on_success_and_errors() {
|
||||
let cors = WebDavCors::new(BROWSER_ORIGIN);
|
||||
for status in [
|
||||
StatusCode::OK,
|
||||
StatusCode::FORBIDDEN,
|
||||
StatusCode::NOT_FOUND,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
] {
|
||||
let response = cors
|
||||
.handle(browser_request("GET", "/"), |_| async {
|
||||
let mut response = error_response(status, "test response");
|
||||
response
|
||||
.headers_mut()
|
||||
.append(VARY, HeaderValue::from_static("Accept-Encoding"));
|
||||
Ok(response)
|
||||
})
|
||||
.await
|
||||
.expect("wrapped response");
|
||||
assert_eq!(response.status(), status);
|
||||
assert_browser_headers(&response);
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.get_all(VARY)
|
||||
.iter()
|
||||
.any(|value| value == "Accept-Encoding")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_headers_drop_credentials_and_s3_auth_spoofing() {
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -1525,12 +971,11 @@ mod tests {
|
||||
|
||||
let conn = tokio::spawn(WebDavServer::<StubStorage>::handle_connection_impl(
|
||||
TokioIo::new(server),
|
||||
StubStorage::default(),
|
||||
StubStorage,
|
||||
TEST_IP,
|
||||
false,
|
||||
1024,
|
||||
Duration::from_secs(30),
|
||||
Arc::new(WebDavCors::new("")),
|
||||
));
|
||||
|
||||
client
|
||||
@@ -1555,7 +1000,7 @@ mod tests {
|
||||
request_timeout_secs: 30,
|
||||
max_connections: 1,
|
||||
};
|
||||
let server = WebDavServer::new(config, StubStorage::default()).await.expect("build server");
|
||||
let server = WebDavServer::new(config, StubStorage).await.expect("build server");
|
||||
|
||||
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.expect("bind listener");
|
||||
let addr = listener.local_addr().expect("listener addr");
|
||||
|
||||
@@ -924,9 +924,6 @@ pub struct GenerallyLockResponse {
|
||||
/// JSON serialized LockInfo
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub lock_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
/// Only populated by unlock RPCs.
|
||||
#[prost(enumeration = "LockReleaseStatus", tag = "4")]
|
||||
pub release_status: i32,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BatchGenerallyLockRequest {
|
||||
@@ -942,9 +939,6 @@ pub struct GenerallyLockResult {
|
||||
/// JSON serialized LockInfo
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub lock_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
/// Per-ID result for unlock batches.
|
||||
#[prost(enumeration = "LockReleaseStatus", tag = "4")]
|
||||
pub release_status: i32,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct BatchGenerallyLockResponse {
|
||||
@@ -1881,37 +1875,6 @@ impl CompareAndUpdateFileOutcome {
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Authoritative unlock result for cleanup of a previously confirmed grant.
|
||||
/// Older peers omit this field. Unspecified does not confirm absence.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum LockReleaseStatus {
|
||||
Unspecified = 0,
|
||||
Released = 1,
|
||||
AlreadyAbsent = 2,
|
||||
}
|
||||
impl LockReleaseStatus {
|
||||
/// String value of the enum field names used in the ProtoBuf definition.
|
||||
///
|
||||
/// The values are not transformed in any way and thus are considered stable
|
||||
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
|
||||
pub fn as_str_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Unspecified => "LOCK_RELEASE_STATUS_UNSPECIFIED",
|
||||
Self::Released => "LOCK_RELEASE_STATUS_RELEASED",
|
||||
Self::AlreadyAbsent => "LOCK_RELEASE_STATUS_ALREADY_ABSENT",
|
||||
}
|
||||
}
|
||||
/// Creates an enum from field names used in the ProtoBuf definition.
|
||||
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
|
||||
match value {
|
||||
"LOCK_RELEASE_STATUS_UNSPECIFIED" => Some(Self::Unspecified),
|
||||
"LOCK_RELEASE_STATUS_RELEASED" => Some(Self::Released),
|
||||
"LOCK_RELEASE_STATUS_ALREADY_ABSENT" => Some(Self::AlreadyAbsent),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum TierMutationPeerState {
|
||||
|
||||
@@ -4090,63 +4090,3 @@ mod tests {
|
||||
evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod lock_release_tests {
|
||||
use super::proto_gen::node_service::{GenerallyLockResponse, GenerallyLockResult, LockReleaseStatus};
|
||||
use prost::Message;
|
||||
|
||||
/// The release response schema before structured outcomes were added.
|
||||
#[derive(Clone, PartialEq, Message)]
|
||||
struct LegacyReleaseResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
success: bool,
|
||||
#[prost(string, optional, tag = "2")]
|
||||
error_info: Option<String>,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
lock_info: Option<String>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_release_response_does_not_confirm_absence() {
|
||||
let legacy = LegacyReleaseResponse {
|
||||
success: false,
|
||||
error_info: Some("lock not found for release".to_string()),
|
||||
lock_info: None,
|
||||
};
|
||||
let encoded = legacy.encode_to_vec();
|
||||
let response = GenerallyLockResponse::decode(encoded.as_slice()).expect("decode legacy release response");
|
||||
let result = GenerallyLockResult::decode(encoded.as_slice()).expect("decode legacy batch entry");
|
||||
assert!(!response.success);
|
||||
assert_eq!(response.error_info, legacy.error_info);
|
||||
assert_eq!(response.release_status(), LockReleaseStatus::Unspecified);
|
||||
assert_eq!(result.release_status(), LockReleaseStatus::Unspecified);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structured_release_results_preserve_legacy_success_and_failure() {
|
||||
for (status, success, error) in [
|
||||
(LockReleaseStatus::Released, true, None),
|
||||
(LockReleaseStatus::AlreadyAbsent, false, Some("lock not found for release".to_string())),
|
||||
] {
|
||||
let response = GenerallyLockResponse {
|
||||
success,
|
||||
error_info: error.clone(),
|
||||
lock_info: None,
|
||||
release_status: status.into(),
|
||||
};
|
||||
let result = GenerallyLockResult {
|
||||
success,
|
||||
error_info: error.clone(),
|
||||
lock_info: None,
|
||||
release_status: status.into(),
|
||||
};
|
||||
for encoded in [response.encode_to_vec(), result.encode_to_vec()] {
|
||||
let legacy = LegacyReleaseResponse::decode(encoded.as_slice()).expect("old peer ignores additive outcome");
|
||||
assert_eq!(legacy.success, success);
|
||||
assert_eq!(legacy.error_info, error);
|
||||
assert!(legacy.lock_info.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -643,19 +643,10 @@ message GenerallyLockRequest {
|
||||
string args = 1;
|
||||
}
|
||||
|
||||
// Authoritative unlock result for cleanup of a previously confirmed grant.
|
||||
// Older peers omit this field. Unspecified does not confirm absence.
|
||||
enum LockReleaseStatus {
|
||||
LOCK_RELEASE_STATUS_UNSPECIFIED = 0;
|
||||
LOCK_RELEASE_STATUS_RELEASED = 1;
|
||||
LOCK_RELEASE_STATUS_ALREADY_ABSENT = 2;
|
||||
}
|
||||
|
||||
message GenerallyLockResponse {
|
||||
bool success = 1;
|
||||
optional string error_info = 2;
|
||||
optional string lock_info = 3; // JSON serialized LockInfo
|
||||
LockReleaseStatus release_status = 4; // Only populated by unlock RPCs.
|
||||
}
|
||||
|
||||
message BatchGenerallyLockRequest {
|
||||
@@ -666,7 +657,6 @@ message GenerallyLockResult {
|
||||
bool success = 1;
|
||||
optional string error_info = 2;
|
||||
optional string lock_info = 3; // JSON serialized LockInfo
|
||||
LockReleaseStatus release_status = 4; // Per-ID result for unlock batches.
|
||||
}
|
||||
|
||||
message BatchGenerallyLockResponse {
|
||||
|
||||
@@ -24,12 +24,6 @@ may therefore remain undetected until real I/O reaches it. Missing slots,
|
||||
invalidated identities, unhealthy drives, and remote endpoints still get probed
|
||||
for recovery. Scanner, heal, and other background services have separate controls.
|
||||
|
||||
## Multipart shard completion
|
||||
|
||||
UploadPart waits for every still-pending shard operation to finish or reach its existing `RUSTFS_OBJECT_DISK_WRITE_STALL_TIMEOUT` budget (default 30 seconds), bounded further by the remaining `RUSTFS_OBJECT_DISK_WRITE_ABSOLUTE_CAP` when configured. Reaching write quorum does not start the one-second straggler cutoff used by ordinary PUT. This keeps finite slow shards from being discarded on different disks for different parts, which could leave too few disks carrying the entire object at CompleteMultipartUpload.
|
||||
|
||||
This does not require every disk to succeed, count an unfinished writer as committed, or reduce the final object's write quorum. Failed and expired writers remain excluded. The tradeoff is that a black-hole peer can delay each part up to the existing operation budget rather than one second after quorum. The stall budget resets per shard-write operation; the optional absolute cap belongs to one part's encoding instance, not the entire multipart upload. Setting both budgets to zero retains the existing unbounded-wait behavior. Actual failures can still leave different parts missing on different disks, in which case completion must fail if a common complete write quorum is unavailable.
|
||||
|
||||
## Background: what these timeouts bound
|
||||
|
||||
Every foreground drive operation carries a liveness budget so a hung disk fails
|
||||
|
||||
@@ -17,11 +17,7 @@ A timed-out request is no longer cancelled. Cancelling sends `RST_STREAM`, and e
|
||||
|
||||
Three consecutive lock-acquisition timeouts open a request-level per-peer breaker. While open, new `lock` and `lock_batch` calls fail fast as retryable contention; after the backoff window one half-open probe is admitted. The breaker does not block release or refresh traffic, and it is separate from channel eviction. A per-peer in-flight admission limit also bounds requests that were already admitted before the breaker opened.
|
||||
|
||||
Unlock cleanup that remains unconfirmed after three quick retries continues in the background with a deferred schedule (1s, 2s, 4s, 8s, 16s). Exhausting that schedule means cleanup was not confirmed; it does not prove that the peer still holds the lock. A server-side lease bounds any grant that remains.
|
||||
|
||||
Cleanup of a previously confirmed grant is idempotent for its exact `LockId`: `Released` and authoritative `AlreadyAbsent` both complete cleanup. The unlock RPCs append a structured `release_status` to each response or batch entry while preserving the existing `success` and `error_info` fields. Ordinary release and force-release still report a missing lock as failure. A new client keeps retrying unknown transport/backend results and absence responses from older peers that omit the structured status; it never infers absence from diagnostic text. Older clients ignore the added field and retain their existing behavior during upgrades or rollback.
|
||||
|
||||
An absence response cannot settle an acquisition that may still grant the same ID later. Pending acquisition completion retains its own cleanup owner, and uncertain remote acquisition failures keep their conservative late-release retries. Only a completed, confirmed grant can use authoritative absence to stop retrying.
|
||||
Unlocks that fail three quick retries no longer stop there. The background task continues with a deferred schedule (1s, 2s, 4s, 8s, 16s) before it gives up and leaves the entry to the server-side lease.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -22,16 +22,6 @@ Each row is a feature whose activation makes older binaries unable to read what
|
||||
- Rolling restart (no downtime): restart one node at a time, wait for the restarted node to report `200` on `/health/ready`, and then wait until the other nodes no longer list its drives under `details.storage.unavailableDrives` before touching the next one. The remaining nodes keep serving traffic.
|
||||
- Sequential cold start (several nodes down): nodes started before the cluster has quorum come up in degraded mode. The process stays alive, answers `503` with the blocking reason, and recovers automatically as soon as enough peers are online. Do not restart-loop them; keep starting the remaining nodes.
|
||||
|
||||
## Returning a recovered node to a load balancer
|
||||
|
||||
Probe each backend directly on its S3 listener, using its configured TLS verification when HTTPS is enabled. Require HTTP `200` from `/health/ready` (or `/minio/health/ready`) before routing object requests to that backend. For a workload that also requires write availability, additionally require HTTP `200` from `/minio/health/cluster` on that backend. A read-only workload can use node readiness alone; `/minio/health/cluster/read` has a more conservative lock check and adds no startup-publication guarantee. See the [probe semantics](../architecture/readiness-matrix.md#probe-semantics).
|
||||
|
||||
Cluster write/read probes can return `200` while a restarted node still returns `503` for object requests: quorum observations can succeed before startup publishes `FullReady`, including while startup waits for peer re-admission. The node readiness aliases remain `503` during that interval, with `startup_finalization_pending` in `degradedReasons`. Liveness and cluster probes alone are therefore insufficient for backend admission. This also applies when every node's cluster probe succeeds.
|
||||
|
||||
Readiness does not certify that healing has completed for every object. The startup peer re-admission wait is bounded and fails open; before stopping another node, also follow the survivor checks in the [rolling restart procedure](#rolling-restart-procedure). No health endpoint is an atomic guarantee that every subsequent object read or write will succeed. The [storage detail contract](../architecture/readiness-matrix.md#storage-detail-contract) describes these sampling limits.
|
||||
|
||||
Clients should use bounded retries with backoff and jitter for transient `503` responses. Honor `Retry-After` when present, within the request's retry/deadline budget; it is a suggested delay, not a recovery deadline. The startup admission gate sends `Retry-After: 5` and `x-rustfs-readiness-pending` naming the pending dependency. Other `503` paths can use different headers, so inspect the actual response. Retrying a GET is safe; retrying a write with an ambiguous outcome requires accounting for versioning, concurrent writers, and request preconditions. For recovery validation, retain the first failed response and use distinct payloads or version IDs for overwrites so a successful retry cannot hide a failed assertion or identify the wrong version as durable.
|
||||
|
||||
## Why a single node cannot serve alone
|
||||
|
||||
Erasure coding shards every object, including internal metadata such as IAM users, groups, and policies under `.rustfs.sys`, across the drives of a set. Reading an object back needs a read quorum of shards online. With the drives of one set spread over several nodes, one node alone can never satisfy the read quorum; this is a property of erasure coding, not a bug. The cluster becomes readable once enough nodes are up (for internal configuration objects, written with maximum parity, typically about half the nodes of a set).
|
||||
@@ -69,10 +59,6 @@ On Kubernetes the dial itself is the slow part. A replaced pod gets a new IP, bu
|
||||
|
||||
After a distributed node reaches local readiness, it asks the peers in every erasure set it serves how they currently see its drives. It uses the `PeerDriveAdmission` internode RPC, and each peer answers with the same predicate its own `/health/ready` uses for `unavailableDrives`. Startup readiness is published once a strict majority of each set's peers count all of this node's drives in that set as online. In a three-node set that means both peers. While the node waits, `/health/ready` stays `503` with `startup_finalization_pending` and the log shows `event="startup_peer_readmission" state="waiting"`.
|
||||
|
||||
Both local dependency checks and peer re-admission rounds are bounded by the startup budget, including in-flight observations and polling delays. An exhausted peer-readmission budget starts no new probe round. Within a round, startup proceeds as soon as every shared set has its required peer majority; pending surplus read-only RPCs are cancelled. Votes are never carried across rounds. The unreachable-peer grace also bounds in-flight rounds: enough reachable replies clear an earlier shortfall before unrelated peers finish, and only a completed round can establish a new unreachable shortfall.
|
||||
|
||||
After deferred IAM initialization succeeds, recovery retains that IAM owner while preparing the topology and claiming the application context once. A later local-readiness timeout retries publication with the prepared context; it does not reinitialize IAM or reinstall the context.
|
||||
|
||||
The wait always ends in publication:
|
||||
|
||||
- `result="admitted"`: the peers re-admitted the node.
|
||||
@@ -107,7 +93,7 @@ When the whole cluster (or several nodes) went down and nodes are brought back o
|
||||
|
||||
| Variable | Default | Effect |
|
||||
| --- | --- | --- |
|
||||
| `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` | `120` (`DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS`) | Budget per readiness-publication attempt for local readiness and subsequent peer re-admission, including in-flight checks and polling delays. Local timeout returns an error to the startup/recovery caller; peer re-admission timeout publishes readiness with a warning. Raising it can delay S3 admission; the HTTP listeners are already bound. |
|
||||
| `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` | `120` (`DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS`) | How long startup waits for full readiness before continuing in degraded mode with background recovery. Raising it delays the listener during genuinely slow starts; lowering it surfaces degraded mode sooner. Recovery retries continue regardless of this limit. |
|
||||
| `RUSTFS_STARTUP_PEER_READMISSION_ENABLE` | `true` (`DEFAULT_STARTUP_PEER_READMISSION_ENABLE`) | Hold startup readiness until a strict majority of each erasure set's peers count this node's drives online. The wait is bounded by `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` and fails open. Set `false` to publish on this node's own view only. |
|
||||
| `RUSTFS_DRIVE_RETURNING_PROBE_INTERVAL_SECS` | `2` (`DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS`) | Interval between recovery probes to a remote drive that is `offline` or `returning`. |
|
||||
| `RUSTFS_DRIVE_RETURNING_SUCCESS_THRESHOLD` | `3` (`DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD`) | Consecutive successful probes before an `offline` remote drive is `online` again. Together with the interval this bounds how long survivors keep a restarted peer out after they can dial it. |
|
||||
|
||||
+24
-41
@@ -184,47 +184,12 @@ if [ "$1" = "/usr/bin/rustfs" ]; then
|
||||
validate_credential_source "RUSTFS_SECRET_KEY" "RUSTFS_SECRET_KEY_FILE" "RUSTFS_ROOT_PASSWORD or MINIO_ROOT_PASSWORD" "$SECRET_SOURCE"
|
||||
fi
|
||||
|
||||
# Existing directories need no ownership scan; newly created directories get
|
||||
# only a best-effort, non-recursive owner change, including on large mounts.
|
||||
ensure_local_directory() {
|
||||
[ -d "$1" ] && return 0
|
||||
|
||||
echo " mkdir -p $1"
|
||||
if ! mkdir -p "$1"; then
|
||||
echo "ERROR: Cannot create $2. Check the configured path and write permissions for the container user (10001:10001 in official images)." >&2
|
||||
if [ "$2" = "RUSTFS_OBS_LOG_DIRECTORY" ]; then
|
||||
case "$1" in
|
||||
\"*\"|\'*\')
|
||||
echo 'HINT: RUSTFS_OBS_LOG_DIRECTORY contains literal surrounding quotes. In Docker Compose, use map syntax: RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/", or list syntax without inner quotes: - RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/' >&2
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -n "$RUSTFS_UID" ] && [ -n "$RUSTFS_GID" ]; then
|
||||
chown "$RUSTFS_UID:$RUSTFS_GID" "$1" 2>/dev/null || true
|
||||
elif [ -n "$RUSTFS_USERNAME" ] && [ -n "$RUSTFS_GROUPNAME" ]; then
|
||||
chown "$RUSTFS_USERNAME:$RUSTFS_GROUPNAME" "$1" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
has_volume_range() {
|
||||
case "$1" in
|
||||
*'{'*) printf '%s\n' "$1" | grep -E -q '\{[0-9]+\.\.\.?[0-9]+\}' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# 2) Process data volumes (separate from log directory)
|
||||
DATA_VOLUMES=""
|
||||
process_data_volumes() {
|
||||
VOLUME_RAW="${RUSTFS_VOLUMES:-/data}"
|
||||
# Normalize comma/tab delimiters only when present (the second pattern is a literal tab).
|
||||
case "$VOLUME_RAW" in
|
||||
*','*|*' '*) VOLUME_LIST_RAW=$(printf '%s\n' "$VOLUME_RAW" | tr ',\t' ' ') ;;
|
||||
*) VOLUME_LIST_RAW="$VOLUME_RAW" ;;
|
||||
esac
|
||||
# Convert comma/tab to space
|
||||
VOLUME_LIST_RAW=$(echo "$VOLUME_RAW" | tr ',\t' ' ')
|
||||
|
||||
# Manually expand {N...M} ranges since sh doesn't support brace expansion on
|
||||
# variables. A single token can carry MORE than one range — the distributed
|
||||
@@ -234,10 +199,10 @@ process_data_volumes() {
|
||||
# multi-range tokens intact; the previous single-pass expander collapsed them
|
||||
# to "http://rustfs1" and silently dropped the disk path.
|
||||
VOLUME_LIST="$VOLUME_LIST_RAW"
|
||||
while has_volume_range "$VOLUME_LIST"; do
|
||||
while echo "$VOLUME_LIST" | grep -E -q "\{[0-9]+\.\.\.?[0-9]+\}"; do
|
||||
NEXT_LIST=""
|
||||
for vol in $VOLUME_LIST; do
|
||||
if has_volume_range "$vol"; then
|
||||
if echo "$vol" | grep -E -q "\{[0-9]+\.\.\.?[0-9]+\}"; then
|
||||
PREFIX=${vol%%\{*}
|
||||
REST=${vol#*\}}
|
||||
RANGE=${vol#*\{}
|
||||
@@ -292,7 +257,16 @@ process_data_volumes() {
|
||||
|
||||
echo "Initializing data directories:$CREATE_DIRS"
|
||||
for vol in $CREATE_DIRS; do
|
||||
ensure_local_directory "$vol" RUSTFS_VOLUMES
|
||||
if [ ! -d "$vol" ]; then
|
||||
echo " mkdir -p $vol"
|
||||
mkdir -p "$vol"
|
||||
# If target user is specified, try to set directory owner to that user (non-recursive to avoid large disk overhead)
|
||||
if [ -n "$RUSTFS_UID" ] && [ -n "$RUSTFS_GID" ]; then
|
||||
chown "$RUSTFS_UID:$RUSTFS_GID" "$vol" 2>/dev/null || true
|
||||
elif [ -n "$RUSTFS_USERNAME" ] && [ -n "$RUSTFS_GROUPNAME" ]; then
|
||||
chown "$RUSTFS_USERNAME:$RUSTFS_GROUPNAME" "$vol" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
@@ -314,7 +288,16 @@ process_log_directory() {
|
||||
LOG_DIR="${RUSTFS_OBS_LOG_DIRECTORY}"
|
||||
|
||||
echo "Initializing log directory: $LOG_DIR"
|
||||
ensure_local_directory "$LOG_DIR" RUSTFS_OBS_LOG_DIRECTORY
|
||||
if [ ! -d "$LOG_DIR" ]; then
|
||||
echo " mkdir -p $LOG_DIR"
|
||||
mkdir -p "$LOG_DIR"
|
||||
# If target user is specified, try to set directory owner to that user (non-recursive to avoid large disk overhead)
|
||||
if [ -n "$RUSTFS_UID" ] && [ -n "$RUSTFS_GID" ]; then
|
||||
chown "$RUSTFS_UID:$RUSTFS_GID" "$LOG_DIR" 2>/dev/null || true
|
||||
elif [ -n "$RUSTFS_USERNAME" ] && [ -n "$RUSTFS_GROUPNAME" ]; then
|
||||
chown "$RUSTFS_USERNAME:$RUSTFS_GROUPNAME" "$LOG_DIR" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Execute the separate processes
|
||||
|
||||
+56
-102
@@ -4574,112 +4574,66 @@ mod tests {
|
||||
async {
|
||||
let object_traffic_health = Arc::new(ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
readiness.mark_stage(rustfs_common::SystemStage::IamReady);
|
||||
let inner = CountingHybridService::default();
|
||||
let calls = inner.calls();
|
||||
let public_health =
|
||||
public_health_layer_with_tracker_and_readiness(object_traffic_health, Arc::clone(&readiness)).await;
|
||||
let mut service =
|
||||
crate::server::ReadinessGateLayer::new(Arc::clone(&readiness)).layer(public_health.layer(inner));
|
||||
let mut service = public_health_layer_with_tracker_and_readiness(object_traffic_health, Arc::clone(&readiness))
|
||||
.await
|
||||
.layer(inner);
|
||||
|
||||
// Cluster quorum can recover before this node publishes S3 admission.
|
||||
for method in [Method::GET, Method::HEAD] {
|
||||
for path in [
|
||||
MINIO_HEALTH_CLUSTER_PATH,
|
||||
MINIO_HEALTH_CLUSTER_READ_PATH,
|
||||
HEALTH_COMPAT_LIVE_PATH,
|
||||
] {
|
||||
let response = service
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(path)
|
||||
.body(Full::<Bytes>::from(Bytes::new()))
|
||||
.expect("probe request before admission publication"),
|
||||
)
|
||||
.await
|
||||
.expect("probe response before admission publication");
|
||||
assert_eq!(response.status(), StatusCode::OK, "{method} {path}");
|
||||
let body = BodyExt::collect(response.into_body()).await.expect("probe body").to_bytes();
|
||||
if method == Method::HEAD {
|
||||
assert!(body.is_empty(), "HEAD {path}");
|
||||
} else {
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("probe JSON");
|
||||
if path == HEALTH_COMPAT_LIVE_PATH {
|
||||
assert_eq!(payload["status"], "ok");
|
||||
assert!(payload.get("ready").is_none());
|
||||
} else {
|
||||
assert_eq!(payload["ready"], true, "{path}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let response = service
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(HEALTH_READY_PATH)
|
||||
.body(Full::<Bytes>::from(Bytes::new()))
|
||||
.expect("readiness request before admission publication"),
|
||||
)
|
||||
.await
|
||||
.expect("readiness response before admission publication");
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
let body = BodyExt::collect(response.into_body())
|
||||
.await
|
||||
.expect("readiness body before admission publication")
|
||||
.to_bytes();
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("readiness JSON");
|
||||
assert_eq!(payload["ready"], false);
|
||||
assert_eq!(payload["details"]["storage"]["ready"], true);
|
||||
assert_eq!(payload["details"]["iam"]["ready"], true);
|
||||
assert_eq!(payload["details"]["lock"]["ready"], true);
|
||||
assert_eq!(payload["degradedReasons"], serde_json::json!(["startup_finalization_pending"]));
|
||||
|
||||
for published in [false, true] {
|
||||
if published {
|
||||
readiness.mark_stage(rustfs_common::SystemStage::FullReady);
|
||||
}
|
||||
for method in [Method::GET, Method::HEAD] {
|
||||
for path in [HEALTH_READY_PATH, MINIO_HEALTH_READY_PATH] {
|
||||
let response = service
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(path)
|
||||
.body(Full::<Bytes>::from(Bytes::new()))
|
||||
.expect("node readiness request"),
|
||||
)
|
||||
.await
|
||||
.expect("node readiness response");
|
||||
let expected = if published {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
};
|
||||
assert_eq!(response.status(), expected, "{method} {path}, published={published}");
|
||||
let body = BodyExt::collect(response.into_body())
|
||||
.await
|
||||
.expect("readiness body")
|
||||
.to_bytes();
|
||||
if method == Method::HEAD {
|
||||
assert!(body.is_empty(), "HEAD {path}");
|
||||
} else {
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("readiness JSON");
|
||||
assert_eq!(payload["ready"], published);
|
||||
for component in ["storage", "iam", "lock"] {
|
||||
assert_eq!(payload["details"][component]["ready"], true, "{component}");
|
||||
}
|
||||
let reasons = if published {
|
||||
serde_json::json!([])
|
||||
} else {
|
||||
serde_json::json!(["startup_finalization_pending"])
|
||||
};
|
||||
assert_eq!(payload["degradedReasons"], reasons);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0, "health probes bypass the inner service");
|
||||
for method in [Method::GET, Method::PUT] {
|
||||
let response = service
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(method.clone())
|
||||
.uri("/bucket/object")
|
||||
.body(Full::<Bytes>::from(Bytes::new()))
|
||||
.expect("object request"),
|
||||
)
|
||||
.await
|
||||
.expect("object admission response");
|
||||
if published {
|
||||
assert_eq!(response.status(), StatusCode::IM_A_TEAPOT, "{method} reaches the inner service");
|
||||
} else {
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{method} stays gated");
|
||||
assert_eq!(response.headers()[http::header::RETRY_AFTER], "5");
|
||||
assert_eq!(response.headers()["x-rustfs-readiness-pending"], "startup_finalization");
|
||||
}
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), if published { 2 } else { 0 });
|
||||
}
|
||||
let response = service
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(HEALTH_COMPAT_LIVE_PATH)
|
||||
.body(Full::<Bytes>::from(Bytes::new()))
|
||||
.expect("liveness request before admission publication"),
|
||||
)
|
||||
.await
|
||||
.expect("liveness response before admission publication");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = BodyExt::collect(response.into_body())
|
||||
.await
|
||||
.expect("liveness body before admission publication")
|
||||
.to_bytes();
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("liveness JSON");
|
||||
assert_eq!(payload["status"], "ok");
|
||||
assert!(payload.get("ready").is_none());
|
||||
|
||||
readiness.mark_stage(rustfs_common::SystemStage::FullReady);
|
||||
let response = service
|
||||
.call(
|
||||
Request::builder()
|
||||
.method(Method::HEAD)
|
||||
.uri(MINIO_HEALTH_READY_PATH)
|
||||
.body(Full::<Bytes>::from(Bytes::new()))
|
||||
.expect("readiness request after admission publication"),
|
||||
)
|
||||
.await
|
||||
.expect("readiness response after admission publication");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -92,8 +92,6 @@ pub(crate) use readiness::collect_node_readiness_report;
|
||||
pub(crate) use readiness::observe_peer_drive_admission;
|
||||
pub use readiness::publish_ready_when_runtime_ready;
|
||||
pub(crate) use readiness::snapshot_dependency_readiness_report;
|
||||
#[cfg(test)]
|
||||
pub(crate) use readiness::wait_for_runtime_readiness_with;
|
||||
pub(crate) use readiness::{collect_cluster_read_health_report, collect_cluster_write_health_report};
|
||||
pub(crate) use ssec_transport::NativeTlsTransport;
|
||||
|
||||
|
||||
+29
-213
@@ -75,9 +75,6 @@ fn startup_runtime_readiness_max_wait() -> Duration {
|
||||
Duration::from_secs(secs)
|
||||
}
|
||||
}
|
||||
const EVENT_STARTUP_RUNTIME_READINESS: &str = "startup_runtime_readiness";
|
||||
const LOG_COMPONENT_SERVER: &str = "server";
|
||||
const LOG_SUBSYSTEM_READINESS: &str = "readiness";
|
||||
const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
|
||||
const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";
|
||||
const METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL: &str = "rustfs_pool_metadata_check_timeouts_total";
|
||||
@@ -1276,15 +1273,6 @@ async fn collect_lock_quorum_observation_uncached() -> LockQuorumObservation {
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for local runtime dependencies without exceeding the observation budget.
|
||||
///
|
||||
/// The budget covers both observations and polling sleeps. An immediately ready
|
||||
/// observation can succeed with a zero budget. Cancellation drops the in-flight
|
||||
/// observation; only success invokes `on_ready`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error on expiry, retaining the last completed observation when
|
||||
/// available. The readiness callback is never invoked after a timeout.
|
||||
pub async fn wait_for_runtime_readiness_with<F, Fut, ReadyFn>(
|
||||
max_wait: Duration,
|
||||
poll_interval: Duration,
|
||||
@@ -1297,60 +1285,35 @@ where
|
||||
ReadyFn: FnMut(DependencyReadiness),
|
||||
{
|
||||
let startup_deadline = tokio::time::Instant::now() + max_wait;
|
||||
let mut last_readiness = None;
|
||||
let result = tokio::time::timeout_at(startup_deadline, async {
|
||||
loop {
|
||||
// Tokio timeouts poll the observation first. Keep the initial
|
||||
// zero-budget observation, but start no later one at expiry.
|
||||
if last_readiness.is_some() && tokio::time::Instant::now() >= startup_deadline {
|
||||
return None;
|
||||
}
|
||||
let readiness = load_readiness().await;
|
||||
last_readiness = Some(readiness);
|
||||
if readiness.storage_ready
|
||||
&& readiness.iam_ready
|
||||
&& readiness.lock_quorum_ready
|
||||
&& readiness.peer_health_ready
|
||||
&& (max_wait.is_zero() || tokio::time::Instant::now() < startup_deadline)
|
||||
{
|
||||
return Some(readiness);
|
||||
}
|
||||
if tokio::time::Instant::now() >= startup_deadline {
|
||||
return None;
|
||||
}
|
||||
info!(
|
||||
event = EVENT_STARTUP_RUNTIME_READINESS,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_READINESS,
|
||||
state = "waiting",
|
||||
storage_ready = readiness.storage_ready,
|
||||
iam_ready = readiness.iam_ready,
|
||||
lock_quorum_ready = readiness.lock_quorum_ready,
|
||||
peer_health_ready = readiness.peer_health_ready,
|
||||
"Runtime node readiness has not been reached yet; delaying ready state publication"
|
||||
);
|
||||
tokio::time::sleep(poll_interval).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
match result {
|
||||
Ok(Some(readiness)) => {
|
||||
|
||||
loop {
|
||||
let readiness = load_readiness().await;
|
||||
if readiness.storage_ready && readiness.iam_ready && readiness.lock_quorum_ready && readiness.peer_health_ready {
|
||||
on_ready(readiness);
|
||||
Ok(())
|
||||
return Ok(());
|
||||
}
|
||||
Ok(None) | Err(_) => {
|
||||
let detail = match last_readiness {
|
||||
Some(readiness) => format!(
|
||||
"storage_ready={}, iam_ready={}, lock_quorum_ready={}, peer_health_ready={}",
|
||||
readiness.storage_ready, readiness.iam_ready, readiness.lock_quorum_ready, readiness.peer_health_ready
|
||||
),
|
||||
None => "no readiness observation completed".to_string(),
|
||||
};
|
||||
Err(std::io::Error::other(format!(
|
||||
"startup readiness timed out after {}s: {detail}",
|
||||
max_wait.as_secs()
|
||||
)))
|
||||
|
||||
if tokio::time::Instant::now() >= startup_deadline {
|
||||
let reason = format!(
|
||||
"startup readiness timed out after {}s: storage_ready={}, iam_ready={}, lock_quorum_ready={}, peer_health_ready={}",
|
||||
max_wait.as_secs(),
|
||||
readiness.storage_ready,
|
||||
readiness.iam_ready,
|
||||
readiness.lock_quorum_ready,
|
||||
readiness.peer_health_ready
|
||||
);
|
||||
return Err(std::io::Error::other(reason));
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "rustfs::server::readiness",
|
||||
storage_ready = readiness.storage_ready,
|
||||
iam_ready = readiness.iam_ready,
|
||||
lock_quorum_ready = readiness.lock_quorum_ready,
|
||||
peer_health_ready = readiness.peer_health_ready,
|
||||
"Runtime node readiness has not been reached yet; delaying ready state publication"
|
||||
);
|
||||
tokio::time::sleep(poll_interval).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1655,7 +1618,6 @@ mod tests {
|
||||
/// predicate counts node-2's drive online, then publish.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn restarted_node_stays_unready_until_survivor_counts_its_drive_online() {
|
||||
use futures::StreamExt;
|
||||
use peer_readmission::{
|
||||
PeerDriveView, PeerProbe, ReadmissionOutcome, UNREACHABLE_PEER_GRACE, host_drive_observations, readmission_plan,
|
||||
wait_for_peer_readmission_with,
|
||||
@@ -1692,8 +1654,8 @@ mod tests {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(120);
|
||||
|
||||
let gate = async {
|
||||
let outcome = wait_for_peer_readmission_with(&plan, deadline, Duration::from_secs(1), UNREACHABLE_PEER_GRACE, || {
|
||||
futures::stream::once(async {
|
||||
let outcome =
|
||||
wait_for_peer_readmission_with(&plan, deadline, Duration::from_secs(1), UNREACHABLE_PEER_GRACE, || async {
|
||||
let mut online_hosts = HashSet::from(["node-0:9000".to_string(), "node-1:9000".to_string()]);
|
||||
if survivor_reaches_restarted.load(Ordering::SeqCst) {
|
||||
online_hosts.insert("node-2:9000".to_string());
|
||||
@@ -1715,9 +1677,7 @@ mod tests {
|
||||
);
|
||||
HashMap::from([("node-0:9000".to_string(), view.clone()), ("node-1:9000".to_string(), view)])
|
||||
})
|
||||
.flat_map(futures::stream::iter)
|
||||
})
|
||||
.await;
|
||||
.await;
|
||||
readiness.mark_stage(rustfs_common::SystemStage::FullReady);
|
||||
outcome
|
||||
};
|
||||
@@ -2019,150 +1979,6 @@ mod tests {
|
||||
assert_eq!(state_manager.current_state(), ServiceState::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_for_runtime_readiness_cancels_a_suspended_observation() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
struct ObservationDrop(Arc<AtomicUsize>);
|
||||
impl Drop for ObservationDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
let dropped = Arc::new(AtomicUsize::new(0));
|
||||
let published = AtomicUsize::new(0);
|
||||
let started = tokio::time::Instant::now();
|
||||
let wait = wait_for_runtime_readiness_with(
|
||||
Duration::from_secs(2),
|
||||
Duration::from_secs(1),
|
||||
|| {
|
||||
let guard = ObservationDrop(Arc::clone(&dropped));
|
||||
async move {
|
||||
let _guard = guard;
|
||||
std::future::pending::<DependencyReadiness>().await
|
||||
}
|
||||
},
|
||||
|_| {
|
||||
published.fetch_add(1, Ordering::SeqCst);
|
||||
},
|
||||
);
|
||||
let err = tokio::time::timeout(Duration::from_secs(3), wait)
|
||||
.await
|
||||
.expect("startup budget bounds a suspended dependency observation")
|
||||
.expect_err("no dependency observation completed");
|
||||
assert!(err.to_string().contains("no readiness observation completed"));
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(2));
|
||||
assert_eq!(dropped.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(published.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_for_runtime_readiness_bounds_the_poll_sleep() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let polls = AtomicUsize::new(0);
|
||||
let started = tokio::time::Instant::now();
|
||||
let wait = wait_for_runtime_readiness_with(
|
||||
Duration::from_secs(2),
|
||||
Duration::from_secs(30),
|
||||
|| {
|
||||
polls.fetch_add(1, Ordering::SeqCst);
|
||||
future::ready(DependencyReadiness::default())
|
||||
},
|
||||
|_| panic!("an unready observation must never publish"),
|
||||
);
|
||||
let err = tokio::time::timeout(Duration::from_secs(3), wait)
|
||||
.await
|
||||
.expect("startup budget bounds the poll sleep")
|
||||
.expect_err("unready observation");
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(2));
|
||||
assert_eq!(polls.load(Ordering::SeqCst), 1, "expiry must not start another observation");
|
||||
assert!(err.to_string().contains("lock_quorum_ready=false"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_for_runtime_readiness_publishes_once_before_the_deadline() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let polls = AtomicUsize::new(0);
|
||||
let published = AtomicUsize::new(0);
|
||||
let started = tokio::time::Instant::now();
|
||||
wait_for_runtime_readiness_with(
|
||||
Duration::from_secs(2),
|
||||
Duration::from_secs(1),
|
||||
|| {
|
||||
let ready = polls.fetch_add(1, Ordering::SeqCst) > 0;
|
||||
future::ready(DependencyReadiness {
|
||||
storage_ready: ready,
|
||||
iam_ready: ready,
|
||||
lock_quorum_ready: ready,
|
||||
peer_health_ready: ready,
|
||||
})
|
||||
},
|
||||
|_| {
|
||||
published.fetch_add(1, Ordering::SeqCst);
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("dependencies recovered before expiry");
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(1));
|
||||
assert_eq!(published.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(polls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_for_runtime_readiness_starts_no_observation_at_expiry() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let polls = AtomicUsize::new(0);
|
||||
let published = AtomicUsize::new(0);
|
||||
let started = tokio::time::Instant::now();
|
||||
wait_for_runtime_readiness_with(
|
||||
Duration::from_secs(2),
|
||||
Duration::from_secs(1),
|
||||
|| {
|
||||
let ready = polls.fetch_add(1, Ordering::SeqCst) >= 2;
|
||||
future::ready(DependencyReadiness {
|
||||
storage_ready: ready,
|
||||
iam_ready: ready,
|
||||
lock_quorum_ready: ready,
|
||||
peer_health_ready: ready,
|
||||
})
|
||||
},
|
||||
|_| {
|
||||
published.fetch_add(1, Ordering::SeqCst);
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("expiry must prevent a third observation even when immediately ready");
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(2));
|
||||
assert_eq!(polls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(published.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn wait_for_runtime_readiness_does_not_publish_an_observation_at_expiry() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let published = AtomicUsize::new(0);
|
||||
let started = tokio::time::Instant::now();
|
||||
wait_for_runtime_readiness_with(
|
||||
Duration::from_secs(2),
|
||||
Duration::from_secs(1),
|
||||
|| async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
DependencyReadiness {
|
||||
storage_ready: true,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
}
|
||||
},
|
||||
|_| {
|
||||
published.fetch_add(1, Ordering::SeqCst);
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("an observation completing at the deadline must not publish");
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(2));
|
||||
assert_eq!(published.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_path_checks_admin_boundaries() {
|
||||
assert!(is_probe_path("/minio/admin/v3/info"));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+45
-251
@@ -85,35 +85,32 @@ where
|
||||
publish_ready().await
|
||||
}
|
||||
|
||||
async fn prepare_iam_startup(
|
||||
async fn finalize_iam_recovery(
|
||||
iam: Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>,
|
||||
store: Arc<ECStore>,
|
||||
kms_interface: Arc<KmsServiceManager>,
|
||||
readiness: Arc<GlobalReadiness>,
|
||||
state_manager: Option<Arc<ServiceStateManager>>,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
) -> Result<HealTopologyReady> {
|
||||
) -> Result<()> {
|
||||
let endpoint_pools = heal_control_endpoint_pools(store.as_ref())?;
|
||||
publish_iam_ready_after_heal_topology(readiness.as_ref(), server_ctx.heal_topology_fingerprint(), endpoint_pools, || {
|
||||
AppContext::ensure_startup_after_iam(store, kms_interface, &server_ctx, iam)
|
||||
AppContext::ensure_startup_after_iam(store, kms_interface, &server_ctx, iam)?;
|
||||
let topology_ready =
|
||||
publish_iam_ready_after_heal_topology(readiness.as_ref(), server_ctx.heal_topology_fingerprint(), endpoint_pools).await?;
|
||||
publish_runtime_ready_after_heal_topology(topology_ready, || async {
|
||||
publish_ready_when_runtime_ready(readiness.as_ref(), state_manager.as_deref()).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn publish_iam_ready_after_heal_topology<InstallFn>(
|
||||
async fn publish_iam_ready_after_heal_topology(
|
||||
readiness: &GlobalReadiness,
|
||||
cache: Arc<tokio::sync::OnceCell<String>>,
|
||||
endpoint_pools: crate::storage_api::cluster::EndpointServerPools,
|
||||
install_context: InstallFn,
|
||||
) -> Result<HealTopologyReady>
|
||||
where
|
||||
InstallFn: FnOnce() -> Result<()>,
|
||||
{
|
||||
) -> Result<HealTopologyReady> {
|
||||
crate::storage_api::startup::heal_control::initialize_heal_topology_fingerprint(cache, endpoint_pools)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
// Finish fallible preparation before claiming the one-time context slot.
|
||||
// No fallible/async step may follow a successful claim before this token.
|
||||
install_context()?;
|
||||
readiness.mark_stage(SystemStage::IamReady);
|
||||
Ok(HealTopologyReady(()))
|
||||
}
|
||||
@@ -148,7 +145,8 @@ fn spawn_iam_recovery_task(
|
||||
let finalize_readiness = readiness;
|
||||
let finalize_state_manager = state_manager;
|
||||
let finalize_server_ctx = server_ctx;
|
||||
let prepare_readiness = finalize_readiness.clone();
|
||||
let recovered_iam = Arc::new(std::sync::Mutex::new(None));
|
||||
let init_recovered_iam = recovered_iam.clone();
|
||||
tokio::spawn(async move {
|
||||
run_iam_recovery_loop(
|
||||
initial_interval,
|
||||
@@ -156,23 +154,25 @@ fn spawn_iam_recovery_task(
|
||||
shutdown_token,
|
||||
move || {
|
||||
let store = init_store.clone();
|
||||
Box::pin(async move { attempt_init_iam_sys(store).await })
|
||||
let recovered_iam = init_recovered_iam.clone();
|
||||
Box::pin(async move {
|
||||
let iam = attempt_init_iam_sys(store).await?;
|
||||
*recovered_iam
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("IAM recovery state poisoned"))? = Some(iam);
|
||||
Ok(())
|
||||
})
|
||||
},
|
||||
move |iam| {
|
||||
move || {
|
||||
let store = finalize_store.clone();
|
||||
let kms_interface = finalize_kms_interface.clone();
|
||||
let readiness = prepare_readiness.clone();
|
||||
let server_ctx = finalize_server_ctx.clone();
|
||||
Box::pin(prepare_iam_startup(iam, store, kms_interface, readiness, server_ctx))
|
||||
},
|
||||
move |topology_ready| {
|
||||
let readiness = finalize_readiness.clone();
|
||||
let state_manager = finalize_state_manager.clone();
|
||||
let server_ctx = finalize_server_ctx.clone();
|
||||
let iam = recovered_iam.lock().ok().and_then(|mut recovered| recovered.take());
|
||||
Box::pin(async move {
|
||||
publish_runtime_ready_after_heal_topology(topology_ready, || async {
|
||||
publish_ready_when_runtime_ready(readiness.as_ref(), state_manager.as_deref()).await
|
||||
})
|
||||
.await
|
||||
let iam = iam.ok_or_else(|| std::io::Error::other("IAM recovered but unavailable"))?;
|
||||
finalize_iam_recovery(iam, store, kms_interface, readiness, state_manager, server_ctx).await
|
||||
})
|
||||
},
|
||||
)
|
||||
@@ -180,27 +180,23 @@ fn spawn_iam_recovery_task(
|
||||
});
|
||||
}
|
||||
|
||||
type RecoveryFuture<T = ()> = Pin<Box<dyn Future<Output = Result<T>> + Send>>;
|
||||
type RecoveryFuture = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
|
||||
|
||||
async fn run_iam_recovery_loop<T, Prepared, InitFn, PrepareFn, FinalizeFn>(
|
||||
async fn run_iam_recovery_loop<InitFn, FinalizeFn>(
|
||||
initial_interval: Duration,
|
||||
max_interval: Duration,
|
||||
shutdown_token: Option<tokio_util::sync::CancellationToken>,
|
||||
mut init_fn: InitFn,
|
||||
mut prepare_fn: PrepareFn,
|
||||
mut finalize_fn: FinalizeFn,
|
||||
) where
|
||||
T: Clone,
|
||||
Prepared: Clone,
|
||||
InitFn: FnMut() -> RecoveryFuture<T>,
|
||||
PrepareFn: FnMut(T) -> RecoveryFuture<Prepared>,
|
||||
FinalizeFn: FnMut(Prepared) -> RecoveryFuture,
|
||||
InitFn: FnMut() -> RecoveryFuture,
|
||||
FinalizeFn: FnMut() -> RecoveryFuture,
|
||||
{
|
||||
let mut attempts: u64 = 0;
|
||||
let degraded_since = std::time::Instant::now();
|
||||
|
||||
// Keep the initialized IAM owner across preparation/publication retries.
|
||||
let recovered = loop {
|
||||
// Phase 1: retry init until it succeeds
|
||||
loop {
|
||||
attempts += 1;
|
||||
let sleep_duration = compute_backoff_interval(attempts, initial_interval, max_interval);
|
||||
if let Some(token) = shutdown_token.as_ref() {
|
||||
@@ -213,7 +209,7 @@ async fn run_iam_recovery_loop<T, Prepared, InitFn, PrepareFn, FinalizeFn>(
|
||||
}
|
||||
|
||||
match init_fn().await {
|
||||
Ok(recovered) => {
|
||||
Ok(()) => {
|
||||
let degraded_secs = degraded_since.elapsed().as_secs();
|
||||
info!(
|
||||
event = EVENT_IAM_BOOTSTRAP_RECOVERED,
|
||||
@@ -223,7 +219,7 @@ async fn run_iam_recovery_loop<T, Prepared, InitFn, PrepareFn, FinalizeFn>(
|
||||
degraded_duration_secs = degraded_secs,
|
||||
"IAM bootstrap recovered after startup; publishing IAM readiness"
|
||||
);
|
||||
break recovered;
|
||||
break;
|
||||
}
|
||||
Err(err) => {
|
||||
let next_interval = compute_backoff_interval(attempts + 1, initial_interval, max_interval);
|
||||
@@ -254,24 +250,13 @@ async fn run_iam_recovery_loop<T, Prepared, InitFn, PrepareFn, FinalizeFn>(
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Preparation claims the context once. Only publication retries after it.
|
||||
let mut prepared: Option<Prepared> = None;
|
||||
// Phase 2: retry finalize until readiness is published
|
||||
let mut finalize_attempts: u64 = 0;
|
||||
loop {
|
||||
let result = match prepared.as_ref() {
|
||||
Some(prepared) => finalize_fn(prepared.clone()).await,
|
||||
None => match prepare_fn(recovered.clone()).await {
|
||||
Ok(value) => {
|
||||
prepared = Some(value);
|
||||
continue;
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
},
|
||||
};
|
||||
finalize_attempts += 1;
|
||||
match result {
|
||||
match finalize_fn().await {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
event = EVENT_IAM_READINESS_PUBLISHED,
|
||||
@@ -399,7 +384,11 @@ pub(crate) async fn bootstrap_or_defer_iam_init(
|
||||
) -> Result<IamBootstrapDisposition> {
|
||||
match attempt_init_iam_sys(store.clone()).await {
|
||||
Ok(iam) => {
|
||||
let topology_ready = prepare_iam_startup(iam, store, kms_interface, readiness, server_ctx).await?;
|
||||
let endpoint_pools = heal_control_endpoint_pools(store.as_ref())?;
|
||||
AppContext::ensure_startup_after_iam(store, kms_interface, &server_ctx, iam)?;
|
||||
let topology_ready =
|
||||
publish_iam_ready_after_heal_topology(readiness.as_ref(), server_ctx.heal_topology_fingerprint(), endpoint_pools)
|
||||
.await?;
|
||||
return Ok(IamBootstrapDisposition::ReadyInline(topology_ready));
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -597,7 +586,7 @@ mod tests {
|
||||
let expected = crate::storage_api::startup::heal_control::heal_topology_fingerprint(&endpoint_pools)
|
||||
.expect("valid topology should hash");
|
||||
|
||||
publish_iam_ready_after_heal_topology(&readiness, Arc::clone(&cache), endpoint_pools, || Ok(()))
|
||||
publish_iam_ready_after_heal_topology(&readiness, Arc::clone(&cache), endpoint_pools)
|
||||
.await
|
||||
.expect("topology initialization should publish IAM readiness");
|
||||
|
||||
@@ -610,7 +599,6 @@ mod tests {
|
||||
&failed_readiness,
|
||||
Arc::clone(&invalid_cache),
|
||||
EndpointServerPools::from(Vec::<PoolEndpoints>::new()),
|
||||
|| panic!("invalid topology must not claim the context"),
|
||||
)
|
||||
.await
|
||||
.expect_err("invalid topology must block IAM readiness");
|
||||
@@ -640,8 +628,7 @@ mod tests {
|
||||
Ok(())
|
||||
})
|
||||
},
|
||||
|()| Box::pin(async { Ok(()) }),
|
||||
move |_| {
|
||||
move || {
|
||||
let finalize_calls = finalize_calls.clone();
|
||||
Box::pin(async move {
|
||||
let call = finalize_calls.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
@@ -684,8 +671,7 @@ mod tests {
|
||||
}
|
||||
})
|
||||
},
|
||||
|()| Box::pin(async { Ok(()) }),
|
||||
move |_| {
|
||||
move || {
|
||||
let readiness = readiness_for_finalize.clone();
|
||||
let observed_iam_ready = observed_iam_ready_for_finalize.clone();
|
||||
Box::pin(async move {
|
||||
@@ -705,197 +691,6 @@ mod tests {
|
||||
assert!(readiness.is_ready());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn recovery_publication_timeout_reuses_the_installed_context() {
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
// The legacy fixture exposes its real disks without instance endpoints.
|
||||
let (paths, store) = crate::app::gating_test_env::shared_gating_ecstore_and_disk_paths().await;
|
||||
assert!(Arc::ptr_eq(&store, &context.object_store()));
|
||||
let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: paths.len(),
|
||||
endpoints: Endpoints::from(
|
||||
paths
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, path)| {
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(path.to_str().expect("fixture disk path")).expect("fixture endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
),
|
||||
cmd_line: "recovery-publication-test".to_string(),
|
||||
platform: String::new(),
|
||||
}]);
|
||||
crate::storage_api::startup::heal_control::initialize_heal_topology_fingerprint(
|
||||
server_ctx.heal_topology_fingerprint(),
|
||||
endpoints.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare fixture fingerprint before pausing time");
|
||||
// A fresh paused runtime keeps real fixture I/O and its timer-wheel
|
||||
// rounding out of the exact retry/deadline observations below.
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.start_paused(true)
|
||||
.build()
|
||||
.expect("paused recovery runtime");
|
||||
runtime.block_on(async move {
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
let init_calls = Arc::new(AtomicUsize::new(0));
|
||||
let prepare_calls = Arc::new(AtomicUsize::new(0));
|
||||
let publish_calls = Arc::new(AtomicUsize::new(0));
|
||||
let started = tokio::time::Instant::now();
|
||||
let recovery = run_iam_recovery_loop(
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(30),
|
||||
None,
|
||||
{
|
||||
let context = Arc::clone(&context);
|
||||
let calls = Arc::clone(&init_calls);
|
||||
move || {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
let context = Arc::clone(&context);
|
||||
Box::pin(async move { Ok(context) })
|
||||
}
|
||||
},
|
||||
{
|
||||
let server_ctx = Arc::clone(&server_ctx);
|
||||
let readiness = Arc::clone(&readiness);
|
||||
let calls = Arc::clone(&prepare_calls);
|
||||
move |context| {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
let server_ctx = Arc::clone(&server_ctx);
|
||||
let readiness = Arc::clone(&readiness);
|
||||
let endpoints = endpoints.clone();
|
||||
Box::pin(async move {
|
||||
publish_iam_ready_after_heal_topology(
|
||||
readiness.as_ref(),
|
||||
server_ctx.heal_topology_fingerprint(),
|
||||
endpoints,
|
||||
|| server_ctx.try_install(context),
|
||||
)
|
||||
.await
|
||||
})
|
||||
}
|
||||
},
|
||||
{
|
||||
let readiness = Arc::clone(&readiness);
|
||||
let calls = Arc::clone(&publish_calls);
|
||||
move |prepared| {
|
||||
let attempt = calls.fetch_add(1, Ordering::SeqCst);
|
||||
let readiness = Arc::clone(&readiness);
|
||||
Box::pin(async move {
|
||||
let result = publish_runtime_ready_after_heal_topology(prepared, || async {
|
||||
crate::server::wait_for_runtime_readiness_with(
|
||||
Duration::from_secs(2),
|
||||
Duration::from_secs(1),
|
||||
|| async move {
|
||||
if attempt == 0 {
|
||||
std::future::pending::<crate::shared_types::DependencyReadiness>().await
|
||||
} else {
|
||||
crate::shared_types::DependencyReadiness {
|
||||
storage_ready: true,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
}
|
||||
}
|
||||
},
|
||||
|_| readiness.mark_stage(SystemStage::FullReady),
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
if attempt == 0 {
|
||||
assert!(result.is_err(), "the first publication must time out");
|
||||
assert!(!readiness.is_ready(), "timeout cannot publish FullReady");
|
||||
}
|
||||
result
|
||||
})
|
||||
}
|
||||
},
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(20), recovery)
|
||||
.await
|
||||
.expect("publication retry must succeed without attempting another context installation");
|
||||
assert_eq!(init_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(prepare_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(publish_calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(12));
|
||||
assert!(readiness.is_ready());
|
||||
assert!(Arc::ptr_eq(&server_ctx.installed_app_context().expect("installed context"), &context));
|
||||
assert_eq!(
|
||||
server_ctx
|
||||
.try_install(context)
|
||||
.expect_err("duplicate installs remain conflicts")
|
||||
.kind(),
|
||||
std::io::ErrorKind::AlreadyExists
|
||||
);
|
||||
});
|
||||
})
|
||||
.await
|
||||
.expect("recovery model completes without panic");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn recovery_preparation_retries_reuse_the_initialized_owner() {
|
||||
let owner = Arc::new(AtomicUsize::new(0));
|
||||
let init_calls = Arc::new(AtomicUsize::new(0));
|
||||
let publish_calls = Arc::new(AtomicUsize::new(0));
|
||||
run_iam_recovery_loop(
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(2),
|
||||
None,
|
||||
{
|
||||
let owner = Arc::clone(&owner);
|
||||
let calls = Arc::clone(&init_calls);
|
||||
move || {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
let owner = Arc::clone(&owner);
|
||||
Box::pin(async move { Ok(owner) })
|
||||
}
|
||||
},
|
||||
{
|
||||
let expected = Arc::clone(&owner);
|
||||
move |owner| {
|
||||
assert!(Arc::ptr_eq(&expected, &owner), "preparation must keep the same initialized IAM owner");
|
||||
Box::pin(async move {
|
||||
if owner.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
Err(Error::other("topology not ready"))
|
||||
} else {
|
||||
Ok(owner)
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
{
|
||||
let calls = Arc::clone(&publish_calls);
|
||||
move |owner| {
|
||||
let calls = Arc::clone(&calls);
|
||||
Box::pin(async move {
|
||||
assert_eq!(owner.load(Ordering::SeqCst), 2, "publication follows successful preparation");
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(init_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(owner.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(publish_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(Arc::strong_count(&owner), 1, "loop releases its initialized and prepared owners");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn recovery_loop_stops_after_shutdown_cancellation() {
|
||||
let init_calls = Arc::new(AtomicUsize::new(0));
|
||||
@@ -916,8 +711,7 @@ mod tests {
|
||||
Err(Error::other("keep retrying"))
|
||||
})
|
||||
},
|
||||
|()| Box::pin(async { Ok(()) }),
|
||||
|()| Box::pin(async { Ok(()) }),
|
||||
move || Box::pin(async { Ok(()) }),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +30,6 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
|
||||
success: false,
|
||||
error_info: Some(error.into()),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,59 +54,29 @@ async fn refresh_lock(
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
},
|
||||
Ok(false) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
},
|
||||
Err(err) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("can not refresh, resource: {0}, err: {1}", args.resource, err)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
|
||||
GenerallyLockResult {
|
||||
success,
|
||||
error_info: (!success).then(|| format!("lock not found for release: {lock_id}")),
|
||||
lock_info: None,
|
||||
release_status: if success {
|
||||
LockReleaseStatus::Released.into()
|
||||
} else {
|
||||
LockReleaseStatus::AlreadyAbsent.into()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn release_lock(
|
||||
lock_client: &std::sync::Arc<dyn rustfs_lock::client::LockClient>,
|
||||
args: &LockRequest,
|
||||
) -> GenerallyLockResponse {
|
||||
match lock_client.release(&args.lock_id).await {
|
||||
Ok(success) => {
|
||||
let result = lock_result_from_release(&args.lock_id, success);
|
||||
GenerallyLockResponse {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
release_status: result.release_status,
|
||||
}
|
||||
}
|
||||
Err(err) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!(
|
||||
"can not unlock, resource: {0}, owner: {1}, err: {2}",
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
if success {
|
||||
GenerallyLockResult {
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
} else {
|
||||
lock_result_from_error(format!("lock not found for release: {lock_id}"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,7 +93,6 @@ impl NodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -146,7 +113,6 @@ impl NodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -159,7 +125,6 @@ impl NodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
release_status: result.release_status,
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -169,7 +134,6 @@ impl NodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -186,13 +150,29 @@ impl NodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
let lock_client = self.get_lock_client()?;
|
||||
Ok(Response::new(release_lock(&lock_client, &args).await))
|
||||
match lock_client.release(&args.lock_id).await {
|
||||
Ok(success) => {
|
||||
let result = lock_result_from_release(&args.lock_id, success);
|
||||
Ok(Response::new(GenerallyLockResponse {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!(
|
||||
"can not unlock, resource: {0}, owner: {1}, err: {2}",
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_lock(
|
||||
@@ -210,7 +190,6 @@ impl NodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -226,7 +205,6 @@ impl NodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -236,7 +214,6 @@ impl NodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -351,7 +328,6 @@ mod tests {
|
||||
let result = lock_result_from_release(&lock_id, false);
|
||||
|
||||
assert!(!result.success);
|
||||
assert_eq!(result.release_status, super::LockReleaseStatus::AlreadyAbsent as i32);
|
||||
assert!(result.lock_info.is_none());
|
||||
assert!(
|
||||
result
|
||||
@@ -369,40 +345,6 @@ mod tests {
|
||||
assert!(!result.success);
|
||||
assert_eq!(result.error_info.as_deref(), Some("lock conflict"));
|
||||
assert!(result.lock_info.is_none());
|
||||
assert_eq!(result.release_status, super::LockReleaseStatus::Unspecified as i32);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn release_lock_preserves_strict_failure_and_reports_authoritative_absence() {
|
||||
use rustfs_lock::{GlobalLockManager, LocalClient, LockClient, LockRequest, LockType, ObjectKey};
|
||||
use std::sync::Arc;
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
let request = LockRequest::new(ObjectKey::new("bucket", "release-result"), LockType::Exclusive, "owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("acquire grant").success);
|
||||
|
||||
let released = super::release_lock(&client, &request).await;
|
||||
assert!(released.success);
|
||||
assert!(released.error_info.is_none());
|
||||
assert_eq!(released.release_status, super::LockReleaseStatus::Released as i32);
|
||||
let absent = super::release_lock(&client, &request).await;
|
||||
assert!(!absent.success, "ordinary release remains strict");
|
||||
assert!(absent.error_info.is_some());
|
||||
assert_eq!(absent.release_status, super::LockReleaseStatus::AlreadyAbsent as i32);
|
||||
|
||||
let successor = LockRequest::new(request.resource.clone(), LockType::Exclusive, "successor");
|
||||
assert!(client.acquire_lock(&successor).await.expect("acquire successor").success);
|
||||
let absent = super::release_lock(&client, &request).await;
|
||||
assert!(!absent.success);
|
||||
assert_eq!(absent.release_status, super::LockReleaseStatus::AlreadyAbsent as i32);
|
||||
assert!(
|
||||
client
|
||||
.check_status(&successor.lock_id)
|
||||
.await
|
||||
.expect("check successor")
|
||||
.is_some()
|
||||
);
|
||||
assert!(client.release(&successor.lock_id).await.expect("release successor"));
|
||||
}
|
||||
|
||||
// B1 -- refresh_lock delegates to LocalClient::refresh for a held lock, returns
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
TMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
mkdir -p "$TMP_DIR/bin" "$TMP_DIR/data" "$TMP_DIR/work"
|
||||
cat >"$TMP_DIR/bin/cargo" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
[ "$RUSTFS_ACCESS_KEY" = '"literal-access"' ]
|
||||
[ "$RUSTFS_SECRET_KEY" = "'literal-secret'" ]
|
||||
printf '%s\n' "$RUSTFS_OBS_LOG_DIRECTORY" >"$STARTED_FILE"
|
||||
printf '%s\n' "$@" >"$ARGS_FILE"
|
||||
SH
|
||||
chmod +x "$TMP_DIR/bin/cargo"
|
||||
|
||||
# Count actual external setup commands without replacing their behavior.
|
||||
for tool in mkdir tr grep sed; do
|
||||
real_tool=$(command -v "$tool")
|
||||
cat >"$TMP_DIR/bin/$tool" <<SH
|
||||
#!/bin/sh
|
||||
printf '%s\n' '$tool' >>"\$COMMAND_LOG"
|
||||
exec '$real_tool' "\$@"
|
||||
SH
|
||||
chmod +x "$TMP_DIR/bin/$tool"
|
||||
done
|
||||
|
||||
# Ownership tests must not change the host user's files or require root.
|
||||
cat >"$TMP_DIR/bin/chown" <<'SH'
|
||||
#!/bin/sh
|
||||
printf '%s\n' "$*" >>"$CHOWN_LOG"
|
||||
exit "${CHOWN_STATUS:-0}"
|
||||
SH
|
||||
chmod +x "$TMP_DIR/bin/chown"
|
||||
|
||||
run_entrypoint() {
|
||||
local log_dir="$1" volumes="${2:-$TMP_DIR/data}"
|
||||
shift
|
||||
if [ "$#" -gt 0 ]; then shift; fi
|
||||
rm -f "$TMP_DIR/started"
|
||||
: >"$TMP_DIR/commands"
|
||||
: >"$TMP_DIR/chown"
|
||||
(
|
||||
cd "$TMP_DIR/work"
|
||||
env -i PATH="$TMP_DIR/bin:$PATH" \
|
||||
RUSTFS_VOLUMES="$volumes" \
|
||||
RUSTFS_BOOTSTRAP_DIR="$TMP_DIR/no-bootstrap" \
|
||||
RUSTFS_OBS_LOG_DIRECTORY="$log_dir" \
|
||||
RUSTFS_ACCESS_KEY='"literal-access"' \
|
||||
RUSTFS_SECRET_KEY="'literal-secret'" \
|
||||
STARTED_FILE="$TMP_DIR/started" \
|
||||
ARGS_FILE="$TMP_DIR/args" \
|
||||
COMMAND_LOG="$TMP_DIR/commands" \
|
||||
CHOWN_LOG="$TMP_DIR/chown" \
|
||||
"$@" \
|
||||
sh "$ROOT_DIR/entrypoint.sh" cargo
|
||||
) >"$TMP_DIR/result.log" 2>&1
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
if ! grep -Fq -- "$1" "$TMP_DIR/result.log"; then
|
||||
echo "Expected entrypoint output to contain: $1" >&2
|
||||
cat "$TMP_DIR/result.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_start() {
|
||||
local log_dir="$1"
|
||||
shift
|
||||
if ! run_entrypoint "$log_dir" "$@"; then
|
||||
cat "$TMP_DIR/result.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
[ "$(cat "$TMP_DIR/started")" = "$log_dir" ]
|
||||
if grep -Eq '^(ERROR|HINT):' "$TMP_DIR/result.log"; then
|
||||
cat "$TMP_DIR/result.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
local log_dir="$1" quoted="$2" source="${3:-RUSTFS_OBS_LOG_DIRECTORY}" volumes="${4:-$TMP_DIR/data}" status=0
|
||||
run_entrypoint "$log_dir" "$volumes" || status=$?
|
||||
[ "$status" -eq 1 ]
|
||||
[ ! -e "$TMP_DIR/started" ]
|
||||
assert_contains "ERROR: Cannot create $source."
|
||||
assert_contains 'write permissions for the container user'
|
||||
if [ "$quoted" = true ]; then
|
||||
assert_contains 'HINT: RUSTFS_OBS_LOG_DIRECTORY contains literal surrounding quotes.'
|
||||
assert_contains 'RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"'
|
||||
assert_contains '- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/'
|
||||
elif grep -Fq 'HINT:' "$TMP_DIR/result.log"; then
|
||||
echo 'Unquoted path failures must not be diagnosed as a quoting problem' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eq 'literal-access|literal-secret' "$TMP_DIR/result.log"; then
|
||||
echo 'Directory diagnostics must not expose credentials' >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Empty and remote sinks do not create a local directory.
|
||||
expect_start ''
|
||||
expect_start 'https://logs.example.invalid/sink'
|
||||
[ ! -e "$TMP_DIR/work/https:" ]
|
||||
|
||||
# Existing, new, spaced and quote-containing paths retain their exact values.
|
||||
expect_start "$TMP_DIR/logs"
|
||||
[ -d "$TMP_DIR/logs" ]
|
||||
expect_start "$TMP_DIR/logs"
|
||||
expect_start "$TMP_DIR/logs with spaces/embedded\"quote"
|
||||
[ -d "$TMP_DIR/logs with spaces/embedded\"quote" ]
|
||||
expect_start '"quoted logs"'
|
||||
[ -d "$TMP_DIR/work/\"quoted logs\"" ]
|
||||
expect_start "'quoted logs'"
|
||||
[ -d "$TMP_DIR/work/'quoted logs'" ]
|
||||
|
||||
# A regular-file parent makes mkdir fail deterministically, even as root.
|
||||
# The quote-wrapped absolute value reproduces the malformed Compose list value.
|
||||
touch "$TMP_DIR/work/\"" "$TMP_DIR/work/'" "$TMP_DIR/blocked"
|
||||
expect_failure '"/var/log/rustfs/"' true
|
||||
expect_failure "'/var/log/rustfs/'" true
|
||||
expect_failure "$TMP_DIR/blocked/logs" false
|
||||
|
||||
# The same directory policy applies to data volumes, before any log setup.
|
||||
expect_failure "$TMP_DIR/not-created-logs" false RUSTFS_VOLUMES "$TMP_DIR/blocked/data"
|
||||
[ ! -e "$TMP_DIR/not-created-logs" ]
|
||||
|
||||
# Existing directories do no external setup work, even with ownership configured.
|
||||
expect_start "$TMP_DIR/logs" "$TMP_DIR/data" RUSTFS_UID=10001 RUSTFS_GID=10001
|
||||
[ ! -s "$TMP_DIR/commands" ]
|
||||
[ ! -s "$TMP_DIR/chown" ]
|
||||
|
||||
# Newly created directories get a non-recursive owner change in data/log order.
|
||||
expect_start "$TMP_DIR/owned-logs" "$TMP_DIR/owned-data" \
|
||||
RUSTFS_UID=10001 RUSTFS_GID=10002 RUSTFS_USERNAME=ignored RUSTFS_GROUPNAME=ignored
|
||||
[ "$(cat "$TMP_DIR/chown")" = "10001:10002 $TMP_DIR/owned-data
|
||||
10001:10002 $TMP_DIR/owned-logs" ]
|
||||
expect_start '' "$TMP_DIR/named-data" RUSTFS_USERNAME=rustfs RUSTFS_GROUPNAME=rustfs
|
||||
[ "$(cat "$TMP_DIR/chown")" = "rustfs:rustfs $TMP_DIR/named-data" ]
|
||||
expect_start '' "$TMP_DIR/chown-failure" RUSTFS_UID=10001 RUSTFS_GID=10001 CHOWN_STATUS=1
|
||||
[ -s "$TMP_DIR/chown" ]
|
||||
|
||||
expect_volumes() {
|
||||
local volumes="$1" expected_args="$2"
|
||||
expect_start '' "$volumes"
|
||||
[ "$(cat "$TMP_DIR/args")" = "$expected_args" ]
|
||||
}
|
||||
|
||||
# Keep delimiter and range parsing on one path, with unchanged child arguments.
|
||||
expect_volumes "$TMP_DIR/comma0,$TMP_DIR/comma1" "$TMP_DIR/comma0
|
||||
$TMP_DIR/comma1"
|
||||
[ -d "$TMP_DIR/comma0" ] && [ -d "$TMP_DIR/comma1" ]
|
||||
expect_volumes "$TMP_DIR/tab0"$'\t'"$TMP_DIR/tab1" "$TMP_DIR/tab0
|
||||
$TMP_DIR/tab1"
|
||||
[ -d "$TMP_DIR/tab0" ] && [ -d "$TMP_DIR/tab1" ]
|
||||
expect_volumes "$TMP_DIR/newline0
|
||||
$TMP_DIR/newline1" "$TMP_DIR/newline0
|
||||
$TMP_DIR/newline1"
|
||||
[ -d "$TMP_DIR/newline0" ] && [ -d "$TMP_DIR/newline1" ]
|
||||
|
||||
for dots in '..' '...'; do
|
||||
expect_volumes "$TMP_DIR/range${dots}{0${dots}1}" "$TMP_DIR/range${dots}0
|
||||
$TMP_DIR/range${dots}1"
|
||||
[ -d "$TMP_DIR/range${dots}0" ] && [ -d "$TMP_DIR/range${dots}1" ]
|
||||
done
|
||||
|
||||
# Distributed endpoints create local paths without becoming positional arguments.
|
||||
expect_volumes "http://node{1...2}:9000$TMP_DIR/url{0...1}" ''
|
||||
[ -d "$TMP_DIR/url0" ] && [ -d "$TMP_DIR/url1" ]
|
||||
expect_volumes 'http://node:9000' ''
|
||||
expect_volumes "$TMP_DIR/literal{abc}" "$TMP_DIR/literal{abc}"
|
||||
[ -d "$TMP_DIR/literal{abc}" ]
|
||||
|
||||
echo 'entrypoint directory and volume tests passed'
|
||||
Reference in New Issue
Block a user