mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(ecstore): add file-sync durability tier (#9422)
* refactor(ecstore): split durability sync capabilities Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * refactor(ecstore): classify durability barriers Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * feat(ecstore): add file-sync durability tier Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(storage): harden durability docker validation Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(storage): isolate durability PUT benchmarks Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(storage): expose file sync grouping controls Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
@@ -34,6 +34,9 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -81,6 +84,9 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -128,6 +134,9 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
@@ -175,6 +184,9 @@ services:
|
||||
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
|
||||
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
|
||||
|
||||
@@ -170,10 +170,17 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
# `RC_RELEASE=disabled` is reserved for local server-only validation when the
|
||||
# GitHub release asset is unavailable. The placeholder fails if invoked, so a
|
||||
# production image that needs the CLI must keep the default release download.
|
||||
# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
|
||||
# example set by docker-compose) and then to the build host, like the cargo
|
||||
# step above, so every path that builds this file passes a real architecture.
|
||||
RUN set -eu; \
|
||||
if [ "${RC_RELEASE}" = "disabled" ]; then \
|
||||
install -m 0755 /bin/false /usr/local/bin/rc; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
rc_arch="${TARGETARCH:-}"; \
|
||||
if [ -z "${rc_arch}" ]; then \
|
||||
case "${TARGETPLATFORM:-}" in \
|
||||
|
||||
@@ -208,7 +208,8 @@ pub mod bucket {
|
||||
|
||||
pub mod durability {
|
||||
pub use crate::bucket::durability::{
|
||||
BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
|
||||
BUCKET_DURABILITY_MODE_FILE_SYNC, BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -22,12 +22,13 @@
|
||||
//! `.minio.sys`) can never carry an override; they stay pinned to `strict`.
|
||||
//! See docs/operations/durability-modes.md for the semantics.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
|
||||
/// Valid per-bucket durability tier names. `legacy-off` is deliberately not
|
||||
/// representable per bucket: it exists only for the process-wide legacy
|
||||
/// switch (`RUSTFS_DRIVE_SYNC_ENABLE=false`).
|
||||
pub const BUCKET_DURABILITY_MODE_STRICT: &str = "strict";
|
||||
pub const BUCKET_DURABILITY_MODE_FILE_SYNC: &str = "file-sync";
|
||||
pub const BUCKET_DURABILITY_MODE_RELAXED: &str = "relaxed";
|
||||
pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
|
||||
|
||||
@@ -35,14 +36,69 @@ pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
|
||||
///
|
||||
/// An absent entry, an empty payload, or an empty `mode` string all mean
|
||||
/// "no override": the bucket follows the process-wide durability mode.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct BucketDurabilityConfig {
|
||||
/// Requested tier: `strict` | `relaxed` | `none` (case-insensitive,
|
||||
/// Requested tier: `strict` | `file-sync` | `relaxed` | `none` (case-insensitive,
|
||||
/// whitespace-tolerant). Empty means "inherit the global mode".
|
||||
#[serde(default)]
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct BucketDurabilityConfigWireRef<'a> {
|
||||
mode: &'a str,
|
||||
#[serde(skip_serializing_if = "is_false")]
|
||||
file_sync: bool,
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BucketDurabilityConfigWire {
|
||||
#[serde(default)]
|
||||
mode: String,
|
||||
#[serde(default)]
|
||||
file_sync: bool,
|
||||
}
|
||||
|
||||
impl Serialize for BucketDurabilityConfig {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let file_sync = self.mode.trim().eq_ignore_ascii_case(BUCKET_DURABILITY_MODE_FILE_SYNC);
|
||||
BucketDurabilityConfigWireRef {
|
||||
mode: if file_sync {
|
||||
BUCKET_DURABILITY_MODE_STRICT
|
||||
} else {
|
||||
&self.mode
|
||||
},
|
||||
file_sync,
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for BucketDurabilityConfig {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let wire = BucketDurabilityConfigWire::deserialize(deserializer)?;
|
||||
let mode = if wire.file_sync {
|
||||
if wire.mode.trim().eq_ignore_ascii_case(BUCKET_DURABILITY_MODE_STRICT) {
|
||||
BUCKET_DURABILITY_MODE_FILE_SYNC.to_string()
|
||||
} else {
|
||||
BUCKET_DURABILITY_MODE_STRICT.to_string()
|
||||
}
|
||||
} else {
|
||||
wire.mode
|
||||
};
|
||||
Ok(Self { mode })
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketDurabilityConfig {
|
||||
pub fn new(mode: &str) -> Self {
|
||||
Self { mode: mode.to_string() }
|
||||
@@ -52,7 +108,10 @@ impl BucketDurabilityConfig {
|
||||
pub fn is_valid_mode(mode: &str) -> bool {
|
||||
matches!(
|
||||
mode.trim().to_ascii_lowercase().as_str(),
|
||||
BUCKET_DURABILITY_MODE_STRICT | BUCKET_DURABILITY_MODE_RELAXED | BUCKET_DURABILITY_MODE_NONE
|
||||
BUCKET_DURABILITY_MODE_STRICT
|
||||
| BUCKET_DURABILITY_MODE_FILE_SYNC
|
||||
| BUCKET_DURABILITY_MODE_RELAXED
|
||||
| BUCKET_DURABILITY_MODE_NONE
|
||||
)
|
||||
}
|
||||
|
||||
@@ -117,6 +176,7 @@ mod tests {
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("file-sync"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("relaxed"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("none"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode(" RELAXED "));
|
||||
@@ -130,6 +190,7 @@ mod tests {
|
||||
fn normalized_mode_canonicalizes_or_rejects() {
|
||||
assert_eq!(BucketDurabilityConfig::new(" Relaxed ").normalized_mode().as_deref(), Some("relaxed"));
|
||||
assert_eq!(BucketDurabilityConfig::new("strict").normalized_mode().as_deref(), Some("strict"));
|
||||
assert_eq!(BucketDurabilityConfig::new(" FILE-SYNC ").normalized_mode().as_deref(), Some("file-sync"));
|
||||
assert_eq!(BucketDurabilityConfig::default().normalized_mode(), None);
|
||||
assert_eq!(BucketDurabilityConfig::new("bogus").normalized_mode(), None);
|
||||
}
|
||||
@@ -141,11 +202,34 @@ mod tests {
|
||||
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize");
|
||||
assert_eq!(back, cfg);
|
||||
|
||||
let file_sync = BucketDurabilityConfig::new("file-sync");
|
||||
let json = serde_json::to_vec(&file_sync).expect("serialize file-sync");
|
||||
assert_eq!(json, br#"{"mode":"strict","file_sync":true}"#);
|
||||
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize file-sync");
|
||||
assert_eq!(back.normalized_mode().as_deref(), Some("file-sync"));
|
||||
|
||||
let malformed = serde_json::from_slice::<BucketDurabilityConfig>(br#"{"mode":"none","file_sync":true}"#)
|
||||
.expect("deserialize malformed file-sync");
|
||||
assert_eq!(malformed.normalized_mode().as_deref(), Some("strict"));
|
||||
|
||||
// Empty object deserializes to the "inherit" default.
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_sync_encoding_fails_closed_on_legacy_readers() {
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyConfig {
|
||||
#[serde(default)]
|
||||
mode: String,
|
||||
}
|
||||
|
||||
let json = serde_json::to_vec(&BucketDurabilityConfig::new("file-sync")).expect("serialize file-sync");
|
||||
let legacy: LegacyConfig = serde_json::from_slice(&json).expect("legacy reader should ignore the marker");
|
||||
assert_eq!(legacy.mode, "strict", "legacy readers must apply the stronger strict fallback");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_inherits_when_unset() {
|
||||
temp_env::with_var_unset(ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
@@ -157,6 +241,7 @@ mod tests {
|
||||
fn new_bucket_default_honors_explicit_tiers() {
|
||||
for mode in [
|
||||
BUCKET_DURABILITY_MODE_STRICT,
|
||||
BUCKET_DURABILITY_MODE_FILE_SYNC,
|
||||
BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_NONE,
|
||||
] {
|
||||
|
||||
+235
-106
@@ -994,7 +994,7 @@ async fn write_delete_rollback_files(
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let staging_root = io_root.join(RUSTFS_META_TMP_BUCKET);
|
||||
let io_root = io_root.to_path_buf();
|
||||
let object_dir_buf = object_dir.to_path_buf();
|
||||
@@ -1249,7 +1249,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&dst, namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -1306,7 +1306,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
}
|
||||
delete_recovery::restore_barrier(volume, object_dir, rollback_dir).await?;
|
||||
// Persist both sides of the restore rename before discarding its evidence.
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(object_dir, namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -1319,7 +1319,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -1371,7 +1371,7 @@ async fn remove_version_delete_markers(
|
||||
}
|
||||
// Also sync on retry: a previous unlink may have succeeded before its
|
||||
// directory sync failed. Keep the journal until those removals persist.
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&entry.path(), namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -2022,7 +2022,7 @@ const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy"
|
||||
const ENV_RUSTFS_DRIVE_SYNC_ENABLE: &str = "RUSTFS_DRIVE_SYNC_ENABLE";
|
||||
const DEFAULT_RUSTFS_DRIVE_SYNC_ENABLE: bool = true;
|
||||
|
||||
/// Durability tier for object data-path writes: `strict` (default) | `relaxed` | `none`.
|
||||
/// Durability tier for object data-path writes: `strict` (default) | `file-sync` | `relaxed` | `none`.
|
||||
/// See docs/operations/durability-modes.md for the power-loss guarantee matrix.
|
||||
const ENV_RUSTFS_DURABILITY_MODE: &str = "RUSTFS_DURABILITY_MODE";
|
||||
|
||||
@@ -2248,10 +2248,15 @@ pub(crate) enum DurabilityMode {
|
||||
/// Every commit point is fsynced: shard/part contents, xl.meta contents,
|
||||
/// rollback backups, and the directory entries of commit renames.
|
||||
Strict,
|
||||
/// Payload and metadata file contents are fdatasynced, while ordinary
|
||||
/// object commit and newly created ancestor directory entries are left to
|
||||
/// the filesystem. Recovery protocol state retains strict barriers.
|
||||
FileSync,
|
||||
/// Object payload bytes (erasure shard files, multipart part files) are
|
||||
/// still fdatasynced before the commit rename, but metadata commits
|
||||
/// (xl.meta contents, rollback backups, directory entries) are left to
|
||||
/// the page cache. Aligned with MinIO's default durability posture.
|
||||
/// the page cache. This is weaker than MinIO's default for xl.meta, which
|
||||
/// syncs the metadata file contents but not each commit parent directory.
|
||||
Relaxed,
|
||||
/// No fsync on the object data path at all. System-critical namespaces
|
||||
/// are still pinned to `Strict`.
|
||||
@@ -2266,6 +2271,7 @@ impl DurabilityMode {
|
||||
pub(crate) fn parse(value: &str) -> Option<Self> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"strict" => Some(Self::Strict),
|
||||
"file-sync" => Some(Self::FileSync),
|
||||
"relaxed" => Some(Self::Relaxed),
|
||||
"none" => Some(Self::None),
|
||||
_ => None,
|
||||
@@ -2275,6 +2281,7 @@ impl DurabilityMode {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Strict => "strict",
|
||||
Self::FileSync => "file-sync",
|
||||
Self::Relaxed => "relaxed",
|
||||
Self::None => "none",
|
||||
Self::LegacyOff => "legacy-off",
|
||||
@@ -2284,14 +2291,51 @@ impl DurabilityMode {
|
||||
/// Whether object payload bytes (erasure shard files, multipart part
|
||||
/// files) must be fdatasynced at commit points.
|
||||
fn syncs_data_shards(self) -> bool {
|
||||
matches!(self, Self::Strict | Self::Relaxed)
|
||||
matches!(self, Self::Strict | Self::FileSync | Self::Relaxed)
|
||||
}
|
||||
|
||||
/// Whether metadata commits must be fsynced: xl.meta contents, rollback
|
||||
/// backups, and the directory entries created by commit renames.
|
||||
fn syncs_commit_metadata(self) -> bool {
|
||||
/// Whether metadata file contents, including xl.meta and rollback
|
||||
/// backups, must be fdatasynced before publication.
|
||||
fn syncs_metadata_files(self) -> bool {
|
||||
matches!(self, Self::Strict | Self::FileSync)
|
||||
}
|
||||
|
||||
/// Whether ordinary commit rename directory entries and newly created
|
||||
/// ancestor directories must be fsynced before acknowledgement.
|
||||
fn syncs_commit_directories(self) -> bool {
|
||||
matches!(self, Self::Strict)
|
||||
}
|
||||
|
||||
/// Whether a target has the complete file-and-directory durability proof
|
||||
/// required by recovery protocols that may otherwise resync it.
|
||||
fn fully_syncs_commit_target(self) -> bool {
|
||||
self.syncs_metadata_files() && self.syncs_commit_directories()
|
||||
}
|
||||
|
||||
/// Whether recovery journals, witnesses, and protocol state must retain
|
||||
/// their complete barriers. This is independent from ordinary object
|
||||
/// namespace publication so a future file-only tier cannot weaken
|
||||
/// crash-recovery ordering.
|
||||
fn syncs_recovery_protocol_state(self) -> bool {
|
||||
matches!(self, Self::Strict | Self::FileSync)
|
||||
}
|
||||
|
||||
fn recovery_sync_mode(self) -> SyncMode {
|
||||
if self.syncs_recovery_protocol_state() {
|
||||
SyncMode::FileAndDir
|
||||
} else {
|
||||
SyncMode::None
|
||||
}
|
||||
}
|
||||
|
||||
fn metadata_sync_mode(self) -> SyncMode {
|
||||
match (self.syncs_metadata_files(), self.syncs_commit_directories()) {
|
||||
(true, true) => SyncMode::FileAndDir,
|
||||
(true, false) => SyncMode::FileOnly,
|
||||
(false, false) => SyncMode::None,
|
||||
(false, true) => SyncMode::FileAndDir,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure resolution of the durability mode from configuration values.
|
||||
@@ -2310,7 +2354,7 @@ fn resolve_durability_mode(mode_env: Option<String>, legacy_drive_sync_enabled:
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
value = %raw,
|
||||
"Invalid RUSTFS_DURABILITY_MODE value; expected strict|relaxed|none, falling back to the legacy drive-sync switch"
|
||||
"Invalid RUSTFS_DURABILITY_MODE value; expected strict|file-sync|relaxed|none, falling back to the legacy drive-sync switch"
|
||||
);
|
||||
}
|
||||
if legacy_drive_sync_enabled {
|
||||
@@ -3912,23 +3956,19 @@ fn validate_format_control_publication(
|
||||
|
||||
/// Durability mode for `write_all_internal`.
|
||||
///
|
||||
/// `FileOnly` is reserved for tmp files the caller immediately renames away.
|
||||
/// The safe-rename recipe (file content fdatasync -> rename -> fsync of the
|
||||
/// destination parent directory) never needs the tmp directory entry to be
|
||||
/// durable: the rename removes it, and a crash before the rename means the
|
||||
/// operation was never acknowledged, so there is nothing to recover. Files
|
||||
/// that stay where they are written (format.json via `write_all_public`, the
|
||||
/// old-metadata rollback backup in `rename_data`, ...) must use `FileAndDir`
|
||||
/// so both the contents and the new directory entry survive power loss.
|
||||
/// For strict durability, `FileOnly` is used for tmp files the caller
|
||||
/// immediately renames away and follows with a destination-parent sync. The
|
||||
/// tmp directory entry need not be durable because the rename removes it and
|
||||
/// a crash before publication was never acknowledged. A file-only durability
|
||||
/// tier may also deliberately omit the later namespace barrier while still
|
||||
/// requiring the file inode contents to reach stable storage.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum SyncMode {
|
||||
/// No fsync; durability is not required (or drive sync is disabled).
|
||||
None,
|
||||
/// fdatasync the file contents, then fsync its parent directory.
|
||||
FileAndDir,
|
||||
/// fdatasync only the file contents. Only valid when the caller renames
|
||||
/// the file away right after the write and fsyncs the rename
|
||||
/// destination's parent directory before acknowledging.
|
||||
/// fdatasync only the file contents.
|
||||
FileOnly,
|
||||
}
|
||||
|
||||
@@ -10720,7 +10760,7 @@ impl LocalDisk {
|
||||
// decision and also covers inline objects with no data directory.
|
||||
let receipt_path = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, namespace_owner.clone()).await;
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
let receipt = match std::fs::OpenOptions::new().write(true).create_new(true).open(&receipt_path) {
|
||||
Ok(receipt) => receipt,
|
||||
@@ -10754,7 +10794,7 @@ impl LocalDisk {
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&self.io_get_object_path(volume, &path)?, namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -10853,7 +10893,7 @@ impl LocalDisk {
|
||||
}
|
||||
let metadata_path = self.io_get_object_path(volume, &format!("{object}/{STORAGE_FORMAT_FILE}"))?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, Some(owner)).await;
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let result = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
if let Some(check) = &multipart_check {
|
||||
check.callback()()?;
|
||||
@@ -10913,7 +10953,7 @@ impl LocalDisk {
|
||||
// after classification and must never be recursively removed.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner.clone()).await;
|
||||
let data_path = file_path.clone();
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let keep_rollback_witnesses = opt.recoverable_delete;
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
@@ -12070,7 +12110,7 @@ impl LocalDisk {
|
||||
}
|
||||
// fsync_dir_std is a no-op outside Unix, so those platforms cannot
|
||||
// yet prove this repair's durable publication requirement.
|
||||
if !cfg!(unix) || !effective_durability(volume).syncs_commit_metadata() {
|
||||
if !cfg!(unix) || !effective_durability(volume).fully_syncs_commit_target() {
|
||||
return Err(DiskError::other(
|
||||
"transition reconciliation requires Unix directory sync and enabled bucket metadata durability",
|
||||
));
|
||||
@@ -12176,7 +12216,9 @@ impl LocalDisk {
|
||||
|
||||
let durability = effective_durability(volume);
|
||||
|
||||
let do_sync = sync && durability.syncs_commit_metadata();
|
||||
let sync_file = sync && durability.syncs_metadata_files();
|
||||
let sync_directory = sync && durability.syncs_commit_directories();
|
||||
let uses_sync_lane = sync_file || sync_directory;
|
||||
if let Some(ref admission) = admission
|
||||
&& !admission.is_initial()
|
||||
{
|
||||
@@ -12186,7 +12228,7 @@ impl LocalDisk {
|
||||
let precondition = admission.as_ref().map(|a| a.precondition());
|
||||
let root = self.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
|
||||
os::run_blocking_namespace_operation_with_sync(lease, uses_sync_lane, move || {
|
||||
let _index_pin = prepared_index.map(|record| record.run_std()).transpose()?;
|
||||
#[cfg(test)]
|
||||
if let Some(pin) = &_index_pin {
|
||||
@@ -12197,7 +12239,7 @@ impl LocalDisk {
|
||||
run_owned_file_before_open(&tmp_file_path);
|
||||
let mut file = root.create_truncate(&tmp_file_path)?;
|
||||
std::io::Write::write_all(&mut file, buf.as_ref())?;
|
||||
if do_sync {
|
||||
if sync_file {
|
||||
file.sync_data()?;
|
||||
}
|
||||
drop(file);
|
||||
@@ -12207,7 +12249,7 @@ impl LocalDisk {
|
||||
&volume_dir,
|
||||
&root,
|
||||
precondition.as_deref(),
|
||||
do_sync,
|
||||
sync_directory,
|
||||
)
|
||||
})
|
||||
.await
|
||||
@@ -12218,9 +12260,9 @@ impl LocalDisk {
|
||||
// The tmp file is renamed to its final location right below, so only
|
||||
// its contents must be durable here (SyncMode::FileOnly): the rename
|
||||
// drops the tmp directory entry, and the destination parent directory
|
||||
// is fsynced after the rename. Both are metadata commits, so relaxed
|
||||
// tiers skip them.
|
||||
let tmp_sync = if do_sync { SyncMode::FileOnly } else { SyncMode::None };
|
||||
// is fsynced after the rename when the selected durability tier
|
||||
// requires namespace persistence.
|
||||
let tmp_sync = if sync_file { SyncMode::FileOnly } else { SyncMode::None };
|
||||
self.write_all_internal(&tmp_file_path, buf, tmp_sync, &tmp_volume_dir)
|
||||
.await?;
|
||||
|
||||
@@ -12245,13 +12287,13 @@ impl LocalDisk {
|
||||
&self.publication_root,
|
||||
publication_owner,
|
||||
admission.as_ref().map(|admission| admission.precondition()),
|
||||
admission.is_some() && sync && durability.syncs_commit_metadata(),
|
||||
admission.is_some() && sync_directory,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if sync
|
||||
&& admission.is_none()
|
||||
&& durability.syncs_commit_metadata()
|
||||
&& sync_directory
|
||||
&& let Some(parent) = file_path.parent()
|
||||
{
|
||||
os::fsync_dir_with_owner(parent, namespace_owner)
|
||||
@@ -12285,11 +12327,8 @@ impl LocalDisk {
|
||||
// rename follows — so the new directory entry must be fsynced too.
|
||||
// System-critical volumes are pinned to strict by effective_durability;
|
||||
// only the legacy full-off switch (historical semantics) skips this.
|
||||
let sync = if effective_durability(volume).syncs_commit_metadata() {
|
||||
SyncMode::FileAndDir
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
let durability = effective_durability(volume);
|
||||
let sync = durability.metadata_sync_mode();
|
||||
self.write_all_private(volume, path, data, sync, &volume_dir).await?;
|
||||
|
||||
Ok(())
|
||||
@@ -14054,7 +14093,7 @@ impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&marker_path);
|
||||
let marker = std::fs::File::create(marker_path)?;
|
||||
let sync = effective_durability(&volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(&volume).syncs_recovery_protocol_state();
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
}
|
||||
@@ -14070,7 +14109,7 @@ impl LocalDisk {
|
||||
return Ok(true);
|
||||
}
|
||||
let marker = File::create(marker_path).await.map_err(to_file_error)?;
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
marker.sync_all().await.map_err(to_file_error)?;
|
||||
os::fsync_dir(&data_path).await.map_err(to_file_error)?;
|
||||
}
|
||||
@@ -14143,7 +14182,7 @@ impl LocalDisk {
|
||||
DELETE_DATA_DIR_MARKER_PREFIX
|
||||
};
|
||||
let committed_name = format!("{prefix}{rollback_dir}");
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
for entry_path in matching_dirs {
|
||||
let reserved_path = entry_path.join(&reserved_name);
|
||||
match os::rename_with_namespace_owner(&reserved_path, &entry_path.join(&committed_name), namespace_owner.clone())
|
||||
@@ -15062,7 +15101,7 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let path = path.to_string();
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync_metadata = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
os::record_native_publication(&file_path);
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = self.mount_lease.clone();
|
||||
@@ -15195,7 +15234,7 @@ impl DiskAPI for LocalDisk {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync_metadata = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let publication_root = self.publication_root.clone();
|
||||
return Ok(spawn_blocking_with_disk_io_permit(None, move || {
|
||||
#[cfg(test)]
|
||||
@@ -15505,7 +15544,7 @@ impl DiskAPI for LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let sync = durability.syncs_data_shards() || durability.syncs_commit_metadata();
|
||||
let sync = durability.syncs_data_shards() || durability.syncs_metadata_files() || durability.syncs_commit_directories();
|
||||
let publication_root = self.publication_root.clone();
|
||||
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
|
||||
if let Some(ref source) = admission_source {
|
||||
@@ -15712,7 +15751,7 @@ impl DiskAPI for LocalDisk {
|
||||
&dst_volume_dir,
|
||||
&publication_root,
|
||||
precondition.as_deref(),
|
||||
durability.syncs_commit_metadata() && !same_parent,
|
||||
durability.syncs_commit_directories() && !same_parent,
|
||||
)
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
@@ -15783,7 +15822,7 @@ impl DiskAPI for LocalDisk {
|
||||
let new_meta_path = staging_path.join(PART_TRANSACTION_NEW_META);
|
||||
let mut new_meta = create_new_in_dir_fd(txn_dir_fd.as_deref(), PART_TRANSACTION_NEW_META, &new_meta_path)?;
|
||||
std::io::Write::write_all(&mut new_meta, &meta)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
if durability.syncs_metadata_files() {
|
||||
new_meta.sync_data()?;
|
||||
}
|
||||
// Windows rejects renaming a directory while one of its children is
|
||||
@@ -15796,7 +15835,7 @@ impl DiskAPI for LocalDisk {
|
||||
PART_TRANSACTION_PUBLISH_META,
|
||||
&staging_path.join(PART_TRANSACTION_PUBLISH_META),
|
||||
);
|
||||
let sync_overwrite_backup = durability.syncs_commit_metadata() && (had_old_data || had_old_meta);
|
||||
let sync_overwrite_backup = durability.syncs_recovery_protocol_state() && (had_old_data || had_old_meta);
|
||||
if sync_overwrite_backup {
|
||||
os::fsync_directory_handle_std(&staging_path, txn_dir_fd.as_deref())?;
|
||||
}
|
||||
@@ -15901,7 +15940,7 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
};
|
||||
let durability = effective_durability(volume);
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
let sync = durability.syncs_recovery_protocol_state();
|
||||
let part_number = current_data_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
@@ -15932,7 +15971,7 @@ impl DiskAPI for LocalDisk {
|
||||
if let Err(err) = std::fs::write(transaction_path.join(PART_TRANSACTION_ROLLBACK), []) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
if durability.syncs_commit_metadata()
|
||||
if durability.syncs_recovery_protocol_state()
|
||||
&& let Err(err) = os::fsync_directory_handle_std(&transaction_path, txn_dir_fd.as_deref())
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
@@ -15953,7 +15992,7 @@ impl DiskAPI for LocalDisk {
|
||||
) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
if durability.syncs_commit_metadata()
|
||||
if durability.syncs_recovery_protocol_state()
|
||||
&& let Some(parent) = current_data_path.parent()
|
||||
&& let Err(err) = os::fsync_directory_handle_std(parent, txn_parent_fd.as_deref())
|
||||
{
|
||||
@@ -16006,7 +16045,7 @@ impl DiskAPI for LocalDisk {
|
||||
) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
if durability.syncs_commit_metadata()
|
||||
if durability.syncs_recovery_protocol_state()
|
||||
&& let Err(err) = os::fsync_directory_handle_std(parent, txn_parent_fd.as_deref())
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
@@ -16178,7 +16217,9 @@ impl DiskAPI for LocalDisk {
|
||||
// its directory entry must be durable before we return. Relaxed keeps the
|
||||
// part payload fdatasync but leaves the directory entry to the page cache.
|
||||
let durability = effective_durability(dst_volume);
|
||||
let sync = (durability.syncs_data_shards() && !src_is_dir) || durability.syncs_commit_metadata();
|
||||
let sync = (durability.syncs_data_shards() && !src_is_dir)
|
||||
|| durability.syncs_metadata_files()
|
||||
|| durability.syncs_commit_directories();
|
||||
let publication_root = self.publication_root.clone();
|
||||
let src_file_for_cleanup = src_file_path.clone();
|
||||
let src_volume_for_cleanup = src_volume_dir.clone();
|
||||
@@ -16331,7 +16372,7 @@ impl DiskAPI for LocalDisk {
|
||||
rename_started,
|
||||
);
|
||||
rename_res.map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() && transaction_publish_meta.is_none() {
|
||||
if durability.syncs_commit_directories() && transaction_publish_meta.is_none() {
|
||||
let parent = dst_file_path
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::other("source metadata parent is missing"))
|
||||
@@ -16352,7 +16393,7 @@ impl DiskAPI for LocalDisk {
|
||||
&dst_volume_dir,
|
||||
&publication_root,
|
||||
precondition.as_deref(),
|
||||
durability.syncs_commit_metadata() && transaction_publish_meta.is_none(),
|
||||
durability.syncs_commit_directories() && transaction_publish_meta.is_none(),
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
@@ -16394,7 +16435,7 @@ impl DiskAPI for LocalDisk {
|
||||
rename_started,
|
||||
);
|
||||
rename_res.map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
if durability.syncs_commit_directories() {
|
||||
let parent = dst_meta_path
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::other("source metadata parent is missing"))
|
||||
@@ -16415,7 +16456,7 @@ impl DiskAPI for LocalDisk {
|
||||
&dst_volume_dir,
|
||||
&publication_root,
|
||||
precondition.as_deref(),
|
||||
durability.syncs_commit_metadata(),
|
||||
durability.syncs_commit_directories(),
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
@@ -16714,7 +16755,7 @@ impl DiskAPI for LocalDisk {
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
self.rename_file(src_volume, src_path, dst_volume, dst_path).await?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
if durability.syncs_commit_directories() {
|
||||
os::fsync_dir_with_owner(destination.parent().ok_or(DiskError::InvalidPath)?, usage_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -17298,48 +17339,55 @@ impl DiskAPI for LocalDisk {
|
||||
let xl_file_path = self.io_get_object_path(volume, &xl_rel_path)?;
|
||||
check_path_length(xl_file_path.to_string_lossy().as_ref())?;
|
||||
let tmp_file_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, Uuid::new_v4().to_string().as_str())?;
|
||||
let sync = !opts.no_persistence && effective_durability(volume).syncs_commit_metadata();
|
||||
let durability = effective_durability(volume);
|
||||
let sync_file = !opts.no_persistence && durability.syncs_metadata_files();
|
||||
let sync_directory = !opts.no_persistence && durability.syncs_commit_directories();
|
||||
let uses_sync_lane = sync_file || sync_directory;
|
||||
let replace_user_metadata = opts.replace_user_metadata;
|
||||
let has_version_id = fi.version_id.is_some();
|
||||
let publication_root = self.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&xl_file_path, namespace_owner).await;
|
||||
return os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<Result<()>> {
|
||||
let buf = match std::fs::read(&xl_file_path) {
|
||||
Ok(buf) => buf,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound && has_version_id => {
|
||||
return os::run_blocking_namespace_operation_with_sync(
|
||||
lease,
|
||||
uses_sync_lane,
|
||||
move || -> std::io::Result<Result<()>> {
|
||||
let buf = match std::fs::read(&xl_file_path) {
|
||||
Ok(buf) => buf,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound && has_version_id => {
|
||||
return Ok(Err(DiskError::FileVersionNotFound));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !FileMeta::is_xl2_v1_format(&buf) {
|
||||
return Ok(Err(DiskError::FileVersionNotFound));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !FileMeta::is_xl2_v1_format(&buf) {
|
||||
return Ok(Err(DiskError::FileVersionNotFound));
|
||||
}
|
||||
let wbuf = match FileMeta::load(&buf).and_then(|mut xl_meta| {
|
||||
xl_meta.update_object_version_with_opts(fi, replace_user_metadata)?;
|
||||
xl_meta.marshal_msg()
|
||||
}) {
|
||||
Ok(wbuf) => wbuf,
|
||||
Err(err) => return Ok(Err(err.into())),
|
||||
};
|
||||
if let Some(parent) = tmp_file_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut tmp_file = std::fs::File::create(&tmp_file_path)?;
|
||||
std::io::Write::write_all(&mut tmp_file, &wbuf)?;
|
||||
if sync && !wbuf.is_empty() {
|
||||
tmp_file.sync_data()?;
|
||||
}
|
||||
drop(tmp_file);
|
||||
os::rename_all_with_precondition_and_parent_sync_std(
|
||||
&tmp_file_path,
|
||||
&xl_file_path,
|
||||
&volume_dir,
|
||||
&publication_root,
|
||||
None,
|
||||
sync,
|
||||
)?;
|
||||
Ok(Ok(()))
|
||||
})
|
||||
let wbuf = match FileMeta::load(&buf).and_then(|mut xl_meta| {
|
||||
xl_meta.update_object_version_with_opts(fi, replace_user_metadata)?;
|
||||
xl_meta.marshal_msg()
|
||||
}) {
|
||||
Ok(wbuf) => wbuf,
|
||||
Err(err) => return Ok(Err(err.into())),
|
||||
};
|
||||
if let Some(parent) = tmp_file_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut tmp_file = std::fs::File::create(&tmp_file_path)?;
|
||||
std::io::Write::write_all(&mut tmp_file, &wbuf)?;
|
||||
if sync_file && !wbuf.is_empty() {
|
||||
tmp_file.sync_data()?;
|
||||
}
|
||||
drop(tmp_file);
|
||||
os::rename_all_with_precondition_and_parent_sync_std(
|
||||
&tmp_file_path,
|
||||
&xl_file_path,
|
||||
&volume_dir,
|
||||
&publication_root,
|
||||
None,
|
||||
sync_directory,
|
||||
)?;
|
||||
Ok(Ok(()))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
@@ -22741,6 +22789,23 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename_data_file_sync_new_object_skips_ancestor_fsync() {
|
||||
let bucket = "new-object-bucket-file-sync";
|
||||
let (disk, _dir) = commit_new_object(DurabilityMode::FileSync, bucket, "prefix/new-object").await;
|
||||
|
||||
let bucket_dir = disk.get_bucket_path(bucket).expect("bucket path should resolve");
|
||||
let prefix_dir = disk.get_object_path(bucket, "prefix").expect("prefix path should resolve");
|
||||
assert!(
|
||||
!os::fsync_dir_recorder::was_fsynced(&prefix_dir),
|
||||
"file-sync must omit the ordinary prefix-directory barrier"
|
||||
);
|
||||
assert!(
|
||||
!os::fsync_dir_recorder::was_fsynced(&bucket_dir),
|
||||
"file-sync must omit the ordinary bucket-directory barrier"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn windows_and_unix_rename_data_new_inline_object_fsyncs_new_ancestor_dirs() {
|
||||
// The inline commit path (fi.data present) has the same mkdir gap as the
|
||||
@@ -24449,6 +24514,7 @@ mod test {
|
||||
assert_eq!(resolve_durability_mode(None, false), DurabilityMode::LegacyOff);
|
||||
// Explicit mode wins over the legacy switch.
|
||||
assert_eq!(resolve_durability_mode(Some("strict".into()), false), DurabilityMode::Strict);
|
||||
assert_eq!(resolve_durability_mode(Some("file-sync".into()), false), DurabilityMode::FileSync);
|
||||
assert_eq!(resolve_durability_mode(Some("relaxed".into()), true), DurabilityMode::Relaxed);
|
||||
assert_eq!(resolve_durability_mode(Some("none".into()), true), DurabilityMode::None);
|
||||
// Case- and whitespace-tolerant.
|
||||
@@ -24463,15 +24529,38 @@ mod test {
|
||||
fn test_durability_mode_sync_gates() {
|
||||
// Strict = current main behavior: every commit point synced.
|
||||
assert!(DurabilityMode::Strict.syncs_data_shards());
|
||||
assert!(DurabilityMode::Strict.syncs_commit_metadata());
|
||||
assert!(DurabilityMode::Strict.syncs_metadata_files());
|
||||
assert!(DurabilityMode::Strict.syncs_commit_directories());
|
||||
assert!(DurabilityMode::Strict.fully_syncs_commit_target());
|
||||
assert!(DurabilityMode::Strict.syncs_recovery_protocol_state());
|
||||
assert_eq!(DurabilityMode::Strict.metadata_sync_mode(), SyncMode::FileAndDir);
|
||||
assert_eq!(DurabilityMode::Strict.recovery_sync_mode(), SyncMode::FileAndDir);
|
||||
// FileSync matches MinIO's file-content boundary while preserving
|
||||
// strict recovery-state ordering.
|
||||
assert!(DurabilityMode::FileSync.syncs_data_shards());
|
||||
assert!(DurabilityMode::FileSync.syncs_metadata_files());
|
||||
assert!(!DurabilityMode::FileSync.syncs_commit_directories());
|
||||
assert!(!DurabilityMode::FileSync.fully_syncs_commit_target());
|
||||
assert!(DurabilityMode::FileSync.syncs_recovery_protocol_state());
|
||||
assert_eq!(DurabilityMode::FileSync.metadata_sync_mode(), SyncMode::FileOnly);
|
||||
assert_eq!(DurabilityMode::FileSync.recovery_sync_mode(), SyncMode::FileAndDir);
|
||||
// Relaxed keeps payload durability, drops metadata-commit fsyncs.
|
||||
assert!(DurabilityMode::Relaxed.syncs_data_shards());
|
||||
assert!(!DurabilityMode::Relaxed.syncs_commit_metadata());
|
||||
assert!(!DurabilityMode::Relaxed.syncs_metadata_files());
|
||||
assert!(!DurabilityMode::Relaxed.syncs_commit_directories());
|
||||
assert!(!DurabilityMode::Relaxed.fully_syncs_commit_target());
|
||||
assert!(!DurabilityMode::Relaxed.syncs_recovery_protocol_state());
|
||||
assert_eq!(DurabilityMode::Relaxed.metadata_sync_mode(), SyncMode::None);
|
||||
assert_eq!(DurabilityMode::Relaxed.recovery_sync_mode(), SyncMode::None);
|
||||
// None and the legacy full-off switch sync nothing on the data path.
|
||||
assert!(!DurabilityMode::None.syncs_data_shards());
|
||||
assert!(!DurabilityMode::None.syncs_commit_metadata());
|
||||
assert!(!DurabilityMode::None.syncs_metadata_files());
|
||||
assert!(!DurabilityMode::None.syncs_commit_directories());
|
||||
assert!(!DurabilityMode::None.syncs_recovery_protocol_state());
|
||||
assert!(!DurabilityMode::LegacyOff.syncs_data_shards());
|
||||
assert!(!DurabilityMode::LegacyOff.syncs_commit_metadata());
|
||||
assert!(!DurabilityMode::LegacyOff.syncs_metadata_files());
|
||||
assert!(!DurabilityMode::LegacyOff.syncs_commit_directories());
|
||||
assert!(!DurabilityMode::LegacyOff.syncs_recovery_protocol_state());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -24492,6 +24581,11 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_effective_durability_pins_system_volumes() {
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::FileSync);
|
||||
assert_eq!(effective_durability("user-bucket"), DurabilityMode::FileSync);
|
||||
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
|
||||
}
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||
assert_eq!(effective_durability("user-bucket"), DurabilityMode::Relaxed);
|
||||
@@ -24532,6 +24626,14 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_effective_durability_bucket_override() {
|
||||
// The file-sync bucket tier can lower ordinary namespace durability
|
||||
// without changing the process-wide strict default.
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-override-file-sync", DurabilityMode::FileSync);
|
||||
assert_eq!(effective_durability("hp5b-override-file-sync"), DurabilityMode::FileSync);
|
||||
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Strict);
|
||||
}
|
||||
// Global strict + per-bucket relaxed: only the named bucket drops.
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
@@ -24691,6 +24793,33 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_meta_file_sync_writes_file_but_skips_dst_parent() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::FileSync);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "file-sync-meta-bucket";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let meta_path = format!("dir/object/{STORAGE_FORMAT_FILE}");
|
||||
let dst_file_path = disk.get_object_path(bucket, &meta_path).expect("dst path should resolve");
|
||||
let dst_parent = dst_file_path.parent().expect("dst file should have a parent").to_path_buf();
|
||||
disk.write_all_meta(bucket, &meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("file-sync metadata write should succeed");
|
||||
|
||||
assert_eq!(tokio::fs::read(&dst_file_path).await.expect("xl.meta should exist"), b"payload");
|
||||
assert!(
|
||||
!os::fsync_dir_recorder::was_fsynced(&dst_parent),
|
||||
"file-sync must omit the ordinary destination-parent barrier"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_public_relaxed_pins_system_volume() {
|
||||
@@ -25074,11 +25203,11 @@ mod test {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_rename_data_writes_old_metadata_backup_for_inline_overwrite() {
|
||||
async fn test_rename_data_file_sync_keeps_recovery_backup_barrier() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let _mode = durability_mode_override::set(DurabilityMode::FileSync);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
@@ -25141,7 +25270,7 @@ mod test {
|
||||
assert!(backup_path.exists());
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(backup_path.parent().expect("backup must have a parent")),
|
||||
"strict inline overwrite must persist the rollback backup directory entry"
|
||||
"file-sync inline overwrite must persist the rollback backup directory entry"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(backup_path.parent().expect("backup must have a parent")),
|
||||
@@ -25150,12 +25279,12 @@ mod test {
|
||||
);
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&dst_object_dir),
|
||||
"strict inline overwrite must persist the committed xl.meta directory entry"
|
||||
"file-sync overwrite must retain the recovery backup publication barrier"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&dst_object_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline commit fsyncs should use the disk file-sync limit"
|
||||
"only Unix recovery backup fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
// The rollback backup must contain the previous metadata bytes verbatim so
|
||||
// that undo_write can restore the prior committed object; guards the inline
|
||||
|
||||
@@ -845,17 +845,20 @@ impl LocalDisk {
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
}
|
||||
let sync_data_shards = no_inline && durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let file_sync_admission = if sync_data_shards || sync_commit_metadata {
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let sync_metadata_files = durability.syncs_metadata_files();
|
||||
let sync_commit_directories = durability.syncs_commit_directories();
|
||||
let sync_recovery_protocol = durability.syncs_recovery_protocol_state();
|
||||
let file_sync_admission =
|
||||
if sync_data_shards || sync_metadata_files || sync_commit_directories || sync_recovery_protocol {
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
@@ -966,7 +969,7 @@ impl LocalDisk {
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
sync_recovery_protocol,
|
||||
) {
|
||||
Ok(path) => path,
|
||||
Err(err) => {
|
||||
@@ -995,7 +998,7 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if sync_metadata_files && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -1014,7 +1017,7 @@ impl LocalDisk {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
} else if sync_metadata_files && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
@@ -1105,8 +1108,8 @@ impl LocalDisk {
|
||||
if let Err(err) = backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
dst_buf.as_ref(),
|
||||
sync_commit_metadata,
|
||||
sync_commit_metadata,
|
||||
sync_recovery_protocol,
|
||||
sync_recovery_protocol,
|
||||
) {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
|
||||
}
|
||||
@@ -1125,7 +1128,7 @@ impl LocalDisk {
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
None,
|
||||
sync_commit_metadata,
|
||||
sync_recovery_protocol,
|
||||
) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
@@ -1178,7 +1181,7 @@ impl LocalDisk {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata {
|
||||
if sync_commit_directories {
|
||||
if staged_rollback_path.is_none()
|
||||
&& rollback_data_dir.is_none()
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
@@ -1320,7 +1323,7 @@ impl LocalDisk {
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let sync_commit_directories = durability.syncs_commit_directories();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
@@ -1472,7 +1475,7 @@ impl LocalDisk {
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if sync_commit_directories {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
@@ -1505,7 +1508,7 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(
|
||||
sync_commit_metadata,
|
||||
durability.fully_syncs_commit_target(),
|
||||
!cfg!(test),
|
||||
staging_cleanup,
|
||||
Some(&directory_guard_owned),
|
||||
@@ -1624,7 +1627,7 @@ impl LocalDisk {
|
||||
.await?;
|
||||
// This is system metadata authority, so use its existing strict
|
||||
// durability policy even when the object payload is relaxed.
|
||||
if effective_durability(src_volume).syncs_commit_metadata() {
|
||||
if effective_durability(src_volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(recovery.parent().ok_or(DiskError::InvalidPath)?, Some(authority_owner.clone()))
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -1731,7 +1734,7 @@ impl LocalDisk {
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
} else {
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() {
|
||||
let tmp_meta_sync = if durability.syncs_metadata_files() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
@@ -1878,11 +1881,7 @@ impl LocalDisk {
|
||||
// it keeps SyncMode::FileAndDir: contents and directory entry both
|
||||
// durable. It is part of the metadata commit machinery, so relaxed
|
||||
// tiers leave it to the page cache like the xl.meta it mirrors.
|
||||
let backup_sync = if durability.syncs_commit_metadata() {
|
||||
SyncMode::FileAndDir
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
let backup_sync = durability.recovery_sync_mode();
|
||||
if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref())
|
||||
&& multipart_publication.as_ref().is_none_or(|publication| publication.legacy)
|
||||
{
|
||||
@@ -1999,7 +1998,7 @@ impl LocalDisk {
|
||||
// Persist the directory entries for both the data dir and xl.meta renames;
|
||||
// without this the commit itself can vanish on power loss. Relaxed tiers
|
||||
// accept that window (documented in docs/operations/durability-modes.md).
|
||||
if durability.syncs_commit_metadata()
|
||||
if durability.syncs_commit_directories()
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
@@ -2039,7 +2038,7 @@ impl LocalDisk {
|
||||
// are durable. Overwrites already have a durable object dir. The
|
||||
// starts_with guard bounds the walk to the bucket subtree. Relaxed/none
|
||||
// accept the wider window, like the commit fsync above.
|
||||
if has_dst_buf.is_none() && durability.syncs_commit_metadata() {
|
||||
if has_dst_buf.is_none() && durability.syncs_commit_directories() {
|
||||
let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir) {
|
||||
@@ -2106,7 +2105,7 @@ impl LocalDisk {
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let target_already_synced = durability.fully_syncs_commit_target();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
@@ -2152,7 +2151,7 @@ impl LocalDisk {
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
let sync = durability.syncs_recovery_protocol_state();
|
||||
#[cfg(unix)]
|
||||
let receipt_identity = if sync
|
||||
&& pool_meta_condition
|
||||
|
||||
@@ -1867,7 +1867,7 @@ impl LocalDisk {
|
||||
let destination = root.join(transaction.to_string());
|
||||
let destination_was_absent = original.is_none();
|
||||
let original = original.map(<[u8]>::to_vec);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
#[cfg(test)]
|
||||
let record_destination = destination.clone();
|
||||
let staging_root = temporary;
|
||||
@@ -2344,7 +2344,7 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&dir, owner.clone()).await.map_err(to_file_error)?;
|
||||
}
|
||||
if self
|
||||
@@ -2372,7 +2372,9 @@ impl LocalDisk {
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() && fs::try_exists(&rollback).await.map_err(to_file_error)? {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state()
|
||||
&& fs::try_exists(&rollback).await.map_err(to_file_error)?
|
||||
{
|
||||
os::fsync_dir_with_owner(&rollback, owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -2456,7 +2458,7 @@ impl LocalDisk {
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&root, owner.clone()).await.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
@@ -2467,7 +2469,7 @@ impl LocalDisk {
|
||||
os::remove_files_and_fsync_with_owner(
|
||||
&rollback,
|
||||
&[STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE],
|
||||
effective_durability(volume).syncs_commit_metadata(),
|
||||
effective_durability(volume).syncs_recovery_protocol_state(),
|
||||
None,
|
||||
owner.clone(),
|
||||
)
|
||||
@@ -2500,7 +2502,7 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
if effective_durability(volume).syncs_recovery_protocol_state() {
|
||||
os::fsync_dir_with_owner(&staged, owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -2581,7 +2583,7 @@ impl LocalDisk {
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&dir, owner.clone()).await;
|
||||
prune_dirs.push((dir, lease));
|
||||
}
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let (rollback_buf, root_buf, volume_str, object_str) =
|
||||
(rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
|
||||
let publication_root = self.publication_root.clone();
|
||||
@@ -2802,7 +2804,7 @@ impl LocalDisk {
|
||||
} else {
|
||||
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?
|
||||
};
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let incarnation = opts.delete_recovery_bucket_incarnation;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
let fi_clone = fi.clone();
|
||||
@@ -3235,7 +3237,7 @@ impl LocalDisk {
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let volume_str = volume.to_owned();
|
||||
let object_str = object.to_owned();
|
||||
@@ -3478,7 +3480,7 @@ impl LocalDisk {
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let sync = effective_durability(volume).syncs_recovery_protocol_state();
|
||||
|
||||
let mut claims: Vec<SnapshotLeaseDeleteClaim> = Vec::with_capacity(journal.data_dirs.len());
|
||||
for data_dir in &journal.data_dirs {
|
||||
|
||||
@@ -1578,7 +1578,7 @@ impl LocalDisk {
|
||||
let object_path = self.io_get_object_path(&volume, &object)?;
|
||||
let directory = object_path.join(proof.publication.final_data_dir.to_string());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&directory, Some(metadata_owner)).await;
|
||||
let sync = super::effective_durability(&volume).syncs_commit_metadata();
|
||||
let sync = super::effective_durability(&volume).syncs_recovery_protocol_state();
|
||||
let result = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
let result = (|| {
|
||||
let base = &proof.base;
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# Durability modes (drive sync tiers)
|
||||
|
||||
**Use this when:** choosing or debugging the fsync tier (`strict|relaxed|none|legacy-off`) for a deployment or a single bucket, or changing any write-path sync behaviour.
|
||||
**Use this when:** choosing or debugging the fsync tier (`strict|file-sync|relaxed|none|legacy-off`) for a deployment or a single bucket, or changing any write-path sync behaviour.
|
||||
**Source of truth:** `crates/ecstore/src/disk/local.rs` (`ENV_RUSTFS_DURABILITY_MODE`, `ENV_RUSTFS_DRIVE_SYNC_ENABLE`, mode resolution and per-write-point sync decisions), `crates/ecstore/src/bucket/durability.rs` (`ENV_NEW_BUCKET_DURABILITY_MODE`, per-bucket override), `rustfs/src/admin/handlers/durability.rs` (admin API), `crates/ecstore/src/bucket/metadata_sys.rs` (`BUCKET_METADATA_REFRESH_INTERVAL`).
|
||||
|
||||
RustFS lets operators choose how much fsync work runs on the object write
|
||||
path. The default (`strict`) preserves the fully synced behavior RustFS has
|
||||
always shipped. `relaxed` and `none` are **opt-in** trades of power-loss
|
||||
always shipped. `file-sync`, `relaxed`, and `none` are **opt-in** trades of power-loss
|
||||
durability for latency/IOPS. Newly created buckets inherit the process-wide
|
||||
mode unless `RUSTFS_NEW_BUCKET_DURABILITY_MODE` explicitly sets an override.
|
||||
|
||||
@@ -13,13 +13,13 @@ mode unless `RUSTFS_NEW_BUCKET_DURABILITY_MODE` explicitly sets an override.
|
||||
|
||||
```bash
|
||||
# New tiered switch (wins when set to a valid value)
|
||||
RUSTFS_DURABILITY_MODE=strict|relaxed|none # default: strict
|
||||
RUSTFS_DURABILITY_MODE=strict|file-sync|relaxed|none # default: strict
|
||||
|
||||
# Legacy binary switch (kept for compatibility, superseded by the above)
|
||||
RUSTFS_DRIVE_SYNC_ENABLE=true|false # default: true
|
||||
|
||||
# Tier seeded into a NEWLY CREATED bucket's own override (see "New-bucket default")
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=relaxed|strict|none|inherit # default: inherit process mode
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=file-sync|relaxed|strict|none|inherit # default: inherit process mode
|
||||
```
|
||||
|
||||
Resolution rules:
|
||||
@@ -29,7 +29,7 @@ Resolution rules:
|
||||
| unset | unset | `strict` (default) |
|
||||
| unset | `true` | `strict` |
|
||||
| unset | `false` | `legacy-off` (the historical "everything off" semantics) |
|
||||
| `strict` / `relaxed` / `none` | anything | the named mode |
|
||||
| `strict` / `file-sync` / `relaxed` / `none` | anything | the named mode |
|
||||
| invalid value | any | logged warning, falls back to the legacy switch, then the default |
|
||||
|
||||
Values are case-insensitive and whitespace-tolerant. The mode is resolved
|
||||
@@ -43,19 +43,21 @@ through `RUSTFS_DRIVE_SYNC_ENABLE=false` so that existing deployments keep
|
||||
their exact current behavior. It is deprecated and will be retained for at
|
||||
least one major version.
|
||||
|
||||
Do not set the process-wide `file-sync` value until every node runs a version that recognizes it. An older node treats the value as invalid and falls back to its legacy switch. Prefer the fail-closed bucket override during a rolling upgrade, and keep `RUSTFS_NEW_BUCKET_DURABILITY_MODE=inherit` until the fleet upgrade is complete.
|
||||
|
||||
## What each mode fsyncs
|
||||
|
||||
Write points on the object path and how each mode treats them:
|
||||
|
||||
| Write point | `strict` | `relaxed` | `none` | `legacy-off` |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Erasure shard files (fdatasync before the commit rename) | yes | **yes** | no | no |
|
||||
| Multipart part payload (fdatasync before `rename_part` commit) | yes | **yes** | no | no |
|
||||
| xl.meta contents (tmp write before the commit rename) | yes | no | no | no |
|
||||
| Inline objects (data embedded in xl.meta) | yes | no | no | no |
|
||||
| Old-metadata rollback backups | yes | no | no | no |
|
||||
| Directory entries of commit renames (fsync of the parent dir) | yes | no | no | no |
|
||||
| System-critical writes (see pinning below) | yes | yes (pinned) | yes (pinned) | **no** |
|
||||
| Write point | `strict` | `file-sync` | `relaxed` | `none` | `legacy-off` |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Erasure shard files (fdatasync before the commit rename) | yes | **yes** | **yes** | no | no |
|
||||
| Multipart part payload (fdatasync before `rename_part` commit) | yes | **yes** | **yes** | no | no |
|
||||
| xl.meta contents (tmp write before the commit rename) | yes | **yes** | no | no | no |
|
||||
| Inline objects (data embedded in xl.meta) | yes | **yes** | no | no | no |
|
||||
| Recovery journals, witnesses, and rollback state | yes | **yes** | no | no | no |
|
||||
| Ordinary commit and new ancestor directory entries | yes | no | no | no | no |
|
||||
| System-critical writes (see pinning below) | yes | yes (pinned) | yes (pinned) | yes (pinned) | **no** |
|
||||
|
||||
## Power-loss guarantees, honestly stated
|
||||
|
||||
@@ -66,6 +68,8 @@ and the directory entries of the commit renames are all fsynced. A whole-node
|
||||
(or whole-cluster) power failure does not lose acknowledged data. This is the
|
||||
current mainline behavior, unchanged.
|
||||
|
||||
**`file-sync`.** Payload files and final xl.meta file contents are synchronized before acknowledgement, including inline object data, while ordinary commit rename and newly created ancestor directory entries are not individually fsynced. Recovery journals, multipart transaction witnesses, rollback state, and system-critical namespaces retain strict barriers. After sudden power loss, synchronized inode contents are protected, but a recently acknowledged fresh object name can disappear and an overwrite can revert to the previous namespace state. This matches MinIO's default object-path boundary more closely than `relaxed`, but it does not provide the complete per-drive acknowledgement guarantee of `strict`.
|
||||
|
||||
**`relaxed`.** Payload bytes of non-inline objects and multipart parts are
|
||||
fdatasynced to the device before the acknowledgement, but the metadata
|
||||
commits — xl.meta contents, rollback backups, and the directory entries of
|
||||
@@ -77,10 +81,7 @@ failure:
|
||||
xl.meta bytes nor the rename's directory entry are synced, the commit
|
||||
itself can vanish; surviving shard bytes become unreferenced orphans on
|
||||
that drive.
|
||||
- **Inline (small) objects receive no per-object fsync at all** in this mode:
|
||||
their data lives inside xl.meta, and xl.meta is not synced. This matches
|
||||
MinIO's default posture (no per-object fsync) but means small objects have
|
||||
the widest loss window.
|
||||
- **Inline (small) objects receive no per-object fsync at all** in this mode: their data lives inside xl.meta, and xl.meta is not synced. This is weaker than MinIO's default object path, which synchronizes final xl.meta file contents while ordinarily omitting a per-object parent-directory fsync. Small objects therefore have the widest loss window under RustFS relaxed.
|
||||
- Durability of acknowledged writes therefore rests on **erasure-coded
|
||||
redundancy across other nodes** plus the unclean-shutdown heal converging
|
||||
the affected drive afterwards.
|
||||
@@ -147,7 +148,7 @@ points, so no restart is needed.
|
||||
```bash
|
||||
# Set an override (admin credentials, ConfigUpdateAdminAction)
|
||||
curl -X PUT "http://<host>/rustfs/admin/v3/bucket-durability/<bucket>" \
|
||||
-d '{"mode":"relaxed"}' # strict | relaxed | none
|
||||
-d '{"mode":"file-sync"}' # strict | file-sync | relaxed | none
|
||||
|
||||
# Read it back ("mode": null means the bucket inherits the global mode)
|
||||
curl "http://<host>/rustfs/admin/v3/bucket-durability/<bucket>"
|
||||
@@ -163,8 +164,8 @@ configuration plane.
|
||||
|
||||
A newly created bucket inherits the process-wide mode by default. On an
|
||||
otherwise unconfigured deployment this means `strict`. Operators can explicitly
|
||||
set `RUSTFS_NEW_BUCKET_DURABILITY_MODE=relaxed` to seed a relaxed override into
|
||||
new buckets, or choose `strict`, `none`, or `inherit`. A non-strict override is
|
||||
set `RUSTFS_NEW_BUCKET_DURABILITY_MODE=file-sync` to seed a file-sync override into
|
||||
new buckets, or choose `strict`, `relaxed`, `none`, or `inherit`. A non-strict override is
|
||||
logged when the bucket is created; an explicitly non-strict process-wide mode
|
||||
is warned at startup. Relaxed remains appropriate only for the multi-node,
|
||||
independent-power-domain deployment described above.
|
||||
@@ -184,6 +185,7 @@ The seeded tier is controlled by an env var read once per bucket creation
|
||||
|
||||
```bash
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=relaxed # explicitly seed `relaxed`
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=file-sync # explicitly seed `file-sync`
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=strict # seed `strict` instead
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=none # seed `none` instead
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=inherit # seed nothing: follow the global mode
|
||||
@@ -238,6 +240,8 @@ Until a peer has reloaded the metadata, its writes use the previous tier.
|
||||
This is the same eventual-consistency window every other bucket config
|
||||
(policy, quota, versioning) already has.
|
||||
|
||||
`file-sync` overrides use a backward-compatible fail-closed encoding: the stored JSON carries `mode: strict` plus a `file_sync` marker. Nodes that understand the marker apply `file-sync`; older nodes ignore the marker and apply the stronger `strict` mode. Roll back by setting the bucket to `strict` or clearing the override before downgrading if inheriting the process mode is intended.
|
||||
|
||||
### Power-loss guarantees
|
||||
|
||||
Identical to the corresponding global tier, scoped to the bucket. In
|
||||
@@ -248,10 +252,7 @@ bucket on power failure.
|
||||
|
||||
## Performance expectations
|
||||
|
||||
Measure `relaxed` on the target platform (Linux ext4/xfs) before relying on a
|
||||
number. Throughput deltas measured with sync fully **off** (`none` or
|
||||
`legacy-off`) do not transfer: `relaxed` keeps the per-shard fdatasync, so its
|
||||
gain is necessarily smaller. Do not use `none` numbers to size `relaxed`.
|
||||
Measure `file-sync` and `relaxed` independently on the target platform (Linux ext4/xfs) before relying on a number. Throughput deltas measured with sync fully **off** (`none` or `legacy-off`) do not transfer: `file-sync` keeps payload and metadata-file synchronization, while `relaxed` keeps payload synchronization. Do not use `none` numbers to size either tier.
|
||||
|
||||
## Related Recovery Guides
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Per-bucket durability tier admin handlers (HP-5 phase 2, rustfs/backlog#938).
|
||||
//!
|
||||
//! A bucket can override the process-wide durability mode
|
||||
//! (`RUSTFS_DURABILITY_MODE`) with its own `strict` | `relaxed` | `none`
|
||||
//! (`RUSTFS_DURABILITY_MODE`) with its own `strict` | `file-sync` | `relaxed` | `none`
|
||||
//! tier. The override is stored in the bucket metadata (`durability.json`
|
||||
//! entry) and consumed by the disk layer at commit points. System buckets
|
||||
//! can never carry an override. See docs/operations/durability-modes.md.
|
||||
@@ -94,7 +94,7 @@ fn normalize_mode(mode: &str) -> Result<String, rustfs_s3_types::S3Error> {
|
||||
} else {
|
||||
Err(s3_error!(
|
||||
InvalidArgument,
|
||||
"invalid durability mode {:?}: expected strict|relaxed|none",
|
||||
"invalid durability mode {:?}: expected strict|file-sync|relaxed|none",
|
||||
mode
|
||||
))
|
||||
}
|
||||
@@ -262,6 +262,7 @@ mod tests {
|
||||
#[test]
|
||||
fn normalize_mode_accepts_tiers_and_rejects_everything_else() {
|
||||
assert_eq!(normalize_mode("strict").unwrap(), "strict");
|
||||
assert_eq!(normalize_mode(" FILE-SYNC ").unwrap(), "file-sync");
|
||||
assert_eq!(normalize_mode(" RELAXED ").unwrap(), "relaxed");
|
||||
assert_eq!(normalize_mode("none").unwrap(), "none");
|
||||
assert!(normalize_mode("").is_err());
|
||||
|
||||
+3
-2
@@ -130,8 +130,9 @@ their issue closes.
|
||||
| `hotpath_warp_ab_gate.sh` | dev-tool | Relative-budget gate evaluated over the warp A/B results | used by `run_hotpath_warp_ab.sh`; hotpath runbook |
|
||||
| `run_internode_grpc_ab_bench.sh` | dev-tool | One-click A/B driver for the internode gRPC optimization stages | `docs/operations/internode-grpc-benchmark-runbook.md` |
|
||||
| `run_internode_transport_baseline.sh` | dev-tool | Internode transport baseline runner | internode runbook; `crates/io-metrics/README.md` |
|
||||
| `run_four_node_cluster_failover_bench.sh` | dev-tool | Four-node cluster failover benchmark | `docker/compose/README.md`; internode runbook |
|
||||
| `run_object_batch_bench.sh` | dev-tool | Batch object benchmark runner (warp/s3bench) | internode + scanner runbooks |
|
||||
| `run_four_node_cluster_failover_bench.sh` | dev-tool | Four-node cluster failover benchmark with optional server-only image build | `docker/compose/README.md`; internode runbook |
|
||||
| `run_object_batch_bench.sh` | dev-tool | Batch object benchmark runner with PUT-specific Warp summary extraction | internode + scanner runbooks |
|
||||
| `test_object_batch_bench.sh` | dev-tool | Regression test for Warp PUT summary extraction | `scripts/test_object_batch_bench.sh` |
|
||||
| `run_object_batch_bench_enhanced.sh` | dev-tool | Enhanced batch benchmark runner; hub used by the smoke rigs | hotpath runbook |
|
||||
| `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` |
|
||||
| `prepare_conditional_read_baseline.py` | dev-tool | Installs only the matching conditional-read benchmark harness in a clean detached baseline | `docs/testing/conditional-read-benchmark.md` |
|
||||
|
||||
@@ -18,6 +18,7 @@ COMPOSE_UP_NO_BUILD="${COMPOSE_UP_NO_BUILD:-false}"
|
||||
RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin-local}"
|
||||
RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfssecret-local}"
|
||||
RUSTFS_DOCKER_PLATFORM="${RUSTFS_DOCKER_PLATFORM:-}"
|
||||
RUSTFS_RC_RELEASE="${RUSTFS_RC_RELEASE:-latest}"
|
||||
RUSTFS_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}"
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK="${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}"
|
||||
|
||||
@@ -34,6 +35,7 @@ BENCH_WARP_HOSTS="${BENCH_WARP_HOSTS:-http://127.0.0.1:9000,http://127.0.0.1:900
|
||||
BENCH_BUCKET="${BENCH_BUCKET:-rustfs-four-node-bench}"
|
||||
BENCH_AUTO_NEW_BUCKET="${BENCH_AUTO_NEW_BUCKET:-true}"
|
||||
BENCH_BUCKET_PREFIX="${BENCH_BUCKET_PREFIX:-rustfs-four-node-bench}"
|
||||
BENCH_WARP_MODE="${BENCH_WARP_MODE:-mixed}"
|
||||
BENCH_CONCURRENCY="${BENCH_CONCURRENCY:-}"
|
||||
BENCH_CONCURRENCIES="${BENCH_CONCURRENCIES:-}"
|
||||
BENCH_DURATION="${BENCH_DURATION:-60s}"
|
||||
@@ -61,6 +63,7 @@ Options:
|
||||
--bench-endpoint <url> benchmark endpoint (default: http://127.0.0.1:9000)
|
||||
--bench-warp-hosts <hosts> comma-separated warp hosts (default: node1..node4)
|
||||
--bench-sizes <sizes> comma list (default: 1KiB,4KiB,11Mi)
|
||||
--bench-warp-mode <mode> warp mode: mixed|get|put (default: mixed)
|
||||
--bench-concurrency <n> benchmark concurrency
|
||||
--bench-concurrencies <list> benchmark concurrency list (default: 8,16,32,64,128)
|
||||
--bench-duration <dur> benchmark duration
|
||||
@@ -73,11 +76,13 @@ Environment:
|
||||
WITH_OBSERVABILITY BUILD_LOCAL_IMAGE RUN_FAILOVER RUN_BENCHMARK KEEP_UP
|
||||
COMPOSE_UP_NO_BUILD (true|false, default: false)
|
||||
RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY RUSTFS_OBS_ENDPOINT
|
||||
RUSTFS_RC_RELEASE (default: latest; disabled creates a server-only validation image)
|
||||
PRECHECK_AUTO_CLEANUP (true|false, default: true)
|
||||
WAIT_PROBE_MODE (service|ready, default: service)
|
||||
WAIT_TIMEOUT_SECS FAILOVER_NODE FAILOVER_WARMUP_SECS FAILOVER_SAMPLE_SECS
|
||||
FAILOVER_INTERVAL_SECS BENCH_ENDPOINT BENCH_BUCKET BENCH_CONCURRENCY
|
||||
BENCH_WARP_HOSTS (default: http://127.0.0.1:9000,http://127.0.0.1:9001,http://127.0.0.1:9002,http://127.0.0.1:9003)
|
||||
BENCH_WARP_MODE (mixed|get|put, default: mixed)
|
||||
BENCH_CONCURRENCIES BENCH_DURATION BENCH_SIZES OUT_DIR
|
||||
BENCH_WAIT_MODE (ready|service, default: ready)
|
||||
BENCH_READY_TIMEOUT_SECS (default: 180)
|
||||
@@ -593,6 +598,7 @@ run_benchmark() {
|
||||
--secret-key "${RUSTFS_SECRET_KEY}" \
|
||||
--bucket "${bench_bucket}" \
|
||||
--bucket-prefix "${BENCH_BUCKET_PREFIX}" \
|
||||
--warp-mode "${BENCH_WARP_MODE}" \
|
||||
"${bench_extra_args[@]}" \
|
||||
--concurrency "${conc}" \
|
||||
--duration "${BENCH_DURATION}" \
|
||||
@@ -675,6 +681,10 @@ parse_args() {
|
||||
BENCH_SIZES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-warp-mode)
|
||||
BENCH_WARP_MODE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-concurrency)
|
||||
BENCH_CONCURRENCY="$2"
|
||||
BENCH_CONCURRENCIES="$2"
|
||||
@@ -758,9 +768,9 @@ main() {
|
||||
log_info "Building local image from Dockerfile.source: ${IMAGE_TAG}"
|
||||
if [[ -n "${RUSTFS_DOCKER_PLATFORM}" ]]; then
|
||||
log_info "Using docker build platform: ${RUSTFS_DOCKER_PLATFORM}"
|
||||
docker build --platform "${RUSTFS_DOCKER_PLATFORM}" -f "${PROJECT_ROOT}/Dockerfile.source" -t "${IMAGE_TAG}" "${PROJECT_ROOT}"
|
||||
docker build --platform "${RUSTFS_DOCKER_PLATFORM}" --build-arg "RC_RELEASE=${RUSTFS_RC_RELEASE}" -f "${PROJECT_ROOT}/Dockerfile.source" -t "${IMAGE_TAG}" "${PROJECT_ROOT}"
|
||||
else
|
||||
docker build -f "${PROJECT_ROOT}/Dockerfile.source" -t "${IMAGE_TAG}" "${PROJECT_ROOT}"
|
||||
docker build --build-arg "RC_RELEASE=${RUSTFS_RC_RELEASE}" -f "${PROJECT_ROOT}/Dockerfile.source" -t "${IMAGE_TAG}" "${PROJECT_ROOT}"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping image build"
|
||||
|
||||
@@ -188,6 +188,13 @@ validate_args() {
|
||||
fi
|
||||
fi
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
case "$WARP_MODE" in
|
||||
mixed|get|put) ;;
|
||||
*)
|
||||
echo "ERROR: --warp-mode must be mixed, get, or put" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$ENDPOINT")"
|
||||
if [[ -z "$warp_host" ]]; then
|
||||
@@ -224,9 +231,35 @@ extract_value() {
|
||||
collect_metrics() {
|
||||
local log_file="$1"
|
||||
local throughput reqps latency
|
||||
throughput="$(extract_value '([0-9]+(\\.[0-9]+)?\\s*(GiB/s|MiB/s|MB/s|KB/s))' "$log_file")"
|
||||
reqps="$(extract_value '([0-9]+(\\.[0-9]+)?\\s*(req/s|ops/s|requests/s))' "$log_file")"
|
||||
latency="$(extract_value '([0-9]+(\\.[0-9]+)?\\s*(ms|us|µs|s))(\\s*(avg|mean))?' "$log_file")"
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
awk '
|
||||
/^Report: PUT[.]/ { in_put = 1; next }
|
||||
in_put && /^Report:/ { in_put = 0 }
|
||||
in_put && /^[[:space:]]*\* Average:/ {
|
||||
line = $0
|
||||
sub(/^[[:space:]]*\* Average:[[:space:]]*/, "", line)
|
||||
split(line, values, /,[[:space:]]*/)
|
||||
throughput = values[1]
|
||||
reqps = values[2]
|
||||
}
|
||||
in_put && /^[[:space:]]*\* Reqs: Avg:/ {
|
||||
line = $0
|
||||
sub(/^[[:space:]]*\* Reqs: Avg:[[:space:]]*/, "", line)
|
||||
split(line, values, /,[[:space:]]*/)
|
||||
latency = values[1]
|
||||
}
|
||||
END {
|
||||
throughput = throughput == "" ? "N/A" : throughput
|
||||
reqps = reqps == "" ? "N/A" : reqps
|
||||
latency = latency == "" ? "N/A" : latency
|
||||
printf "%s,%s,%s\n", throughput, reqps, latency
|
||||
}
|
||||
' "$log_file"
|
||||
return
|
||||
fi
|
||||
throughput="$(extract_value '([0-9]+(\.[0-9]+)?\s*(GiB/s|MiB/s|MB/s|KB/s))' "$log_file")"
|
||||
reqps="$(extract_value '([0-9]+(\.[0-9]+)?\s*(req/s|ops/s|requests/s))' "$log_file")"
|
||||
latency="$(extract_value '([0-9]+(\.[0-9]+)?\s*(ms|us|µs|s))(\s*(avg|mean))?' "$log_file")"
|
||||
echo "${throughput:-N/A},${reqps:-N/A},${latency:-N/A}"
|
||||
}
|
||||
|
||||
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${TMP_DIR}"' EXIT
|
||||
|
||||
FAKE_WARP="${TMP_DIR}/warp"
|
||||
cat >"${FAKE_WARP}" <<'WARP'
|
||||
#!/usr/bin/env bash
|
||||
cat <<'REPORT'
|
||||
Report: DELETE. Concurrency: 16. Ran: 12s
|
||||
* Average: 34.41 obj/s
|
||||
* Reqs: Avg: 338.1ms, 99%: 873.4ms
|
||||
|
||||
Report: GET. Concurrency: 16. Ran: 12s
|
||||
* Average: 15.02 MiB/s, 153.76 obj/s
|
||||
* Reqs: Avg: 2.8ms, 99%: 7.7ms
|
||||
|
||||
Report: PUT. Concurrency: 16. Ran: 12s
|
||||
* Average: 5.08 MiB/s, 52.00 obj/s
|
||||
* Reqs: Avg: 93.9ms, 99%: 492.7ms
|
||||
|
||||
Report: STAT. Concurrency: 16. Ran: 12s
|
||||
* Average: 102.88 obj/s
|
||||
* Reqs: Avg: 2.6ms, 99%: 8.4ms
|
||||
REPORT
|
||||
WARP
|
||||
chmod +x "${FAKE_WARP}"
|
||||
|
||||
"${SCRIPT_DIR}/run_object_batch_bench.sh" \
|
||||
--tool warp \
|
||||
--warp-bin "${FAKE_WARP}" \
|
||||
--endpoint http://127.0.0.1:9000 \
|
||||
--access-key test \
|
||||
--secret-key test \
|
||||
--sizes 100KiB \
|
||||
--concurrency 16 \
|
||||
--duration 1s \
|
||||
--out-dir "${TMP_DIR}/out" >/dev/null
|
||||
|
||||
expected="100KiB,warp,16,ok,5.08 MiB/s,52.00 obj/s,93.9ms,0,${TMP_DIR}/out/warp_100KiB.log"
|
||||
actual="$(tail -n1 "${TMP_DIR}/out/summary.csv")"
|
||||
if [[ "${actual}" != "${expected}" ]]; then
|
||||
echo "unexpected warp PUT summary" >&2
|
||||
echo "expected: ${expected}" >&2
|
||||
echo "actual: ${actual}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat >"${FAKE_WARP}" <<'WARP'
|
||||
#!/usr/bin/env bash
|
||||
cat <<'REPORT'
|
||||
Report: GET. Concurrency: 16. Ran: 12s
|
||||
* Average: 15.02 MiB/s, 153.76 obj/s
|
||||
* Reqs: Avg: 2.8ms, 99%: 7.7ms
|
||||
REPORT
|
||||
WARP
|
||||
chmod +x "${FAKE_WARP}"
|
||||
|
||||
"${SCRIPT_DIR}/run_object_batch_bench.sh" \
|
||||
--tool warp \
|
||||
--warp-bin "${FAKE_WARP}" \
|
||||
--endpoint http://127.0.0.1:9000 \
|
||||
--access-key test \
|
||||
--secret-key test \
|
||||
--sizes 100KiB \
|
||||
--concurrency 16 \
|
||||
--duration 1s \
|
||||
--out-dir "${TMP_DIR}/missing-put" >/dev/null
|
||||
|
||||
expected="100KiB,warp,16,ok,N/A,N/A,N/A,0,${TMP_DIR}/missing-put/warp_100KiB.log"
|
||||
actual="$(tail -n1 "${TMP_DIR}/missing-put/summary.csv")"
|
||||
if [[ "${actual}" != "${expected}" ]]; then
|
||||
echo "missing PUT report must not reuse GET metrics" >&2
|
||||
echo "expected: ${expected}" >&2
|
||||
echo "actual: ${actual}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if "${SCRIPT_DIR}/run_object_batch_bench.sh" \
|
||||
--tool warp \
|
||||
--warp-bin "${FAKE_WARP}" \
|
||||
--warp-mode invalid \
|
||||
--endpoint http://127.0.0.1:9000 \
|
||||
--access-key test \
|
||||
--secret-key test \
|
||||
--sizes 100KiB \
|
||||
--out-dir "${TMP_DIR}/invalid-mode" >/dev/null 2>&1; then
|
||||
echo "invalid warp mode must be rejected" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "object batch benchmark tests passed"
|
||||
Reference in New Issue
Block a user