Compare commits

..

2 Commits

Author SHA1 Message Date
唐小鸭 cddc003c16 fix(admin): emit madmin bandwidthlimit tag in list-remote-targets
madmin-go v3.0.109 `BucketTarget` (bucket-targets.go) tags the limit
`json:"bandwidthlimit,omitempty"`, not `bandwidth`, so the previous rename
still left mc decoding a zero bandwidth limit for every nonzero target.
Emit the exact tag in remote_target_admin_json, and add the same
`bandwidthlimit` alias on the ecstore `BucketTarget` reader so a
MinIO-written bucket-targets.json keeps its limit on migration (the legacy
`bandwidth` alias stays for compatibility). The persisted snake_case wire
key and the inbound RemoteTargetRequest aliases are unchanged.

Cover it with a madmin-shaped decode regression: the admin response is
decoded through a mirror of the v3.0.109 json tags and must yield the
nonzero bandwidth limit plus the other renamed fields.
2026-08-23 00:59:45 +08:00
唐小鸭 fbecd38e2d fix(admin): use madmin key names in list-remote-targets response
The admin list-remote-targets response serialized five fields under the
persisted snake_case keys (bandwidth_limit, storage_class, reset_id,
deployment_id, credentials.session_token), which madmin/mc decode to zero
values, blanking the bandwidth and reset-id columns of mc replicate ls.
Rename them to the madmin JSON tags (bandwidth, storageclass, resetID,
deploymentID, sessionToken) in remote_target_admin_json only, following
the existing Go-duration overwrite precedent there.

Symmetrically, BucketTarget deserialization only aliased bandwidth, so a
MinIO-written bucket-targets.json silently dropped the other four fields
to defaults on migration; add serde aliases for them. Aliases and the
response-layer rename leave the persisted bucket-targets.json and the
msgpack struct-map wire format untouched, pinned by new tests.

Refs rustfs/backlog#1951, spec rustfs/backlog#1946.
2026-08-21 18:53:48 +08:00
14 changed files with 242 additions and 1261 deletions
-5
View File
@@ -61,11 +61,6 @@ mod get_codec_streaming_compat_test;
#[cfg(test)]
mod version_id_regression_test;
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
// replication metadata must not overwrite a newer local category state.
#[cfg(test)]
mod replication_lww_receiver_test;
// Data usage regression tests
#[cfg(test)]
mod data_usage_test;
@@ -1,152 +0,0 @@
#![cfg(test)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
//! A4/P1-6).
//!
//! In an active-active topology both sites' metadata states arrive at the
//! peer as authorized replication PUTs carrying per-category source
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
//! family). Before the fix the receiver applied them unconditionally, so a
//! stale delivery overwrote a newer local state and the two sites diverged
//! permanently while both reported COMPLETED. This test drives one live
//! `rustfs` server with simulated inbound replication PUTs for the same
//! object version and asserts the newer tagging state wins regardless of
//! delivery order, while a stale delivery still succeeds at the object level
//! (a failure would loop through MRF re-delivering the stale value).
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use serial_test::serial;
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
const T_STALE: &str = "2026-01-01T00:00:01Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEWER: &str = "2026-03-01T00:00:00Z";
/// Simulated inbound authorized replication PUT: same object version, tags and
/// the source-authored tagging timestamp carried in transport headers.
async fn inbound_replication_put(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tags: &str,
tagging_timestamp: &str,
) -> TestResult {
let version_id = version_id.to_string();
let tagging_timestamp = tagging_timestamp.to_string();
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"lww-e2e-body"))
.tagging(tags)
.customize()
.mutate_request(move |req| {
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
req.headers_mut()
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
})
.send()
.await?;
Ok(())
}
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await
.expect("object tagging should be readable");
tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string())
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "replication-lww-receiver";
let key = "object";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
// First delivery establishes version V with tags stamped T_LOCAL.
let version_id = uuid::Uuid::new_v4().to_string();
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"the first delivery must establish the tagged version"
);
// A stale delivery (older source timestamp) must succeed at the object
// level but must NOT overwrite the newer tags.
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
);
// A newer delivery still converges the version onto the newest state.
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("newer"),
"a newer inbound delivery must overwrite older tags"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(&version_id)
.send()
.await?;
env.delete_test_bucket(bucket).await.ok();
Ok(())
}
+1 -1
View File
@@ -161,7 +161,7 @@ pub mod bucket {
pub mod objectlock_sys {
pub use crate::bucket::object_lock::objectlock_sys::{
BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
check_retention_for_modification, is_retention_active,
};
}
}
@@ -15,7 +15,7 @@
use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
use crate::bucket::object_lock::objectlock;
use crate::error::{Error, Result, StorageError};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::object_api::ObjectInfo;
use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
use std::sync::Arc;
@@ -136,41 +136,12 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
/// Check if an object has legal hold enabled.
/// Returns true if legal hold is ON.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
let lhold = objectlock::get_object_legalhold_meta(user_defined);
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
}
/// Whether an authorized replication write (`ObjectOptions::replication_request`)
/// may overwrite a locked destination version.
///
/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
/// skips the existing-version check for replicas), and a source-side hold
/// release or retention change reaches this site only through this write. The
/// overwrite is allowed only when the write carries the source timestamp of
/// every category that currently locks the version, so receiver-side LWW
/// (`merge_replication_metadata_lww`) judges each of them: a category locked
/// more recently here is kept, otherwise the source's newer state wins. A write
/// without that timestamp carries no source decision for the category — the
/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
pub fn replication_write_may_pass_worm_gate(
user_defined: &std::collections::HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
if !opts.replication_request {
return false;
}
if has_legal_hold(user_defined) && opts.replication_legalhold_timestamp.is_none() {
return false;
}
let ret = objectlock::get_object_retention_meta(user_defined);
let retention_locked = ret
.mode
.as_ref()
.is_some_and(|mode| is_retention_active(mode.as_str(), ret.retain_until_date.as_ref()));
!(retention_locked && opts.replication_retention_timestamp.is_none())
}
/// Check if an object is locked based on its metadata.
/// This is a common function used by both lifecycle evaluation and deletion checks.
///
@@ -268,12 +239,7 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
return Ok(None);
}
// A cleared retention / legal hold is persisted as empty strings (the
// MinIO on-disk shape, `parse_object_lock_retention`); read it as "no lock"
// rather than as corrupt metadata.
let persisted = |key: &str| obj_info.user_defined.get(key).filter(|value| !value.is_empty());
if let Some(status) = persisted(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
return Ok(Some(ObjectLockBlockReason::LegalHold));
}
@@ -282,8 +248,8 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
}
}
let mode = persisted(X_AMZ_OBJECT_LOCK_MODE.as_str());
let retain_until = persisted(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
let explicit_ret = match (mode, retain_until) {
(None, None) => None,
(Some(mode), Some(retain_until)) => {
@@ -520,98 +486,6 @@ mod tests {
}
}
/// A replication write passes the WORM gate only when it carries the
/// source timestamp of every category that currently locks the version.
#[test]
fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
let retention = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
];
let expired = [
(AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
];
let metadata = |entries: &[&[(&str, &str)]]| -> std::collections::HashMap<String, String> {
entries
.iter()
.flat_map(|entries| entries.iter())
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect()
};
let opts = |hold_ts: bool, retention_ts: bool| ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let locked_by_both = metadata(&[&hold, &retention]);
assert!(replication_write_may_pass_worm_gate(&locked_by_both, &opts(true, true)));
assert!(!replication_write_may_pass_worm_gate(&locked_by_both, &opts(true, false)));
assert!(!replication_write_may_pass_worm_gate(&locked_by_both, &opts(false, true)));
assert!(replication_write_may_pass_worm_gate(&metadata(&[&hold]), &opts(true, false)));
assert!(!replication_write_may_pass_worm_gate(&metadata(&[&hold]), &opts(false, true)));
assert!(replication_write_may_pass_worm_gate(&metadata(&[&retention]), &opts(false, true)));
assert!(!replication_write_may_pass_worm_gate(&metadata(&[&retention]), &opts(true, false)));
// Expired retention and a released hold no longer lock anything.
let unlocked = metadata(&[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]]);
assert!(replication_write_may_pass_worm_gate(&unlocked, &opts(false, false)));
// Never for a non-replication write, whatever it carries.
let local = ObjectOptions {
replication_request: false,
..opts(true, true)
};
assert!(!replication_write_may_pass_worm_gate(&metadata(&[&hold]), &local));
}
/// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
/// lock keys as empty strings (the MinIO on-disk shape, see
/// `parse_object_lock_retention`); that is "no lock", not corruption, and
/// must not wedge later explicit-version PUTs or deletes
/// (rustfs/backlog#1953).
#[test]
fn deletion_treats_cleared_empty_lock_metadata_as_unlocked() {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
let cases: [(&str, &[&str]); 3] = [
(
"cleared retention",
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
(
"all cleared",
&[
AMZ_OBJECT_LOCK_MODE_LOWER,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
],
),
];
for (case, keys) in cases {
let user_defined = keys.iter().map(|key| (key.to_string(), String::new())).collect();
let obj_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
};
let result = check_object_lock_for_deletion_with_config(None, &obj_info, false);
assert!(matches!(result, Ok(None)), "{case}: empty lock keys must read as unlocked: {result:?}");
}
}
#[test]
fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
let mut user_defined = std::collections::HashMap::new();
@@ -3868,7 +3868,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
actual_size,
object_info.etag.clone().unwrap_or_default(),
object_info.mod_time,
&put_opts.internal,
),
)
.await
@@ -472,7 +472,6 @@ pub(crate) fn replication_complete_multipart_options(
actual_size: String,
source_etag: String,
source_mtime: Option<OffsetDateTime>,
source_internal: &AdvancedPutOptions,
) -> PutObjectOptions {
let mut user_metadata = HashMap::new();
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -485,14 +484,6 @@ pub(crate) fn replication_complete_multipart_options(
// mtime must degrade to epoch so header() suppresses the header
// instead of asserting the replication time as the object's mtime.
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
// Carry the per-category LWW timestamps on the complete request as
// well: the receiver's CompleteMultipartUpload options builder
// parses the same headers, so the multipart transport gets the
// same receiver-side LWW as the single-PUT transport
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
tagging_timestamp: source_internal.tagging_timestamp,
retention_timestamp: source_internal.retention_timestamp,
legalhold_timestamp: source_internal.legalhold_timestamp,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
..Default::default()
@@ -658,39 +649,20 @@ mod tests {
#[test]
fn replication_complete_multipart_options_sets_actual_size() {
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
let source_internal = AdvancedPutOptions {
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
..Default::default()
};
let options = replication_complete_multipart_options(
"1024".to_string(),
"0123456789abcdef0123456789abcdef-3".to_string(),
Some(source_mtime),
&source_internal,
);
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
assert_eq!(options.internal.source_mtime, source_mtime);
// The complete request must carry the same per-category LWW timestamps
// as the initiate request; the receiver reads them from the complete
// headers (rustfs/backlog#1953).
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
// Absent source mtime must degrade to epoch (header suppressed), not
// the AdvancedPutOptions default of now_utc() — that default would
// stamp the replication time as the replica's mtime and break the
// multipart HEAD convergence. Unset category timestamps stay epoch so
// header() keeps suppressing them.
let options_no_mtime =
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
// multipart HEAD convergence.
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
assert_eq!(
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -32,6 +32,10 @@ pub struct Credentials {
pub access_key: String,
#[serde(rename = "secretKey")]
pub secret_key: String,
// The aliases accept madmin's JSON tags (MinIO-written bucket-targets
// metadata and mc request bodies) without changing the snake_case
// persisted/peer wire format this struct serializes to.
#[serde(alias = "sessionToken")]
pub session_token: Option<String>,
pub expiration: Option<Timestamp>,
}
@@ -202,12 +206,14 @@ pub struct BucketTarget {
#[serde(default)]
pub region: String,
#[serde(alias = "bandwidth", default)]
// madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
// alias kept for inputs written before the madmin tag was verified.
#[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
pub bandwidth_limit: i64,
#[serde(rename = "replicationSync", default)]
pub replication_sync: bool,
#[serde(default)]
#[serde(alias = "storageclass", default)]
pub storage_class: String,
#[serde(rename = "skipTlsVerify", default)]
pub skip_tls_verify: bool,
@@ -220,7 +226,7 @@ pub struct BucketTarget {
#[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
pub reset_before_date: Option<OffsetDateTime>,
#[serde(default)]
#[serde(alias = "resetID", default)]
pub reset_id: String,
#[serde(rename = "totalDowntime", with = "duration_seconds", default)]
pub total_downtime: Duration,
@@ -233,7 +239,7 @@ pub struct BucketTarget {
#[serde(default)]
pub latency: LatencyStat,
#[serde(default)]
#[serde(alias = "deploymentID", default)]
pub deployment_id: String,
#[serde(default)]
@@ -531,6 +537,85 @@ mod tests {
assert_eq!(value["totalDowntime"], 90);
}
#[test]
fn bucket_target_persisted_wire_keys_stay_snake_case() {
// bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
// in the admin set/remove handlers) and the msgpack struct-map form
// (`BucketTargets::marshal_msg`) both come straight from this struct's
// serde field names. madmin naming is applied only in the admin
// response layer (`remote_target_admin_json`); renaming here would
// silently break every existing deployment's persisted metadata.
let targets = BucketTargets {
targets: vec![BucketTarget {
credentials: Some(Credentials {
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
session_token: Some("token".to_string()),
expiration: None,
}),
bandwidth_limit: 5,
storage_class: "STANDARD".to_string(),
reset_id: "reset-1".to_string(),
deployment_id: "deploy-1".to_string(),
..Default::default()
}],
};
let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
let msgpack: serde_json::Value =
rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
.expect("msgpack struct map should decode into a JSON value");
for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
}
}
#[test]
fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
// A MinIO-written bucket-targets.json carries madmin's JSON tags
// (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
// `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
// On migration these must land in the matching fields instead of
// silently defaulting (backlog#1951).
let targets: BucketTargets = serde_json::from_value(serde_json::json!({
"targets": [{
"sourcebucket": "src",
"endpoint": "minio.example:9000",
"credentials": {
"accessKey": "ak",
"secretKey": "sk",
"sessionToken": "minio-session-token"
},
"targetbucket": "dst",
"type": "replication",
"replicationSync": true,
"bandwidthlimit": 107374182400i64,
"storageclass": "STANDARD",
"resetID": "reset-789",
"deploymentID": "deploy-123"
}]
}))
.expect("MinIO-written bucket-targets.json must deserialize");
let target = &targets.targets[0];
assert_eq!(target.bandwidth_limit, 107374182400);
assert_eq!(target.storage_class, "STANDARD");
assert_eq!(target.reset_id, "reset-789");
assert_eq!(target.deployment_id, "deploy-123");
assert_eq!(
target
.credentials
.as_ref()
.and_then(|credentials| credentials.session_token.as_deref()),
Some("minio-session-token")
);
}
#[test]
fn test_bucket_target_debug_redacts_credentials() {
let target = BucketTarget {
-1
View File
@@ -47,7 +47,6 @@ use crate::bucket::metadata_sys;
use crate::bucket::metadata_sys::ObjectLockConfigState;
use crate::bucket::object_lock::objectlock_sys::{
check_object_lock_for_deletion_with_config, check_object_lock_for_deletion_with_state, check_retention_for_modification,
replication_write_may_pass_worm_gate,
};
use crate::bucket::replication::{
ReplicateDecision, ReplicationObjectBridge, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
@@ -2223,57 +2223,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.set_data_moved();
}
// Receiver-side LWW (rustfs/backlog#1953): the multipart replication
// transport carries the category values at CreateMultipartUpload (in
// the staged upload metadata) and the source category timestamps on
// the complete request. Read the destination version under the held
// object write lock and keep any category this site modified more
// recently. Read failures (version absent on first replication, quorum
// errors) keep today's overwrite semantics: failing the complete would
// loop through MRF, re-delivering the stale value forever.
if crate::set_disk::ops::object::replication_lww_applicable(opts)
&& let Some(version_id) = fi.version_id
{
match self
.get_object_info(
bucket,
object,
&ObjectOptions {
version_id: Some(version_id.to_string()),
no_lock: true,
metadata_cache_safe: false,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
..Default::default()
},
)
.await
{
Ok(existing) => {
let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
}
// Version absent: first replication of this version, nothing
// local to compare — the normal path, not a degraded one.
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => {
// Degraded path: without the stored state the inbound
// metadata is applied unchanged — exactly the overwrite
// LWW exists to prevent — so this must be operator-visible.
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
version_id = %version_id,
error = %err,
state = "replication_lww_read_unavailable",
"SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
);
}
}
}
for meta in parts_metadatas.iter_mut() {
if meta.has_valid_erasure_geometry() {
meta.size = fi.size;
@@ -7011,97 +6960,6 @@ mod tests {
.await
}
/// Receiver-side LWW on the multipart replication transport
/// (rustfs/backlog#1953): a metadata-only replication of a multipart
/// source object rides CreateMultipartUpload (category values in the
/// upload metadata) + CompleteMultipartUpload (category timestamps in
/// the complete options). A stale inbound tagging timestamp must not
/// overwrite a newer locally-tagged destination version.
#[tokio::test]
#[serial]
async fn complete_multipart_upload_stale_replication_tags_keep_local() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-lww-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
// Local destination version with newer tags.
let version_id = Uuid::new_v4();
let mut local_metadata = HashMap::new();
local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
set_disks
.put_object(
bucket,
object,
&mut local_reader,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined: local_metadata,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await
.expect("local versioned put should commit");
// Inbound replication upload carrying older tags for the same version.
let mut inbound_metadata = HashMap::new();
inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let create_opts = ObjectOptions {
versioned: true,
user_defined: inbound_metadata,
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
let complete_opts = ObjectOptions {
versioned: true,
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
..Default::default()
};
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
.await
.expect("replication multipart completion should succeed even when a category keeps local values");
let info = set_disks
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
},
)
.await
.expect("completed version should be readable");
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound multipart tags must not overwrite newer local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
#[tokio::test]
#[serial]
async fn complete_multipart_upload_assigns_completion_version_id() {
+1 -729
View File
@@ -1880,110 +1880,6 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
transported
}
/// True when an authorized replication write carries at least one per-category
/// source timestamp, i.e. receiver-side LWW has something to judge.
pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
opts.replication_request
&& (opts.replication_tagging_timestamp.is_some()
|| opts.replication_retention_timestamp.is_some()
|| opts.replication_legalhold_timestamp.is_some())
}
/// The stored per-category state of a destination version, as compared by
/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
/// externalizes tags into `user_tags` (stripping the metadata key), so the
/// tag value is folded back into map form here.
pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
let mut stored = (*existing.user_defined).clone();
if !existing.user_tags.is_empty() {
stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
}
stored
}
/// Receiver-side last-writer-wins for authorized replication writes
/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
/// whole-object transports, so in active-active topologies an inbound write
/// carries the source's tags / retention / legal hold verbatim and would
/// otherwise overwrite a category the destination modified more recently —
/// both sites end up permanently diverged while reporting COMPLETED.
///
/// Judged per category, only when the inbound request carries that category's
/// source timestamp (`ObjectOptions::replication_*_timestamp`):
/// - stored timestamp newer than inbound: the local category values and
/// timestamp are kept; the rest of the write proceeds per the inbound
/// metadata and the object-level result stays successful (failing the write
/// instead would loop through MRF, re-delivering the stale value forever);
/// - otherwise the inbound category wins and its internal timestamp key is
/// pinned to the source-authored time — the PUT path re-stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
/// `now()` via `eval_metadata`), which would make the replica's clock the
/// LWW authority and wedge later convergence;
/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
/// overwrite behavior is preserved.
///
/// Returns whether `inbound` was modified. Callers must hold the object write
/// lock so the stored values compared here are the ones being replaced.
pub(in crate::set_disk) fn merge_replication_metadata_lww(
inbound: &mut HashMap<String, String>,
existing: &HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::metadata_compat::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
remove_str,
};
use time::format_description::well_known::Rfc3339;
let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
(
opts.replication_retention_timestamp,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
(
opts.replication_legalhold_timestamp,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
&[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
),
];
let mut changed = false;
for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
let Some(inbound_timestamp) = inbound_timestamp else { continue };
let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
inbound.retain(|key, _| !is_category_value_key(key));
remove_str(inbound, timestamp_suffix);
for (key, value) in existing {
if is_category_value_key(key) {
inbound.insert(key.clone(), value.clone());
}
}
// Restore the winning timestamp via insert_str, not a verbatim key
// copy: a MinIO-written version may carry only the
// x-minio-internal- key, and the dual-key invariant requires every
// write to produce both keys.
if let Some(stored_value) = get_str(existing, timestamp_suffix) {
rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
}
changed = true;
} else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
&& get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
{
rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
changed = true;
}
}
changed
}
impl SetDisks {
pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
&self,
@@ -2661,33 +2557,9 @@ impl SetDisks {
let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
})?;
// The WORM gate protects the locked version from local
// overwrites; an authorized replication write passes it
// only when the LWW merge below will judge every
// locking category (see
// `replication_write_may_pass_worm_gate`). Gate first so
// malformed lock metadata still fails closed.
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some()
&& !replication_write_may_pass_worm_gate(&existing.user_defined, opts)
{
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
}
// Receiver-side LWW (rustfs/backlog#1953): reuse this
// commit-lock read of the destination version so a
// category (tags / retention / legal hold) modified
// more recently on this site is kept instead of being
// overwritten by the inbound replication metadata.
if replication_lww_applicable(opts) {
let stored = stored_replication_category_metadata(&existing);
let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
if merge_replication_metadata_lww(&mut merged, &stored, opts) {
for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
if disk.is_some() {
pfi.metadata = merged.clone();
}
}
}
}
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
@@ -8015,540 +7887,6 @@ mod replication_quota_safety_tests {
}
}
#[cfg(test)]
mod replication_lww_tests {
//! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
//! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
//! (tags / retention / legal hold) is older than the destination version's
//! stored timestamp must keep the local category values instead of
//! overwriting them; categories are judged independently and the write
//! itself still succeeds.
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
insert_str,
};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEW: &str = "2026-03-01T00:00:00Z";
fn parse_ts(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
}
async fn make_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
}
async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, opts)
.await
.expect("versioned put should commit");
assert_eq!(opts.version_id.as_deref(), Some(version_id));
}
fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
}
}
/// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
}
fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(timestamp)),
..versioned_opts(version_id, user_defined)
}
}
async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
set_disks
.get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
.await
.expect("version should be readable")
}
#[tokio::test]
async fn inbound_stale_tagging_keeps_newer_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound tags must not overwrite newer local tags"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the winning local tagging timestamp must be preserved"
);
}
#[tokio::test]
async fn inbound_newer_tagging_overwrites_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-newer";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_NEW),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"newer inbound tags must overwrite older local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_wins_when_local_has_no_tagging_timestamp() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-no-local-ts";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Pre-P1-6 data: local tags without a stored tagging timestamp.
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"without a local timestamp the inbound category must win (pre-LWW data compatibility)"
);
}
#[tokio::test]
async fn categories_are_judged_independently() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-category-independent";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
// timestamp key only, the shape a replicated retention clear stores.
// (An active local retention would already block the overwrite at the
// WORM gate; the LWW-reachable retention states are cleared/expired.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound: older tags (T_OLD), newer retention (T_NEW).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
replication_retention_timestamp: Some(parse_ts(T_NEW)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
Some("COMPLIANCE"),
"the newer retention category must be applied in the same write"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_stale_legal_hold_keeps_local_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-legalhold-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
// still ON already blocks the overwrite at the WORM gate; the
// LWW-reachable divergence is a stale inbound ON resurrecting a hold
// that was released more recently on this site.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
Some("OFF"),
"a stale inbound legal hold must not resurrect a hold released more recently"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
Some(T_LOCAL)
);
}
/// Dual-key invariant under LWW: a MinIO-written destination version may
/// carry only the x-minio-internal timestamp key; when the local category
/// wins, the restored map must still hold BOTH compatibility keys.
#[test]
fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let existing = HashMap::from([
(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
]);
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
..Default::default()
};
assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
assert_eq!(
inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
Some(T_LOCAL),
"the RustFS twin key must be materialized even when the source version only had the MinIO key"
);
assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
}
/// When the inbound category wins, the stored timestamp must be the
/// source-authored one: the PUT path's eval_metadata stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention`), which would otherwise make this
/// replica's clock the LWW authority and wedge later convergence.
#[tokio::test]
async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-retention-ts-pinned";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local cleared retention at T_OLD.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound newer retention: the source authored T_LOCAL, but the PUT
// path's eval_metadata stomped the metadata key with receiver-now
// (simulated by T_NEW here).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the stored category timestamp must be the source-authored time, not the receiver's clock"
);
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
}
#[tokio::test]
async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-deleted";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert!(
info.user_tags.is_empty(),
"a newer local tag deletion must not be resurrected by older inbound tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
/// Destination version under an active legal hold at `hold_timestamp`,
/// plus an active COMPLIANCE retention (no retention timestamp).
async fn seed_locked_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, hold_timestamp: &str) {
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, hold_timestamp.to_string());
local.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
local.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, local)).await;
}
/// Inbound legal-hold release from a source that also carries the (same)
/// COMPLIANCE retention; the sender stamps a source timestamp for every
/// category the source version has.
fn inbound_legal_hold_release_opts(version_id: &str, timestamp: &str) -> ObjectOptions {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, timestamp.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: Some(parse_ts(timestamp)),
replication_retention_timestamp: Some(parse_ts(T_OLD)),
..versioned_opts(version_id, inbound)
}
}
/// The source's lock state governs the replica: a legal-hold release (or a
/// retention change) can only reach this site through the authorized
/// replication write, so the commit-time WORM gate must not reject it
/// because the destination version is currently locked.
#[tokio::test]
async fn inbound_newer_legal_hold_release_updates_locked_version() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-locked-release-newer";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_legal_hold_release_opts(&version_id, T_NEW),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
Some("OFF"),
"a newer source-side legal hold release must be applied to the locked replica"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(), Some(T_NEW));
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
Some("COMPLIANCE"),
"the untouched retention category must survive the write"
);
}
/// Skipping the WORM gate for replication writes must not weaken LWW: a
/// stale inbound release still loses to a hold applied more recently here.
#[tokio::test]
async fn inbound_stale_legal_hold_release_keeps_newer_local_hold() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-locked-release-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_locked_version(&set_disks, bucket, object, &version_id, T_LOCAL).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_legal_hold_release_opts(&version_id, T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
Some("ON"),
"a stale inbound release must not lift a hold applied more recently on this site"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
Some(T_LOCAL)
);
}
/// A replication write that carries no source decision for a locking
/// category (here: tags changed at a source that never held the object)
/// must not lift the destination's hold by replacing the metadata
/// unjudged; it stays WORM-rejected like a local overwrite.
#[tokio::test]
async fn inbound_without_legal_hold_timestamp_stays_rejected_on_held_version() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-locked-unjudged-category";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "k=v".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_NEW.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_NEW)),
replication_retention_timestamp: Some(parse_ts(T_NEW)),
replication_legalhold_timestamp: None,
..versioned_opts(&version_id, inbound)
};
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
let err = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect_err("a replication write without the legal-hold source timestamp must stay rejected");
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
}
/// The gate runs before the replication bypass, so malformed persisted
/// lock metadata still fails closed for an authorized replication write.
#[tokio::test]
async fn replication_write_on_malformed_lock_metadata_still_fails_closed() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-locked-malformed";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "MAYBE".to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
let err = set_disks
.put_object(bucket, object, &mut reader, &inbound_legal_hold_release_opts(&version_id, T_NEW))
.await
.expect_err("malformed persisted lock metadata must fail the replication write closed");
assert!(!matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("MAYBE"));
}
/// The bypass is scoped to authorized replication writes: the same
/// explicit-version PUT without `replication_request` stays WORM-rejected.
#[tokio::test]
async fn non_replication_overwrite_of_locked_version_is_still_rejected() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-locked-plain-put";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
let opts = ObjectOptions {
replication_request: false,
..inbound_legal_hold_release_opts(&version_id, T_NEW)
};
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
let err = set_disks
.put_object(bucket, object, &mut reader, &opts)
.await
.expect_err("a non-replication overwrite of a locked version must stay rejected");
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
}
}
#[cfg(test)]
mod inline_put_commit_path_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
@@ -14535,72 +13873,6 @@ mod put_object_tmp_cleanup_tests {
assert_eq!(body, original_body);
}
/// A local PutObjectRetention / PutObjectLegalHold clear persists empty
/// lock keys (`parse_object_lock_retention`). The commit-time WORM gate
/// must read that as unlocked: an explicit-version PUT (the inbound
/// replication transport) and a version delete both have to succeed
/// (rustfs/backlog#1953).
#[tokio::test]
async fn explicit_version_overwrite_and_delete_succeed_after_local_lock_clear() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "put-explicit-version-cleared-lock";
let object = "object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut initial_reader = PutObjReader::from_vec(b"original".to_vec());
let initial = set_disks
.put_object(
bucket,
object,
&mut initial_reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("initial version should be written");
let version_id = initial
.version_id
.expect("versioned PUT should return a version ID")
.to_string();
let version_opts = ObjectOptions {
versioned: true,
version_id: Some(version_id.clone()),
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())),
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
..Default::default()
};
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
eval_metadata: Some(HashMap::from([
(X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), String::new()),
(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), String::new()),
(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), String::new()),
])),
..version_opts.clone()
},
)
.await
.expect("cleared lock metadata should be written");
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
set_disks
.put_object(bucket, object, &mut replacement, &version_opts)
.await
.expect("explicit-version PUT must not be wedged by cleared lock metadata");
set_disks
.delete_object(bucket, object, version_opts)
.await
.expect("version delete must not be wedged by cleared lock metadata");
}
#[tokio::test]
async fn version_only_copy_checks_the_destination_version_object_lock() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+139 -2
View File
@@ -374,7 +374,10 @@ impl RemoteTargetRequest {
/// Admin-response encoding of a remote target: the persisted bucket-targets
/// format keeps `healthCheckDuration`/`totalDowntime` in seconds and the
/// `latency` stats in milliseconds, but madmin decodes all of them as Go
/// `time.Duration` (nanoseconds) — re-encode just those fields without
/// `time.Duration` (nanoseconds) — and it looks the fields up under its own
/// JSON tags (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
/// `credentials.sessionToken` — madmin-go v3.0.109 `bucket-targets.go`), not
/// the persisted snake_case keys. Re-encode just those fields here without
/// touching the persistence wire format.
fn remote_target_admin_json(target: &BucketTarget) -> Result<serde_json::Value, serde_json::Error> {
fn go_duration_nanos(duration: Duration) -> serde_json::Value {
@@ -383,6 +386,12 @@ fn remote_target_admin_json(target: &BucketTarget) -> Result<serde_json::Value,
u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX).into()
}
fn rename_key(value: &mut serde_json::Value, from: &str, to: &str) {
if let Some(moved) = value.as_object_mut().and_then(|object| object.remove(from)) {
value[to] = moved;
}
}
let mut value = serde_json::to_value(target)?;
value["healthCheckDuration"] = go_duration_nanos(target.health_check_duration);
value["totalDowntime"] = go_duration_nanos(target.total_downtime);
@@ -391,6 +400,11 @@ fn remote_target_admin_json(target: &BucketTarget) -> Result<serde_json::Value,
"avg": go_duration_nanos(target.latency.avg),
"max": go_duration_nanos(target.latency.max),
});
rename_key(&mut value, "bandwidth_limit", "bandwidthlimit");
rename_key(&mut value, "storage_class", "storageclass");
rename_key(&mut value, "reset_id", "resetID");
rename_key(&mut value, "deployment_id", "deploymentID");
rename_key(&mut value["credentials"], "session_token", "sessionToken");
Ok(value)
}
@@ -1473,7 +1487,7 @@ mod tests {
parse_remote_target_update_ops, render_mrf_backlog, render_replication_diff, unique_replication_peers,
validate_remote_target_tls_settings,
};
use crate::admin::storage_api::bucket::target::{BucketTarget, LatencyStat};
use crate::admin::storage_api::bucket::target::{BucketTarget, Credentials as TargetCredentials, LatencyStat};
use crate::admin::storage_api::replication::{BucketStats, DurableMrfBacklog, MrfOpKind, MrfReplicateEntry};
use http::Uri;
@@ -2268,6 +2282,129 @@ mod tests {
assert_eq!(persisted["latency"]["max"], 250);
}
#[test]
fn list_remote_targets_response_uses_madmin_key_names() {
// madmin-go v3.0.109 BucketTarget JSON tags are `bandwidthlimit`,
// `storageclass`, `resetID`, `deploymentID`, and
// `credentials.sessionToken` (backlog#1951); the persisted snake_case
// keys decode to zero values in mc, blanking the bandwidth and
// reset-id columns of `mc replicate ls`.
let target = BucketTarget {
endpoint: "192.168.1.10:9000".to_string(),
target_bucket: "target".to_string(),
credentials: Some(TargetCredentials {
access_key: "access".to_string(),
secret_key: String::new(),
session_token: Some("session-token".to_string()),
expiration: None,
}),
bandwidth_limit: 107_374_182_400,
storage_class: "STANDARD".to_string(),
reset_id: "reset-123".to_string(),
deployment_id: "deploy-456".to_string(),
..Default::default()
};
let value = super::remote_target_admin_json(&target).expect("admin response should serialize");
assert_eq!(value["bandwidthlimit"], 107_374_182_400i64);
assert_eq!(value["storageclass"], "STANDARD");
assert_eq!(value["resetID"], "reset-123");
assert_eq!(value["deploymentID"], "deploy-456");
assert_eq!(value["credentials"]["sessionToken"], "session-token");
// The madmin keys replace the snake_case ones rather than duplicating
// them next to each other.
for stale in ["bandwidth_limit", "bandwidth", "storage_class", "reset_id", "deployment_id"] {
assert!(value.get(stale).is_none(), "admin response must not carry `{stale}`");
}
assert!(value["credentials"].get("session_token").is_none());
}
/// Decode-side mirror of madmin-go v3.0.109 `BucketTarget`/`Credentials`
/// (`bucket-targets.go`): the exact `json:"..."` tags mc's `encoding/json`
/// looks fields up under. Unknown keys are ignored like Go does, and a
/// missing key leaves the Go zero value, which is exactly how a misnamed
/// key turns into a blank column in `mc replicate ls`.
#[derive(Debug, Default, serde::Deserialize)]
#[serde(default)]
struct MadminBucketTarget {
sourcebucket: String,
endpoint: String,
credentials: Option<MadminCredentials>,
targetbucket: String,
arn: String,
bandwidthlimit: i64,
#[serde(rename = "replicationSync")]
replication_sync: bool,
storageclass: String,
#[serde(rename = "healthCheckDuration")]
health_check_duration: i64,
#[serde(rename = "resetID")]
reset_id: String,
#[serde(rename = "totalDowntime")]
total_downtime: i64,
#[serde(rename = "deploymentID")]
deployment_id: String,
}
#[derive(Debug, Default, serde::Deserialize)]
#[serde(default)]
struct MadminCredentials {
#[serde(rename = "accessKey")]
access_key: String,
#[serde(rename = "secretKey")]
secret_key: String,
#[serde(rename = "sessionToken")]
session_token: String,
}
#[test]
fn list_remote_targets_response_decodes_through_madmin_tags() {
// Regression for the review on backlog#1951: the response must decode
// a nonzero bandwidth limit through madmin's `bandwidthlimit` tag (not
// `bandwidth`, which Go would silently drop as an unknown key).
let target = BucketTarget {
source_bucket: "src".to_string(),
endpoint: "192.168.1.10:9000".to_string(),
target_bucket: "target".to_string(),
arn: "arn:rustfs:replication:us-east-1:dep:target".to_string(),
credentials: Some(TargetCredentials {
access_key: "access".to_string(),
secret_key: String::new(),
session_token: Some("session-token".to_string()),
expiration: None,
}),
bandwidth_limit: 1_073_741_824,
replication_sync: true,
storage_class: "STANDARD".to_string(),
health_check_duration: std::time::Duration::from_secs(60),
reset_id: "reset-123".to_string(),
total_downtime: std::time::Duration::from_secs(90),
deployment_id: "deploy-456".to_string(),
..Default::default()
};
let wire = serde_json::to_string(&super::remote_target_admin_json(&target).expect("admin response should serialize"))
.expect("admin response should encode");
let decoded: MadminBucketTarget = serde_json::from_str(&wire).expect("madmin-shaped decode must succeed");
assert_eq!(decoded.bandwidthlimit, 1_073_741_824, "mc must see the nonzero bandwidth limit");
assert_eq!(decoded.sourcebucket, "src");
assert_eq!(decoded.endpoint, "192.168.1.10:9000");
assert_eq!(decoded.targetbucket, "target");
assert_eq!(decoded.arn, "arn:rustfs:replication:us-east-1:dep:target");
assert!(decoded.replication_sync);
assert_eq!(decoded.storageclass, "STANDARD");
assert_eq!(decoded.health_check_duration, 60_000_000_000);
assert_eq!(decoded.reset_id, "reset-123");
assert_eq!(decoded.total_downtime, 90_000_000_000);
assert_eq!(decoded.deployment_id, "deploy-456");
let credentials = decoded.credentials.expect("credentials must decode");
assert_eq!(credentials.access_key, "access");
assert_eq!(credentials.secret_key, "");
assert_eq!(credentials.session_token, "session-token");
}
#[test]
fn remote_target_admin_json_latency_round_trips_through_go_duration() {
// Round trip: a madmin reader decodes the latency values as Go
+1 -48
View File
@@ -39,7 +39,7 @@ use super::storage_api::object_usecase::bucket::{
metadata_sys,
object_lock::{
objectlock::{get_object_legalhold_meta, get_object_retention_meta},
objectlock_sys::{check_object_lock_for_deletion, is_retention_active, replication_write_may_pass_worm_gate},
objectlock_sys::{check_object_lock_for_deletion, is_retention_active},
},
predict_lifecycle_expiration,
quota::{QuotaCheckResult, QuotaError, QuotaOperation},
@@ -3938,12 +3938,6 @@ pub(crate) fn validate_existing_object_lock_for_write(existing_obj_info: &Object
if put_like_write_creates_new_version(opts) {
return Ok(());
}
// An authorized replication write may replace the locked version when the
// set layer's commit-lock LWW will judge every locking category; the set
// layer re-checks the same rule under the lock.
if replication_write_may_pass_worm_gate(&existing_obj_info.user_defined, opts) {
return Ok(());
}
let legal_hold = get_object_legalhold_meta(&existing_obj_info.user_defined);
if legal_hold
@@ -11043,47 +11037,6 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
/// The source's lock state governs the replica (rustfs/backlog#1953):
/// an authorized replication write carrying the locking category's source
/// timestamp may overwrite a locked version; the set layer's LWW then
/// decides per category.
#[test]
fn validate_existing_object_lock_allows_authorized_replication_overwrite() {
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
replication_request: true,
replication_retention_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
replication_legalhold_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
validate_existing_object_lock_for_write(&compliance_retained_object_info(), &opts)
.expect("replication write must bypass the destination COMPLIANCE lock");
validate_existing_object_lock_for_write(&legal_hold_object_info(), &opts)
.expect("replication write must bypass the destination legal hold");
}
/// Without the locking category's source timestamp the LWW merge cannot
/// judge it, so the write stays rejected instead of lifting the lock.
#[test]
fn validate_existing_object_lock_rejects_replication_overwrite_without_lock_timestamp() {
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let err = validate_existing_object_lock_for_write(&compliance_retained_object_info(), &opts)
.expect_err("COMPLIANCE lock must hold without a retention source timestamp");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = validate_existing_object_lock_for_write(&legal_hold_object_info(), &opts)
.expect_err("legal hold must hold without a legal-hold source timestamp");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
#[test]
fn is_put_object_extract_requested_accepts_meta_header() {
let mut headers = HeaderMap::new();
-10
View File
@@ -587,16 +587,6 @@ pub(crate) mod bucket {
retain_until_date,
)
}
pub(crate) fn replication_write_may_pass_worm_gate(
user_defined: &std::collections::HashMap<String, String>,
opts: &crate::storage::storage_api::StorageObjectOptions,
) -> bool {
crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::replication_write_may_pass_worm_gate(
user_defined,
opts,
)
}
}
}
+4 -5
View File
@@ -554,11 +554,10 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
// Persist into the internal metadata keys so a later outbound replication
// pass (replication_target_boundary) reads the source's modification
// times instead of falling back to mod_time. Receiver-side LWW happens at
// the set layer under the object write lock
// (ecstore set_disk::ops::object::merge_replication_metadata_lww,
// rustfs/backlog#1953): a category whose stored timestamp is newer than
// the inbound one keeps the local values.
// times instead of falling back to mod_time.
// TODO(P1-6): receiver-side LWW is still missing — when the stored
// per-category timestamp is newer than the inbound one, the existing
// tags/retention/legal-hold should win instead of being overwritten.
for (timestamp, suffix) in [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
(opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),