Compare commits

..

2 Commits

Author SHA1 Message Date
Zhengchao An 39e4aa1e39 Merge branch 'main' into reatang/wasabi-replication-compatibility-c9e5ec 2026-09-07 02:11:25 +08:00
唐小鸭 6141694c10 fix(replication): stop duplicate re-drives on own-version-id targets
A target that mints its own version ids (Wasabi, AWS S3) answers a
version-addressed HEAD for the source uuid with 404, which the worker read
as "replica missing": every heal, MRF retry and existing-object resync
re-drive PUT the object again and minted one more target version, while
the source reported COMPLETED.

Record a per-target VersionIdentityCapability verdict from every
replication write response and from replication-check's VersionFidelity
phase. On a 404 from a target proven to mint its own ids, locate the
replica by exact key and ETag through ListObjectVersions, HEAD the id the
target assigned, and reuse the ETag-comparison fallback in the heal
decision, the existing-object resync path and the resync verification.
Keep the verdict across target edits that address the same service, since
a resync start rewrites the target entry in place.

Refs: rustfs/backlog#2340
2026-09-07 00:58:06 +08:00
12 changed files with 737 additions and 124 deletions
@@ -368,23 +368,23 @@ impl Drop for SlowReplicationTargetGuard {
// Mirrors madmin-go `ResyncTargetsInfo`/`ResyncTarget` json tags — the same
// shape `mc replicate resync status` decodes.
#[derive(Debug, Clone, serde::Deserialize)]
struct ReplicationResetStatusResponse {
pub(crate) struct ReplicationResetStatusResponse {
#[serde(rename = "target", default)]
targets: Vec<ReplicationResetStatusTarget>,
pub(crate) targets: Vec<ReplicationResetStatusTarget>,
}
#[derive(Debug, Clone, serde::Deserialize)]
struct ReplicationResetStatusTarget {
pub(crate) struct ReplicationResetStatusTarget {
#[serde(rename = "arn", default)]
arn: String,
pub(crate) arn: String,
#[serde(rename = "resetid", default)]
reset_id: String,
pub(crate) reset_id: String,
#[serde(rename = "resyncStatus", default)]
status: String,
pub(crate) status: String,
#[serde(rename = "replicationCount", default)]
replicated_count: i64,
pub(crate) replicated_count: i64,
#[serde(rename = "object", default)]
object: String,
pub(crate) object: String,
}
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
@@ -2294,7 +2294,7 @@ async fn site_replication_state_edit(
/// return the target `(arn, reset_id)`, asserting the response carries the
/// madmin `ResyncTargetsInfo` shape (`target[0].arn` / `target[0].resetid`)
/// that `mc replicate resync start` decodes.
async fn start_bucket_replication_reset(
pub(crate) async fn start_bucket_replication_reset(
env: &RustFSTestEnvironment,
bucket: &str,
) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
@@ -2314,7 +2314,7 @@ async fn start_bucket_replication_reset(
Ok((arn, reset_id))
}
async fn get_replication_reset_status(
pub(crate) async fn get_replication_reset_status(
env: &RustFSTestEnvironment,
bucket: &str,
arn: &str,
@@ -3837,77 +3837,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
Ok(())
}
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a directory
/// marker (`prefix/` with a body) in a versioned bucket is stored as the null
/// version, like MinIO (`putOpts`: "for directory objects skip creating new
/// versions"), and must still replicate to completion instead of staying
/// `PENDING`.
#[tokio::test]
async fn test_bucket_replication_replicates_directory_marker_in_versioned_bucket() -> TestResult {
init_logging();
let mut source_env = RustFSTestEnvironment::new().await?;
let mut source_env_vars = replication_fast_env();
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
let mut target_env = RustFSTestEnvironment::new().await?;
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
let source_bucket = "replication-dir-marker-src";
let target_bucket = "replication-dir-marker-dst";
let source_client = source_env.create_s3_client();
let target_client = target_env.create_s3_client();
source_client.create_bucket().bucket(source_bucket).send().await?;
target_client.create_bucket().bucket(target_bucket).send().await?;
enable_bucket_versioning(&source_env, source_bucket).await?;
enable_bucket_versioning(&target_env, target_bucket).await?;
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
let marker_key = "dir/trailing/";
let body = b"directory marker body";
let put = source_client
.put_object()
.bucket(source_bucket)
.key(marker_key)
.body(ByteStream::from_static(body))
.send()
.await?;
assert!(
put.version_id()
.is_none_or(|id| id == "null" || id == uuid::Uuid::nil().to_string()),
"a directory marker is the null version even in a versioned bucket: {:?}",
put.version_id()
);
wait_for_source_replication_status(&source_client, source_bucket, marker_key, "COMPLETED", false).await?;
let replica = target_client
.get_object()
.bucket(target_bucket)
.key(marker_key)
.send()
.await?;
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body);
let listed = target_client
.list_object_versions()
.bucket(target_bucket)
.prefix(marker_key)
.send()
.await?;
let marker_versions: Vec<_> = listed.versions().iter().filter(|v| v.key() == Some(marker_key)).collect();
assert_eq!(marker_versions.len(), 1, "the marker must land exactly once: {marker_versions:?}");
assert_eq!(
marker_versions[0].version_id(),
Some("null"),
"the replica keeps the null version identity"
);
Ok(())
}
#[tokio::test]
async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> TestResult {
init_logging();
@@ -33,11 +33,11 @@
use crate::common::{init_logging, replication_fast_env};
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
use crate::fake_s3_target::{FakeS3Target, Operation as FakeTargetOperation, RequestRecord};
use crate::fake_s3_target::{FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, RequestRecord};
use crate::on_demand_migration::common::{OdmEnvOptions, OdmTestEnv, fake_source_client};
use crate::replication_extension_test::{
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, put_bucket_replication,
set_replication_target_with_options,
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, get_replication_reset_status,
put_bucket_replication, set_replication_target_with_options, start_bucket_replication_reset,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::{ByteStream, DateTime};
@@ -227,6 +227,170 @@ fn expectation(mode: TargetMode, shape: ObjectShape) -> Expectation {
.unwrap_or(Expectation::Completed)
}
/// rustfs/backlog#2340: a target that mints its own version ids (Wasabi,
/// AWS S3) answers 404 to a HEAD by the source uuid, which the worker used to
/// read as "replica missing" and re-drive the PUT — one more target version
/// per heal, MRF retry or resync. Two re-drive shapes, both must converge on
/// the single version the first PUT created:
/// - the first PUT lands but its response is lost, so the object is FAILED
/// and the scanner heal pass re-drives it;
/// - an existing-object resync re-drives a COMPLETED object unconditionally.
#[tokio::test]
async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult {
init_logging();
let target = FakeS3Target::start().await?;
let target_bucket = "matrix-mint-own-redrive-dst".to_string();
target.create_bucket_with_object_lock(target_bucket.clone());
target.assign_own_version_ids(true);
let mut env_vars = replication_fast_env();
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
env_vars.extend_from_slice(&[
("NO_PROXY", "127.0.0.1,localhost"),
("HTTP_PROXY", ""),
("HTTPS_PROXY", ""),
// The scanner heal pass is what re-drives a FAILED object.
("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
]);
let env = OdmTestEnv::start_with(OdmEnvOptions {
env: env_vars,
..OdmEnvOptions::default()
})
.await?;
let source_env = &env.rustfs;
let source_bucket = "matrix-mint-own-redrive-src";
let source_client = source_env.create_s3_client();
source_client
.create_bucket()
.bucket(source_bucket)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
enable_bucket_versioning(source_env, source_bucket).await?;
let target_arn = set_replication_target_with_options(
source_env,
source_bucket,
ReplicationTargetOptions {
endpoint: &target.address(),
access_key: FAKE_ACCESS_KEY,
secret_key: FAKE_SECRET_KEY,
target_bucket: &target_bucket,
secure: false,
skip_tls_verify: false,
ca_cert_pem: None,
},
)
.await?;
put_bucket_replication(source_env, source_bucket, &target_arn).await?;
// Teach the worker the target's identity contract with one ordinary
// write, exactly as production learns it (the PUT response carries the
// minted id).
let probe_key = "redrive/identity-probe.bin";
source_client
.put_object()
.bucket(source_bucket)
.key(probe_key)
.body(ByteStream::from(payload(4 * 1024, 0x01)))
.send()
.await?;
assert_eq!(
wait_for_terminal_replication_status(&source_client, source_bucket, probe_key).await?,
"COMPLETED"
);
// Shape 1: the PUT is stored, its response never arrives, heal re-drives.
let heal_key = "redrive/heal.bin";
target.inject_for_key(FakeTargetOperation::PutObject, heal_key, FakeTargetFault::DisconnectAfterResponse, 1);
source_client
.put_object()
.bucket(source_bucket)
.key(heal_key)
.body(ByteStream::from(payload(8 * 1024, 0x02)))
.send()
.await?;
wait_for_replication_status_and_single_version(&source_client, source_bucket, &target, &target_bucket, heal_key).await?;
// Shape 2: an existing-object resync re-drives a COMPLETED object.
let resync_key = "redrive/resync.bin";
source_client
.put_object()
.bucket(source_bucket)
.key(resync_key)
.body(ByteStream::from(payload(8 * 1024, 0x03)))
.send()
.await?;
assert_eq!(
wait_for_terminal_replication_status(&source_client, source_bucket, resync_key).await?,
"COMPLETED"
);
let (reset_arn, _reset_id) = start_bucket_replication_reset(source_env, source_bucket).await?;
assert_eq!(reset_arn, target_arn);
let resync = async {
loop {
let status = get_replication_reset_status(source_env, source_bucket, &target_arn).await?;
if let Some(entry) = status.targets.iter().find(|entry| entry.arn == target_arn)
&& entry.status == "Completed"
{
return Ok::<_, Box<dyn Error + Send + Sync>>(entry.replicated_count);
}
sleep(Duration::from_millis(250)).await;
}
};
let replicated = timeout(Duration::from_secs(90), resync)
.await
.map_err(|_| "existing-object resync did not complete within 90 seconds")??;
assert!(replicated >= 3, "resync must count the located replicas as replicated, got {replicated}");
for key in [probe_key, heal_key, resync_key] {
let versions = target.stored_versions(&target_bucket, key);
assert_eq!(
versions.len(),
1,
"{key}: a re-drive against a target that mints its own version ids must not mint another one: {versions:?}"
);
}
target.shutdown().await;
Ok(())
}
/// Wait until `key` is COMPLETED on the source and, for the observation
/// window after that, the target still holds exactly one live version of it.
async fn wait_for_replication_status_and_single_version(
source_client: &Client,
source_bucket: &str,
target: &FakeS3Target,
target_bucket: &str,
key: &str,
) -> TestResult {
// The lost PUT response first settles the object FAILED; only the next
// scanner heal pass can turn that into COMPLETED, so FAILED is transient
// here and the wait is for COMPLETED alone.
let converged = async {
loop {
let head = source_client.head_object().bucket(source_bucket).key(key).send().await?;
if head.replication_status().is_some_and(|status| status.as_str() == "COMPLETED") {
return Ok::<_, Box<dyn Error + Send + Sync>>(());
}
sleep(Duration::from_millis(250)).await;
}
};
timeout(Duration::from_secs(90), converged)
.await
.map_err(|_| format!("{key}: heal re-drive did not converge to COMPLETED within 90 seconds"))??;
// The heal pass keeps visiting the key for a few scanner cycles; a
// duplicate would show up here as a second stored version.
for _ in 0..12 {
let versions = target.stored_versions(target_bucket, key);
assert_eq!(versions.len(), 1, "{key}: target minted another version on re-drive: {versions:?}");
sleep(Duration::from_millis(500)).await;
}
Ok(())
}
#[tokio::test]
async fn matrix_baseline_target() -> TestResult {
run_row(TargetMode::Baseline).await
+1 -1
View File
@@ -32,7 +32,7 @@ pub mod bucket {
pub mod bucket_target_sys {
pub use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
SsecPassthroughCapability, TargetClient, append_version_id_query,
SsecPassthroughCapability, TargetClient, VersionIdentityCapability, append_version_id_query,
};
}
+194 -1
View File
@@ -18,7 +18,7 @@ use crate::bucket::metadata_sys::get_replication_config;
use crate::bucket::remote_s3_client::{
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
};
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity};
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
use crate::bucket::target::ARN;
use crate::bucket::target::BucketTargetType;
@@ -126,6 +126,22 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
}
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
/// Whether an edited bucket target still addresses the same remote service
/// (endpoint, bucket, path style, TLS and identity), so a verdict learned
/// about that service stays valid across the edit.
fn same_replication_service(edited: &BucketTarget, previous: &BucketTarget) -> bool {
let access_key = |target: &BucketTarget| target.credentials.as_ref().map(|credentials| credentials.access_key.clone());
edited.endpoint == previous.endpoint
&& edited.target_bucket == previous.target_bucket
&& edited.secure == previous.secure
&& edited.path == previous.path
&& access_key(edited) == access_key(previous)
}
/// Page size and page budget for [`TargetClient::find_version_by_etag`].
const FIND_VERSION_BY_ETAG_PAGE_SIZE: i32 = 1000;
const FIND_VERSION_BY_ETAG_MAX_PAGES: usize = 8;
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
@@ -349,6 +365,13 @@ struct TargetClientBuildProbe {
/// their import path while the verdict vocabulary lives with the
/// replication decision logic.
pub use crate::bucket::replication::SsecPassthroughCapability;
/// Version-identity verdicts (see the enum's own docs in
/// `rustfs-replication`) are cached here per target ARN and follow the same
/// `arn_remotes_map` lifecycle. They carry no TTL: the verdict is refreshed
/// by every replication write's response, so it can only go stale on a
/// target that receives no writes — and a stale `MintsOwn` costs one extra
/// content-identity lookup before a PUT, never a lost replica.
pub use crate::bucket::replication::VersionIdentityCapability;
/// How long an audited SSE-C passthrough verdict stays authoritative.
///
@@ -375,6 +398,11 @@ pub struct BucketTargetSys {
/// SSE-C passthrough capability verdicts keyed by target ARN. See
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
/// Version-identity verdicts keyed by target ARN. See
/// [`VersionIdentityCapability`]; reset alongside `arn_remotes_map`. A std
/// lock (never held across an await) so the replication worker can record
/// a verdict from inside its synchronous PUT-response audit.
version_identity_map: Arc<std::sync::RwLock<HashMap<String, VersionIdentityCapability>>>,
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
@@ -423,6 +451,7 @@ impl BucketTargetSys {
Self {
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
version_identity_map: Arc::new(std::sync::RwLock::new(HashMap::new())),
targets_map: Arc::new(RwLock::new(HashMap::new())),
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
h_mutex: Arc::new(RwLock::new(HashMap::new())),
@@ -746,10 +775,40 @@ impl BucketTargetSys {
arn_remotes_map.remove(&target.arn);
health_map.remove(&target.arn);
ssec_map.remove(&target.arn);
self.forget_version_identity_capability(&target.arn);
}
}
}
/// Cached version-identity verdict for a target ARN; `Unknown` until a
/// replication write or a replication-check VersionFidelity probe judged
/// it since the target was built.
pub fn version_identity_capability(&self, arn: &str) -> VersionIdentityCapability {
self.version_identity_map
.read()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.get(arn)
.copied()
.unwrap_or_default()
}
/// Record a version-identity verdict for a target ARN. Written by the
/// replication worker after every PutObject / CompleteMultipartUpload
/// response and by the replication-check VersionFidelity phase.
pub fn record_version_identity_capability(&self, arn: &str, capability: VersionIdentityCapability) {
self.version_identity_map
.write()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.insert(arn.to_string(), capability);
}
fn forget_version_identity_capability(&self, arn: &str) {
self.version_identity_map
.write()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(arn);
}
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
/// false)` when no verdict has been recorded since the target was built.
@@ -1162,12 +1221,32 @@ impl BucketTargetSys {
// Remove existing targets
if let Some(existing_targets) = targets_map.remove(bucket) {
let mut ssec_map = self.ssec_passthrough_map.write().await;
let unchanged_service: HashMap<&str, &BucketTarget> = targets
.map(|new_targets| {
new_targets
.targets
.iter()
.map(|target| (target.arn.as_str(), target))
.collect()
})
.unwrap_or_default();
for target in existing_targets {
arn_remotes_map.remove(&target.arn);
health_map.remove(&target.arn);
// A rebuilt/edited target may point at a different service:
// the SSE-C passthrough verdict must be re-audited from Unknown.
ssec_map.remove(&target.arn);
// The version-identity verdict survives an edit that keeps the
// same remote service (a resync start or a bandwidth change
// rewrites the entry in place): forgetting it there would make
// the very resync that follows re-drive every object as a
// duplicate on a target that mints its own version ids.
if unchanged_service
.get(target.arn.as_str())
.is_none_or(|edited| !same_replication_service(edited, &target))
{
self.forget_version_identity_capability(&target.arn);
}
self.update_bandwidth_limit(bucket, &target.arn, 0);
}
}
@@ -1892,6 +1971,62 @@ impl TargetClient {
.map_err(Box::new)
}
/// Locate a replica by content identity on a target that mints its own
/// version ids: page `ListObjectVersions` under the exact key and return
/// the newest live version whose ETag matches `source_etag`. Delete
/// markers and prefix siblings never match. Bounded to
/// [`FIND_VERSION_BY_ETAG_MAX_PAGES`] pages so a key with a very deep
/// history cannot turn one convergence check into an unbounded scan; a
/// replica beyond that window reads as missing, which only costs a
/// re-PUT (today's behaviour), never a lost object.
pub async fn find_version_by_etag(
&self,
bucket: &str,
object: &str,
source_etag: &str,
) -> Result<Option<String>, Box<SdkError<aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError>>> {
let mut key_marker: Option<String> = None;
let mut version_id_marker: Option<String> = None;
for _ in 0..FIND_VERSION_BY_ETAG_MAX_PAGES {
let page = self
.client
.list_object_versions()
.bucket(bucket)
.prefix(object)
.max_keys(FIND_VERSION_BY_ETAG_PAGE_SIZE)
.set_key_marker(key_marker.take())
.set_version_id_marker(version_id_marker.take())
.send()
.await
.map_err(Box::new)?;
if let Some(version) = page.versions().iter().find(|version| {
version.key() == Some(object)
&& version.version_id().is_some_and(|id| !id.is_empty())
&& replication_etags_match(Some(source_etag), version.e_tag())
}) {
return Ok(version.version_id().map(str::to_string));
}
// Every listed key is >= the prefix; once the listing moved past
// the exact key there is nothing left to find.
if page
.versions()
.iter()
.any(|version| version.key().is_some_and(|key| key > object))
{
return Ok(None);
}
if !page.is_truncated().unwrap_or(false) {
return Ok(None);
}
key_marker = page.next_key_marker().map(str::to_string);
version_id_marker = page.next_version_id_marker().map(str::to_string);
if key_marker.is_none() {
return Ok(None);
}
}
Ok(None)
}
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
/// replicated locally, MinIO `proxyHeadToRepTarget`).
///
@@ -3221,6 +3356,64 @@ mod tests {
assert!(message.contains("connection refused"));
}
#[test]
fn same_replication_service_ignores_resync_and_bandwidth_edits() {
let base = BucketTarget {
endpoint: "target.example:9000".to_string(),
target_bucket: "replica".to_string(),
secure: true,
path: "on".to_string(),
arn: "arn:rustfs:replication:us-east-1:bucket:same".to_string(),
credentials: Some(Credentials {
access_key: "access".to_string(),
..Default::default()
}),
..Default::default()
};
let resync_edit = BucketTarget {
reset_id: "reset-1".to_string(),
bandwidth_limit: 1024,
..base.clone()
};
assert!(same_replication_service(&resync_edit, &base));
for moved in [
BucketTarget {
endpoint: "other.example:9000".to_string(),
..base.clone()
},
BucketTarget {
target_bucket: "other".to_string(),
..base.clone()
},
BucketTarget {
secure: false,
..base.clone()
},
BucketTarget {
credentials: Some(Credentials {
access_key: "rotated".to_string(),
..Default::default()
}),
..base.clone()
},
] {
assert!(!same_replication_service(&moved, &base));
}
}
#[test]
fn version_identity_verdict_is_per_arn_and_forgotten_with_the_target() {
let sys = BucketTargetSys::default();
let arn = "arn:rustfs:replication:us-east-1:bucket:identity";
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
sys.record_version_identity_capability(arn, VersionIdentityCapability::MintsOwn);
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::MintsOwn);
assert_eq!(sys.version_identity_capability("other"), VersionIdentityCapability::Unknown);
// A rebuilt target may point at a different service.
sys.forget_version_identity_capability(arn);
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
}
#[test]
fn endpoint_health_key_preserves_explicit_port() {
let url = Url::parse("https://remote.example:9443").expect("url should parse");
@@ -66,6 +66,7 @@ pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
pub use replication_object_bridge::ReplicationObjectBridge;
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
pub(crate) use replication_object_decision_boundary::replication_etags_match;
pub use replication_object_decision_boundary::{
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
@@ -88,5 +89,6 @@ pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
pub use replication_target_boundary::SsecPassthroughCapability;
pub use replication_target_boundary::VersionIdentityCapability;
pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
@@ -56,6 +56,8 @@ use super::replication_storage_boundary::{
};
#[cfg(test)]
use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject};
#[cfg(test)]
use super::replication_target_boundary::VersionIdentityCapability;
use super::replication_target_boundary::{
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
RemotePutObjectResponse, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
@@ -63,7 +65,7 @@ use super::replication_target_boundary::{
replication_delete_marker_purge_remove_options, replication_delete_remove_options, replication_force_delete_remove_options,
replication_object_is_ssec_encrypted, replication_put_object_header_size, replication_put_object_options,
replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
ssec_passthrough_gate, version_identity_drifted,
ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted,
};
use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources;
@@ -123,6 +125,7 @@ const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_
const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
const EVENT_REPLICATION_DRIFTED_REPLICA_LOCATED: &str = "replication_drifted_replica_located";
const EVENT_REPLICATION_OBJECT_FAILED: &str = "replication_object_failed";
const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
@@ -332,6 +335,12 @@ fn verify_single_part_replica(
const REPLICA_ETAG_MISMATCH_ERROR: &str = "replica etag mismatch: the target persisted different bytes than were sent";
fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) {
// Every write refreshes the cached verdict, so the convergence fallback
// below (`replica_head_fallback`) knows whether a 404 on a
// version-addressed HEAD can mean "replica missing" on this target.
if let Some(capability) = version_identity_capability_from_put(source_version_id, assigned_version_id) {
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, capability);
}
if !version_identity_drifted(source_version_id, assigned_version_id) {
return;
}
@@ -404,11 +413,70 @@ async fn head_object_fallback(
) -> std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError> {
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
Ok(oi) => Ok(Some(oi)),
Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None),
Err(e) if head_object_not_found(&e) => Ok(None),
Err(e) => Err(e),
}
}
fn head_object_not_found(err: &HeadObjectSdkError) -> bool {
err.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(err, 404)
}
/// Second look at a replica whose version-addressed HEAD failed, for the two
/// target shapes where that failure is not a verdict on the replica:
///
/// - AWS-style 400/403 (the RustFS uuid is rejected as malformed): HEAD the
/// current version without a version id; callers compare ETags.
/// - 404 on a target known to mint its own version ids (the Wasabi shape,
/// rustfs/backlog#2340): the source id never existed there, so locate the
/// replica by exact key and ETag through ListObjectVersions and HEAD the id
/// the target assigned. Without this, every heal, MRF retry and
/// existing-object resync re-drive PUTs the object again and mints one
/// more target version.
///
/// `None` when the error stands as-is: a real miss on an adopting target, or
/// a target whose identity contract is still unknown. A failed lookup is
/// returned as a HEAD-shaped error so callers keep their "target operation
/// failed" handling (retry later) instead of re-driving the PUT.
async fn replica_head_fallback(
tgt_client: &TargetClient,
object: &str,
source_etag: Option<&str>,
err: &HeadObjectSdkError,
) -> Option<std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError>> {
if is_version_id_format_mismatch(err) {
return Some(head_object_fallback(tgt_client, object).await);
}
if !head_object_not_found(err)
|| !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable()
{
return None;
}
let etag = source_etag.filter(|etag| !etag.trim().is_empty())?;
Some(match tgt_client.find_version_by_etag(&tgt_client.bucket, object, etag).await {
Ok(Some(assigned_version_id)) => {
debug!(
event = EVENT_REPLICATION_DRIFTED_REPLICA_LOCATED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %tgt_client.bucket,
object = %object,
arn = %tgt_client.arn,
assigned_version_id = %assigned_version_id,
"Located replica by content identity on a target that mints its own version ids"
);
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, Some(assigned_version_id)).await {
Ok(oi) => Ok(Some(oi)),
// The located version disappeared between LIST and HEAD.
Err(e) if head_object_not_found(&e) => Ok(None),
Err(e) => Err(e),
}
}
Ok(None) => Ok(None),
Err(list_err) => Err(Box::new(SdkError::construction_failure(*list_err))),
})
}
/// Resolve the N2 fail-closed gate for an SSE-C passthrough attempt against
/// this target. Returns `Some(audit_required)` when replication may proceed;
/// on a freshly-flagged header-dropping target it settles `rinfo` as FAILED
@@ -1400,31 +1468,26 @@ async fn verify_resync_head_result(
(0, None)
}
}
Err(err) if is_version_id_format_mismatch(&err) => {
// AWS-style target rejects the RustFS UUID versionId
// (400). Re-verify without the versionId before
// concluding the object failed to replicate, instead
// of counting a well-replicated object as failed.
match head_object_fallback(target_client.as_ref(), &roi.name).await {
Ok(Some(_)) => {
Err(err) => {
// A version-addressed HEAD is not the last word on every target:
// re-verify through the fallback before counting a well-replicated
// object as failed (see `replica_head_fallback`).
match replica_head_fallback(target_client.as_ref(), &roi.name, roi.etag.as_deref(), &err).await {
Some(Ok(Some(_))) => {
st.replicated_count += 1;
st.replicated_size += roi.size;
(roi.size, None)
}
Ok(None) => {
Some(Ok(None)) | None => {
st.failed_count += 1;
(0, Some(err))
}
Err(e2) => {
Some(Err(e2)) => {
st.failed_count += 1;
(0, Some(e2))
}
}
}
Err(err) => {
st.failed_count += 1;
(0, Some(err))
}
}
}
@@ -3597,11 +3660,8 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
}
}
Err(e) => {
if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
// Object not on target yet → fall through to PUT.
} else if is_version_id_format_mismatch(&e) {
// Version-ID format mismatch: retry without versionId and compare ETags.
match head_object_fallback(&tgt_client, &object).await {
if let Some(fallback) = replica_head_fallback(&tgt_client, &object, object_info.etag.as_deref(), &e).await {
match fallback {
Ok(Some(oi)) if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) => {
if ssec_audit_required
&& !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo).await
@@ -3631,6 +3691,8 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
return rinfo;
}
}
} else if head_object_not_found(&e) {
// Object not on target yet → fall through to PUT.
} else {
rinfo.error = Some(e.to_string());
warn!(
@@ -4230,9 +4292,8 @@ async fn resolve_replicate_all_action(
}
}
Err(e) => {
if is_version_id_format_mismatch(&e) {
// Version-ID format mismatch: retry without versionId and compare ETags.
match head_object_fallback(tgt_client, object).await {
if let Some(fallback) = replica_head_fallback(tgt_client, object, object_info.etag.as_deref(), &e).await {
match fallback {
Ok(Some(oi)) => {
let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
if require_existing_target && !etags_match {
@@ -4284,7 +4345,7 @@ async fn resolve_replicate_all_action(
return None;
}
}
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
} else if head_object_not_found(&e) {
if require_existing_target {
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
@@ -5383,6 +5444,178 @@ mod tests {
ReplicationTargetStore::register_test_target(target).await;
}
const DRIFTED_ASSIGNED_VERSION_ID: &str = "001788697733811332140-fR6j6uXKV-";
const DRIFTED_ETAG: &str = "9a0364b9e99bb480dd25e1f0284c8555";
/// The Wasabi shape (rustfs/backlog#2340): a version-addressed HEAD with
/// the source uuid answers 404 (not the AWS 400), ListObjectVersions shows
/// the id the target minted, and a HEAD by that id succeeds. Serves exactly
/// `requests` connections and returns the request lines it saw.
fn spawn_drifted_target_server(requests: usize) -> (String, std::thread::JoinHandle<Vec<String>>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
let handle = std::thread::spawn(move || {
let mut seen = Vec::new();
for _ in 0..requests {
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
let mut request = [0_u8; 8192];
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
let text = String::from_utf8_lossy(&request[..bytes_read]).to_string();
let request_line = text.lines().next().unwrap_or_default().to_string();
let response = if request_line.starts_with("HEAD ") {
if request_line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}")) {
format!(
"HTTP/1.1 200 OK\r\nETag: \"{DRIFTED_ETAG}\"\r\nContent-Length: 4\r\nLast-Modified: Sun, 06 Sep 2026 10:00:00 GMT\r\nConnection: close\r\n\r\n"
)
} else {
"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()
}
} else if request_line.starts_with("GET ") && request_line.contains("versions") {
let body = format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>target-bucket</Name><Prefix>object</Prefix><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Version><Key>object</Key><VersionId>{DRIFTED_ASSIGNED_VERSION_ID}</VersionId><IsLatest>true</IsLatest><LastModified>2026-09-06T10:00:00.000Z</LastModified><ETag>&quot;{DRIFTED_ETAG}&quot;</ETag><Size>4</Size><StorageClass>STANDARD</StorageClass></Version></ListVersionsResult>"
);
format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
)
} else {
"HTTP/1.1 500 Unexpected\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()
};
stream
.write_all(response.as_bytes())
.expect("test HTTP response should be written");
seen.push(request_line);
}
seen
});
(endpoint, handle)
}
fn drifted_roi_and_object() -> (ReplicateObjectInfo, ObjectInfo) {
let roi = ReplicateObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
op_type: ReplicationType::Heal,
replication_status: ReplicationStatusType::Pending,
etag: Some(DRIFTED_ETAG.to_string()),
size: 4,
..Default::default()
};
let object_info = ObjectInfo {
bucket: roi.bucket.clone(),
name: roi.name.clone(),
version_id: roi.version_id,
etag: Some(DRIFTED_ETAG.to_string()),
size: 4,
..Default::default()
};
(roi, object_info)
}
#[tokio::test]
async fn heal_redrive_locates_replica_by_etag_on_target_that_mints_own_version_ids() {
let (endpoint, server) = spawn_drifted_target_server(3);
let target = test_target_client(endpoint);
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
let (roi, object_info) = drifted_roi_and_object();
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(
matches!(action, Some((ReplicationAction::None, _))),
"a replica located by content identity must not be re-driven: {action:?}"
);
assert!(rinfo.error.is_none(), "{:?}", rinfo.error);
let seen = server.join().expect("test HTTP server should finish");
assert_eq!(seen.len(), 3, "HEAD by source id, ListObjectVersions, HEAD by assigned id: {seen:?}");
assert!(seen[0].starts_with("HEAD ") && seen[0].contains(&roi.version_id.unwrap().to_string()));
assert!(seen[1].starts_with("GET ") && seen[1].contains("prefix=object"), "{}", seen[1]);
assert!(seen[2].starts_with("HEAD ") && seen[2].contains(DRIFTED_ASSIGNED_VERSION_ID));
}
#[tokio::test]
async fn head_not_found_still_replicates_when_identity_contract_is_unknown() {
// Same 404, but the target never revealed whether it adopts version
// ids: a 404 keeps meaning "replica missing" (adopting targets, e.g.
// RustFS/MinIO peers, must not skip a genuinely missing version).
let (endpoint, server) = spawn_head_status_server(404);
let target = test_target_client(endpoint);
let (roi, object_info) = drifted_roi_and_object();
let mut rinfo = replicate_all_target_info(&roi, &target);
let action = resolve_replicate_all_action(
ReplicateAllActionContext {
roi: &roi,
tgt_client: &target,
bucket: &roi.bucket,
object: &roi.name,
start_time: OffsetDateTime::now_utc(),
ssec_audit_required: false,
},
object_info,
&mut rinfo,
)
.await;
assert!(matches!(action, Some((ReplicationAction::All, _))));
server.join().expect("test HTTP server should finish");
}
#[tokio::test]
async fn resync_verification_counts_drifted_replica_as_replicated() {
let (endpoint, server) = spawn_drifted_target_server(3);
let target = test_target_client(endpoint);
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
let (roi, _) = drifted_roi_and_object();
let mut st = TargetReplicationResyncStatus::default();
let head_result =
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
assert!(err.is_none(), "{err:?}");
assert_eq!((size, st.replicated_count, st.failed_count), (4, 1, 0));
server.join().expect("test HTTP server should finish");
}
#[test]
fn put_response_audit_records_identity_verdict() {
let target = test_target_client("http://127.0.0.1:1".to_string());
let source = Uuid::new_v4().to_string();
audit_target_version_identity(&target, &source, Some(DRIFTED_ASSIGNED_VERSION_ID));
assert_eq!(
ReplicationTargetStore::version_identity_capability(&target.arn),
VersionIdentityCapability::MintsOwn
);
audit_target_version_identity(&target, &source, Some(&source));
assert_eq!(
ReplicationTargetStore::version_identity_capability(&target.arn),
VersionIdentityCapability::Adopts
);
// An unversioned write carries no contract and must not overwrite it.
audit_target_version_identity(&target, "null", None);
assert_eq!(
ReplicationTargetStore::version_identity_capability(&target.arn),
VersionIdentityCapability::Adopts
);
}
#[test]
fn resync_admission_configuration_is_bounded() {
assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS");
@@ -48,6 +48,7 @@ pub use rustfs_replication::{ObjectLockIntegrity, object_lock_put_integrity};
pub(crate) use rustfs_replication::{
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
};
pub use rustfs_replication::{VersionIdentityCapability, version_identity_capability_from_put};
use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error, Result};
@@ -192,6 +193,14 @@ impl ReplicationTargetStore {
.await
}
pub(crate) fn version_identity_capability(arn: &str) -> VersionIdentityCapability {
BucketTargetSys::get().version_identity_capability(arn)
}
pub(crate) fn record_version_identity_capability(arn: &str, capability: VersionIdentityCapability) {
BucketTargetSys::get().record_version_identity_capability(arn, capability)
}
#[cfg(test)]
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
BucketTargetSys::get().arn_remotes_map.write().await.insert(
+4 -3
View File
@@ -65,9 +65,10 @@ pub use multipart::{
};
pub use object::{
ObjectLockIntegrity, ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity, replication_action_for_target,
replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
target_is_newer_than_source_null_version, version_identity_drifted,
VersionIdentityCapability, content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity,
replication_action_for_target, replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present,
ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_capability_from_put,
version_identity_drifted,
};
pub use operation::{
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
+79 -9
View File
@@ -234,18 +234,62 @@ fn comparable_metadata(metadata: Option<&HashMap<String, String>>) -> HashMap<St
/// real (non-nil) version uuid, and drift means the target answered with
/// anything else — including nothing at all.
pub fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool {
if source_version_id.is_empty() {
return false;
version_identity_capability_from_put(source_version_id, assigned_version_id) == Some(VersionIdentityCapability::MintsOwn)
}
/// Whether a replication target adopts the source version id it is handed on
/// PutObject / CompleteMultipartUpload, or mints its own.
///
/// A target that mints its own ids (AWS S3, Wasabi, Impossible Cloud) still
/// stores the bytes, but every later version-addressed request from the
/// source names an id the target never had. Its HEAD then answers 404 —
/// indistinguishable from a replica that is really missing — so a heal, MRF
/// retry or existing-object resync re-drive would PUT the object again and
/// mint yet another target version (rustfs/backlog#2340). The replication
/// worker learns the verdict from each PUT response (and replication-check's
/// VersionFidelity phase) and, once `MintsOwn` is known, locates a replica by
/// exact key and ETag before concluding that it is missing. The verdict cache
/// is owned by the runtime's bucket target system; this crate owns only the
/// vocabulary and the judgment.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum VersionIdentityCapability {
#[default]
Unknown,
Adopts,
MintsOwn,
}
impl VersionIdentityCapability {
/// True when a 404 from a version-addressed HEAD on this target cannot be
/// read as "replica missing": the source-side id was never the target's.
pub fn version_addressing_unreliable(self) -> bool {
self == VersionIdentityCapability::MintsOwn
}
}
/// Judge the identity contract from one replication write: `None` when no
/// contract applies (the source addressed no real version — an empty or nil
/// uuid travels as the literal "null", unversioned-source semantics),
/// otherwise whether the target echoed the source id or answered with
/// anything else — including nothing at all.
pub fn version_identity_capability_from_put(
source_version_id: &str,
assigned_version_id: Option<&str>,
) -> Option<VersionIdentityCapability> {
if source_version_id.is_empty() {
return None;
}
// A nil source uuid travels as the literal "null" (unversioned-source
// semantics); no identity contract applies to it.
if uuid::Uuid::parse_str(source_version_id)
.map(|uuid| uuid.is_nil())
.unwrap_or(true)
{
return false;
return None;
}
assigned_version_id != Some(source_version_id)
Some(if assigned_version_id == Some(source_version_id) {
VersionIdentityCapability::Adopts
} else {
VersionIdentityCapability::MintsOwn
})
}
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
@@ -377,9 +421,9 @@ mod tests {
use super::{
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match,
single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
target_is_newer_than_source_null_version, version_identity_drifted,
VersionIdentityCapability, content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target,
replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
target_is_newer_than_source_null_version, version_identity_capability_from_put, version_identity_drifted,
};
use crate::filemeta::{ReplicationAction, ReplicationType};
use crate::http::AMZ_OBJECT_LOCK_MODE;
@@ -508,6 +552,32 @@ mod tests {
}
}
#[test]
fn version_identity_capability_is_judged_only_for_real_source_versions() {
let source = "8e4d2f4c-2d5c-4f1b-9d0a-9c8b7a6f5e4d";
assert_eq!(
version_identity_capability_from_put(source, Some(source)),
Some(VersionIdentityCapability::Adopts)
);
// Wasabi / AWS shape: a minted id, or no id at all, both mean the
// source-side id is not addressable on the target.
assert_eq!(
version_identity_capability_from_put(source, Some("001788697733811332140-fR6j6uXKV-")),
Some(VersionIdentityCapability::MintsOwn)
);
assert_eq!(
version_identity_capability_from_put(source, None),
Some(VersionIdentityCapability::MintsOwn)
);
// No contract for an unversioned source write.
assert_eq!(version_identity_capability_from_put("", Some("anything")), None);
assert_eq!(version_identity_capability_from_put("00000000-0000-0000-0000-000000000000", None), None);
assert_eq!(version_identity_capability_from_put("null", Some("null")), None);
assert!(VersionIdentityCapability::MintsOwn.version_addressing_unreliable());
assert!(!VersionIdentityCapability::Adopts.version_addressing_unreliable());
assert!(!VersionIdentityCapability::Unknown.version_addressing_unreliable());
}
#[test]
fn replication_target_offline_error_classifier_is_network_scoped() {
assert!(is_replication_target_offline_error("put_object dispatch failure: connector error"));
@@ -62,7 +62,6 @@ Object keys are stored as file-system paths under each drive (`{drive}/{bucket}/
| Behavior | RustFS | AWS S3 | Why |
|---|---|---|---|
| Object key with a `.` or `..` path segment, or an empty segment (`//`), such as `a//b/./c/../d` | `400 InvalidArgument` (`check_object_args` in `crates/ecstore/src/bucket/utils.rs`, mirroring MinIO `IsValidObjectPrefix`) | Accepted as an opaque key | A `..` segment would resolve to a parent directory and `.`/`//` segments would alias other keys on disk; encoding them would change the MinIO-compatible on-disk format. |
| Directory marker (key ending in `/`, with or without a body) in a versioned bucket | Stored as the null version: `PutObject`/`HeadObject` report version id `00000000-0000-0000-0000-000000000000`, `ListObjectVersions` reports `null`, and a later PUT of the same key overwrites in place (`put_opts` in `rustfs/src/storage/options.rs`, mirroring MinIO `putOpts`: "for directory objects skip creating new versions") | A real version id per PUT, with a version history | The marker only exists to make an empty prefix listable; keeping a history for it would leave hidden versions behind every prefix delete. Replication still copies the marker as its null version (`test_bucket_replication_replicates_directory_marker_in_versioned_bucket` in `crates/e2e_test/src/replication_extension_test.rs`). |
## Update Rule
+13 -1
View File
@@ -18,7 +18,7 @@ use super::storage_api::bucket::replication::{self, BucketReplicationResyncStatu
use super::storage_api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
use super::storage_api::bucket::target_sys::{
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, SsecPassthroughCapability, TargetClient,
append_version_id_query,
VersionIdentityCapability, append_version_id_query,
};
use super::storage_api::bucket::versioning_sys::BucketVersioningSys;
use super::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _};
@@ -2100,6 +2100,18 @@ async fn check_replication_target(
}
_ => {}
}
// Same for the identity verdict: once a target is known to mint its own
// version ids, the worker locates replicas by content identity instead of
// re-driving PUTs whenever a version-addressed HEAD answers 404.
match (result.phases.version_fidelity.status, result.phases.version_fidelity.code) {
("OK", _) => {
BucketTargetSys::get().record_version_identity_capability(&target.arn, VersionIdentityCapability::Adopts);
}
("FAILED", Some(REPLICATION_CHECK_CODE_VERSION_MISMATCH)) => {
BucketTargetSys::get().record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
}
_ => {}
}
result
}
+1
View File
@@ -211,6 +211,7 @@ pub(crate) mod bucket_target_sys {
pub(crate) type RemoveObjectOptions = super::ecstore_bucket::bucket_target_sys::RemoveObjectOptions;
pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError;
pub(crate) type SsecPassthroughCapability = super::ecstore_bucket::bucket_target_sys::SsecPassthroughCapability;
pub(crate) type VersionIdentityCapability = super::ecstore_bucket::bucket_target_sys::VersionIdentityCapability;
pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient;
}