mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-15 17:43:13 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 74b1189016 | |||
| 77a536093e | |||
| 8d191c7653 | |||
| faf735896b | |||
| 72fd7339c9 | |||
| 71e83aeec4 | |||
| 9138c24571 |
@@ -94,6 +94,7 @@ jobs:
|
||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -118,6 +119,7 @@ jobs:
|
||||
short_sha=""
|
||||
is_prerelease=false
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
@@ -137,6 +139,7 @@ jobs:
|
||||
# Extract version info from commit message or use commit SHA
|
||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||
source_ref="$HEAD_SHA"
|
||||
|
||||
# Determine build type based on triggering workflow event and ref
|
||||
triggering_event="$TRIGGERING_EVENT"
|
||||
@@ -261,6 +264,23 @@ jobs:
|
||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||
tag_ref="refs/tags/$input_version"
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
if [[ "$input_version" == v* ]]; then
|
||||
tag_ref="refs/tags/${input_version#v}"
|
||||
else
|
||||
tag_ref="refs/tags/v$input_version"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
echo "❌ Release tag not found for Docker build: $input_version"
|
||||
exit 1
|
||||
fi
|
||||
source_ref="$tag_ref"
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
@@ -271,6 +291,7 @@ jobs:
|
||||
echo "short_sha=$short_sha"
|
||||
echo "is_prerelease=$is_prerelease"
|
||||
echo "create_latest=$create_latest"
|
||||
echo "source_ref=$source_ref"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Docker Build Summary:"
|
||||
@@ -281,6 +302,7 @@ jobs:
|
||||
echo " - Short SHA: $short_sha"
|
||||
echo " - Is prerelease: $is_prerelease"
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
@@ -308,6 +330,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
@@ -397,7 +420,8 @@ jobs:
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
||||
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||
|
||||
@@ -76,6 +76,18 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-tagging-timestamp",
|
||||
];
|
||||
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-minio-source-replication-retention-timestamp",
|
||||
];
|
||||
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||
|
||||
@@ -118,6 +130,25 @@ pub enum FaultAction {
|
||||
WrongEtag,
|
||||
}
|
||||
|
||||
/// Replication LWW timestamp headers observed on a request, journaled so
|
||||
/// sender-side tests can assert what a real target would receive.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ReplicationTimestampHeaders {
|
||||
pub tagging: Option<String>,
|
||||
pub retention: Option<String>,
|
||||
pub legalhold: Option<String>,
|
||||
}
|
||||
|
||||
impl ReplicationTimestampHeaders {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Credential-free request metadata retained for deterministic assertions.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestRecord {
|
||||
@@ -131,6 +162,7 @@ pub struct RequestRecord {
|
||||
pub part_number: Option<i32>,
|
||||
pub content_length: Option<u64>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
@@ -536,7 +568,15 @@ impl S3Access for FaultAccess {
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok());
|
||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||
let fault = record_request(
|
||||
&self.control,
|
||||
operation,
|
||||
context.method().clone(),
|
||||
parsed,
|
||||
content_length,
|
||||
replication_timestamps,
|
||||
);
|
||||
if let Some(RequestFault {
|
||||
action: FaultAction::Status(status),
|
||||
..
|
||||
@@ -589,6 +629,7 @@ fn record_request(
|
||||
method: Method,
|
||||
parsed: ParsedRequest,
|
||||
content_length: Option<u64>,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
) -> Option<RequestFault> {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
@@ -613,6 +654,7 @@ fn record_request(
|
||||
part_number: parsed.part_number,
|
||||
content_length,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps,
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
@@ -1699,6 +1741,52 @@ mod tests {
|
||||
.await?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("plain")
|
||||
.body(ByteStream::from_static(b"plain"))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key("stamped")
|
||||
.body(ByteStream::from_static(b"stamped"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let requests = target.requests();
|
||||
let plain = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
||||
.expect("plain PUT must be journaled");
|
||||
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
||||
|
||||
let stamped = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
||||
.expect("stamped PUT must be journaled");
|
||||
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
||||
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
||||
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! assert_sdk_error {
|
||||
($error:expr, $status:expr, $code:expr) => {{
|
||||
let error = &$error;
|
||||
@@ -2985,6 +3073,7 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
Some(0),
|
||||
ReplicationTimestampHeaders::default(),
|
||||
);
|
||||
}
|
||||
let records = lock(&control).requests.clone();
|
||||
@@ -3006,6 +3095,7 @@ mod tests {
|
||||
part_number: None,
|
||||
},
|
||||
None,
|
||||
ReplicationTimestampHeaders::default(),
|
||||
);
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
|
||||
@@ -58,7 +58,9 @@ use rustfs_utils::http::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||
insert_header,
|
||||
};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -1476,9 +1478,12 @@ impl Default for AdvancedPutOptions {
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
source_mtime: OffsetDateTime::now_utc(),
|
||||
replication_request: false,
|
||||
retention_timestamp: OffsetDateTime::now_utc(),
|
||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
||||
// UNIX_EPOCH means "never modified": header() must not emit a
|
||||
// timestamp header for it, otherwise a receiver would treat an
|
||||
// unset category as a modification made right now.
|
||||
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||
replication_validity_check: false,
|
||||
}
|
||||
}
|
||||
@@ -1675,6 +1680,16 @@ impl PutObjectOptions {
|
||||
);
|
||||
}
|
||||
|
||||
for (suffix, timestamp) in [
|
||||
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
||||
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
||||
] {
|
||||
if timestamp.unix_timestamp() != 0 {
|
||||
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
||||
}
|
||||
}
|
||||
|
||||
if self.internal.replication_request {
|
||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
@@ -2842,6 +2857,57 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_carry_replication_timestamp_headers() {
|
||||
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
||||
// last-writer-wins on these headers (object-api-options.go parses them
|
||||
// as RFC3339); a replica without them loses every conflict resolution.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
||||
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
||||
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
||||
opts.internal.tagging_timestamp = tagging;
|
||||
opts.internal.retention_timestamp = retention;
|
||||
opts.internal.legalhold_timestamp = legalhold;
|
||||
|
||||
let header = opts.header();
|
||||
for (suffix, expected) in [
|
||||
("source-replication-tagging-timestamp", tagging),
|
||||
("source-replication-retention-timestamp", retention),
|
||||
("source-replication-legalhold-timestamp", legalhold),
|
||||
] {
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
||||
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
||||
"replication put requests must carry the {suffix} header"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_omit_unset_replication_timestamps() {
|
||||
// UNIX_EPOCH means "never modified on the source"; sending it would
|
||||
// make the receiver treat an unset category as a fresh modification.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.replication_request = true;
|
||||
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||
|
||||
let header = opts.header();
|
||||
for suffix in [
|
||||
"source-replication-tagging-timestamp",
|
||||
"source-replication-retention-timestamp",
|
||||
"source-replication-legalhold-timestamp",
|
||||
] {
|
||||
assert!(
|
||||
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
||||
"unset {suffix} must not be sent to replication targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||
let sys = BucketTargetSys::default();
|
||||
|
||||
@@ -259,15 +259,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
|
||||
if !tags.is_empty() {
|
||||
put_options.user_tags = tags;
|
||||
put_options.internal.tagging_timestamp =
|
||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
};
|
||||
}
|
||||
}
|
||||
// Load the stored tagging timestamp independently of whether any tags
|
||||
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
||||
// key, and the deletion's LWW timestamp must still reach the replica.
|
||||
// With no stored key, fall back to mod_time only while tags exist
|
||||
// (MinIO parity); a tagless object without the key was never tagged and
|
||||
// keeps the epoch default (no header).
|
||||
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
||||
{
|
||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||
} else if !put_options.user_tags.is_empty() {
|
||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
let metadata = &*object_info.user_defined;
|
||||
|
||||
@@ -694,6 +702,44 @@ mod tests {
|
||||
assert!(options.internal.replication_request);
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging leaves the object tagless but stamps the
|
||||
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
||||
/// still be loaded (and therefore sent) so the replica can order the
|
||||
/// deletion against concurrent tag edits.
|
||||
#[test]
|
||||
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||
|
||||
assert!(options.user_tags.is_empty());
|
||||
assert_eq!(
|
||||
options.internal.tagging_timestamp,
|
||||
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
||||
"the stored tagging timestamp must load independently of remaining tags"
|
||||
);
|
||||
|
||||
// A tagless object without the stored key was never tagged: the epoch
|
||||
// default keeps the header unsent.
|
||||
let untagged = ObjectInfo {
|
||||
user_tags: Arc::new(String::new()),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
||||
version_id: Some(Uuid::nil()),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
||||
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||
|
||||
@@ -277,6 +277,12 @@ pub struct ObjectOptions {
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
pub replication_request: bool,
|
||||
/// Source-cluster LWW timestamps carried by an authorized replication
|
||||
/// request; None when the source never modified the category. Only the
|
||||
/// replication-authorized options builders may set these.
|
||||
pub replication_tagging_timestamp: Option<OffsetDateTime>,
|
||||
pub replication_retention_timestamp: Option<OffsetDateTime>,
|
||||
pub replication_legalhold_timestamp: Option<OffsetDateTime>,
|
||||
/// Authorized SSE-C replication passthrough: the body is already
|
||||
/// ciphertext, so the write path must not encrypt or compress it and
|
||||
/// stores the restored encryption metadata verbatim. Only the
|
||||
|
||||
@@ -225,8 +225,6 @@ async fn nothing_readable_leaves_the_bundle_unwrapped() {
|
||||
"artifact {} carries the raw on-disk record",
|
||||
artifact.path
|
||||
);
|
||||
// A cheap structural check too: an encrypted payload is not JSON.
|
||||
assert_ne!(payload.first(), Some(&b'{'), "artifact {} looks like plaintext JSON", artifact.path);
|
||||
}
|
||||
|
||||
// The manifest itself is not encrypted, so assert directly that it carries
|
||||
|
||||
@@ -50,6 +50,14 @@ pub const SUFFIX_SOURCE_DELETEMARKER: &str = "source-deletemarker";
|
||||
pub const SUFFIX_SOURCE_PROXY_REQUEST: &str = "source-proxy-request";
|
||||
pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request";
|
||||
pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
|
||||
// LWW timestamps for replicated tag/retention/legal-hold modifications. MinIO
|
||||
// declares these with mixed case (internal/http/headers.go:
|
||||
// X-Minio-Source-Replication-Tagging-Timestamp / -Retention-Timestamp /
|
||||
// -LegalHold-Timestamp); HTTP header names compare case-insensitively, so the
|
||||
// lowercase suffix forms interoperate. Values are RFC3339 on the wire.
|
||||
pub const SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP: &str = "source-replication-tagging-timestamp";
|
||||
pub const SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP: &str = "source-replication-retention-timestamp";
|
||||
pub const SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP: &str = "source-replication-legalhold-timestamp";
|
||||
pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
|
||||
|
||||
/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO.
|
||||
@@ -196,6 +204,27 @@ mod tests {
|
||||
assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_timestamp_headers_match_minio_wire_names() {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z");
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, "2026-01-02T03:04:06Z");
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, "2026-01-02T03:04:07Z");
|
||||
|
||||
// The exact names MinIO's object-api-options.go reads (its Get()
|
||||
// canonicalizes case, so a case-insensitive match is wire-equivalent).
|
||||
for name in [
|
||||
"X-Minio-Source-Replication-Tagging-Timestamp",
|
||||
"X-Minio-Source-Replication-Retention-Timestamp",
|
||||
"X-Minio-Source-Replication-LegalHold-Timestamp",
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
] {
|
||||
assert!(headers.contains_key(name), "replication timestamp header {name} must be written");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -659,9 +659,6 @@ mod test {
|
||||
// Port should be in valid range (u16 max is always <= 65535)
|
||||
assert!(port1 > 0);
|
||||
assert!(port2 > 0);
|
||||
|
||||
// Different calls should typically return different ports
|
||||
assert_ne!(port1, port2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1067,9 +1067,13 @@ fn parse_site_replication_state(data: &[u8]) -> S3Result<SiteReplicationState> {
|
||||
state.peers = normalize_peer_map_by_identity(state.peers);
|
||||
// A peer-edit high-water mark only fences a CURRENT peer. A site that
|
||||
// leaves drops below two peers, which clears its own state object and
|
||||
// restarts its generation counter at zero — a mark left over from the
|
||||
// previous membership would then reject every edit it sends after it
|
||||
// rejoins. Dropping departed origins on load also keeps the map bounded.
|
||||
// restarts its generation counter — a mark left over from the previous
|
||||
// membership must not reject the edits it sends after it rejoins. This
|
||||
// pruning covers departures THIS site observed; an origin removed
|
||||
// unilaterally elsewhere stays in this peer map with its mark, and the
|
||||
// wall-clock floor in `next_peer_edit_generation` is what lifts its
|
||||
// restarted counter over that mark. Dropping departed origins on load
|
||||
// also keeps the map bounded.
|
||||
state
|
||||
.applied_edit_generations
|
||||
.retain(|origin, _| state.peers.contains_key(origin));
|
||||
@@ -5935,11 +5939,51 @@ fn summarize_peer_error_detail(detail: &str) -> String {
|
||||
summary
|
||||
}
|
||||
|
||||
/// Allocate the next peer-edit generation. Called inside the state
|
||||
/// transaction, so the counter is handed out under the distributed
|
||||
/// state-object lock and two nodes of this site can never take the same one.
|
||||
/// The wall clock in unix nanoseconds, clamped into u64. A pre-1970 (or
|
||||
/// post-2554) clock yields 0, which makes the hybrid allocation below
|
||||
/// degrade to the plain `previous + 1` counter — monotone, never panicking.
|
||||
fn edit_generation_wall_clock() -> u64 {
|
||||
u64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Allocate the next peer-edit generation as a hybrid logical clock:
|
||||
/// `max(wall clock in unix nanoseconds, previous + 1)`. Called inside the
|
||||
/// state transaction, so the value is handed out under the distributed
|
||||
/// state-object lock and two nodes of this site can never take the same one
|
||||
/// (`previous + 1` keeps the sequence strictly increasing even when two
|
||||
/// allocations land in one clock tick, and keeps it monotone on a node
|
||||
/// whose clock stepped backwards mid-lifetime).
|
||||
///
|
||||
/// The wall-clock floor is what survives the counter's death. A site
|
||||
/// removed while unreachable — the receiver never dropped it from its peer
|
||||
/// map, so the load-time mark pruning in `parse_site_replication_state`
|
||||
/// never fired — that later rejoins recreates its state object with the
|
||||
/// counter back at zero. A plain counter would then hand out generations
|
||||
/// below the receiver's stale high-water mark and every delivery would be
|
||||
/// silently fenced until the counter caught up. Jumping to wall time clears
|
||||
/// that mark: every value the deleted lifetime handed out was capped by the
|
||||
/// wall clock at its own allocation (or by a prior lifetime's cap, applied
|
||||
/// inductively), so the recreated lifetime's first allocation exceeds them
|
||||
/// all — while a pre-removal delivery still in flight stays below the new
|
||||
/// floor and remains correctly fenced. Marks recorded by pre-hybrid
|
||||
/// receivers (small plain-counter values) sit far below any wall-clock
|
||||
/// value, so a restarted origin passes those too — the fix needs only the
|
||||
/// sender upgraded, nothing on the wire or in the receiver changed.
|
||||
///
|
||||
/// A wall clock that regresses across a delete/recreate (the recreating
|
||||
/// node's clock behind the clock that fed the previous lifetime) mints
|
||||
/// below the stale mark and the origin stays fenced — but only until real
|
||||
/// time passes the previous lifetime's last allocation, because every later
|
||||
/// allocation takes the wall-clock floor again. Bounded by the skew,
|
||||
/// self-healing, and no rollback window beyond the plain counter's: a
|
||||
/// delivery applies only at or above the receiver's mark, so the one
|
||||
/// cross-lifetime interleaving that can apply stale content — a
|
||||
/// pre-removal delivery whose generation lands above everything the
|
||||
/// regressed new lifetime has minted — required the same straggler landing
|
||||
/// above the mark under the plain counter, where the recreated counter's
|
||||
/// low restart made it strictly easier to hit.
|
||||
fn next_peer_edit_generation(state: &mut SiteReplicationState) -> u64 {
|
||||
state.edit_generation = state.edit_generation.saturating_add(1);
|
||||
state.edit_generation = edit_generation_wall_clock().max(state.edit_generation.saturating_add(1));
|
||||
state.edit_generation
|
||||
}
|
||||
|
||||
@@ -13244,6 +13288,104 @@ mod tests {
|
||||
assert!(!peer_edit_delivery_is_stale(&reloaded, "origin-site", 1));
|
||||
}
|
||||
|
||||
/// The unilateral-removal rejoin gap the hybrid clock closes. The origin
|
||||
/// was removed while unreachable, but THIS site never dropped it from
|
||||
/// its peer map, so the load-time mark pruning never fired and the mark
|
||||
/// from the previous membership survives. The origin's recreated state
|
||||
/// object restarts its counter, and with a plain `previous + 1` counter
|
||||
/// every delivery it sent — generations 1, 2, … below the stale mark —
|
||||
/// would be silently acked-and-dropped until the counter caught up. The
|
||||
/// wall-clock floor in `next_peer_edit_generation` lifts the restarted
|
||||
/// counter over every value the deleted lifetime handed out. Reverting
|
||||
/// the allocation to the plain counter (dropping the wall-clock max)
|
||||
/// turns the not-stale assertion red.
|
||||
#[test]
|
||||
fn hybrid_generation_unfences_a_rejoined_origin_whose_counter_restarted() {
|
||||
// First lifetime of the origin's state object: two allocations, both
|
||||
// capped by the wall clock at their own allocation.
|
||||
let mut first_life = SiteReplicationState::default();
|
||||
let straggler = next_peer_edit_generation(&mut first_life);
|
||||
let last_applied = next_peer_edit_generation(&mut first_life);
|
||||
assert!(last_applied > straggler, "allocations must be strictly increasing");
|
||||
|
||||
// The receiver applied up to `last_applied` and keeps the origin in
|
||||
// its peer map across the unilateral removal — reloading must keep
|
||||
// the mark, which is exactly why pruning cannot cover this case.
|
||||
let mut receiver = SiteReplicationState::default();
|
||||
receiver.peers.insert(
|
||||
"origin-site".to_string(),
|
||||
PeerInfo {
|
||||
deployment_id: "origin-site".to_string(),
|
||||
..peer("origin", "https://origin.example:9000")
|
||||
},
|
||||
);
|
||||
record_applied_peer_edit_generation(&mut receiver, "origin-site", last_applied);
|
||||
let mut receiver = parse_site_replication_state(&serde_json::to_vec(&receiver).expect("serialize")).expect("reload");
|
||||
assert_eq!(receiver.applied_edit_generations.get("origin-site"), Some(&last_applied));
|
||||
|
||||
// The origin rejoins with a RECREATED state object: counter back at
|
||||
// zero. The wall-clock floor must lift its first allocation over the
|
||||
// previous lifetime's mark…
|
||||
let mut second_life = SiteReplicationState::default();
|
||||
let restarted = next_peer_edit_generation(&mut second_life);
|
||||
assert!(
|
||||
!peer_edit_delivery_is_stale(&receiver, "origin-site", restarted),
|
||||
"the recreated lifetime's first allocation ({restarted}) must not be fenced by the previous lifetime's mark ({last_applied})"
|
||||
);
|
||||
record_applied_peer_edit_generation(&mut receiver, "origin-site", restarted);
|
||||
|
||||
// …while a pre-removal delivery still in flight stays below the new
|
||||
// floor and remains correctly fenced — the rollback the fence exists
|
||||
// to reject.
|
||||
assert!(
|
||||
peer_edit_delivery_is_stale(&receiver, "origin-site", straggler),
|
||||
"a pre-removal in-flight delivery ({straggler}) must stay fenced after the rejoin"
|
||||
);
|
||||
}
|
||||
|
||||
/// Marks recorded before the hybrid clock existed are small plain-counter
|
||||
/// values, far below any wall-clock allocation: a restarted origin passes
|
||||
/// them as soon as the SENDER runs the hybrid clock — nothing changes on
|
||||
/// the wire or in the receiver, so pre-hybrid receivers get the fix too.
|
||||
/// The other direction is unchanged: among plain-counter values the
|
||||
/// generation order still fences the delivery that lost the race.
|
||||
#[test]
|
||||
fn hybrid_generation_passes_marks_recorded_by_plain_counter_receivers() {
|
||||
let mut receiver = SiteReplicationState::default();
|
||||
record_applied_peer_edit_generation(&mut receiver, "origin-site", 57);
|
||||
assert!(peer_edit_delivery_is_stale(&receiver, "origin-site", 56));
|
||||
assert!(!peer_edit_delivery_is_stale(&receiver, "origin-site", 57));
|
||||
|
||||
let mut rejoined = SiteReplicationState::default();
|
||||
let restarted = next_peer_edit_generation(&mut rejoined);
|
||||
assert!(
|
||||
!peer_edit_delivery_is_stale(&receiver, "origin-site", restarted),
|
||||
"a wall-clock allocation ({restarted}) must clear a plain-counter mark (57)"
|
||||
);
|
||||
}
|
||||
|
||||
/// The `previous + 1` half of the hybrid clock: allocations stay strictly
|
||||
/// increasing even when the wall clock cannot move them forward — two
|
||||
/// allocations inside one clock tick, or a clock that stepped backwards
|
||||
/// mid-lifetime (a counter already ahead of the wall clock advances by
|
||||
/// exactly one per allocation instead of jumping back). Dropping the
|
||||
/// `previous + 1` half (allocating bare wall time) turns this red.
|
||||
#[test]
|
||||
fn hybrid_generation_is_strictly_increasing_when_the_clock_stalls() {
|
||||
let mut state = SiteReplicationState {
|
||||
// A counter far ahead of any wall clock this test will see.
|
||||
edit_generation: u64::MAX / 2,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(next_peer_edit_generation(&mut state), u64::MAX / 2 + 1);
|
||||
assert_eq!(next_peer_edit_generation(&mut state), u64::MAX / 2 + 2);
|
||||
// Saturation pins at the ceiling instead of wrapping; the equal-value
|
||||
// escape (`applied > generation` is false for equal) keeps deliveries
|
||||
// applying rather than fencing the origin out.
|
||||
state.edit_generation = u64::MAX;
|
||||
assert_eq!(next_peer_edit_generation(&mut state), u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_stats_for_state_counts_pending_and_failed() {
|
||||
let state = SiteReplicationState {
|
||||
@@ -16044,10 +16186,77 @@ mod tests {
|
||||
generations.len(),
|
||||
"two nodes took the same edit generation, so their deliveries cannot be ordered: {generations:?}"
|
||||
);
|
||||
// The hybrid clock allocates `max(wall nanos, previous + 1)` — the
|
||||
// persisted counter is the largest allocation, and the `+ 1` half
|
||||
// keeps allocations distinct even inside one clock tick.
|
||||
assert_eq!(
|
||||
Some(&load_site_replication_state().await.expect("reload").edit_generation),
|
||||
unique.last(),
|
||||
"the persisted counter must be the largest allocation handed out"
|
||||
);
|
||||
}
|
||||
|
||||
/// The unilateral-removal rejoin, end to end across the state object's
|
||||
/// real lifecycle: dropping below two peers clears the object (the
|
||||
/// counter dies with it), and the recreated object's first allocation —
|
||||
/// raced by two nodes — must clear the previous lifetime's values via
|
||||
/// the wall-clock floor, so a receiver still holding the old mark
|
||||
/// accepts the restarted counter instead of fencing it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_recreated_state_object_allocates_over_the_previous_lifetimes_mark() {
|
||||
publish_ready_iam_context().await;
|
||||
let seed = || SiteReplicationState {
|
||||
peers: ["site-a", "site-b"]
|
||||
.into_iter()
|
||||
.map(|name| (name.to_string(), peer(name, &format!("https://{name}.example:9000"))))
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
save_site_replication_state(&seed()).await.expect("seed state");
|
||||
let straggler = update_site_replication_state(|state| Ok(next_peer_edit_generation(state)))
|
||||
.await
|
||||
.expect("first-life allocation");
|
||||
let last_applied = update_site_replication_state(|state| Ok(next_peer_edit_generation(state)))
|
||||
.await
|
||||
.expect("first-life allocation");
|
||||
// A receiver that never dropped this site from its peer map holds
|
||||
// this mark across the removal.
|
||||
let mut receiver = SiteReplicationState::default();
|
||||
record_applied_peer_edit_generation(&mut receiver, "origin-site", last_applied);
|
||||
|
||||
// Unilateral removal: the site drops below two peers, which clears
|
||||
// its state object and the counter with it.
|
||||
let mut departed = seed();
|
||||
departed.peers.remove("site-b");
|
||||
save_site_replication_state(&departed).await.expect("clear state");
|
||||
assert_eq!(
|
||||
load_site_replication_state().await.expect("reload").edit_generation,
|
||||
generations.len() as u64,
|
||||
"the persisted counter must account for every allocation"
|
||||
0,
|
||||
"clearing the state object must take the counter with it"
|
||||
);
|
||||
|
||||
// Rejoin recreates the state object; two nodes race the first
|
||||
// allocation of the new life.
|
||||
save_site_replication_state(&seed()).await.expect("recreate state");
|
||||
let node_a = tokio::spawn(update_site_replication_state(|state| Ok(next_peer_edit_generation(state))));
|
||||
let node_b = tokio::spawn(update_site_replication_state(|state| Ok(next_peer_edit_generation(state))));
|
||||
let generation_a = node_a.await.expect("node a task").expect("node a allocation");
|
||||
let generation_b = node_b.await.expect("node b task").expect("node b allocation");
|
||||
assert_ne!(generation_a, generation_b, "racing allocations must stay distinct");
|
||||
|
||||
// The receiver's stale mark must not fence the restarted counter…
|
||||
let restarted = generation_a.min(generation_b);
|
||||
assert!(
|
||||
!peer_edit_delivery_is_stale(&receiver, "origin-site", restarted),
|
||||
"the recreated life's first allocation ({restarted}) must clear the previous life's mark ({last_applied})"
|
||||
);
|
||||
record_applied_peer_edit_generation(&mut receiver, "origin-site", restarted);
|
||||
// …while the cleared life's in-flight leftovers stay fenced.
|
||||
assert!(
|
||||
peer_edit_delivery_is_stale(&receiver, "origin-site", straggler),
|
||||
"a pre-removal in-flight delivery ({straggler}) must stay fenced after the rejoin"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ use rustfs_targets::EventName;
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
};
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, insert_str};
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Debug;
|
||||
@@ -461,6 +461,11 @@ impl S3 for FS {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_TAGGING_TIMESTAMP,
|
||||
OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
|
||||
);
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
}
|
||||
|
||||
@@ -1645,6 +1650,11 @@ impl S3 for FS {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_TAGGING_TIMESTAMP,
|
||||
OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
|
||||
);
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
}
|
||||
|
||||
|
||||
@@ -17,11 +17,13 @@ use crate::storage::storage_api::options_consumer::contract::{object::HTTPPrecon
|
||||
use http::header::{IF_MATCH, IF_NONE_MATCH};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
|
||||
SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_VERSION_ID, get_header,
|
||||
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
|
||||
SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
||||
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID, SUFFIX_TAGGING_TIMESTAMP, get_header,
|
||||
header_compat::{MINIO_ENCRYPTION_PREFIX, RUSTFS_ENCRYPTION_PREFIX},
|
||||
insert_header_map,
|
||||
insert_header_map, insert_str,
|
||||
metadata_compat::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX},
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
@@ -422,6 +424,9 @@ pub fn get_complete_multipart_upload_opts_with_replication_authorization(
|
||||
preserve_etag,
|
||||
..Default::default()
|
||||
};
|
||||
if replication_request {
|
||||
apply_replication_timestamps_from_headers(headers, &mut opts);
|
||||
}
|
||||
apply_replica_status_from_headers(headers, &mut opts, replication_request_authorized);
|
||||
|
||||
fill_conditional_writes_opts_from_header(headers, &mut opts)?;
|
||||
@@ -479,6 +484,7 @@ pub fn put_opts_from_headers_with_replication_authorization(
|
||||
if let Some(crc) = get_header(headers, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||
insert_header_map(&mut opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC, crc.into_owned());
|
||||
}
|
||||
apply_replication_timestamps_from_headers(headers, &mut opts);
|
||||
}
|
||||
Ok(opts)
|
||||
}
|
||||
@@ -504,6 +510,47 @@ fn replication_source_mtime(headers: &HeaderMap<HeaderValue>) -> Option<time::Of
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses one replication LWW timestamp header. Invalid values are dropped
|
||||
/// with a warning (same tolerance as [`replication_source_mtime`]) so a
|
||||
/// malformed source header cannot wedge the replication queue.
|
||||
fn replication_timestamp_header(headers: &HeaderMap<HeaderValue>, suffix: &str) -> Option<time::OffsetDateTime> {
|
||||
let value = get_header(headers, suffix)?;
|
||||
let value = value.trim();
|
||||
match time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339) {
|
||||
Ok(timestamp) => Some(timestamp),
|
||||
Err(err) => {
|
||||
tracing::warn!("Invalid {} value '{}' (replication request=true): {}", suffix, value, err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Callers must gate on an authorized replication request: these headers are
|
||||
/// trusted source-cluster state, not client input.
|
||||
fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, opts: &mut ObjectOptions) {
|
||||
opts.replication_tagging_timestamp = replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP);
|
||||
opts.replication_retention_timestamp = replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP);
|
||||
opts.replication_legalhold_timestamp = replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP);
|
||||
|
||||
// Persist into the internal metadata keys so a later outbound replication
|
||||
// pass (replication_target_boundary) reads the source's modification
|
||||
// times instead of falling back to mod_time.
|
||||
// TODO(P1-6): receiver-side LWW is still missing — when the stored
|
||||
// per-category timestamp is newer than the inbound one, the existing
|
||||
// tags/retention/legal-hold should win instead of being overwritten.
|
||||
for (timestamp, suffix) in [
|
||||
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
|
||||
(opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),
|
||||
(opts.replication_legalhold_timestamp, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP),
|
||||
] {
|
||||
if let Some(timestamp) = timestamp
|
||||
&& let Ok(value) = timestamp.format(&time::format_description::well_known::Rfc3339)
|
||||
{
|
||||
insert_str(&mut opts.user_defined, suffix, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_replica_status_from_headers(headers: &HeaderMap<HeaderValue>, opts: &mut ObjectOptions, authorized: bool) {
|
||||
if !authorized {
|
||||
return;
|
||||
@@ -663,6 +710,13 @@ fn is_reserved_user_metadata_key(key: &str) -> bool {
|
||||
|| starts_with_ignore_ascii_case(key, MINIO_INTERNAL_PREFIX)
|
||||
|| starts_with_ignore_ascii_case(key, RUSTFS_ENCRYPTION_PREFIX)
|
||||
|| starts_with_ignore_ascii_case(key, MINIO_ENCRYPTION_PREFIX)
|
||||
// Replication transport names (source-replication timestamps,
|
||||
// source-mtime/-etag/-version-id, ...). A bare stored key with one of
|
||||
// these names is forwarded verbatim by the outbound replication
|
||||
// header builder on a server-authorized request, so the receiver
|
||||
// would persist attacker-chosen values as trusted internal LWW state.
|
||||
|| starts_with_ignore_ascii_case(key, "x-rustfs-source-")
|
||||
|| starts_with_ignore_ascii_case(key, "x-minio-source-")
|
||||
}
|
||||
|
||||
fn stored_user_metadata_key(key: &str) -> String {
|
||||
@@ -1599,6 +1653,136 @@ mod tests {
|
||||
assert!(opts_invalid.mod_time.is_none());
|
||||
}
|
||||
|
||||
/// A client PUT must not materialize the replication transport names as
|
||||
/// bare stored user-metadata keys: the outbound replication header
|
||||
/// builder forwards user metadata verbatim on a server-authorized
|
||||
/// request, so a bare `x-rustfs-source-replication-*-timestamp` key would
|
||||
/// deliver an attacker-chosen value into the replica's trusted internal
|
||||
/// LWW state (for a tagless object nothing later overwrites it).
|
||||
#[test]
|
||||
fn test_replication_transport_names_cannot_be_forged_via_user_metadata() {
|
||||
let mut headers = HeaderMap::new();
|
||||
for name in [
|
||||
"x-amz-meta-x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-amz-meta-x-minio-source-replication-legalhold-timestamp",
|
||||
"x-rustfs-meta-x-rustfs-source-replication-retention-timestamp",
|
||||
"x-amz-meta-x-rustfs-source-mtime",
|
||||
] {
|
||||
headers.insert(
|
||||
http::header::HeaderName::from_static(name),
|
||||
HeaderValue::from_static("2026-01-02T03:04:05Z"),
|
||||
);
|
||||
}
|
||||
|
||||
let metadata = extract_metadata(&headers);
|
||||
|
||||
for forged in [
|
||||
"x-rustfs-source-replication-tagging-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
"x-rustfs-source-replication-retention-timestamp",
|
||||
"x-rustfs-source-mtime",
|
||||
] {
|
||||
assert!(
|
||||
metadata.get(forged).is_none(),
|
||||
"{forged} must not be storable as a bare user-metadata key"
|
||||
);
|
||||
}
|
||||
// The values survive, namespaced back under the user-metadata prefix.
|
||||
assert_eq!(
|
||||
metadata
|
||||
.get("x-amz-meta-x-rustfs-source-replication-tagging-timestamp")
|
||||
.map(String::as_str),
|
||||
Some("2026-01-02T03:04:05Z")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_opts_from_headers_gates_replication_timestamp_persistence_on_authorization() {
|
||||
// Sender-side LWW state (replication_target_boundary.rs) is read back
|
||||
// from these internal metadata keys, so an authorized replication PUT
|
||||
// must persist the inbound timestamp headers; an unauthorized client
|
||||
// must not be able to forge them.
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, "source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||
insert_header(&mut headers, "source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||
insert_header(&mut headers, "source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||
|
||||
let untrusted = put_opts_from_headers(&headers, HashMap::new()).expect("ordinary PUT options should be created");
|
||||
for suffix in [
|
||||
"tagging-timestamp",
|
||||
"objectlock-retention-timestamp",
|
||||
"objectlock-legalhold-timestamp",
|
||||
] {
|
||||
assert!(
|
||||
rustfs_utils::http::get_str(&untrusted.user_defined, suffix).is_none(),
|
||||
"unauthorized clients must not persist the {suffix} internal key"
|
||||
);
|
||||
}
|
||||
assert!(untrusted.replication_tagging_timestamp.is_none());
|
||||
assert!(untrusted.replication_retention_timestamp.is_none());
|
||||
assert!(untrusted.replication_legalhold_timestamp.is_none());
|
||||
|
||||
let trusted = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("authorized replication request should parse");
|
||||
let parse = |value: &str| {
|
||||
time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).expect("valid RFC3339")
|
||||
};
|
||||
assert_eq!(trusted.replication_tagging_timestamp, Some(parse("2026-01-02T03:04:05Z")));
|
||||
assert_eq!(trusted.replication_retention_timestamp, Some(parse("2026-01-02T03:04:06Z")));
|
||||
assert_eq!(trusted.replication_legalhold_timestamp, Some(parse("2026-01-02T03:04:07Z")));
|
||||
for (suffix, expected) in [
|
||||
("tagging-timestamp", "2026-01-02T03:04:05Z"),
|
||||
("objectlock-retention-timestamp", "2026-01-02T03:04:06Z"),
|
||||
("objectlock-legalhold-timestamp", "2026-01-02T03:04:07Z"),
|
||||
] {
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_str(&trusted.user_defined, suffix).as_deref(),
|
||||
Some(expected),
|
||||
"authorized replication must persist the {suffix} internal key"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_complete_multipart_opts_persist_replication_timestamps_when_authorized() {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, "replication-actual-object-size", "1");
|
||||
insert_header(&mut headers, "source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||
insert_header(&mut headers, "source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||
insert_header(&mut headers, "source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||
|
||||
let untrusted = get_complete_multipart_upload_opts(&headers).expect("ordinary multipart options should be created");
|
||||
for suffix in [
|
||||
"tagging-timestamp",
|
||||
"objectlock-retention-timestamp",
|
||||
"objectlock-legalhold-timestamp",
|
||||
] {
|
||||
assert!(
|
||||
rustfs_utils::http::get_str(&untrusted.user_defined, suffix).is_none(),
|
||||
"unauthorized multipart completes must not persist the {suffix} internal key"
|
||||
);
|
||||
}
|
||||
|
||||
let trusted = get_complete_multipart_upload_opts_with_replication_authorization(&headers, true)
|
||||
.expect("authorized multipart complete should parse");
|
||||
for (suffix, expected) in [
|
||||
("tagging-timestamp", "2026-01-02T03:04:05Z"),
|
||||
("objectlock-retention-timestamp", "2026-01-02T03:04:06Z"),
|
||||
("objectlock-legalhold-timestamp", "2026-01-02T03:04:07Z"),
|
||||
] {
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_str(&trusted.user_defined, suffix).as_deref(),
|
||||
Some(expected),
|
||||
"authorized multipart completes must persist the {suffix} internal key"
|
||||
);
|
||||
}
|
||||
assert!(trusted.replication_tagging_timestamp.is_some());
|
||||
assert!(trusted.replication_retention_timestamp.is_some());
|
||||
assert!(trusted.replication_legalhold_timestamp.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_opts_from_headers_with_replica_status() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -195,6 +195,13 @@ IFS= read -r -d '' expected_docker_automatic_guard <<'EOF' || true
|
||||
EOF
|
||||
expected_docker_automatic_guard=${expected_docker_automatic_guard%$'\n'}
|
||||
require_job_if "$docker_workflow" "build-check" "$expected_docker_automatic_guard"
|
||||
require_line "$docker_workflow" ' source_ref: ${{ steps.check.outputs.source_ref }}' "Docker source ref output"
|
||||
require_line "$docker_workflow" ' source_ref="$HEAD_SHA"' "automatic Docker source ref"
|
||||
require_line "$docker_workflow" ' source_ref="$tag_ref"' "manual Docker source ref"
|
||||
require_line "$docker_workflow" ' ref: ${{ needs.build-check.outputs.source_ref }}' "Docker release source checkout"
|
||||
require_line "$docker_workflow" ' SOURCE_REVISION="$(git rev-parse HEAD)"' "Docker source revision resolution"
|
||||
require_line "$docker_workflow" ' LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"' "Docker revision label"
|
||||
require_absent "$docker_workflow" 'org.opencontainers.image.revision=${{ github.sha }}' "Docker revision must not use the workflow branch SHA"
|
||||
|
||||
docker_manual_guard=$(awk '
|
||||
$0 == " *-preview*)" { in_preview = 1 }
|
||||
|
||||
Reference in New Issue
Block a user