mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 18:27:49 +00:00
Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 21c2fb42bb | |||
| d091554ffe | |||
| c04ee41cf0 | |||
| a9691b6797 | |||
| 7db3882777 | |||
| 3f3e3f4f05 | |||
| 01e0af6312 | |||
| 3377688dab | |||
| 890ddea94b | |||
| 9f02ca6c36 | |||
| 33cd11472a | |||
| 3ff250f1cd | |||
| d795729585 | |||
| 9e6e02ea09 | |||
| 39274fc37c | |||
| 33eff4c3c4 | |||
| a2f16aa066 | |||
| 4c8b9f87e1 | |||
| 3272730c13 | |||
| 1862112d0c | |||
| cd0ac02879 | |||
| 6cf9cf7bb5 | |||
| f1f86ee9d0 | |||
| 1eef0de003 | |||
| a118d7e4fd | |||
| ed1bedf1fb | |||
| 4392f94e1a | |||
| 81d7b7d07a | |||
| e26668e62c | |||
| 8d3511c1b3 | |||
| d172d05e86 | |||
| 0d86c50760 | |||
| 526d6f667e | |||
| dcf3e4b9e8 | |||
| 04b9c8fd36 | |||
| c1f66969d7 | |||
| cfa9276fad | |||
| db8f55cb97 | |||
| 7f23a1ba91 | |||
| 1619c4be60 | |||
| 72fd7339c9 | |||
| 71e83aeec4 | |||
| 9138c24571 |
@@ -94,6 +94,7 @@ jobs:
|
|||||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||||
|
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
@@ -118,6 +119,7 @@ jobs:
|
|||||||
short_sha=""
|
short_sha=""
|
||||||
is_prerelease=false
|
is_prerelease=false
|
||||||
create_latest=false
|
create_latest=false
|
||||||
|
source_ref="$GITHUB_SHA"
|
||||||
|
|
||||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||||
# Triggered by build workflow completion
|
# Triggered by build workflow completion
|
||||||
@@ -137,6 +139,7 @@ jobs:
|
|||||||
# Extract version info from commit message or use commit SHA
|
# Extract version info from commit message or use commit SHA
|
||||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||||
|
source_ref="$HEAD_SHA"
|
||||||
|
|
||||||
# Determine build type based on triggering workflow event and ref
|
# Determine build type based on triggering workflow event and ref
|
||||||
triggering_event="$TRIGGERING_EVENT"
|
triggering_event="$TRIGGERING_EVENT"
|
||||||
@@ -261,6 +264,23 @@ jobs:
|
|||||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||||
|
tag_ref="refs/tags/$input_version"
|
||||||
|
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||||
|
if [[ "$input_version" == v* ]]; then
|
||||||
|
tag_ref="refs/tags/${input_version#v}"
|
||||||
|
else
|
||||||
|
tag_ref="refs/tags/v$input_version"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||||
|
echo "❌ Release tag not found for Docker build: $input_version"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
source_ref="$tag_ref"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -271,6 +291,7 @@ jobs:
|
|||||||
echo "short_sha=$short_sha"
|
echo "short_sha=$short_sha"
|
||||||
echo "is_prerelease=$is_prerelease"
|
echo "is_prerelease=$is_prerelease"
|
||||||
echo "create_latest=$create_latest"
|
echo "create_latest=$create_latest"
|
||||||
|
echo "source_ref=$source_ref"
|
||||||
} >> "$GITHUB_OUTPUT"
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
echo "🐳 Docker Build Summary:"
|
echo "🐳 Docker Build Summary:"
|
||||||
@@ -281,6 +302,7 @@ jobs:
|
|||||||
echo " - Short SHA: $short_sha"
|
echo " - Short SHA: $short_sha"
|
||||||
echo " - Is prerelease: $is_prerelease"
|
echo " - Is prerelease: $is_prerelease"
|
||||||
echo " - Create latest: $create_latest"
|
echo " - Create latest: $create_latest"
|
||||||
|
echo " - Source ref: $source_ref"
|
||||||
|
|
||||||
# Build multi-arch Docker images
|
# Build multi-arch Docker images
|
||||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||||
@@ -308,6 +330,7 @@ jobs:
|
|||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||||
|
|
||||||
- name: Login to Docker Hub
|
- name: Login to Docker Hub
|
||||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
@@ -397,7 +420,8 @@ jobs:
|
|||||||
LABELS="org.opencontainers.image.title=RustFS"
|
LABELS="org.opencontainers.image.title=RustFS"
|
||||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||||
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||||
|
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||||
|
|||||||
Generated
+4
@@ -278,6 +278,7 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"bigdecimal",
|
"bigdecimal",
|
||||||
"bon",
|
"bon",
|
||||||
|
"crc32fast",
|
||||||
"digest 0.11.3",
|
"digest 0.11.3",
|
||||||
"log",
|
"log",
|
||||||
"miniz_oxide 0.9.1",
|
"miniz_oxide 0.9.1",
|
||||||
@@ -289,9 +290,11 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"snap",
|
||||||
"strum",
|
"strum",
|
||||||
"thiserror 2.0.20",
|
"thiserror 2.0.20",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
"zstd",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -9200,6 +9203,7 @@ dependencies = [
|
|||||||
"serial_test",
|
"serial_test",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"shadow-rs",
|
"shadow-rs",
|
||||||
|
"snap",
|
||||||
"socket2",
|
"socket2",
|
||||||
"subtle",
|
"subtle",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
|
|||||||
+1
-1
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
|||||||
tower-http = { version = "0.7.0" }
|
tower-http = { version = "0.7.0" }
|
||||||
|
|
||||||
# Serialization and Data Formats
|
# Serialization and Data Formats
|
||||||
apache-avro = "0.22.0"
|
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||||
bytes = { version = "1.12.1" }
|
bytes = { version = "1.12.1" }
|
||||||
bytesize = "2.7.0"
|
bytesize = "2.7.0"
|
||||||
byteorder = "1.5.0"
|
byteorder = "1.5.0"
|
||||||
|
|||||||
@@ -11,7 +11,6 @@
|
|||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
use base64_simd::STANDARD;
|
use base64_simd::STANDARD;
|
||||||
|
|
||||||
|
|||||||
@@ -38,7 +38,10 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
|||||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||||
|
|
||||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||||
|
|||||||
@@ -476,13 +476,19 @@ impl Checksum for Xxhash64 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||||
|
)]
|
||||||
struct Md5 {
|
struct Md5 {
|
||||||
hasher: md5::Md5,
|
hasher: md5::Md5,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||||
|
)]
|
||||||
impl Md5 {
|
impl Md5 {
|
||||||
fn update(&mut self, bytes: &[u8]) {
|
fn update(&mut self, bytes: &[u8]) {
|
||||||
use md5::Digest;
|
use md5::Digest;
|
||||||
|
|||||||
@@ -76,6 +76,18 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
|||||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||||
|
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||||
|
"x-rustfs-source-replication-tagging-timestamp",
|
||||||
|
"x-minio-source-replication-tagging-timestamp",
|
||||||
|
];
|
||||||
|
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||||
|
"x-rustfs-source-replication-retention-timestamp",
|
||||||
|
"x-minio-source-replication-retention-timestamp",
|
||||||
|
];
|
||||||
|
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||||
|
"x-rustfs-source-replication-legalhold-timestamp",
|
||||||
|
"x-minio-source-replication-legalhold-timestamp",
|
||||||
|
];
|
||||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||||
|
|
||||||
@@ -118,6 +130,25 @@ pub enum FaultAction {
|
|||||||
WrongEtag,
|
WrongEtag,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Replication LWW timestamp headers observed on a request, journaled so
|
||||||
|
/// sender-side tests can assert what a real target would receive.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct ReplicationTimestampHeaders {
|
||||||
|
pub tagging: Option<String>,
|
||||||
|
pub retention: Option<String>,
|
||||||
|
pub legalhold: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReplicationTimestampHeaders {
|
||||||
|
fn from_headers(headers: &HeaderMap) -> Self {
|
||||||
|
Self {
|
||||||
|
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||||
|
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||||
|
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Credential-free request metadata retained for deterministic assertions.
|
/// Credential-free request metadata retained for deterministic assertions.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct RequestRecord {
|
pub struct RequestRecord {
|
||||||
@@ -131,6 +162,7 @@ pub struct RequestRecord {
|
|||||||
pub part_number: Option<i32>,
|
pub part_number: Option<i32>,
|
||||||
pub content_length: Option<u64>,
|
pub content_length: Option<u64>,
|
||||||
pub consumed_bytes: Option<usize>,
|
pub consumed_bytes: Option<usize>,
|
||||||
|
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||||
pub fault: Option<FaultAction>,
|
pub fault: Option<FaultAction>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -536,7 +568,15 @@ impl S3Access for FaultAccess {
|
|||||||
.get(CONTENT_LENGTH)
|
.get(CONTENT_LENGTH)
|
||||||
.and_then(|value| value.to_str().ok())
|
.and_then(|value| value.to_str().ok())
|
||||||
.and_then(|value| value.parse().ok());
|
.and_then(|value| value.parse().ok());
|
||||||
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||||
|
let fault = record_request(
|
||||||
|
&self.control,
|
||||||
|
operation,
|
||||||
|
context.method().clone(),
|
||||||
|
parsed,
|
||||||
|
content_length,
|
||||||
|
replication_timestamps,
|
||||||
|
);
|
||||||
if let Some(RequestFault {
|
if let Some(RequestFault {
|
||||||
action: FaultAction::Status(status),
|
action: FaultAction::Status(status),
|
||||||
..
|
..
|
||||||
@@ -589,6 +629,7 @@ fn record_request(
|
|||||||
method: Method,
|
method: Method,
|
||||||
parsed: ParsedRequest,
|
parsed: ParsedRequest,
|
||||||
content_length: Option<u64>,
|
content_length: Option<u64>,
|
||||||
|
replication_timestamps: ReplicationTimestampHeaders,
|
||||||
) -> Option<RequestFault> {
|
) -> Option<RequestFault> {
|
||||||
let mut state = lock(control);
|
let mut state = lock(control);
|
||||||
let action = parsed
|
let action = parsed
|
||||||
@@ -613,6 +654,7 @@ fn record_request(
|
|||||||
part_number: parsed.part_number,
|
part_number: parsed.part_number,
|
||||||
content_length,
|
content_length,
|
||||||
consumed_bytes: None,
|
consumed_bytes: None,
|
||||||
|
replication_timestamps,
|
||||||
fault: action.clone(),
|
fault: action.clone(),
|
||||||
});
|
});
|
||||||
action.map(|action| RequestFault { sequence, action })
|
action.map(|action| RequestFault { sequence, action })
|
||||||
@@ -1699,6 +1741,52 @@ mod tests {
|
|||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
||||||
|
let target = FakeS3Target::start().await?;
|
||||||
|
target.create_bucket("target-bucket");
|
||||||
|
let client = client(&target);
|
||||||
|
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket("target-bucket")
|
||||||
|
.key("plain")
|
||||||
|
.body(ByteStream::from_static(b"plain"))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket("target-bucket")
|
||||||
|
.key("stamped")
|
||||||
|
.body(ByteStream::from_static(b"stamped"))
|
||||||
|
.customize()
|
||||||
|
.map_request(move |mut request| {
|
||||||
|
let headers = request.headers_mut();
|
||||||
|
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
||||||
|
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
||||||
|
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
||||||
|
Ok::<_, std::convert::Infallible>(request)
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let requests = target.requests();
|
||||||
|
let plain = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
||||||
|
.expect("plain PUT must be journaled");
|
||||||
|
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
||||||
|
|
||||||
|
let stamped = requests
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
||||||
|
.expect("stamped PUT must be journaled");
|
||||||
|
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
||||||
|
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
||||||
|
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
macro_rules! assert_sdk_error {
|
macro_rules! assert_sdk_error {
|
||||||
($error:expr, $status:expr, $code:expr) => {{
|
($error:expr, $status:expr, $code:expr) => {{
|
||||||
let error = &$error;
|
let error = &$error;
|
||||||
@@ -2985,6 +3073,7 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
Some(0),
|
Some(0),
|
||||||
|
ReplicationTimestampHeaders::default(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let records = lock(&control).requests.clone();
|
let records = lock(&control).requests.clone();
|
||||||
@@ -3006,6 +3095,7 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
None,
|
None,
|
||||||
|
ReplicationTimestampHeaders::default(),
|
||||||
);
|
);
|
||||||
{
|
{
|
||||||
let bounded_records = lock(&bounded_control);
|
let bounded_records = lock(&bounded_control);
|
||||||
|
|||||||
@@ -135,7 +135,8 @@ pub mod bucket {
|
|||||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||||
pub use crate::bucket::metadata_sys::{
|
pub use crate::bucket::metadata_sys::{
|
||||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||||
|
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||||
@@ -184,17 +185,18 @@ pub mod bucket {
|
|||||||
mrf_backlog_observability_snapshot,
|
mrf_backlog_observability_snapshot,
|
||||||
};
|
};
|
||||||
pub use crate::bucket::replication::{
|
pub use crate::bucket::replication::{
|
||||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
||||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
||||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
|
||||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
||||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||||
|
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||||
|
|||||||
@@ -58,7 +58,9 @@ use rustfs_utils::http::{
|
|||||||
};
|
};
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||||
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
|
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||||
|
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||||
|
insert_header,
|
||||||
};
|
};
|
||||||
use rustls_pki_types::pem::PemObject;
|
use rustls_pki_types::pem::PemObject;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -80,7 +82,6 @@ use tracing::warn;
|
|||||||
use url::Url;
|
use url::Url;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
|
||||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||||
|
|
||||||
@@ -1476,9 +1477,12 @@ impl Default for AdvancedPutOptions {
|
|||||||
replication_status: ReplicationStatusType::Pending,
|
replication_status: ReplicationStatusType::Pending,
|
||||||
source_mtime: OffsetDateTime::now_utc(),
|
source_mtime: OffsetDateTime::now_utc(),
|
||||||
replication_request: false,
|
replication_request: false,
|
||||||
retention_timestamp: OffsetDateTime::now_utc(),
|
// UNIX_EPOCH means "never modified": header() must not emit a
|
||||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
// timestamp header for it, otherwise a receiver would treat an
|
||||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
// unset category as a modification made right now.
|
||||||
|
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||||
|
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||||
|
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
|
||||||
replication_validity_check: false,
|
replication_validity_check: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1675,6 +1679,16 @@ impl PutObjectOptions {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (suffix, timestamp) in [
|
||||||
|
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
|
||||||
|
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
|
||||||
|
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
|
||||||
|
] {
|
||||||
|
if timestamp.unix_timestamp() != 0 {
|
||||||
|
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if self.internal.replication_request {
|
if self.internal.replication_request {
|
||||||
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||||
}
|
}
|
||||||
@@ -2842,6 +2856,57 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn put_object_headers_carry_replication_timestamp_headers() {
|
||||||
|
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
|
||||||
|
// last-writer-wins on these headers (object-api-options.go parses them
|
||||||
|
// as RFC3339); a replica without them loses every conflict resolution.
|
||||||
|
let mut opts = PutObjectOptions::default();
|
||||||
|
opts.internal.replication_request = true;
|
||||||
|
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
|
||||||
|
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
|
||||||
|
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
|
||||||
|
opts.internal.tagging_timestamp = tagging;
|
||||||
|
opts.internal.retention_timestamp = retention;
|
||||||
|
opts.internal.legalhold_timestamp = legalhold;
|
||||||
|
|
||||||
|
let header = opts.header();
|
||||||
|
for (suffix, expected) in [
|
||||||
|
("source-replication-tagging-timestamp", tagging),
|
||||||
|
("source-replication-retention-timestamp", retention),
|
||||||
|
("source-replication-legalhold-timestamp", legalhold),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_header(&header, suffix).as_deref(),
|
||||||
|
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
|
||||||
|
"replication put requests must carry the {suffix} header"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn put_object_headers_omit_unset_replication_timestamps() {
|
||||||
|
// UNIX_EPOCH means "never modified on the source"; sending it would
|
||||||
|
// make the receiver treat an unset category as a fresh modification.
|
||||||
|
let mut opts = PutObjectOptions::default();
|
||||||
|
opts.internal.replication_request = true;
|
||||||
|
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||||
|
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||||
|
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
|
||||||
|
|
||||||
|
let header = opts.header();
|
||||||
|
for suffix in [
|
||||||
|
"source-replication-tagging-timestamp",
|
||||||
|
"source-replication-retention-timestamp",
|
||||||
|
"source-replication-legalhold-timestamp",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
rustfs_utils::http::get_header(&header, suffix).is_none(),
|
||||||
|
"unset {suffix} must not be sent to replication targets"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
|
||||||
let sys = BucketTargetSys::default();
|
let sys = BucketTargetSys::default();
|
||||||
|
|||||||
@@ -126,11 +126,23 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
|
|||||||
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
||||||
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
||||||
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
||||||
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
||||||
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub type TraceFn =
|
pub type TraceFn =
|
||||||
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||||
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||||
@@ -140,9 +152,21 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
|||||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||||
|
|
||||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||||
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
||||||
|
|
||||||
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
||||||
@@ -280,6 +304,10 @@ impl LifecycleSys {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
||||||
let bucket = oi.bucket.clone();
|
let bucket = oi.bucket.clone();
|
||||||
let name = oi.name.clone();
|
let name = oi.name.clone();
|
||||||
@@ -570,6 +598,10 @@ async fn delete_free_version_remote_object(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
||||||
oi: &ObjectInfo,
|
oi: &ObjectInfo,
|
||||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||||
@@ -2868,6 +2900,10 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
|
|||||||
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
||||||
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
||||||
}
|
}
|
||||||
@@ -3352,6 +3388,10 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
||||||
if remote_delete_succeeded {
|
if remote_delete_succeeded {
|
||||||
opts.skip_decommissioned = true;
|
opts.skip_decommissioned = true;
|
||||||
@@ -4339,6 +4379,10 @@ pub async fn expire_transitioned_object(
|
|||||||
Ok(dobj)
|
Ok(dobj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||||
let us = Uuid::new_v4().to_string();
|
let us = Uuid::new_v4().to_string();
|
||||||
let mut hasher = Sha256::new();
|
let mut hasher = Sha256::new();
|
||||||
@@ -4373,6 +4417,10 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||||
let tier = _tier.to_string();
|
let tier = _tier.to_string();
|
||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
@@ -4391,6 +4439,10 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn get_transitioned_object_reader(
|
pub async fn get_transitioned_object_reader(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
@@ -5145,6 +5197,10 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res
|
|||||||
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||||
let mut success = false;
|
let mut success = false;
|
||||||
match event.action {
|
match event.action {
|
||||||
@@ -7422,6 +7478,10 @@ mod tests {
|
|||||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||||
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
||||||
#[allow(unsafe_code)]
|
#[allow(unsafe_code)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
||||||
where
|
where
|
||||||
F: FnOnce() -> Fut,
|
F: FnOnce() -> Fut,
|
||||||
|
|||||||
@@ -759,6 +759,10 @@ pub struct ManualTransitionWorkerResultRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ManualTransitionWorkerResultRecord {
|
impl ManualTransitionWorkerResultRecord {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||||
Self::new_with_reason(job_id, task_key, result, None)
|
Self::new_with_reason(job_id, task_key, result, None)
|
||||||
}
|
}
|
||||||
@@ -1257,6 +1261,10 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn load_manual_transition_task_record(
|
pub async fn load_manual_transition_task_record(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
@@ -1320,6 +1328,10 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn load_manual_transition_worker_result_stats(
|
pub async fn load_manual_transition_worker_result_stats(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
@@ -1455,6 +1467,10 @@ async fn scan_manual_transition_worker_result_journal(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn reconcile_manual_transition_worker_results(
|
pub async fn reconcile_manual_transition_worker_results(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
|
|||||||
@@ -15,25 +15,35 @@
|
|||||||
use rustfs_common::metrics::IlmAction;
|
use rustfs_common::metrics::IlmAction;
|
||||||
|
|
||||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||||
|
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||||
pub(crate) use crate::bucket::replication::{
|
pub(crate) use crate::bucket::replication::{
|
||||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||||
};
|
};
|
||||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
|
||||||
use crate::storage_api_contracts::object::DeletedObject;
|
use crate::storage_api_contracts::object::DeletedObject;
|
||||||
|
|
||||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||||
obj.version_purge_status.is_pending()
|
obj.version_purge_status.is_pending()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,10 @@
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -331,6 +331,10 @@ where
|
|||||||
persist_tier_delete_journal_entry(api, &committed).await
|
persist_tier_delete_journal_entry(api, &committed).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||||
where
|
where
|
||||||
S: ObjectOperations<
|
S: ObjectOperations<
|
||||||
|
|||||||
@@ -148,6 +148,10 @@ struct RecoveryCursor {
|
|||||||
object: String,
|
object: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn recover_tier_free_versions(
|
pub async fn recover_tier_free_versions(
|
||||||
api: Arc<ECStore>,
|
api: Arc<ECStore>,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
|
|||||||
@@ -385,6 +385,10 @@ impl ExpiryOp for Jentry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||||
if let Err(err) = &result
|
if let Err(err) = &result
|
||||||
@@ -395,6 +399,10 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||||
@@ -405,6 +413,10 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
|||||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||||
obj_name: &str,
|
obj_name: &str,
|
||||||
rv_id: &str,
|
rv_id: &str,
|
||||||
@@ -485,6 +497,10 @@ pub enum RemoteTierDeleteOutcome {
|
|||||||
AlreadyRemoved,
|
AlreadyRemoved,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn delete_object_from_remote_tier_idempotent(
|
pub async fn delete_object_from_remote_tier_idempotent(
|
||||||
obj_name: &str,
|
obj_name: &str,
|
||||||
rv_id: &str,
|
rv_id: &str,
|
||||||
|
|||||||
@@ -50,8 +50,16 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
|||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum TransitionTransactionError {
|
pub enum TransitionTransactionError {
|
||||||
#[error("transition transaction already exists")]
|
#[error("transition transaction already exists")]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
AlreadyExists,
|
AlreadyExists,
|
||||||
#[error("transition transaction is not found")]
|
#[error("transition transaction is not found")]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||||
|
)]
|
||||||
NotFound,
|
NotFound,
|
||||||
#[error("transition transaction is corrupt: {0}")]
|
#[error("transition transaction is corrupt: {0}")]
|
||||||
Corrupt(&'static str),
|
Corrupt(&'static str),
|
||||||
|
|||||||
@@ -60,12 +60,14 @@ struct ConfigWriteLockProbeState {
|
|||||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||||
pub struct ConfigWriteLockProbe {
|
pub struct ConfigWriteLockProbe {
|
||||||
state: Arc<ConfigWriteLockProbeState>,
|
state: Arc<ConfigWriteLockProbeState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
impl ConfigWriteLockProbe {
|
impl ConfigWriteLockProbe {
|
||||||
|
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||||
pub fn install(bucket: &str) -> Self {
|
pub fn install(bucket: &str) -> Self {
|
||||||
let state = Arc::new(ConfigWriteLockProbeState {
|
let state = Arc::new(ConfigWriteLockProbeState {
|
||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
@@ -84,6 +86,7 @@ impl ConfigWriteLockProbe {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||||
pub async fn wait_until_attempted(&self) {
|
pub async fn wait_until_attempted(&self) {
|
||||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||||
.await
|
.await
|
||||||
@@ -656,6 +659,16 @@ pub async fn update_under_transaction_lock(
|
|||||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||||
|
pub async fn delete_under_transaction_lock(
|
||||||
|
guard: &BucketMetadataMutationGuard,
|
||||||
|
bucket: &str,
|
||||||
|
config_file: &str,
|
||||||
|
) -> Result<OffsetDateTime> {
|
||||||
|
guard.ensure_valid(bucket)?;
|
||||||
|
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn update_quota_if_incarnation(
|
pub async fn update_quota_if_incarnation(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
data: Vec<u8>,
|
data: Vec<u8>,
|
||||||
@@ -795,6 +808,14 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
|
|||||||
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acquire the bucket transaction lock only if its incarnation still matches.
|
||||||
|
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
|
||||||
|
bucket: &str,
|
||||||
|
expected_incarnation_id: Uuid,
|
||||||
|
) -> Result<BucketMetadataMutationGuard> {
|
||||||
|
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
|
||||||
ctx: &crate::runtime::instance::InstanceContext,
|
ctx: &crate::runtime::instance::InstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -872,6 +893,10 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
|||||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
@@ -1086,6 +1111,10 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
|||||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
@@ -1599,6 +1628,7 @@ impl BucketMetadataSys {
|
|||||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||||
/// instead of supplied up front. Loads through this system's own store so
|
/// instead of supplied up front. Loads through this system's own store so
|
||||||
/// the read and the persisted write target the same instance.
|
/// the read and the persisted write target the same instance.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||||
where
|
where
|
||||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||||
@@ -1703,6 +1733,7 @@ impl BucketMetadataSys {
|
|||||||
/// A miss is never published as an authoritative default, and a snapshot
|
/// A miss is never published as an authoritative default, and a snapshot
|
||||||
/// read before delete plus same-name recreation cannot replace the new
|
/// read before delete plus same-name recreation cannot replace the new
|
||||||
/// generation.
|
/// generation.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||||
if is_meta_bucketname(bucket) {
|
if is_meta_bucketname(bucket) {
|
||||||
return Err(Error::other("errInvalidArgument"));
|
return Err(Error::other("errInvalidArgument"));
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
pub mod bandwidth;
|
pub mod bandwidth;
|
||||||
pub mod bucket_target_sys;
|
pub mod bucket_target_sys;
|
||||||
|
|||||||
@@ -136,6 +136,7 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
|||||||
|
|
||||||
/// Check if an object has legal hold enabled.
|
/// Check if an object has legal hold enabled.
|
||||||
/// Returns true if legal hold is ON.
|
/// Returns true if legal hold is ON.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||||
@@ -151,6 +152,7 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||||
/// * `false` if the object is not locked
|
/// * `false` if the object is not locked
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||||
// Delete markers are never locked
|
// Delete markers are never locked
|
||||||
if is_delete_marker {
|
if is_delete_marker {
|
||||||
|
|||||||
@@ -193,6 +193,7 @@ pub enum QuotaError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub struct QuotaErrorResponse {
|
pub struct QuotaErrorResponse {
|
||||||
#[serde(rename = "Code")]
|
#[serde(rename = "Code")]
|
||||||
pub code: String,
|
pub code: String,
|
||||||
@@ -208,6 +209,7 @@ pub struct QuotaErrorResponse {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl QuotaErrorResponse {
|
impl QuotaErrorResponse {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||||
match quota_error {
|
match quota_error {
|
||||||
QuotaError::QuotaExceeded { .. } => Self {
|
QuotaError::QuotaExceeded { .. } => Self {
|
||||||
|
|||||||
@@ -899,6 +899,7 @@ async fn save_ledger_locked(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn fail_next_quota_ledger_save_for_test() {
|
pub fn fail_next_quota_ledger_save_for_test() {
|
||||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ pub use replication_filemeta_boundary::{
|
|||||||
pub(crate) use replication_filemeta_boundary::{
|
pub(crate) use replication_filemeta_boundary::{
|
||||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||||
};
|
};
|
||||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||||
@@ -81,6 +81,6 @@ pub use replication_queue_boundary::{
|
|||||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||||
|
|||||||
@@ -37,6 +37,10 @@ impl ReplicationConfigStore {
|
|||||||
com::read_config_limited(api, file, max_bytes).await
|
com::read_config_limited(api, file, max_bytes).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||||
where
|
where
|
||||||
S: ReplicationObjectIO,
|
S: ReplicationObjectIO,
|
||||||
|
|||||||
@@ -24,15 +24,27 @@ use super::replication_storage_boundary::{
|
|||||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||||
|
|
||||||
pub(crate) struct ReplicationLifecycleBridge;
|
pub(crate) struct ReplicationLifecycleBridge;
|
||||||
|
|
||||||
impl ReplicationLifecycleBridge {
|
impl ReplicationLifecycleBridge {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||||
ReplicationConfig::new(Some(config), None)
|
ReplicationConfig::new(Some(config), None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn has_pending_version_purge(
|
pub(crate) fn has_pending_version_purge(
|
||||||
config: &ReplicationLifecycleConfig,
|
config: &ReplicationLifecycleConfig,
|
||||||
object_name: &str,
|
object_name: &str,
|
||||||
@@ -45,6 +57,10 @@ impl ReplicationLifecycleBridge {
|
|||||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn check_delete_replication(
|
pub(crate) async fn check_delete_replication(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &ObjectToDelete,
|
object: &ObjectToDelete,
|
||||||
@@ -54,6 +70,10 @@ impl ReplicationLifecycleBridge {
|
|||||||
check_replicate_delete(bucket, object, source, opts, None).await
|
check_replicate_delete(bucket, object, source, opts, None).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||||
let pending_status = decision.pending_status();
|
let pending_status = decision.pending_status();
|
||||||
ReplicationState {
|
ReplicationState {
|
||||||
|
|||||||
@@ -19,17 +19,33 @@ use time::OffsetDateTime;
|
|||||||
use super::replication_error_boundary::Result;
|
use super::replication_error_boundary::Result;
|
||||||
use crate::bucket::msgp_decode;
|
use crate::bucket::msgp_decode;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) struct ReplicationMsgpCodec;
|
pub(crate) struct ReplicationMsgpCodec;
|
||||||
|
|
||||||
impl ReplicationMsgpCodec {
|
impl ReplicationMsgpCodec {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||||
msgp_decode::read_msgp_ext8_time(rd)
|
msgp_decode::read_msgp_ext8_time(rd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||||
msgp_decode::skip_msgp_value(rd)
|
msgp_decode::skip_msgp_value(rd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||||
msgp_decode::write_msgp_time(wr, time)
|
msgp_decode::write_msgp_time(wr, time)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,6 +77,10 @@ impl ReplicationObjectBridge {
|
|||||||
load_delete_request_config_in(ctx, bucket).await
|
load_delete_request_config_in(ctx, bucket).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn delete_config_snapshot_in(
|
pub(crate) async fn delete_config_snapshot_in(
|
||||||
ctx: &ReplicationInstanceContext,
|
ctx: &ReplicationInstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -231,6 +231,10 @@ pub(crate) async fn load_delete_replication_config(
|
|||||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn load_delete_replication_config_in(
|
pub(crate) async fn load_delete_replication_config_in(
|
||||||
ctx: &ReplicationInstanceContext,
|
ctx: &ReplicationInstanceContext,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -217,6 +217,10 @@ impl DurableMrfBacklogTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||||
let mut tracker = DurableMrfBacklogTracker {
|
let mut tracker = DurableMrfBacklogTracker {
|
||||||
available: true,
|
available: true,
|
||||||
@@ -663,6 +667,368 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
|||||||
Err(EcstoreError::PreconditionFailed)
|
Err(EcstoreError::PreconditionFailed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||||
|
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||||
|
/// node is already processing the backlog.
|
||||||
|
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||||
|
let recovery_lock = match storage
|
||||||
|
.new_ns_lock(
|
||||||
|
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||||
|
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(lock) => lock,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to create the MRF recovery leader lock"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match recovery_lock
|
||||||
|
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(guard) => Some(guard),
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
"Another node is already processing the MRF recovery backlog"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads and decodes the on-disk MRF recovery file.
|
||||||
|
/// Returns `None` when there is nothing to replay: missing file (publishes an
|
||||||
|
/// empty available summary), read failure, or corrupt data (quarantined).
|
||||||
|
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
|
||||||
|
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(EcstoreError::ConfigNotFound) => {
|
||||||
|
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||||
|
available: true,
|
||||||
|
buckets: Vec::new(),
|
||||||
|
});
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %e,
|
||||||
|
"Failed to load MRF recovery file"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match decode_mrf_file(&data) {
|
||||||
|
Ok(v) => Some(v),
|
||||||
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %e,
|
||||||
|
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||||
|
);
|
||||||
|
quarantine_mrf_file(storage, &data).await;
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays one MRF recovery entry by operation kind.
|
||||||
|
/// Returns `None` when the entry is skipped entirely (no admission outcome);
|
||||||
|
/// entries that must be retried later are pushed onto `retry_entries`.
|
||||||
|
async fn replay_mrf_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
match entry.op {
|
||||||
|
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
|
||||||
|
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||||
|
replay_mrf_object_entry(entry, storage, retry_entries).await
|
||||||
|
}
|
||||||
|
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
|
||||||
|
/// stale force-delete generations are skipped, and plain deletes are
|
||||||
|
/// reconstructed as heal deletes.
|
||||||
|
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
if should_replay_force_delete_intent(entry) {
|
||||||
|
let operation_id = entry.force_delete_id?;
|
||||||
|
let delete = force_delete_heal_replication_info(entry, operation_id);
|
||||||
|
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
} else if entry.force_delete_id.is_some() {
|
||||||
|
Some(ReplicationQueueAdmission::Skipped)
|
||||||
|
} else {
|
||||||
|
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
|
||||||
|
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
|
||||||
|
DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: entry.target_arns.clone(),
|
||||||
|
force_delete_generation: entry.force_delete_generation,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||||
|
/// get_object_info here because the delete-marker or version may
|
||||||
|
/// already be absent from the local store — that is expected.
|
||||||
|
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||||
|
let oi = ObjectInfo {
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
delete_marker: entry.delete_marker,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
|
||||||
|
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
|
||||||
|
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The MRF entry does not persist the replication decision and the
|
||||||
|
/// source object is gone, so re-derive the decision from the live
|
||||||
|
/// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||||
|
/// it on the reconstructed delete. Without this the decision string
|
||||||
|
/// is empty and the delete replicates to zero targets — a silent
|
||||||
|
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||||
|
async fn resolve_mrf_delete_replicate_decision(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
oi: &ObjectInfo,
|
||||||
|
versioned: bool,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicateDecision> {
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||||
|
Ok(None) => None,
|
||||||
|
Err(_) => {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||||
|
&entry.bucket,
|
||||||
|
&ObjectToDelete {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
oi,
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(dsc) => Some(dsc),
|
||||||
|
Err(_) => {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
|
||||||
|
/// replication decision.
|
||||||
|
fn reconstructed_heal_delete_info(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
oi: &ObjectInfo,
|
||||||
|
dsc: &ReplicateDecision,
|
||||||
|
) -> DeletedObjectReplicationInfo {
|
||||||
|
let mut rstate = oi.replication_state();
|
||||||
|
rstate.replicate_decision_str = dsc.to_string();
|
||||||
|
|
||||||
|
let delete_marker_mtime = entry
|
||||||
|
.delete_marker_mtime
|
||||||
|
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||||
|
|
||||||
|
DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
delete_marker_version_id: entry.delete_marker_version_id,
|
||||||
|
delete_marker: entry.delete_marker,
|
||||||
|
delete_marker_mtime,
|
||||||
|
force_delete: entry.force_delete,
|
||||||
|
replication_state: Some(rstate),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
|
||||||
|
async fn replay_mrf_object_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_id: entry.version_id.map(|u| u.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||||
|
Ok(oi) => oi,
|
||||||
|
Err(e) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
bucket = %entry.bucket,
|
||||||
|
object = %entry.object,
|
||||||
|
error = %e,
|
||||||
|
"MRF recovery: source object lookup failed"
|
||||||
|
);
|
||||||
|
if should_retry_mrf_source_lookup(&e) {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
// Legacy entries predate target admission persistence. They cannot
|
||||||
|
// be safely attributed, so retain the old live-config fallback.
|
||||||
|
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||||
|
} else {
|
||||||
|
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
|
||||||
|
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays a metadata-kind MRF entry against the live source object.
|
||||||
|
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_id: entry.version_id.map(|u| u.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||||
|
Ok(oi) => oi,
|
||||||
|
Err(e) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
bucket = %entry.bucket,
|
||||||
|
object = %entry.object,
|
||||||
|
error = %e,
|
||||||
|
"MRF metadata recovery: source object lookup failed"
|
||||||
|
);
|
||||||
|
if should_retry_mrf_source_lookup(&e) {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||||
|
} else {
|
||||||
|
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
|
||||||
|
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: replicate-object info for an entry with persisted
|
||||||
|
/// admitted targets, carrying over the entry's retry count.
|
||||||
|
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||||
|
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||||
|
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||||
|
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||||
|
roi
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||||
|
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||||
|
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||||
|
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||||
|
storage: &Arc<S>,
|
||||||
|
recovery_guard: &rustfs_lock::NamespaceLockGuard,
|
||||||
|
entries: &[MrfReplicateEntry],
|
||||||
|
retry_entries: &[MrfReplicateEntry],
|
||||||
|
) -> Vec<MrfReplicateEntry> {
|
||||||
|
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
|
||||||
|
Ok(retained) => retained,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||||
|
);
|
||||||
|
match read_mrf_entries(storage.clone()).await {
|
||||||
|
Ok(current) => current,
|
||||||
|
Err(read_error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %read_error,
|
||||||
|
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||||
|
);
|
||||||
|
entries.to_vec()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||||
struct ResyncActiveConflictError {
|
struct ResyncActiveConflictError {
|
||||||
@@ -712,6 +1078,10 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
|||||||
|
|
||||||
// MRF worker lifecycle
|
// MRF worker lifecycle
|
||||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
mrf_stop_tx: Sender<()>,
|
mrf_stop_tx: Sender<()>,
|
||||||
|
|
||||||
// Worker size tracking
|
// Worker size tracking
|
||||||
@@ -940,6 +1310,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Resizes worker priority and counts
|
/// Resizes worker priority and counts
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn resize_worker_priority(
|
pub async fn resize_worker_priority(
|
||||||
&self,
|
&self,
|
||||||
pri: ReplicationPriority,
|
pri: ReplicationPriority,
|
||||||
@@ -1180,6 +1554,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Queues an MRF save operation
|
/// Queues an MRF save operation
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||||
}
|
}
|
||||||
@@ -1205,71 +1583,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
let storage = self.storage.clone();
|
let storage = self.storage.clone();
|
||||||
|
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
let recovery_lock = match storage
|
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||||
.new_ns_lock(
|
|
||||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
|
||||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(lock) => lock,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %error,
|
|
||||||
"Failed to create the MRF recovery leader lock"
|
|
||||||
);
|
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
};
|
|
||||||
let recovery_guard = match recovery_lock
|
|
||||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(guard) => guard,
|
|
||||||
Err(_) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
"Another node is already processing the MRF recovery backlog"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||||
Ok(d) => d,
|
|
||||||
Err(EcstoreError::ConfigNotFound) => {
|
|
||||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
|
||||||
available: true,
|
|
||||||
buckets: Vec::new(),
|
|
||||||
});
|
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %e,
|
|
||||||
"Failed to load MRF recovery file"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let entries = match decode_mrf_file(&data) {
|
|
||||||
Ok(v) => v,
|
|
||||||
Err(e) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %e,
|
|
||||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
|
||||||
);
|
|
||||||
quarantine_mrf_file(&storage, &data).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||||
|
|
||||||
@@ -1278,188 +1597,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
let mut retry_entries = Vec::new();
|
let mut retry_entries = Vec::new();
|
||||||
|
|
||||||
for entry in entries.iter() {
|
for entry in entries.iter() {
|
||||||
let admission = match entry.op {
|
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||||
MrfOpKind::Delete => {
|
|
||||||
if should_replay_force_delete_intent(entry) {
|
|
||||||
let Some(operation_id) = entry.force_delete_id else {
|
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let delete = DeletedObjectReplicationInfo {
|
|
||||||
delete_object: ReplicationDeletedObject {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
force_delete: true,
|
|
||||||
force_delete_id: Some(operation_id),
|
|
||||||
force_delete_target_arns: entry.target_arns.clone(),
|
|
||||||
force_delete_generation: entry.force_delete_generation,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
op_type: ReplicationType::Heal,
|
|
||||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
} else if entry.force_delete_id.is_some() {
|
|
||||||
ReplicationQueueAdmission::Skipped
|
|
||||||
} else {
|
|
||||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
|
||||||
// get_object_info here because the delete-marker or version may
|
|
||||||
// already be absent from the local store — that is expected.
|
|
||||||
//
|
|
||||||
// The MRF entry does not persist the replication decision and the
|
|
||||||
// source object is gone, so re-derive the decision from the live
|
|
||||||
// bucket config (mirroring get_heal_replicate_object_info) and set
|
|
||||||
// it on the reconstructed delete. Without this the decision string
|
|
||||||
// is empty and the delete replicates to zero targets — a silent
|
|
||||||
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
|
||||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
|
||||||
let oi = ObjectInfo {
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
delete_marker: entry.delete_marker,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let dsc = if entry.target_arns.is_empty() {
|
|
||||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
|
||||||
Ok(None) => continue,
|
|
||||||
Err(_) => {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
|
||||||
&entry.bucket,
|
|
||||||
&ObjectToDelete {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&oi,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(dsc) => dsc,
|
|
||||||
Err(_) => {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
replicate_decision_for_admitted_targets(&entry.target_arns)
|
|
||||||
};
|
|
||||||
let mut rstate = oi.replication_state();
|
|
||||||
rstate.replicate_decision_str = dsc.to_string();
|
|
||||||
|
|
||||||
let delete_marker_mtime = entry
|
|
||||||
.delete_marker_mtime
|
|
||||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
|
||||||
|
|
||||||
let dv = DeletedObjectReplicationInfo {
|
|
||||||
delete_object: ReplicationDeletedObject {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
delete_marker_version_id: entry.delete_marker_version_id,
|
|
||||||
delete_marker: entry.delete_marker,
|
|
||||||
delete_marker_mtime,
|
|
||||||
force_delete: entry.force_delete,
|
|
||||||
replication_state: Some(rstate),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
op_type: ReplicationType::Heal,
|
|
||||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
version_id: entry.version_id.map(|u| u.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
|
||||||
Ok(oi) => oi,
|
|
||||||
Err(e) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
bucket = %entry.bucket,
|
|
||||||
object = %entry.object,
|
|
||||||
error = %e,
|
|
||||||
"MRF recovery: source object lookup failed"
|
|
||||||
);
|
|
||||||
if should_retry_mrf_source_lookup(&e) {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if entry.target_arns.is_empty() {
|
|
||||||
// Legacy entries predate target admission persistence. They cannot
|
|
||||||
// be safely attributed, so retain the old live-config fallback.
|
|
||||||
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
|
||||||
} else {
|
|
||||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
|
||||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
|
|
||||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
|
||||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MrfOpKind::Metadata => {
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
version_id: entry.version_id.map(|u| u.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
|
||||||
Ok(oi) => oi,
|
|
||||||
Err(e) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
bucket = %entry.bucket,
|
|
||||||
object = %entry.object,
|
|
||||||
error = %e,
|
|
||||||
"MRF metadata recovery: source object lookup failed"
|
|
||||||
);
|
|
||||||
if should_retry_mrf_source_lookup(&e) {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if entry.target_arns.is_empty() {
|
|
||||||
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
|
||||||
} else {
|
|
||||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
|
||||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
|
||||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
|
||||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if admission == ReplicationQueueAdmission::Missed {
|
if admission == ReplicationQueueAdmission::Missed {
|
||||||
retry_entries.push(entry.clone());
|
retry_entries.push(entry.clone());
|
||||||
@@ -1468,29 +1608,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
|
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||||
Ok(retained) => retained,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %error,
|
|
||||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
|
||||||
);
|
|
||||||
match read_mrf_entries(storage.clone()).await {
|
|
||||||
Ok(current) => current,
|
|
||||||
Err(read_error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %read_error,
|
|
||||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
|
||||||
);
|
|
||||||
entries.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let retained_count = retained.len();
|
let retained_count = retained.len();
|
||||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||||
|
|
||||||
@@ -1651,6 +1769,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling regular replication operations
|
/// Worker function for handling regular replication operations
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn add_worker(
|
async fn add_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1664,6 +1786,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling large object replication operations
|
/// Worker function for handling large object replication operations
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn add_large_worker(
|
async fn add_large_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1678,6 +1804,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn add_mrf_worker(
|
async fn add_mrf_worker(
|
||||||
&self,
|
&self,
|
||||||
mut rx: Receiver<ReplicationOperation>,
|
mut rx: Receiver<ReplicationOperation>,
|
||||||
@@ -1691,6 +1821,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Delete resync metadata from replication resync state in memory
|
/// Delete resync metadata from replication resync state in memory
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||||
let mut status_map = self.resyncer.status_map.write().await;
|
let mut status_map = self.resyncer.status_map.write().await;
|
||||||
status_map.remove(bucket);
|
status_map.remove(bucket);
|
||||||
|
|||||||
@@ -21,11 +21,31 @@ pub(crate) use rustfs_replication::{
|
|||||||
should_count_head_proxy_failure,
|
should_count_head_proxy_failure,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||||
|
|
||||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -340,6 +340,10 @@ impl ReplicationStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Site replication update replica statistics
|
/// Site replication update replica statistics
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn sr_update_replica_stat(&self, size: i64) {
|
fn sr_update_replica_stat(&self, size: i64) {
|
||||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -704,6 +708,12 @@ impl ReplicationStats {
|
|||||||
} else {
|
} else {
|
||||||
BucketReplicationStats::new()
|
BucketReplicationStats::new()
|
||||||
};
|
};
|
||||||
|
// Stamp the serializable failure windows from the live samples: the
|
||||||
|
// samples themselves do not cross the peer-RPC wire, so this snapshot
|
||||||
|
// is what cluster aggregation and the metrics endpoints see.
|
||||||
|
for stat in replication_stats.stats.values_mut() {
|
||||||
|
stat.fail_stats.refresh_windows();
|
||||||
|
}
|
||||||
let uptime = if cache.contains_key(bucket) {
|
let uptime = if cache.contains_key(bucket) {
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(SystemTime::UNIX_EPOCH)
|
.duration_since(SystemTime::UNIX_EPOCH)
|
||||||
|
|||||||
@@ -15,7 +15,9 @@
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_replication::FailStats;
|
pub(crate) use rustfs_replication::FailStats;
|
||||||
pub(crate) use rustfs_replication::{
|
pub(crate) use rustfs_replication::{
|
||||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary,
|
||||||
SRMetricsSummary, XferStats,
|
|
||||||
};
|
};
|
||||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs)
|
||||||
|
// can project the internal stats onto the minio-go response shapes through
|
||||||
|
// the storage_api facade chain.
|
||||||
|
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||||
|
|||||||
@@ -27,8 +27,10 @@ use rustfs_utils::http::{
|
|||||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||||
is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
|
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP,
|
||||||
|
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
|
||||||
|
ssec_replication_transport_header,
|
||||||
};
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use time::format_description::well_known::Rfc3339;
|
use time::format_description::well_known::Rfc3339;
|
||||||
@@ -119,6 +121,27 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
|
||||||
|
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
|
||||||
|
let key = key.as_bytes();
|
||||||
|
key.len() == prefix.len() + suffix.len()
|
||||||
|
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
|
||||||
|
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
[
|
||||||
|
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
|
||||||
|
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
|
||||||
|
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|suffix| {
|
||||||
|
["x-rustfs-", "x-minio-"]
|
||||||
|
.iter()
|
||||||
|
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||||
}
|
}
|
||||||
@@ -176,6 +199,11 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if is_legacy_source_replication_timestamp_key(key) {
|
||||||
|
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if is_internal_key(key) || is_standard_header(key) {
|
if is_internal_key(key) || is_standard_header(key) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -259,15 +287,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
|
|
||||||
if !tags.is_empty() {
|
if !tags.is_empty() {
|
||||||
put_options.user_tags = tags;
|
put_options.user_tags = tags;
|
||||||
put_options.internal.tagging_timestamp =
|
}
|
||||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
|
}
|
||||||
|
// Load the stored tagging timestamp independently of whether any tags
|
||||||
|
// remain: DeleteObjectTagging leaves the object tagless but stamps this
|
||||||
|
// key, and the deletion's LWW timestamp must still reach the replica.
|
||||||
|
// With no stored key, fall back to mod_time only while tags exist
|
||||||
|
// (MinIO parity); a tagless object without the key was never tagged and
|
||||||
|
// keeps the epoch default (no header).
|
||||||
|
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
|
||||||
|
{
|
||||||
OffsetDateTime::parse(×tamp, &Rfc3339)
|
OffsetDateTime::parse(×tamp, &Rfc3339)
|
||||||
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
|
||||||
} else {
|
} else if !put_options.user_tags.is_empty() {
|
||||||
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||||
|
} else {
|
||||||
|
OffsetDateTime::UNIX_EPOCH
|
||||||
};
|
};
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let metadata = &*object_info.user_defined;
|
let metadata = &*object_info.user_defined;
|
||||||
|
|
||||||
@@ -283,13 +319,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
|||||||
put_options.cache_control = cache_control.to_string();
|
put_options.cache_control = cache_control.to_string();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
|
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) {
|
||||||
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
|
||||||
|
if !retain_until_date.is_empty() {
|
||||||
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
|
||||||
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
|
||||||
|
}
|
||||||
put_options.internal.retention_timestamp =
|
put_options.internal.retention_timestamp =
|
||||||
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
|
||||||
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||||
@@ -694,6 +732,110 @@ mod tests {
|
|||||||
assert!(options.internal.replication_request);
|
assert!(options.internal.replication_request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// DeleteObjectTagging leaves the object tagless but stamps the
|
||||||
|
/// tagging-timestamp internal key; the deletion's LWW timestamp must
|
||||||
|
/// still be loaded (and therefore sent) so the replica can order the
|
||||||
|
/// deletion against concurrent tag edits.
|
||||||
|
#[test]
|
||||||
|
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
|
||||||
|
let mut metadata = std::collections::HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||||
|
|
||||||
|
let object_info = ObjectInfo {
|
||||||
|
user_defined: Arc::new(metadata),
|
||||||
|
user_tags: Arc::new(String::new()),
|
||||||
|
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||||
|
version_id: Some(Uuid::nil()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||||
|
|
||||||
|
assert!(options.user_tags.is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
options.internal.tagging_timestamp,
|
||||||
|
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
|
||||||
|
"the stored tagging timestamp must load independently of remaining tags"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A tagless object without the stored key was never tagged: the epoch
|
||||||
|
// default keeps the header unsent.
|
||||||
|
let untagged = ObjectInfo {
|
||||||
|
user_tags: Arc::new(String::new()),
|
||||||
|
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
|
||||||
|
version_id: Some(Uuid::nil()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
|
||||||
|
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
|
||||||
|
let legacy_keys = [
|
||||||
|
"x-rustfs-source-replication-tagging-timestamp",
|
||||||
|
"x-rustfs-source-replication-retention-timestamp",
|
||||||
|
"x-rustfs-source-replication-legalhold-timestamp",
|
||||||
|
"x-minio-source-replication-tagging-timestamp",
|
||||||
|
"x-minio-source-replication-retention-timestamp",
|
||||||
|
"x-minio-source-replication-legalhold-timestamp",
|
||||||
|
];
|
||||||
|
let object_info = ObjectInfo {
|
||||||
|
user_defined: Arc::new(
|
||||||
|
legacy_keys
|
||||||
|
.iter()
|
||||||
|
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
|
||||||
|
|
||||||
|
for legacy_key in legacy_keys {
|
||||||
|
assert!(!options.user_metadata.contains_key(legacy_key));
|
||||||
|
assert_eq!(
|
||||||
|
options
|
||||||
|
.user_metadata
|
||||||
|
.get(&format!("x-amz-meta-{legacy_key}"))
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("2099-01-02T03:04:05Z")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replication_put_options_carry_retention_timestamp_after_clear() {
|
||||||
|
let mut metadata = HashMap::from([
|
||||||
|
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
|
||||||
|
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
|
||||||
|
]);
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
|
||||||
|
let object_info = ObjectInfo {
|
||||||
|
user_defined: Arc::new(metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
|
||||||
|
|
||||||
|
assert!(options.mode.is_none());
|
||||||
|
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
|
||||||
|
assert_eq!(
|
||||||
|
options.internal.retention_timestamp,
|
||||||
|
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
|
||||||
|
);
|
||||||
|
let headers = options.header();
|
||||||
|
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
|
||||||
|
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
|
||||||
|
assert_eq!(
|
||||||
|
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
|
||||||
|
Some("2026-01-02T03:04:05Z")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
|
||||||
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
|
||||||
|
|||||||
@@ -40,7 +40,14 @@ impl ARN {
|
|||||||
|
|
||||||
impl Display for ARN {
|
impl Display for ARN {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
// The `minio` partition is deliberate: madmin-go's ParseARN
|
||||||
|
// hard-rejects any other partition, so native mc/madmin tooling can
|
||||||
|
// only decode remote-target ARNs minted in this form (backlog#1675
|
||||||
|
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
|
||||||
|
// readable via the FromStr whitelist below; runtime matching between
|
||||||
|
// targets and replication rules is by full-string equality, so mixed
|
||||||
|
// partitions coexist safely.
|
||||||
|
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,7 +55,12 @@ impl FromStr for ARN {
|
|||||||
type Err = std::io::Error;
|
type Err = std::io::Error;
|
||||||
|
|
||||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||||
if !s.starts_with("arn:rustfs:") {
|
// Partition whitelist, not just an `arn:` check: `BucketTargetType::
|
||||||
|
// from_str(...).unwrap_or_default()` below never fails, so this is
|
||||||
|
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
|
||||||
|
// the legacy partition and must stay accepted forever (persisted
|
||||||
|
// bucket-targets.json / replication configs from older releases).
|
||||||
|
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
|
||||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,14 +113,50 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// RustFS commonly generates ARNs with an empty region:
|
/// RustFS commonly generates ARNs with an empty region:
|
||||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
/// `arn:minio:replication::<deployment_id>:<bucket>`.
|
||||||
#[test]
|
#[test]
|
||||||
fn from_str_handles_empty_region_segment() {
|
fn from_str_handles_empty_region_segment() {
|
||||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||||
|
|
||||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||||
assert_eq!(parsed.id, "depl-123");
|
assert_eq!(parsed.id, "depl-123");
|
||||||
assert_eq!(parsed.bucket, "bucket-a");
|
assert_eq!(parsed.bucket, "bucket-a");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
|
||||||
|
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
|
||||||
|
/// native mc/madmin tooling cannot decode remote-target listings.
|
||||||
|
#[test]
|
||||||
|
fn display_emits_minio_partition() {
|
||||||
|
let arn = ARN::new(
|
||||||
|
BucketTargetType::ReplicationService,
|
||||||
|
"depl-123".to_string(),
|
||||||
|
String::new(),
|
||||||
|
"bucket-a".to_string(),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persisted bucket-targets.json files from older RustFS releases carry
|
||||||
|
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
|
||||||
|
#[test]
|
||||||
|
fn from_str_accepts_legacy_rustfs_partition() {
|
||||||
|
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
|
||||||
|
|
||||||
|
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||||
|
assert_eq!(parsed.region, "us-east-1");
|
||||||
|
assert_eq!(parsed.id, "depl-123");
|
||||||
|
assert_eq!(parsed.bucket, "bucket-a");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The partition whitelist is the only structural gate: `BucketTargetType::
|
||||||
|
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
|
||||||
|
/// would otherwise parse as `type=None`.
|
||||||
|
#[test]
|
||||||
|
fn from_str_rejects_unknown_partition() {
|
||||||
|
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
|
||||||
|
assert!(ARN::from_str("not-an-arn").is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ impl fmt::Debug for Credentials {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub enum ServiceType {
|
pub enum ServiceType {
|
||||||
#[default]
|
#[default]
|
||||||
Replication,
|
Replication,
|
||||||
|
|||||||
@@ -73,23 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
|||||||
check_bucket_name_common(bucket_name, true)
|
check_bucket_name_common(bucket_name, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
|
||||||
if object_name.len() > 1024 {
|
|
||||||
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
|
||||||
}
|
|
||||||
if !object_name.is_ascii() {
|
|
||||||
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
|
||||||
if object_name.trim().is_empty() {
|
|
||||||
return Err(Error::other("Object name cannot be empty"));
|
|
||||||
}
|
|
||||||
check_valid_object_name_prefix(object_name)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||||
where
|
where
|
||||||
T: for<'xml> xml::Deserialize<'xml>,
|
T: for<'xml> xml::Deserialize<'xml>,
|
||||||
@@ -100,6 +83,10 @@ where
|
|||||||
Ok(ans)
|
Ok(ans)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||||
let mut buf = Vec::with_capacity(256);
|
let mut buf = Vec::with_capacity(256);
|
||||||
{
|
{
|
||||||
@@ -186,15 +173,27 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
|||||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||||
/// accept the full S3 key character set.
|
/// accept the full S3 key character set.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||||
|
|
||||||
/// Client-facing reason for path segments Windows can store but not address
|
/// Client-facing reason for path segments Windows can store but not address
|
||||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||||
|
|
||||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||||
@@ -204,6 +203,10 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
|||||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||||
object.split(['/', '\\']).any(|segment| {
|
object.split(['/', '\\']).any(|segment| {
|
||||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||||
|
|||||||
@@ -90,6 +90,10 @@ impl BucketVersioningSys {
|
|||||||
/// caller's own instance context so a second in-process store never
|
/// caller's own instance context so a second in-process store never
|
||||||
/// answers with the first instance's versioning state; falls back to the
|
/// answers with the first instance's versioning state; falls back to the
|
||||||
/// ambient system when the instance cell is not initialized.
|
/// ambient system when the instance cell is not initialized.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||||
return Ok(VersioningConfiguration::default());
|
return Ok(VersioningConfiguration::default());
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||||
use crate::disk::error::DiskError;
|
use crate::disk::error::DiskError;
|
||||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||||
|
use futures::future::join_all;
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
|
||||||
use std::{
|
use std::{
|
||||||
@@ -655,6 +656,7 @@ async fn list_path_raw_inner(
|
|||||||
errs.push(None);
|
errs.push(None);
|
||||||
}
|
}
|
||||||
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||||
|
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let mut current = MetaCacheEntry::default();
|
let mut current = MetaCacheEntry::default();
|
||||||
@@ -676,6 +678,21 @@ async fn list_path_raw_inner(
|
|||||||
let mut has_err = 0;
|
let mut has_err = 0;
|
||||||
let mut agree = 0;
|
let mut agree = 0;
|
||||||
|
|
||||||
|
// Start every missing head read in the same round so one stalled
|
||||||
|
// disk cannot multiply the wait budget by the erasure-set width.
|
||||||
|
// Outcomes are still consumed below in stable disk-index order.
|
||||||
|
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||||
|
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cancel = &revjob_rx;
|
||||||
|
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
|
||||||
|
});
|
||||||
|
for (i, outcome) in join_all(concurrent_peeks).await {
|
||||||
|
peek_outcomes[i] = Some(outcome);
|
||||||
|
}
|
||||||
|
|
||||||
for (i, r) in readers.iter_mut().enumerate() {
|
for (i, r) in readers.iter_mut().enumerate() {
|
||||||
if errs[i].is_some() {
|
if errs[i].is_some() {
|
||||||
has_err += 1;
|
has_err += 1;
|
||||||
@@ -685,7 +702,10 @@ async fn list_path_raw_inner(
|
|||||||
let entry = if let Some(entry) = pending_entries[i].take() {
|
let entry = if let Some(entry) = pending_entries[i].take() {
|
||||||
entry
|
entry
|
||||||
} else {
|
} else {
|
||||||
match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
|
let Some(outcome) = peek_outcomes[i].take() else {
|
||||||
|
return Err(DiskError::Unexpected);
|
||||||
|
};
|
||||||
|
match outcome {
|
||||||
PeekOutcome::Ready(res) => {
|
PeekOutcome::Ready(res) => {
|
||||||
if let Some(entry) = res {
|
if let Some(entry) = res {
|
||||||
// info!("read entry disk: {}, name: {}", i, entry.name);
|
// info!("read entry disk: {}, name: {}", i, entry.name);
|
||||||
@@ -1295,6 +1315,36 @@ mod tests {
|
|||||||
assert_eq!(err, DiskError::Timeout);
|
assert_eq!(err, DiskError::Timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
|
||||||
|
let peek_timeout = Duration::from_millis(20);
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
let err = list_path_raw(
|
||||||
|
CancellationToken::new(),
|
||||||
|
ListPathRawOptions {
|
||||||
|
disks: vec![None, None, None, None],
|
||||||
|
min_disks: 1,
|
||||||
|
test_reader_behaviors: vec![
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
TestReaderBehavior::Stall,
|
||||||
|
],
|
||||||
|
peek_timeout: Some(peek_timeout),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("all stalled readers should fail the listing");
|
||||||
|
|
||||||
|
assert_eq!(err, DiskError::Timeout);
|
||||||
|
assert_eq!(
|
||||||
|
started.elapsed(),
|
||||||
|
peek_timeout,
|
||||||
|
"reader deadlines must overlap instead of accumulating once per disk"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||||
let entry = MetaCacheEntry {
|
let entry = MetaCacheEntry {
|
||||||
|
|||||||
@@ -229,17 +229,6 @@ pub fn http_resp_to_error_response(
|
|||||||
err_resp
|
err_resp
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
|
||||||
ErrorResponse {
|
|
||||||
status_code: StatusCode::BAD_REQUEST,
|
|
||||||
code: S3ErrorCode::InvalidArgument,
|
|
||||||
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
|
||||||
.to_string(),
|
|
||||||
bucket_name: bucket_name.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||||
let msg = format!(
|
let msg = format!(
|
||||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||||
@@ -295,16 +284,6 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
|
||||||
ErrorResponse {
|
|
||||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
|
||||||
code: S3ErrorCode::Custom("APINotSupported".into()),
|
|
||||||
message: message.to_string(),
|
|
||||||
request_id: "rustfs".to_string(),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -135,6 +135,10 @@ impl Object {
|
|||||||
Self { ..Default::default() }
|
Self { ..Default::default() }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||||
let _ = request.did_offset_change;
|
let _ = request.did_offset_change;
|
||||||
let _ = request.offset;
|
let _ = request.offset;
|
||||||
@@ -150,12 +154,20 @@ impl Object {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||||
self.curr_offset += bytes_read;
|
self.curr_offset += bytes_read;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||||
let mut read_req = GetRequest {
|
let mut read_req = GetRequest {
|
||||||
is_read_op: true,
|
is_read_op: true,
|
||||||
@@ -180,6 +192,10 @@ impl Object {
|
|||||||
Ok(response.size)
|
Ok(response.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||||
if !self.is_started || !self.object_info_set {
|
if !self.is_started || !self.object_info_set {
|
||||||
let _ = self.do_get_request(&GetRequest {
|
let _ = self.do_get_request(&GetRequest {
|
||||||
@@ -192,6 +208,10 @@ impl Object {
|
|||||||
Ok(self.object_info.clone())
|
Ok(self.object_info.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||||
self.curr_offset = offset;
|
self.curr_offset = offset;
|
||||||
|
|
||||||
@@ -219,6 +239,10 @@ impl Object {
|
|||||||
Ok(response.size)
|
Ok(response.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||||
if !self.is_started || !self.object_info_set {
|
if !self.is_started || !self.object_info_set {
|
||||||
let seek_req = GetRequest {
|
let seek_req = GetRequest {
|
||||||
@@ -253,6 +277,10 @@ impl Object {
|
|||||||
Ok(self.curr_offset)
|
Ok(self.curr_offset)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||||
self.is_closed = true;
|
self.is_closed = true;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ use crate::client::{
|
|||||||
api_put_object_common::optimal_part_info,
|
api_put_object_common::optimal_part_info,
|
||||||
api_put_object_multipart::UploadPartParams,
|
api_put_object_multipart::UploadPartParams,
|
||||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE},
|
||||||
credentials::SignatureType,
|
credentials::SignatureType,
|
||||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||||
|
|||||||
@@ -30,10 +30,6 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
|
|||||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
|
||||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||||
let unknown_size;
|
let unknown_size;
|
||||||
let mut object_size = object_size;
|
let mut object_size = object_size;
|
||||||
|
|||||||
@@ -81,18 +81,6 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct UploadedPartRes {
|
|
||||||
pub error: std::io::Error,
|
|
||||||
pub part_num: i64,
|
|
||||||
pub size: i64,
|
|
||||||
pub part: ObjectPart,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct UploadPartReq {
|
|
||||||
pub part_num: i64,
|
|
||||||
pub part: ObjectPart,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TransitionClient {
|
impl TransitionClient {
|
||||||
pub async fn put_object_multipart_stream(
|
pub async fn put_object_multipart_stream(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
|
|||||||
@@ -29,10 +29,6 @@ use crate::client::utils::base64_decode;
|
|||||||
|
|
||||||
use super::transition_api;
|
use super::transition_api;
|
||||||
|
|
||||||
pub struct ListAllMyBucketsResult {
|
|
||||||
pub owner: Owner,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
pub struct CommonPrefix {
|
pub struct CommonPrefix {
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
@@ -89,6 +85,10 @@ pub struct ListVersionsResult {
|
|||||||
pub next_version_id_marker: String,
|
pub next_version_id_marker: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct ListBucketResult {
|
pub struct ListBucketResult {
|
||||||
common_prefixes: Vec<CommonPrefix>,
|
common_prefixes: Vec<CommonPrefix>,
|
||||||
contents: Vec<transition_api::ObjectInfo>,
|
contents: Vec<transition_api::ObjectInfo>,
|
||||||
@@ -102,6 +102,10 @@ pub struct ListBucketResult {
|
|||||||
prefix: String,
|
prefix: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct ListMultipartUploadsResult {
|
pub struct ListMultipartUploadsResult {
|
||||||
bucket: String,
|
bucket: String,
|
||||||
key_marker: String,
|
key_marker: String,
|
||||||
@@ -117,16 +121,15 @@ pub struct ListMultipartUploadsResult {
|
|||||||
common_prefixes: Vec<CommonPrefix>,
|
common_prefixes: Vec<CommonPrefix>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct Initiator {
|
pub struct Initiator {
|
||||||
id: String,
|
id: String,
|
||||||
display_name: String,
|
display_name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CopyObjectResult {
|
|
||||||
pub etag: String,
|
|
||||||
pub last_modified: OffsetDateTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ObjectPart {
|
pub struct ObjectPart {
|
||||||
pub etag: String,
|
pub etag: String,
|
||||||
@@ -260,6 +263,7 @@ pub struct CompletePart {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CompletePart {
|
impl CompletePart {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||||
match t {
|
match t {
|
||||||
ChecksumMode::ChecksumCRC32C => {
|
ChecksumMode::ChecksumCRC32C => {
|
||||||
@@ -284,11 +288,6 @@ impl CompletePart {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CopyObjectPartResult {
|
|
||||||
pub etag: String,
|
|
||||||
pub last_modified: OffsetDateTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Default, serde::Serialize)]
|
#[derive(Debug, Default, serde::Serialize)]
|
||||||
#[serde(rename = "CompleteMultipartUpload")]
|
#[serde(rename = "CompleteMultipartUpload")]
|
||||||
pub struct CompleteMultipartUpload {
|
pub struct CompleteMultipartUpload {
|
||||||
@@ -357,10 +356,10 @@ impl CompleteMultipartUpload {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CreateBucketConfiguration {
|
#[allow(
|
||||||
pub location: String,
|
dead_code,
|
||||||
}
|
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)"
|
||||||
|
)]
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct DeleteObject {
|
pub struct DeleteObject {
|
||||||
//api has
|
//api has
|
||||||
@@ -368,21 +367,6 @@ pub struct DeleteObject {
|
|||||||
pub version_id: String,
|
pub version_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct DeletedObject {
|
|
||||||
//s3s has
|
|
||||||
pub key: String,
|
|
||||||
pub version_id: String,
|
|
||||||
pub deletemarker: bool,
|
|
||||||
pub deletemarker_version_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct NonDeletedObject {
|
|
||||||
pub key: String,
|
|
||||||
pub code: String,
|
|
||||||
pub message: String,
|
|
||||||
pub version_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct DeleteMultiObjects {
|
pub struct DeleteMultiObjects {
|
||||||
pub quiet: bool,
|
pub quiet: bool,
|
||||||
@@ -402,6 +386,7 @@ impl DeleteMultiObjects {
|
|||||||
Ok(buf)
|
Ok(buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
|
||||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct WireDeleteObject {
|
struct WireDeleteObject {
|
||||||
@@ -436,8 +421,3 @@ impl DeleteMultiObjects {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct DeleteMultiObjectsResult {
|
|
||||||
pub deleted_objects: Vec<DeletedObject>,
|
|
||||||
pub undeleted_objects: Vec<NonDeletedObject>,
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -365,6 +365,10 @@ mod tests {
|
|||||||
pub struct Checksum {
|
pub struct Checksum {
|
||||||
checksum_type: ChecksumMode,
|
checksum_type: ChecksumMode,
|
||||||
r: Vec<u8>,
|
r: Vec<u8>,
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
|
||||||
|
)]
|
||||||
computed: bool,
|
computed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,8 +32,5 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
|||||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||||
|
|
||||||
pub const TOTAL_WORKERS: i64 = 4;
|
|
||||||
|
|
||||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
|
||||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||||
|
|||||||
@@ -67,6 +67,10 @@ impl<P: Provider + Default> Credentials<P> {
|
|||||||
Ok(self.creds.clone())
|
Ok(self.creds.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn expire(&mut self) {
|
fn expire(&mut self) {
|
||||||
self.force_refresh = true;
|
self.force_refresh = true;
|
||||||
}
|
}
|
||||||
@@ -133,6 +137,10 @@ impl Provider for Static {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct STSError {
|
pub struct STSError {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub r#type: String,
|
pub r#type: String,
|
||||||
pub code: String,
|
pub code: String,
|
||||||
pub message: String,
|
pub message: String,
|
||||||
@@ -141,6 +149,10 @@ pub struct STSError {
|
|||||||
#[derive(Debug, Clone, thiserror::Error)]
|
#[derive(Debug, Clone, thiserror::Error)]
|
||||||
pub struct ErrorResponse {
|
pub struct ErrorResponse {
|
||||||
pub sts_error: STSError,
|
pub sts_error: STSError,
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
|
||||||
|
)]
|
||||||
pub request_id: String,
|
pub request_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,22 +170,3 @@ impl ErrorResponse {
|
|||||||
return self.sts_error.message.clone();
|
return self.sts_error.message.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
|
||||||
where
|
|
||||||
for<'de> T: Deserialize<'de>,
|
|
||||||
{
|
|
||||||
match std::str::from_utf8(body) {
|
|
||||||
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
|
||||||
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
|
||||||
where
|
|
||||||
for<'de> T: Deserialize<'de>,
|
|
||||||
{
|
|
||||||
let body = body_reader.to_vec();
|
|
||||||
let parsed = xml_decoder(&body)?;
|
|
||||||
Ok((body, parsed))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
pub mod admin_handler_utils;
|
pub mod admin_handler_utils;
|
||||||
pub mod api_error_response;
|
pub mod api_error_response;
|
||||||
|
|||||||
@@ -77,39 +77,6 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
|
|
||||||
let mut skip_length: i64 = 0;
|
|
||||||
let mut cumulative_actual_size: i64 = 0;
|
|
||||||
let mut first_part_idx: i64 = 0;
|
|
||||||
let mut compressed_offset: i64 = 0;
|
|
||||||
let mut part_skip: i64 = 0;
|
|
||||||
let mut decrypt_skip: i64 = 0;
|
|
||||||
let mut seq_num: u64 = 0;
|
|
||||||
for (i, part) in oi.parts.iter().enumerate() {
|
|
||||||
cumulative_actual_size += part.actual_size as i64;
|
|
||||||
if cumulative_actual_size <= offset {
|
|
||||||
compressed_offset += part.size as i64;
|
|
||||||
} else {
|
|
||||||
first_part_idx = i as i64;
|
|
||||||
skip_length = cumulative_actual_size - part.actual_size as i64;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
skip_length = offset - skip_length;
|
|
||||||
|
|
||||||
let parts: &[ObjectPartInfo] = &oi.parts;
|
|
||||||
if skip_length > 0
|
|
||||||
&& parts.len() > first_part_idx as usize
|
|
||||||
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
|
|
||||||
{
|
|
||||||
let _ = part_skip;
|
|
||||||
let _ = decrypt_skip;
|
|
||||||
let _ = seq_num;
|
|
||||||
}
|
|
||||||
|
|
||||||
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_getobjectreader<'a>(
|
pub fn new_getobjectreader<'a>(
|
||||||
rs: &Option<HTTPRangeSpec>,
|
rs: &Option<HTTPRangeSpec>,
|
||||||
oi: &'a ObjectInfo,
|
oi: &'a ObjectInfo,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id";
|
|||||||
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
||||||
pub(crate) enum BucketVersioningState {
|
pub(crate) enum BucketVersioningState {
|
||||||
Unknown,
|
Unknown,
|
||||||
Disabled,
|
Disabled,
|
||||||
@@ -47,6 +48,7 @@ impl RemoteVersion {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||||
match self {
|
match self {
|
||||||
Self::Unknown => Err(Error::new(
|
Self::Unknown => Err(Error::new(
|
||||||
|
|||||||
@@ -101,6 +101,10 @@ where
|
|||||||
|
|
||||||
const C_UNKNOWN: i32 = -1;
|
const C_UNKNOWN: i32 = -1;
|
||||||
const C_OFFLINE: i32 = 0;
|
const C_OFFLINE: i32 = 0;
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reachable only from the unused transition client methods below (backlog#1823)"
|
||||||
|
)]
|
||||||
const C_ONLINE: i32 = 1;
|
const C_ONLINE: i32 = 1;
|
||||||
|
|
||||||
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
|
||||||
@@ -320,6 +324,10 @@ impl TransitionClient {
|
|||||||
Ok(client)
|
Ok(client)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn endpoint_url(&self) -> Url {
|
fn endpoint_url(&self) -> Url {
|
||||||
self.endpoint_url.clone()
|
self.endpoint_url.clone()
|
||||||
}
|
}
|
||||||
@@ -348,12 +356,20 @@ impl TransitionClient {
|
|||||||
.to_string())
|
.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn trace_errors_only_off(&self) {
|
fn trace_errors_only_off(&self) {
|
||||||
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
|
||||||
*trace_errors_only = false;
|
*trace_errors_only = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn trace_off(&self) {
|
fn trace_off(&self) {
|
||||||
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
|
||||||
*is_trace_enabled = false;
|
*is_trace_enabled = false;
|
||||||
@@ -363,12 +379,20 @@ impl TransitionClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||||
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
|
||||||
*endpoint = accelerate_endpoint.to_string();
|
*endpoint = accelerate_endpoint.to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||||
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
|
||||||
*dual_stack = enabled;
|
*dual_stack = enabled;
|
||||||
@@ -398,10 +422,18 @@ impl TransitionClient {
|
|||||||
(hash_algos, hash_sums)
|
(hash_algos, hash_sums)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn is_online(&self) -> bool {
|
fn is_online(&self) -> bool {
|
||||||
!self.is_offline()
|
!self.is_offline()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn mark_offline(&self) {
|
fn mark_offline(&self) {
|
||||||
self.health_status
|
self.health_status
|
||||||
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
|
||||||
@@ -411,10 +443,18 @@ impl TransitionClient {
|
|||||||
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
self.health_status.load(Ordering::SeqCst) == C_OFFLINE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn health_check(hc_duration: Duration) {
|
fn health_check(hc_duration: Duration) {
|
||||||
let _ = hc_duration;
|
let _ = hc_duration;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
|
||||||
let mut resp_trace: Vec<u8>;
|
let mut resp_trace: Vec<u8>;
|
||||||
|
|
||||||
@@ -1102,6 +1142,7 @@ impl Default for ObjectInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectInfo {
|
impl ObjectInfo {
|
||||||
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
pub(crate) fn remote_version(
|
pub(crate) fn remote_version(
|
||||||
&self,
|
&self,
|
||||||
capabilities: ProviderVersionCapabilities,
|
capabilities: ProviderVersionCapabilities,
|
||||||
|
|||||||
@@ -48,10 +48,6 @@ lazy_static! {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_standard_query_value(qs_key: &str) -> bool {
|
|
||||||
SUPPORTED_QUERY_VALUES[qs_key]
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn is_storageclass_header(header_key: &str) -> bool {
|
pub fn is_storageclass_header(header_key: &str) -> bool {
|
||||||
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -190,6 +190,17 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su
|
|||||||
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size";
|
||||||
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
||||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
||||||
@@ -551,6 +562,32 @@ mod tests {
|
|||||||
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||||
|
"data_read_inline_body_verify"
|
||||||
|
);
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry");
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
||||||
|
"data_read_inline_identity_mismatch"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
||||||
|
"data_read_inline_missing_payload"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
|
||||||
|
"data_read_inline_missing_shard"
|
||||||
|
);
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote");
|
||||||
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size");
|
||||||
|
assert_eq!(
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||||
|
"data_read_inline_transformed"
|
||||||
|
);
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
||||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
||||||
|
|||||||
@@ -637,14 +637,23 @@ impl Default for DiskOperationMetrics {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl DiskOperationMetrics {
|
impl DiskOperationMetrics {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||||
|
)]
|
||||||
fn record_call(&mut self) {
|
fn record_call(&mut self) {
|
||||||
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||||
|
)]
|
||||||
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
||||||
self.record_latency_atomic(now_sec, elapsed);
|
self.record_latency_atomic(now_sec, elapsed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")]
|
||||||
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
||||||
self.record_call();
|
self.record_call();
|
||||||
self.record_latency(now_sec, elapsed);
|
self.record_latency(now_sec, elapsed);
|
||||||
@@ -770,6 +779,7 @@ impl DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Set disk as faulty
|
/// Set disk as faulty
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn set_faulty(&self) {
|
pub fn set_faulty(&self) {
|
||||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||||
}
|
}
|
||||||
@@ -850,6 +860,7 @@ impl DiskHealthTracker {
|
|||||||
became_offline
|
became_offline
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||||
let current = self.runtime_state();
|
let current = self.runtime_state();
|
||||||
if current == RuntimeDriveHealthState::Offline {
|
if current == RuntimeDriveHealthState::Offline {
|
||||||
@@ -980,11 +991,13 @@ impl DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get waiting operations count
|
/// Get waiting operations count
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn waiting_count(&self) -> u32 {
|
pub fn waiting_count(&self) -> u32 {
|
||||||
self.waiting.load(Ordering::Relaxed)
|
self.waiting.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get last success timestamp
|
/// Get last success timestamp
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn last_success(&self) -> i64 {
|
pub fn last_success(&self) -> i64 {
|
||||||
self.last_success.load(Ordering::Acquire)
|
self.last_success.load(Ordering::Acquire)
|
||||||
}
|
}
|
||||||
@@ -1026,21 +1039,6 @@ impl Default for DiskHealthTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Health check context key for tracking disk operations
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
struct HealthDiskCtxKey;
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct HealthDiskCtxValue {
|
|
||||||
last_success: Arc<AtomicI64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl HealthDiskCtxValue {
|
|
||||||
fn log_success(&self) {
|
|
||||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -1072,10 +1070,6 @@ impl LocalDiskWrapper {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn new_with_health(disk: Arc<LocalDisk>, health_check: bool, health: Arc<DiskHealthTracker>) -> Self {
|
|
||||||
Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default()))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn new_with_reconnect_state(
|
pub(crate) fn new_with_reconnect_state(
|
||||||
disk: Arc<LocalDisk>,
|
disk: Arc<LocalDisk>,
|
||||||
health_check: bool,
|
health_check: bool,
|
||||||
@@ -1438,20 +1432,6 @@ impl LocalDiskWrapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn check_id(&self, want_id: Option<Uuid>) -> Result<()> {
|
|
||||||
if want_id.is_none() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
let stored_disk_id = self.disk.get_disk_id().await?;
|
|
||||||
|
|
||||||
if stored_disk_id != want_id {
|
|
||||||
return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}")));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Check if disk ID is stale
|
/// Check if disk ID is stale
|
||||||
async fn check_disk_stale(&self) -> Result<()> {
|
async fn check_disk_stale(&self) -> Result<()> {
|
||||||
let Some(current_disk_id) = *self.disk_id.read().await else {
|
let Some(current_disk_id) = *self.disk_id.read().await else {
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
||||||
match io_err.kind() {
|
match io_err.kind() {
|
||||||
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
||||||
|
|||||||
@@ -178,6 +178,7 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||||
match fs::remove_file(path.as_ref()).await {
|
match fs::remove_file(path.as_ref()).await {
|
||||||
|
|||||||
@@ -665,6 +665,7 @@ async fn remove_empty_directory_tree_under_mount_lease(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn remove_empty_directory_tree_with(
|
async fn remove_empty_directory_tree_with(
|
||||||
root: &Path,
|
root: &Path,
|
||||||
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||||
@@ -1016,13 +1017,29 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str,
|
|||||||
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
||||||
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
||||||
/// Default: false (uses page cache via mmap/pread).
|
/// Default: false (uses page cache via mmap/pread).
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
||||||
|
|
||||||
/// Minimum shard size threshold for O_DIRECT reads.
|
/// Minimum shard size threshold for O_DIRECT reads.
|
||||||
/// Only shards larger than this threshold will use O_DIRECT.
|
/// Only shards larger than this threshold will use O_DIRECT.
|
||||||
/// Default: 4MB.
|
/// Default: 4MB.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||||
|
|
||||||
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
||||||
@@ -1036,7 +1053,15 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
|||||||
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
||||||
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
||||||
/// Default: false (buffered writes via the page cache, as before).
|
/// Default: false (buffered writes via the page cache, as before).
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
||||||
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
||||||
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
||||||
@@ -1095,12 +1120,14 @@ macro_rules! cached_read_env {
|
|||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Check if O_DIRECT reads are enabled.
|
/// Check if O_DIRECT reads are enabled.
|
||||||
|
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||||
fn is_direct_io_read_enabled() -> bool =
|
fn is_direct_io_read_enabled() -> bool =
|
||||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
||||||
}
|
}
|
||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Check if O_DIRECT shard/part data writes are enabled.
|
/// Check if O_DIRECT shard/part data writes are enabled.
|
||||||
|
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||||
fn is_direct_io_write_enabled() -> bool =
|
fn is_direct_io_write_enabled() -> bool =
|
||||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
||||||
}
|
}
|
||||||
@@ -1456,6 +1483,7 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
|||||||
|
|
||||||
cached_read_env! {
|
cached_read_env! {
|
||||||
/// Get the O_DIRECT read threshold size.
|
/// Get the O_DIRECT read threshold size.
|
||||||
|
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||||
fn get_direct_io_read_threshold() -> usize =
|
fn get_direct_io_read_threshold() -> usize =
|
||||||
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
||||||
}
|
}
|
||||||
@@ -1673,12 +1701,20 @@ impl DirectIoWriteState {
|
|||||||
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
||||||
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
||||||
/// into one positioned write to keep the syscall count low.
|
/// into one positioned write to keep the syscall count low.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
||||||
|
|
||||||
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
||||||
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
||||||
/// flushed batch length, and every write offset stay alignment-correct.
|
/// flushed batch length, and every write offset stay alignment-correct.
|
||||||
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
fn direct_write_staging_capacity(align: usize) -> usize {
|
fn direct_write_staging_capacity(align: usize) -> usize {
|
||||||
debug_assert!(align.is_power_of_two() && align >= 512);
|
debug_assert!(align.is_power_of_two() && align >= 512);
|
||||||
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
||||||
@@ -1687,6 +1723,10 @@ fn direct_write_staging_capacity(align: usize) -> usize {
|
|||||||
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
||||||
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
||||||
/// so the tail-boundary math is unit-tested on any host.
|
/// so the tail-boundary math is unit-tested on any host.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||||
|
)]
|
||||||
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
||||||
let aligned = filled - (filled % align);
|
let aligned = filled - (filled % align);
|
||||||
(aligned, filled - aligned)
|
(aligned, filled - aligned)
|
||||||
@@ -2142,6 +2182,7 @@ fn set_delete_version_fail_after_data_staged(path: &str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
||||||
DELETE_VERSION_FAIL_AFTER_COMMIT
|
DELETE_VERSION_FAIL_AFTER_COMMIT
|
||||||
.lock()
|
.lock()
|
||||||
@@ -2447,6 +2488,10 @@ enum SyncMode {
|
|||||||
FileOnly,
|
FileOnly,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||||
|
)]
|
||||||
struct FileCacheReclaimWriter {
|
struct FileCacheReclaimWriter {
|
||||||
inner: File,
|
inner: File,
|
||||||
reclaim_len: usize,
|
reclaim_len: usize,
|
||||||
@@ -2454,6 +2499,10 @@ struct FileCacheReclaimWriter {
|
|||||||
reclaimed: bool,
|
reclaimed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||||
|
)]
|
||||||
struct FileCacheReclaimReader {
|
struct FileCacheReclaimReader {
|
||||||
inner: File,
|
inner: File,
|
||||||
reclaim_offset: u64,
|
reclaim_offset: u64,
|
||||||
@@ -2519,6 +2568,10 @@ impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)"
|
||||||
|
)]
|
||||||
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
||||||
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
||||||
// (three metric-key constructions) when general metrics are disabled.
|
// (three metric-key constructions) when general metrics are disabled.
|
||||||
@@ -3071,6 +3124,7 @@ impl LocalIoBackend for StdBackend {
|
|||||||
use memmap2::MmapOptions;
|
use memmap2::MmapOptions;
|
||||||
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")]
|
||||||
struct MmapCopyReadResult {
|
struct MmapCopyReadResult {
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
access_check_duration: StdDuration,
|
access_check_duration: StdDuration,
|
||||||
@@ -4704,6 +4758,10 @@ fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
|||||||
Arc::new(StdBackend::new(root))
|
Arc::new(StdBackend::new(root))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct LocalDisk {
|
pub struct LocalDisk {
|
||||||
pub root: PathBuf,
|
pub root: PathBuf,
|
||||||
publication_root: os::PublicationRoot,
|
publication_root: os::PublicationRoot,
|
||||||
@@ -5490,6 +5548,7 @@ impl LocalDisk {
|
|||||||
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
||||||
let path_ref = path.as_ref();
|
let path_ref = path.as_ref();
|
||||||
let path_str = path_ref.to_string_lossy();
|
let path_str = path_ref.to_string_lossy();
|
||||||
@@ -5567,15 +5626,24 @@ impl LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check if a path is valid
|
// Check if a path is valid
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||||
check_local_disk_valid_path(self.io_root(), path)
|
check_local_disk_valid_path(self.io_root(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
||||||
reject_local_disk_symlink_components(self.io_root(), path)
|
reject_local_disk_symlink_components(self.io_root(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch path generation with single lock acquisition
|
// Batch path generation with single lock acquisition
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
||||||
let mut results = Vec::with_capacity(requests.len());
|
let mut results = Vec::with_capacity(requests.len());
|
||||||
let mut cache_misses = Vec::new();
|
let mut cache_misses = Vec::new();
|
||||||
@@ -6488,6 +6556,7 @@ impl LocalDisk {
|
|||||||
Ok(f)
|
Ok(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
||||||
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
||||||
Ok(f)
|
Ok(f)
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
pub mod disk_store;
|
pub mod disk_store;
|
||||||
pub mod endpoint;
|
pub mod endpoint;
|
||||||
@@ -1114,6 +1113,10 @@ pub struct DiskInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default)]
|
#[derive(Clone, Debug, Default)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub struct Info {
|
pub struct Info {
|
||||||
pub total: u64,
|
pub total: u64,
|
||||||
pub free: u64,
|
pub free: u64,
|
||||||
@@ -1372,6 +1375,7 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
||||||
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -571,6 +571,10 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
|||||||
|
|
||||||
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
||||||
/// itself.
|
/// itself.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||||
for entry in std::fs::read_dir(dir.as_ref())? {
|
for entry in std::fs::read_dir(dir.as_ref())? {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
@@ -583,6 +587,7 @@ pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
|||||||
|
|
||||||
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
||||||
/// concurrently, bounded both per directory and process-wide.
|
/// concurrently, bounded both per directory and process-wide.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||||
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
||||||
}
|
}
|
||||||
@@ -1809,10 +1814,6 @@ impl RenameCommitGuard {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result<RenameDestinationPathGuard> {
|
|
||||||
self.destination_directory_guard(directory, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn create_destination_directory_for_path_access(
|
pub(crate) fn create_destination_directory_for_path_access(
|
||||||
&self,
|
&self,
|
||||||
directory: &Path,
|
directory: &Path,
|
||||||
@@ -2858,13 +2859,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a file exists.
|
|
||||||
/// Returns true if the file exists, false otherwise.
|
|
||||||
#[tracing::instrument(level = "debug", skip_all)]
|
|
||||||
pub fn file_exists(path: impl AsRef<Path>) -> bool {
|
|
||||||
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether an [`io::Error`] means "the directory is not empty".
|
/// Whether an [`io::Error`] means "the directory is not empty".
|
||||||
///
|
///
|
||||||
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
||||||
|
|||||||
@@ -704,6 +704,7 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn create_deferred_bitrot_reader(
|
pub fn create_deferred_bitrot_reader(
|
||||||
inline_data: Option<Bytes>,
|
inline_data: Option<Bytes>,
|
||||||
disk: Option<DiskStore>,
|
disk: Option<DiskStore>,
|
||||||
|
|||||||
@@ -277,6 +277,12 @@ pub struct ObjectOptions {
|
|||||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||||
pub replication_request: bool,
|
pub replication_request: bool,
|
||||||
|
/// Source-cluster LWW timestamps carried by an authorized replication
|
||||||
|
/// request; None when the source never modified the category. Only the
|
||||||
|
/// replication-authorized options builders may set these.
|
||||||
|
pub replication_tagging_timestamp: Option<OffsetDateTime>,
|
||||||
|
pub replication_retention_timestamp: Option<OffsetDateTime>,
|
||||||
|
pub replication_legalhold_timestamp: Option<OffsetDateTime>,
|
||||||
/// Authorized SSE-C replication passthrough: the body is already
|
/// Authorized SSE-C replication passthrough: the body is already
|
||||||
/// ciphertext, so the write path must not encrypt or compress it and
|
/// ciphertext, so the write path must not encrypt or compress it and
|
||||||
/// stores the restored encryption metadata verbatim. Only the
|
/// stores the restored encryption metadata verbatim. Only the
|
||||||
|
|||||||
@@ -32,15 +32,22 @@ use crate::diagnostics::get::{
|
|||||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
||||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
||||||
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
||||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||||
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
||||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
||||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
||||||
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
|
||||||
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
|
||||||
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||||
GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK,
|
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
|
||||||
GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||||
|
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||||
|
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||||
|
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||||
|
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
||||||
|
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
||||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
||||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
||||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||||
@@ -173,11 +180,13 @@ pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome {
|
|||||||
Fallback(GetCodecStreamingFallbackReason),
|
Fallback(GetCodecStreamingFallbackReason),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
||||||
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MultipartCodecStreamingReader {
|
impl MultipartCodecStreamingReader {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
readers: VecDeque::from(readers),
|
readers: VecDeque::from(readers),
|
||||||
@@ -652,36 +661,15 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
|||||||
&& left.erasure.distribution == right.erasure.distribution
|
&& left.erasure.distribution == right.erasure.distribution
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
object: &str,
|
object: &str,
|
||||||
candidate: &FileInfo,
|
candidate: &FileInfo,
|
||||||
parts_metadata: &[FileInfo],
|
parts_metadata: &[FileInfo],
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
) -> bool {
|
) -> Option<&'static str> {
|
||||||
if !candidate.inline_data()
|
if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
|
||||||
|| candidate.is_compressed()
|
return Some(reason);
|
||||||
|| candidate
|
|
||||||
.metadata
|
|
||||||
.keys()
|
|
||||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
|
||||||
|| candidate.is_remote()
|
|
||||||
|| candidate.deleted
|
|
||||||
|| candidate.size <= 0
|
|
||||||
|| candidate.parts.len() != 1
|
|
||||||
|| !candidate.has_valid_erasure_geometry()
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
let Ok(object_size) = usize::try_from(candidate.size) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||||
@@ -690,18 +678,21 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
|||||||
candidate.erasure.block_size,
|
candidate.erasure.block_size,
|
||||||
candidate.uses_legacy_checksum,
|
candidate.uses_legacy_checksum,
|
||||||
) else {
|
) else {
|
||||||
return false;
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||||
};
|
};
|
||||||
let Some(data_files) =
|
let data_files =
|
||||||
collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| {
|
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| {
|
||||||
disks.get(index).is_some_and(Option::is_some)
|
disks.get(index).is_some_and(Option::is_some)
|
||||||
})
|
}) {
|
||||||
else {
|
Ok(data_files) => data_files,
|
||||||
return false;
|
Err(reason) => return Some(reason),
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(part) = candidate.parts.first() else {
|
let Some(part) = candidate.parts.first() else {
|
||||||
return false;
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||||
|
};
|
||||||
|
let Ok(object_size) = usize::try_from(candidate.size) else {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
};
|
};
|
||||||
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
||||||
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||||
@@ -721,12 +712,111 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
|
|||||||
let Ok(mut readers) =
|
let Ok(mut readers) =
|
||||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||||
else {
|
else {
|
||||||
return false;
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||||
};
|
};
|
||||||
|
|
||||||
try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size)
|
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
||||||
.await
|
Some(body) if body.len() == object_size => None,
|
||||||
.is_some_and(|body| body.len() == object_size)
|
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
|
||||||
|
// `inline_data` excludes remote objects; this diagnostic reports them separately.
|
||||||
|
if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE);
|
||||||
|
}
|
||||||
|
if candidate.is_compressed()
|
||||||
|
|| candidate
|
||||||
|
.metadata
|
||||||
|
.keys()
|
||||||
|
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||||
|
{
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED);
|
||||||
|
}
|
||||||
|
if candidate.is_remote() {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE);
|
||||||
|
}
|
||||||
|
if candidate.deleted {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED);
|
||||||
|
}
|
||||||
|
if candidate.size <= 0 {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
|
}
|
||||||
|
if candidate.parts.len() != 1 {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||||
|
}
|
||||||
|
if !candidate.has_valid_erasure_geometry() {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||||
|
}
|
||||||
|
|
||||||
|
let Ok(object_size) = usize::try_from(candidate.size) else {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
|
};
|
||||||
|
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||||
|
}
|
||||||
|
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
||||||
|
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn data_read_inline_missing_shards_are_pending(
|
||||||
|
candidate: &FileInfo,
|
||||||
|
parts_metadata: &[FileInfo],
|
||||||
|
errors: &[Option<DiskError>],
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
fanout_order: &[usize],
|
||||||
|
scheduled_fanout_len: usize,
|
||||||
|
) -> bool {
|
||||||
|
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||||
|
candidate.erasure.data_blocks,
|
||||||
|
candidate.erasure.parity_blocks,
|
||||||
|
candidate.erasure.block_size,
|
||||||
|
candidate.uses_legacy_checksum,
|
||||||
|
) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let distribution = &candidate.erasure.distribution;
|
||||||
|
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
|
||||||
|
let mut missing_pending_data_shards = 0usize;
|
||||||
|
|
||||||
|
for (disk_index, file_info) in parts_metadata.iter().enumerate() {
|
||||||
|
let Some(&block_index) = distribution.get(disk_index) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if block_index == 0 || block_index > erasure.data_shards {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !disks.get(disk_index).is_some_and(Option::is_some) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data_slot = block_index - 1;
|
||||||
|
if file_info.name.is_empty() {
|
||||||
|
let scheduled_and_not_failed = fanout_order
|
||||||
|
.get(..scheduled_fanout_len)
|
||||||
|
.is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index))
|
||||||
|
&& errors.get(disk_index).is_some_and(Option::is_none);
|
||||||
|
if scheduled_and_not_failed {
|
||||||
|
data_shards_seen_or_pending[data_slot] = true;
|
||||||
|
missing_pending_data_shards = missing_pending_data_shards.saturating_add(1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if file_info.erasure.index != block_index
|
||||||
|
|| !file_info.has_valid_erasure_geometry()
|
||||||
|
|| !metadata_early_stop_candidate_matches(file_info, candidate)
|
||||||
|
|| file_info.data.as_ref().is_none_or(|data| data.is_empty())
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
data_shards_seen_or_pending[data_slot] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
|
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
|
||||||
@@ -1758,6 +1848,7 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
||||||
files: &[FileInfo],
|
files: &[FileInfo],
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
@@ -2048,6 +2139,7 @@ pub(in crate::set_disk) async fn create_data_block_bitrot_readers(
|
|||||||
setup
|
setup
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
||||||
tasks: Vec<F>,
|
tasks: Vec<F>,
|
||||||
read_quorum: usize,
|
read_quorum: usize,
|
||||||
@@ -2364,6 +2456,7 @@ impl SetDisks {
|
|||||||
let bucket: Arc<str> = Arc::from(bucket);
|
let bucket: Arc<str> = Arc::from(bucket);
|
||||||
let object: Arc<str> = Arc::from(object);
|
let object: Arc<str> = Arc::from(object);
|
||||||
let version_id: Arc<str> = Arc::from(version_id);
|
let version_id: Arc<str> = Arc::from(version_id);
|
||||||
|
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||||
let disk = disk.clone();
|
let disk = disk.clone();
|
||||||
let task_opts = opts;
|
let task_opts = opts;
|
||||||
@@ -2371,10 +2464,14 @@ impl SetDisks {
|
|||||||
let bucket = bucket.clone();
|
let bucket = bucket.clone();
|
||||||
let object = object.clone();
|
let object = object.clone();
|
||||||
let version_id = version_id.clone();
|
let version_id = version_id.clone();
|
||||||
|
let slowtail_fault = slowtail_fault.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let response_start = observe.then(Instant::now);
|
let response_start = observe.then(Instant::now);
|
||||||
let result = if let Some(disk) = disk {
|
let result = if let Some(disk) = disk {
|
||||||
Self::record_read_version_call(&object, disk_index);
|
Self::record_read_version_call(&object, disk_index);
|
||||||
|
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -2469,6 +2566,8 @@ impl SetDisks {
|
|||||||
let mut next_fanout_index = 0usize;
|
let mut next_fanout_index = 0usize;
|
||||||
let mut scheduled_count = 0usize;
|
let mut scheduled_count = 0usize;
|
||||||
let mut force_full_wait = false;
|
let mut force_full_wait = false;
|
||||||
|
let mut final_miss_reason_override = None;
|
||||||
|
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||||
let spawn_read_version =
|
let spawn_read_version =
|
||||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||||
let task_opts = opts;
|
let task_opts = opts;
|
||||||
@@ -2476,6 +2575,7 @@ impl SetDisks {
|
|||||||
let bucket = bucket.clone();
|
let bucket = bucket.clone();
|
||||||
let object = object.clone();
|
let object = object.clone();
|
||||||
let version_id = version_id.clone();
|
let version_id = version_id.clone();
|
||||||
|
let slowtail_fault = slowtail_fault.clone();
|
||||||
join_set.spawn(async move {
|
join_set.spawn(async move {
|
||||||
let response_start = Instant::now();
|
let response_start = Instant::now();
|
||||||
let result = if let Some(disk) = disk {
|
let result = if let Some(disk) = disk {
|
||||||
@@ -2484,6 +2584,9 @@ impl SetDisks {
|
|||||||
Self::record_read_version_call(&object, index);
|
Self::record_read_version_call(&object, index);
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
Self::read_version_fanout_barrier(&object, index).await;
|
Self::read_version_fanout_barrier(&object, index).await;
|
||||||
|
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
@@ -2511,11 +2614,20 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
while let Some(result) = join_set.join_next().await {
|
while let Some(result) = join_set.join_next().await {
|
||||||
|
let mut defer_pending_inline_data_shard = false;
|
||||||
match result {
|
match result {
|
||||||
Ok((index, res, elapsed)) => match res {
|
Ok((index, res, elapsed)) => match res {
|
||||||
Ok(file_info) => {
|
Ok(file_info) => {
|
||||||
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
||||||
accumulator.observe_file_info(&file_info);
|
accumulator.observe_file_info(&file_info);
|
||||||
|
if bounded_fanout
|
||||||
|
&& read_data
|
||||||
|
&& !force_full_wait
|
||||||
|
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
|
||||||
|
{
|
||||||
|
force_full_wait = true;
|
||||||
|
final_miss_reason_override.get_or_insert(reason);
|
||||||
|
}
|
||||||
if let Some(slot) = ress.get_mut(index) {
|
if let Some(slot) = ress.get_mut(index) {
|
||||||
*slot = file_info;
|
*slot = file_info;
|
||||||
}
|
}
|
||||||
@@ -2541,17 +2653,43 @@ impl SetDisks {
|
|||||||
.or_else(|| accumulator.version_early_stop_decision())
|
.or_else(|| accumulator.version_early_stop_decision())
|
||||||
{
|
{
|
||||||
let should_return_early = if read_data {
|
let should_return_early = if read_data {
|
||||||
let allow_data_read_early_stop = match accumulator.candidate.as_ref() {
|
match accumulator.candidate.as_ref() {
|
||||||
Some(candidate) => {
|
Some(candidate) => match data_read_early_stop_inline_body_miss_reason(
|
||||||
data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks)
|
bucket.as_ref(),
|
||||||
|
object.as_ref(),
|
||||||
|
candidate,
|
||||||
|
&ress,
|
||||||
|
disks,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
}
|
{
|
||||||
None => false,
|
None => true,
|
||||||
};
|
Some(reason) => {
|
||||||
if !allow_data_read_early_stop {
|
final_miss_reason_override = Some(reason);
|
||||||
|
if bounded_fanout
|
||||||
|
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
|
||||||
|
&& data_read_inline_missing_shards_are_pending(
|
||||||
|
candidate,
|
||||||
|
&ress,
|
||||||
|
&errors,
|
||||||
|
disks,
|
||||||
|
&fanout_order,
|
||||||
|
next_fanout_index,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
defer_pending_inline_data_shard = true;
|
||||||
|
} else {
|
||||||
force_full_wait = true;
|
force_full_wait = true;
|
||||||
}
|
}
|
||||||
allow_data_read_early_stop
|
false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
None => {
|
||||||
|
force_full_wait = true;
|
||||||
|
final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
true
|
true
|
||||||
};
|
};
|
||||||
@@ -2588,6 +2726,7 @@ impl SetDisks {
|
|||||||
let pending_responses = join_set.len();
|
let pending_responses = join_set.len();
|
||||||
let should_hedge_single_pending_data_read = read_data
|
let should_hedge_single_pending_data_read = read_data
|
||||||
&& !force_full_wait
|
&& !force_full_wait
|
||||||
|
&& !defer_pending_inline_data_shard
|
||||||
&& pending_responses == 1
|
&& pending_responses == 1
|
||||||
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
||||||
if bounded_fanout && force_full_wait {
|
if bounded_fanout && force_full_wait {
|
||||||
@@ -2600,6 +2739,7 @@ impl SetDisks {
|
|||||||
next_fanout_index = next_fanout_index.saturating_add(1);
|
next_fanout_index = next_fanout_index.saturating_add(1);
|
||||||
}
|
}
|
||||||
} else if bounded_fanout
|
} else if bounded_fanout
|
||||||
|
&& !defer_pending_inline_data_shard
|
||||||
&& next_fanout_index < disks.len()
|
&& next_fanout_index < disks.len()
|
||||||
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
||||||
|| should_hedge_single_pending_data_read)
|
|| should_hedge_single_pending_data_read)
|
||||||
@@ -2613,7 +2753,12 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason());
|
let accumulator_miss_reason = accumulator.final_miss_reason();
|
||||||
|
let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) {
|
||||||
|
(Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason,
|
||||||
|
_ => accumulator_miss_reason,
|
||||||
|
};
|
||||||
|
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
|
||||||
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
||||||
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
||||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||||
@@ -2842,6 +2987,7 @@ impl SetDisks {
|
|||||||
(meta_file_infos, errs)
|
(meta_file_infos, errs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn read_multiple_files(
|
pub(in crate::set_disk) async fn read_multiple_files(
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
req: ReadMultipleReq,
|
req: ReadMultipleReq,
|
||||||
@@ -2875,14 +3021,11 @@ impl SetDisks {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||||
Ok(collected) => collected,
|
Ok(collected) => collected,
|
||||||
Err(()) => return empty_quorum_result(),
|
Err(()) => return empty_quorum_result(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// debug!("ReadMultipleResp ress {:?}", ress);
|
|
||||||
// debug!("ReadMultipleResp errors {:?}", errors);
|
|
||||||
|
|
||||||
let mut ret = Vec::with_capacity(req.files.len());
|
let mut ret = Vec::with_capacity(req.files.len());
|
||||||
|
|
||||||
for want in req.files.iter() {
|
for want in req.files.iter() {
|
||||||
@@ -3021,6 +3164,7 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
|||||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
type RenameDataLegacyTuple = (
|
type RenameDataLegacyTuple = (
|
||||||
Vec<Option<DiskStore>>,
|
Vec<Option<DiskStore>>,
|
||||||
RenameConvergence,
|
RenameConvergence,
|
||||||
@@ -3030,6 +3174,7 @@ type RenameDataLegacyTuple = (
|
|||||||
);
|
);
|
||||||
|
|
||||||
impl RenameDataCommit {
|
impl RenameDataCommit {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
||||||
(
|
(
|
||||||
self.online_disks,
|
self.online_disks,
|
||||||
@@ -3148,6 +3293,7 @@ impl SetDisks {
|
|||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||||
#[allow(clippy::type_complexity)]
|
#[allow(clippy::type_complexity)]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn rename_data(
|
pub(in crate::set_disk) async fn rename_data(
|
||||||
disks: &[Option<DiskStore>],
|
disks: &[Option<DiskStore>],
|
||||||
src_bucket: &str,
|
src_bucket: &str,
|
||||||
@@ -4960,6 +5106,7 @@ fn is_cleanup_not_found(e: &DiskError) -> bool {
|
|||||||
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
||||||
/// must not be silently swallowed as an ignorable error (fixes the historical
|
/// must not be silently swallowed as an ignorable error (fixes the historical
|
||||||
/// `Unexpected`/`DiskNotFound` misclassification).
|
/// `Unexpected`/`DiskNotFound` misclassification).
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
||||||
match joined {
|
match joined {
|
||||||
Ok(res) => res,
|
Ok(res) => res,
|
||||||
@@ -5184,6 +5331,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
|
|||||||
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
||||||
pub const CLEANUP: &str = "cleanup";
|
pub const CLEANUP: &str = "cleanup";
|
||||||
/// The per-disk `read_version` phase of metadata read fan-out.
|
/// The per-disk `read_version` phase of metadata read fan-out.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub const READ_VERSION: &str = "read_version";
|
pub const READ_VERSION: &str = "read_version";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5621,6 +5769,130 @@ mod tests {
|
|||||||
(dirs, disks)
|
(dirs, disks)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
assert_eq!(
|
||||||
|
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
|
||||||
|
Some(Duration::from_millis(25))
|
||||||
|
);
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
|
||||||
|
const DISKS: usize = 4;
|
||||||
|
let bucket = "metadata-slowtail-fault-bucket";
|
||||||
|
let object = "objects/metadata-slowtail-fault-object";
|
||||||
|
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||||
|
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||||
|
|
||||||
|
temp_env::async_with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||||
|
],
|
||||||
|
async {
|
||||||
|
let read_without_data =
|
||||||
|
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
|
||||||
|
tokio::time::timeout(Duration::from_millis(100), read_without_data)
|
||||||
|
.await
|
||||||
|
.expect("non-data metadata fanout must not be delayed by the data-read slowtail hook")
|
||||||
|
.expect("metadata fanout without read_data should resolve");
|
||||||
|
|
||||||
|
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||||
|
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"data-read metadata fanout must wait for the injected slow read_version response"
|
||||||
|
);
|
||||||
|
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||||
|
.await
|
||||||
|
.expect("injected slowtail should eventually complete")
|
||||||
|
.expect("data-read metadata fanout should resolve");
|
||||||
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
assert!(errs.iter().all(Option::is_none));
|
||||||
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
drop(dirs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
|
||||||
|
const DISKS: usize = 4;
|
||||||
|
let bucket = "metadata-slowtail-early-stop-bucket";
|
||||||
|
let object = "objects/metadata-slowtail-early-stop-object";
|
||||||
|
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||||
|
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||||
|
|
||||||
|
temp_env::async_with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||||
|
],
|
||||||
|
async {
|
||||||
|
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||||
|
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
|
||||||
|
);
|
||||||
|
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||||
|
.await
|
||||||
|
.expect("injected early-stop slowtail should eventually complete")
|
||||||
|
.expect("early-stop metadata fanout should resolve");
|
||||||
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
assert!(errs.iter().all(Option::is_none));
|
||||||
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
drop(dirs);
|
||||||
|
}
|
||||||
|
|
||||||
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
||||||
///
|
///
|
||||||
/// The metadata fan-out issues each `read_version` inside its own
|
/// The metadata fan-out issues each `read_version` inside its own
|
||||||
@@ -5752,9 +6024,20 @@ mod tests {
|
|||||||
object: &str,
|
object: &str,
|
||||||
payload: &[u8],
|
payload: &[u8],
|
||||||
uses_legacy_checksum: bool,
|
uses_legacy_checksum: bool,
|
||||||
|
) -> Vec<FileInfo> {
|
||||||
|
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, uses_legacy_checksum, 2, 2).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn inline_metadata_fanout_fileinfos_with_geometry(
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
payload: &[u8],
|
||||||
|
uses_legacy_checksum: bool,
|
||||||
|
data_shards: usize,
|
||||||
|
parity_shards: usize,
|
||||||
) -> Vec<FileInfo> {
|
) -> Vec<FileInfo> {
|
||||||
let distribution_key = metadata_distribution_key(bucket, object);
|
let distribution_key = metadata_distribution_key(bucket, object);
|
||||||
let mut base = FileInfo::new(&distribution_key, 2, 2);
|
let mut base = FileInfo::new(&distribution_key, data_shards, parity_shards);
|
||||||
base.volume = bucket.to_string();
|
base.volume = bucket.to_string();
|
||||||
base.name = object.to_string();
|
base.name = object.to_string();
|
||||||
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
||||||
@@ -5817,6 +6100,21 @@ mod tests {
|
|||||||
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn install_inline_metadata_fanout_fileinfo_with_geometry(
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
payload: &[u8],
|
||||||
|
data_shards: usize,
|
||||||
|
parity_shards: usize,
|
||||||
|
mutate: impl FnOnce(&mut [FileInfo]),
|
||||||
|
) {
|
||||||
|
let mut files =
|
||||||
|
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, false, data_shards, parity_shards).await;
|
||||||
|
mutate(&mut files);
|
||||||
|
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
|
||||||
|
}
|
||||||
|
|
||||||
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
|
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
|
||||||
let distribution = files
|
let distribution = files
|
||||||
.first()
|
.first()
|
||||||
@@ -6037,6 +6335,118 @@ mod tests {
|
|||||||
drop(dirs);
|
drop(dirs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn bounded_metadata_early_stop_waits_for_pending_inline_data_shard() {
|
||||||
|
const DISKS: usize = 6;
|
||||||
|
const DATA_SHARDS: usize = 4;
|
||||||
|
const PARITY_SHARDS: usize = 2;
|
||||||
|
let bucket = "bounded-inline-data-get-pending-shard-bucket";
|
||||||
|
let object =
|
||||||
|
object_with_initial_data_shards(bucket, "bounded-inline-data-get-pending-shard-object", DATA_SHARDS, DATA_SHARDS);
|
||||||
|
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||||
|
install_inline_metadata_fanout_fileinfo_with_geometry(
|
||||||
|
&disks,
|
||||||
|
bucket,
|
||||||
|
&object,
|
||||||
|
b"verified inline payload",
|
||||||
|
DATA_SHARDS,
|
||||||
|
PARITY_SHARDS,
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
temp_env::async_with_vars(
|
||||||
|
[
|
||||||
|
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||||
|
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||||
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||||
|
],
|
||||||
|
async {
|
||||||
|
let fanout_order = bounded_metadata_fanout_order(bucket, &object, DISKS, PARITY_SHARDS);
|
||||||
|
let distribution_key = metadata_distribution_key(bucket, &object);
|
||||||
|
let distribution = FileInfo::new(&distribution_key, DATA_SHARDS, PARITY_SHARDS)
|
||||||
|
.erasure
|
||||||
|
.distribution;
|
||||||
|
let paused_data_disk = *fanout_order
|
||||||
|
.iter()
|
||||||
|
.take(DATA_SHARDS)
|
||||||
|
.find(|disk_index| {
|
||||||
|
distribution
|
||||||
|
.get(**disk_index)
|
||||||
|
.is_some_and(|block_index| (1..=DATA_SHARDS).contains(block_index))
|
||||||
|
})
|
||||||
|
.expect("initial fanout should include a data shard to pause");
|
||||||
|
let hedged_parity_disk = fanout_order[DATA_SHARDS];
|
||||||
|
let unscheduled_parity_disk = fanout_order[DATA_SHARDS + 1];
|
||||||
|
|
||||||
|
let barrier = rename_fanout_barrier::arm(&object, paused_data_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||||
|
let tracker = rename_fanout_barrier::observe_tasks(&object);
|
||||||
|
let calls = disk_call_counters::observe(&object);
|
||||||
|
let disks_for_read = disks.clone();
|
||||||
|
let object_for_read = object.clone();
|
||||||
|
let mut read = tokio::spawn(async move {
|
||||||
|
SetDisks::read_all_fileinfo_observed(
|
||||||
|
&disks_for_read,
|
||||||
|
bucket,
|
||||||
|
bucket,
|
||||||
|
&object_for_read,
|
||||||
|
"",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
PARITY_SHARDS,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||||
|
.await
|
||||||
|
.expect("initial data shard should pause before returning");
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||||
|
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, hedged_parity_disk) == 0 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded fanout should hedge one parity disk while the data shard is pending");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
|
||||||
|
"inline data-read early-stop must wait for a scheduled missing data shard instead of forcing full wait"
|
||||||
|
);
|
||||||
|
|
||||||
|
barrier.release();
|
||||||
|
let (parts_metadata, errs, diagnostics) = read
|
||||||
|
.await
|
||||||
|
.expect("metadata read task should not panic")
|
||||||
|
.expect("pending data shard should let the inline verifier finish");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
|
5,
|
||||||
|
"pending data-shard defer should not schedule the final parity disk"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
calls.for_disk(disk_call_counters::KIND_READ_VERSION, unscheduled_parity_disk),
|
||||||
|
0,
|
||||||
|
"the remaining parity disk must stay unissued when pending data verification succeeds"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tracker.running(),
|
||||||
|
0,
|
||||||
|
"early-stop should drain spawned read_version tasks before returning"
|
||||||
|
);
|
||||||
|
assert_eq!(diagnostics.total_responses(), 5);
|
||||||
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 5);
|
||||||
|
assert!(errs.iter().all(Option::is_none));
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
drop(dirs);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
|
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
|
||||||
let bucket = "legacy-inline-data-get-fanout-bucket";
|
let bucket = "legacy-inline-data-get-fanout-bucket";
|
||||||
@@ -6067,11 +6477,133 @@ mod tests {
|
|||||||
.clone();
|
.clone();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await,
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks)
|
||||||
|
.await
|
||||||
|
.is_none(),
|
||||||
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
|
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn data_read_early_stop_reports_inline_miss_reasons() {
|
||||||
|
let bucket = "inline-data-get-miss-reason-bucket";
|
||||||
|
let object = "inline-data-get-miss-reason-object";
|
||||||
|
let payload = b"verified inline payload";
|
||||||
|
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
|
||||||
|
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await;
|
||||||
|
let distribution = files
|
||||||
|
.first()
|
||||||
|
.map(|file| file.erasure.distribution.clone())
|
||||||
|
.expect("fixture should include metadata");
|
||||||
|
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
|
||||||
|
let mut parts_metadata = vec![FileInfo::default(); 4];
|
||||||
|
for disk_index in order.into_iter().take(3) {
|
||||||
|
let block_index = distribution
|
||||||
|
.get(disk_index)
|
||||||
|
.copied()
|
||||||
|
.expect("fixture distribution should cover every disk");
|
||||||
|
parts_metadata[disk_index] = files
|
||||||
|
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
|
||||||
|
.expect("fixture should include every distributed shard")
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
let candidate = parts_metadata
|
||||||
|
.iter()
|
||||||
|
.find(|file| file.name == object)
|
||||||
|
.expect("fixture should include observed metadata")
|
||||||
|
.clone();
|
||||||
|
let data_disk = distribution
|
||||||
|
.iter()
|
||||||
|
.position(|block_index| *block_index == 1)
|
||||||
|
.expect("fixture distribution should include first data shard");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut not_inline = candidate.clone();
|
||||||
|
rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA);
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, ¬_inline, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut remote = candidate.clone();
|
||||||
|
remote.transition_status = TRANSITION_COMPLETE.to_string();
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &remote, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut transformed = candidate.clone();
|
||||||
|
rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut deleted = candidate.clone();
|
||||||
|
deleted.deleted = true;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut zero_size = candidate.clone();
|
||||||
|
zero_size.size = 0;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut multipart = candidate.clone();
|
||||||
|
multipart.parts.push(multipart.parts[0].clone());
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut invalid_geometry = candidate.clone();
|
||||||
|
invalid_geometry.erasure.data_blocks = 0;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut missing_shard = parts_metadata.clone();
|
||||||
|
missing_shard[data_disk] = FileInfo::default();
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut missing_payload = parts_metadata.clone();
|
||||||
|
missing_payload[data_disk].data = None;
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut identity_mismatch = parts_metadata.clone();
|
||||||
|
identity_mismatch[data_disk].version_id = Some(Uuid::new_v4());
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut corrupt = parts_metadata.clone();
|
||||||
|
if let Some(data) = corrupt[data_disk].data.as_mut() {
|
||||||
|
let mut corrupt_data = data.to_vec();
|
||||||
|
corrupt_data[0] ^= 0x01;
|
||||||
|
*data = Bytes::from(corrupt_data);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await,
|
||||||
|
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
|
fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
|
||||||
@@ -6161,7 +6693,7 @@ mod tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_INTERNAL_META),
|
("path", GET_OBJECT_PATH_INTERNAL_META),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
|
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
1,
|
1,
|
||||||
@@ -6173,7 +6705,7 @@ mod tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
|
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
0,
|
0,
|
||||||
@@ -6708,7 +7240,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() {
|
async fn bounded_non_inline_data_get_immediately_forces_full_fanout() {
|
||||||
const DISKS: usize = 4;
|
const DISKS: usize = 4;
|
||||||
let bucket = "bounded-data-get-hedge-bucket";
|
let bucket = "bounded-data-get-hedge-bucket";
|
||||||
let object = "bounded-data-get-hedge-object";
|
let object = "bounded-data-get-hedge-object";
|
||||||
@@ -6739,7 +7271,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("bounded data-read fanout should hedge by starting the spare disk");
|
.expect("bounded non-inline data-read fanout should immediately schedule the spare disk");
|
||||||
|
|
||||||
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||||
assert!(
|
assert!(
|
||||||
@@ -6755,7 +7287,7 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
DISKS as u64,
|
DISKS as u64,
|
||||||
"bounded data-read fanout should issue the paused disk plus one spare hedge"
|
"bounded non-inline data-read fanout should issue the paused disk plus the remaining spare"
|
||||||
);
|
);
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
@@ -6768,7 +7300,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() {
|
async fn bounded_metadata_early_stop_defaults_keep_non_inline_data_get_full_fanout() {
|
||||||
const DISKS: usize = 4;
|
const DISKS: usize = 4;
|
||||||
let bucket = "bounded-data-get-default-bucket";
|
let bucket = "bounded-data-get-default-bucket";
|
||||||
let object = "bounded-data-get-default-object";
|
let object = "bounded-data-get-default-object";
|
||||||
@@ -6782,16 +7314,42 @@ mod tests {
|
|||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
|
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||||
let calls = disk_call_counters::observe(object);
|
let calls = disk_call_counters::observe(object);
|
||||||
let (parts_metadata, errs, diagnostics) =
|
let disks_for_read = disks.clone();
|
||||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
|
let mut read = tokio::spawn(async move {
|
||||||
|
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
|
||||||
.await
|
.await
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||||
|
.await
|
||||||
|
.expect("default bounded non-inline read should schedule the paused metadata task");
|
||||||
|
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||||
|
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect(
|
||||||
|
"default bounded non-inline read should immediately force full fanout after the first non-inline response",
|
||||||
|
);
|
||||||
|
|
||||||
|
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||||
|
assert!(
|
||||||
|
pending.is_err(),
|
||||||
|
"default non-inline data reads must not return before the paused metadata response"
|
||||||
|
);
|
||||||
|
barrier.release();
|
||||||
|
let (parts_metadata, errs, diagnostics) = read
|
||||||
|
.await
|
||||||
|
.expect("metadata read task should not panic")
|
||||||
.expect("default data-read metadata should resolve");
|
.expect("default data-read metadata should resolve");
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||||
DISKS as u64,
|
DISKS as u64,
|
||||||
"default GET data-read metadata must keep full fanout for read-failure tolerance"
|
"default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first"
|
||||||
);
|
);
|
||||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||||
|
|||||||
@@ -42,12 +42,20 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The borrowed core, for state not yet fronted by a typed accessor.
|
/// The borrowed core, for state not yet fronted by a typed accessor.
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn core(&self) -> &'a SetDisks {
|
pub(crate) fn core(&self) -> &'a SetDisks {
|
||||||
self.core
|
self.core
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Immutable topology / config (fixed after construction) ---
|
// --- Immutable topology / config (fixed after construction) ---
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn set_index(&self) -> usize {
|
pub(crate) fn set_index(&self) -> usize {
|
||||||
self.core.set_index
|
self.core.set_index
|
||||||
}
|
}
|
||||||
@@ -56,14 +64,26 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
self.core.pool_index
|
self.core.pool_index
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn set_drive_count(&self) -> usize {
|
pub(crate) fn set_drive_count(&self) -> usize {
|
||||||
self.core.set_drive_count
|
self.core.set_drive_count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn default_parity_count(&self) -> usize {
|
pub(crate) fn default_parity_count(&self) -> usize {
|
||||||
self.core.default_parity_count
|
self.core.default_parity_count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
||||||
&self.core.set_endpoints
|
&self.core.set_endpoints
|
||||||
}
|
}
|
||||||
@@ -72,6 +92,10 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
&self.core.format
|
&self.core.format
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn locker_owner(&self) -> &'a str {
|
pub(crate) fn locker_owner(&self) -> &'a str {
|
||||||
&self.core.locker_owner
|
&self.core.locker_owner
|
||||||
}
|
}
|
||||||
@@ -84,6 +108,10 @@ impl<'a> SetDisksCtx<'a> {
|
|||||||
|
|
||||||
// --- Locker trio ---
|
// --- Locker trio ---
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
||||||
&self.core.lockers
|
&self.core.lockers
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,7 +39,6 @@
|
|||||||
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
||||||
|
|
||||||
// #730: SetDisks still hosts staged read/heal/write migration helpers.
|
// #730: SetDisks still hosts staged read/heal/write migration helpers.
|
||||||
#![allow(dead_code)]
|
|
||||||
#![allow(unused_imports)]
|
#![allow(unused_imports)]
|
||||||
#![allow(unused_variables)]
|
#![allow(unused_variables)]
|
||||||
|
|
||||||
@@ -59,7 +58,10 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
|
|||||||
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
||||||
use crate::data_usage::record_compression_total_memory;
|
use crate::data_usage::record_compression_total_memory;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
|
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
|
||||||
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
|
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||||
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
|
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||||
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
|
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
|
||||||
@@ -100,9 +102,7 @@ use crate::storage_api_contracts::{
|
|||||||
};
|
};
|
||||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||||
use crate::{
|
use crate::{
|
||||||
bucket::lifecycle::bucket_lifecycle_ops::{
|
bucket::lifecycle::bucket_lifecycle_ops::{LifecycleOps, get_transitioned_object_reader_with_tier_manager, put_restore_opts},
|
||||||
LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts,
|
|
||||||
},
|
|
||||||
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
|
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
|
||||||
config::storageclass,
|
config::storageclass,
|
||||||
disk::{
|
disk::{
|
||||||
@@ -174,15 +174,14 @@ use std::future::Future;
|
|||||||
use std::hash::{BuildHasher, Hash, Hasher};
|
use std::hash::{BuildHasher, Hash, Hasher};
|
||||||
use std::mem::{self};
|
use std::mem::{self};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::OnceLock;
|
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::task::{Context, Poll};
|
use std::task::{Context, Poll};
|
||||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet},
|
collections::{HashMap, HashSet},
|
||||||
io::{Cursor, Write},
|
io::{Cursor, Write},
|
||||||
path::Path,
|
path::Path,
|
||||||
sync::Arc,
|
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
@@ -621,7 +620,9 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
|
|||||||
// Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout.
|
// Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout.
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
|
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
||||||
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
|
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
|
||||||
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
|
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
|
||||||
@@ -689,22 +690,36 @@ const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 102
|
|||||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
|
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
|
||||||
// Enabled by default (backlog#872): the early-stop path only engages for
|
// Enabled by default (backlog#872): the early-stop path only engages for
|
||||||
// requests `should_allow_metadata_early_stop` classifies as safe (latest-version
|
// requests `should_allow_metadata_early_stop` classifies as safe (latest-version
|
||||||
// metadata-only reads by default, without version_id / healing / free-version
|
// reads by default, without version_id / healing / free-version needs) and still
|
||||||
// needs) and still requires a full read-quorum agreement before stopping. Set
|
// requires a full read-quorum agreement before stopping. Data-read requests add
|
||||||
|
// a separate inline-shard verifier before cancelling the remaining fanout. Set
|
||||||
// the env var to `false` to fall back to full-wait metadata fanout.
|
// the env var to `false` to fall back to full-wait metadata fanout.
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
|
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||||
|
)]
|
||||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
|
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||||
|
)]
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100;
|
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE";
|
const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE";
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE";
|
const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE";
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = false;
|
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true;
|
||||||
|
|
||||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
||||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false;
|
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true;
|
||||||
|
|
||||||
|
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";
|
||||||
|
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS";
|
||||||
|
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET";
|
||||||
|
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX";
|
||||||
|
|
||||||
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
|
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
|
||||||
|
|
||||||
@@ -906,18 +921,16 @@ mod prepared_get_object_metadata_tests {
|
|||||||
.expect("test should find an object whose initial fanout covers both data shards")
|
.expect("test should find an object whose initial fanout covers both data shards")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
|
||||||
|
)]
|
||||||
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
||||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||||
.get(3)
|
.get(3)
|
||||||
.expect("4-disk test geometry should leave one bounded spare disk")
|
.expect("4-disk test geometry should leave one bounded spare disk")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn bounded_slow_initial_disk_index(bucket: &str, object: &str) -> usize {
|
|
||||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
|
||||||
.get(2)
|
|
||||||
.expect("4-disk test geometry should include a third initial metadata disk")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn prepared_metadata_is_consumed_exactly_once() {
|
async fn prepared_metadata_is_consumed_exactly_once() {
|
||||||
let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new());
|
let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new());
|
||||||
@@ -1036,7 +1049,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial(body_cache_hook)]
|
#[serial_test::serial(body_cache_hook)]
|
||||||
fn inline_data_read_early_stop_reader_returns_exact_body() {
|
fn inline_data_read_early_stop_defaults_return_exact_body() {
|
||||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()
|
.build()
|
||||||
@@ -1068,14 +1081,14 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
temp_env::async_with_vars(
|
temp_env::async_with_vars(
|
||||||
[
|
[
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
|
||||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
let slow_initial_disk = bounded_slow_initial_disk_index(bucket, &object);
|
let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
|
||||||
let barrier =
|
let barrier =
|
||||||
rename_fanout_barrier::arm(&object, slow_initial_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||||
let calls = disk_call_counters::observe(&object);
|
let calls = disk_call_counters::observe(&object);
|
||||||
let set_disks_for_read = Arc::clone(&set_disks);
|
let set_disks_for_read = Arc::clone(&set_disks);
|
||||||
let opts_for_read = opts.clone();
|
let opts_for_read = opts.clone();
|
||||||
@@ -1088,10 +1101,10 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
|
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
|
||||||
.await
|
.await
|
||||||
.expect("bounded inline GET should pause a slow initial metadata read");
|
.expect("default inline GET should pause a slow parity metadata read");
|
||||||
let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader)
|
let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader)
|
||||||
.await
|
.await
|
||||||
.expect("production inline GET should return before the paused metadata response")
|
.expect("default production inline GET should return before the paused parity metadata response")
|
||||||
.expect("inline GET reader task should not panic")
|
.expect("inline GET reader task should not panic")
|
||||||
.expect("inline GET reader should open");
|
.expect("inline GET reader should open");
|
||||||
let object_size = reader.object_info.size;
|
let object_size = reader.object_info.size;
|
||||||
@@ -1112,14 +1125,17 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
assert_eq!(object_size, payload.len() as i64);
|
assert_eq!(object_size, payload.len() as i64);
|
||||||
assert_eq!(restored, payload);
|
assert_eq!(restored, payload);
|
||||||
assert_eq!(calls_total, 4, "bounded production GET should schedule the initial quorum plus one spare");
|
assert_eq!(
|
||||||
|
calls_total, 4,
|
||||||
|
"default production inline GET should schedule the initial bounded quorum plus one hedge"
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
"rustfs_io_get_object_metadata_fanout_scheduled",
|
"rustfs_io_get_object_metadata_fanout_scheduled",
|
||||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||||
),
|
),
|
||||||
vec![4.0],
|
vec![4.0],
|
||||||
"bounded production GET should record all scheduled metadata tasks"
|
"default production GET should record all scheduled metadata tasks"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
@@ -1127,7 +1143,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||||
),
|
),
|
||||||
vec![3.0],
|
vec![3.0],
|
||||||
"bounded production GET should record only observed metadata responses as completed"
|
"default production GET should record only observed metadata responses as completed"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
@@ -1135,7 +1151,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||||
),
|
),
|
||||||
vec![1.0],
|
vec![1.0],
|
||||||
"bounded production GET should record the aborted slow metadata task"
|
"default production GET should record the aborted slow parity metadata task"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1282,9 +1298,9 @@ mod prepared_get_object_metadata_tests {
|
|||||||
|
|
||||||
temp_env::async_with_vars(
|
temp_env::async_with_vars(
|
||||||
[
|
[
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
|
||||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
|
||||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||||
],
|
],
|
||||||
async {
|
async {
|
||||||
let calls = disk_call_counters::observe(&object);
|
let calls = disk_call_counters::observe(&object);
|
||||||
@@ -1686,6 +1702,95 @@ fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct GetMetadataSlowtailFaultConfig {
|
||||||
|
delay: Duration,
|
||||||
|
disks: Arc<[usize]>,
|
||||||
|
bucket: Option<String>,
|
||||||
|
object_prefix: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct GetMetadataSlowtailFaultRequest {
|
||||||
|
delay: Duration,
|
||||||
|
disks: Arc<[usize]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GetMetadataSlowtailFaultRequest {
|
||||||
|
fn delay_for_disk(&self, disk_index: usize) -> Option<Duration> {
|
||||||
|
self.disks.contains(&disk_index).then_some(self.delay)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_get_metadata_slowtail_fault_disks(raw: &str) -> Option<Vec<usize>> {
|
||||||
|
let mut disks = Vec::new();
|
||||||
|
for item in raw.split(',').map(str::trim).filter(|item| !item.is_empty()) {
|
||||||
|
let Ok(index) = item.parse::<usize>() else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
if !disks.contains(&index) {
|
||||||
|
disks.push(index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(!disks.is_empty()).then_some(disks)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_get_metadata_slowtail_fault_config() -> Option<GetMetadataSlowtailFaultConfig> {
|
||||||
|
let delay_ms = rustfs_utils::get_env_u64(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, 0);
|
||||||
|
if delay_ms == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let disks = parse_get_metadata_slowtail_fault_disks(&std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS).ok()?)?;
|
||||||
|
let bucket = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET)
|
||||||
|
.ok()
|
||||||
|
.filter(|value| !value.is_empty());
|
||||||
|
let object_prefix = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX)
|
||||||
|
.ok()
|
||||||
|
.filter(|value| !value.is_empty());
|
||||||
|
Some(GetMetadataSlowtailFaultConfig {
|
||||||
|
delay: Duration::from_millis(delay_ms),
|
||||||
|
disks: Arc::from(disks.into_boxed_slice()),
|
||||||
|
bucket,
|
||||||
|
object_prefix,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_metadata_slowtail_fault_request(bucket: &str, object: &str, read_data: bool) -> Option<GetMetadataSlowtailFaultRequest> {
|
||||||
|
if !read_data {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
let config = load_get_metadata_slowtail_fault_config();
|
||||||
|
#[cfg(test)]
|
||||||
|
let config = config.as_ref()?;
|
||||||
|
#[cfg(not(test))]
|
||||||
|
let config = ({
|
||||||
|
static CACHED: OnceLock<Option<GetMetadataSlowtailFaultConfig>> = OnceLock::new();
|
||||||
|
CACHED.get_or_init(load_get_metadata_slowtail_fault_config).as_ref()
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if let Some(expected_bucket) = &config.bucket
|
||||||
|
&& expected_bucket != bucket
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if let Some(expected_prefix) = &config.object_prefix
|
||||||
|
&& !object.starts_with(expected_prefix)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(GetMetadataSlowtailFaultRequest {
|
||||||
|
delay: config.delay,
|
||||||
|
disks: config.disks.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn get_metadata_slowtail_fault_delay(bucket: &str, object: &str, disk_index: usize, read_data: bool) -> Option<Duration> {
|
||||||
|
get_metadata_slowtail_fault_request(bucket, object, read_data)?.delay_for_disk(disk_index)
|
||||||
|
}
|
||||||
|
|
||||||
/// Check if multipart reads prefetch the next part's bitrot reader setup
|
/// Check if multipart reads prefetch the next part's bitrot reader setup
|
||||||
/// while the current part decodes (backlog#870).
|
/// while the current part decodes (backlog#870).
|
||||||
///
|
///
|
||||||
@@ -1711,6 +1816,10 @@ fn is_multipart_reader_setup_prefetch_enabled() -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||||
|
)]
|
||||||
fn get_metadata_early_stop_rollout_pct() -> u32 {
|
fn get_metadata_early_stop_rollout_pct() -> u32 {
|
||||||
static CACHED: OnceLock<u32> = OnceLock::new();
|
static CACHED: OnceLock<u32> = OnceLock::new();
|
||||||
*CACHED.get_or_init(|| {
|
*CACHED.get_or_init(|| {
|
||||||
@@ -1750,6 +1859,10 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Should this specific request use metadata early-stop?
|
/// Should this specific request use metadata early-stop?
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||||
|
)]
|
||||||
pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool {
|
pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool {
|
||||||
let base = is_get_metadata_early_stop_enabled();
|
let base = is_get_metadata_early_stop_enabled();
|
||||||
let pct = get_metadata_early_stop_rollout_pct();
|
let pct = get_metadata_early_stop_rollout_pct();
|
||||||
@@ -2183,6 +2296,7 @@ fn classify_get_codec_streaming_object_class(
|
|||||||
GetCodecStreamingObjectClass::PlainSinglePart
|
GetCodecStreamingObjectClass::PlainSinglePart
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn is_get_small_object_direct_memory_eligible_with_threshold(
|
fn is_get_small_object_direct_memory_eligible_with_threshold(
|
||||||
range: &Option<HTTPRangeSpec>,
|
range: &Option<HTTPRangeSpec>,
|
||||||
object_info: &ObjectInfo,
|
object_info: &ObjectInfo,
|
||||||
@@ -2788,6 +2902,7 @@ pub struct SetDisks {
|
|||||||
/// Stable namespace shared by every object lock created for this set.
|
/// Stable namespace shared by every object lock created for this set.
|
||||||
set_lock_namespace: Arc<str>,
|
set_lock_namespace: Arc<str>,
|
||||||
pub format: FormatV3,
|
pub format: FormatV3,
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
||||||
@@ -3063,11 +3178,13 @@ struct GetObjectMetadataCacheEntry {
|
|||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct DiskHealthEntry {
|
struct DiskHealthEntry {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
last_check: Instant,
|
last_check: Instant,
|
||||||
online: bool,
|
online: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DiskHealthEntry {
|
impl DiskHealthEntry {
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn cached_value(&self) -> Option<bool> {
|
fn cached_value(&self) -> Option<bool> {
|
||||||
if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL {
|
if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL {
|
||||||
Some(self.online)
|
Some(self.online)
|
||||||
@@ -3661,6 +3778,7 @@ fn multipart_put_large_batch_min_size_bytes() -> usize {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
|
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
|
||||||
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
|
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
|
||||||
SmallWritePath::Inline
|
SmallWritePath::Inline
|
||||||
@@ -3866,8 +3984,17 @@ fn collect_inline_data_shard_fileinfos_by_index<'a>(
|
|||||||
parts_metadata: &'a [FileInfo],
|
parts_metadata: &'a [FileInfo],
|
||||||
fi: &FileInfo,
|
fi: &FileInfo,
|
||||||
data_shards: usize,
|
data_shards: usize,
|
||||||
mut disk_is_online: impl FnMut(usize) -> bool,
|
disk_is_online: impl FnMut(usize) -> bool,
|
||||||
) -> Option<Vec<&'a FileInfo>> {
|
) -> Option<Vec<&'a FileInfo>> {
|
||||||
|
collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, fi, data_shards, disk_is_online).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
|
||||||
|
parts_metadata: &'a [FileInfo],
|
||||||
|
fi: &FileInfo,
|
||||||
|
data_shards: usize,
|
||||||
|
mut disk_is_online: impl FnMut(usize) -> bool,
|
||||||
|
) -> std::result::Result<Vec<&'a FileInfo>, &'static str> {
|
||||||
let distribution = &fi.erasure.distribution;
|
let distribution = &fi.erasure.distribution;
|
||||||
let mut data_files = vec![None; data_shards];
|
let mut data_files = vec![None; data_shards];
|
||||||
|
|
||||||
@@ -3875,27 +4002,35 @@ fn collect_inline_data_shard_fileinfos_by_index<'a>(
|
|||||||
if !disk_is_online(disk_index) {
|
if !disk_is_online(disk_index) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let block_index = *distribution.get(disk_index)?;
|
let Some(&block_index) = distribution.get(disk_index) else {
|
||||||
|
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||||
|
};
|
||||||
if block_index == 0 || block_index > data_shards {
|
if block_index == 0 || block_index > data_shards {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if file_info.name.is_empty() {
|
||||||
|
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD);
|
||||||
|
}
|
||||||
if file_info.erasure.index != block_index {
|
if file_info.erasure.index != block_index {
|
||||||
continue;
|
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH);
|
||||||
}
|
}
|
||||||
if !file_info.has_valid_erasure_geometry() {
|
if !file_info.has_valid_erasure_geometry() {
|
||||||
continue;
|
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||||
}
|
}
|
||||||
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
|
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
|
||||||
continue;
|
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH);
|
||||||
}
|
}
|
||||||
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
|
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
|
||||||
continue;
|
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD);
|
||||||
}
|
}
|
||||||
|
|
||||||
data_files[block_index - 1] = Some(file_info);
|
data_files[block_index - 1] = Some(file_info);
|
||||||
}
|
}
|
||||||
|
|
||||||
data_files.into_iter().collect()
|
data_files
|
||||||
|
.into_iter()
|
||||||
|
.collect::<Option<Vec<_>>>()
|
||||||
|
.ok_or(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
@@ -4222,6 +4357,7 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj
|
|||||||
///
|
///
|
||||||
/// Fail closed: when bucket metadata cannot be resolved the check stays on, so
|
/// Fail closed: when bucket metadata cannot be resolved the check stays on, so
|
||||||
/// object-lock protection is never skipped because of a metadata lookup miss.
|
/// object-lock protection is never skipped because of a metadata lookup miss.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool {
|
pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool {
|
||||||
bucket_meta.is_none_or(|meta| meta.object_locking())
|
bucket_meta.is_none_or(|meta| meta.object_locking())
|
||||||
}
|
}
|
||||||
@@ -4497,15 +4633,6 @@ impl Hash for ObjProps {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default, Clone, Debug)]
|
|
||||||
pub struct HealEntryResult {
|
|
||||||
pub bytes: usize,
|
|
||||||
pub success: bool,
|
|
||||||
pub skipped: bool,
|
|
||||||
pub entry_done: bool,
|
|
||||||
pub name: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_object_dangling(
|
fn is_object_dangling(
|
||||||
meta_arr: &[FileInfo],
|
meta_arr: &[FileInfo],
|
||||||
errs: &[Option<DiskError>],
|
errs: &[Option<DiskError>],
|
||||||
@@ -5282,6 +5409,7 @@ pub fn is_valid_storage_class(storage_class: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns true if the storage class is a cold storage tier that requires special handling
|
/// Returns true if the storage class is a cold storage tier that requires special handling
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
storage_class,
|
storage_class,
|
||||||
@@ -5290,6 +5418,7 @@ pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns true if the storage class is an infrequent access tier
|
/// Returns true if the storage class is an infrequent access tier
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub fn is_infrequent_access_class(storage_class: &str) -> bool {
|
pub fn is_infrequent_access_class(storage_class: &str) -> bool {
|
||||||
matches!(
|
matches!(
|
||||||
storage_class,
|
storage_class,
|
||||||
|
|||||||
@@ -453,7 +453,9 @@ impl SetDisks {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let write_lock_guard = if !opts.no_lock {
|
// Bound, not `_`: this guard must live to the end of the scope. A bare
|
||||||
|
// `_` would drop it here and release the namespace write lock.
|
||||||
|
let _write_lock_guard = if !opts.no_lock {
|
||||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||||
Some(
|
Some(
|
||||||
ns_lock
|
ns_lock
|
||||||
@@ -996,7 +998,7 @@ impl SetDisks {
|
|||||||
readers.push(None);
|
readers.push(None);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(_e) => {
|
||||||
readers.push(None);
|
readers.push(None);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1545,6 +1547,9 @@ impl SetDisks {
|
|||||||
|
|
||||||
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
||||||
for (disk_index, disk) in disks.iter().enumerate() {
|
for (disk_index, disk) in disks.iter().enumerate() {
|
||||||
|
// Only the #[cfg(test)] fault-injection branch below reads this.
|
||||||
|
#[cfg(not(test))]
|
||||||
|
let _ = disk_index;
|
||||||
let Some(disk) = disk else {
|
let Some(disk) = disk else {
|
||||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||||
};
|
};
|
||||||
@@ -1716,6 +1721,10 @@ impl SetDisks {
|
|||||||
Ok((result, None))
|
Ok((result, None))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "lock-taking wrapper over the live heal_object_dir_locked; only comments reference it (backlog#1823)"
|
||||||
|
)]
|
||||||
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
|
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
|
||||||
pub(in crate::set_disk) async fn heal_object_dir(
|
pub(in crate::set_disk) async fn heal_object_dir(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -66,6 +66,7 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
|
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||||
match err {
|
match err {
|
||||||
LockResult::Timeout => {
|
LockResult::Timeout => {
|
||||||
@@ -79,6 +80,7 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||||
pub(in crate::set_disk) fn format_lock_error_from_error(
|
pub(in crate::set_disk) fn format_lock_error_from_error(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -143,6 +145,7 @@ impl SetDisks {
|
|||||||
disks
|
disks
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
|
pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
|
||||||
let snapshot = self.drive_membership_snapshot().await;
|
let snapshot = self.drive_membership_snapshot().await;
|
||||||
let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
|
let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
|
||||||
@@ -153,6 +156,10 @@ impl SetDisks {
|
|||||||
disks
|
disks
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "local-only sibling of the test-covered get_online_disks; no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
|
pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
|
||||||
let snapshot = self.drive_membership_snapshot().await;
|
let snapshot = self.drive_membership_snapshot().await;
|
||||||
let mut disks = snapshot
|
let mut disks = snapshot
|
||||||
@@ -432,6 +439,10 @@ impl SetDisks {
|
|||||||
Ok((disk, fm))
|
Ok((disk, fm))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "MinIO-parity healing-disk accessor with no caller in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(in crate::set_disk) async fn get_online_disk_with_healing(
|
pub(in crate::set_disk) async fn get_online_disk_with_healing(
|
||||||
&self,
|
&self,
|
||||||
incl_healing: bool,
|
incl_healing: bool,
|
||||||
@@ -440,6 +451,10 @@ impl SetDisks {
|
|||||||
Ok((new_disks, healing > 0))
|
Ok((new_disks, healing > 0))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "reached only from get_online_disk_with_healing, itself uncalled in this port (backlog#1823)"
|
||||||
|
)]
|
||||||
pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info(
|
pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info(
|
||||||
&self,
|
&self,
|
||||||
incl_healing: bool,
|
incl_healing: bool,
|
||||||
|
|||||||
@@ -415,6 +415,7 @@ fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize
|
|||||||
/// never returned, but flips `is_truncated` to `true` and yields a
|
/// never returned, but flips `is_truncated` to `true` and yields a
|
||||||
/// `next_upload_id_marker` pointing at the last returned upload so the caller can
|
/// `next_upload_id_marker` pointing at the last returned upload so the caller can
|
||||||
/// resume paging.
|
/// resume paging.
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) {
|
fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) {
|
||||||
let is_truncated = remaining.len() > max_uploads;
|
let is_truncated = remaining.len() > max_uploads;
|
||||||
let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect();
|
let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect();
|
||||||
@@ -557,6 +558,7 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(self))]
|
#[tracing::instrument(level = "debug", skip(self))]
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
pub(super) async fn check_upload_id_exists(
|
pub(super) async fn check_upload_id_exists(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -1398,7 +1400,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
|
|
||||||
let mut count = max_parts;
|
let mut count = max_parts;
|
||||||
|
|
||||||
for (i, part) in object_parts.iter().enumerate() {
|
for part in object_parts.iter() {
|
||||||
if let Some(err) = &part.error {
|
if let Some(err) = &part.error {
|
||||||
warn!("list_object_parts part error: {:?}", &err);
|
warn!("list_object_parts part error: {:?}", &err);
|
||||||
}
|
}
|
||||||
@@ -2041,8 +2043,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
|
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
|
||||||
{
|
{
|
||||||
error!(
|
error!(
|
||||||
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
|
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}, err={}",
|
||||||
p.part_num, bucket, object
|
p.part_num, bucket, object, err
|
||||||
);
|
);
|
||||||
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
||||||
}
|
}
|
||||||
@@ -2087,8 +2089,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
} else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) {
|
} else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) {
|
||||||
error!(
|
error!(
|
||||||
"complete_multipart_upload checksum matches failed want={}, got={}",
|
"complete_multipart_upload checksum matches failed want={}, got={}, err={}",
|
||||||
wtcs.encoded, checksum.encoded
|
wtcs.encoded, checksum.encoded, err
|
||||||
);
|
);
|
||||||
return Err(Error::other(format!(
|
return Err(Error::other(format!(
|
||||||
"complete_multipart_upload checksum matches failed want={}, got={}",
|
"complete_multipart_upload checksum matches failed want={}, got={}",
|
||||||
|
|||||||
@@ -3507,6 +3507,10 @@ struct TransitionUploadedSaveProbeState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
struct TransitionUploadedSaveProbe {
|
struct TransitionUploadedSaveProbe {
|
||||||
state: Arc<TransitionUploadedSaveProbeState>,
|
state: Arc<TransitionUploadedSaveProbeState>,
|
||||||
}
|
}
|
||||||
@@ -3517,6 +3521,10 @@ static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Opti
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
impl TransitionUploadedSaveProbe {
|
impl TransitionUploadedSaveProbe {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
fn install(bucket: &str, object: &str) -> Self {
|
fn install(bucket: &str, object: &str) -> Self {
|
||||||
let state = Arc::new(TransitionUploadedSaveProbeState {
|
let state = Arc::new(TransitionUploadedSaveProbeState {
|
||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
@@ -3533,6 +3541,10 @@ impl TransitionUploadedSaveProbe {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
fn attempts(&self) -> usize {
|
fn attempts(&self) -> usize {
|
||||||
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
||||||
}
|
}
|
||||||
@@ -3738,6 +3750,10 @@ struct TransitionCommitBarrierState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
struct TransitionCommitBarrier {
|
struct TransitionCommitBarrier {
|
||||||
state: Arc<TransitionCommitBarrierState>,
|
state: Arc<TransitionCommitBarrierState>,
|
||||||
}
|
}
|
||||||
@@ -3748,14 +3764,26 @@ static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Ar
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
impl TransitionCommitBarrier {
|
impl TransitionCommitBarrier {
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self {
|
fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self {
|
||||||
Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost)
|
Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
fn install(bucket: &str, object: &str) -> Self {
|
fn install(bucket: &str, object: &str) -> Self {
|
||||||
Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation)
|
Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
fn install_after_lease_check(bucket: &str, object: &str) -> Self {
|
fn install_after_lease_check(bucket: &str, object: &str) -> Self {
|
||||||
Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation)
|
Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation)
|
||||||
}
|
}
|
||||||
@@ -3778,12 +3806,20 @@ impl TransitionCommitBarrier {
|
|||||||
Self { state }
|
Self { state }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
async fn wait_until_paused(&self) {
|
async fn wait_until_paused(&self) {
|
||||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||||
.await
|
.await
|
||||||
.expect("transition should reach the deterministic commit barrier");
|
.expect("transition should reach the deterministic commit barrier");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||||
|
)]
|
||||||
fn release(&self) {
|
fn release(&self) {
|
||||||
self.state.release.notify_one();
|
self.state.release.notify_one();
|
||||||
}
|
}
|
||||||
@@ -5620,7 +5656,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
// TODO: Lifecycle
|
// TODO: Lifecycle
|
||||||
|
|
||||||
let mut version_found = true;
|
let mut version_found = true;
|
||||||
let (mut goi, write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
|
// delete_object_version below derives its own majority quorum from the
|
||||||
|
// disk array, so the object-derived quorum here is unused.
|
||||||
|
let (mut goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
|
||||||
if let Some(err) = &gerr
|
if let Some(err) = &gerr
|
||||||
&& goi.name.is_empty()
|
&& goi.name.is_empty()
|
||||||
{
|
{
|
||||||
@@ -6374,7 +6412,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||||
|
|
||||||
for disk in disks.iter() {
|
for disk in disks.iter() {
|
||||||
if let Some(disk) = disk {
|
if disk.is_some() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let _ = self
|
let _ = self
|
||||||
@@ -7505,7 +7543,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT,
|
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT,
|
||||||
GetObjectFailureReason,
|
GetObjectFailureReason,
|
||||||
};
|
};
|
||||||
use crate::disk::RUSTFS_META_BUCKET;
|
use crate::disk::RUSTFS_META_BUCKET;
|
||||||
@@ -7637,8 +7675,8 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
legacy_completed,
|
legacy_completed,
|
||||||
internal_cancelled,
|
internal_cancelled,
|
||||||
legacy_cancelled,
|
legacy_cancelled,
|
||||||
internal_unsafe_miss,
|
internal_not_found_miss,
|
||||||
legacy_unsafe_miss,
|
legacy_not_found_miss,
|
||||||
internal_saved,
|
internal_saved,
|
||||||
legacy_saved,
|
legacy_saved,
|
||||||
) = metrics::with_local_recorder(&recorder, || {
|
) = metrics::with_local_recorder(&recorder, || {
|
||||||
@@ -7714,7 +7752,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_INTERNAL_META),
|
("path", GET_OBJECT_PATH_INTERNAL_META),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST),
|
("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
recorder.counter_value(
|
recorder.counter_value(
|
||||||
@@ -7722,7 +7760,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
&[
|
&[
|
||||||
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
|
||||||
("decision", "miss"),
|
("decision", "miss"),
|
||||||
("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST),
|
("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
recorder.histogram_values(
|
recorder.histogram_values(
|
||||||
@@ -7773,21 +7811,21 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
"internal metadata lifecycle cancelled count must not leak into legacy_duplex"
|
"internal metadata lifecycle cancelled count must not leak into legacy_duplex"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
internal_unsafe_miss, 1,
|
internal_not_found_miss, 1,
|
||||||
"internal metadata unsafe early-stop miss must retain its path label"
|
"internal metadata not-found early-stop miss must retain its path label"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
legacy_unsafe_miss, 0,
|
legacy_not_found_miss, 0,
|
||||||
"internal metadata unsafe early-stop miss must not leak into legacy_duplex"
|
"internal metadata not-found early-stop miss must not leak into legacy_duplex"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
internal_saved,
|
internal_saved,
|
||||||
vec![0.0],
|
vec![0.0],
|
||||||
"internal metadata unsafe miss must record zero saved responses on internal_meta"
|
"internal metadata not-found miss must record zero saved responses on internal_meta"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
legacy_saved.is_empty(),
|
legacy_saved.is_empty(),
|
||||||
"internal metadata unsafe miss saved responses must not leak into legacy_duplex"
|
"internal metadata not-found miss saved responses must not leak into legacy_duplex"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -10170,6 +10208,288 @@ mod transition_upload_integrity_tests {
|
|||||||
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Compresses `plaintext` with the codec the PUT path uses, so the stored
|
||||||
|
/// bytes round-trip through the read path's decompressor.
|
||||||
|
async fn compress_for_storage(plaintext: &[u8]) -> Vec<u8> {
|
||||||
|
let mut reader = crate::io_support::rio::compression_reader(
|
||||||
|
Cursor::new(plaintext.to_vec()),
|
||||||
|
rustfs_utils::CompressionAlgorithm::default(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
let mut compressed = Vec::new();
|
||||||
|
reader.read_to_end(&mut compressed).await.expect("plaintext should compress");
|
||||||
|
assert!(compressed.len() < plaintext.len(), "test payload must actually compress");
|
||||||
|
compressed
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Writes a genuinely compressed object: stored data is `compressed`, and the
|
||||||
|
/// metadata marks it compressed with the plaintext length as its actual size,
|
||||||
|
/// exactly as the app-layer compress path records it.
|
||||||
|
async fn write_compressed_source(
|
||||||
|
set_disks: &Arc<SetDisks>,
|
||||||
|
disk_stores: &[DiskStore],
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
plaintext: &[u8],
|
||||||
|
compressed: &[u8],
|
||||||
|
) -> ObjectInfo {
|
||||||
|
for disk in disk_stores {
|
||||||
|
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||||
|
}
|
||||||
|
let mut user_defined = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut user_defined,
|
||||||
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
|
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
|
||||||
|
let stream = crate::io_support::rio::HashReader::from_stream(
|
||||||
|
Cursor::new(compressed.to_vec()),
|
||||||
|
compressed.len() as i64,
|
||||||
|
plaintext.len() as i64,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.expect("hash reader over compressed bytes");
|
||||||
|
let mut reader = PutObjReader::new(stream);
|
||||||
|
set_disks
|
||||||
|
.put_object(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&mut reader,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
user_defined,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("compressed object should be written")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_transitioned(
|
||||||
|
set_disks: &Arc<SetDisks>,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
range: Option<HTTPRangeSpec>,
|
||||||
|
opts: &ObjectOptions,
|
||||||
|
) -> (Vec<u8>, i64) {
|
||||||
|
let mut reader = set_disks
|
||||||
|
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
|
||||||
|
.await
|
||||||
|
.expect("transitioned object reader should open");
|
||||||
|
let published_size = reader.object_info.size;
|
||||||
|
let mut body = Vec::new();
|
||||||
|
reader
|
||||||
|
.stream
|
||||||
|
.read_to_end(&mut body)
|
||||||
|
.await
|
||||||
|
.expect("transitioned body should drain");
|
||||||
|
(body, published_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Transition uploads the object's STORED bytes, so a tiered read has to
|
||||||
|
/// apply the same transform an erasure read would. #6107 routed this path
|
||||||
|
/// through `ReadPlan` to stop serving an encrypted object's ciphertext;
|
||||||
|
/// compression rides the same plan, and nothing pinned it (backlog#1851).
|
||||||
|
/// Without the transform this GET returns the compressed bytes under the
|
||||||
|
/// compressed size — silent corruption for every client of a compressed
|
||||||
|
/// object that ILM has moved to a warm tier.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn transitioned_compressed_object_get_returns_plaintext() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "transitioned-compressed-get-bucket";
|
||||||
|
let object = "object.txt";
|
||||||
|
let plaintext = b"transitioned compressed objects must decompress on read ".repeat(20_000);
|
||||||
|
let compressed = compress_for_storage(&plaintext).await;
|
||||||
|
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
||||||
|
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (local_body, local_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
||||||
|
assert_eq!(local_body, plaintext, "control: the pre-transition read must decompress");
|
||||||
|
assert_eq!(
|
||||||
|
local_size,
|
||||||
|
plaintext.len() as i64,
|
||||||
|
"control: the pre-transition read publishes the plaintext size"
|
||||||
|
);
|
||||||
|
|
||||||
|
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||||
|
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||||
|
set_disks
|
||||||
|
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||||
|
.await
|
||||||
|
.expect("transition should commit");
|
||||||
|
|
||||||
|
let put_versions = backend.put_versions().await;
|
||||||
|
assert_eq!(put_versions.len(), 1, "transition should upload one remote candidate");
|
||||||
|
let remote_bytes = backend
|
||||||
|
.bytes(&put_versions[0].0)
|
||||||
|
.await
|
||||||
|
.expect("remote candidate should be stored");
|
||||||
|
assert_eq!(
|
||||||
|
remote_bytes, compressed,
|
||||||
|
"transition uploads the stored representation; the read side is what has to decode it"
|
||||||
|
);
|
||||||
|
|
||||||
|
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
||||||
|
assert_eq!(body, plaintext, "a tiered read must return the object's content, not its stored bytes");
|
||||||
|
assert_eq!(
|
||||||
|
published_size,
|
||||||
|
plaintext.len() as i64,
|
||||||
|
"a tiered read must publish the plaintext size, not the compressed one"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A ranged tiered read is expressed in plaintext coordinates, so the plan
|
||||||
|
/// has to translate it into the remote copy's compressed extent and skip
|
||||||
|
/// into the decompressed stream — the same translation the erasure path does.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn transitioned_compressed_object_range_get_returns_plaintext_slice() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "transitioned-compressed-range-bucket";
|
||||||
|
let object = "object.txt";
|
||||||
|
let plaintext = b"ranged reads of transitioned compressed objects must land in plaintext ".repeat(20_000);
|
||||||
|
let compressed = compress_for_storage(&plaintext).await;
|
||||||
|
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
||||||
|
|
||||||
|
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||||
|
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||||
|
set_disks
|
||||||
|
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||||
|
.await
|
||||||
|
.expect("transition should commit");
|
||||||
|
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
// Deliberately past the compressed size, so a range still measured in
|
||||||
|
// stored coordinates could not produce this slice.
|
||||||
|
let start = compressed.len() as i64 + 4096;
|
||||||
|
let end = start + 511;
|
||||||
|
let range = HTTPRangeSpec {
|
||||||
|
is_suffix_length: false,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
};
|
||||||
|
let (body, published_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
|
||||||
|
|
||||||
|
let expected = &plaintext[start as usize..=end as usize];
|
||||||
|
assert_eq!(body, expected, "a ranged tiered read must return that plaintext slice");
|
||||||
|
assert_eq!(published_size, expected.len() as i64, "a ranged tiered read publishes the slice length");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The restore copy-back re-writes the object under its original metadata,
|
||||||
|
/// which still says "compressed". It therefore has to keep receiving the
|
||||||
|
/// STORED bytes: `restore_request_active` holds it on the plan's `Plain`
|
||||||
|
/// branch, and decompressing there would write plaintext under compressed
|
||||||
|
/// metadata.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn restore_read_of_transitioned_compressed_object_keeps_stored_bytes() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "transitioned-compressed-restore-bucket";
|
||||||
|
let object = "object.txt";
|
||||||
|
let plaintext = b"restore copy-back must keep the stored representation intact ".repeat(20_000);
|
||||||
|
let compressed = compress_for_storage(&plaintext).await;
|
||||||
|
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
||||||
|
|
||||||
|
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||||
|
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||||
|
set_disks
|
||||||
|
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||||
|
.await
|
||||||
|
.expect("transition should commit");
|
||||||
|
|
||||||
|
let oi = set_disks
|
||||||
|
.get_object_info(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("transitioned metadata should resolve");
|
||||||
|
let restore_opts = ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
part_number: Some(1),
|
||||||
|
transition: TransitionOptions {
|
||||||
|
restore_request: s3s::dto::RestoreRequest {
|
||||||
|
days: Some(1),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
&None,
|
||||||
|
&HeaderMap::new(),
|
||||||
|
&oi,
|
||||||
|
&restore_opts,
|
||||||
|
&set_disks.ctx.tier_config_mgr(),
|
||||||
|
set_disks.ctx.object_encryption_resolver(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("restore read of the tiered copy should open");
|
||||||
|
let published_size = reader.object_info.size;
|
||||||
|
let mut body = Vec::new();
|
||||||
|
reader.stream.read_to_end(&mut body).await.expect("restore body should drain");
|
||||||
|
|
||||||
|
assert_eq!(body, compressed, "a restore read must copy the stored bytes back verbatim");
|
||||||
|
assert_eq!(
|
||||||
|
published_size,
|
||||||
|
compressed.len() as i64,
|
||||||
|
"a restore read must keep publishing the stored size"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Plain objects must keep streaming the remote bytes through untouched:
|
||||||
|
/// their plan is `Plain`, so the tiered read stays byte-identical.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn transitioned_plain_object_get_is_unchanged() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "transitioned-plain-get-bucket";
|
||||||
|
let object = "object.bin";
|
||||||
|
let payload = b"plain transitioned objects must keep reading back byte-identical ".repeat(1024);
|
||||||
|
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||||
|
|
||||||
|
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||||
|
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||||
|
set_disks
|
||||||
|
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||||
|
.await
|
||||||
|
.expect("transition should commit");
|
||||||
|
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
no_lock: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
||||||
|
assert_eq!(body, payload);
|
||||||
|
assert_eq!(published_size, payload.len() as i64);
|
||||||
|
|
||||||
|
let range = HTTPRangeSpec {
|
||||||
|
is_suffix_length: false,
|
||||||
|
start: 100,
|
||||||
|
end: 611,
|
||||||
|
};
|
||||||
|
let (ranged_body, ranged_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
|
||||||
|
assert_eq!(ranged_body, &payload[100..=611]);
|
||||||
|
assert_eq!(ranged_size, payload.len() as i64, "a plain ranged read keeps publishing the object size");
|
||||||
|
}
|
||||||
|
|
||||||
async fn corrupt_beyond_read_quorum(
|
async fn corrupt_beyond_read_quorum(
|
||||||
temp_dirs: &[tempfile::TempDir],
|
temp_dirs: &[tempfile::TempDir],
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -116,6 +116,7 @@ impl SetDisks {
|
|||||||
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
||||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||||
MetadataCacheLookup::Hit(entry) => Some(entry),
|
MetadataCacheLookup::Hit(entry) => Some(entry),
|
||||||
@@ -1826,6 +1827,7 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp
|
|||||||
.then_some(GET_METADATA_CACHE_REASON_META_BUCKET)
|
.then_some(GET_METADATA_CACHE_REASON_META_BUCKET)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||||
fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool {
|
fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool {
|
||||||
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
|
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
|
||||||
}
|
}
|
||||||
@@ -3886,13 +3888,15 @@ mod tests {
|
|||||||
assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
|
assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
|
||||||
// observe=false (non-observed fanout) also disables early-stop.
|
// observe=false (non-observed fanout) also disables early-stop.
|
||||||
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
|
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
|
||||||
assert!(!metadata_early_stop_permitted(true, true, true, "", false, false));
|
// Whole/latest data-read metadata is now allowed by default;
|
||||||
|
// the inline verifier still decides whether it can stop early.
|
||||||
|
assert!(metadata_early_stop_permitted(true, true, true, "", false, false));
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn metadata_early_stop_keeps_data_reads_opt_in_by_default() {
|
fn metadata_early_stop_allows_safe_data_reads_by_default() {
|
||||||
temp_env::with_vars(
|
temp_env::with_vars(
|
||||||
[
|
[
|
||||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||||
@@ -3900,7 +3904,7 @@ mod tests {
|
|||||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
||||||
],
|
],
|
||||||
|| {
|
|| {
|
||||||
assert!(!should_allow_metadata_early_stop(true, "", false, false));
|
assert!(should_allow_metadata_early_stop(true, "", false, false));
|
||||||
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
|
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
|
||||||
assert!(should_allow_metadata_early_stop(false, "", false, false));
|
assert!(should_allow_metadata_early_stop(false, "", false, false));
|
||||||
assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
|
assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
|
||||||
@@ -3932,6 +3936,34 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metadata_early_stop_bounded_fanout_defaults_to_enabled() {
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, None),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
assert!(is_get_metadata_data_read_early_stop_enabled());
|
||||||
|
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||||
|
},
|
||||||
|
);
|
||||||
|
temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false"))], || {
|
||||||
|
assert!(!is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||||
|
});
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")),
|
||||||
|
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
assert!(!is_get_metadata_data_read_early_stop_enabled());
|
||||||
|
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn metadata_early_stop_rejects_healing_and_free_version_requests() {
|
fn metadata_early_stop_rejects_healing_and_free_version_requests() {
|
||||||
temp_env::with_vars(
|
temp_env::with_vars(
|
||||||
|
|||||||
@@ -135,10 +135,7 @@ impl FileMeta {
|
|||||||
let i = buf.len() as u64;
|
let i = buf.len() as u64;
|
||||||
|
|
||||||
// check version, buf = buf[8..]
|
// check version, buf = buf[8..]
|
||||||
let (buf, _, _) = Self::check_xl2_v1(buf).map_err(|e| {
|
let (buf, _, _) = Self::check_xl2_v1(buf)?;
|
||||||
error!("failed to check XL2 v1 format: {}", e);
|
|
||||||
e
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if buf.len() < 5 {
|
if buf.len() < 5 {
|
||||||
error!(
|
error!(
|
||||||
|
|||||||
@@ -82,8 +82,8 @@ impl Error {
|
|||||||
/// Whether a heal operation can be retried without changing its inputs.
|
/// Whether a heal operation can be retried without changing its inputs.
|
||||||
pub(crate) fn is_recoverable_heal(&self) -> bool {
|
pub(crate) fn is_recoverable_heal(&self) -> bool {
|
||||||
match self {
|
match self {
|
||||||
Error::TaskCancelled => false,
|
Error::TaskCancelled | Error::TaskTimeout => false,
|
||||||
Error::TaskTimeout | Error::TransientSkip { .. } => true,
|
Error::TransientSkip { .. } => true,
|
||||||
Error::Storage(err) => {
|
Error::Storage(err) => {
|
||||||
err.is_quorum_error()
|
err.is_quorum_error()
|
||||||
|| matches!(
|
|| matches!(
|
||||||
@@ -165,4 +165,9 @@ mod tests {
|
|||||||
assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal());
|
assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal());
|
||||||
assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal());
|
assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn task_timeout_is_terminal() {
|
||||||
|
assert!(!Error::TaskTimeout.is_recoverable_heal());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -673,6 +673,12 @@ fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(Hea
|
|||||||
Some((request, delay, error))
|
Some((request, delay, error))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
|
||||||
|
let (_, delay, error) = retry_request_for_result(task, result)?;
|
||||||
|
let request = task.retry_request_with_remaining_timeout().await.ok()?;
|
||||||
|
Some((request, delay, error))
|
||||||
|
}
|
||||||
|
|
||||||
fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
|
fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
|
||||||
let retry_attempt = retry_attempt.clamp(1, 5);
|
let retry_attempt = retry_attempt.clamp(1, 5);
|
||||||
let delay = Duration::from_secs(2_u64.saturating_pow(retry_attempt));
|
let delay = Duration::from_secs(2_u64.saturating_pow(retry_attempt));
|
||||||
@@ -690,7 +696,7 @@ pub struct HealConfig {
|
|||||||
pub max_concurrent_heals: usize,
|
pub max_concurrent_heals: usize,
|
||||||
/// Maximum concurrent heal tasks allowed for a single erasure set
|
/// Maximum concurrent heal tasks allowed for a single erasure set
|
||||||
pub max_concurrent_per_set: usize,
|
pub max_concurrent_per_set: usize,
|
||||||
/// Task timeout
|
/// Aggregate task execution timeout across recoverable retries
|
||||||
pub task_timeout: Duration,
|
pub task_timeout: Duration,
|
||||||
/// Queue size
|
/// Queue size
|
||||||
pub queue_size: usize,
|
pub queue_size: usize,
|
||||||
@@ -3106,7 +3112,7 @@ impl HealManager {
|
|||||||
"Heal scheduler task started"
|
"Heal scheduler task started"
|
||||||
);
|
);
|
||||||
let result = task.execute().await;
|
let result = task.execute().await;
|
||||||
let retry_request = retry_request_for_result(task.as_ref(), &result);
|
let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await;
|
||||||
match &result {
|
match &result {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -4539,6 +4545,25 @@ mod tests {
|
|||||||
assert!(retry_error.contains("Lock acquisition timeout"));
|
assert!(retry_error.contains("Lock acquisition timeout"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn retry_request_for_result_preserves_remaining_timeout_budget() {
|
||||||
|
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||||
|
let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
|
||||||
|
request.options.timeout = Some(Duration::from_secs(60));
|
||||||
|
let task = HealTask::from_request(request, storage);
|
||||||
|
let result = task.execute().await;
|
||||||
|
|
||||||
|
let (retry_request, _, _) = retry_request_for_result_with_budget(&task, &result)
|
||||||
|
.await
|
||||||
|
.expect("read quorum failure should retain the unused timeout budget");
|
||||||
|
let remaining = retry_request
|
||||||
|
.options
|
||||||
|
.timeout
|
||||||
|
.expect("configured timeout should remain present");
|
||||||
|
assert!(remaining < Duration::from_secs(60));
|
||||||
|
assert!(remaining > Duration::from_secs(59));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_retry_request_for_incomplete_heal_rename() {
|
fn test_retry_request_for_incomplete_heal_rename() {
|
||||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||||
@@ -6054,7 +6079,7 @@ mod tests {
|
|||||||
process_manager_queue_once(&manager).await;
|
process_manager_queue_once(&manager).await;
|
||||||
let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async {
|
let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async {
|
||||||
loop {
|
loop {
|
||||||
if let Ok(status @ HealTaskStatus::Retrying { .. }) = manager.get_task_status(&defaulted_id).await {
|
if let Ok(status @ HealTaskStatus::Timeout) = manager.get_task_status(&defaulted_id).await {
|
||||||
break status;
|
break status;
|
||||||
}
|
}
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
@@ -6062,23 +6087,8 @@ mod tests {
|
|||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("configured timeout should finish the task");
|
.expect("configured timeout should finish the task");
|
||||||
assert!(matches!(defaulted_status, HealTaskStatus::Retrying { .. }));
|
assert_eq!(defaulted_status, HealTaskStatus::Timeout);
|
||||||
assert_eq!(
|
assert!(manager.retrying_heals.lock().await.get(&defaulted_id).is_none());
|
||||||
manager
|
|
||||||
.retrying_heals
|
|
||||||
.lock()
|
|
||||||
.await
|
|
||||||
.get(&defaulted_id)
|
|
||||||
.expect("timed out task should retain its retry request")
|
|
||||||
.request
|
|
||||||
.options
|
|
||||||
.timeout,
|
|
||||||
Some(Duration::ZERO)
|
|
||||||
);
|
|
||||||
manager
|
|
||||||
.cancel_task(&defaulted_id)
|
|
||||||
.await
|
|
||||||
.expect("retrying timeout task should be cancelled");
|
|
||||||
|
|
||||||
let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin);
|
let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin);
|
||||||
explicit.options.timeout = Some(Duration::from_secs(60));
|
explicit.options.timeout = Some(Duration::from_secs(60));
|
||||||
|
|||||||
@@ -196,7 +196,7 @@ pub struct HealOptions {
|
|||||||
/// Whether to skip namespace locking
|
/// Whether to skip namespace locking
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub no_lock: bool,
|
pub no_lock: bool,
|
||||||
/// Timeout
|
/// Aggregate execution timeout across recoverable manager retries
|
||||||
pub timeout: Option<Duration>,
|
pub timeout: Option<Duration>,
|
||||||
/// pool index
|
/// pool index
|
||||||
pub pool_index: Option<usize>,
|
pub pool_index: Option<usize>,
|
||||||
@@ -442,6 +442,14 @@ impl HealTask {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn retry_request_with_remaining_timeout(&self) -> Result<HealRequest> {
|
||||||
|
let mut request = self.retry_request();
|
||||||
|
if self.options.timeout.is_some() {
|
||||||
|
request.options.timeout = self.remaining_timeout().await?;
|
||||||
|
}
|
||||||
|
Ok(request)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn from_replacement_recovery_request(
|
pub(crate) fn from_replacement_recovery_request(
|
||||||
request: HealRequest,
|
request: HealRequest,
|
||||||
storage: Arc<dyn HealStorageAPI>,
|
storage: Arc<dyn HealStorageAPI>,
|
||||||
@@ -2657,6 +2665,36 @@ mod tests {
|
|||||||
|
|
||||||
use super::super::storage_api::status::BucketInfo;
|
use super::super::storage_api::status::BucketInfo;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn retry_request_carries_remaining_timeout_budget() {
|
||||||
|
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage::default());
|
||||||
|
let mut request = HealRequest::bucket("bucket".to_string());
|
||||||
|
request.options.timeout = Some(Duration::from_secs(100));
|
||||||
|
let task = HealTask::from_request(request, storage.clone());
|
||||||
|
*task.task_start_instant.write().await = Some(Instant::now() - Duration::from_secs(40));
|
||||||
|
|
||||||
|
let retry = task
|
||||||
|
.retry_request_with_remaining_timeout()
|
||||||
|
.await
|
||||||
|
.expect("first retry should retain the unused timeout budget");
|
||||||
|
let first_remaining = retry.options.timeout.expect("configured timeout should remain present");
|
||||||
|
assert!(first_remaining <= Duration::from_secs(60));
|
||||||
|
assert!(first_remaining > Duration::from_secs(59));
|
||||||
|
|
||||||
|
let retry_task = HealTask::from_request(retry, storage);
|
||||||
|
*retry_task.task_start_instant.write().await = Some(Instant::now() - Duration::from_secs(20));
|
||||||
|
let second_retry = retry_task
|
||||||
|
.retry_request_with_remaining_timeout()
|
||||||
|
.await
|
||||||
|
.expect("second retry should retain only the unused aggregate budget");
|
||||||
|
let second_remaining = second_retry
|
||||||
|
.options
|
||||||
|
.timeout
|
||||||
|
.expect("configured timeout should remain present");
|
||||||
|
assert!(second_remaining <= Duration::from_secs(40));
|
||||||
|
assert!(second_remaining > Duration::from_secs(39));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn format_result_requires_every_requested_target_to_be_ok() {
|
fn format_result_requires_every_requested_target_to_be_ok() {
|
||||||
let result = HealResultItem {
|
let result = HealResultItem {
|
||||||
|
|||||||
@@ -487,22 +487,21 @@ pub fn record_get_object_completion(total_duration_secs: f64, response_size_byte
|
|||||||
|
|
||||||
/// Record the streaming strategy chosen for a GetObject response body.
|
/// Record the streaming strategy chosen for a GetObject response body.
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub fn record_get_object_stream_strategy(strategy: &str, buffer_size_bytes: usize, response_size_bytes: i64) {
|
pub fn record_get_object_stream_strategy(strategy: &'static str, buffer_size_bytes: usize, response_size_bytes: i64) {
|
||||||
if !get_stage_metrics_enabled() {
|
if !get_stage_metrics_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy.to_string()).increment(1);
|
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy).increment(1);
|
||||||
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy.to_string())
|
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy).record(usize_to_f64(buffer_size_bytes));
|
||||||
.record(usize_to_f64(buffer_size_bytes));
|
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy)
|
||||||
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy.to_string())
|
|
||||||
.record(i64_non_negative_to_f64(response_size_bytes));
|
.record(i64_non_negative_to_f64(response_size_bytes));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Record the response-body handoff shape from a GetObject reader into the S3 streaming body.
|
/// Record the response-body handoff shape from a GetObject reader into the S3 streaming body.
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub fn record_get_object_response_handoff(
|
pub fn record_get_object_response_handoff(
|
||||||
strategy: &str,
|
strategy: &'static str,
|
||||||
buffer_source: &str,
|
buffer_source: &'static str,
|
||||||
buffer_size_bytes: usize,
|
buffer_size_bytes: usize,
|
||||||
response_size_bytes: i64,
|
response_size_bytes: i64,
|
||||||
duration_secs: f64,
|
duration_secs: f64,
|
||||||
@@ -512,26 +511,26 @@ pub fn record_get_object_response_handoff(
|
|||||||
}
|
}
|
||||||
counter!(
|
counter!(
|
||||||
"rustfs_io_get_object_response_handoff_total",
|
"rustfs_io_get_object_response_handoff_total",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string()
|
"buffer_source" => buffer_source
|
||||||
)
|
)
|
||||||
.increment(1);
|
.increment(1);
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_response_handoff_buffer_size_bytes",
|
"rustfs_io_get_object_response_handoff_buffer_size_bytes",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string()
|
"buffer_source" => buffer_source
|
||||||
)
|
)
|
||||||
.record(usize_to_f64(buffer_size_bytes));
|
.record(usize_to_f64(buffer_size_bytes));
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_response_handoff_response_size_bytes",
|
"rustfs_io_get_object_response_handoff_response_size_bytes",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string()
|
"buffer_source" => buffer_source
|
||||||
)
|
)
|
||||||
.record(i64_non_negative_to_f64(response_size_bytes));
|
.record(i64_non_negative_to_f64(response_size_bytes));
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_response_handoff_duration_seconds",
|
"rustfs_io_get_object_response_handoff_duration_seconds",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string()
|
"buffer_source" => buffer_source
|
||||||
)
|
)
|
||||||
.record(duration_secs);
|
.record(duration_secs);
|
||||||
record_get_object_response_handoff_duration("s3_handler", duration_secs);
|
record_get_object_response_handoff_duration("s3_handler", duration_secs);
|
||||||
@@ -539,14 +538,18 @@ pub fn record_get_object_response_handoff(
|
|||||||
|
|
||||||
/// Record ReaderStream capacity chosen for GetObject handoff.
|
/// Record ReaderStream capacity chosen for GetObject handoff.
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source: &str, buffer_size_bytes: usize) {
|
pub fn record_get_object_reader_stream_buffer_size(
|
||||||
|
strategy: &'static str,
|
||||||
|
buffer_source: &'static str,
|
||||||
|
buffer_size_bytes: usize,
|
||||||
|
) {
|
||||||
if !get_stage_metrics_enabled() {
|
if !get_stage_metrics_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_reader_stream_buffer_size_bytes",
|
"rustfs_io_get_object_reader_stream_buffer_size_bytes",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string()
|
"buffer_source" => buffer_source
|
||||||
)
|
)
|
||||||
.record(usize_to_f64(buffer_size_bytes));
|
.record(usize_to_f64(buffer_size_bytes));
|
||||||
}
|
}
|
||||||
@@ -554,8 +557,8 @@ pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source
|
|||||||
/// Record ReaderStream poll outcomes for GetObject handoff attribution.
|
/// Record ReaderStream poll outcomes for GetObject handoff attribution.
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub fn record_get_object_reader_stream_poll(
|
pub fn record_get_object_reader_stream_poll(
|
||||||
strategy: &str,
|
strategy: &'static str,
|
||||||
buffer_source: &str,
|
buffer_source: &'static str,
|
||||||
outcome: &'static str,
|
outcome: &'static str,
|
||||||
remaining_before: usize,
|
remaining_before: usize,
|
||||||
bytes: usize,
|
bytes: usize,
|
||||||
@@ -567,36 +570,36 @@ pub fn record_get_object_reader_stream_poll(
|
|||||||
let bytes = u64::try_from(bytes).unwrap_or(u64::MAX);
|
let bytes = u64::try_from(bytes).unwrap_or(u64::MAX);
|
||||||
counter!(
|
counter!(
|
||||||
"rustfs_io_get_object_reader_stream_poll_total",
|
"rustfs_io_get_object_reader_stream_poll_total",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string(),
|
"buffer_source" => buffer_source,
|
||||||
"outcome" => outcome
|
"outcome" => outcome
|
||||||
)
|
)
|
||||||
.increment(1);
|
.increment(1);
|
||||||
counter!(
|
counter!(
|
||||||
"rustfs_io_get_object_reader_stream_poll_bytes_total",
|
"rustfs_io_get_object_reader_stream_poll_bytes_total",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string(),
|
"buffer_source" => buffer_source,
|
||||||
"outcome" => outcome
|
"outcome" => outcome
|
||||||
)
|
)
|
||||||
.increment(bytes);
|
.increment(bytes);
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_reader_stream_poll_remaining_bytes",
|
"rustfs_io_get_object_reader_stream_poll_remaining_bytes",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string(),
|
"buffer_source" => buffer_source,
|
||||||
"outcome" => outcome
|
"outcome" => outcome
|
||||||
)
|
)
|
||||||
.record(usize_to_f64(remaining_before));
|
.record(usize_to_f64(remaining_before));
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_reader_stream_poll_bytes",
|
"rustfs_io_get_object_reader_stream_poll_bytes",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string(),
|
"buffer_source" => buffer_source,
|
||||||
"outcome" => outcome
|
"outcome" => outcome
|
||||||
)
|
)
|
||||||
.record(usize_to_f64(bytes as usize));
|
.record(usize_to_f64(bytes as usize));
|
||||||
histogram!(
|
histogram!(
|
||||||
"rustfs_io_get_object_reader_stream_poll_duration_seconds",
|
"rustfs_io_get_object_reader_stream_poll_duration_seconds",
|
||||||
"strategy" => strategy.to_string(),
|
"strategy" => strategy,
|
||||||
"buffer_source" => buffer_source.to_string(),
|
"buffer_source" => buffer_source,
|
||||||
"outcome" => outcome
|
"outcome" => outcome
|
||||||
)
|
)
|
||||||
.record(duration_secs);
|
.record(duration_secs);
|
||||||
|
|||||||
@@ -31,7 +31,10 @@ pub struct KeystoneClient {
|
|||||||
admin_password: Option<String>,
|
admin_password: Option<String>,
|
||||||
admin_project: Option<String>,
|
admin_project: Option<String>,
|
||||||
admin_domain: String,
|
admin_domain: String,
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "TLS verification flag parsed from config; the reqwest client is built before it is consulted, so nothing reads it back (backlog#1823)"
|
||||||
|
)]
|
||||||
verify_ssl: bool,
|
verify_ssl: bool,
|
||||||
/// Request timeout applied to the underlying HTTP client.
|
/// Request timeout applied to the underlying HTTP client.
|
||||||
timeout: std::time::Duration,
|
timeout: std::time::Duration,
|
||||||
|
|||||||
@@ -20,7 +20,10 @@ use tracing::{debug, info};
|
|||||||
|
|
||||||
/// Maps Keystone identities to RustFS concepts
|
/// Maps Keystone identities to RustFS concepts
|
||||||
pub struct KeystoneIdentityMapper {
|
pub struct KeystoneIdentityMapper {
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "keeps the Keystone client alive for the mapper's lifetime; the mapping paths do not call through it yet (backlog#1823)"
|
||||||
|
)]
|
||||||
client: Arc<KeystoneClient>,
|
client: Arc<KeystoneClient>,
|
||||||
role_policy_map: HashMap<String, String>,
|
role_policy_map: HashMap<String, String>,
|
||||||
enable_tenant_prefix: bool,
|
enable_tenant_prefix: bool,
|
||||||
|
|||||||
@@ -293,6 +293,15 @@ enum StrictVaultAuthMethod {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
refresh_safety_window_secs: Option<u64>,
|
refresh_safety_window_secs: Option<u64>,
|
||||||
},
|
},
|
||||||
|
Kubernetes {
|
||||||
|
role: String,
|
||||||
|
#[serde(default)]
|
||||||
|
mount: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
jwt_path: Option<std::path::PathBuf>,
|
||||||
|
#[serde(default)]
|
||||||
|
refresh_safety_window_secs: Option<u64>,
|
||||||
|
},
|
||||||
TokenFile {
|
TokenFile {
|
||||||
path: std::path::PathBuf,
|
path: std::path::PathBuf,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -319,6 +328,17 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
|
|||||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
||||||
refresh_safety_window_secs,
|
refresh_safety_window_secs,
|
||||||
},
|
},
|
||||||
|
StrictVaultAuthMethod::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount,
|
||||||
|
jwt_path,
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
} => Self::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
|
||||||
|
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
},
|
||||||
StrictVaultAuthMethod::TokenFile {
|
StrictVaultAuthMethod::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
|||||||
auth_method_type: match &vault_config.auth_method {
|
auth_method_type: match &vault_config.auth_method {
|
||||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||||
|
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||||
},
|
},
|
||||||
has_stored_credentials: true,
|
has_stored_credentials: true,
|
||||||
@@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
|||||||
auth_method_type: match &vault_config.auth_method {
|
auth_method_type: match &vault_config.auth_method {
|
||||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||||
|
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||||
},
|
},
|
||||||
has_stored_credentials: true,
|
has_stored_credentials: true,
|
||||||
@@ -901,6 +923,42 @@ mod tests {
|
|||||||
assert!(request.to_kms_config().validate().is_ok());
|
assert!(request.to_kms_config().validate().is_ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The admin API reaches Kubernetes auth with the role alone; the mount and
|
||||||
|
/// the projected token path fall back to the cluster defaults, so a Tenant
|
||||||
|
/// manifest carries no credential and no cluster-specific paths.
|
||||||
|
#[test]
|
||||||
|
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
|
||||||
|
let raw = serde_json::json!({
|
||||||
|
"backend_type": "vault-transit",
|
||||||
|
"address": "https://vault.example.com:8200",
|
||||||
|
"mount_path": "rustfs",
|
||||||
|
"auth_method": { "Kubernetes": { "role": "rustfs" } }
|
||||||
|
});
|
||||||
|
|
||||||
|
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
|
||||||
|
let config = request.to_kms_config();
|
||||||
|
config.validate().expect("kubernetes auth must validate");
|
||||||
|
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
let VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} = &vault.auth_method
|
||||||
|
else {
|
||||||
|
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
|
||||||
|
};
|
||||||
|
assert_eq!(role, "rustfs");
|
||||||
|
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||||
|
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||||
|
|
||||||
|
let unknown_field = serde_json::json!({
|
||||||
|
"backend_type": "vault-transit",
|
||||||
|
"address": "https://vault.example.com:8200",
|
||||||
|
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
|
||||||
|
});
|
||||||
|
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
|
||||||
|
.expect_err("an unknown auth field must be rejected rather than silently dropped");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
||||||
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
||||||
|
|||||||
@@ -550,6 +550,7 @@ impl VaultKmsClient {
|
|||||||
address: config.address.clone(),
|
address: config.address.clone(),
|
||||||
namespace: config.namespace.clone(),
|
namespace: config.namespace.clone(),
|
||||||
attempt_timeout: kms_config.effective_timeout(),
|
attempt_timeout: kms_config.effective_timeout(),
|
||||||
|
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||||
};
|
};
|
||||||
let source = token_source_for(&config.auth_method, &settings)?;
|
let source = token_source_for(&config.auth_method, &settings)?;
|
||||||
let policy = VaultCredentialPolicy::from_kms_config(
|
let policy = VaultCredentialPolicy::from_kms_config(
|
||||||
|
|||||||
@@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
|
||||||
|
/// projected ServiceAccount token for a lease-bound Vault token.
|
||||||
|
///
|
||||||
|
/// The JWT is re-read on every login because the kubelet rotates a projected
|
||||||
|
/// token well inside the pod's lifetime; caching it would strand the source on
|
||||||
|
/// an expired assertion once the current Vault token can no longer be renewed.
|
||||||
|
///
|
||||||
|
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
|
||||||
|
/// the projected token and mounts it world-readable by default, so rejecting
|
||||||
|
/// group/other bits would refuse every standard pod rather than catch a
|
||||||
|
/// deployment error.
|
||||||
|
pub(crate) struct KubernetesLogin {
|
||||||
|
/// Unauthenticated client used only for the login exchange.
|
||||||
|
login_client: VaultClient,
|
||||||
|
mount: String,
|
||||||
|
role: String,
|
||||||
|
jwt_path: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl KubernetesLogin {
|
||||||
|
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
|
||||||
|
Ok(Self {
|
||||||
|
login_client: settings.build_login_client()?,
|
||||||
|
mount,
|
||||||
|
role,
|
||||||
|
jwt_path,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the ServiceAccount token for one login attempt.
|
||||||
|
///
|
||||||
|
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
|
||||||
|
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
|
||||||
|
/// has not projected yet heals the source without a restart.
|
||||||
|
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
|
||||||
|
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
|
||||||
|
.await
|
||||||
|
.map_err(|error| AttemptError {
|
||||||
|
class: ErrorClass::Fatal,
|
||||||
|
error: KmsError::configuration_error(format!(
|
||||||
|
"Failed to read Kubernetes ServiceAccount token {}: {error}",
|
||||||
|
self.jwt_path.display()
|
||||||
|
)),
|
||||||
|
})?;
|
||||||
|
let trimmed = raw.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
raw.zeroize();
|
||||||
|
return Err(AttemptError {
|
||||||
|
class: ErrorClass::Fatal,
|
||||||
|
error: KmsError::configuration_error(format!(
|
||||||
|
"Kubernetes ServiceAccount token {} is empty",
|
||||||
|
self.jwt_path.display()
|
||||||
|
)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let jwt = SecretString::new(trimmed.to_string());
|
||||||
|
raw.zeroize();
|
||||||
|
Ok(jwt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl TokenSource for KubernetesLogin {
|
||||||
|
async fn acquire(&self) -> AttemptResult<TokenLease> {
|
||||||
|
let jwt = self.resolve_jwt().await?;
|
||||||
|
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
|
||||||
|
.await
|
||||||
|
.map_err(|error| attempt_error("Kubernetes login", error))?;
|
||||||
|
Ok(TokenLease::from_auth(auth))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
|
||||||
|
let auth = vaultrs::token::renew_self(client, None)
|
||||||
|
.await
|
||||||
|
.map_err(|error| attempt_error("token renewal", error))?;
|
||||||
|
Ok(TokenLease::from_auth(auth))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for KubernetesLogin {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
// The login client embeds Vault client settings and must stay out of
|
||||||
|
// Debug output; the role name is not a secret, and the JWT is never held.
|
||||||
|
f.debug_struct("KubernetesLogin")
|
||||||
|
.field("mount", &self.mount)
|
||||||
|
.field("role", &self.role)
|
||||||
|
.field("jwt_path", &self.jwt_path)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
||||||
/// token file (for example a Vault Agent auto-auth sink).
|
/// token file (for example a Vault Agent auto-auth sink).
|
||||||
///
|
///
|
||||||
@@ -464,6 +555,9 @@ pub(crate) fn token_source_for(
|
|||||||
secret_id.clone(),
|
secret_id.clone(),
|
||||||
secret_id_file.clone(),
|
secret_id_file.clone(),
|
||||||
)?)),
|
)?)),
|
||||||
|
VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
|
||||||
VaultAuthMethod::TokenFile {
|
VaultAuthMethod::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings {
|
|||||||
pub(crate) namespace: Option<String>,
|
pub(crate) namespace: Option<String>,
|
||||||
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
||||||
pub(crate) attempt_timeout: Duration,
|
pub(crate) attempt_timeout: Duration,
|
||||||
|
/// Whether to accept an unverified Vault server certificate. Gated on
|
||||||
|
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
|
||||||
|
pub(crate) skip_tls_verify: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VaultConnectionSettings {
|
impl VaultConnectionSettings {
|
||||||
@@ -499,6 +596,11 @@ impl VaultConnectionSettings {
|
|||||||
// operation-level retry policy.
|
// operation-level retry policy.
|
||||||
settings_builder.timeout(Some(self.attempt_timeout));
|
settings_builder.timeout(Some(self.attempt_timeout));
|
||||||
settings_builder.token(token);
|
settings_builder.token(token);
|
||||||
|
// Always set explicitly: left unset, vaultrs derives this from its own
|
||||||
|
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
|
||||||
|
// disable certificate verification behind the KMS configuration and its
|
||||||
|
// insecure-defaults gate.
|
||||||
|
settings_builder.verify(!self.skip_tls_verify);
|
||||||
|
|
||||||
if let Some(namespace) = &self.namespace {
|
if let Some(namespace) = &self.namespace {
|
||||||
settings_builder.namespace(Some(namespace.clone()));
|
settings_builder.namespace(Some(namespace.clone()));
|
||||||
@@ -551,6 +653,10 @@ impl VaultCredentialPolicy {
|
|||||||
refresh_safety_window_secs: Some(secs),
|
refresh_safety_window_secs: Some(secs),
|
||||||
..
|
..
|
||||||
}
|
}
|
||||||
|
| VaultAuthMethod::Kubernetes {
|
||||||
|
refresh_safety_window_secs: Some(secs),
|
||||||
|
..
|
||||||
|
}
|
||||||
| VaultAuthMethod::TokenFile {
|
| VaultAuthMethod::TokenFile {
|
||||||
refresh_safety_window_secs: Some(secs),
|
refresh_safety_window_secs: Some(secs),
|
||||||
..
|
..
|
||||||
@@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle {
|
|||||||
|
|
||||||
impl VaultClientHandle {
|
impl VaultClientHandle {
|
||||||
/// Absolute expiry of this generation's token.
|
/// Absolute expiry of this generation's token.
|
||||||
|
///
|
||||||
|
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
|
||||||
|
/// a value too large to add to `issued_at` would panic on the bare `+`. A
|
||||||
|
/// TTL that cannot be represented is indistinguishable from no expiry, so it
|
||||||
|
/// collapses to `None` — the same answer already given for the zero-lease
|
||||||
|
/// tokens Vault issues, which keeps the token in use and still fully
|
||||||
|
/// validated by Vault on every call.
|
||||||
fn expires_at(&self) -> Option<Instant> {
|
fn expires_at(&self) -> Option<Instant> {
|
||||||
self.lease.map(|lease| self.issued_at + lease.ttl)
|
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// When the renewal task should refresh this generation: half the TTL,
|
/// When the renewal task should refresh this generation: half the TTL,
|
||||||
/// leaving the second half as budget for retries before the fail-closed
|
/// leaving the second half as budget for retries before the fail-closed
|
||||||
/// window is reached.
|
/// window is reached.
|
||||||
|
///
|
||||||
|
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
|
||||||
|
/// leaving a token that never expires with nothing to renew.
|
||||||
fn renew_at(&self) -> Option<Instant> {
|
fn renew_at(&self) -> Option<Instant> {
|
||||||
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
|
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -662,7 +778,7 @@ impl VaultCredentialProvider {
|
|||||||
let handle = self.current.load_full();
|
let handle = self.current.load_full();
|
||||||
if let Some(expires_at) = handle.expires_at() {
|
if let Some(expires_at) = handle.expires_at() {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
if now + self.policy.safety_window >= expires_at {
|
if self.inside_safety_window(now, expires_at) {
|
||||||
return Err(KmsError::credentials_unavailable(format!(
|
return Err(KmsError::credentials_unavailable(format!(
|
||||||
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
||||||
handle.generation, self.policy.safety_window
|
handle.generation, self.policy.safety_window
|
||||||
@@ -672,6 +788,18 @@ impl VaultCredentialProvider {
|
|||||||
Ok(handle)
|
Ok(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the token expiring at `expires_at` is close enough to refuse.
|
||||||
|
///
|
||||||
|
/// `safety_window` reaches here from persisted configuration, so it is not
|
||||||
|
/// guaranteed to have passed this version's validation: a window too large
|
||||||
|
/// to add to the current instant would panic on the bare `+`. Such a window
|
||||||
|
/// means every token is always inside it, so saturating to "refuse" is both
|
||||||
|
/// the fail-closed answer and the one the arithmetic was reaching for.
|
||||||
|
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
|
||||||
|
now.checked_add(self.policy.safety_window)
|
||||||
|
.is_none_or(|deadline| deadline >= expires_at)
|
||||||
|
}
|
||||||
|
|
||||||
/// Publish the credential gauges for the generation currently installed.
|
/// Publish the credential gauges for the generation currently installed.
|
||||||
///
|
///
|
||||||
/// The fail-closed gauge re-evaluates the very gate
|
/// The fail-closed gauge re-evaluates the very gate
|
||||||
@@ -683,7 +811,7 @@ impl VaultCredentialProvider {
|
|||||||
let fail_closed = match handle.expires_at() {
|
let fail_closed = match handle.expires_at() {
|
||||||
Some(expires_at) => {
|
Some(expires_at) => {
|
||||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||||
now + self.policy.safety_window >= expires_at
|
self.inside_safety_window(now, expires_at)
|
||||||
}
|
}
|
||||||
// A generation without an expiry has no remaining TTL to report
|
// A generation without an expiry has no remaining TTL to report
|
||||||
// and can never lapse, so it can never fail closed either.
|
// and can never lapse, so it can never fail closed either.
|
||||||
@@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::REDACTED_SECRET;
|
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||||
|
|
||||||
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
||||||
@@ -871,6 +999,7 @@ mod tests {
|
|||||||
address: "http://127.0.0.1:8200".to_string(),
|
address: "http://127.0.0.1:8200".to_string(),
|
||||||
namespace: Some("team-namespace".to_string()),
|
namespace: Some("team-namespace".to_string()),
|
||||||
attempt_timeout: Duration::from_secs(30),
|
attempt_timeout: Duration::from_secs(30),
|
||||||
|
skip_tls_verify: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1057,6 +1186,143 @@ mod tests {
|
|||||||
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_kubernetes_auth_method_maps_to_login_source() {
|
||||||
|
let settings = test_settings();
|
||||||
|
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
|
||||||
|
.expect("kubernetes auth must map to a login source");
|
||||||
|
|
||||||
|
assert!(format!("{source:?}").contains("KubernetesLogin"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
|
||||||
|
/// path from persisted configuration, so the fail-closed comparison must
|
||||||
|
/// survive a window too large to add to the current instant. Before the
|
||||||
|
/// checked arithmetic this panicked with "overflow when adding duration to
|
||||||
|
/// instant" on the first request after a lease-bearing login.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
|
||||||
|
let (provider, _state) = scripted_provider(
|
||||||
|
Duration::from_secs(60),
|
||||||
|
true,
|
||||||
|
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let error = provider
|
||||||
|
.current()
|
||||||
|
.expect_err("a window wider than any lease must refuse the token");
|
||||||
|
assert!(
|
||||||
|
matches!(error, KmsError::CredentialsUnavailable { .. }),
|
||||||
|
"expected CredentialsUnavailable, got {error:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `lease_duration` is a bare u64 straight off the Vault response and forms
|
||||||
|
/// the other side of the same comparison, so an absurd one must not panic
|
||||||
|
/// either. It is indistinguishable from a non-expiring token, which is how
|
||||||
|
/// the zero-lease case already behaves.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
|
||||||
|
let (provider, _state) = scripted_provider(
|
||||||
|
Duration::from_secs(u64::MAX),
|
||||||
|
true,
|
||||||
|
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
provider
|
||||||
|
.current()
|
||||||
|
.expect("a token whose expiry cannot be represented must stay usable");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The configured flag has to reach the HTTP client, not just the config
|
||||||
|
/// struct: every generation (authenticated and login) builds its own client,
|
||||||
|
/// and a Vault with a self-signed certificate fails the handshake unless
|
||||||
|
/// each one carries the setting.
|
||||||
|
#[test]
|
||||||
|
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
|
||||||
|
for skip_tls_verify in [false, true] {
|
||||||
|
let settings = VaultConnectionSettings {
|
||||||
|
address: "https://vault.example.com:8200".to_string(),
|
||||||
|
namespace: None,
|
||||||
|
attempt_timeout: Duration::from_secs(30),
|
||||||
|
skip_tls_verify,
|
||||||
|
};
|
||||||
|
|
||||||
|
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
|
||||||
|
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
|
||||||
|
|
||||||
|
let login = settings.build_login_client().expect("login client must build");
|
||||||
|
assert_eq!(login.settings.verify, !skip_tls_verify);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
|
||||||
|
/// builder leaves it unset, which would disable certificate verification
|
||||||
|
/// without passing the KMS insecure-defaults gate.
|
||||||
|
#[test]
|
||||||
|
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
|
||||||
|
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
|
||||||
|
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
|
||||||
|
assert!(
|
||||||
|
client.settings.verify,
|
||||||
|
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The projected token is read fresh per login attempt and trimmed, so a
|
||||||
|
/// kubelet rotation is picked up without a restart and a trailing newline
|
||||||
|
/// does not corrupt the assertion sent to Vault.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
|
||||||
|
let dir = tempfile::tempdir().expect("temp dir");
|
||||||
|
let path = dir.path().join("token");
|
||||||
|
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
|
||||||
|
|
||||||
|
let login = KubernetesLogin::new(
|
||||||
|
&test_settings(),
|
||||||
|
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||||
|
"rustfs".to_string(),
|
||||||
|
path.clone(),
|
||||||
|
)
|
||||||
|
.expect("login source must build");
|
||||||
|
|
||||||
|
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
|
||||||
|
|
||||||
|
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
|
||||||
|
assert_eq!(
|
||||||
|
login.resolve_jwt().await.expect("second read").expose(),
|
||||||
|
"rotated-jwt",
|
||||||
|
"a rotated projected token must be picked up without a restart"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ServiceAccount token is re-read per attempt, so an unreadable or
|
||||||
|
/// empty one fails that attempt without reaching Vault; the refresh loop
|
||||||
|
/// keeps retrying, which is what lets a late projection heal the source.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
|
||||||
|
let dir = tempfile::tempdir().expect("temp dir");
|
||||||
|
let missing = dir.path().join("absent-token");
|
||||||
|
let empty = dir.path().join("empty-token");
|
||||||
|
tokio::fs::write(&empty, " \n").await.expect("write empty token");
|
||||||
|
|
||||||
|
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
|
||||||
|
let login =
|
||||||
|
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
|
||||||
|
.expect("login source must build");
|
||||||
|
|
||||||
|
let error = login
|
||||||
|
.acquire()
|
||||||
|
.await
|
||||||
|
.expect_err("an unusable ServiceAccount token must fail the attempt");
|
||||||
|
assert!(matches!(error.class, ErrorClass::Fatal));
|
||||||
|
assert!(error.error.to_string().contains(expected), "got {}", error.error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn test_renewal_task_renews_at_half_ttl() {
|
async fn test_renewal_task_renews_at_half_ttl() {
|
||||||
let (provider, state) = scripted_provider(
|
let (provider, state) = scripted_provider(
|
||||||
|
|||||||
@@ -415,6 +415,7 @@ impl VaultTransitKmsClient {
|
|||||||
address: config.address.clone(),
|
address: config.address.clone(),
|
||||||
namespace: config.namespace.clone(),
|
namespace: config.namespace.clone(),
|
||||||
attempt_timeout: kms_config.effective_timeout(),
|
attempt_timeout: kms_config.effective_timeout(),
|
||||||
|
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||||
};
|
};
|
||||||
let source = token_source_for(&config.auth_method, &settings)?;
|
let source = token_source_for(&config.auth_method, &settings)?;
|
||||||
let policy = VaultCredentialPolicy::from_kms_config(
|
let policy = VaultCredentialPolicy::from_kms_config(
|
||||||
|
|||||||
@@ -450,6 +450,10 @@ impl VaultRestoreClient {
|
|||||||
address: target.address.clone(),
|
address: target.address.clone(),
|
||||||
namespace: target.namespace.clone(),
|
namespace: target.namespace.clone(),
|
||||||
attempt_timeout: kms_config.effective_timeout(),
|
attempt_timeout: kms_config.effective_timeout(),
|
||||||
|
// A restore target carries no TLS settings, so certificates are
|
||||||
|
// always verified: recovery is the last path that should accept an
|
||||||
|
// unauthenticated Vault.
|
||||||
|
skip_tls_verify: false,
|
||||||
};
|
};
|
||||||
let source = token_source_for(&target.auth_method, &settings)?;
|
let source = token_source_for(&target.auth_method, &settings)?;
|
||||||
let policy = VaultCredentialPolicy::from_kms_config(
|
let policy = VaultCredentialPolicy::from_kms_config(
|
||||||
|
|||||||
+292
-51
@@ -25,6 +25,10 @@ use url::Url;
|
|||||||
|
|
||||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||||
|
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
|
||||||
|
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
|
||||||
|
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
|
||||||
|
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
|
||||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||||
@@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
|||||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||||
|
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
|
||||||
|
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
|
||||||
|
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
|
||||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||||
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
||||||
@@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
|
|||||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||||
|
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
|
||||||
|
/// Where the kubelet projects a pod's ServiceAccount token by default.
|
||||||
|
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||||
|
|
||||||
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
||||||
///
|
///
|
||||||
@@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String {
|
|||||||
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_vault_kubernetes_mount() -> String {
|
||||||
|
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_vault_kubernetes_jwt_path() -> PathBuf {
|
||||||
|
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
|
||||||
|
}
|
||||||
|
|
||||||
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
||||||
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
||||||
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
||||||
@@ -490,6 +508,23 @@ pub enum VaultAuthMethod {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
refresh_safety_window_secs: Option<u64>,
|
refresh_safety_window_secs: Option<u64>,
|
||||||
},
|
},
|
||||||
|
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
|
||||||
|
/// for a lease-bound Vault token that is renewed in the background.
|
||||||
|
Kubernetes {
|
||||||
|
/// Vault role bound to this ServiceAccount.
|
||||||
|
role: String,
|
||||||
|
/// Kubernetes auth engine mount path.
|
||||||
|
#[serde(default = "default_vault_kubernetes_mount")]
|
||||||
|
mount: String,
|
||||||
|
/// Projected ServiceAccount token to present. Re-read on every login so
|
||||||
|
/// a token the kubelet rotates is picked up without a restart.
|
||||||
|
#[serde(default = "default_vault_kubernetes_jwt_path")]
|
||||||
|
jwt_path: PathBuf,
|
||||||
|
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
|
||||||
|
/// per-attempt timeout.
|
||||||
|
#[serde(default)]
|
||||||
|
refresh_safety_window_secs: Option<u64>,
|
||||||
|
},
|
||||||
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
||||||
/// the token is read from `path` and re-read periodically so a token
|
/// the token is read from `path` and re-read periodically so a token
|
||||||
/// rotated by the agent is picked up without a restart.
|
/// rotated by the agent is picked up without a restart.
|
||||||
@@ -520,6 +555,16 @@ impl VaultAuthMethod {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Kubernetes authentication with the default mount and projected token path.
|
||||||
|
pub fn kubernetes(role: String) -> Self {
|
||||||
|
Self::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount: default_vault_kubernetes_mount(),
|
||||||
|
jwt_path: default_vault_kubernetes_jwt_path(),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Agent-managed token file with the default poll interval.
|
/// Agent-managed token file with the default poll interval.
|
||||||
pub fn token_file(path: PathBuf) -> Self {
|
pub fn token_file(path: PathBuf) -> Self {
|
||||||
Self::TokenFile {
|
Self::TokenFile {
|
||||||
@@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod {
|
|||||||
.field("mount", mount)
|
.field("mount", mount)
|
||||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||||
.finish(),
|
.finish(),
|
||||||
|
// No redaction: the role and mount name a Vault binding, and the
|
||||||
|
// ServiceAccount token itself is never held on this type.
|
||||||
|
Self::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount,
|
||||||
|
jwt_path,
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
} => f
|
||||||
|
.debug_struct("Kubernetes")
|
||||||
|
.field("role", role)
|
||||||
|
.field("mount", mount)
|
||||||
|
.field("jwt_path", jwt_path)
|
||||||
|
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||||
|
.finish(),
|
||||||
Self::TokenFile {
|
Self::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -1028,50 +1087,12 @@ impl KmsConfig {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
KmsBackend::VaultKv2 => {
|
KmsBackend::VaultKv2 => {
|
||||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
config.backend_config =
|
||||||
let auth_method = vault_auth_method_from_env()?;
|
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
|
||||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
|
||||||
|
|
||||||
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
|
|
||||||
Some(path) => {
|
|
||||||
tracing::warn!(
|
|
||||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
|
||||||
);
|
|
||||||
path
|
|
||||||
}
|
|
||||||
None => default_vault_kv2_mount_path(),
|
|
||||||
};
|
|
||||||
|
|
||||||
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
|
|
||||||
address,
|
|
||||||
auth_method,
|
|
||||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
|
||||||
mount_path,
|
|
||||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
|
||||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
|
||||||
tls: vault_tls_config(skip_tls_verify),
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
KmsBackend::VaultTransit => {
|
KmsBackend::VaultTransit => {
|
||||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
config.backend_config =
|
||||||
let auth_method = vault_auth_method_from_env()?;
|
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
|
||||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
|
||||||
|
|
||||||
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
|
||||||
address,
|
|
||||||
auth_method,
|
|
||||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
|
||||||
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
|
|
||||||
metadata_kv_mount: get_env_str(
|
|
||||||
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
|
||||||
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
|
||||||
),
|
|
||||||
metadata_key_prefix: get_env_str(
|
|
||||||
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
|
|
||||||
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
|
|
||||||
),
|
|
||||||
tls: vault_tls_config(skip_tls_verify),
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
KmsBackend::Static => {
|
KmsBackend::Static => {
|
||||||
// Read from file first, then fall back to direct env var
|
// Read from file first, then fall back to direct env var
|
||||||
@@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
|||||||
path.starts_with(std::env::temp_dir())
|
path.starts_with(std::env::temp_dir())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Command-line values that take precedence over the matching environment
|
||||||
|
/// variables when assembling a Vault backend configuration.
|
||||||
|
///
|
||||||
|
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
|
||||||
|
/// reads, so these are only set when the operator passed an explicit flag.
|
||||||
|
///
|
||||||
|
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
|
||||||
|
/// redacting `Debug` impls elsewhere in this module exist because a derived one
|
||||||
|
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
|
||||||
|
/// error instead of a leak.
|
||||||
|
#[derive(Default, Clone, Copy)]
|
||||||
|
pub struct VaultCliOverrides<'a> {
|
||||||
|
pub address: Option<&'a str>,
|
||||||
|
pub token: Option<&'a str>,
|
||||||
|
pub mount_path: Option<&'a str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Assemble the Vault KV2 backend configuration from the environment.
|
||||||
|
///
|
||||||
|
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
|
||||||
|
/// so both resolve the same auth method, namespace, TLS and mount settings.
|
||||||
|
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
|
||||||
|
let mount_path = match overrides
|
||||||
|
.mount_path
|
||||||
|
.map(str::to_string)
|
||||||
|
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
|
||||||
|
{
|
||||||
|
Some(path) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||||
|
);
|
||||||
|
path
|
||||||
|
}
|
||||||
|
None => default_vault_kv2_mount_path(),
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(VaultConfig {
|
||||||
|
address: vault_address_from_env(overrides.address),
|
||||||
|
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||||
|
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||||
|
mount_path,
|
||||||
|
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||||
|
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||||
|
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Assemble the Vault Transit backend configuration from the environment.
|
||||||
|
///
|
||||||
|
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
|
||||||
|
/// points share it.
|
||||||
|
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
|
||||||
|
Ok(VaultTransitConfig {
|
||||||
|
address: vault_address_from_env(overrides.address),
|
||||||
|
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||||
|
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||||
|
mount_path: overrides
|
||||||
|
.mount_path
|
||||||
|
.map(str::to_string)
|
||||||
|
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
|
||||||
|
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
|
||||||
|
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
|
||||||
|
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||||
|
override_value
|
||||||
|
.map(str::to_string)
|
||||||
|
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolve the Vault auth method from environment variables.
|
/// Resolve the Vault auth method from environment variables.
|
||||||
///
|
///
|
||||||
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
||||||
@@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
|||||||
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
||||||
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
||||||
/// file taking precedence. Without a role id the legacy token flow applies.
|
/// file taking precedence. Without a role id the legacy token flow applies.
|
||||||
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
|
///
|
||||||
|
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
|
||||||
|
/// presents the pod's projected ServiceAccount token.
|
||||||
|
///
|
||||||
|
/// `token_override` carries a token supplied on the command line; it stands in
|
||||||
|
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
|
||||||
|
/// so a flag and the variable it mirrors select the same method.
|
||||||
|
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
|
||||||
|
let token = token_override
|
||||||
|
.map(str::to_string)
|
||||||
|
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
|
||||||
|
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
|
||||||
|
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
|
||||||
|
|
||||||
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
||||||
// A token file names one authoritative credential source; combining it
|
// A token file names one authoritative credential source; combining it
|
||||||
// with another one would leave the effective identity ambiguous, so
|
// with another one would leave the effective identity ambiguous, so
|
||||||
// that is a configuration error rather than a precedence rule.
|
// that is a configuration error rather than a precedence rule.
|
||||||
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
|
for (name, configured) in [
|
||||||
|
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
|
||||||
|
(ENV_KMS_VAULT_TOKEN, token.is_some()),
|
||||||
|
] {
|
||||||
|
if configured {
|
||||||
return Err(KmsError::configuration_error(format!(
|
return Err(KmsError::configuration_error(format!(
|
||||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
|
|
||||||
return Err(KmsError::configuration_error(format!(
|
|
||||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
|
|
||||||
)));
|
|
||||||
}
|
}
|
||||||
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
|
if let Some(role) = kubernetes_role {
|
||||||
|
// Unlike a leftover static token, a second login method is never a
|
||||||
|
// stale remnant: both were configured deliberately and neither can be
|
||||||
|
// ranked over the other.
|
||||||
|
if role_id.is_some() {
|
||||||
|
return Err(KmsError::configuration_error(format!(
|
||||||
|
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
return Ok(VaultAuthMethod::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
|
||||||
|
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
|
||||||
|
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(role_id) = role_id else {
|
||||||
return Ok(VaultAuthMethod::Token {
|
return Ok(VaultAuthMethod::Token {
|
||||||
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
|
token: token.unwrap_or_else(|| "dev-token".to_string()),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
|
|||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} => {
|
||||||
|
if role.is_empty() {
|
||||||
|
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
|
||||||
|
}
|
||||||
|
if mount.is_empty() {
|
||||||
|
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
|
||||||
|
}
|
||||||
|
if jwt_path.as_os_str().is_empty() {
|
||||||
|
return Err(KmsError::configuration_error(format!(
|
||||||
|
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
VaultAuthMethod::TokenFile {
|
VaultAuthMethod::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -1976,6 +2117,106 @@ mod tests {
|
|||||||
.expect("well-formed token file auth must validate");
|
.expect("well-formed token file auth must validate");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A Kubernetes role alone configures the method: the credential is the
|
||||||
|
/// pod's projected ServiceAccount token, so nothing secret is in the
|
||||||
|
/// environment and the mount and token path fall back to the cluster
|
||||||
|
/// defaults.
|
||||||
|
#[test]
|
||||||
|
fn test_from_env_selects_kubernetes() {
|
||||||
|
with_vars(
|
||||||
|
vec![
|
||||||
|
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||||
|
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
|
||||||
|
(ENV_KMS_VAULT_TOKEN, None),
|
||||||
|
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||||
|
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
let VaultAuthMethod::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount,
|
||||||
|
jwt_path,
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
} = &vault.auth_method
|
||||||
|
else {
|
||||||
|
panic!(
|
||||||
|
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||||
|
vault.auth_method
|
||||||
|
);
|
||||||
|
};
|
||||||
|
assert_eq!(role, "rustfs");
|
||||||
|
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||||
|
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||||
|
assert_eq!(refresh_safety_window_secs, &None);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
|
||||||
|
with_vars(
|
||||||
|
vec![
|
||||||
|
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||||
|
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
|
||||||
|
(ENV_KMS_VAULT_TOKEN, None),
|
||||||
|
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
|
||||||
|
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
|
||||||
|
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_rejects_bad_kubernetes_settings() {
|
||||||
|
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
|
||||||
|
backend: KmsBackend::VaultTransit,
|
||||||
|
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||||
|
address: "https://vault.example.com:8200".to_string(),
|
||||||
|
auth_method,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
|
||||||
|
.validate()
|
||||||
|
.expect_err("an empty kubernetes role must be rejected");
|
||||||
|
assert!(error.to_string().contains("role"), "got {error}");
|
||||||
|
|
||||||
|
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||||
|
role: "rustfs".to_string(),
|
||||||
|
mount: String::new(),
|
||||||
|
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
})
|
||||||
|
.validate()
|
||||||
|
.expect_err("an empty kubernetes mount must be rejected");
|
||||||
|
assert!(error.to_string().contains("mount"), "got {error}");
|
||||||
|
|
||||||
|
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||||
|
role: "rustfs".to_string(),
|
||||||
|
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||||
|
jwt_path: PathBuf::new(),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
})
|
||||||
|
.validate()
|
||||||
|
.expect_err("an empty ServiceAccount token path must be rejected");
|
||||||
|
assert!(error.to_string().contains("token path"), "got {error}");
|
||||||
|
|
||||||
|
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
|
||||||
|
.validate()
|
||||||
|
.expect("well-formed kubernetes auth must validate");
|
||||||
|
}
|
||||||
|
|
||||||
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
||||||
/// empty one names a path no Vault engine answers. The Transit backend
|
/// empty one names a path no Vault engine answers. The Transit backend
|
||||||
/// already rejects its own empty mounts; this closes the same gap on the
|
/// already rejects its own empty mounts; this closes the same gap on the
|
||||||
|
|||||||
@@ -18,8 +18,6 @@
|
|||||||
//! data encryption keys using master keys. It abstracts the encryption
|
//! data encryption keys using master keys. It abstracts the encryption
|
||||||
//! operations so that different backends can share the same encryption logic.
|
//! operations so that different backends can share the same encryption logic.
|
||||||
|
|
||||||
#![allow(dead_code)] // Trait methods may be used by implementations
|
|
||||||
|
|
||||||
use crate::error::{KmsError, Result};
|
use crate::error::{KmsError, Result};
|
||||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|||||||
@@ -225,8 +225,6 @@ async fn nothing_readable_leaves_the_bundle_unwrapped() {
|
|||||||
"artifact {} carries the raw on-disk record",
|
"artifact {} carries the raw on-disk record",
|
||||||
artifact.path
|
artifact.path
|
||||||
);
|
);
|
||||||
// A cheap structural check too: an encrypted payload is not JSON.
|
|
||||||
assert_ne!(payload.first(), Some(&b'{'), "artifact {} looks like plaintext JSON", artifact.path);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The manifest itself is not encrypted, so assert directly that it carries
|
// The manifest itself is not encrypted, so assert directly that it carries
|
||||||
|
|||||||
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
|
|||||||
pub api_version: Option<String>,
|
pub api_version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
pub struct SRIAMUser {
|
pub struct SRIAMUser {
|
||||||
#[serde(rename = "accessKey", default)]
|
#[serde(rename = "accessKey", default)]
|
||||||
pub access_key: String,
|
pub access_key: String,
|
||||||
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
|
|||||||
pub api_version: Option<String>,
|
pub api_version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
pub struct SRGroupInfo {
|
pub struct SRGroupInfo {
|
||||||
#[serde(rename = "updateReq", default)]
|
#[serde(rename = "updateReq", default)]
|
||||||
pub update_req: GroupAddRemove,
|
pub update_req: GroupAddRemove,
|
||||||
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
|
|||||||
pub api_version: Option<String>,
|
pub api_version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
pub struct SRIAMItem {
|
pub struct SRIAMItem {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub r#type: String,
|
pub r#type: String,
|
||||||
|
|||||||
@@ -40,7 +40,10 @@ impl RuleEvents for RuleView {
|
|||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct CompiledRules {
|
struct CompiledRules {
|
||||||
// Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase)
|
// Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase)
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "speculative retention: the comment above keeps it for richer snapshot-time judgements that no code performs yet (backlog#1823)"
|
||||||
|
)]
|
||||||
rules_map: RulesMap,
|
rules_map: RulesMap,
|
||||||
// for RulesContainer::iter_rules
|
// for RulesContainer::iter_rules
|
||||||
rule_views: Vec<RuleView>,
|
rule_views: Vec<RuleView>,
|
||||||
|
|||||||
@@ -187,7 +187,6 @@ impl RulesMap {
|
|||||||
/// # Parameters
|
/// # Parameters
|
||||||
/// * `event_name` - The EventName from which to remove the rule.
|
/// * `event_name` - The EventName from which to remove the rule.
|
||||||
/// * `pattern` - The pattern of the rule to be removed.
|
/// * `pattern` - The pattern of the rule to be removed.
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) {
|
pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) {
|
||||||
let mut remove_event = false;
|
let mut remove_event = false;
|
||||||
|
|
||||||
@@ -209,7 +208,6 @@ impl RulesMap {
|
|||||||
///
|
///
|
||||||
/// # Parameters
|
/// # Parameters
|
||||||
/// * `event_names` - A slice of EventNames to be removed.
|
/// * `event_names` - A slice of EventNames to be removed.
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn remove_rules(&mut self, event_names: &[EventName]) {
|
pub fn remove_rules(&mut self, event_names: &[EventName]) {
|
||||||
for event_name in event_names {
|
for event_name in event_names {
|
||||||
self.map.remove(event_name);
|
self.map.remove(event_name);
|
||||||
@@ -223,7 +221,6 @@ impl RulesMap {
|
|||||||
/// * `event_name` - The EventName to update.
|
/// * `event_name` - The EventName to update.
|
||||||
/// * `pattern` - The pattern of the rule to be updated.
|
/// * `pattern` - The pattern of the rule to be updated.
|
||||||
/// * `target_id` - The TargetID to be added.
|
/// * `target_id` - The TargetID to be added.
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) {
|
pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) {
|
||||||
self.map.entry(event_name).or_default().add(pattern, target_id);
|
self.map.entry(event_name).or_default().add(pattern, target_id);
|
||||||
self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask
|
self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask
|
||||||
|
|||||||
@@ -18,12 +18,6 @@ use rustfs_targets::arn::TargetID;
|
|||||||
/// TargetIDSet - A collection representation of TargetID.
|
/// TargetIDSet - A collection representation of TargetID.
|
||||||
pub type TargetIdSet = HashSet<TargetID>;
|
pub type TargetIdSet = HashSet<TargetID>;
|
||||||
|
|
||||||
/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed)
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub(crate) fn new_target_id_set(target_ids: Vec<TargetID>) -> TargetIdSet {
|
|
||||||
target_ids.into_iter().collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
// HashSet has built-in clone, union, difference and other operations.
|
// HashSet has built-in clone, union, difference and other operations.
|
||||||
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
|
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
|
||||||
// If you need to exactly match Go's API style, you can add wrapper functions.
|
// If you need to exactly match Go's API style, you can add wrapper functions.
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ use std::time::Duration;
|
|||||||
/// Environment variable key for the global default metrics interval (seconds).
|
/// Environment variable key for the global default metrics interval (seconds).
|
||||||
pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC";
|
pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC";
|
||||||
/// Default interval for metrics collection if not specified otherwise.
|
/// Default interval for metrics collection if not specified otherwise.
|
||||||
#[allow(dead_code)]
|
|
||||||
pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60);
|
pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
/// Environment variable key for cluster metrics interval (seconds).
|
/// Environment variable key for cluster metrics interval (seconds).
|
||||||
|
|||||||
@@ -145,21 +145,18 @@ impl PrometheusMetric {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self {
|
pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self {
|
||||||
self.labels.push((key, value.into()));
|
self.labels.push((key, value.into()));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self {
|
pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self {
|
||||||
self.labels.push((key, Cow::Owned(value)));
|
self.labels.push((key, Cow::Owned(value)));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self {
|
pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self {
|
||||||
self.labels = labels;
|
self.labels = labels;
|
||||||
self
|
self
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType};
|
|||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
|
||||||
/// MetricDescriptor - Metric descriptors
|
/// MetricDescriptor - Metric descriptors
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct MetricDescriptor {
|
pub struct MetricDescriptor {
|
||||||
pub name: MetricName,
|
pub name: MetricName,
|
||||||
@@ -52,7 +51,6 @@ impl MetricDescriptor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get the full metric name in Prometheus style: <namespace>_<subsystem>_<name>
|
/// Get the full metric name in Prometheus style: <namespace>_<subsystem>_<name>
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn get_full_metric_name(&self) -> String {
|
pub fn get_full_metric_name(&self) -> String {
|
||||||
let namespace = self.namespace.as_str();
|
let namespace = self.namespace.as_str();
|
||||||
let formatted_subsystem = self.subsystem.as_str();
|
let formatted_subsystem = self.subsystem.as_str();
|
||||||
@@ -61,7 +59,6 @@ impl MetricDescriptor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// check whether the label is in the label set
|
/// check whether the label is in the label set
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn has_label(&mut self, label: &str) -> bool {
|
pub fn has_label(&mut self, label: &str) -> bool {
|
||||||
self.get_label_set().contains(label)
|
self.get_label_set().contains(label)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
/// The metric name is the individual name of the metric
|
/// The metric name is the individual name of the metric
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum MetricName {
|
pub enum MetricName {
|
||||||
// The generic metric name
|
// The generic metric name
|
||||||
@@ -443,7 +442,6 @@ pub enum MetricName {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl MetricName {
|
impl MetricName {
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn as_str(&self) -> String {
|
pub fn as_str(&self) -> String {
|
||||||
match self {
|
match self {
|
||||||
Self::AuthTotal => "auth_total".to_string(),
|
Self::AuthTotal => "auth_total".to_string(),
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
/// MetricType - Indicates the type of indicator
|
/// MetricType - Indicates the type of indicator
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum MetricType {
|
pub enum MetricType {
|
||||||
Counter,
|
Counter,
|
||||||
@@ -23,7 +22,6 @@ pub enum MetricType {
|
|||||||
|
|
||||||
impl MetricType {
|
impl MetricType {
|
||||||
/// convert the metric type to a string representation
|
/// convert the metric type to a string representation
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn as_str(&self) -> &'static str {
|
pub fn as_str(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Counter => "counter",
|
Self::Counter => "counter",
|
||||||
@@ -34,7 +32,6 @@ impl MetricType {
|
|||||||
|
|
||||||
/// Convert the metric type to the Prometheus value type
|
/// Convert the metric type to the Prometheus value type
|
||||||
/// In a Rust implementation, this might return the corresponding Prometheus Rust client type
|
/// In a Rust implementation, this might return the corresponding Prometheus Rust client type
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn as_prom(&self) -> &'static str {
|
pub fn as_prom(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Counter => "counter.",
|
Self::Counter => "counter.",
|
||||||
|
|||||||
@@ -56,7 +56,6 @@ pub fn new_gauge_md(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// create a new histogram indicator descriptor
|
/// create a new histogram indicator descriptor
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn new_histogram_md(
|
pub fn new_histogram_md(
|
||||||
name: impl Into<MetricName>,
|
name: impl Into<MetricName>,
|
||||||
help: impl Into<String>,
|
help: impl Into<String>,
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ pub enum MetricNamespace {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl MetricNamespace {
|
impl MetricNamespace {
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn as_str(&self) -> &'static str {
|
pub fn as_str(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::RustFS => "rustfs",
|
Self::RustFS => "rustfs",
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
|
|
||||||
/// Format the path to the metric name format
|
/// Format the path to the metric name format
|
||||||
/// Replace '/' and '-' with '_'
|
/// Replace '/' and '-' with '_'
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn format_path_to_metric_name(path: &str) -> String {
|
pub fn format_path_to_metric_name(path: &str) -> String {
|
||||||
path.trim_start_matches('/').replace(['/', '-'], "_")
|
path.trim_start_matches('/').replace(['/', '-'], "_")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,7 +102,6 @@ impl MetricSubsystem {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get the formatted metric name format string
|
/// Get the formatted metric name format string
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn as_str(&self) -> String {
|
pub fn as_str(&self) -> String {
|
||||||
format_path_to_metric_name(self.path())
|
format_path_to_metric_name(self.path())
|
||||||
}
|
}
|
||||||
@@ -151,7 +150,6 @@ impl MetricSubsystem {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// A convenient way to create custom subsystems directly
|
/// A convenient way to create custom subsystems directly
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn new(path: impl Into<String>) -> Self {
|
pub fn new(path: impl Into<String>) -> Self {
|
||||||
Self::Custom(path.into())
|
Self::Custom(path.into())
|
||||||
}
|
}
|
||||||
@@ -176,7 +174,6 @@ impl std::fmt::Display for MetricSubsystem {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub mod subsystems {
|
pub mod subsystems {
|
||||||
use super::MetricSubsystem;
|
use super::MetricSubsystem;
|
||||||
|
|
||||||
|
|||||||
@@ -38,7 +38,10 @@ pub enum Rotation {
|
|||||||
Minutely,
|
Minutely,
|
||||||
Hourly,
|
Hourly,
|
||||||
Daily,
|
Daily,
|
||||||
#[allow(dead_code)]
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "constructed only by this file's rolling-appender tests; the lib target cannot see them (backlog#1823)"
|
||||||
|
)]
|
||||||
Never,
|
Never,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -219,10 +219,6 @@ impl PartialEq for Functions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Serialize, Deserialize)]
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct Value;
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::policy::Functions;
|
use crate::policy::Functions;
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user