mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
fix(tier): preserve credentials during backend validation (#7127)
* fix(tier): preserve credentials during backend validation * test(tier): make clone redaction assertion clippy-clean
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -18,25 +18,203 @@
|
|||||||
#![allow(unused_must_use)]
|
#![allow(unused_must_use)]
|
||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Default, Clone)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub struct TierServicePrincipalAuth {
|
||||||
|
#[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")]
|
||||||
|
pub tenant_id: String,
|
||||||
|
#[serde(rename = "ClientID", alias = "clientID", alias = "client_id")]
|
||||||
|
pub client_id: String,
|
||||||
|
#[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")]
|
||||||
|
pub client_secret: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TierServicePrincipalAuth {
|
||||||
|
pub(crate) fn is_empty(&self) -> bool {
|
||||||
|
self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for TierServicePrincipalAuth {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("TierServicePrincipalAuth")
|
||||||
|
.field("tenant_id", &self.tenant_id)
|
||||||
|
.field("client_id", &self.client_id)
|
||||||
|
.field("client_secret", &"REDACTED")
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Default, Clone)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub struct TierCreds {
|
pub struct TierCreds {
|
||||||
#[serde(rename = "accessKey")]
|
#[serde(rename = "access", alias = "accessKey")]
|
||||||
pub access_key: String,
|
pub access_key: String,
|
||||||
#[serde(rename = "secretKey")]
|
#[serde(rename = "secret", alias = "secretKey")]
|
||||||
pub secret_key: String,
|
pub secret_key: String,
|
||||||
|
|
||||||
#[serde(rename = "awsRole")]
|
#[serde(rename = "awsrole", alias = "awsRole")]
|
||||||
pub aws_role: bool,
|
pub aws_role: bool,
|
||||||
#[serde(rename = "awsRoleWebIdentityTokenFile")]
|
#[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")]
|
||||||
pub aws_role_web_identity_token_file: String,
|
pub aws_role_web_identity_token_file: String,
|
||||||
#[serde(rename = "awsRoleArn")]
|
#[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")]
|
||||||
pub aws_role_arn: String,
|
pub aws_role_arn: String,
|
||||||
|
|
||||||
//azsp: ServicePrincipalAuth,
|
#[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")]
|
||||||
|
pub azure_service_principal: TierServicePrincipalAuth,
|
||||||
|
|
||||||
//#[serde(rename = "credsJson")]
|
#[serde(
|
||||||
|
rename = "creds",
|
||||||
|
alias = "credsJson",
|
||||||
|
alias = "credsJSON",
|
||||||
|
alias = "creds_json",
|
||||||
|
default,
|
||||||
|
skip_serializing_if = "Vec::is_empty",
|
||||||
|
with = "base64_bytes"
|
||||||
|
)]
|
||||||
pub creds_json: Vec<u8>,
|
pub creds_json: Vec<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for TierCreds {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("TierCreds")
|
||||||
|
.field("access_key", &self.access_key)
|
||||||
|
.field("secret_key", &"REDACTED")
|
||||||
|
.field("aws_role", &self.aws_role)
|
||||||
|
.field(
|
||||||
|
"aws_role_web_identity_token_file",
|
||||||
|
&(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"),
|
||||||
|
)
|
||||||
|
.field("aws_role_arn", &self.aws_role_arn)
|
||||||
|
.field("azure_service_principal", &self.azure_service_principal)
|
||||||
|
.field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED"))
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mod base64_bytes {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) fn serialize<S>(value: &[u8], serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: Serializer,
|
||||||
|
{
|
||||||
|
serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum EncodedBytes {
|
||||||
|
Base64(String),
|
||||||
|
Legacy(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
match EncodedBytes::deserialize(deserializer)? {
|
||||||
|
EncodedBytes::Base64(value) => base64_simd::STANDARD
|
||||||
|
.decode_to_vec(value.as_bytes())
|
||||||
|
.or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes()))
|
||||||
|
.or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes()))
|
||||||
|
.or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes()))
|
||||||
|
.map_err(de::Error::custom),
|
||||||
|
EncodedBytes::Legacy(value) => Ok(value),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() {
|
||||||
|
let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes();
|
||||||
|
let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ==";
|
||||||
|
let creds: TierCreds = serde_json::from_value(serde_json::json!({
|
||||||
|
"access": "access",
|
||||||
|
"secret": "secret",
|
||||||
|
"awsrole": false,
|
||||||
|
"creds": encoded,
|
||||||
|
}))
|
||||||
|
.expect("madmin tier credentials should decode");
|
||||||
|
|
||||||
|
assert_eq!(creds.access_key, "access");
|
||||||
|
assert_eq!(creds.secret_key, "secret");
|
||||||
|
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
|
||||||
|
|
||||||
|
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
|
||||||
|
assert_eq!(wire["access"], "access");
|
||||||
|
assert_eq!(wire["secret"], "secret");
|
||||||
|
assert_eq!(wire["creds"], encoded);
|
||||||
|
assert!(wire.get("accessKey").is_none());
|
||||||
|
assert!(wire.get("secretKey").is_none());
|
||||||
|
|
||||||
|
let legacy: TierCreds = serde_json::from_value(serde_json::json!({
|
||||||
|
"accessKey": "legacy-access",
|
||||||
|
"secretKey": "legacy-secret",
|
||||||
|
"credsJson": service_account,
|
||||||
|
}))
|
||||||
|
.expect("the former RustFS field names and byte-array encoding should remain readable");
|
||||||
|
assert_eq!(legacy.access_key, "legacy-access");
|
||||||
|
assert_eq!(legacy.secret_key, "legacy-secret");
|
||||||
|
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() {
|
||||||
|
let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes();
|
||||||
|
for encoder in [
|
||||||
|
base64_simd::STANDARD,
|
||||||
|
base64_simd::STANDARD_NO_PAD,
|
||||||
|
base64_simd::URL_SAFE,
|
||||||
|
base64_simd::URL_SAFE_NO_PAD,
|
||||||
|
] {
|
||||||
|
let encoded = encoder.encode_to_string(service_account);
|
||||||
|
let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded }))
|
||||||
|
.expect("all supported madmin base64 forms should decode");
|
||||||
|
assert_eq!(creds.creds_json, service_account);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tier_creds_debug_redacts_secret_payloads() {
|
||||||
|
let creds = TierCreds {
|
||||||
|
access_key: "access".to_string(),
|
||||||
|
secret_key: "tier-secret-value".to_string(),
|
||||||
|
aws_role_web_identity_token_file: "/var/run/private-token".to_string(),
|
||||||
|
creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let rendered = format!("{creds:?}");
|
||||||
|
assert!(!rendered.contains("tier-secret-value"));
|
||||||
|
assert!(!rendered.contains("/var/run/private-token"));
|
||||||
|
assert!(!rendered.contains("gcs-private-key-value"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() {
|
||||||
|
let creds: TierCreds = serde_json::from_value(serde_json::json!({
|
||||||
|
"azSP": {
|
||||||
|
"TenantID": "tenant",
|
||||||
|
"ClientID": "client",
|
||||||
|
"ClientSecret": "service-principal-secret"
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("canonical madmin azure service principal credentials should decode");
|
||||||
|
|
||||||
|
assert_eq!(creds.azure_service_principal.tenant_id, "tenant");
|
||||||
|
assert_eq!(creds.azure_service_principal.client_id, "client");
|
||||||
|
assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret");
|
||||||
|
let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode");
|
||||||
|
assert_eq!(wire["azSP"]["TenantID"], "tenant");
|
||||||
|
assert_eq!(wire["azSP"]["ClientID"], "client");
|
||||||
|
assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret");
|
||||||
|
assert!(!format!("{creds:?}").contains("service-principal-secret"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
|||||||
pub enum TierType {
|
pub enum TierType {
|
||||||
#[default]
|
#[default]
|
||||||
Unsupported,
|
Unsupported,
|
||||||
#[serde(rename = "s3")]
|
#[serde(rename = "s3", alias = "S3")]
|
||||||
S3,
|
S3,
|
||||||
#[serde(rename = "wasabi")]
|
#[serde(rename = "wasabi")]
|
||||||
Wasabi,
|
Wasabi,
|
||||||
@@ -58,7 +58,7 @@ pub enum TierType {
|
|||||||
Huaweicloud,
|
Huaweicloud,
|
||||||
#[serde(rename = "azure")]
|
#[serde(rename = "azure")]
|
||||||
Azure,
|
Azure,
|
||||||
#[serde(rename = "gcs")]
|
#[serde(rename = "gcs", alias = "GCS")]
|
||||||
GCS,
|
GCS,
|
||||||
#[serde(rename = "r2")]
|
#[serde(rename = "r2")]
|
||||||
R2,
|
R2,
|
||||||
@@ -138,16 +138,18 @@ impl TierType {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default, Debug, Serialize, Deserialize)]
|
pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
|
||||||
|
|
||||||
|
#[derive(Default, Serialize, Deserialize)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub struct TierConfig {
|
pub struct TierConfig {
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub version: String,
|
pub version: String,
|
||||||
#[serde(rename = "type")]
|
#[serde(rename = "type", alias = "Type")]
|
||||||
pub tier_type: TierType,
|
pub tier_type: TierType,
|
||||||
#[serde(skip)]
|
#[serde(rename = "Name", alias = "name", skip_serializing)]
|
||||||
pub name: String,
|
pub name: String,
|
||||||
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")]
|
||||||
pub s3: Option<TierS3>,
|
pub s3: Option<TierS3>,
|
||||||
#[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
|
||||||
pub wasabi: Option<TierWasabi>,
|
pub wasabi: Option<TierWasabi>,
|
||||||
@@ -159,7 +161,7 @@ pub struct TierConfig {
|
|||||||
pub huaweicloud: Option<TierHuaweicloud>,
|
pub huaweicloud: Option<TierHuaweicloud>,
|
||||||
#[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
|
||||||
pub azure: Option<TierAzure>,
|
pub azure: Option<TierAzure>,
|
||||||
#[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")]
|
||||||
pub gcs: Option<TierGCS>,
|
pub gcs: Option<TierGCS>,
|
||||||
#[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
|
||||||
pub r2: Option<TierR2>,
|
pub r2: Option<TierR2>,
|
||||||
@@ -170,109 +172,91 @@ pub struct TierConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Clone for TierConfig {
|
impl Clone for TierConfig {
|
||||||
fn clone(&self) -> TierConfig {
|
fn clone(&self) -> Self {
|
||||||
let mut s3 = None;
|
self.redacted()
|
||||||
let mut wasabi = None;
|
|
||||||
let mut r = None;
|
|
||||||
let mut compatible_backend = None;
|
|
||||||
let mut aliyun = None;
|
|
||||||
let mut tencent = None;
|
|
||||||
let mut huaweicloud = None;
|
|
||||||
let mut azure = None;
|
|
||||||
let mut gcs = None;
|
|
||||||
let mut r2 = None;
|
|
||||||
match self.tier_type {
|
|
||||||
TierType::S3 => {
|
|
||||||
if let Some(s3_) = self.s3.as_ref() {
|
|
||||||
let mut s3_clone = s3_.clone();
|
|
||||||
s3_clone.secret_key = "REDACTED".to_string();
|
|
||||||
s3 = Some(s3_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::Wasabi => {
|
|
||||||
if let Some(wasabi_) = self.wasabi.as_ref() {
|
|
||||||
let mut wasabi_clone = wasabi_.clone();
|
|
||||||
wasabi_clone.secret_key = "REDACTED".to_string();
|
|
||||||
wasabi = Some(wasabi_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::RustFS => {
|
|
||||||
if let Some(r_) = self.rustfs.as_ref() {
|
|
||||||
let mut r_clone = r_.clone();
|
|
||||||
r_clone.secret_key = "REDACTED".to_string();
|
|
||||||
r = Some(r_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::MinIO => {
|
|
||||||
if let Some(compatible_backend_) = self.minio.as_ref() {
|
|
||||||
let mut compatible_backend_clone = compatible_backend_.clone();
|
|
||||||
compatible_backend_clone.secret_key = "REDACTED".to_string();
|
|
||||||
compatible_backend = Some(compatible_backend_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::Aliyun => {
|
|
||||||
if let Some(aliyun_) = self.aliyun.as_ref() {
|
|
||||||
let mut aliyun_clone = aliyun_.clone();
|
|
||||||
aliyun_clone.secret_key = "REDACTED".to_string();
|
|
||||||
aliyun = Some(aliyun_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::Tencent => {
|
|
||||||
if let Some(tencent_) = self.tencent.as_ref() {
|
|
||||||
let mut tencent_clone = tencent_.clone();
|
|
||||||
tencent_clone.secret_key = "REDACTED".to_string();
|
|
||||||
tencent = Some(tencent_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::Huaweicloud => {
|
|
||||||
if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
|
|
||||||
let mut huaweicloud_clone = huaweicloud_.clone();
|
|
||||||
huaweicloud_clone.secret_key = "REDACTED".to_string();
|
|
||||||
huaweicloud = Some(huaweicloud_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::Azure => {
|
|
||||||
if let Some(azure_) = self.azure.as_ref() {
|
|
||||||
let mut azure_clone = azure_.clone();
|
|
||||||
azure_clone.secret_key = "REDACTED".to_string();
|
|
||||||
azure = Some(azure_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::GCS => {
|
|
||||||
if let Some(gcs_) = self.gcs.as_ref() {
|
|
||||||
let mut gcs_clone = gcs_.clone();
|
|
||||||
gcs_clone.creds = "REDACTED".to_string();
|
|
||||||
gcs = Some(gcs_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TierType::R2 => {
|
|
||||||
if let Some(r2_) = self.r2.as_ref() {
|
|
||||||
let mut r2_clone = r2_.clone();
|
|
||||||
r2_clone.secret_key = "REDACTED".to_string();
|
|
||||||
r2 = Some(r2_clone);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => (),
|
|
||||||
}
|
|
||||||
TierConfig {
|
|
||||||
version: self.version.clone(),
|
|
||||||
tier_type: self.tier_type.clone(),
|
|
||||||
name: self.name.clone(),
|
|
||||||
s3,
|
|
||||||
wasabi,
|
|
||||||
rustfs: r,
|
|
||||||
minio: compatible_backend,
|
|
||||||
aliyun,
|
|
||||||
tencent,
|
|
||||||
huaweicloud,
|
|
||||||
azure,
|
|
||||||
gcs,
|
|
||||||
r2,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TierConfig {
|
impl TierConfig {
|
||||||
|
pub(crate) fn redacted(&self) -> Self {
|
||||||
|
let mut redacted = Self {
|
||||||
|
version: self.version.clone(),
|
||||||
|
tier_type: self.tier_type.clone(),
|
||||||
|
name: self.name.clone(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
match self.tier_type {
|
||||||
|
TierType::S3 => {
|
||||||
|
redacted.s3 = self.s3.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
if !backend.aws_role_web_identity_token_file.is_empty() {
|
||||||
|
backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
}
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::Wasabi => {
|
||||||
|
redacted.wasabi = self.wasabi.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::RustFS => {
|
||||||
|
redacted.rustfs = self.rustfs.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::MinIO => {
|
||||||
|
redacted.minio = self.minio.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::Aliyun => {
|
||||||
|
redacted.aliyun = self.aliyun.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::Tencent => {
|
||||||
|
redacted.tencent = self.tencent.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::Huaweicloud => {
|
||||||
|
redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::Azure => {
|
||||||
|
redacted.azure = self.azure.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
if !backend.sp_auth.client_secret.is_empty() {
|
||||||
|
backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
}
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::GCS => {
|
||||||
|
redacted.gcs = self.gcs.clone().map(|mut backend| {
|
||||||
|
backend.creds = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::R2 => {
|
||||||
|
redacted.r2 = self.r2.clone().map(|mut backend| {
|
||||||
|
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
|
||||||
|
backend
|
||||||
|
});
|
||||||
|
}
|
||||||
|
TierType::Unsupported => {}
|
||||||
|
}
|
||||||
|
redacted
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn clone_with_credentials(&self) -> Self {
|
pub(crate) fn clone_with_credentials(&self) -> Self {
|
||||||
Self {
|
Self {
|
||||||
version: self.version.clone(),
|
version: self.version.clone(),
|
||||||
@@ -372,31 +356,61 @@ impl TierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for TierConfig {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let redacted = self.redacted();
|
||||||
|
f.debug_struct("TierConfig")
|
||||||
|
.field("version", &redacted.version)
|
||||||
|
.field("tier_type", &redacted.tier_type)
|
||||||
|
.field("name", &redacted.name)
|
||||||
|
.field("s3", &redacted.s3)
|
||||||
|
.field("wasabi", &redacted.wasabi)
|
||||||
|
.field("aliyun", &redacted.aliyun)
|
||||||
|
.field("tencent", &redacted.tencent)
|
||||||
|
.field("huaweicloud", &redacted.huaweicloud)
|
||||||
|
.field("azure", &redacted.azure)
|
||||||
|
.field("gcs", &redacted.gcs)
|
||||||
|
.field("r2", &redacted.r2)
|
||||||
|
.field("rustfs", &redacted.rustfs)
|
||||||
|
.field("minio", &redacted.minio)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
|
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub struct TierS3 {
|
pub struct TierS3 {
|
||||||
|
#[serde(alias = "Name")]
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
#[serde(alias = "Endpoint")]
|
||||||
pub endpoint: String,
|
pub endpoint: String,
|
||||||
#[serde(rename = "accessKey")]
|
#[serde(rename = "accessKey", alias = "AccessKey")]
|
||||||
pub access_key: String,
|
pub access_key: String,
|
||||||
#[serde(rename = "secretKey")]
|
#[serde(rename = "secretKey", alias = "SecretKey")]
|
||||||
pub secret_key: String,
|
pub secret_key: String,
|
||||||
|
#[serde(alias = "Bucket")]
|
||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
|
#[serde(alias = "Prefix")]
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
|
#[serde(alias = "Region")]
|
||||||
pub region: String,
|
pub region: String,
|
||||||
#[serde(rename = "storageClass")]
|
#[serde(rename = "storageClass", alias = "StorageClass")]
|
||||||
pub storage_class: String,
|
pub storage_class: String,
|
||||||
#[serde(skip)]
|
#[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)]
|
||||||
pub aws_role: bool,
|
pub aws_role: bool,
|
||||||
#[serde(skip)]
|
#[serde(
|
||||||
|
rename = "AWSRoleWebIdentityTokenFile",
|
||||||
|
alias = "awsRoleWebIdentityTokenFile",
|
||||||
|
skip_serializing
|
||||||
|
)]
|
||||||
pub aws_role_web_identity_token_file: String,
|
pub aws_role_web_identity_token_file: String,
|
||||||
#[serde(skip)]
|
#[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)]
|
||||||
pub aws_role_arn: String,
|
pub aws_role_arn: String,
|
||||||
#[serde(skip)]
|
#[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)]
|
||||||
pub aws_role_session_name: String,
|
pub aws_role_session_name: String,
|
||||||
#[serde(skip)]
|
#[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)]
|
||||||
pub aws_role_duration_seconds: i32,
|
pub aws_role_duration_seconds: i32,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -623,8 +637,11 @@ pub struct TierHuaweicloud {
|
|||||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub struct ServicePrincipalAuth {
|
pub struct ServicePrincipalAuth {
|
||||||
|
#[serde(alias = "TenantID")]
|
||||||
pub tenant_id: String,
|
pub tenant_id: String,
|
||||||
|
#[serde(alias = "ClientID")]
|
||||||
pub client_id: String,
|
pub client_id: String,
|
||||||
|
#[serde(alias = "ClientSecret")]
|
||||||
pub client_secret: String,
|
pub client_secret: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -640,9 +657,9 @@ pub struct TierAzure {
|
|||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
pub region: String,
|
pub region: String,
|
||||||
#[serde(rename = "storageClass")]
|
#[serde(rename = "storageClass", alias = "StorageClass")]
|
||||||
pub storage_class: String,
|
pub storage_class: String,
|
||||||
#[serde(rename = "spAuth")]
|
#[serde(rename = "spAuth", alias = "SPAuth")]
|
||||||
pub sp_auth: ServicePrincipalAuth,
|
pub sp_auth: ServicePrincipalAuth,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error {
|
|||||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub struct TierGCS {
|
pub struct TierGCS {
|
||||||
|
#[serde(alias = "Name")]
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
#[serde(alias = "Endpoint")]
|
||||||
pub endpoint: String,
|
pub endpoint: String,
|
||||||
#[serde(rename = "creds")]
|
#[serde(rename = "creds", alias = "Creds")]
|
||||||
pub creds: String,
|
pub creds: String,
|
||||||
|
#[serde(alias = "Bucket")]
|
||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
|
#[serde(alias = "Prefix")]
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
|
#[serde(alias = "Region")]
|
||||||
pub region: String,
|
pub region: String,
|
||||||
#[serde(rename = "storageClass")]
|
#[serde(rename = "storageClass", alias = "StorageClass")]
|
||||||
pub storage_class: String,
|
pub storage_class: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -725,6 +747,43 @@ pub struct TierR2 {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() {
|
||||||
|
let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode");
|
||||||
|
let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode");
|
||||||
|
assert!(matches!(s3, TierType::S3));
|
||||||
|
assert!(matches!(gcs, TierType::GCS));
|
||||||
|
assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#);
|
||||||
|
assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn azure_service_principal_accepts_canonical_madmin_field_names() {
|
||||||
|
for field in ["TenantID", "ClientID", "ClientSecret"] {
|
||||||
|
let mut sp_auth = serde_json::Map::new();
|
||||||
|
sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string()));
|
||||||
|
let config: TierConfig = serde_json::from_value(serde_json::json!({
|
||||||
|
"type": "azure",
|
||||||
|
"Name": "COLD-AZURE",
|
||||||
|
"azure": {
|
||||||
|
"name": "COLD-AZURE",
|
||||||
|
"endpoint": "https://azure.example.invalid",
|
||||||
|
"accessKey": "account",
|
||||||
|
"secretKey": "key",
|
||||||
|
"bucket": "archive",
|
||||||
|
"SPAuth": sp_auth
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("mixed RustFS/madmin Azure payload should decode");
|
||||||
|
let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth;
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
!sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(),
|
||||||
|
"canonical {field} must not be silently discarded"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn wasabi_config() -> TierWasabi {
|
fn wasabi_config() -> TierWasabi {
|
||||||
TierWasabi {
|
TierWasabi {
|
||||||
name: "COLD-WASABI".to_string(),
|
name: "COLD-WASABI".to_string(),
|
||||||
@@ -838,9 +897,14 @@ mod tests {
|
|||||||
let config = TierConfig {
|
let config = TierConfig {
|
||||||
tier_type: TierType::Wasabi,
|
tier_type: TierType::Wasabi,
|
||||||
wasabi: Some(wasabi_config()),
|
wasabi: Some(wasabi_config()),
|
||||||
|
rustfs: Some(TierRustFS {
|
||||||
|
access_key: "inactive-access".to_string(),
|
||||||
|
secret_key: "inactive-secret".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let redacted = config.clone();
|
let redacted = config.redacted();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
redacted
|
redacted
|
||||||
.wasabi
|
.wasabi
|
||||||
@@ -849,21 +913,46 @@ mod tests {
|
|||||||
.secret_key,
|
.secret_key,
|
||||||
"REDACTED"
|
"REDACTED"
|
||||||
);
|
);
|
||||||
|
assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider");
|
||||||
|
let cloned = config.clone();
|
||||||
|
assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED");
|
||||||
|
assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics");
|
||||||
|
let preserved = config.clone_with_credentials();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
config
|
preserved
|
||||||
.clone_with_credentials()
|
|
||||||
.wasabi
|
.wasabi
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.expect("credential-bearing Wasabi payload should remain")
|
.expect("credential-bearing Wasabi snapshot should remain")
|
||||||
.secret_key,
|
.secret_key,
|
||||||
"secret"
|
"secret"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
preserved
|
||||||
|
.rustfs
|
||||||
|
.expect("credential-bearing snapshots should preserve inactive provider data")
|
||||||
|
.secret_key,
|
||||||
|
"inactive-secret"
|
||||||
|
);
|
||||||
|
|
||||||
let mut debug_config = wasabi_config();
|
let mut debug_config = wasabi_config();
|
||||||
debug_config.secret_key = "wasabi-debug-secret-value".to_string();
|
debug_config.secret_key = "wasabi-debug-secret-value".to_string();
|
||||||
let debug = format!("{debug_config:?}");
|
let debug = format!("{debug_config:?}");
|
||||||
assert!(debug.contains("REDACTED"));
|
assert!(debug.contains("REDACTED"));
|
||||||
assert!(!debug.contains("wasabi-debug-secret-value"));
|
assert!(!debug.contains("wasabi-debug-secret-value"));
|
||||||
|
|
||||||
|
let debug = format!(
|
||||||
|
"{:?}",
|
||||||
|
TierConfig {
|
||||||
|
tier_type: TierType::RustFS,
|
||||||
|
rustfs: Some(TierRustFS {
|
||||||
|
secret_key: "rustfs-debug-secret-value".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(debug.contains("REDACTED"));
|
||||||
|
assert!(!debug.contains("rustfs-debug-secret-value"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -894,7 +983,7 @@ mod tests {
|
|||||||
assert_eq!(encoded, expected);
|
assert_eq!(encoded, expected);
|
||||||
let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
|
let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
|
||||||
assert!(matches!(decoded.tier_type, TierType::Wasabi));
|
assert!(matches!(decoded.tier_type, TierType::Wasabi));
|
||||||
let redacted = config.clone();
|
let redacted = config.redacted();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
config
|
config
|
||||||
.wasabi
|
.wasabi
|
||||||
@@ -916,4 +1005,87 @@ mod tests {
|
|||||||
"REDACTED"
|
"REDACTED"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() {
|
||||||
|
let cases = [
|
||||||
|
(
|
||||||
|
"s3",
|
||||||
|
TierConfig {
|
||||||
|
tier_type: TierType::S3,
|
||||||
|
s3: Some(TierS3 {
|
||||||
|
secret_key: "s3-secret-bytes".to_string(),
|
||||||
|
aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
vec!["s3-secret-bytes", "/var/run/s3-private-token"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"gcs",
|
||||||
|
TierConfig {
|
||||||
|
tier_type: TierType::GCS,
|
||||||
|
gcs: Some(TierGCS {
|
||||||
|
creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
vec!["gcs-private-key-bytes"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"azure",
|
||||||
|
TierConfig {
|
||||||
|
tier_type: TierType::Azure,
|
||||||
|
azure: Some(TierAzure {
|
||||||
|
secret_key: "azure-account-secret-bytes".to_string(),
|
||||||
|
sp_auth: ServicePrincipalAuth {
|
||||||
|
client_secret: "azure-client-secret-bytes".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
vec!["azure-account-secret-bytes", "azure-client-secret-bytes"],
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (provider, config, secrets) in cases {
|
||||||
|
let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize");
|
||||||
|
let debug = format!("{config:?}");
|
||||||
|
assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted");
|
||||||
|
assert!(
|
||||||
|
debug.contains(TIER_CREDENTIAL_REDACTED),
|
||||||
|
"{provider} Debug output should be visibly redacted"
|
||||||
|
);
|
||||||
|
for secret in secrets {
|
||||||
|
assert!(!api.contains(secret), "{provider} API output exposed credential bytes");
|
||||||
|
assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() {
|
||||||
|
let config = TierConfig {
|
||||||
|
tier_type: TierType::Azure,
|
||||||
|
azure: Some(TierAzure {
|
||||||
|
secret_key: "azure-account-secret-bytes".to_string(),
|
||||||
|
sp_auth: ServicePrincipalAuth::default(),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize");
|
||||||
|
let debug = format!("{config:?}");
|
||||||
|
|
||||||
|
assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED);
|
||||||
|
assert_eq!(api["azure"]["spAuth"]["client_secret"], "");
|
||||||
|
assert!(debug.contains("client_secret: \"\""));
|
||||||
|
assert!(!debug.contains("client_secret: \"REDACTED\""));
|
||||||
|
assert!(!debug.contains("azure-account-secret-bytes"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,7 @@
|
|||||||
|
|
||||||
use crate::error::is_err_bucket_not_found;
|
use crate::error::is_err_bucket_not_found;
|
||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
|
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
|
||||||
tier_config::{TierConfig, TierType},
|
tier_config::{TierConfig, TierType},
|
||||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||||
warm_backend_aliyun::WarmBackendAliyun,
|
warm_backend_aliyun::WarmBackendAliyun,
|
||||||
@@ -55,18 +55,21 @@ use s3s::header::{
|
|||||||
};
|
};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use time::format_description::well_known::{Rfc2822, Rfc3339};
|
use time::format_description::well_known::{Rfc2822, Rfc3339};
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
|
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
|
||||||
|
|
||||||
const PROBE_OBJECT: &str = "probeobject";
|
|
||||||
|
|
||||||
/// Largest object the S3-compatible warm backends accept for a multipart put.
|
/// Largest object the S3-compatible warm backends accept for a multipart put.
|
||||||
pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||||
/// Part-count ceiling S3-compatible services impose on a multipart upload.
|
/// Part-count ceiling S3-compatible services impose on a multipart upload.
|
||||||
pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
|
pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
|
||||||
|
pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1);
|
||||||
|
const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub struct WarmBackendGetOpts {
|
pub struct WarmBackendGetOpts {
|
||||||
@@ -260,6 +263,23 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
|
|||||||
pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
|
pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return the authority format accepted by `TransitionClient::new` while
|
||||||
|
/// retaining an explicitly configured port. `url::Url::host_str()` omits the
|
||||||
|
/// brackets needed when an IPv6 literal is combined with a port.
|
||||||
|
pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Error> {
|
||||||
|
let host = url
|
||||||
|
.host_str()
|
||||||
|
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||||
|
let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
|
||||||
|
if host.starts_with('[') && host.ends_with(']') {
|
||||||
|
Ok(format!("{host}:{port}"))
|
||||||
|
} else if host.contains(':') {
|
||||||
|
Ok(format!("[{host}]:{port}"))
|
||||||
|
} else {
|
||||||
|
Ok(format!("{host}:{port}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
|
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
|
||||||
///
|
///
|
||||||
/// Credential, bucket, and endpoint validation run in this order because the
|
/// Credential, bucket, and endpoint validation run in this order because the
|
||||||
@@ -298,17 +318,11 @@ pub(crate) async fn new_s3_compatible_warm_backend(
|
|||||||
bucket_lookup: params.bucket_lookup,
|
bucket_lookup: params.bucket_lookup,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let scheme = u.scheme();
|
let endpoint = endpoint_authority(&u)?;
|
||||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
// Run the SSRF guard after the host-presence check so a host-less endpoint
|
||||||
let host = u
|
// keeps this constructor's stable error text.
|
||||||
.host_str()
|
|
||||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
|
||||||
// Runs after the host-presence check above (not immediately after Url::parse) so a
|
|
||||||
// host-less endpoint still reports this constructor's own "missing host" text instead of
|
|
||||||
// validate_endpoint's differently-worded rejection for the same input.
|
|
||||||
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||||
let client =
|
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
|
||||||
TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?;
|
|
||||||
|
|
||||||
let client = Arc::new(client);
|
let client = Arc::new(client);
|
||||||
let core = TransitionCore(Arc::clone(&client));
|
let core = TransitionCore(Arc::clone(&client));
|
||||||
@@ -451,25 +465,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
|
async fn remove_discovered_probe_candidate(
|
||||||
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
|
w: &WarmBackendImpl,
|
||||||
w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
|
probe_object: &str,
|
||||||
let remote_version_id = w
|
candidate: TransitionCandidateProbe,
|
||||||
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
|
) -> Result<bool, std::io::Error> {
|
||||||
.await
|
match candidate {
|
||||||
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
|
TransitionCandidateProbe::Missing => Ok(false),
|
||||||
|
TransitionCandidateProbe::VersionedPresent(remote_version_id) => {
|
||||||
|
w.remove_exact(probe_object, &remote_version_id).await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
TransitionCandidateProbe::UnversionedPresent => {
|
||||||
|
w.remove(probe_object, "").await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
TransitionCandidateProbe::Ambiguous => {
|
||||||
|
Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions"))
|
||||||
|
}
|
||||||
|
TransitionCandidateProbe::Unsupported => Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::Unsupported,
|
||||||
|
"remote tier cannot discover the outcome of a probe PUT",
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if w.validate_remote_version_id(&remote_version_id).is_err() {
|
async fn compensate_uncertain_probe_put(
|
||||||
w.remove_exact(PROBE_OBJECT, &remote_version_id)
|
w: &WarmBackendImpl,
|
||||||
|
probe_object: &str,
|
||||||
|
settle_deadline: tokio::time::Instant,
|
||||||
|
) -> Result<(), std::io::Error> {
|
||||||
|
let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT;
|
||||||
|
let mut removed_any = false;
|
||||||
|
while tokio::time::Instant::now() < settle_deadline {
|
||||||
|
let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await {
|
||||||
|
Ok(candidate) => candidate?,
|
||||||
|
Err(_) => break,
|
||||||
|
};
|
||||||
|
if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
|
||||||
.await
|
.await
|
||||||
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
|
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??;
|
||||||
return Err(ERR_TIER_INVALID_CONFIG.clone());
|
|
||||||
|
let now = tokio::time::Instant::now();
|
||||||
|
if now >= settle_deadline {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
|
let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
|
||||||
let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
|
.await
|
||||||
//xhttp.DrainBody(r);
|
.map_err(|_| {
|
||||||
if read_result.is_err() || remove_result.is_err() {
|
std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state")
|
||||||
|
})??;
|
||||||
|
if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate")
|
||||||
|
})??
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
|
||||||
|
.await
|
||||||
|
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??;
|
||||||
|
match final_candidate {
|
||||||
|
TransitionCandidateProbe::Missing => Ok(()),
|
||||||
|
_ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn probe_cleanup_incomplete_error() -> AdminError {
|
||||||
|
let mut err = ERR_TIER_PERM_ERR.clone();
|
||||||
|
err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string();
|
||||||
|
err
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_warm_backend_with_deadlines(
|
||||||
|
w: Option<&WarmBackendImpl>,
|
||||||
|
deadline: tokio::time::Instant,
|
||||||
|
cleanup_deadline: tokio::time::Instant,
|
||||||
|
) -> Result<(), AdminError> {
|
||||||
|
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
|
||||||
|
let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4());
|
||||||
|
let timeout_error = || {
|
||||||
|
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||||
|
err.message = "Timed out validating the remote tier mutation".to_string();
|
||||||
|
err
|
||||||
|
};
|
||||||
|
tokio::time::timeout_at(deadline, w.validate())
|
||||||
|
.await
|
||||||
|
.map_err(|_| timeout_error())?
|
||||||
|
.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
|
||||||
|
let put_result =
|
||||||
|
tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await;
|
||||||
|
let remote_version_id = match put_result {
|
||||||
|
Ok(Ok(remote_version_id)) => remote_version_id,
|
||||||
|
Ok(Err(_)) => {
|
||||||
|
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
|
||||||
|
Ok(()) => ERR_TIER_PERM_ERR.clone(),
|
||||||
|
Err(_) => probe_cleanup_incomplete_error(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
let err = timeout_error();
|
||||||
|
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
|
||||||
|
Ok(()) => err,
|
||||||
|
Err(_) => probe_cleanup_incomplete_error(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// S3-family backends do not replay a failed request before returning `Ok`,
|
||||||
|
// while GCS discovers every matching generation. The authoritative probe
|
||||||
|
// below therefore closes the acknowledged-PUT path; only an error or
|
||||||
|
// timeout needs the longer visibility reconciliation above.
|
||||||
|
let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await {
|
||||||
|
Ok(Ok(candidate)) => candidate,
|
||||||
|
Ok(Err(_)) | Err(_) => {
|
||||||
|
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
|
||||||
|
Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
|
||||||
|
Err(_) => probe_cleanup_incomplete_error(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok();
|
||||||
|
let response_matches_candidate = match &authoritative_candidate {
|
||||||
|
TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(),
|
||||||
|
TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id,
|
||||||
|
TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false,
|
||||||
|
};
|
||||||
|
if !response_version_is_valid || !response_matches_candidate {
|
||||||
|
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
|
||||||
|
Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
|
||||||
|
Err(_) => probe_cleanup_incomplete_error(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let read_result = tokio::time::timeout_at(deadline, async {
|
||||||
|
let mut reader = w
|
||||||
|
.get(
|
||||||
|
&probe_object,
|
||||||
|
&remote_version_id,
|
||||||
|
WarmBackendGetOpts {
|
||||||
|
start_offset: 0,
|
||||||
|
length: 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
|
||||||
|
let mut body = Vec::new();
|
||||||
|
reader
|
||||||
|
.take(7)
|
||||||
|
.read_to_end(&mut body)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
|
||||||
|
if body != b"RustFS" {
|
||||||
|
return Err(ERR_TIER_PERM_ERR.clone());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| timeout_error())
|
||||||
|
.and_then(|result| result);
|
||||||
|
let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async {
|
||||||
|
if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? {
|
||||||
|
return Err(std::io::Error::other("remote tier probe disappeared before cleanup"));
|
||||||
|
}
|
||||||
|
match w.probe_transition_candidate(&probe_object).await? {
|
||||||
|
TransitionCandidateProbe::Missing => Ok(()),
|
||||||
|
_ => Err(std::io::Error::other("remote tier probe remained after cleanup")),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
if !matches!(cleanup_result, Ok(Ok(()))) {
|
||||||
|
return Err(probe_cleanup_incomplete_error());
|
||||||
|
}
|
||||||
|
if let Err(err) = read_result {
|
||||||
//if is_err_bucket_not_found(&err) {
|
//if is_err_bucket_not_found(&err) {
|
||||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||||
//}
|
//}
|
||||||
@@ -477,12 +653,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
|
|||||||
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
||||||
}*/
|
}*/
|
||||||
//else {
|
//else {
|
||||||
return Err(ERR_TIER_PERM_ERR.clone());
|
return Err(err);
|
||||||
//}
|
//}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Validate a backend using a caller-owned deadline while retaining a bounded
|
||||||
|
/// reconciliation window for an uncertain probe PUT. The validation future is
|
||||||
|
/// kept alive through cleanup so an outer timeout cannot abandon the remote
|
||||||
|
/// probe object.
|
||||||
|
pub(crate) async fn check_warm_backend_until(
|
||||||
|
w: Option<&WarmBackendImpl>,
|
||||||
|
deadline: tokio::time::Instant,
|
||||||
|
) -> Result<(), AdminError> {
|
||||||
|
check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
|
||||||
|
let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT;
|
||||||
|
check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
|
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
|
||||||
let mut d: Option<WarmBackendImpl> = None;
|
let mut d: Option<WarmBackendImpl> = None;
|
||||||
match tier.tier_type {
|
match tier.tier_type {
|
||||||
@@ -701,7 +893,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
|||||||
let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
|
let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
|
||||||
|
|
||||||
if probe {
|
if probe {
|
||||||
d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
|
check_warm_backend(Some(&d)).await?;
|
||||||
}
|
}
|
||||||
Ok(d)
|
Ok(d)
|
||||||
}
|
}
|
||||||
@@ -754,6 +946,7 @@ pub(crate) async fn new_transition_candidate_reconciler(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::services::tier::test_util::{MockWarmBackend, MockWarmOp};
|
||||||
use crate::services::tier::tier_config::TierWasabi;
|
use crate::services::tier::tier_config::TierWasabi;
|
||||||
use std::sync::{
|
use std::sync::{
|
||||||
Arc,
|
Arc,
|
||||||
@@ -920,13 +1113,38 @@ mod tests {
|
|||||||
|
|
||||||
struct RejectingProbeVersionBackend {
|
struct RejectingProbeVersionBackend {
|
||||||
gets: Arc<AtomicUsize>,
|
gets: Arc<AtomicUsize>,
|
||||||
|
present: Arc<std::sync::atomic::AtomicBool>,
|
||||||
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||||
|
returned_version: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct RecordingProbeBackend {
|
struct RecordingProbeBackend {
|
||||||
get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||||
|
present: Arc<std::sync::atomic::AtomicBool>,
|
||||||
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||||
|
remove_clears_candidate: bool,
|
||||||
fail_get: bool,
|
fail_get: bool,
|
||||||
|
body: ProbeBody,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct HangingProbePutBackend {
|
||||||
|
put_started: Arc<tokio::sync::Notify>,
|
||||||
|
present: Arc<std::sync::atomic::AtomicBool>,
|
||||||
|
probes: Arc<AtomicUsize>,
|
||||||
|
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct LateVisibleProbeBackend {
|
||||||
|
visible_at: tokio::time::Instant,
|
||||||
|
removed: Arc<std::sync::atomic::AtomicBool>,
|
||||||
|
probes: Arc<AtomicUsize>,
|
||||||
|
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
enum ProbeBody {
|
||||||
|
Exact,
|
||||||
|
Mismatch,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
@@ -976,7 +1194,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||||
Ok(uuid::Uuid::nil().to_string())
|
Ok(self.returned_version.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn put_with_meta(
|
async fn put_with_meta(
|
||||||
@@ -999,10 +1217,19 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||||
|
self.present.store(false, Ordering::SeqCst);
|
||||||
self.removed_versions.lock().await.push(rv.to_string());
|
self.removed_versions.lock().await.push(rv.to_string());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
if self.present.load(Ordering::SeqCst) {
|
||||||
|
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
|
||||||
|
} else {
|
||||||
|
Ok(TransitionCandidateProbe::Missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
@@ -1029,15 +1256,121 @@ mod tests {
|
|||||||
if self.fail_get {
|
if self.fail_get {
|
||||||
Err(std::io::Error::other("probe GET failed"))
|
Err(std::io::Error::other("probe GET failed"))
|
||||||
} else {
|
} else {
|
||||||
Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
|
match self.body {
|
||||||
|
ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))),
|
||||||
|
ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||||
|
if self.remove_clears_candidate {
|
||||||
|
self.present.store(false, Ordering::SeqCst);
|
||||||
|
}
|
||||||
self.removed_versions.lock().await.push(rv.to_string());
|
self.removed_versions.lock().await.push(rv.to_string());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
if self.present.load(Ordering::SeqCst) {
|
||||||
|
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
|
||||||
|
} else {
|
||||||
|
Ok(TransitionCandidateProbe::Missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl WarmBackend for HangingProbePutBackend {
|
||||||
|
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||||
|
self.put_started.notify_one();
|
||||||
|
std::future::pending().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_with_meta(
|
||||||
|
&self,
|
||||||
|
object: &str,
|
||||||
|
r: ReaderImpl,
|
||||||
|
length: i64,
|
||||||
|
_meta: HashMap<String, String>,
|
||||||
|
) -> Result<String, std::io::Error> {
|
||||||
|
self.put(object, r, length).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||||
|
Err(std::io::Error::other("GET must not run after a timed out probe PUT"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
|
||||||
|
Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||||
|
self.present.store(false, Ordering::SeqCst);
|
||||||
|
self.removed_versions.lock().await.push(rv.to_string());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
self.probes.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if self.present.load(Ordering::SeqCst) {
|
||||||
|
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
|
||||||
|
} else {
|
||||||
|
Ok(TransitionCandidateProbe::Missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl WarmBackend for LateVisibleProbeBackend {
|
||||||
|
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||||
|
Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::ConnectionReset,
|
||||||
|
"probe PUT response was lost before the object became visible",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_with_meta(
|
||||||
|
&self,
|
||||||
|
object: &str,
|
||||||
|
r: ReaderImpl,
|
||||||
|
length: i64,
|
||||||
|
_meta: HashMap<String, String>,
|
||||||
|
) -> Result<String, std::io::Error> {
|
||||||
|
self.put(object, r, length).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||||
|
Err(std::io::Error::other("GET must not run after a lost probe PUT response"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
|
||||||
|
Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||||
|
self.removed.store(true, Ordering::SeqCst);
|
||||||
|
self.removed_versions.lock().await.push(rv.to_string());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
self.probes.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) {
|
||||||
|
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
|
||||||
|
} else {
|
||||||
|
Ok(TransitionCandidateProbe::Missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
@@ -1098,13 +1431,15 @@ mod tests {
|
|||||||
assert_eq!(probe, TransitionCandidateProbe::Unsupported);
|
assert_eq!(probe, TransitionCandidateProbe::Unsupported);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
|
async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
|
||||||
let gets = Arc::new(AtomicUsize::new(0));
|
let gets = Arc::new(AtomicUsize::new(0));
|
||||||
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
|
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
|
||||||
gets: gets.clone(),
|
gets: gets.clone(),
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
removed_versions: removed_versions.clone(),
|
removed_versions: removed_versions.clone(),
|
||||||
|
returned_version: uuid::Uuid::nil().to_string(),
|
||||||
});
|
});
|
||||||
|
|
||||||
let err = check_warm_backend(Some(&backend))
|
let err = check_warm_backend(Some(&backend))
|
||||||
@@ -1113,7 +1448,27 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
||||||
assert_eq!(gets.load(Ordering::SeqCst), 0);
|
assert_eq!(gets.load(Ordering::SeqCst), 0);
|
||||||
assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() {
|
||||||
|
let gets = Arc::new(AtomicUsize::new(0));
|
||||||
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
|
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
|
||||||
|
gets: gets.clone(),
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
|
removed_versions: removed_versions.clone(),
|
||||||
|
returned_version: String::new(),
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&backend))
|
||||||
|
.await
|
||||||
|
.expect_err("an empty PUT version must not read or generically delete a versioned object");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
||||||
|
assert_eq!(gets.load(Ordering::SeqCst), 0);
|
||||||
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1122,8 +1477,11 @@ mod tests {
|
|||||||
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
||||||
get_versions: get_versions.clone(),
|
get_versions: get_versions.clone(),
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
removed_versions: removed_versions.clone(),
|
removed_versions: removed_versions.clone(),
|
||||||
|
remove_clears_candidate: true,
|
||||||
fail_get: false,
|
fail_get: false,
|
||||||
|
body: ProbeBody::Exact,
|
||||||
});
|
});
|
||||||
|
|
||||||
check_warm_backend(Some(&backend))
|
check_warm_backend(Some(&backend))
|
||||||
@@ -1140,8 +1498,11 @@ mod tests {
|
|||||||
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
||||||
get_versions: get_versions.clone(),
|
get_versions: get_versions.clone(),
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
removed_versions: removed_versions.clone(),
|
removed_versions: removed_versions.clone(),
|
||||||
|
remove_clears_candidate: true,
|
||||||
fail_get: true,
|
fail_get: true,
|
||||||
|
body: ProbeBody::Exact,
|
||||||
});
|
});
|
||||||
|
|
||||||
let err = check_warm_backend(Some(&backend))
|
let err = check_warm_backend(Some(&backend))
|
||||||
@@ -1153,6 +1514,169 @@ mod tests {
|
|||||||
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn check_warm_backend_removes_probe_after_body_mismatch() {
|
||||||
|
let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
|
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
||||||
|
get_versions,
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
|
removed_versions: removed_versions.clone(),
|
||||||
|
remove_clears_candidate: true,
|
||||||
|
fail_get: false,
|
||||||
|
body: ProbeBody::Mismatch,
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&backend))
|
||||||
|
.await
|
||||||
|
.expect_err("a mismatched body should fail after cleanup");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||||
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() {
|
||||||
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
|
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
||||||
|
get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())),
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
|
removed_versions: removed_versions.clone(),
|
||||||
|
remove_clears_candidate: false,
|
||||||
|
fail_get: false,
|
||||||
|
body: ProbeBody::Exact,
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&backend))
|
||||||
|
.await
|
||||||
|
.expect_err("cleanup must not succeed while the deleted candidate remains visible");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||||
|
assert!(err.message.contains("cleanup is incomplete"));
|
||||||
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn check_warm_backend_reconciles_a_lost_put_response() {
|
||||||
|
let backend = MockWarmBackend::new();
|
||||||
|
backend.lose_next_put_response();
|
||||||
|
let driver: WarmBackendImpl = Box::new(backend.clone());
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&driver))
|
||||||
|
.await
|
||||||
|
.expect_err("a lost probe PUT response must fail after compensation");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||||
|
assert_eq!(backend.object_count().await, 0);
|
||||||
|
assert_eq!(backend.exact_remove_count(), 1);
|
||||||
|
let operations = backend.op_log().await;
|
||||||
|
let put = operations.iter().find_map(|operation| match operation {
|
||||||
|
MockWarmOp::Put { object } => Some(object),
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
let probe = operations.iter().find_map(|operation| match operation {
|
||||||
|
MockWarmOp::Probe { object } => Some(object),
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
let remove = operations.iter().find_map(|operation| match operation {
|
||||||
|
MockWarmOp::Remove { object } => Some(object),
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else {
|
||||||
|
panic!("lost-response compensation should PUT, probe, and remove");
|
||||||
|
};
|
||||||
|
assert_eq!(put, probe);
|
||||||
|
assert_eq!(probe, remove);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn check_warm_backend_retries_until_a_late_put_becomes_visible() {
|
||||||
|
let probes = Arc::new(AtomicUsize::new(0));
|
||||||
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
|
let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend {
|
||||||
|
visible_at: tokio::time::Instant::now() + Duration::from_secs(5),
|
||||||
|
removed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
|
probes: probes.clone(),
|
||||||
|
removed_versions: removed_versions.clone(),
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&driver))
|
||||||
|
.await
|
||||||
|
.expect_err("a late-visible probe PUT must still report the lost response");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||||
|
assert!(
|
||||||
|
probes.load(Ordering::SeqCst) > 5,
|
||||||
|
"reconciliation must not stop at the first Missing result"
|
||||||
|
);
|
||||||
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() {
|
||||||
|
for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] {
|
||||||
|
let backend = MockWarmBackend::new();
|
||||||
|
backend.set_transition_candidate_probe_override(Some(candidate)).await;
|
||||||
|
backend.lose_next_put_response();
|
||||||
|
let driver: WarmBackendImpl = Box::new(backend.clone());
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&driver))
|
||||||
|
.await
|
||||||
|
.expect_err("an uncertain candidate must fail without a guessed delete");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||||
|
assert!(err.message.contains("cleanup is incomplete"));
|
||||||
|
assert_eq!(backend.remove_count().await, 0);
|
||||||
|
assert_eq!(backend.object_count().await, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn check_warm_backend_reports_an_exact_cleanup_failure() {
|
||||||
|
let backend = MockWarmBackend::new();
|
||||||
|
backend.set_remove_failure(true);
|
||||||
|
backend.lose_next_put_response();
|
||||||
|
let driver: WarmBackendImpl = Box::new(backend.clone());
|
||||||
|
|
||||||
|
let err = check_warm_backend(Some(&driver))
|
||||||
|
.await
|
||||||
|
.expect_err("an exact cleanup failure must replace the ambiguous PUT error");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||||
|
assert!(err.message.contains("cleanup is incomplete"));
|
||||||
|
assert_eq!(backend.exact_remove_count(), 1);
|
||||||
|
assert_eq!(backend.object_count().await, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn check_warm_backend_reconciles_a_timed_out_put() {
|
||||||
|
let put_started = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let probes = Arc::new(AtomicUsize::new(0));
|
||||||
|
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||||
|
let driver: WarmBackendImpl = Box::new(HangingProbePutBackend {
|
||||||
|
put_started: put_started.clone(),
|
||||||
|
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||||
|
probes: probes.clone(),
|
||||||
|
removed_versions: removed_versions.clone(),
|
||||||
|
});
|
||||||
|
let check = check_warm_backend(Some(&driver));
|
||||||
|
tokio::pin!(check);
|
||||||
|
tokio::select! {
|
||||||
|
_ = put_started.notified() => {}
|
||||||
|
result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await;
|
||||||
|
let err = check.await.expect_err("a timed out probe PUT must fail after compensation");
|
||||||
|
|
||||||
|
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
|
||||||
|
assert!(
|
||||||
|
probes.load(Ordering::SeqCst) > 1,
|
||||||
|
"timed-out PUT reconciliation must keep checking through the visibility window"
|
||||||
|
);
|
||||||
|
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn new_wasabi_backend_honors_probe_flag() {
|
async fn new_wasabi_backend_honors_probe_flag() {
|
||||||
let tier = TierConfig {
|
let tier = TierConfig {
|
||||||
@@ -1298,6 +1822,15 @@ mod tests {
|
|||||||
assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
|
assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() {
|
||||||
|
let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse");
|
||||||
|
assert_eq!(
|
||||||
|
endpoint_authority(&url).expect("the endpoint should have an authority"),
|
||||||
|
"[2001:db8::1]:9443"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
|
async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
|
||||||
let mut params = s3_compatible_params("http://tier.example.com:9000");
|
let mut params = s3_compatible_params("http://tier.example.com:9000");
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use std::collections::HashMap;
|
|||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier_config::TierAliyun,
|
tier_config::TierAliyun,
|
||||||
warm_backend::{
|
warm_backend::{
|
||||||
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||||
new_s3_compatible_warm_backend, optimal_part_size,
|
new_s3_compatible_warm_backend, optimal_part_size,
|
||||||
},
|
},
|
||||||
warm_backend_s3::WarmBackendS3,
|
warm_backend_s3::WarmBackendS3,
|
||||||
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendAliyun {
|
|||||||
self.0.remove(object, rv).await
|
self.0.remove(object, rv).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
self.0.probe_transition_candidate(object).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
self.0.in_use().await
|
self.0.in_use().await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use std::collections::HashMap;
|
|||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier_config::TierAzure,
|
tier_config::TierAzure,
|
||||||
warm_backend::{
|
warm_backend::{
|
||||||
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||||
new_s3_compatible_warm_backend, optimal_part_size,
|
new_s3_compatible_warm_backend, optimal_part_size,
|
||||||
},
|
},
|
||||||
warm_backend_s3::WarmBackendS3,
|
warm_backend_s3::WarmBackendS3,
|
||||||
@@ -89,6 +89,16 @@ impl WarmBackend for WarmBackendAzure {
|
|||||||
self.0.remove(object, rv).await
|
self.0.remove(object, rv).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
// Azure currently uses the shared S3/SigV4 transport, but its normal
|
||||||
|
// object path cannot persist exact remote versions across mixed
|
||||||
|
// RustFS releases. The mutation probe may still detect and precisely
|
||||||
|
// remove a versioned test object before rejecting that configuration.
|
||||||
|
self.0
|
||||||
|
.probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id")
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
self.0.in_use().await
|
self.0.in_use().await
|
||||||
}
|
}
|
||||||
@@ -98,6 +108,23 @@ impl WarmBackend for WarmBackendAzure {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::services::tier::tier_config::TierAzure;
|
use crate::services::tier::tier_config::TierAzure;
|
||||||
|
use rustfs_s3_client::{
|
||||||
|
credentials::{Credentials, SignatureType, Static, Value},
|
||||||
|
transition_api::{Options, TransitionClient, TransitionCore},
|
||||||
|
};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
|
async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
|
||||||
|
let mut request = Vec::new();
|
||||||
|
let mut buffer = [0_u8; 1024];
|
||||||
|
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||||
|
let read = stream.read(&mut buffer).await.expect("fixture request should be readable");
|
||||||
|
assert_ne!(read, 0, "connection closed before request headers were received");
|
||||||
|
request.extend_from_slice(&buffer[..read]);
|
||||||
|
}
|
||||||
|
String::from_utf8_lossy(&request).into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
/// The SSRF guard itself is exercised once, generically, in
|
/// The SSRF guard itself is exercised once, generically, in
|
||||||
/// `warm_backend::tests` (see backlog#2040/backlog#2041 and
|
/// `warm_backend::tests` (see backlog#2040/backlog#2041 and
|
||||||
@@ -119,4 +146,85 @@ mod tests {
|
|||||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() {
|
||||||
|
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||||
|
Ok(listener) => listener,
|
||||||
|
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
|
||||||
|
Err(err) => panic!("test listener should bind: {err}"),
|
||||||
|
};
|
||||||
|
let endpoint = listener
|
||||||
|
.local_addr()
|
||||||
|
.expect("listener local address should be available")
|
||||||
|
.to_string();
|
||||||
|
let fixture = tokio::spawn(async move {
|
||||||
|
let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
|
||||||
|
let get_request = read_request_head(&mut get_stream).await;
|
||||||
|
get_stream
|
||||||
|
.write_all(
|
||||||
|
b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fixture should write candidate GET response");
|
||||||
|
|
||||||
|
let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE");
|
||||||
|
let delete_request = read_request_head(&mut delete_stream).await;
|
||||||
|
delete_stream
|
||||||
|
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||||
|
.await
|
||||||
|
.expect("fixture should write exact DELETE response");
|
||||||
|
(get_request, delete_request)
|
||||||
|
});
|
||||||
|
let client = Arc::new(
|
||||||
|
TransitionClient::new(
|
||||||
|
&endpoint,
|
||||||
|
Options {
|
||||||
|
creds: Credentials::new(Static(Value {
|
||||||
|
access_key_id: "access-key".to_string(),
|
||||||
|
secret_access_key: "secret-key".to_string(),
|
||||||
|
signer_type: SignatureType::SignatureV4,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
bucket_lookup: BucketLookupType::BucketLookupPath,
|
||||||
|
max_retries: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
"azure",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fixture client should build"),
|
||||||
|
);
|
||||||
|
let backend = WarmBackendAzure(WarmBackendS3 {
|
||||||
|
core: TransitionCore(Arc::clone(&client)),
|
||||||
|
client,
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
prefix: String::new(),
|
||||||
|
storage_class: String::new(),
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
!backend.0.client.provider_version_capabilities().exact_get_delete,
|
||||||
|
"probe-only version discovery must not change Azure's persisted data-path contract"
|
||||||
|
);
|
||||||
|
|
||||||
|
let candidate = backend
|
||||||
|
.probe_transition_candidate("probe")
|
||||||
|
.await
|
||||||
|
.expect("Azure candidate should be discovered");
|
||||||
|
assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string()));
|
||||||
|
backend
|
||||||
|
.remove_exact("probe", "azure-version")
|
||||||
|
.await
|
||||||
|
.expect("Azure candidate should be deleted by exact version");
|
||||||
|
|
||||||
|
let (get_request, delete_request) = fixture.await.expect("fixture should join");
|
||||||
|
assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n"));
|
||||||
|
assert!(
|
||||||
|
delete_request
|
||||||
|
.lines()
|
||||||
|
.next()
|
||||||
|
.is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version "))
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,13 +18,13 @@
|
|||||||
#![allow(unused_must_use)]
|
#![allow(unused_must_use)]
|
||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use std::future::Future;
|
||||||
use std::io::{Error, ErrorKind};
|
use std::io::{Error, ErrorKind};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use google_cloud_auth::credentials::Credentials;
|
use google_cloud_auth::credentials::service_account::Builder;
|
||||||
use google_cloud_auth::credentials::user_account::Builder;
|
|
||||||
use google_cloud_storage as gcs;
|
use google_cloud_storage as gcs;
|
||||||
use google_cloud_storage::client::Storage;
|
use google_cloud_storage::client::Storage;
|
||||||
use google_cloud_storage::client::StorageControl;
|
use google_cloud_storage::client::StorageControl;
|
||||||
@@ -32,7 +32,7 @@ use std::convert::TryFrom;
|
|||||||
|
|
||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier_config::TierGCS,
|
tier_config::TierGCS,
|
||||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
|
||||||
};
|
};
|
||||||
use rustfs_s3_client::{
|
use rustfs_s3_client::{
|
||||||
admin_handler_utils::AdminError,
|
admin_handler_utils::AdminError,
|
||||||
@@ -43,6 +43,7 @@ use rustfs_utils::egress::validate_outbound_url;
|
|||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||||
|
const MAX_GCS_CANDIDATE_PAGES: usize = 64;
|
||||||
|
|
||||||
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
||||||
if remote_version.is_empty() {
|
if remote_version.is_empty() {
|
||||||
@@ -57,6 +58,85 @@ fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
|||||||
Ok(Some(generation))
|
Ok(Some(generation))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn append_gcs_chunk<E: std::fmt::Display>(
|
||||||
|
contents: &mut Vec<u8>,
|
||||||
|
chunk: Result<Bytes, E>,
|
||||||
|
max_response_bytes: Option<usize>,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?;
|
||||||
|
if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
ErrorKind::InvalidData,
|
||||||
|
"GCS object response exceeded the configured byte limit",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
contents.extend_from_slice(&chunk);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn gcs_bucket_resource_name(bucket: &str) -> String {
|
||||||
|
format!("projects/_/buckets/{bucket}")
|
||||||
|
}
|
||||||
|
|
||||||
|
struct GcsCandidateObject {
|
||||||
|
name: String,
|
||||||
|
generation: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct GcsCandidatePage {
|
||||||
|
objects: Vec<GcsCandidateObject>,
|
||||||
|
next_page_token: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn probe_exact_gcs_candidate<F, Fut>(
|
||||||
|
remote_object: &str,
|
||||||
|
mut fetch_page: F,
|
||||||
|
) -> Result<TransitionCandidateProbe, std::io::Error>
|
||||||
|
where
|
||||||
|
F: FnMut(String) -> Fut,
|
||||||
|
Fut: Future<Output = Result<GcsCandidatePage, std::io::Error>>,
|
||||||
|
{
|
||||||
|
let mut page_token = String::new();
|
||||||
|
let mut seen_page_tokens = HashSet::new();
|
||||||
|
let mut generation = None;
|
||||||
|
let mut pages_seen = 0_usize;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
pages_seen += 1;
|
||||||
|
if pages_seen > MAX_GCS_CANDIDATE_PAGES {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
ErrorKind::InvalidData,
|
||||||
|
"GCS candidate listing exceeded the page limit",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let response = fetch_page(page_token.clone()).await?;
|
||||||
|
for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) {
|
||||||
|
if candidate.generation <= 0 {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
ErrorKind::InvalidData,
|
||||||
|
"GCS candidate listing returned a non-positive generation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if generation.replace(candidate.generation).is_some() {
|
||||||
|
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if response.next_page_token.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if !seen_page_tokens.insert(response.next_page_token.clone()) {
|
||||||
|
return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token"));
|
||||||
|
}
|
||||||
|
page_token = response.next_page_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(match generation {
|
||||||
|
Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()),
|
||||||
|
None => TransitionCandidateProbe::Missing,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub struct WarmBackendGCS {
|
pub struct WarmBackendGCS {
|
||||||
pub client: Arc<Storage>,
|
pub client: Arc<Storage>,
|
||||||
pub control: Arc<StorageControl>,
|
pub control: Arc<StorageControl>,
|
||||||
@@ -80,8 +160,8 @@ impl WarmBackendGCS {
|
|||||||
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let authorized_user = serde_json::from_str(&conf.creds)?;
|
let service_account = serde_json::from_str(&conf.creds)?;
|
||||||
let credentials = Builder::new(authorized_user)
|
let credentials = Builder::new(service_account)
|
||||||
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
|
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
|
||||||
//.with_backoff_policy(backoff)
|
//.with_backoff_policy(backoff)
|
||||||
.build()
|
.build()
|
||||||
@@ -98,7 +178,11 @@ impl WarmBackendGCS {
|
|||||||
let client = Arc::new(client);
|
let client = Arc::new(client);
|
||||||
// Control-plane client: the data-plane `Storage` client cannot delete or list objects;
|
// Control-plane client: the data-plane `Storage` client cannot delete or list objects;
|
||||||
// delete_object/list_objects live on StorageControl.
|
// delete_object/list_objects live on StorageControl.
|
||||||
let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else {
|
let mut control_builder = StorageControl::builder().with_credentials(credentials);
|
||||||
|
if !conf.endpoint.is_empty() {
|
||||||
|
control_builder = control_builder.with_endpoint(conf.endpoint.clone());
|
||||||
|
}
|
||||||
|
let Ok(control) = control_builder.build().await else {
|
||||||
return Err(std::io::Error::other("StorageControl::builder error"));
|
return Err(std::io::Error::other("StorageControl::builder error"));
|
||||||
};
|
};
|
||||||
let control = Arc::new(control);
|
let control = Arc::new(control);
|
||||||
@@ -136,9 +220,10 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
ReaderImpl::Body(content_body) => content_body.to_vec(),
|
ReaderImpl::Body(content_body) => content_body.to_vec(),
|
||||||
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
|
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
|
||||||
};
|
};
|
||||||
|
let bucket = gcs_bucket_resource_name(&self.bucket);
|
||||||
let Ok(res) = Box::pin(
|
let Ok(res) = Box::pin(
|
||||||
self.client
|
self.client
|
||||||
.write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
|
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
|
||||||
.send_buffered(),
|
.send_buffered(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -154,7 +239,9 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||||
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
|
let bucket = gcs_bucket_resource_name(&self.bucket);
|
||||||
|
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
|
||||||
|
let mut max_response_bytes = None;
|
||||||
if let Some(generation) = parse_generation(rv)? {
|
if let Some(generation) = parse_generation(rv)? {
|
||||||
req = req.set_generation(generation);
|
req = req.set_generation(generation);
|
||||||
}
|
}
|
||||||
@@ -170,6 +257,11 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
.length
|
.length
|
||||||
.try_into()
|
.try_into()
|
||||||
.map_err(|_| std::io::Error::other("invalid range: negative length"))?;
|
.map_err(|_| std::io::Error::other("invalid range: negative length"))?;
|
||||||
|
max_response_bytes = Some(
|
||||||
|
opts.length
|
||||||
|
.try_into()
|
||||||
|
.map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?,
|
||||||
|
);
|
||||||
req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
|
req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,8 +269,8 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
return Err(std::io::Error::other("read_object error"));
|
return Err(std::io::Error::other("read_object error"));
|
||||||
};
|
};
|
||||||
let mut contents = Vec::new();
|
let mut contents = Vec::new();
|
||||||
while let Ok(Some(chunk)) = reader.next().await.transpose() {
|
while let Some(chunk) = reader.next().await {
|
||||||
contents.extend_from_slice(&chunk);
|
append_gcs_chunk(&mut contents, chunk, max_response_bytes)?;
|
||||||
}
|
}
|
||||||
Ok(ReadCloser::new(std::io::Cursor::new(contents)))
|
Ok(ReadCloser::new(std::io::Cursor::new(contents)))
|
||||||
}
|
}
|
||||||
@@ -190,7 +282,7 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
let mut req = self
|
let mut req = self
|
||||||
.control
|
.control
|
||||||
.delete_object()
|
.delete_object()
|
||||||
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
|
.set_bucket(gcs_bucket_resource_name(&self.bucket))
|
||||||
.set_object(self.get_dest(object));
|
.set_object(self.get_dest(object));
|
||||||
if let Some(generation) = parse_generation(rv)? {
|
if let Some(generation) = parse_generation(rv)? {
|
||||||
req = req.set_generation(generation);
|
req = req.set_generation(generation);
|
||||||
@@ -199,13 +291,47 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
let remote_object = self.get_dest(object);
|
||||||
|
let parent = gcs_bucket_resource_name(&self.bucket);
|
||||||
|
probe_exact_gcs_candidate(&remote_object, |page_token| {
|
||||||
|
let control = self.control.clone();
|
||||||
|
let parent = parent.clone();
|
||||||
|
let prefix = remote_object.clone();
|
||||||
|
async move {
|
||||||
|
let response = control
|
||||||
|
.list_objects()
|
||||||
|
.set_parent(parent)
|
||||||
|
.set_prefix(prefix)
|
||||||
|
.set_versions(true)
|
||||||
|
.set_page_size(2)
|
||||||
|
.set_page_token(page_token)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| std::io::Error::other(err.to_string()))?;
|
||||||
|
Ok(GcsCandidatePage {
|
||||||
|
objects: response
|
||||||
|
.objects
|
||||||
|
.into_iter()
|
||||||
|
.map(|candidate| GcsCandidateObject {
|
||||||
|
name: candidate.name,
|
||||||
|
generation: candidate.generation,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
next_page_token: response.next_page_token,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
// Scope the listing to this tier's prefix (matching the other warm backends) and only
|
// Scope the listing to this tier's prefix (matching the other warm backends) and only
|
||||||
// need to know whether a single object exists.
|
// need to know whether a single object exists.
|
||||||
let resp = self
|
let resp = self
|
||||||
.control
|
.control
|
||||||
.list_objects()
|
.list_objects()
|
||||||
.set_parent(format!("projects/_/buckets/{}", self.bucket))
|
.set_parent(gcs_bucket_resource_name(&self.bucket))
|
||||||
.set_prefix(self.prefix.clone())
|
.set_prefix(self.prefix.clone())
|
||||||
.set_page_size(1)
|
.set_page_size(1)
|
||||||
.send()
|
.send()
|
||||||
@@ -218,10 +344,126 @@ impl WarmBackend for WarmBackendGCS {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use super::GcsCandidateObject;
|
||||||
|
use super::GcsCandidatePage;
|
||||||
|
use super::MAX_GCS_CANDIDATE_PAGES;
|
||||||
use super::WarmBackendGCS;
|
use super::WarmBackendGCS;
|
||||||
|
use super::append_gcs_chunk;
|
||||||
|
use super::gcs_bucket_resource_name;
|
||||||
use super::parse_generation;
|
use super::parse_generation;
|
||||||
|
use super::probe_exact_gcs_candidate;
|
||||||
use crate::services::tier::tier_config::TierGCS;
|
use crate::services::tier::tier_config::TierGCS;
|
||||||
|
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts};
|
||||||
|
use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
|
||||||
|
use google_cloud_storage::client::{Storage, StorageControl};
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
async fn serve_data_plane_fixture(listener: TcpListener) -> Vec<String> {
|
||||||
|
let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#;
|
||||||
|
let responses = [
|
||||||
|
format!(
|
||||||
|
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}",
|
||||||
|
upload_body.len()
|
||||||
|
),
|
||||||
|
"HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!"
|
||||||
|
.to_string(),
|
||||||
|
"HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!"
|
||||||
|
.to_string(),
|
||||||
|
];
|
||||||
|
let mut requests = Vec::new();
|
||||||
|
|
||||||
|
for response in responses {
|
||||||
|
let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request");
|
||||||
|
let mut request = Vec::new();
|
||||||
|
loop {
|
||||||
|
let mut chunk = [0_u8; 1024];
|
||||||
|
let count = stream
|
||||||
|
.read(&mut chunk)
|
||||||
|
.await
|
||||||
|
.expect("the GCS fixture should read request headers");
|
||||||
|
if count == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
request.extend_from_slice(&chunk[..count]);
|
||||||
|
if request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let header_end = request
|
||||||
|
.windows(4)
|
||||||
|
.position(|window| window == b"\r\n\r\n")
|
||||||
|
.map(|position| position + 4)
|
||||||
|
.expect("the GCS fixture should receive complete request headers");
|
||||||
|
let headers = String::from_utf8_lossy(&request[..header_end]);
|
||||||
|
if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) {
|
||||||
|
stream
|
||||||
|
.write_all(b"HTTP/1.1 100 Continue\r\n\r\n")
|
||||||
|
.await
|
||||||
|
.expect("the GCS fixture should acknowledge 100-continue");
|
||||||
|
}
|
||||||
|
let content_length = headers
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| {
|
||||||
|
let (name, value) = line.split_once(':')?;
|
||||||
|
name.eq_ignore_ascii_case("content-length")
|
||||||
|
.then(|| value.trim().parse::<usize>().expect("content-length should be numeric"))
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
while request.len() < header_end.saturating_add(content_length) {
|
||||||
|
let mut chunk = [0_u8; 1024];
|
||||||
|
let count = stream
|
||||||
|
.read(&mut chunk)
|
||||||
|
.await
|
||||||
|
.expect("the GCS fixture should read the request body");
|
||||||
|
if count == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
request.extend_from_slice(&chunk[..count]);
|
||||||
|
}
|
||||||
|
requests.push(String::from_utf8_lossy(&request).into_owned());
|
||||||
|
stream
|
||||||
|
.write_all(response.as_bytes())
|
||||||
|
.await
|
||||||
|
.expect("the GCS fixture should write its response");
|
||||||
|
}
|
||||||
|
|
||||||
|
requests
|
||||||
|
}
|
||||||
|
|
||||||
|
fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage {
|
||||||
|
GcsCandidatePage {
|
||||||
|
objects: objects
|
||||||
|
.iter()
|
||||||
|
.map(|(name, generation)| GcsCandidateObject {
|
||||||
|
name: (*name).to_string(),
|
||||||
|
generation: *generation,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
next_page_token: next_page_token.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn probe_candidate_pages(
|
||||||
|
remote_object: &str,
|
||||||
|
pages: Vec<GcsCandidatePage>,
|
||||||
|
) -> (Result<TransitionCandidateProbe, std::io::Error>, Vec<String>) {
|
||||||
|
let mut pages = pages.into_iter();
|
||||||
|
let mut requested_tokens = Vec::new();
|
||||||
|
let result = probe_exact_gcs_candidate(remote_object, |page_token| {
|
||||||
|
requested_tokens.push(page_token);
|
||||||
|
std::future::ready(
|
||||||
|
pages
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
(result, requested_tokens)
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn generation_parser_preserves_exact_numeric_versions() {
|
fn generation_parser_preserves_exact_numeric_versions() {
|
||||||
@@ -241,6 +483,243 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn body_collection_propagates_an_error_after_a_complete_prefix() {
|
||||||
|
let mut contents = Vec::new();
|
||||||
|
append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7))
|
||||||
|
.expect("the prefix chunk should be collected");
|
||||||
|
let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7))
|
||||||
|
.expect_err("a trailing stream error must not be mistaken for EOF");
|
||||||
|
|
||||||
|
assert_eq!(contents, b"RustFS");
|
||||||
|
assert!(err.to_string().contains("trailing stream failure"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() {
|
||||||
|
let mut contents = Vec::new();
|
||||||
|
let err = append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7))
|
||||||
|
.expect_err("the GCS collection layer must reject an oversized probe response");
|
||||||
|
|
||||||
|
assert!(contents.is_empty());
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidData);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_finds_exact_name_on_first_or_later_page() {
|
||||||
|
let (first, first_tokens) =
|
||||||
|
probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await;
|
||||||
|
assert_eq!(
|
||||||
|
first.expect("an exact first-page object should be discovered"),
|
||||||
|
TransitionCandidateProbe::VersionedPresent("7".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(first_tokens, [""]);
|
||||||
|
|
||||||
|
let (later, later_tokens) = probe_candidate_pages(
|
||||||
|
"prefix/object",
|
||||||
|
vec![
|
||||||
|
candidate_page(&[("prefix/object-shadow", 8)], "next"),
|
||||||
|
candidate_page(&[("prefix/object", 9)], ""),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
later.expect("an exact later-page object should be discovered"),
|
||||||
|
TransitionCandidateProbe::VersionedPresent("9".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(later_tokens, ["", "next"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_ignores_non_exact_prefix_matches() {
|
||||||
|
let (probe, _) = probe_candidate_pages(
|
||||||
|
"prefix/object",
|
||||||
|
vec![candidate_page(
|
||||||
|
&[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)],
|
||||||
|
"",
|
||||||
|
)],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
probe.expect("prefix-only matches should not hide the exact object"),
|
||||||
|
TransitionCandidateProbe::VersionedPresent("7".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() {
|
||||||
|
let (probe, _) = probe_candidate_pages(
|
||||||
|
"prefix/object",
|
||||||
|
vec![
|
||||||
|
candidate_page(&[("prefix/object", 7)], "next"),
|
||||||
|
candidate_page(&[("prefix/object", 8)], ""),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
probe.expect("multiple exact generations should produce a conservative result"),
|
||||||
|
TransitionCandidateProbe::Ambiguous
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_reports_missing_without_an_exact_name() {
|
||||||
|
let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
probe.expect("a complete listing without an exact name should be definitive"),
|
||||||
|
TransitionCandidateProbe::Missing
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_rejects_non_positive_generations() {
|
||||||
|
for generation in [0, -1] {
|
||||||
|
let (probe, _) =
|
||||||
|
probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await;
|
||||||
|
let err = probe.expect_err("a non-positive GCS generation must fail closed");
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_rejects_a_page_token_that_does_not_advance() {
|
||||||
|
let (probe, requested_tokens) =
|
||||||
|
probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await;
|
||||||
|
|
||||||
|
let err = probe.expect_err("a repeated GCS page token must fail closed");
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidData);
|
||||||
|
assert_eq!(requested_tokens, ["", "next"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() {
|
||||||
|
let (probe, requested_tokens) = probe_candidate_pages(
|
||||||
|
"prefix/object",
|
||||||
|
vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed");
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidData);
|
||||||
|
assert_eq!(requested_tokens, ["", "a", "b"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_rejects_an_unbounded_unique_token_chain() {
|
||||||
|
let pages = (0..MAX_GCS_CANDIDATE_PAGES)
|
||||||
|
.map(|index| candidate_page(&[], &format!("token-{index}")))
|
||||||
|
.collect();
|
||||||
|
let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await;
|
||||||
|
|
||||||
|
let err = probe.expect_err("an unbounded unique page-token chain must fail closed");
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidData);
|
||||||
|
assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0")
|
||||||
|
.await
|
||||||
|
.expect("the GCS fixture should bind a loopback port");
|
||||||
|
let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address"));
|
||||||
|
let fixture = tokio::spawn(serve_data_plane_fixture(listener));
|
||||||
|
let credentials = Anonymous::new().build();
|
||||||
|
let client = Storage::builder()
|
||||||
|
.with_endpoint(endpoint.clone())
|
||||||
|
.with_credentials(credentials.clone())
|
||||||
|
.build()
|
||||||
|
.await
|
||||||
|
.expect("the GCS data client should build");
|
||||||
|
let control = StorageControl::builder()
|
||||||
|
.with_endpoint(endpoint)
|
||||||
|
.with_credentials(credentials)
|
||||||
|
.build()
|
||||||
|
.await
|
||||||
|
.expect("the GCS control client should build");
|
||||||
|
let backend = WarmBackendGCS {
|
||||||
|
client: Arc::new(client),
|
||||||
|
control: Arc::new(control),
|
||||||
|
bucket: "tier-bucket".to_string(),
|
||||||
|
prefix: String::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
let version = backend
|
||||||
|
.put(
|
||||||
|
"probe",
|
||||||
|
rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")),
|
||||||
|
6,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("a plain configured bucket should reach the GCS upload transport");
|
||||||
|
let mut reader = backend
|
||||||
|
.get(
|
||||||
|
"probe",
|
||||||
|
&version,
|
||||||
|
WarmBackendGetOpts {
|
||||||
|
start_offset: 0,
|
||||||
|
length: 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("a plain configured bucket should reach the GCS read transport");
|
||||||
|
let mut body = Vec::new();
|
||||||
|
reader
|
||||||
|
.read_to_end(&mut body)
|
||||||
|
.await
|
||||||
|
.expect("the fixture body should be readable");
|
||||||
|
let oversized_error = match backend
|
||||||
|
.get(
|
||||||
|
"probe",
|
||||||
|
&version,
|
||||||
|
WarmBackendGetOpts {
|
||||||
|
start_offset: 0,
|
||||||
|
length: 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"),
|
||||||
|
Err(err) => err,
|
||||||
|
};
|
||||||
|
let requests = fixture.await.expect("the GCS fixture task should finish");
|
||||||
|
(version, body, oversized_error.kind(), requests)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("the GCS data-plane requests should not be rejected before transport");
|
||||||
|
|
||||||
|
assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket");
|
||||||
|
assert_eq!(version, "123");
|
||||||
|
assert_eq!(body, b"RustFS!");
|
||||||
|
assert_eq!(oversized_error_kind, ErrorKind::InvalidData);
|
||||||
|
assert!(
|
||||||
|
requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"),
|
||||||
|
"unexpected upload request line: {}",
|
||||||
|
requests[0].lines().next().unwrap_or_default()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
|
||||||
|
"unexpected read request line: {}",
|
||||||
|
requests[1].lines().next().unwrap_or_default()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
|
||||||
|
"the GCS probe read must preserve its seven-byte range"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
|
||||||
|
"unexpected oversized read request line: {}",
|
||||||
|
requests[2].lines().next().unwrap_or_default()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
|
||||||
|
"the oversized response must be fetched under the same seven-byte request boundary"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn new_rejects_loopback_endpoint_before_credential_setup() {
|
async fn new_rejects_loopback_endpoint_before_credential_setup() {
|
||||||
let conf = TierGCS {
|
let conf = TierGCS {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use std::collections::HashMap;
|
|||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier_config::TierHuaweicloud,
|
tier_config::TierHuaweicloud,
|
||||||
warm_backend::{
|
warm_backend::{
|
||||||
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||||
new_s3_compatible_warm_backend, optimal_part_size,
|
new_s3_compatible_warm_backend, optimal_part_size,
|
||||||
},
|
},
|
||||||
warm_backend_s3::WarmBackendS3,
|
warm_backend_s3::WarmBackendS3,
|
||||||
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendHuaweicloud {
|
|||||||
self.0.remove(object, rv).await
|
self.0.remove(object, rv).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
self.0.probe_transition_candidate(object).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
self.0.in_use().await
|
self.0.in_use().await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,11 +26,12 @@ use crate::services::tier::{
|
|||||||
tier_config::TierS3,
|
tier_config::TierS3,
|
||||||
warm_backend::{
|
warm_backend::{
|
||||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||||
build_transition_put_options,
|
build_transition_put_options, endpoint_authority,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_s3_client::{
|
use rustfs_s3_client::{
|
||||||
|
api_error_response::to_error_response,
|
||||||
api_get_options::GetObjectOptions,
|
api_get_options::GetObjectOptions,
|
||||||
api_list::ListObjectsOptions,
|
api_list::ListObjectsOptions,
|
||||||
api_put_object::PutObjectOptions,
|
api_put_object::PutObjectOptions,
|
||||||
@@ -43,7 +44,7 @@ use rustfs_s3_client::{
|
|||||||
};
|
};
|
||||||
use rustfs_utils::egress::validate_outbound_url;
|
use rustfs_utils::egress::validate_outbound_url;
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||||
use s3s::dto::BucketVersioningStatus;
|
use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
|
||||||
|
|
||||||
pub struct WarmBackendS3 {
|
pub struct WarmBackendS3 {
|
||||||
pub client: Arc<TransitionClient>,
|
pub client: Arc<TransitionClient>,
|
||||||
@@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBu
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bounded_get_range(opts: &WarmBackendGetOpts) -> Result<Option<(i64, i64)>, std::io::Error> {
|
||||||
|
if opts.start_offset < 0 || opts.length <= 0 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
usize::try_from(opts.length)
|
||||||
|
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?;
|
||||||
|
let end_offset = opts
|
||||||
|
.start_offset
|
||||||
|
.checked_add(opts.length - 1)
|
||||||
|
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?;
|
||||||
|
Ok(Some((opts.start_offset, end_offset)))
|
||||||
|
}
|
||||||
|
|
||||||
impl WarmBackendS3 {
|
impl WarmBackendS3 {
|
||||||
pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
|
pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
|
||||||
Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
|
Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
|
||||||
@@ -132,10 +146,8 @@ impl WarmBackendS3 {
|
|||||||
bucket_lookup,
|
bucket_lookup,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let host = u
|
let endpoint = endpoint_authority(&u)?;
|
||||||
.host()
|
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
|
||||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
|
||||||
let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
|
|
||||||
|
|
||||||
let client = Arc::new(client);
|
let client = Arc::new(client);
|
||||||
let core = TransitionCore(Arc::clone(&client));
|
let core = TransitionCore(Arc::clone(&client));
|
||||||
@@ -177,10 +189,8 @@ impl WarmBackendS3 {
|
|||||||
if !rv.is_empty() {
|
if !rv.is_empty() {
|
||||||
gopts.version_id = rv.to_string();
|
gopts.version_id = rv.to_string();
|
||||||
}
|
}
|
||||||
if opts.start_offset >= 0 && opts.length > 0 {
|
if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? {
|
||||||
gopts
|
gopts.set_range(start_offset, end_offset)?;
|
||||||
.set_range(opts.start_offset, opts.start_offset + opts.length - 1)
|
|
||||||
.map_err(std::io::Error::other)?;
|
|
||||||
}
|
}
|
||||||
let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
||||||
Ok((headers, reader))
|
Ok((headers, reader))
|
||||||
@@ -191,34 +201,62 @@ impl WarmBackendS3 {
|
|||||||
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn probe_transition_candidate_versions(
|
async fn probe_current_transition_candidate_with_header(
|
||||||
&self,
|
&self,
|
||||||
object: &str,
|
object: &str,
|
||||||
bucket_versioning: RemoteBucketVersioning,
|
raw_version_header: Option<&'static str>,
|
||||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
let remote_object = self.get_dest(object);
|
match self
|
||||||
let mut opts = ListObjectsOptions::default();
|
.get_with_headers(
|
||||||
opts.set("prefix", &remote_object);
|
object,
|
||||||
opts.set("max-keys", "1000");
|
"",
|
||||||
|
WarmBackendGetOpts {
|
||||||
let mut key_marker = String::new();
|
start_offset: 0,
|
||||||
let mut version_id_marker = String::new();
|
length: 1,
|
||||||
let mut candidates = TransitionCandidateVersions::default();
|
},
|
||||||
loop {
|
)
|
||||||
let versions = self
|
.await
|
||||||
.client
|
{
|
||||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
Ok((headers, _)) => {
|
||||||
.await?;
|
let version_id = match raw_version_header {
|
||||||
candidates.extend(&remote_object, &versions);
|
Some(header_name) => match headers.get(header_name) {
|
||||||
if candidates.is_ambiguous() {
|
Some(value) => {
|
||||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
let version_id = value.to_str().map_err(|_| {
|
||||||
|
std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidData,
|
||||||
|
"remote object version id is not valid ASCII",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
validate_remote_version_id(version_id)?;
|
||||||
|
Some(version_id)
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
},
|
||||||
|
None => self.client.raw_version_id(&headers)?,
|
||||||
|
};
|
||||||
|
Ok(match version_id {
|
||||||
|
Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()),
|
||||||
|
None => TransitionCandidateProbe::UnversionedPresent,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
let response = to_error_response(&err);
|
||||||
|
if response.code == S3ErrorCode::NoSuchKey {
|
||||||
|
Ok(TransitionCandidateProbe::Missing)
|
||||||
|
} else {
|
||||||
|
Err(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !versions.is_truncated {
|
|
||||||
return classify_transition_candidates(candidates, bucket_versioning);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
pub(crate) async fn probe_transition_candidate_with_raw_version_header(
|
||||||
}
|
&self,
|
||||||
|
object: &str,
|
||||||
|
raw_version_header: &'static str,
|
||||||
|
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
self.probe_current_transition_candidate_with_header(object, Some(raw_version_header))
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn probe_transition_candidate_identity(
|
async fn probe_transition_candidate_identity(
|
||||||
@@ -343,6 +381,7 @@ struct TransitionCandidateVersions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TransitionCandidateVersions {
|
impl TransitionCandidateVersions {
|
||||||
|
#[cfg(test)]
|
||||||
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
||||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||||
if self.version_id.is_some() {
|
if self.version_id.is_some() {
|
||||||
@@ -353,10 +392,6 @@ impl TransitionCandidateVersions {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_ambiguous(&self) -> bool {
|
|
||||||
self.ambiguous
|
|
||||||
}
|
|
||||||
|
|
||||||
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
|
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
|
||||||
if self.ambiguous {
|
if self.ambiguous {
|
||||||
return TransitionCandidateProbe::Ambiguous;
|
return TransitionCandidateProbe::Ambiguous;
|
||||||
@@ -380,6 +415,8 @@ impl TransitionCandidateVersions {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
|
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||||
@@ -398,6 +435,204 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn new_preserves_an_explicit_endpoint_port() {
|
||||||
|
let conf = TierS3 {
|
||||||
|
endpoint: "https://tier.example.com:9443".to_string(),
|
||||||
|
bucket: "tier-bucket".to_string(),
|
||||||
|
access_key: "access".to_string(),
|
||||||
|
secret_key: "secret".to_string(),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let backend = WarmBackendS3::new(&conf, "tier")
|
||||||
|
.await
|
||||||
|
.expect("a well-formed S3 endpoint should initialize without network I/O");
|
||||||
|
assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com"));
|
||||||
|
assert_eq!(backend.client.endpoint_url.port(), Some(9443));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn overflowing_get_range_is_rejected_before_network_io() {
|
||||||
|
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||||
|
Ok(listener) => listener,
|
||||||
|
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
|
||||||
|
Err(err) => panic!("test listener should bind: {err}"),
|
||||||
|
};
|
||||||
|
let endpoint = listener
|
||||||
|
.local_addr()
|
||||||
|
.expect("listener local address should be available")
|
||||||
|
.to_string();
|
||||||
|
let client = Arc::new(
|
||||||
|
TransitionClient::new(
|
||||||
|
&endpoint,
|
||||||
|
Options {
|
||||||
|
creds: Credentials::new(Static(Value {
|
||||||
|
access_key_id: "access-key".to_string(),
|
||||||
|
secret_access_key: "secret-key".to_string(),
|
||||||
|
signer_type: SignatureType::SignatureV4,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
bucket_lookup: BucketLookupType::BucketLookupPath,
|
||||||
|
max_retries: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
"s3",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fixture client should build"),
|
||||||
|
);
|
||||||
|
let backend = WarmBackendS3 {
|
||||||
|
core: TransitionCore(Arc::clone(&client)),
|
||||||
|
client,
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
prefix: String::new(),
|
||||||
|
storage_class: String::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let err = backend
|
||||||
|
.get_with_headers(
|
||||||
|
"probe",
|
||||||
|
"",
|
||||||
|
WarmBackendGetOpts {
|
||||||
|
start_offset: i64::MAX,
|
||||||
|
length: 2,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("an overflowing range must fail before issuing a GET");
|
||||||
|
|
||||||
|
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(100), listener.accept())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||||
|
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||||
|
Ok(listener) => listener,
|
||||||
|
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||||
|
Err(err) => panic!("test listener should bind: {err}"),
|
||||||
|
};
|
||||||
|
let endpoint = listener
|
||||||
|
.local_addr()
|
||||||
|
.expect("listener local address should be available")
|
||||||
|
.to_string();
|
||||||
|
let fixture = tokio::spawn(async move {
|
||||||
|
let responses = [
|
||||||
|
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
|
||||||
|
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
|
||||||
|
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||||
|
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
|
||||||
|
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
|
||||||
|
];
|
||||||
|
let mut requests = Vec::new();
|
||||||
|
for response in responses {
|
||||||
|
let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
|
||||||
|
let mut request = Vec::new();
|
||||||
|
let mut buffer = [0; 1024];
|
||||||
|
loop {
|
||||||
|
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
|
||||||
|
assert_ne!(read, 0, "connection closed before request headers were received");
|
||||||
|
request.extend_from_slice(&buffer[..read]);
|
||||||
|
if request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
requests.push(String::from_utf8_lossy(&request).into_owned());
|
||||||
|
stream
|
||||||
|
.write_all(response.as_bytes())
|
||||||
|
.await
|
||||||
|
.expect("fixture should write candidate response");
|
||||||
|
}
|
||||||
|
requests
|
||||||
|
});
|
||||||
|
let client = Arc::new(
|
||||||
|
TransitionClient::new(
|
||||||
|
&endpoint,
|
||||||
|
Options {
|
||||||
|
creds: Credentials::new(Static(Value {
|
||||||
|
access_key_id: "access-key".to_string(),
|
||||||
|
secret_access_key: "secret-key".to_string(),
|
||||||
|
signer_type: SignatureType::SignatureV4,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
bucket_lookup: BucketLookupType::BucketLookupPath,
|
||||||
|
max_retries: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
"s3",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fixture client should build"),
|
||||||
|
);
|
||||||
|
Some((
|
||||||
|
WarmBackendS3 {
|
||||||
|
core: TransitionCore(Arc::clone(&client)),
|
||||||
|
client,
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
prefix: String::new(),
|
||||||
|
storage_class: String::new(),
|
||||||
|
},
|
||||||
|
fixture,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_probe_uses_only_exact_bounded_get_permissions() {
|
||||||
|
let Some((backend, fixture)) = candidate_probe_fixture().await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
backend
|
||||||
|
.probe_transition_candidate("versioned-probe")
|
||||||
|
.await
|
||||||
|
.expect("versioned candidate should be discovered"),
|
||||||
|
TransitionCandidateProbe::VersionedPresent("opaque-version".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend
|
||||||
|
.probe_transition_candidate("unversioned-probe")
|
||||||
|
.await
|
||||||
|
.expect("unversioned candidate should be discovered"),
|
||||||
|
TransitionCandidateProbe::UnversionedPresent
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend
|
||||||
|
.probe_transition_candidate("missing-probe")
|
||||||
|
.await
|
||||||
|
.expect("a missing key should be classified"),
|
||||||
|
TransitionCandidateProbe::Missing
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend
|
||||||
|
.probe_transition_candidate("provider-missing-probe")
|
||||||
|
.await
|
||||||
|
.expect("a provider-specific missing code should be classified"),
|
||||||
|
TransitionCandidateProbe::Missing
|
||||||
|
);
|
||||||
|
let err = backend
|
||||||
|
.probe_transition_candidate("forbidden-probe")
|
||||||
|
.await
|
||||||
|
.expect_err("an authorization failure must not be mistaken for a missing key");
|
||||||
|
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
|
||||||
|
|
||||||
|
let requests = fixture.await.expect("candidate fixture should join");
|
||||||
|
for request in requests {
|
||||||
|
let request = request.to_ascii_lowercase();
|
||||||
|
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
|
||||||
|
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
|
||||||
|
assert!(!request.contains("?versioning"));
|
||||||
|
assert!(!request.contains("?versions"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
|
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
|
||||||
ListVersionsResult {
|
ListVersionsResult {
|
||||||
versions: versions
|
versions: versions
|
||||||
@@ -631,8 +866,7 @@ impl WarmBackend for WarmBackendS3 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
self.probe_current_transition_candidate_with_header(object, None).await
|
||||||
self.probe_transition_candidate_versions(object, bucket_versioning).await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use std::collections::HashMap;
|
|||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier_config::TierTencent,
|
tier_config::TierTencent,
|
||||||
warm_backend::{
|
warm_backend::{
|
||||||
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||||
new_s3_compatible_warm_backend, optimal_part_size,
|
new_s3_compatible_warm_backend, optimal_part_size,
|
||||||
},
|
},
|
||||||
warm_backend_s3::WarmBackendS3,
|
warm_backend_s3::WarmBackendS3,
|
||||||
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendTencent {
|
|||||||
self.0.remove(object, rv).await
|
self.0.remove(object, rv).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||||
|
self.0.probe_transition_candidate(object).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
self.0.in_use().await
|
self.0.in_use().await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ use uuid::Uuid;
|
|||||||
|
|
||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier_config::{TierS3, TierWasabi},
|
tier_config::{TierS3, TierWasabi},
|
||||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
|
||||||
warm_backend_s3::WarmBackendS3,
|
warm_backend_s3::WarmBackendS3,
|
||||||
};
|
};
|
||||||
use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
|
use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
|
||||||
@@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi {
|
|||||||
self.s3.remove(object, rv).await
|
self.s3.remove(object, rv).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
|
||||||
|
self.s3.probe_transition_candidate(object).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn in_use(&self) -> io::Result<bool> {
|
async fn in_use(&self) -> io::Result<bool> {
|
||||||
self.check_remote_bucket_unversioned().await?;
|
self.check_remote_bucket_unversioned().await?;
|
||||||
let in_use = self.s3.in_use().await?;
|
let in_use = self.s3.in_use().await?;
|
||||||
|
|||||||
@@ -57,7 +57,11 @@ fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
|
|||||||
where
|
where
|
||||||
D: Deserializer<'de>,
|
D: Deserializer<'de>,
|
||||||
{
|
{
|
||||||
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
|
let code = String::deserialize(d)?;
|
||||||
|
if code == "NoSuchObject" {
|
||||||
|
return Ok(S3ErrorCode::NoSuchKey);
|
||||||
|
}
|
||||||
|
Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ErrorResponse {
|
impl Default for ErrorResponse {
|
||||||
@@ -325,4 +329,21 @@ mod tests {
|
|||||||
assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
|
assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
|
||||||
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
|
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalizes_provider_specific_missing_object_code() {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
|
||||||
|
|
||||||
|
let response = http_resp_to_error_response(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
&headers,
|
||||||
|
b"<Error><Code>NoSuchObject</Code><Message>remote detail</Message></Error>".to_vec(),
|
||||||
|
"bucket",
|
||||||
|
"object",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(response.code, S3ErrorCode::NoSuchKey);
|
||||||
|
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,9 @@ use tokio_util::io::StreamReader;
|
|||||||
use crate::{
|
use crate::{
|
||||||
api_error_response::err_invalid_argument,
|
api_error_response::err_invalid_argument,
|
||||||
api_get_options::GetObjectOptions,
|
api_get_options::GetObjectOptions,
|
||||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
|
transition_api::{
|
||||||
|
ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
use futures_util::StreamExt;
|
use futures_util::StreamExt;
|
||||||
use http_body_util::BodyExt;
|
use http_body_util::BodyExt;
|
||||||
@@ -39,6 +41,42 @@ use hyper::body::Bytes;
|
|||||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||||
use tokio_util::io::ReaderStream;
|
use tokio_util::io::ReaderStream;
|
||||||
|
|
||||||
|
fn response_limit_from_range(opts: &GetObjectOptions) -> Result<Option<usize>, std::io::Error> {
|
||||||
|
let Some(range) = opts
|
||||||
|
.headers
|
||||||
|
.iter()
|
||||||
|
.find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str()))
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let Some((unit, bounds)) = range.split_once('=') else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
if !unit.eq_ignore_ascii_case("bytes") {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let Some((start, end)) = bounds.split_once('-') else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
if start.is_empty() || end.is_empty() || end.contains(',') {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let start = start
|
||||||
|
.parse::<u64>()
|
||||||
|
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?;
|
||||||
|
let end = end
|
||||||
|
.parse::<u64>()
|
||||||
|
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?;
|
||||||
|
let length = end
|
||||||
|
.checked_sub(start)
|
||||||
|
.and_then(|length| length.checked_add(1))
|
||||||
|
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?;
|
||||||
|
let limit = usize::try_from(length).map_err(|_| {
|
||||||
|
std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory")
|
||||||
|
})?;
|
||||||
|
Ok(Some(limit))
|
||||||
|
}
|
||||||
|
|
||||||
impl TransitionClient {
|
impl TransitionClient {
|
||||||
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
|
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
|
||||||
let _ = opts;
|
let _ = opts;
|
||||||
@@ -54,6 +92,7 @@ impl TransitionClient {
|
|||||||
object_name: &str,
|
object_name: &str,
|
||||||
opts: &GetObjectOptions,
|
opts: &GetObjectOptions,
|
||||||
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
|
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
|
||||||
|
let max_response_bytes = response_limit_from_range(opts)?;
|
||||||
let resp = self
|
let resp = self
|
||||||
.execute_method(
|
.execute_method(
|
||||||
http::Method::GET,
|
http::Method::GET,
|
||||||
@@ -81,18 +120,214 @@ impl TransitionClient {
|
|||||||
|
|
||||||
let h = resp.headers().clone();
|
let h = resp.headers().clone();
|
||||||
|
|
||||||
let mut body_vec = Vec::new();
|
|
||||||
let mut body = resp.into_body();
|
let mut body = resp.into_body();
|
||||||
|
let body_vec = if let Some(limit) = max_response_bytes {
|
||||||
|
collect_response_body(body, limit).await?
|
||||||
|
} else {
|
||||||
|
let mut body_vec = Vec::new();
|
||||||
while let Some(frame) = body.frame().await {
|
while let Some(frame) = body.frame().await {
|
||||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||||
if let Some(data) = frame.data_ref() {
|
if let Some(data) = frame.data_ref() {
|
||||||
body_vec.extend_from_slice(data);
|
body_vec.extend_from_slice(data);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
body_vec
|
||||||
|
};
|
||||||
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
|
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod bounded_response_tests {
|
||||||
|
use super::response_limit_from_range;
|
||||||
|
use crate::{
|
||||||
|
api_get_options::GetObjectOptions,
|
||||||
|
credentials::{Credentials, SignatureType, Static, Value},
|
||||||
|
transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
|
||||||
|
};
|
||||||
|
use http_body_util::Full;
|
||||||
|
use hyper::body::Bytes;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::{
|
||||||
|
io::{AsyncReadExt, AsyncWriteExt},
|
||||||
|
net::TcpListener,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn closed_range_derives_a_collection_limit_without_new_public_options() {
|
||||||
|
let mut opts = GetObjectOptions::default();
|
||||||
|
opts.set_range(5, 11).expect("the closed range should be valid");
|
||||||
|
|
||||||
|
assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() {
|
||||||
|
let mut opts = GetObjectOptions::default();
|
||||||
|
opts.set_range(0, 6).expect("the probe range should be valid");
|
||||||
|
let max_response_bytes = response_limit_from_range(&opts)
|
||||||
|
.expect("the range should parse")
|
||||||
|
.expect("the closed range should have a limit");
|
||||||
|
let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes)
|
||||||
|
.await
|
||||||
|
.expect_err("the collection layer must reject a response larger than its limit");
|
||||||
|
|
||||||
|
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
|
||||||
|
let listener = match TcpListener::bind("127.0.0.1:0").await {
|
||||||
|
Ok(listener) => listener,
|
||||||
|
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||||
|
Err(err) => panic!("test listener should bind: {err}"),
|
||||||
|
};
|
||||||
|
let endpoint = listener
|
||||||
|
.local_addr()
|
||||||
|
.expect("listener local address should be available")
|
||||||
|
.to_string();
|
||||||
|
let request = tokio::spawn(async move {
|
||||||
|
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
|
||||||
|
let mut request = Vec::new();
|
||||||
|
let mut buffer = [0; 1024];
|
||||||
|
loop {
|
||||||
|
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
|
||||||
|
assert_ne!(read, 0, "connection closed before request headers were received");
|
||||||
|
request.extend_from_slice(&buffer[..read]);
|
||||||
|
if request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let request = String::from_utf8_lossy(&request).into_owned();
|
||||||
|
let response = format!(
|
||||||
|
"HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
body.len()
|
||||||
|
);
|
||||||
|
stream
|
||||||
|
.write_all(response.as_bytes())
|
||||||
|
.await
|
||||||
|
.expect("fixture should write response headers");
|
||||||
|
stream.write_all(body).await.expect("fixture should write response body");
|
||||||
|
request
|
||||||
|
});
|
||||||
|
let client = TransitionClient::new(
|
||||||
|
&endpoint,
|
||||||
|
Options {
|
||||||
|
creds: Credentials::new(Static(Value {
|
||||||
|
access_key_id: "access-key".to_string(),
|
||||||
|
secret_access_key: "secret-key".to_string(),
|
||||||
|
signer_type: SignatureType::SignatureV4,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
bucket_lookup: BucketLookupType::BucketLookupPath,
|
||||||
|
max_retries: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fixture client should build");
|
||||||
|
Some((client, request))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn real_transport_accepts_the_exact_closed_range_length() {
|
||||||
|
let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let mut opts = GetObjectOptions::default();
|
||||||
|
opts.set_range(0, 6).expect("the probe range should be valid");
|
||||||
|
|
||||||
|
let (_, _, mut reader) = client
|
||||||
|
.get_object_inner("bucket", "probe", &opts)
|
||||||
|
.await
|
||||||
|
.expect("a seven-byte response should fit the requested range");
|
||||||
|
let mut body = Vec::new();
|
||||||
|
reader
|
||||||
|
.read_to_end(&mut body)
|
||||||
|
.await
|
||||||
|
.expect("bounded response should be readable");
|
||||||
|
|
||||||
|
assert_eq!(body, b"RustFS!");
|
||||||
|
assert!(
|
||||||
|
request
|
||||||
|
.await
|
||||||
|
.expect("fixture should join")
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.contains("\r\nrange: bytes=0-6\r\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn real_transport_rejects_a_body_larger_than_the_closed_range() {
|
||||||
|
let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let mut opts = GetObjectOptions::default();
|
||||||
|
opts.set_range(0, 6).expect("the probe range should be valid");
|
||||||
|
|
||||||
|
let err = client
|
||||||
|
.get_object_inner("bucket", "probe", &opts)
|
||||||
|
.await
|
||||||
|
.expect_err("an eight-byte response must exceed the seven-byte range limit");
|
||||||
|
|
||||||
|
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||||
|
assert!(
|
||||||
|
request
|
||||||
|
.await
|
||||||
|
.expect("fixture should join")
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.contains("\r\nrange: bytes=0-6\r\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn overflowing_closed_range_is_rejected_before_network_io() {
|
||||||
|
let listener = match TcpListener::bind("127.0.0.1:0").await {
|
||||||
|
Ok(listener) => listener,
|
||||||
|
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
|
||||||
|
Err(err) => panic!("test listener should bind: {err}"),
|
||||||
|
};
|
||||||
|
let endpoint = listener
|
||||||
|
.local_addr()
|
||||||
|
.expect("listener local address should be available")
|
||||||
|
.to_string();
|
||||||
|
let client = TransitionClient::new(
|
||||||
|
&endpoint,
|
||||||
|
Options {
|
||||||
|
creds: Credentials::new(Static(Value {
|
||||||
|
access_key_id: "access-key".to_string(),
|
||||||
|
secret_access_key: "secret-key".to_string(),
|
||||||
|
signer_type: SignatureType::SignatureV4,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
bucket_lookup: BucketLookupType::BucketLookupPath,
|
||||||
|
max_retries: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("fixture client should build");
|
||||||
|
let mut opts = GetObjectOptions::default();
|
||||||
|
opts.headers
|
||||||
|
.insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
|
||||||
|
|
||||||
|
let err = client
|
||||||
|
.get_object_inner("bucket", "probe", &opts)
|
||||||
|
.await
|
||||||
|
.expect_err("an overflowing closed range must be rejected locally");
|
||||||
|
|
||||||
|
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(100), listener.accept())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub struct GetRequest {
|
pub struct GetRequest {
|
||||||
pub buffer: Vec<u8>,
|
pub buffer: Vec<u8>,
|
||||||
|
|||||||
@@ -80,6 +80,82 @@ fn wasabi_payload_name(config: &TierConfig) -> S3Result<String> {
|
|||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration"))
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn normalize_add_tier_payload_name(config: &mut TierConfig) -> S3Result<()> {
|
||||||
|
match config.tier_type {
|
||||||
|
TierType::S3 => {
|
||||||
|
let _ = config
|
||||||
|
.s3
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?;
|
||||||
|
}
|
||||||
|
TierType::Wasabi => config.name = wasabi_payload_name(config)?,
|
||||||
|
TierType::RustFS => {
|
||||||
|
config.name = config
|
||||||
|
.rustfs
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::MinIO => {
|
||||||
|
config.name = config
|
||||||
|
.minio
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::Aliyun => {
|
||||||
|
config.name = config
|
||||||
|
.aliyun
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::Tencent => {
|
||||||
|
config.name = config
|
||||||
|
.tencent
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::Huaweicloud => {
|
||||||
|
config.name = config
|
||||||
|
.huaweicloud
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::Azure => {
|
||||||
|
config.name = config
|
||||||
|
.azure
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::GCS => {
|
||||||
|
let _ = config
|
||||||
|
.gcs
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?;
|
||||||
|
}
|
||||||
|
TierType::R2 => {
|
||||||
|
config.name = config
|
||||||
|
.r2
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
|
||||||
|
.name
|
||||||
|
.clone();
|
||||||
|
}
|
||||||
|
TierType::Unsupported => {}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn spawn_transition_tier_config_propagation(action: &'static str) {
|
fn spawn_transition_tier_config_propagation(action: &'static str) {
|
||||||
if let Some(notification_sys) = current_notification_system() {
|
if let Some(notification_sys) = current_notification_system() {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -263,75 +339,7 @@ impl Operation for AddTier {
|
|||||||
let mut args: TierConfig = serde_json::from_slice(&body)
|
let mut args: TierConfig = serde_json::from_slice(&body)
|
||||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
|
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
|
||||||
|
|
||||||
match args.tier_type {
|
normalize_add_tier_payload_name(&mut args)?;
|
||||||
TierType::S3 => {
|
|
||||||
args.name = args
|
|
||||||
.s3
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::Wasabi => {
|
|
||||||
args.name = wasabi_payload_name(&args)?;
|
|
||||||
}
|
|
||||||
TierType::RustFS => {
|
|
||||||
args.name = args
|
|
||||||
.rustfs
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::MinIO => {
|
|
||||||
args.name = args
|
|
||||||
.minio
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::Aliyun => {
|
|
||||||
args.name = args
|
|
||||||
.aliyun
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::Tencent => {
|
|
||||||
args.name = args
|
|
||||||
.tencent
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::Huaweicloud => {
|
|
||||||
args.name = args
|
|
||||||
.huaweicloud
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::Azure => {
|
|
||||||
args.name = args
|
|
||||||
.azure
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::GCS => {
|
|
||||||
args.name = args
|
|
||||||
.gcs
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
TierType::R2 => {
|
|
||||||
args.name = args
|
|
||||||
.r2
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
|
|
||||||
.name;
|
|
||||||
}
|
|
||||||
_ => (),
|
|
||||||
}
|
|
||||||
debug!(
|
debug!(
|
||||||
event = EVENT_ADMIN_TIER_STATE,
|
event = EVENT_ADMIN_TIER_STATE,
|
||||||
component = LOG_COMPONENT_ADMIN,
|
component = LOG_COMPONENT_ADMIN,
|
||||||
@@ -1150,6 +1158,44 @@ mod tests {
|
|||||||
assert_eq!(err.message(), Some("missing Wasabi configuration"));
|
assert_eq!(err.message(), Some("missing Wasabi configuration"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn add_tier_payload_preserves_canonical_madmin_s3_and_gcs_names() {
|
||||||
|
for (provider, wire) in [
|
||||||
|
(
|
||||||
|
"S3",
|
||||||
|
serde_json::json!({
|
||||||
|
"Type": "s3",
|
||||||
|
"Name": "COLD-S3",
|
||||||
|
"S3": {
|
||||||
|
"Endpoint": "https://s3.example.invalid",
|
||||||
|
"AccessKey": "access",
|
||||||
|
"SecretKey": "secret",
|
||||||
|
"Bucket": "archive"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"GCS",
|
||||||
|
serde_json::json!({
|
||||||
|
"Type": "gcs",
|
||||||
|
"Name": "COLD-GCS",
|
||||||
|
"GCS": {
|
||||||
|
"Endpoint": "https://storage.googleapis.com",
|
||||||
|
"Creds": "e30=",
|
||||||
|
"Bucket": "archive"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let mut config: TierConfig = serde_json::from_value(wire).expect("canonical madmin payload should decode");
|
||||||
|
let expected = config.name.clone();
|
||||||
|
|
||||||
|
normalize_add_tier_payload_name(&mut config).expect("canonical madmin payload should pass the handler boundary");
|
||||||
|
|
||||||
|
assert_eq!(config.name, expected, "{provider} top-level Name must not be cleared");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_tier_name_prefers_path_parameter() {
|
fn resolve_tier_name_prefers_path_parameter() {
|
||||||
let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse");
|
let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse");
|
||||||
@@ -1756,5 +1802,11 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert!(!production.contains("check_key_valid(get_session_token"));
|
assert!(!production.contains("check_key_valid(get_session_token"));
|
||||||
|
|
||||||
|
let add_tier = source_block(production, "impl Operation for AddTier");
|
||||||
|
assert!(
|
||||||
|
add_tier.contains("normalize_add_tier_payload_name(&mut args)?;"),
|
||||||
|
"AddTier must preserve canonical top-level provider names through the tested boundary helper"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user