mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 13:06:00 +00:00
refactor(odm): move migration orchestration into application (#7226)
* refactor(odm): move migration orchestration into application * style(odm): format relocated listing test imports
This commit is contained in:
Zhengchao An
+4
-1
@@ -58,7 +58,7 @@ required-features = ["swift"]
|
||||
|
||||
[features]
|
||||
default = ["ftps", "webdav", "gcs"]
|
||||
gcs = ["rustfs-ecstore/gcs"]
|
||||
gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
|
||||
metrics-gpu = ["rustfs-obs/gpu"]
|
||||
ftps = ["rustfs-protocols/ftps"]
|
||||
swift = ["rustfs-protocols/swift"]
|
||||
@@ -288,6 +288,9 @@ reqwest = { workspace = true, features = ["json", "stream"] }
|
||||
socket2 = { workspace = true, features = ["all"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util", "fs"] }
|
||||
tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
|
||||
aws-smithy-types = { workspace = true }
|
||||
google-cloud-auth = { workspace = true, optional = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
tokio-stream.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "time"] }
|
||||
|
||||
@@ -38,16 +38,6 @@ use crate::admin::runtime_sources::{
|
||||
};
|
||||
use crate::admin::storage_api::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
|
||||
use crate::admin::storage_api::bucket::metadata_sys;
|
||||
use crate::admin::storage_api::bucket::on_demand_migration::backfill::{
|
||||
BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::on_demand_migration::{
|
||||
OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
|
||||
source_backend_spec,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::remote_s3_client::{
|
||||
PathStyle as RemotePathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
|
||||
};
|
||||
@@ -58,6 +48,16 @@ use crate::admin::utils::{extract_query_params, read_compatible_admin_body};
|
||||
use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
use crate::module_switches::{ENV_ON_DEMAND_MIGRATION_ENABLED, on_demand_migration_enabled_from_env};
|
||||
use crate::on_demand_migration::backfill::{
|
||||
BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
|
||||
};
|
||||
use crate::on_demand_migration::source_client::{
|
||||
SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
|
||||
};
|
||||
use crate::on_demand_migration::{
|
||||
OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
|
||||
source_backend_spec,
|
||||
};
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
@@ -580,7 +580,7 @@ async fn validate_config(bucket: &str, config: &OnDemandMigrationConfig) -> S3Re
|
||||
}
|
||||
|
||||
fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider {
|
||||
use crate::admin::storage_api::bucket::on_demand_migration::Provider;
|
||||
use crate::on_demand_migration::Provider;
|
||||
match config.source.provider {
|
||||
Provider::S3 => SourceProvider::S3,
|
||||
Provider::Aws => SourceProvider::Aws,
|
||||
@@ -787,7 +787,7 @@ impl Operation for GetBucketOnDemandMigrationHandler {
|
||||
let bucket = bucket_from_params(¶ms)?;
|
||||
let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;
|
||||
|
||||
let Some((config, updated_at)) = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
|
||||
let Some((config, updated_at)) = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
|
||||
admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
|
||||
})?
|
||||
else {
|
||||
@@ -846,7 +846,7 @@ impl Operation for GetBucketOnDemandMigrationStatusHandler {
|
||||
let bucket = bucket_from_params(¶ms)?;
|
||||
let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;
|
||||
|
||||
let config = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
|
||||
let config = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
|
||||
admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
|
||||
})?;
|
||||
let runtime = OnDemandMigrationSys::get().bucket_snapshot(&bucket);
|
||||
@@ -1763,6 +1763,19 @@ mod store_tests {
|
||||
assert_eq!(body["inflight_pulls"], Value::from(0));
|
||||
assert_eq!(body["queue_depth"], Value::from(0));
|
||||
|
||||
// A malformed replacement cannot overwrite the saved source.
|
||||
let before = metadata_sys::get(BUCKET).await.expect("saved metadata");
|
||||
for invalid in [b"not-json".to_vec(), br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec()] {
|
||||
let err = SetBucketOnDemandMigrationHandler {}
|
||||
.call(root_request(Method::PUT, config_uri(""), invalid), bucket_params(&router))
|
||||
.await
|
||||
.expect_err("malformed replacement must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
let after = metadata_sys::get(BUCKET).await.expect("saved metadata remains readable");
|
||||
assert_eq!(after.on_demand_migration_config_json, before.on_demand_migration_config_json);
|
||||
assert_eq!(after.on_demand_migration_config_updated_at, before.on_demand_migration_config_updated_at);
|
||||
}
|
||||
|
||||
// The peer fan-out ran: the single unreachable peer is reported.
|
||||
let context = crate::admin::runtime_sources::current_app_context();
|
||||
let err = reload_peers(context.as_deref(), BUCKET)
|
||||
|
||||
@@ -20,8 +20,8 @@ use time::OffsetDateTime;
|
||||
|
||||
mod ecstore_bucket {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::{
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, on_demand_migration, quota,
|
||||
remote_s3_client, replication, target, utils, versioning, versioning_sys,
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, quota, remote_s3_client,
|
||||
replication, target, utils, versioning, versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -284,35 +284,6 @@ pub(crate) mod durability {
|
||||
pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
|
||||
}
|
||||
|
||||
pub(crate) mod on_demand_migration {
|
||||
pub(crate) type OdmBucketSnapshot = super::ecstore_bucket::on_demand_migration::OdmBucketSnapshot;
|
||||
pub(crate) type OnDemandMigrationConfig = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfig;
|
||||
pub(crate) type OnDemandMigrationConfigError = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfigError;
|
||||
pub(crate) type OnDemandMigrationSys = super::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
|
||||
pub(crate) type PathStyle = super::ecstore_bucket::on_demand_migration::PathStyle;
|
||||
pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider;
|
||||
pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>;
|
||||
pub(crate) use super::ecstore_bucket::on_demand_migration::source_backend_spec;
|
||||
|
||||
pub(crate) mod backfill {
|
||||
pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint;
|
||||
pub(crate) type BackfillError = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillError;
|
||||
pub(crate) type BackfillRequest = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillRequest;
|
||||
pub(crate) type BackfillState = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillState;
|
||||
pub(crate) type SkipExisting = super::super::ecstore_bucket::on_demand_migration::backfill::SkipExisting;
|
||||
pub(crate) use super::super::ecstore_bucket::on_demand_migration::backfill::global_backfill_runner;
|
||||
}
|
||||
|
||||
pub(crate) mod source_client {
|
||||
pub(crate) type SourceClient = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClient;
|
||||
pub(crate) type SourceClientSpec = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClientSpec;
|
||||
pub(crate) type SourceError = super::super::ecstore_bucket::on_demand_migration::source_client::SourceError;
|
||||
pub(crate) type SourceProbe = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProbe;
|
||||
pub(crate) type SourceProvider = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProvider;
|
||||
pub(crate) type SourceTimeouts = super::super::ecstore_bucket::on_demand_migration::source_client::SourceTimeouts;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod remote_s3_client {
|
||||
pub(crate) type PathStyle = super::ecstore_bucket::remote_s3_client::PathStyle;
|
||||
pub(crate) type RemoteCredentials = super::ecstore_bucket::remote_s3_client::RemoteCredentials;
|
||||
@@ -455,12 +426,6 @@ pub(crate) mod metadata_sys {
|
||||
super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_on_demand_migration_config(
|
||||
bucket: &str,
|
||||
) -> Result<Option<(super::on_demand_migration::OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
super::ecstore_bucket::metadata_sys::get_on_demand_migration_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
|
||||
}
|
||||
@@ -913,7 +878,6 @@ pub(crate) mod bucket {
|
||||
pub(crate) use super::lifecycle;
|
||||
pub(crate) use super::metadata;
|
||||
pub(crate) use super::metadata_sys;
|
||||
pub(crate) use super::on_demand_migration;
|
||||
pub(crate) use super::quota;
|
||||
pub(crate) use super::remote_s3_client;
|
||||
pub(crate) use super::replication;
|
||||
|
||||
@@ -25,11 +25,6 @@
|
||||
use super::storage_api::bucket_usecase::ECStore;
|
||||
use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
|
||||
use super::storage_api::bucket_usecase::StorageObjectOptions;
|
||||
use super::storage_api::bucket_usecase::bucket::on_demand_migration::{
|
||||
BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
|
||||
MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
|
||||
SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
|
||||
use super::storage_api::bucket_usecase::contract::list::{ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _};
|
||||
use super::storage_api::bucket_usecase::contract::object::ObjectOperations as _;
|
||||
@@ -37,6 +32,11 @@ use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Result};
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params;
|
||||
use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class};
|
||||
use crate::error::ApiError;
|
||||
use crate::on_demand_migration::{
|
||||
BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
|
||||
MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
|
||||
SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header};
|
||||
@@ -443,14 +443,14 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
|
||||
use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
|
||||
FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
|
||||
SourceCredentials, TlsConfig,
|
||||
};
|
||||
use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
|
||||
use crate::app::storage_api::test::StoragePutObjReader;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::contract::object::ObjectIO as _;
|
||||
use crate::on_demand_migration::{
|
||||
FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
|
||||
SourceCredentials, TlsConfig,
|
||||
};
|
||||
use s3s::dto::ListObjectsInput;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
//! GetObject / GetObjectAttributes read path: cold fill, resume, stream tuning.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
|
||||
use crate::on_demand_migration::WriteBackBody;
|
||||
use crate::on_demand_migration::{
|
||||
BucketOdmState, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, PullError, PullLeader, PullOutcome, PullReason, PullSlot,
|
||||
RangeGetPolicy, SourceBody, SourceClient, SourceError, SourceGet, SourceHead, commit_inline, idle_guarded_body,
|
||||
};
|
||||
use crate::app::storage_api::object_usecase::on_demand_migration::WriteBackBody;
|
||||
use rustfs_rio::{TeeOptions, TeePrimary, tee_reader_with_options};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
@@ -4798,11 +4798,11 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
|
||||
#[cfg(test)]
|
||||
mod on_demand_migration_tests {
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
|
||||
use crate::on_demand_migration::{
|
||||
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
|
||||
Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
|
||||
};
|
||||
use crate::app::storage_api::object_usecase::on_demand_migration::{
|
||||
use crate::on_demand_migration::{
|
||||
LocalObject, OdmWriteBack, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! HeadObject path.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
|
||||
use crate::on_demand_migration::{
|
||||
BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, SourceClient, SourceError, SourceHead,
|
||||
};
|
||||
|
||||
@@ -638,7 +638,7 @@ impl DefaultObjectUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
|
||||
use crate::on_demand_migration::{
|
||||
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
|
||||
Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
|
||||
};
|
||||
@@ -739,11 +739,9 @@ mod tests {
|
||||
user_metadata: HashMap::from([("owner".to_string(), "alice".to_string())]),
|
||||
version_id: Some("v1".to_string()),
|
||||
storage_class: Some("STANDARD_IA".to_string()),
|
||||
sse: Some(
|
||||
crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::SourceSse::Kms {
|
||||
key_id: Some("key-1".to_string()),
|
||||
},
|
||||
),
|
||||
sse: Some(crate::on_demand_migration::source_client::SourceSse::Kms {
|
||||
key_id: Some("key-1".to_string()),
|
||||
}),
|
||||
is_multipart_etag: true,
|
||||
etag_is_opaque: false,
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! On-demand migration write-back (rustfs/backlog#2153): the app-layer
|
||||
//! [`OdmWriteBack`] the ecstore pull pipeline stores source objects with.
|
||||
//! [`OdmWriteBack`] the migration service stores source objects with.
|
||||
//!
|
||||
//! Every write goes through the internal put entry points, so a pulled
|
||||
//! object is indistinguishable from a client PUT: bucket default SSE, quota,
|
||||
@@ -34,7 +34,7 @@
|
||||
use super::*;
|
||||
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
|
||||
use crate::app::storage_api::object_usecase::on_demand_migration::{
|
||||
use crate::on_demand_migration::{
|
||||
LocalObject, OdmWriteBack, SourceHead, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
|
||||
is_multipart_etag,
|
||||
};
|
||||
@@ -297,7 +297,6 @@ impl OdmWriteBack for OnDemandMigrationWriteBack {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::MultipartOperations as _;
|
||||
use crate::app::storage_api::object_usecase::on_demand_migration::{PullFailureReason, SourceSse};
|
||||
use crate::app::storage_api::s3::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
|
||||
@@ -306,6 +305,7 @@ mod tests {
|
||||
use crate::app::storage_api::test::bucket::utils::serialize;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
use crate::on_demand_migration::{PullFailureReason, SourceSse};
|
||||
use http::Method;
|
||||
use rustfs_utils::http::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, contains_key_str, get_str};
|
||||
use sha2::{Digest as Sha256Digest, Sha256};
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
//! Cross-cutting helpers shared by the object use-case modules.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
|
||||
OdmStateError, PolicyConfig, SourceErrorPolicy, SourceHead,
|
||||
};
|
||||
use crate::on_demand_migration::{OdmStateError, PolicyConfig, SourceErrorPolicy, SourceHead};
|
||||
|
||||
pub(super) const RUSTFS_EXPECTED_CURRENT_VERSION_ID: &str = "x-rustfs-expected-current-version-id";
|
||||
|
||||
|
||||
@@ -628,26 +628,6 @@ pub(crate) mod bucket {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod on_demand_migration {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
|
||||
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig,
|
||||
PathStyle, Provider, SourceConfig, SourceCredentials, TlsConfig,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
|
||||
BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OdmStateError, OnDemandMigrationSys, PolicyConfig,
|
||||
PullError, PullLeader, PullOutcome, PullReason, PullSlot, RangeGetPolicy, SourceBody, SourceErrorPolicy,
|
||||
commit_inline, idle_guarded_body,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
|
||||
ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
|
||||
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod policy_sys {
|
||||
pub(crate) type PolicySys = crate::storage::storage_api::ecstore_bucket::policy_sys::PolicySys;
|
||||
}
|
||||
@@ -1181,19 +1161,6 @@ pub(crate) mod bucket_usecase {
|
||||
pub(crate) mod object_usecase {
|
||||
pub(crate) use super::storage_contracts::BUCKET_LIFECYCLE_LOCK_OBJECT;
|
||||
|
||||
pub(crate) mod on_demand_migration {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::PullFailureReason;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::SourceSse;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::{
|
||||
SourceHead, is_multipart_etag,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
|
||||
LocalObject, OdmWriteBack, WriteBackBody, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod object_cache {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::ecstore_object::GetObjectBodySource;
|
||||
|
||||
@@ -93,6 +93,7 @@ pub(crate) mod kms_rekey;
|
||||
pub mod license;
|
||||
pub mod memory_observability;
|
||||
pub mod module_switches;
|
||||
pub mod on_demand_migration;
|
||||
pub mod profiling;
|
||||
#[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
|
||||
pub mod protocols;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,172 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! One contract every [`SourceBackend`] implementation must satisfy.
|
||||
//!
|
||||
//! The migration pipeline talks to a source only through the trait, so a new
|
||||
//! provider is correct exactly when it answers the same questions the same way:
|
||||
//! the same head fields, the same range semantics, the same page shape, the
|
||||
//! same error classes. Each backend supplies a fixture that answers this fixed
|
||||
//! corpus in its own dialect and then runs [`assert_backend_contract`], so a
|
||||
//! provider-specific mapping bug shows up as a contract failure rather than as
|
||||
//! a surprise in the pull pipeline.
|
||||
//!
|
||||
//! Backends differ in two documented ways, declared through
|
||||
//! [`BackendCapabilities`]: whether the provider's ETag is a content digest,
|
||||
//! and whether the provider can resume a listing from a key.
|
||||
|
||||
use super::source_client::{SourceBackend, SourceError, SourceListRequest};
|
||||
use super::storage_api::HTTPRangeSpec;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// The single object every fixture serves.
|
||||
pub(super) const OBJECT_KEY: &str = "dir/a.txt";
|
||||
pub(super) const OBJECT_BODY: &[u8] = b"hello";
|
||||
/// MD5 of [`OBJECT_BODY`]; the ETag of the object on a digest provider.
|
||||
pub(super) const OBJECT_MD5: &str = "5d41402abc4b2a76b9719d911017c592";
|
||||
/// The second key the fixture's listing returns, on its second page.
|
||||
pub(super) const SECOND_KEY: &str = "dir/b.txt";
|
||||
pub(super) const COMMON_PREFIX: &str = "dir/sub/";
|
||||
pub(super) const LIST_CURSOR: &str = "cursor-1";
|
||||
/// A key the fixture answers with the provider's "no such object".
|
||||
pub(super) const MISSING_KEY: &str = "missing";
|
||||
/// A key the fixture answers with the provider's "not authorized".
|
||||
pub(super) const FORBIDDEN_KEY: &str = "secret";
|
||||
|
||||
/// Where backends are allowed to differ.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(super) struct BackendCapabilities {
|
||||
/// The provider's ETag is an opaque token, not a digest of the bytes.
|
||||
pub(super) etag_is_opaque: bool,
|
||||
/// The provider can resume a listing from a key rather than only from an
|
||||
/// opaque cursor.
|
||||
pub(super) supports_start_after: bool,
|
||||
/// The provider has an object-tagging concept at all. GCS does not, and
|
||||
/// answers with an empty map instead of failing a pull.
|
||||
pub(super) supports_tagging: bool,
|
||||
}
|
||||
|
||||
/// Drives `backend` through the shared corpus. Fixtures are scripted in
|
||||
/// request order, so the call order here is part of the contract.
|
||||
pub(super) async fn assert_backend_contract(backend: &dyn SourceBackend, caps: BackendCapabilities) {
|
||||
// 1. HEAD maps the object's shared fields.
|
||||
let head = backend.head(OBJECT_KEY).await.expect("HEAD of the fixture object");
|
||||
assert_eq!(head.size, OBJECT_BODY.len() as u64, "HEAD reports the object size");
|
||||
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
|
||||
assert_eq!(
|
||||
head.user_metadata,
|
||||
HashMap::from([("owner".to_string(), "alice".to_string())]),
|
||||
"user metadata is keyed without the provider prefix"
|
||||
);
|
||||
assert!(head.storage_class.is_some(), "the provider's tier is recorded");
|
||||
assert!(head.last_modified.is_some(), "the provider's timestamp is parsed");
|
||||
assert!(head.sse.is_none(), "the fixture object is not server-side encrypted");
|
||||
assert!(!head.is_multipart_etag);
|
||||
assert_eq!(head.etag_is_opaque, caps.etag_is_opaque);
|
||||
match caps.etag_is_opaque {
|
||||
false => assert_eq!(head.etag.as_deref(), Some(OBJECT_MD5), "a digest ETag is mapped verbatim"),
|
||||
true => assert!(head.etag.is_some(), "an opaque ETag is still recorded"),
|
||||
}
|
||||
|
||||
// 2. An unranged GET streams the whole object and reports no range.
|
||||
let got = backend.get(OBJECT_KEY, None).await.expect("unranged GET");
|
||||
assert_eq!(got.head.size, OBJECT_BODY.len() as u64);
|
||||
assert!(got.content_range.is_none(), "an unranged GET has no content-range");
|
||||
assert_eq!(got.head.etag_is_opaque, caps.etag_is_opaque, "GET and HEAD agree about the ETag");
|
||||
let body = got.body.collect().await.expect("body streams").into_bytes();
|
||||
assert_eq!(body.as_ref(), OBJECT_BODY);
|
||||
|
||||
// 3. A ranged GET returns exactly the requested interval, and `size` is
|
||||
// the length of the returned bytes rather than of the object.
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 1,
|
||||
end: 3,
|
||||
};
|
||||
let got = backend.get(OBJECT_KEY, Some(&range)).await.expect("ranged GET");
|
||||
assert_eq!(got.head.size, 3, "a ranged GET reports the range length");
|
||||
assert_eq!(got.content_range.as_deref(), Some("bytes 1-3/5"));
|
||||
let body = got.body.collect().await.expect("body streams").into_bytes();
|
||||
assert_eq!(body.as_ref(), &OBJECT_BODY[1..=3]);
|
||||
|
||||
// 4. A delimiter listing rolls prefixes up and hands back a cursor.
|
||||
let page = backend
|
||||
.list(&SourceListRequest {
|
||||
prefix: Some("dir/"),
|
||||
delimiter: Some("/"),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("first listing page");
|
||||
assert_eq!(page.objects.len(), 1, "the first page holds one object");
|
||||
assert_eq!(page.objects[0].key, OBJECT_KEY, "listing keys are in the source namespace");
|
||||
assert_eq!(page.objects[0].size, OBJECT_BODY.len() as u64);
|
||||
assert!(page.objects[0].last_modified.is_some());
|
||||
assert_eq!(page.common_prefixes, vec![COMMON_PREFIX.to_string()]);
|
||||
assert!(page.is_truncated);
|
||||
assert_eq!(page.next_continuation_token.as_deref(), Some(LIST_CURSOR));
|
||||
|
||||
// 5. The cursor is passed back verbatim and the last page ends the walk.
|
||||
let page = backend
|
||||
.list(&SourceListRequest {
|
||||
prefix: Some("dir/"),
|
||||
delimiter: Some("/"),
|
||||
continuation_token: Some(LIST_CURSOR),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("second listing page");
|
||||
assert_eq!(page.objects.len(), 1);
|
||||
assert_eq!(page.objects[0].key, SECOND_KEY);
|
||||
assert!(!page.is_truncated);
|
||||
assert!(page.next_continuation_token.is_none(), "a complete listing carries no cursor");
|
||||
|
||||
// 6. Tags come back as a flat map, empty on a provider without tags.
|
||||
let tags = backend.tagging(OBJECT_KEY).await.expect("object tags");
|
||||
match caps.supports_tagging {
|
||||
true => assert_eq!(tags, HashMap::from([("env".to_string(), "prod".to_string())])),
|
||||
false => assert!(tags.is_empty(), "a provider without tags reports none: {tags:?}"),
|
||||
}
|
||||
|
||||
// 7. The probe confirms the bucket or container answers.
|
||||
backend.probe().await.expect("probe of the fixture bucket");
|
||||
|
||||
// 8. A missing object is `NotFound`, and never retried.
|
||||
let err = backend.head(MISSING_KEY).await.expect_err("a missing object must fail");
|
||||
assert!(matches!(err, SourceError::NotFound), "{err:?}");
|
||||
assert_eq!(err.class_label(), "not_found");
|
||||
assert!(!err.is_retryable());
|
||||
|
||||
// 9. A denied object is `AccessDenied`, and never retried.
|
||||
let err = backend.head(FORBIDDEN_KEY).await.expect_err("a denied object must fail");
|
||||
assert!(matches!(err, SourceError::AccessDenied), "{err:?}");
|
||||
assert_eq!(err.class_label(), "access_denied");
|
||||
assert!(!err.is_retryable());
|
||||
|
||||
// 10. A provider without a key cursor must refuse one instead of listing
|
||||
// from the wrong position. This issues no request either way.
|
||||
if !caps.supports_start_after {
|
||||
let err = backend
|
||||
.list(&SourceListRequest {
|
||||
start_after: Some(OBJECT_KEY),
|
||||
max_keys: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("a backend without a key cursor must refuse start_after");
|
||||
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,367 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket three-state circuit breaker protecting an on-demand migration
|
||||
//! source (rustfs/backlog#2152).
|
||||
//!
|
||||
//! `Closed` lets every request through and counts consecutive failures
|
||||
//! inside a sliding window; reaching the threshold opens the breaker. `Open`
|
||||
//! rejects everything until the open duration elapses, then moves to
|
||||
//! `HalfOpen`, which admits a single probe: success closes the breaker,
|
||||
//! failure re-opens it. Timing uses `tokio::time::Instant` so tests can drive
|
||||
//! it with `tokio::time::pause`.
|
||||
//!
|
||||
//! Only transport-level failures count (`Throttled`, `Timeout`, `Connect`,
|
||||
//! `ServerError`). `NotFound` is a healthy answer and resets the failure
|
||||
//! streak; `AccessDenied`, `Unsupported` and `Other` are configuration or
|
||||
//! object problems that neither open nor close the breaker.
|
||||
|
||||
use super::source_client::SourceError;
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// Consecutive counted failures that open the breaker.
|
||||
pub const BREAKER_FAILURE_THRESHOLD: u32 = 5;
|
||||
/// Failures further apart than this do not accumulate.
|
||||
pub const BREAKER_FAILURE_WINDOW: Duration = Duration::from_secs(30);
|
||||
/// How long an open breaker rejects before admitting a probe.
|
||||
pub const BREAKER_OPEN_DURATION: Duration = Duration::from_secs(30);
|
||||
/// Probes admitted while half-open.
|
||||
pub const BREAKER_HALF_OPEN_MAX_PROBES: u32 = 1;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BreakerState {
|
||||
Closed,
|
||||
Open,
|
||||
HalfOpen,
|
||||
}
|
||||
|
||||
impl BreakerState {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
BreakerState::Closed => "closed",
|
||||
BreakerState::Open => "open",
|
||||
BreakerState::HalfOpen => "half_open",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A state change the caller may want to log.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct BreakerTransition {
|
||||
pub from: BreakerState,
|
||||
pub to: BreakerState,
|
||||
}
|
||||
|
||||
/// How a source result is scored by the breaker.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum BreakerVerdict {
|
||||
/// Resets the failure streak; closes a half-open breaker.
|
||||
Success,
|
||||
/// Counts toward the threshold; re-opens a half-open breaker.
|
||||
Failure,
|
||||
/// Leaves the breaker untouched.
|
||||
Neutral,
|
||||
}
|
||||
|
||||
impl BreakerVerdict {
|
||||
/// `None` is a successful source call.
|
||||
pub fn for_result(error: Option<&SourceError>) -> Self {
|
||||
match error {
|
||||
None | Some(SourceError::NotFound) => BreakerVerdict::Success,
|
||||
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
|
||||
BreakerVerdict::Failure
|
||||
}
|
||||
Some(
|
||||
SourceError::AccessDenied
|
||||
| SourceError::Unsupported(_)
|
||||
| SourceError::InvalidPagination(_)
|
||||
| SourceError::Other(_),
|
||||
) => BreakerVerdict::Neutral,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Inner {
|
||||
state: BreakerState,
|
||||
consecutive_failures: u32,
|
||||
last_failure_at: Option<Instant>,
|
||||
opened_at: Option<Instant>,
|
||||
half_open_probes: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Breaker {
|
||||
inner: Mutex<Inner>,
|
||||
}
|
||||
|
||||
impl Default for Breaker {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Breaker {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(Inner {
|
||||
state: BreakerState::Closed,
|
||||
consecutive_failures: 0,
|
||||
last_failure_at: None,
|
||||
opened_at: None,
|
||||
half_open_probes: 0,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Current state after applying the open-duration timeout.
|
||||
pub fn state(&self) -> BreakerState {
|
||||
let mut inner = self.inner.lock();
|
||||
Self::advance(&mut inner, Instant::now());
|
||||
inner.state
|
||||
}
|
||||
|
||||
/// Whether a request may reach the source right now. Consumes the
|
||||
/// half-open probe budget when it grants one.
|
||||
pub fn allow_request(&self) -> bool {
|
||||
let mut inner = self.inner.lock();
|
||||
Self::advance(&mut inner, Instant::now());
|
||||
match inner.state {
|
||||
BreakerState::Closed => true,
|
||||
BreakerState::Open => false,
|
||||
BreakerState::HalfOpen => {
|
||||
if inner.half_open_probes < BREAKER_HALF_OPEN_MAX_PROBES {
|
||||
inner.half_open_probes += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scores a source result; returns the transition it caused, if any.
|
||||
pub fn record(&self, verdict: BreakerVerdict) -> Option<BreakerTransition> {
|
||||
match verdict {
|
||||
BreakerVerdict::Success => self.record_success(),
|
||||
BreakerVerdict::Failure => self.record_failure(),
|
||||
BreakerVerdict::Neutral => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_success(&self) -> Option<BreakerTransition> {
|
||||
let mut inner = self.inner.lock();
|
||||
let now = Instant::now();
|
||||
Self::advance(&mut inner, now);
|
||||
inner.consecutive_failures = 0;
|
||||
inner.last_failure_at = None;
|
||||
match inner.state {
|
||||
BreakerState::Closed => None,
|
||||
// A success while open can only come from a request admitted
|
||||
// before the breaker opened; it says nothing about recovery.
|
||||
BreakerState::Open => None,
|
||||
BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Closed, now)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_failure(&self) -> Option<BreakerTransition> {
|
||||
let mut inner = self.inner.lock();
|
||||
let now = Instant::now();
|
||||
Self::advance(&mut inner, now);
|
||||
match inner.state {
|
||||
BreakerState::Closed => {
|
||||
let within_window = inner
|
||||
.last_failure_at
|
||||
.is_some_and(|last| now.saturating_duration_since(last) <= BREAKER_FAILURE_WINDOW);
|
||||
inner.consecutive_failures = if within_window { inner.consecutive_failures + 1 } else { 1 };
|
||||
inner.last_failure_at = Some(now);
|
||||
if inner.consecutive_failures >= BREAKER_FAILURE_THRESHOLD {
|
||||
Some(Self::transition(&mut inner, BreakerState::Open, now))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
BreakerState::Open => None,
|
||||
BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Open, now)),
|
||||
}
|
||||
}
|
||||
|
||||
fn advance(inner: &mut Inner, now: Instant) {
|
||||
if inner.state == BreakerState::Open
|
||||
&& inner
|
||||
.opened_at
|
||||
.is_some_and(|opened| now.saturating_duration_since(opened) >= BREAKER_OPEN_DURATION)
|
||||
{
|
||||
Self::transition(inner, BreakerState::HalfOpen, now);
|
||||
}
|
||||
}
|
||||
|
||||
fn transition(inner: &mut Inner, to: BreakerState, now: Instant) -> BreakerTransition {
|
||||
let from = inner.state;
|
||||
inner.state = to;
|
||||
match to {
|
||||
BreakerState::Open => {
|
||||
inner.opened_at = Some(now);
|
||||
inner.half_open_probes = 0;
|
||||
}
|
||||
BreakerState::HalfOpen => {
|
||||
inner.half_open_probes = 0;
|
||||
}
|
||||
BreakerState::Closed => {
|
||||
inner.opened_at = None;
|
||||
inner.half_open_probes = 0;
|
||||
inner.consecutive_failures = 0;
|
||||
inner.last_failure_at = None;
|
||||
}
|
||||
}
|
||||
BreakerTransition { from, to }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn server_error() -> SourceError {
|
||||
SourceError::ServerError(503)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn five_failures_open_then_half_open_after_timeout() {
|
||||
let breaker = Breaker::new();
|
||||
for i in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&server_error()))), None, "failure {i}");
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
assert_eq!(
|
||||
breaker.record(BreakerVerdict::for_result(Some(&server_error()))),
|
||||
Some(BreakerTransition {
|
||||
from: BreakerState::Closed,
|
||||
to: BreakerState::Open
|
||||
})
|
||||
);
|
||||
assert_eq!(breaker.state(), BreakerState::Open);
|
||||
assert!(!breaker.allow_request());
|
||||
|
||||
tokio::time::advance(BREAKER_OPEN_DURATION - Duration::from_secs(1)).await;
|
||||
assert!(!breaker.allow_request());
|
||||
assert_eq!(breaker.state(), BreakerState::Open);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(breaker.state(), BreakerState::HalfOpen);
|
||||
assert!(breaker.allow_request(), "one probe is admitted");
|
||||
assert!(!breaker.allow_request(), "second probe is rejected");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn half_open_probe_success_closes_and_failure_reopens() {
|
||||
let breaker = Breaker::new();
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD {
|
||||
breaker.record_failure();
|
||||
}
|
||||
tokio::time::advance(BREAKER_OPEN_DURATION).await;
|
||||
assert!(breaker.allow_request());
|
||||
assert_eq!(
|
||||
breaker.record_failure(),
|
||||
Some(BreakerTransition {
|
||||
from: BreakerState::HalfOpen,
|
||||
to: BreakerState::Open
|
||||
})
|
||||
);
|
||||
assert!(!breaker.allow_request());
|
||||
|
||||
tokio::time::advance(BREAKER_OPEN_DURATION).await;
|
||||
assert!(breaker.allow_request());
|
||||
assert_eq!(
|
||||
breaker.record_success(),
|
||||
Some(BreakerTransition {
|
||||
from: BreakerState::HalfOpen,
|
||||
to: BreakerState::Closed
|
||||
})
|
||||
);
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
assert!(breaker.allow_request());
|
||||
// The streak restarts from zero after closing.
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
assert_eq!(breaker.record_failure(), None);
|
||||
}
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn failures_outside_window_do_not_accumulate() {
|
||||
let breaker = Breaker::new();
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
breaker.record_failure();
|
||||
}
|
||||
tokio::time::advance(BREAKER_FAILURE_WINDOW + Duration::from_secs(1)).await;
|
||||
assert_eq!(breaker.record_failure(), None, "stale streak restarts at one");
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_found_and_access_denied_do_not_count() {
|
||||
let breaker = Breaker::new();
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
breaker.record(BreakerVerdict::for_result(Some(&server_error())));
|
||||
}
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::AccessDenied))), None);
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
// AccessDenied is neutral: the streak is still one short of opening.
|
||||
assert_eq!(
|
||||
breaker.record(BreakerVerdict::for_result(Some(&SourceError::Unsupported("sse-c".into())))),
|
||||
None
|
||||
);
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Other("x".into())))), None);
|
||||
// NotFound is a healthy answer and resets the streak entirely.
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::NotFound))), None);
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Timeout))), None);
|
||||
}
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdicts_cover_every_source_error_class() {
|
||||
assert_eq!(BreakerVerdict::for_result(None), BreakerVerdict::Success);
|
||||
assert_eq!(BreakerVerdict::for_result(Some(&SourceError::NotFound)), BreakerVerdict::Success);
|
||||
for failure in [
|
||||
SourceError::Throttled,
|
||||
SourceError::Timeout,
|
||||
SourceError::Connect("refused".into()),
|
||||
SourceError::ServerError(500),
|
||||
] {
|
||||
assert_eq!(BreakerVerdict::for_result(Some(&failure)), BreakerVerdict::Failure, "{failure:?}");
|
||||
}
|
||||
for neutral in [
|
||||
SourceError::AccessDenied,
|
||||
SourceError::Unsupported("sse-c".into()),
|
||||
SourceError::Other("x".into()),
|
||||
] {
|
||||
assert_eq!(BreakerVerdict::for_result(Some(&neutral)), BreakerVerdict::Neutral, "{neutral:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_labels_are_stable() {
|
||||
assert_eq!(BreakerState::Closed.as_str(), "closed");
|
||||
assert_eq!(BreakerState::Open.as_str(), "open");
|
||||
assert_eq!(BreakerState::HalfOpen.as_str(), "half_open");
|
||||
assert_eq!(serde_json::to_string(&BreakerState::HalfOpen).unwrap(), "\"half_open\"");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,564 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Native Google Cloud Storage source backend.
|
||||
//!
|
||||
//! The `gcs` provider already reaches GCS through its S3 interoperability API,
|
||||
//! which needs an HMAC key pair. This backend is the other half: it authorizes
|
||||
//! with a service-account key, the credential most GCS projects actually issue,
|
||||
//! by minting OAuth tokens through the shared `google-cloud-auth` credential
|
||||
//! machinery the tier layer already uses.
|
||||
//!
|
||||
//! Two GCS surfaces are involved, each for the half it describes best. The read
|
||||
//! path uses the XML API (`/{bucket}/{object}`), whose responses carry
|
||||
//! `x-goog-meta-*` user metadata and the `x-goog-hash` digest in one round trip.
|
||||
//! Listing uses the JSON API (`objects.list`), whose `pageToken` maps directly
|
||||
//! onto the shared page cursor and whose `prefixes` are the delimiter roll-up.
|
||||
//! Both accept the same bearer token.
|
||||
//!
|
||||
//! Every call this backend makes needs only `storage.objects.get` and
|
||||
//! `storage.objects.list`, the two permissions of the `objectViewer` role, so a
|
||||
//! key scoped to exactly the migration's needs works.
|
||||
//!
|
||||
//! `x-goog-hash` carries a base64 MD5 for every non-composite object; it is
|
||||
//! converted to hex and becomes the head's ETag, so a pulled object is checked
|
||||
//! against the digest GCS itself computed. A composite object has no MD5, and
|
||||
//! its ETag is then marked opaque rather than checked.
|
||||
|
||||
use super::native_http::{
|
||||
NativeHeadFields, NativeHttp, base64_md5_to_hex, header, native_source_head, parse_http_timestamp, read_text, response_body,
|
||||
};
|
||||
use super::source_client::{
|
||||
GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
SourceTimeouts, range_header_value,
|
||||
};
|
||||
use super::storage_api::HTTPRangeSpec;
|
||||
use super::storage_api::remote_s3_client::RemoteS3ClientError;
|
||||
use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder};
|
||||
use google_cloud_auth::credentials::{CacheableResource, Credentials};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use url::Url;
|
||||
|
||||
/// Read-only object scope: this backend never writes to the source.
|
||||
const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
|
||||
const METADATA_PREFIX: &str = "x-goog-meta-";
|
||||
/// GCS reports its error code in the response body, not a header; the shared
|
||||
/// transport takes a header name, so it is given one that never matches and
|
||||
/// classification falls back to the status.
|
||||
const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
|
||||
/// One `objects.list` page is small; refuse an unbounded document.
|
||||
const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
pub struct GcsNativeSourceBackend {
|
||||
http: NativeHttp,
|
||||
bucket: String,
|
||||
credentials: Credentials,
|
||||
}
|
||||
|
||||
impl GcsNativeSourceBackend {
|
||||
pub fn new(
|
||||
endpoint: &str,
|
||||
bucket: &str,
|
||||
spec: &GcsSourceSpec,
|
||||
timeouts: SourceTimeouts,
|
||||
skip_tls_verify: bool,
|
||||
ca_cert_pem: Option<&str>,
|
||||
) -> Result<Self, RemoteS3ClientError> {
|
||||
let key: serde_json::Value = serde_json::from_str(&spec.service_account_json)
|
||||
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not valid JSON"))?;
|
||||
let credentials = ServiceAccountBuilder::new(key)
|
||||
.with_access_specifier(AccessSpecifier::from_scopes([READ_ONLY_SCOPE]))
|
||||
.build()
|
||||
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not usable"))?;
|
||||
Ok(Self {
|
||||
http: NativeHttp::new(endpoint, timeouts, skip_tls_verify, ca_cert_pem)?,
|
||||
bucket: bucket.to_string(),
|
||||
credentials,
|
||||
})
|
||||
}
|
||||
|
||||
/// Authorization headers for one request. A credential failure is reported
|
||||
/// as `AccessDenied` with no message: the renderer of a credential error
|
||||
/// has the key material in scope, and the class is what callers act on.
|
||||
async fn auth_headers(&self) -> Result<HeaderMap, SourceError> {
|
||||
match self.credentials.headers(http::Extensions::new()).await {
|
||||
Ok(CacheableResource::New { data, .. }) => Ok(data),
|
||||
// Only returned when the caller passes an entity tag, which this
|
||||
// backend never does; an empty set is still the honest answer.
|
||||
Ok(CacheableResource::NotModified) => Ok(HeaderMap::new()),
|
||||
Err(_) => Err(SourceError::AccessDenied),
|
||||
}
|
||||
}
|
||||
|
||||
/// XML API URL of one object; `/` in the key stay path separators.
|
||||
fn object_url(&self, key: &str) -> Result<Url, SourceError> {
|
||||
self.http.url(std::iter::once(self.bucket.as_str()).chain(key.split('/')))
|
||||
}
|
||||
|
||||
/// JSON API URL of the bucket's object collection.
|
||||
fn objects_url(&self) -> Result<Url, SourceError> {
|
||||
self.http.url(["storage", "v1", "b", self.bucket.as_str(), "o"])
|
||||
}
|
||||
|
||||
async fn request(&self, method: Method, url: Url, mut headers: HeaderMap) -> Result<reqwest::Request, SourceError> {
|
||||
for (name, value) in self.auth_headers().await? {
|
||||
if let Some(name) = name {
|
||||
headers.insert(name, value);
|
||||
}
|
||||
}
|
||||
let mut request = reqwest::Request::new(method, url);
|
||||
*request.headers_mut() = headers;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
async fn send_object(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
|
||||
match self.http.send_object(request, NO_ERROR_CODE_HEADER).await {
|
||||
Err(SourceError::NotFound) => {
|
||||
// An XML object URL also returns 404 when its bucket is gone.
|
||||
// Reuse the read-only listing probe before caching a key miss.
|
||||
self.probe().await?;
|
||||
Err(SourceError::NotFound)
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared mapping for the XML API's HEAD and GET responses.
|
||||
fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
|
||||
if header(headers, "x-goog-encryption-key-sha256").is_some() {
|
||||
return Err(SourceError::Unsupported(
|
||||
"source object uses a customer-supplied encryption key; customer-key sources are not supported".to_string(),
|
||||
));
|
||||
}
|
||||
// `x-goog-hash` lists digests as `name=base64`, comma separated, and may
|
||||
// repeat across header lines. Only the MD5 describes the whole object.
|
||||
let md5 = headers
|
||||
.get_all("x-goog-hash")
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.filter_map(|digest| digest.trim().strip_prefix("md5="))
|
||||
.find_map(base64_md5_to_hex);
|
||||
|
||||
let (etag, etag_is_opaque) = match md5 {
|
||||
Some(md5) => (Some(md5), false),
|
||||
// A composite object has no MD5; its ETag describes the composition
|
||||
// rather than the bytes, so it is provenance only.
|
||||
None => (header(headers, "etag").map(str::to_string), true),
|
||||
};
|
||||
native_source_head(
|
||||
headers,
|
||||
METADATA_PREFIX,
|
||||
NativeHeadFields {
|
||||
etag,
|
||||
etag_is_opaque,
|
||||
version_id: header(headers, "x-goog-generation").map(str::to_string),
|
||||
storage_class: header(headers, "x-goog-storage-class").map(str::to_string),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SourceBackend for GcsNativeSourceBackend {
|
||||
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
|
||||
let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?;
|
||||
let response = self.send_object(request).await?;
|
||||
Self::head_from_response(response.headers())
|
||||
}
|
||||
|
||||
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(range) = range.map(range_header_value).transpose()? {
|
||||
headers.insert(
|
||||
http::header::RANGE,
|
||||
HeaderValue::from_str(&range).map_err(|_| SourceError::Other("invalid range header".to_string()))?,
|
||||
);
|
||||
}
|
||||
let request = self.request(Method::GET, self.object_url(key)?, headers).await?;
|
||||
let response = self.send_object(request).await?;
|
||||
let head = Self::head_from_response(response.headers())?;
|
||||
let content_range = header(response.headers(), "content-range").map(str::to_string);
|
||||
Ok(SourceGet {
|
||||
head,
|
||||
body: response_body(response),
|
||||
content_range,
|
||||
})
|
||||
}
|
||||
|
||||
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
|
||||
// `objects.list` offers `startOffset`, which is inclusive, so it cannot
|
||||
// express "resume after this key" without silently repeating it.
|
||||
if request.start_after.is_some() {
|
||||
return Err(SourceError::Unsupported(
|
||||
"gcs sources cannot resume a listing from a key; use the continuation token".to_string(),
|
||||
));
|
||||
}
|
||||
let mut url = self.objects_url()?;
|
||||
{
|
||||
let mut query = url.query_pairs_mut();
|
||||
if let Some(prefix) = request.prefix.filter(|prefix| !prefix.is_empty()) {
|
||||
query.append_pair("prefix", prefix);
|
||||
}
|
||||
if let Some(delimiter) = request.delimiter.filter(|delimiter| !delimiter.is_empty()) {
|
||||
query.append_pair("delimiter", delimiter);
|
||||
}
|
||||
if let Some(token) = request.continuation_token.filter(|token| !token.is_empty()) {
|
||||
query.append_pair("pageToken", token);
|
||||
}
|
||||
if request.max_keys > 0 {
|
||||
query.append_pair("maxResults", &request.max_keys.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
|
||||
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
|
||||
let body = read_text(response, MAX_JSON_BYTES).await?;
|
||||
parse_objects_list(&body)
|
||||
}
|
||||
|
||||
/// GCS has no object tagging API; user metadata is already carried by the
|
||||
/// head mapping. An empty map keeps `policy.copy_tags` from failing a pull
|
||||
/// over a concept the provider does not have.
|
||||
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
|
||||
Ok(HashMap::new())
|
||||
}
|
||||
|
||||
/// A one-object listing, not `buckets.get`: the migration pipeline only
|
||||
/// ever needs `storage.objects.list` and `storage.objects.get`, and a key
|
||||
/// scoped to exactly those (the `objectViewer` role) cannot read the bucket
|
||||
/// resource. Probing with `buckets.get` would reject a correct key.
|
||||
async fn probe(&self) -> Result<(), SourceError> {
|
||||
let mut url = self.objects_url()?;
|
||||
url.query_pairs_mut().append_pair("maxResults", "1");
|
||||
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
|
||||
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
|
||||
read_text(response, MAX_JSON_BYTES)
|
||||
.await
|
||||
.and_then(|body| parse_objects_list(&body))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ObjectsList {
|
||||
#[serde(default)]
|
||||
items: Vec<ListedObject>,
|
||||
#[serde(default)]
|
||||
prefixes: Vec<String>,
|
||||
#[serde(default)]
|
||||
next_page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ListedObject {
|
||||
name: String,
|
||||
/// GCS renders the size as a decimal string, not a JSON number.
|
||||
#[serde(default)]
|
||||
size: Option<String>,
|
||||
#[serde(default)]
|
||||
updated: Option<String>,
|
||||
#[serde(default)]
|
||||
md5_hash: Option<String>,
|
||||
#[serde(default)]
|
||||
etag: Option<String>,
|
||||
#[serde(default)]
|
||||
storage_class: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
|
||||
let listing: ObjectsList =
|
||||
serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
|
||||
let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
|
||||
let objects = listing
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let etag = item
|
||||
.md5_hash
|
||||
.as_deref()
|
||||
.and_then(base64_md5_to_hex)
|
||||
.or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
|
||||
.filter(|etag| !etag.is_empty());
|
||||
SourceObject {
|
||||
key: item.name,
|
||||
etag,
|
||||
size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
|
||||
last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
|
||||
storage_class: item.storage_class,
|
||||
// GCS never encodes a part count in a digest or an ETag.
|
||||
is_multipart_etag: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(SourcePage {
|
||||
objects,
|
||||
common_prefixes: listing.prefixes,
|
||||
is_truncated: next_continuation_token.is_some(),
|
||||
next_continuation_token,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
|
||||
use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
|
||||
use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
|
||||
|
||||
const LIST_PAGE_ONE: &str = r#"{
|
||||
"kind": "storage#objects",
|
||||
"nextPageToken": "cursor-1",
|
||||
"prefixes": ["dir/sub/"],
|
||||
"items": [
|
||||
{
|
||||
"name": "dir/a.txt",
|
||||
"size": "5",
|
||||
"updated": "2015-10-21T07:28:00.000Z",
|
||||
"md5Hash": "XUFAKrxLKna5cZ2REBfFkg==",
|
||||
"etag": "CJizy9Wq0McCEAE=",
|
||||
"storageClass": "STANDARD"
|
||||
}
|
||||
]
|
||||
}"#;
|
||||
|
||||
const LIST_PAGE_TWO: &str = r#"{
|
||||
"kind": "storage#objects",
|
||||
"items": [
|
||||
{
|
||||
"name": "dir/b.txt",
|
||||
"size": "7",
|
||||
"updated": "2015-10-21T07:28:00.000Z",
|
||||
"etag": "\"CJizy9Wq0McCEAI=\""
|
||||
}
|
||||
]
|
||||
}"#;
|
||||
|
||||
fn backend(endpoint: &Url) -> GcsNativeSourceBackend {
|
||||
GcsNativeSourceBackend {
|
||||
http: NativeHttp::for_test(endpoint.clone()),
|
||||
bucket: "legacy".to_string(),
|
||||
// Anonymous credentials add no headers, so the fixture sees exactly
|
||||
// the request this backend builds.
|
||||
credentials: AnonymousBuilder::new().build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn object_headers() -> Vec<(&'static str, String)> {
|
||||
vec![
|
||||
("Content-Type", "text/plain".to_string()),
|
||||
("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
|
||||
("ETag", "\"CJizy9Wq0McCEAE=\"".to_string()),
|
||||
("x-goog-hash", "crc32c=AAAAAA==,md5=XUFAKrxLKna5cZ2REBfFkg==".to_string()),
|
||||
("x-goog-meta-owner", "alice".to_string()),
|
||||
("x-goog-storage-class", "STANDARD".to_string()),
|
||||
("x-goog-generation", "1445412480000000".to_string()),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn objects_list_maps_items_prefixes_and_the_page_token() {
|
||||
let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse");
|
||||
assert_eq!(page.common_prefixes, vec!["dir/sub/"]);
|
||||
assert!(page.is_truncated);
|
||||
assert_eq!(page.next_continuation_token.as_deref(), Some("cursor-1"));
|
||||
assert_eq!(page.objects.len(), 1);
|
||||
assert_eq!(page.objects[0].key, "dir/a.txt");
|
||||
assert_eq!(page.objects[0].size, 5, "the string size is parsed");
|
||||
assert_eq!(
|
||||
page.objects[0].etag.as_deref(),
|
||||
Some("5d41402abc4b2a76b9719d911017c592"),
|
||||
"the base64 md5Hash becomes a hex ETag"
|
||||
);
|
||||
assert_eq!(page.objects[0].storage_class.as_deref(), Some("STANDARD"));
|
||||
assert!(page.objects[0].last_modified.is_some(), "RFC 3339 `updated` is parsed");
|
||||
|
||||
let page = parse_objects_list(LIST_PAGE_TWO).expect("page should parse");
|
||||
assert!(!page.is_truncated);
|
||||
assert!(page.next_continuation_token.is_none());
|
||||
assert_eq!(
|
||||
page.objects[0].etag.as_deref(),
|
||||
Some("CJizy9Wq0McCEAI="),
|
||||
"without md5Hash the raw etag is carried"
|
||||
);
|
||||
|
||||
assert!(parse_objects_list("not json").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_prefers_the_goog_hash_md5_over_the_etag() {
|
||||
let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, object_headers(), String::new())]).await;
|
||||
let head = backend(&endpoint).head("dir/a b.txt").await.expect("HEAD should map");
|
||||
|
||||
let recorded = recorded.lock().expect("recorder lock").clone();
|
||||
assert_eq!(recorded[0].method, "HEAD");
|
||||
assert_eq!(recorded[0].target, "/legacy/dir/a%20b.txt", "the XML API addresses the object by path");
|
||||
assert_eq!(
|
||||
head.etag.as_deref(),
|
||||
Some("5d41402abc4b2a76b9719d911017c592"),
|
||||
"the x-goog-hash md5 is the content digest"
|
||||
);
|
||||
assert!(!head.etag_is_opaque, "a GCS md5 may be checked against the pulled bytes");
|
||||
assert_eq!(head.user_metadata, HashMap::from([("owner".to_string(), "alice".to_string())]));
|
||||
assert_eq!(head.version_id.as_deref(), Some("1445412480000000"));
|
||||
assert_eq!(head.storage_class.as_deref(), Some("STANDARD"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_composite_object_without_an_md5_keeps_an_opaque_etag() {
|
||||
let headers = object_headers()
|
||||
.into_iter()
|
||||
.map(|(name, value)| {
|
||||
if name == "x-goog-hash" {
|
||||
(name, "crc32c=AAAAAA==".to_string())
|
||||
} else {
|
||||
(name, value)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
|
||||
let head = backend(&endpoint).head("composed").await.expect("HEAD should map");
|
||||
assert_eq!(head.etag.as_deref(), Some("CJizy9Wq0McCEAE="));
|
||||
assert!(head.etag_is_opaque, "a composite ETag describes the composition, not the bytes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn customer_supplied_key_objects_are_refused() {
|
||||
let mut headers = object_headers();
|
||||
headers.push(("x-goog-encryption-key-sha256", "abc".to_string()));
|
||||
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
|
||||
let err = backend(&endpoint)
|
||||
.head("a.txt")
|
||||
.await
|
||||
.expect_err("CSEK objects are unsupported");
|
||||
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_and_probe_address_the_json_api() {
|
||||
let (endpoint, recorded) = scripted_server(vec![
|
||||
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
|
||||
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
|
||||
])
|
||||
.await;
|
||||
let backend = backend(&endpoint);
|
||||
|
||||
backend
|
||||
.list(&SourceListRequest {
|
||||
prefix: Some("dir/"),
|
||||
delimiter: Some("/"),
|
||||
continuation_token: Some("cursor-0"),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("listing should succeed");
|
||||
backend.probe().await.expect("probe should succeed");
|
||||
|
||||
let recorded = recorded.lock().expect("recorder lock").clone();
|
||||
assert!(recorded[0].target.starts_with("/storage/v1/b/legacy/o?"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("prefix=dir%2F"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("delimiter=%2F"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("pageToken=cursor-0"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("maxResults=2"), "{}", recorded[0].target);
|
||||
assert_eq!(
|
||||
recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1",
|
||||
"the probe uses the listing permission the pipeline already needs"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gcs_native_backend_satisfies_the_shared_backend_contract() {
|
||||
let mut ranged = object_headers();
|
||||
ranged.push(("Content-Range", "bytes 1-3/5".to_string()));
|
||||
// A HEAD reports the object size with no body, exactly as GCS does.
|
||||
let mut head_only = object_headers();
|
||||
head_only.push(("Content-Length", "5".to_string()));
|
||||
let (endpoint, _) = scripted_server(vec![
|
||||
ScriptedResponse::new(200, head_only, String::new()),
|
||||
ScriptedResponse::new(200, object_headers(), "hello".to_string()),
|
||||
ScriptedResponse::new(206, ranged, "ell".to_string()),
|
||||
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
|
||||
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_TWO.to_string()),
|
||||
// GCS has no tagging call, so the contract's tag step issues no
|
||||
// request; the probe is the next one on the wire.
|
||||
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
|
||||
ScriptedResponse::new(404, Vec::new(), String::new()),
|
||||
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
|
||||
ScriptedResponse::new(403, Vec::new(), String::new()),
|
||||
])
|
||||
.await;
|
||||
|
||||
assert_backend_contract(
|
||||
&backend(&endpoint),
|
||||
BackendCapabilities {
|
||||
etag_is_opaque: false,
|
||||
supports_start_after: false,
|
||||
// GCS objects have no tags; the contract's tag step is skipped.
|
||||
supports_tagging: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_404_is_not_an_object_not_found() {
|
||||
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(404, Vec::new(), String::new())]).await;
|
||||
let err = backend(&endpoint)
|
||||
.list(&SourceListRequest {
|
||||
max_keys: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("a failed bucket listing is not a per-object miss");
|
||||
assert_eq!(err.class_label(), "other", "{err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_404_requires_a_readable_source_bucket() {
|
||||
for method in [Method::HEAD, Method::GET] {
|
||||
for (probe_status, expected_class) in [
|
||||
(200, "not_found"),
|
||||
(404, "other"),
|
||||
(403, "access_denied"),
|
||||
(503, "throttled"),
|
||||
(500, "server_error"),
|
||||
] {
|
||||
let (endpoint, recorded) = scripted_server(vec![
|
||||
ScriptedResponse::new(404, Vec::new(), String::new()),
|
||||
ScriptedResponse::new(probe_status, Vec::new(), "{}".to_string()),
|
||||
])
|
||||
.await;
|
||||
let backend = backend(&endpoint);
|
||||
let result = if method == Method::HEAD {
|
||||
backend.head("missing").await.map(|_| ())
|
||||
} else {
|
||||
backend.get("missing", None).await.map(|_| ())
|
||||
};
|
||||
let error = result.expect_err("the object 404 must remain an error");
|
||||
assert_eq!(error.class_label(), expected_class, "{method} with probe HTTP {probe_status}: {error:?}");
|
||||
let recorded = recorded.lock().expect("recorder lock");
|
||||
assert_eq!(recorded.len(), 2, "one bounded read-only probe per ambiguous object miss");
|
||||
assert_eq!(recorded[0].method, method.as_str());
|
||||
assert_eq!(recorded[1].method, "GET");
|
||||
assert_eq!(recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,147 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Projection of application runtime state onto observability-owned DTOs.
|
||||
|
||||
use crate::on_demand_migration::backfill::{
|
||||
BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
|
||||
};
|
||||
use crate::on_demand_migration::{
|
||||
BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
|
||||
OnDemandMigrationSys as SourceOnDemandMigrationSys,
|
||||
};
|
||||
use rustfs_obs::metrics::{
|
||||
OdmBackfillBucketStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
|
||||
register_on_demand_migration_metrics_source,
|
||||
};
|
||||
|
||||
pub(super) fn register() {
|
||||
register_on_demand_migration_metrics_source(snapshot, backfill_snapshot);
|
||||
}
|
||||
|
||||
fn on_demand_migration_stats_from_snapshot(snapshot: SourceOdmBucketSnapshot) -> OnDemandMigrationBucketStats {
|
||||
let stats = snapshot.stats;
|
||||
OnDemandMigrationBucketStats {
|
||||
bucket: snapshot.bucket,
|
||||
requests_total: stats.requests_total,
|
||||
pulled_bytes_total: stats.pulled_bytes_total,
|
||||
pulled_objects_total: stats.pulled_objects_total,
|
||||
pull_failures_total: stats.pull_failures_total,
|
||||
inflight_pulls: stats.inflight_pulls,
|
||||
queue_depth: stats.queue_depth,
|
||||
source_latency_buckets: stats
|
||||
.source_latency
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.le_ms, bucket.count))
|
||||
.collect(),
|
||||
source_latency_count: stats.source_latency.count,
|
||||
source_latency_sum_ms: stats.source_latency.sum_ms,
|
||||
breaker_state: match stats.breaker_state {
|
||||
SourceOdmBreakerState::Closed => OnDemandMigrationBreakerState::Closed,
|
||||
SourceOdmBreakerState::HalfOpen => OnDemandMigrationBreakerState::HalfOpen,
|
||||
SourceOdmBreakerState::Open => OnDemandMigrationBreakerState::Open,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Every bucket with live on-demand migration state on this node, sorted by
|
||||
/// name. Empty while the module switch is off.
|
||||
fn snapshot() -> Vec<OnDemandMigrationBucketStats> {
|
||||
SourceOnDemandMigrationSys::get()
|
||||
.snapshot()
|
||||
.into_iter()
|
||||
.map(on_demand_migration_stats_from_snapshot)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn on_demand_migration_backfill_stats_from_checkpoint(
|
||||
bucket: String,
|
||||
checkpoint: SourceBackfillCheckpoint,
|
||||
) -> OdmBackfillBucketStats {
|
||||
OdmBackfillBucketStats {
|
||||
bucket,
|
||||
state: checkpoint.state.as_str().to_string(),
|
||||
listed: checkpoint.listed,
|
||||
enqueued: checkpoint.enqueued,
|
||||
pulled: checkpoint.pulled,
|
||||
skipped_existing: checkpoint.skipped_existing,
|
||||
failed: checkpoint.failed,
|
||||
bytes: checkpoint.bytes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Backfill jobs running on this node, sorted by bucket. Empty until the
|
||||
/// runner is installed, and empty again once a job finishes: the series are
|
||||
/// per-node job progress, not a cluster-wide history.
|
||||
fn backfill_snapshot() -> Vec<OdmBackfillBucketStats> {
|
||||
source_global_backfill_runner()
|
||||
.map(|runner| {
|
||||
runner
|
||||
.local_job_snapshots()
|
||||
.into_iter()
|
||||
.map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn on_demand_migration_snapshot_projects_counters_and_breaker_state() {
|
||||
// Pin the runtime wire snapshot and its observability projection together.
|
||||
let snapshot: SourceOdmBucketSnapshot = serde_json::from_value(serde_json::json!({
|
||||
"bucket": "photos",
|
||||
"provider": "minio",
|
||||
"endpoint_host": "source.example.com",
|
||||
"applied_at": "2026-09-02T10:00:00Z",
|
||||
"client_error": null,
|
||||
"negative_cache_entries": 0,
|
||||
"inflight_keys": 1,
|
||||
"max_concurrent_pulls": 8,
|
||||
"stats": {
|
||||
"requests_total": {"get": {"source_hit": 2}},
|
||||
"pulled_bytes_total": 4096,
|
||||
"pulled_objects_total": {"inline": 1},
|
||||
"pull_failures_total": {"source_timeout": 1},
|
||||
"inflight_pulls": 1,
|
||||
"queue_depth": 2,
|
||||
"source_latency": {
|
||||
"buckets": [{"le_ms": 5, "count": 1}, {"le_ms": 10, "count": 2}],
|
||||
"count": 3,
|
||||
"sum_ms": 90753
|
||||
},
|
||||
"last_source_error": {"class": "server_error", "at": "2026-09-02T10:00:00Z"},
|
||||
"breaker_state": "open"
|
||||
}
|
||||
}))
|
||||
.expect("runtime snapshot decodes");
|
||||
|
||||
let stats = on_demand_migration_stats_from_snapshot(snapshot);
|
||||
|
||||
assert_eq!(stats.bucket, "photos");
|
||||
assert_eq!(stats.requests_total["get"]["source_hit"], 2);
|
||||
assert_eq!(stats.pulled_bytes_total, 4096);
|
||||
assert_eq!(stats.pulled_objects_total["inline"], 1);
|
||||
assert_eq!(stats.pull_failures_total["source_timeout"], 1);
|
||||
assert_eq!(stats.inflight_pulls, 1);
|
||||
assert_eq!(stats.queue_depth, 2);
|
||||
assert_eq!(stats.source_latency_buckets, vec![(5, 1), (10, 2)]);
|
||||
assert_eq!(stats.source_latency_count, 3);
|
||||
assert_eq!(stats.source_latency_sum_ms, 90_753);
|
||||
assert_eq!(stats.breaker_state, OnDemandMigrationBreakerState::Open);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! On-Demand Migration (ODM): a bucket can name an external S3-compatible
|
||||
//! source bucket; GET misses are served from that source and backfilled
|
||||
//! locally. This module owns the bucket-level configuration model
|
||||
//! (`on-demand-migration.json` in the bucket metadata file), the source
|
||||
//! client, and the per-node runtime (`sys`) that turns configs into live
|
||||
//! clients guarded by a breaker, a negative cache, singleflight and a pull
|
||||
//! concurrency limit (rustfs/backlog#2147).
|
||||
//!
|
||||
//! A source is reached through one `SourceBackend`: the S3 dialect for every
|
||||
//! S3-compatible provider, and a native backend for the providers that have no
|
||||
//! S3 API (`azure`, `gcs_native`).
|
||||
|
||||
pub mod azure;
|
||||
#[cfg(test)]
|
||||
mod backend_contract;
|
||||
pub mod backfill;
|
||||
pub mod breaker;
|
||||
pub mod config;
|
||||
#[cfg(feature = "gcs")]
|
||||
pub mod gcs;
|
||||
pub mod list_through;
|
||||
mod metrics;
|
||||
mod native_http;
|
||||
pub mod negative_cache;
|
||||
pub mod pull;
|
||||
pub mod source_client;
|
||||
pub mod stats;
|
||||
mod storage_api;
|
||||
pub mod sys;
|
||||
#[cfg(test)]
|
||||
mod test_http_fixture;
|
||||
|
||||
pub use breaker::{
|
||||
BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
|
||||
BreakerState, BreakerTransition, BreakerVerdict,
|
||||
};
|
||||
pub use config::{
|
||||
AzureSourceConfig, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig,
|
||||
OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig, SourceCredentials,
|
||||
SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub use list_through::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
|
||||
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
|
||||
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
|
||||
decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
|
||||
pub use pull::{
|
||||
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion,
|
||||
PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody, WriteBackError,
|
||||
WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, idle_guarded_body,
|
||||
};
|
||||
pub use source_client::{
|
||||
SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceSse, is_multipart_etag,
|
||||
};
|
||||
pub use stats::{
|
||||
GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
|
||||
PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
|
||||
};
|
||||
pub use sys::{
|
||||
ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
|
||||
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
|
||||
source_client_spec,
|
||||
};
|
||||
|
||||
pub(crate) fn register_metrics() {
|
||||
metrics::register();
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared HTTP transport for the on-demand migration source backends that do
|
||||
//! not speak S3 (Azure Blob, native GCS).
|
||||
//!
|
||||
//! The S3 backend rides the AWS SDK; these providers have no SigV4 dialect, so
|
||||
//! they talk plain HTTP through one `reqwest` client that carries the same
|
||||
//! connect/read timeouts and TLS policy the operator configured for the source.
|
||||
//! Redirects are refused: the endpoint passed the outbound policy gate once, and
|
||||
//! following a source-chosen `Location` would leave that gate behind.
|
||||
//!
|
||||
//! Errors never render the request URL. A SAS token lives in the query string,
|
||||
//! so a `reqwest` error rendered with its URL would print the credential into
|
||||
//! the log line and the admin response.
|
||||
|
||||
use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag};
|
||||
use super::storage_api::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use futures::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use std::collections::HashMap;
|
||||
use std::time::SystemTime;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::{Rfc2822, Rfc3339};
|
||||
use url::Url;
|
||||
|
||||
/// Origin the native backends are allowed to address, plus the HTTP client
|
||||
/// that reaches it.
|
||||
pub(super) struct NativeHttp {
|
||||
client: reqwest::Client,
|
||||
endpoint: Url,
|
||||
}
|
||||
|
||||
impl NativeHttp {
|
||||
/// `endpoint` must be a bare `scheme://host[:port]` origin; it is checked
|
||||
/// against the outbound policy exactly like an S3 source endpoint.
|
||||
pub(super) fn new(
|
||||
endpoint: &str,
|
||||
timeouts: SourceTimeouts,
|
||||
skip_tls_verify: bool,
|
||||
ca_cert_pem: Option<&str>,
|
||||
) -> Result<Self, RemoteS3ClientError> {
|
||||
let endpoint = Url::parse(endpoint.trim()).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if !matches!(endpoint.scheme(), "http" | "https") {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint(format!(
|
||||
"unsupported scheme {}; expected http or https",
|
||||
endpoint.scheme()
|
||||
)));
|
||||
}
|
||||
if endpoint.host_str().is_none_or(str::is_empty) {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint has no host".to_string()));
|
||||
}
|
||||
if !endpoint.username().is_empty() || endpoint.password().is_some() {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint must not carry userinfo".to_string()));
|
||||
}
|
||||
if !matches!(endpoint.path(), "" | "/") || endpoint.query().is_some() || endpoint.fragment().is_some() {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint(
|
||||
"endpoint must be an origin without path, query or fragment".to_string(),
|
||||
));
|
||||
}
|
||||
validate_remote_endpoint(&endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
|
||||
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.connect_timeout(timeouts.connect)
|
||||
.read_timeout(timeouts.read)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent(USER_AGENT_SUFFIX);
|
||||
if skip_tls_verify {
|
||||
builder = builder.danger_accept_invalid_certs(true);
|
||||
} else if let Some(pem) = ca_cert_pem.map(str::trim).filter(|pem| !pem.is_empty()) {
|
||||
// Reject a malformed bundle the same way the S3 path does, so the
|
||||
// operator sees "invalid CA PEM" instead of a TLS handshake failure.
|
||||
validate_target_ca_pem(pem)?;
|
||||
let certificate = reqwest::Certificate::from_pem(pem.as_bytes())
|
||||
.map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
|
||||
builder = builder.add_root_certificate(certificate);
|
||||
}
|
||||
|
||||
let client = builder
|
||||
.build()
|
||||
.map_err(|err| RemoteS3ClientError::InvalidEndpoint(format!("http client cannot be built: {err}")))?;
|
||||
Ok(Self { client, endpoint })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn for_test(endpoint: Url) -> Self {
|
||||
Self {
|
||||
client: reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("test http client should build"),
|
||||
endpoint,
|
||||
}
|
||||
}
|
||||
|
||||
/// A URL under the endpoint origin. `segments` are percent-encoded as
|
||||
/// path segments, so a key containing `?`, `#` or a space cannot rewrite
|
||||
/// the request target.
|
||||
pub(super) fn url<'a>(&self, segments: impl IntoIterator<Item = &'a str>) -> Result<Url, SourceError> {
|
||||
let mut url = self.endpoint.clone();
|
||||
{
|
||||
let mut path = url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?;
|
||||
path.clear();
|
||||
path.extend(segments);
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
/// Sends the request and returns the response only for a 2xx status.
|
||||
/// Non-2xx statuses are classified from the status and the provider's own
|
||||
/// error-code header; response bodies are not read, so no provider message
|
||||
/// can smuggle credentials or markup into a log line.
|
||||
pub(super) async fn send(
|
||||
&self,
|
||||
request: reqwest::Request,
|
||||
error_code_header: &str,
|
||||
) -> Result<reqwest::Response, SourceError> {
|
||||
self.send_classified(request, error_code_header, false).await
|
||||
}
|
||||
|
||||
pub(super) async fn send_object(
|
||||
&self,
|
||||
request: reqwest::Request,
|
||||
error_code_header: &str,
|
||||
) -> Result<reqwest::Response, SourceError> {
|
||||
self.send_classified(request, error_code_header, true).await
|
||||
}
|
||||
|
||||
async fn send_classified(
|
||||
&self,
|
||||
request: reqwest::Request,
|
||||
error_code_header: &str,
|
||||
not_found_on_404_without_code: bool,
|
||||
) -> Result<reqwest::Response, SourceError> {
|
||||
let response = self.client.execute(request).await.map_err(classify_transport_error)?;
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
let code = response
|
||||
.headers()
|
||||
.get(error_code_header)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
let message = match &code {
|
||||
Some(code) => format!("source returned HTTP {status} ({code})"),
|
||||
None => format!("source returned HTTP {status}"),
|
||||
};
|
||||
match classify_status(status.as_u16(), code.as_deref(), message) {
|
||||
SourceError::Other(_) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
|
||||
err => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Renders a transport failure without the request URL: a SAS token or a
|
||||
/// signed query would otherwise reach logs and admin responses.
|
||||
pub(super) fn classify_transport_error(err: reqwest::Error) -> SourceError {
|
||||
let is_timeout = err.is_timeout();
|
||||
let is_connect = err.is_connect();
|
||||
let message = err.without_url().to_string();
|
||||
if is_timeout {
|
||||
SourceError::Timeout
|
||||
} else if is_connect {
|
||||
SourceError::Connect(message)
|
||||
} else {
|
||||
SourceError::Other(message)
|
||||
}
|
||||
}
|
||||
|
||||
/// Streams the response body without buffering it.
|
||||
pub(super) fn response_body(response: reqwest::Response) -> ByteStream {
|
||||
let stream = response.bytes_stream().map(|chunk| {
|
||||
chunk
|
||||
.map(http_body::Frame::data)
|
||||
.map_err(|err| std::io::Error::other(err.without_url().to_string()))
|
||||
});
|
||||
ByteStream::new(SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream)))
|
||||
}
|
||||
|
||||
/// Reads a bounded response body as UTF-8, for the XML and JSON listings.
|
||||
pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) -> Result<String, SourceError> {
|
||||
let mut body = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(classify_transport_error)?;
|
||||
if body.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(SourceError::Other("source listing response exceeded the size limit".to_string()));
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
String::from_utf8(body).map_err(|_| SourceError::Other("source listing response is not valid UTF-8".to_string()))
|
||||
}
|
||||
|
||||
/// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
|
||||
/// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
|
||||
/// an MD5.
|
||||
pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
|
||||
let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
|
||||
(raw.len() == 16).then(|| faster_hex::hex_string(&raw))
|
||||
}
|
||||
|
||||
pub(super) fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
|
||||
headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim)
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
header(headers, name).filter(|value| !value.is_empty()).map(str::to_string)
|
||||
}
|
||||
|
||||
/// `Last-Modified` and friends arrive as an HTTP date; the JSON dialects use
|
||||
/// RFC 3339 for the same field, so both are accepted.
|
||||
pub(super) fn parse_http_timestamp(value: &str) -> Option<SystemTime> {
|
||||
OffsetDateTime::parse(value, &Rfc2822)
|
||||
.or_else(|_| OffsetDateTime::parse(value, &Rfc3339))
|
||||
.ok()
|
||||
.map(SystemTime::from)
|
||||
}
|
||||
|
||||
/// Provider-specific fields the shared header mapping cannot infer.
|
||||
pub(super) struct NativeHeadFields {
|
||||
pub(super) etag: Option<String>,
|
||||
/// The ETag is an opaque token rather than a digest of the bytes.
|
||||
pub(super) etag_is_opaque: bool,
|
||||
pub(super) version_id: Option<String>,
|
||||
pub(super) storage_class: Option<String>,
|
||||
}
|
||||
|
||||
/// Maps a HEAD or GET response onto [`SourceHead`]. `metadata_prefix` is the
|
||||
/// provider's user-metadata header prefix (`x-ms-meta-`, `x-goog-meta-`); the
|
||||
/// stored shape drops it, matching the `x-amz-meta-` handling of the S3 path.
|
||||
pub(super) fn native_source_head(
|
||||
headers: &HeaderMap,
|
||||
metadata_prefix: &str,
|
||||
fields: NativeHeadFields,
|
||||
) -> Result<SourceHead, SourceError> {
|
||||
let size = header(headers, "content-length")
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.ok_or_else(|| SourceError::Other("source response has no valid content-length".to_string()))?;
|
||||
|
||||
let mut user_metadata = HashMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = name.as_str();
|
||||
if let Some(key) = name.strip_prefix(metadata_prefix)
|
||||
&& !key.is_empty()
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
user_metadata.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let etag = fields
|
||||
.etag
|
||||
.map(|etag| etag.trim().trim_matches('"').to_string())
|
||||
.filter(|etag| !etag.is_empty());
|
||||
// An opaque ETag never encodes a part count, so the multipart flag stays
|
||||
// false for it however the provider happens to spell the token.
|
||||
let is_multipart_etag = !fields.etag_is_opaque && etag.as_deref().is_some_and(is_multipart_etag);
|
||||
|
||||
Ok(SourceHead {
|
||||
etag,
|
||||
size,
|
||||
last_modified: header(headers, "last-modified").and_then(parse_http_timestamp),
|
||||
content_type: header_string(headers, "content-type"),
|
||||
content_encoding: header_string(headers, "content-encoding"),
|
||||
content_disposition: header_string(headers, "content-disposition"),
|
||||
content_language: header_string(headers, "content-language"),
|
||||
cache_control: header_string(headers, "cache-control"),
|
||||
expires: header_string(headers, "expires"),
|
||||
user_metadata,
|
||||
version_id: fields.version_id,
|
||||
storage_class: fields.storage_class,
|
||||
// Neither native provider hands back ciphertext: a customer-key object
|
||||
// is refused by the backend before it reaches this mapping, and the
|
||||
// service-managed encryption is transparent to the reader.
|
||||
sse: None,
|
||||
is_multipart_etag,
|
||||
etag_is_opaque: fields.etag_is_opaque,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::HeaderValue;
|
||||
|
||||
fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in pairs {
|
||||
headers.insert(
|
||||
http::HeaderName::from_bytes(name.as_bytes()).expect("test header name"),
|
||||
HeaderValue::from_str(value).expect("test header value"),
|
||||
);
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
fn fields() -> NativeHeadFields {
|
||||
NativeHeadFields {
|
||||
etag: None,
|
||||
etag_is_opaque: false,
|
||||
version_id: None,
|
||||
storage_class: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_source_head_maps_content_headers_and_prefixed_metadata() {
|
||||
let headers = headers(&[
|
||||
("content-length", "1234"),
|
||||
("content-type", "text/plain"),
|
||||
("content-encoding", "gzip"),
|
||||
("content-language", "en"),
|
||||
("content-disposition", "attachment"),
|
||||
("cache-control", "max-age=60"),
|
||||
("expires", "Thu, 01 Jan 2026 00:00:00 GMT"),
|
||||
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"),
|
||||
("x-ms-meta-owner", "alice"),
|
||||
("x-goog-meta-owner", "not-mine"),
|
||||
]);
|
||||
let head = native_source_head(
|
||||
&headers,
|
||||
"x-ms-meta-",
|
||||
NativeHeadFields {
|
||||
etag: Some("\"0x8DCE1D2\"".to_string()),
|
||||
etag_is_opaque: true,
|
||||
version_id: Some("2026-01-01T00:00:00.0000000Z".to_string()),
|
||||
storage_class: Some("Hot".to_string()),
|
||||
},
|
||||
)
|
||||
.expect("head should map");
|
||||
|
||||
assert_eq!(head.size, 1234);
|
||||
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
|
||||
assert_eq!(head.content_encoding.as_deref(), Some("gzip"));
|
||||
assert_eq!(head.content_language.as_deref(), Some("en"));
|
||||
assert_eq!(head.content_disposition.as_deref(), Some("attachment"));
|
||||
assert_eq!(head.cache_control.as_deref(), Some("max-age=60"));
|
||||
assert_eq!(head.expires.as_deref(), Some("Thu, 01 Jan 2026 00:00:00 GMT"));
|
||||
assert_eq!(
|
||||
head.last_modified,
|
||||
Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_445_412_480)),
|
||||
"HTTP-date Last-Modified must parse"
|
||||
);
|
||||
assert_eq!(
|
||||
head.user_metadata,
|
||||
HashMap::from([("owner".to_string(), "alice".to_string())]),
|
||||
"only the provider's own metadata prefix is read"
|
||||
);
|
||||
assert_eq!(head.etag.as_deref(), Some("0x8DCE1D2"), "quotes are stripped, the token is kept");
|
||||
assert!(head.etag_is_opaque);
|
||||
assert!(!head.is_multipart_etag);
|
||||
assert_eq!(head.storage_class.as_deref(), Some("Hot"));
|
||||
assert!(head.sse.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_source_head_requires_a_content_length() {
|
||||
let err = native_source_head(&headers(&[("content-type", "text/plain")]), "x-ms-meta-", fields())
|
||||
.expect_err("a response without content-length is unusable");
|
||||
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_etag_never_reads_as_a_multipart_etag() {
|
||||
// A digest-shaped ETag keeps the S3 reading; the same string marked
|
||||
// opaque must not be split into "digest-partcount".
|
||||
for (opaque, expected) in [(false, true), (true, false)] {
|
||||
let head = native_source_head(
|
||||
&headers(&[("content-length", "1")]),
|
||||
"x-ms-meta-",
|
||||
NativeHeadFields {
|
||||
etag: Some("d41d8cd98f00b204e9800998ecf8427e-3".to_string()),
|
||||
etag_is_opaque: opaque,
|
||||
..fields()
|
||||
},
|
||||
)
|
||||
.expect("head should map");
|
||||
assert_eq!(head.is_multipart_etag, expected, "opaque = {opaque}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base64_md5_converts_only_sixteen_byte_digests() {
|
||||
assert_eq!(
|
||||
base64_md5_to_hex("1B2M2Y8AsgTpgAmY7PhCfg==").as_deref(),
|
||||
Some("d41d8cd98f00b204e9800998ecf8427e")
|
||||
);
|
||||
assert_eq!(base64_md5_to_hex("not base64!").as_deref(), None);
|
||||
// A CRC32C digest is four bytes: it must not pass as an MD5.
|
||||
assert_eq!(base64_md5_to_hex("AAAAAA==").as_deref(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_http_rejects_endpoints_that_are_not_bare_origins() {
|
||||
for bad in [
|
||||
"ftp://source.example.com",
|
||||
"https://user:pw@source.example.com",
|
||||
"https://source.example.com/container",
|
||||
"https://source.example.com/?x=1",
|
||||
"not a url",
|
||||
] {
|
||||
assert!(
|
||||
NativeHttp::new(bad, SourceTimeouts::default(), false, None).is_err(),
|
||||
"{bad} must be rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_http_percent_encodes_every_path_segment() {
|
||||
let http = NativeHttp::for_test(Url::parse("https://acct.blob.core.windows.net").expect("origin"));
|
||||
let url = http.url(["container", "dir", "a b?c#d.txt"]).expect("url should build");
|
||||
assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt");
|
||||
assert_eq!(url.query(), None, "a key with '?' must not become a query");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket cache of keys the source answered 404 for
|
||||
//! (rustfs/backlog#2152). A hit short-circuits the source lookup for
|
||||
//! `policy.negative_cache_ttl_secs`; a TTL of zero disables the cache.
|
||||
//!
|
||||
//! Entries are never invalidated on a local PUT: once the object exists
|
||||
//! locally the handler never consults ODM for it, so a stale negative entry
|
||||
//! is harmless.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// Upper bound on remembered keys per bucket; LRU eviction beyond it.
|
||||
pub const NEGATIVE_CACHE_MAX_ENTRIES: u64 = 100_000;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct NegativeCache {
|
||||
cache: Option<moka::sync::Cache<String, ()>>,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl NegativeCache {
|
||||
/// `ttl == 0` builds a disabled cache that never records anything.
|
||||
pub fn new(ttl: Duration) -> Self {
|
||||
Self::with_capacity(ttl, NEGATIVE_CACHE_MAX_ENTRIES)
|
||||
}
|
||||
|
||||
pub fn with_capacity(ttl: Duration, max_entries: u64) -> Self {
|
||||
let cache = (!ttl.is_zero()).then(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(max_entries)
|
||||
.time_to_live(ttl)
|
||||
.build()
|
||||
});
|
||||
Self { cache, ttl }
|
||||
}
|
||||
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.cache.is_some()
|
||||
}
|
||||
|
||||
pub fn ttl(&self) -> Duration {
|
||||
self.ttl
|
||||
}
|
||||
|
||||
/// Whether `key` is currently remembered as absent on the source.
|
||||
pub fn contains(&self, key: &str) -> bool {
|
||||
self.cache.as_ref().is_some_and(|cache| cache.get(key).is_some())
|
||||
}
|
||||
|
||||
/// Remembers `key` as absent; no-op when disabled.
|
||||
pub fn insert(&self, key: &str) {
|
||||
if let Some(cache) = &self.cache {
|
||||
cache.insert(key.to_string(), ());
|
||||
}
|
||||
}
|
||||
|
||||
/// Forgets `key` (e.g. after an admin-triggered backfill found it).
|
||||
pub fn remove(&self, key: &str) {
|
||||
if let Some(cache) = &self.cache {
|
||||
cache.invalidate(key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Approximate live entry count, for status snapshots only.
|
||||
pub fn len(&self) -> u64 {
|
||||
self.cache.as_ref().map_or(0, |cache| {
|
||||
cache.run_pending_tasks();
|
||||
cache.entry_count()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn entry_expires_after_ttl() {
|
||||
let cache = NegativeCache::new(Duration::from_millis(80));
|
||||
assert!(cache.is_enabled());
|
||||
cache.insert("a/x");
|
||||
assert!(cache.contains("a/x"));
|
||||
assert!(!cache.contains("a/y"));
|
||||
std::thread::sleep(Duration::from_millis(160));
|
||||
assert!(!cache.contains("a/x"), "entry must expire after the TTL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_ttl_disables_the_cache() {
|
||||
let cache = NegativeCache::new(Duration::ZERO);
|
||||
assert!(!cache.is_enabled());
|
||||
cache.insert("a/x");
|
||||
assert!(!cache.contains("a/x"));
|
||||
assert!(cache.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_forgets_a_key() {
|
||||
let cache = NegativeCache::new(Duration::from_secs(30));
|
||||
cache.insert("a/x");
|
||||
cache.remove("a/x");
|
||||
assert!(!cache.contains("a/x"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capacity_bounds_entries() {
|
||||
let cache = NegativeCache::with_capacity(Duration::from_secs(30), 4);
|
||||
for i in 0..64 {
|
||||
cache.insert(&format!("k{i}"));
|
||||
}
|
||||
assert!(cache.len() <= 4, "len {} exceeds capacity", cache.len());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,531 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket on-demand migration counters (rustfs/backlog#2152).
|
||||
//!
|
||||
//! `OdmStats` is lock-free and survives config rebuilds; `snapshot()` turns
|
||||
//! it into the serializable `OdmStatsSnapshot` that the metrics collector
|
||||
//! and the admin status route (ODM-10/14/15) consume. Field names and label
|
||||
//! values are a wire contract: the golden JSON test below pins them.
|
||||
|
||||
use super::breaker::BreakerState;
|
||||
use super::source_client::SourceError;
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Request operations that can enter ODM.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OdmOp {
|
||||
Get,
|
||||
Head,
|
||||
}
|
||||
|
||||
impl OdmOp {
|
||||
pub const ALL: [OdmOp; 2] = [OdmOp::Get, OdmOp::Head];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
OdmOp::Get => "get",
|
||||
OdmOp::Head => "head",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How a request that entered ODM ended. `local_hit` is deliberately absent:
|
||||
/// requests served locally never reach the runtime.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OdmOutcome {
|
||||
SourceHit,
|
||||
SourceMiss,
|
||||
SourceError,
|
||||
BreakerOpen,
|
||||
NegativeCached,
|
||||
Filtered,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl OdmOutcome {
|
||||
pub const ALL: [OdmOutcome; 7] = [
|
||||
OdmOutcome::SourceHit,
|
||||
OdmOutcome::SourceMiss,
|
||||
OdmOutcome::SourceError,
|
||||
OdmOutcome::BreakerOpen,
|
||||
OdmOutcome::NegativeCached,
|
||||
OdmOutcome::Filtered,
|
||||
OdmOutcome::Unsupported,
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
OdmOutcome::SourceHit => "source_hit",
|
||||
OdmOutcome::SourceMiss => "source_miss",
|
||||
OdmOutcome::SourceError => "source_error",
|
||||
OdmOutcome::BreakerOpen => "breaker_open",
|
||||
OdmOutcome::NegativeCached => "negative_cached",
|
||||
OdmOutcome::Filtered => "filtered",
|
||||
OdmOutcome::Unsupported => "unsupported",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which pipeline stored a pulled object locally.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PullPath {
|
||||
/// Streamed to the client and written locally in one pass.
|
||||
Inline,
|
||||
/// Pulled by a background task after a partial/large read.
|
||||
Background,
|
||||
/// Pulled by the backfill job.
|
||||
Backfill,
|
||||
}
|
||||
|
||||
impl PullPath {
|
||||
pub const ALL: [PullPath; 3] = [PullPath::Inline, PullPath::Background, PullPath::Backfill];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
PullPath::Inline => "inline",
|
||||
PullPath::Background => "background",
|
||||
PullPath::Backfill => "backfill",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a pull did not produce a local object.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PullFailureReason {
|
||||
SourceNotFound,
|
||||
SourceAccessDenied,
|
||||
SourceThrottled,
|
||||
SourceTimeout,
|
||||
SourceConnect,
|
||||
SourceServerError,
|
||||
SourceUnsupported,
|
||||
SourceOther,
|
||||
/// Source bytes did not match the ETag advertised by HEAD/GET.
|
||||
EtagMismatch,
|
||||
/// The local write (internal PUT) failed.
|
||||
LocalWrite,
|
||||
/// The bucket quota rejected the write-back.
|
||||
Quota,
|
||||
/// The bucket state was removed or the process is shutting down.
|
||||
Canceled,
|
||||
/// The background pull queue was full.
|
||||
QueueFull,
|
||||
}
|
||||
|
||||
impl PullFailureReason {
|
||||
pub const ALL: [PullFailureReason; 13] = [
|
||||
PullFailureReason::SourceNotFound,
|
||||
PullFailureReason::SourceAccessDenied,
|
||||
PullFailureReason::SourceThrottled,
|
||||
PullFailureReason::SourceTimeout,
|
||||
PullFailureReason::SourceConnect,
|
||||
PullFailureReason::SourceServerError,
|
||||
PullFailureReason::SourceUnsupported,
|
||||
PullFailureReason::SourceOther,
|
||||
PullFailureReason::EtagMismatch,
|
||||
PullFailureReason::LocalWrite,
|
||||
PullFailureReason::Quota,
|
||||
PullFailureReason::Canceled,
|
||||
PullFailureReason::QueueFull,
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
PullFailureReason::SourceNotFound => "source_not_found",
|
||||
PullFailureReason::SourceAccessDenied => "source_access_denied",
|
||||
PullFailureReason::SourceThrottled => "source_throttled",
|
||||
PullFailureReason::SourceTimeout => "source_timeout",
|
||||
PullFailureReason::SourceConnect => "source_connect",
|
||||
PullFailureReason::SourceServerError => "source_server_error",
|
||||
PullFailureReason::SourceUnsupported => "source_unsupported",
|
||||
PullFailureReason::SourceOther => "source_other",
|
||||
PullFailureReason::EtagMismatch => "etag_mismatch",
|
||||
PullFailureReason::LocalWrite => "local_write",
|
||||
PullFailureReason::Quota => "quota",
|
||||
PullFailureReason::Canceled => "canceled",
|
||||
PullFailureReason::QueueFull => "queue_full",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&SourceError> for PullFailureReason {
|
||||
fn from(err: &SourceError) -> Self {
|
||||
match err {
|
||||
SourceError::NotFound => PullFailureReason::SourceNotFound,
|
||||
SourceError::AccessDenied => PullFailureReason::SourceAccessDenied,
|
||||
SourceError::Throttled => PullFailureReason::SourceThrottled,
|
||||
SourceError::Timeout => PullFailureReason::SourceTimeout,
|
||||
SourceError::Connect(_) => PullFailureReason::SourceConnect,
|
||||
SourceError::ServerError(_) => PullFailureReason::SourceServerError,
|
||||
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
|
||||
SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bounds (milliseconds) of the source latency histogram buckets; the
|
||||
/// implicit last bucket is unbounded. Roughly logarithmic from 5 ms to 60 s.
|
||||
pub const SOURCE_LATENCY_BUCKET_BOUNDS_MS: [u64; 14] = [
|
||||
5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000,
|
||||
];
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct LatencyHistogram {
|
||||
/// One counter per bound plus one for the overflow bucket.
|
||||
buckets: [AtomicU64; SOURCE_LATENCY_BUCKET_BOUNDS_MS.len() + 1],
|
||||
count: AtomicU64,
|
||||
sum_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl LatencyHistogram {
|
||||
fn observe(&self, latency: Duration) {
|
||||
let ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
|
||||
let index = SOURCE_LATENCY_BUCKET_BOUNDS_MS
|
||||
.iter()
|
||||
.position(|bound| ms <= *bound)
|
||||
.unwrap_or(SOURCE_LATENCY_BUCKET_BOUNDS_MS.len());
|
||||
self.buckets[index].fetch_add(1, Ordering::Relaxed);
|
||||
self.count.fetch_add(1, Ordering::Relaxed);
|
||||
self.sum_ms.fetch_add(ms, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> SourceLatencySnapshot {
|
||||
let mut cumulative = 0;
|
||||
let buckets = SOURCE_LATENCY_BUCKET_BOUNDS_MS
|
||||
.iter()
|
||||
.zip(self.buckets.iter())
|
||||
.map(|(bound, counter)| {
|
||||
cumulative += counter.load(Ordering::Relaxed);
|
||||
LatencyBucketSnapshot {
|
||||
le_ms: *bound,
|
||||
count: cumulative,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
SourceLatencySnapshot {
|
||||
buckets,
|
||||
count: self.count.load(Ordering::Relaxed),
|
||||
sum_ms: self.sum_ms.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The most recent source failure, kept for operators: class only, never the
|
||||
/// key or the message (which may echo attacker-controlled input).
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LastSourceError {
|
||||
pub class: String,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub at: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct OdmStats {
|
||||
requests_total: [[AtomicU64; OdmOutcome::ALL.len()]; OdmOp::ALL.len()],
|
||||
pulled_bytes_total: AtomicU64,
|
||||
pulled_objects_total: [AtomicU64; PullPath::ALL.len()],
|
||||
pull_failures_total: [AtomicU64; PullFailureReason::ALL.len()],
|
||||
inflight_pulls: AtomicU64,
|
||||
queue_depth: AtomicU64,
|
||||
source_latency: LatencyHistogram,
|
||||
last_source_error: Mutex<Option<LastSourceError>>,
|
||||
}
|
||||
|
||||
impl OdmStats {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn record_request(&self, op: OdmOp, outcome: OdmOutcome) {
|
||||
self.requests_total[op as usize][outcome as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_pulled_bytes(&self, bytes: u64) {
|
||||
self.pulled_bytes_total.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_pulled_object(&self, path: PullPath) {
|
||||
self.pulled_objects_total[path as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_pull_failure(&self, reason: PullFailureReason) {
|
||||
self.pull_failures_total[reason as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_source_latency(&self, latency: Duration) {
|
||||
self.source_latency.observe(latency);
|
||||
}
|
||||
|
||||
pub fn record_source_error(&self, err: &SourceError) {
|
||||
self.record_source_error_at(err, OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
pub fn record_source_error_at(&self, err: &SourceError, at: OffsetDateTime) {
|
||||
*self.last_source_error.lock() = Some(LastSourceError {
|
||||
class: err.class_label().to_string(),
|
||||
at,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn last_source_error(&self) -> Option<LastSourceError> {
|
||||
self.last_source_error.lock().clone()
|
||||
}
|
||||
|
||||
pub fn inflight_pulls(&self) -> u64 {
|
||||
self.inflight_pulls.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn queue_depth(&self) -> u64 {
|
||||
self.queue_depth.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// RAII increment of `inflight_pulls`.
|
||||
pub fn inflight_guard(self: &Arc<Self>) -> GaugeGuard {
|
||||
GaugeGuard::new(Arc::clone(self), OdmGauge::InflightPulls)
|
||||
}
|
||||
|
||||
/// RAII increment of `queue_depth`.
|
||||
pub fn queue_guard(self: &Arc<Self>) -> GaugeGuard {
|
||||
GaugeGuard::new(Arc::clone(self), OdmGauge::QueueDepth)
|
||||
}
|
||||
|
||||
fn gauge(&self, gauge: OdmGauge) -> &AtomicU64 {
|
||||
match gauge {
|
||||
OdmGauge::InflightPulls => &self.inflight_pulls,
|
||||
OdmGauge::QueueDepth => &self.queue_depth,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only, side-effect-free copy of every counter. The breaker lives
|
||||
/// next to the stats in the bucket state; its state is passed in so the
|
||||
/// snapshot stays a single document.
|
||||
pub fn snapshot(&self, breaker_state: BreakerState) -> OdmStatsSnapshot {
|
||||
let mut requests_total = BTreeMap::new();
|
||||
for op in OdmOp::ALL {
|
||||
let mut by_outcome = BTreeMap::new();
|
||||
for outcome in OdmOutcome::ALL {
|
||||
by_outcome.insert(
|
||||
outcome.as_str().to_string(),
|
||||
self.requests_total[op as usize][outcome as usize].load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
requests_total.insert(op.as_str().to_string(), by_outcome);
|
||||
}
|
||||
let pulled_objects_total = PullPath::ALL
|
||||
.iter()
|
||||
.map(|path| {
|
||||
(
|
||||
path.as_str().to_string(),
|
||||
self.pulled_objects_total[*path as usize].load(Ordering::Relaxed),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let pull_failures_total = PullFailureReason::ALL
|
||||
.iter()
|
||||
.map(|reason| {
|
||||
(
|
||||
reason.as_str().to_string(),
|
||||
self.pull_failures_total[*reason as usize].load(Ordering::Relaxed),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
OdmStatsSnapshot {
|
||||
requests_total,
|
||||
pulled_bytes_total: self.pulled_bytes_total.load(Ordering::Relaxed),
|
||||
pulled_objects_total,
|
||||
pull_failures_total,
|
||||
inflight_pulls: self.inflight_pulls(),
|
||||
queue_depth: self.queue_depth(),
|
||||
source_latency: self.source_latency.snapshot(),
|
||||
last_source_error: self.last_source_error(),
|
||||
breaker_state,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum OdmGauge {
|
||||
InflightPulls,
|
||||
QueueDepth,
|
||||
}
|
||||
|
||||
/// Increments a gauge on creation and decrements it on drop. Owns its
|
||||
/// `OdmStats` so it can live inside the pull slot handed to callers.
|
||||
#[derive(Debug)]
|
||||
pub struct GaugeGuard {
|
||||
stats: Arc<OdmStats>,
|
||||
gauge: OdmGauge,
|
||||
}
|
||||
|
||||
impl GaugeGuard {
|
||||
fn new(stats: Arc<OdmStats>, gauge: OdmGauge) -> Self {
|
||||
stats.gauge(gauge).fetch_add(1, Ordering::Relaxed);
|
||||
Self { stats, gauge }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GaugeGuard {
|
||||
fn drop(&mut self) {
|
||||
self.stats.gauge(self.gauge).fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LatencyBucketSnapshot {
|
||||
/// Upper bound of the bucket in milliseconds.
|
||||
pub le_ms: u64,
|
||||
/// Cumulative observations at or below `le_ms`.
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SourceLatencySnapshot {
|
||||
pub buckets: Vec<LatencyBucketSnapshot>,
|
||||
/// Total observations, including those above the last bound.
|
||||
pub count: u64,
|
||||
pub sum_ms: u64,
|
||||
}
|
||||
|
||||
/// Serializable copy of [`OdmStats`]. Every key is snake_case and every
|
||||
/// label set is fixed, so consumers can rely on the document shape.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OdmStatsSnapshot {
|
||||
/// `op -> outcome -> count`.
|
||||
pub requests_total: BTreeMap<String, BTreeMap<String, u64>>,
|
||||
pub pulled_bytes_total: u64,
|
||||
/// `path -> count`.
|
||||
pub pulled_objects_total: BTreeMap<String, u64>,
|
||||
/// `reason -> count`.
|
||||
pub pull_failures_total: BTreeMap<String, u64>,
|
||||
pub inflight_pulls: u64,
|
||||
pub queue_depth: u64,
|
||||
pub source_latency: SourceLatencySnapshot,
|
||||
pub last_source_error: Option<LastSourceError>,
|
||||
pub breaker_state: BreakerState,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
use time::macros::datetime;
|
||||
|
||||
#[test]
|
||||
fn snapshot_matches_golden_json() {
|
||||
let stats = Arc::new(OdmStats::new());
|
||||
stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
|
||||
stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
|
||||
stats.record_request(OdmOp::Head, OdmOutcome::NegativeCached);
|
||||
stats.record_pulled_bytes(4096);
|
||||
stats.record_pulled_object(PullPath::Inline);
|
||||
stats.record_pull_failure(PullFailureReason::from(&SourceError::Timeout));
|
||||
stats.record_source_latency(Duration::from_millis(3));
|
||||
stats.record_source_latency(Duration::from_millis(750));
|
||||
stats.record_source_latency(Duration::from_secs(90));
|
||||
stats.record_source_error_at(&SourceError::ServerError(502), datetime!(2026-09-02 10:00:00 UTC));
|
||||
let _inflight = stats.inflight_guard();
|
||||
let _queued = stats.queue_guard();
|
||||
|
||||
let snapshot = stats.snapshot(BreakerState::HalfOpen);
|
||||
let actual = serde_json::to_value(&snapshot).unwrap();
|
||||
let expected = json!({
|
||||
"requests_total": {
|
||||
"get": {
|
||||
"breaker_open": 0, "filtered": 0, "negative_cached": 0, "source_error": 0,
|
||||
"source_hit": 2, "source_miss": 0, "unsupported": 0
|
||||
},
|
||||
"head": {
|
||||
"breaker_open": 0, "filtered": 0, "negative_cached": 1, "source_error": 0,
|
||||
"source_hit": 0, "source_miss": 0, "unsupported": 0
|
||||
}
|
||||
},
|
||||
"pulled_bytes_total": 4096,
|
||||
"pulled_objects_total": { "backfill": 0, "background": 0, "inline": 1 },
|
||||
"pull_failures_total": {
|
||||
"canceled": 0, "etag_mismatch": 0, "local_write": 0, "queue_full": 0, "quota": 0,
|
||||
"source_access_denied": 0, "source_connect": 0, "source_not_found": 0, "source_other": 0,
|
||||
"source_server_error": 0, "source_throttled": 0, "source_timeout": 1, "source_unsupported": 0
|
||||
},
|
||||
"inflight_pulls": 1,
|
||||
"queue_depth": 1,
|
||||
"source_latency": {
|
||||
"buckets": [
|
||||
{ "le_ms": 5, "count": 1 }, { "le_ms": 10, "count": 1 }, { "le_ms": 20, "count": 1 },
|
||||
{ "le_ms": 50, "count": 1 }, { "le_ms": 100, "count": 1 }, { "le_ms": 200, "count": 1 },
|
||||
{ "le_ms": 500, "count": 1 }, { "le_ms": 1000, "count": 2 }, { "le_ms": 2000, "count": 2 },
|
||||
{ "le_ms": 5000, "count": 2 }, { "le_ms": 10000, "count": 2 }, { "le_ms": 20000, "count": 2 },
|
||||
{ "le_ms": 30000, "count": 2 }, { "le_ms": 60000, "count": 2 }
|
||||
],
|
||||
"count": 3,
|
||||
"sum_ms": 90753
|
||||
},
|
||||
"last_source_error": { "class": "server_error", "at": "2026-09-02T10:00:00Z" },
|
||||
"breaker_state": "half_open"
|
||||
});
|
||||
assert_eq!(actual, expected);
|
||||
|
||||
let round_trip: OdmStatsSnapshot = serde_json::from_value(actual).unwrap();
|
||||
assert_eq!(round_trip, snapshot);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gauges_return_to_zero_when_guards_drop() {
|
||||
let stats = Arc::new(OdmStats::new());
|
||||
{
|
||||
let _a = stats.inflight_guard();
|
||||
let _b = stats.inflight_guard();
|
||||
let _c = stats.queue_guard();
|
||||
assert_eq!(stats.inflight_pulls(), 2);
|
||||
assert_eq!(stats.queue_depth(), 1);
|
||||
}
|
||||
assert_eq!(stats.inflight_pulls(), 0);
|
||||
assert_eq!(stats.queue_depth(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pull_failure_reason_covers_every_source_error_class() {
|
||||
let cases = [
|
||||
(SourceError::NotFound, PullFailureReason::SourceNotFound),
|
||||
(SourceError::AccessDenied, PullFailureReason::SourceAccessDenied),
|
||||
(SourceError::Throttled, PullFailureReason::SourceThrottled),
|
||||
(SourceError::Timeout, PullFailureReason::SourceTimeout),
|
||||
(SourceError::Connect("x".into()), PullFailureReason::SourceConnect),
|
||||
(SourceError::ServerError(500), PullFailureReason::SourceServerError),
|
||||
(SourceError::Unsupported("x".into()), PullFailureReason::SourceUnsupported),
|
||||
(SourceError::Other("x".into()), PullFailureReason::SourceOther),
|
||||
];
|
||||
for (err, reason) in cases {
|
||||
assert_eq!(PullFailureReason::from(&err), reason, "{err:?}");
|
||||
assert_eq!(serde_json::to_string(&reason).unwrap(), format!("\"{}\"", reason.as_str()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_lists_are_exhaustive_and_unique() {
|
||||
let outcomes: std::collections::BTreeSet<_> = OdmOutcome::ALL.iter().map(|o| o.as_str()).collect();
|
||||
assert_eq!(outcomes.len(), OdmOutcome::ALL.len());
|
||||
let reasons: std::collections::BTreeSet<_> = PullFailureReason::ALL.iter().map(|r| r.as_str()).collect();
|
||||
assert_eq!(reasons.len(), PullFailureReason::ALL.len());
|
||||
let paths: std::collections::BTreeSet<_> = PullPath::ALL.iter().map(|p| p.as_str()).collect();
|
||||
assert_eq!(paths.len(), PullPath::ALL.len());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Storage capabilities used by on-demand migration orchestration.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) use crate::storage_api::on_demand_migration::test_support;
|
||||
pub(super) use crate::storage_api::on_demand_migration::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_META_PREFIX, BUCKET_ON_DEMAND_MIGRATION_CONFIG, ECStore, HTTPPreconditions, HTTPRangeSpec,
|
||||
NamespaceLocking, ObjectOperations, ObjectOptions, RUSTFS_META_BUCKET, StorageError, WriteCompletion,
|
||||
get_lock_acquire_timeout, get_on_demand_migration_config, get_on_demand_migration_config_in, read_config_with_metadata,
|
||||
remote_s3_client, save_config_with_opts,
|
||||
};
|
||||
|
||||
pub(super) async fn local_node_name() -> String {
|
||||
rustfs_common::get_global_local_node_name().await
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Scripted HTTP server for the native source backends' tests.
|
||||
//!
|
||||
//! The S3 backend can be driven through the SDK's own connector; the native
|
||||
//! backends talk to a real socket, so their tests need a server that answers a
|
||||
//! fixed script and records what it was asked. Every response closes its
|
||||
//! connection, which keeps one request on one socket and makes the script order
|
||||
//! exactly the request order.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use url::Url;
|
||||
|
||||
pub(super) struct ScriptedResponse {
|
||||
status: u16,
|
||||
headers: Vec<(&'static str, String)>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl ScriptedResponse {
|
||||
pub(super) fn new(status: u16, headers: Vec<(&'static str, String)>, body: String) -> Self {
|
||||
Self { status, headers, body }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct RecordedRequest {
|
||||
pub(super) method: String,
|
||||
/// Request target as it appeared on the wire: path plus query.
|
||||
pub(super) target: String,
|
||||
pub(super) headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl RecordedRequest {
|
||||
pub(super) fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;
|
||||
|
||||
/// Binds a loopback listener that answers `responses` in order and returns its
|
||||
/// origin plus the recorder. The task ends once the script is exhausted.
|
||||
pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind");
|
||||
let port = listener.local_addr().expect("fixture address").port();
|
||||
let recorder: Recorder = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&recorder);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for response in responses {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 2048];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
match stream.read(&mut buffer).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(read) => request.extend_from_slice(&buffer[..read]),
|
||||
}
|
||||
}
|
||||
let text = String::from_utf8_lossy(&request).into_owned();
|
||||
let mut lines = text.lines();
|
||||
let start = lines.next().unwrap_or_default().to_string();
|
||||
let mut parts = start.split_whitespace();
|
||||
sink.lock().expect("recorder lock").push(RecordedRequest {
|
||||
method: parts.next().unwrap_or_default().to_string(),
|
||||
target: parts.next().unwrap_or_default().to_string(),
|
||||
headers: lines
|
||||
.take_while(|line| !line.is_empty())
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
|
||||
.collect(),
|
||||
});
|
||||
|
||||
// A scripted HEAD declares the object size in its own headers while
|
||||
// carrying no body, so an explicit `Content-Length` wins over the
|
||||
// body length.
|
||||
let declares_length = response
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("content-length"));
|
||||
let mut rendered = match declares_length {
|
||||
true => format!("HTTP/1.1 {} Scripted\r\nConnection: close\r\n", response.status),
|
||||
false => format!(
|
||||
"HTTP/1.1 {} Scripted\r\nContent-Length: {}\r\nConnection: close\r\n",
|
||||
response.status,
|
||||
response.body.len()
|
||||
),
|
||||
};
|
||||
for (name, value) in response.headers {
|
||||
rendered.push_str(&format!("{name}: {value}\r\n"));
|
||||
}
|
||||
rendered.push_str("\r\n");
|
||||
rendered.push_str(&response.body);
|
||||
let _ = stream.write_all(rendered.as_bytes()).await;
|
||||
let _ = stream.flush().await;
|
||||
}
|
||||
});
|
||||
|
||||
(Url::parse(&format!("http://127.0.0.1:{port}")).expect("fixture endpoint"), recorder)
|
||||
}
|
||||
@@ -17,10 +17,11 @@ use crate::module_switches::{
|
||||
bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env,
|
||||
is_on_demand_migration_module_enabled, scanner_enabled_from_env,
|
||||
};
|
||||
use crate::storage_api::startup::background::{
|
||||
BackfillRunner, ECStore, OnDemandMigrationSys, SysBackfillContexts, install_global_backfill_runner,
|
||||
set_workload_admission_snapshot_provider, spawn_backfill_recovery_loop,
|
||||
use crate::on_demand_migration::OnDemandMigrationSys;
|
||||
use crate::on_demand_migration::backfill::{
|
||||
BackfillRunner, SysBackfillContexts, install_global_backfill_runner, spawn_backfill_recovery_loop,
|
||||
};
|
||||
use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider};
|
||||
use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_heal::{
|
||||
|
||||
@@ -14,10 +14,11 @@
|
||||
|
||||
use crate::app::object::OnDemandMigrationWriteBack;
|
||||
use crate::module_switches::{on_demand_migration_enabled_from_env, set_on_demand_migration_module_enabled};
|
||||
use crate::on_demand_migration::OnDemandMigrationSys;
|
||||
use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage_api::startup::bucket_metadata::{
|
||||
ECStore, Error as StorageError, OnDemandMigrationSys, Result as StorageResult, get_global_replication_pool,
|
||||
init_bucket_metadata_sys, reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
ECStore, Error as StorageError, Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys,
|
||||
reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
};
|
||||
use std::{
|
||||
io::{Error as IoError, Result as IoResult},
|
||||
@@ -98,6 +99,7 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
|
||||
/// `OnDemandMigrationSys` with a usable write-back (rustfs/backlog#2152).
|
||||
/// Idempotent across embedded and server startups.
|
||||
fn init_on_demand_migration_runtime() {
|
||||
crate::on_demand_migration::register_metrics();
|
||||
let enabled = on_demand_migration_enabled_from_env();
|
||||
set_on_demand_migration_module_enabled(enabled);
|
||||
let sys = OnDemandMigrationSys::get();
|
||||
|
||||
@@ -407,8 +407,8 @@ pub(crate) mod ecstore_bucket {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, on_demand_migration,
|
||||
policy_sys, remote_s3_client, replication, tagging, target, utils,
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
|
||||
remote_s3_client, replication, tagging, target, utils,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys};
|
||||
}
|
||||
@@ -466,10 +466,10 @@ pub(crate) mod ecstore_data_usage {
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_disk {
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq,
|
||||
ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
get_object_disk_read_timeout, validate_batch_read_version_item_count,
|
||||
BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo,
|
||||
DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction,
|
||||
RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
|
||||
}
|
||||
@@ -560,7 +560,7 @@ pub(crate) mod ecstore_object {
|
||||
pub(crate) use rustfs_ecstore::api::object::{
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup,
|
||||
ObjectEncryptionResolver, ObjectMutationHook, PrepareSelectObjectSnapshotError, ReadEncryptionMaterial,
|
||||
ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, get_object_body_cache_plaintext_len,
|
||||
ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, WriteCompletion, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
|
||||
@@ -278,10 +278,6 @@ pub(crate) mod startup {
|
||||
}
|
||||
|
||||
pub(crate) mod background {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::backfill::{
|
||||
BackfillRunner, SysBackfillContexts, install_global_backfill_runner, spawn_backfill_recovery_loop,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider,
|
||||
};
|
||||
@@ -294,7 +290,6 @@ pub(crate) mod startup {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, Error, Result, get_global_replication_pool, init_bucket_metadata_sys,
|
||||
reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
@@ -415,3 +410,30 @@ pub(crate) mod table {
|
||||
get_lock_acquire_timeout, table_catalog_path_hash,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod on_demand_migration {
|
||||
pub(crate) use crate::storage::storage_api::ECStore;
|
||||
pub(crate) use crate::storage::storage_api::StorageObjectOptions as ObjectOptions;
|
||||
pub(crate) use crate::storage::storage_api::contract::{
|
||||
namespace::NamespaceLocking, object::HTTPPreconditions, object::ObjectOperations, range::HTTPRangeSpec,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, get_on_demand_migration_config, get_on_demand_migration_config_in,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::remote_s3_client;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_config::com::{read_config_with_metadata, save_config_with_opts};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_error::Error as StorageError;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_object::WriteCompletion;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_set_disk::get_lock_acquire_timeout;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::{
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG, BucketMetadata,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::BucketMetadataSys;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user