mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 12:09:12 +00:00
Merge branch 'main' into cxymds/fix/tier-remove-backend-in-use
This commit is contained in:
Zhengchao An
@@ -20,9 +20,10 @@
|
||||
//! journal (`count_requests`) carries the assertion in every one of them.
|
||||
|
||||
use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
|
||||
use crate::fake_s3_target::Operation;
|
||||
use crate::fake_s3_target::{FaultAction, Operation};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use std::time::Duration;
|
||||
|
||||
type TestResult = Result<(), BoxError>;
|
||||
@@ -145,14 +146,38 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
.await?;
|
||||
|
||||
let body = payload(128 * 1024);
|
||||
let blocker = "queue/blocker.bin";
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
|
||||
// The one-chunk range completes immediately; its full background pull
|
||||
// occupies the only slot while the remaining requests fill the queue.
|
||||
env.source.inject_for_key(
|
||||
Operation::GetObject,
|
||||
blocker,
|
||||
FaultAction::SlowSendBody {
|
||||
chunk_bytes: 1024,
|
||||
delay: Duration::from_millis(100),
|
||||
},
|
||||
2,
|
||||
);
|
||||
let response = env
|
||||
.raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
|
||||
.await?;
|
||||
assert_eq!(response.status, 206);
|
||||
assert_eq!(response.body, body.slice(0..1024));
|
||||
env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
|
||||
|
||||
let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
|
||||
let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
|
||||
env.seed_source(SOURCE_BUCKET, &seeds);
|
||||
|
||||
let responses: Vec<RawResponse> = futures::future::try_join_all(
|
||||
// Bound source connections below the fixture's limit while still
|
||||
// submitting all 100 requests to the eight-slot background queue.
|
||||
let responses: Vec<RawResponse> = futures::stream::iter(
|
||||
keys.iter()
|
||||
.map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
|
||||
)
|
||||
.buffered(16)
|
||||
.try_collect()
|
||||
.await?;
|
||||
for (key, response) in keys.iter().zip(&responses) {
|
||||
assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
|
||||
@@ -168,6 +193,15 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
.wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
|
||||
.await?;
|
||||
assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
|
||||
let queue_full = usize::try_from(queue_full)?;
|
||||
assert!(queue_full <= REQUESTS);
|
||||
env.wait_for_status_counter(
|
||||
bucket,
|
||||
"/counters/pulled_objects_total/background",
|
||||
u64::try_from(REQUESTS + 1 - queue_full)?,
|
||||
SETTLE,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
|
||||
assert!(
|
||||
@@ -175,9 +209,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
"every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
|
||||
);
|
||||
let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
|
||||
assert!(
|
||||
dropped > 0,
|
||||
"the overflowed keys are the ones with no backfill GET, but every key got one"
|
||||
);
|
||||
assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -265,16 +265,13 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
|
||||
assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
|
||||
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
|
||||
let rejected = env
|
||||
.raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rejected.status,
|
||||
400,
|
||||
"a bumped token version is a client error: {}",
|
||||
String::from_utf8_lossy(&rejected.body)
|
||||
);
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
|
||||
assert_ne!(tampered, token, "the test must change the token version");
|
||||
let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
|
||||
let rejected = env.raw_list_objects_v2(bucket, &query).await?;
|
||||
let error_body = String::from_utf8_lossy(&rejected.body);
|
||||
assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
|
||||
assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -12,21 +12,34 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use crate::common::{
|
||||
RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
|
||||
};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
|
||||
ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::{Instant, sleep};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -40,6 +53,32 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
|
||||
// comfortably covers that window plus CI scheduling jitter.
|
||||
const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089).
|
||||
const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain";
|
||||
const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted";
|
||||
const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated";
|
||||
const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked";
|
||||
const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica";
|
||||
const ROLLBACK_BUCKET: &str = "rollback-config-data";
|
||||
const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica";
|
||||
const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024;
|
||||
const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs";
|
||||
const LIFECYCLE_PREFIX: &str = "logs/";
|
||||
const LIFECYCLE_DAYS: i32 = 30;
|
||||
const BUCKET_TAG_KEY: &str = "owner";
|
||||
const BUCKET_TAG_VALUE: &str = "upgrade-compatibility";
|
||||
const OBJECT_LOCK_DAYS: i32 = 1;
|
||||
// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage
|
||||
// authoritative; the quota test uses the same 30s budget.
|
||||
const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Quota admission fails closed while a freshly started server has neither
|
||||
// authoritative usage nor a persisted degraded baseline for the bucket
|
||||
// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that
|
||||
// window. It is a restart property, not an upgrade property — the same window
|
||||
// opens on the very first start — so the write assertions ride it out instead
|
||||
// of treating it as an upgrade failure.
|
||||
const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
|
||||
|
||||
fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = std::env::var_os(SOURCE_BINARY_ENV)
|
||||
.map(PathBuf::from)
|
||||
@@ -240,6 +279,93 @@ async fn exercise_mixed_cluster(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pins the published old writer's limitation and the supported recovery
|
||||
/// procedure. This is not a promise that mixed-version ODM is supported.
|
||||
/// Replace the loss assertion when ODM gains independent persistence;
|
||||
/// preserving configuration across rc.5 writes is then an improvement.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let version = tokio::process::Command::new(&previous_binary)
|
||||
.arg("--version")
|
||||
.output()
|
||||
.await?;
|
||||
assert!(version.status.success(), "previous binary must report its version");
|
||||
assert!(
|
||||
String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
|
||||
"this compatibility scenario requires the published rc.5 writer"
|
||||
);
|
||||
let mut env = OdmTestEnv::start().await?;
|
||||
let bucket = "odm-rc5-rollback";
|
||||
let source_bucket = "odm-rc5-source";
|
||||
env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
|
||||
env.seed_source(
|
||||
source_bucket,
|
||||
&[SeedObject::new(
|
||||
"source-only",
|
||||
bytes::Bytes::from_static(b"source read after recovery"),
|
||||
)],
|
||||
);
|
||||
env.rustfs.create_test_bucket(bucket).await?;
|
||||
let saved_config = env.fake_source_spec(source_bucket);
|
||||
assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
|
||||
let before = env.get_config(bucket).await?;
|
||||
assert_eq!(before.status, 200);
|
||||
let expected_config = before
|
||||
.json()?
|
||||
.get("config")
|
||||
.cloned()
|
||||
.ok_or("configuration response omitted config")?;
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("local")
|
||||
.body(ByteStream::from_static(b"local data survives rollback"))
|
||||
.send()
|
||||
.await?;
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
let restarted = env.get_config(bucket).await?;
|
||||
assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
|
||||
assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
|
||||
|
||||
restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
|
||||
env.client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("writer").value("rc5").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
let missing = env.get_config(bucket).await?;
|
||||
assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
|
||||
assert!(missing.body.contains("NoSuchConfiguration"));
|
||||
assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
|
||||
let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
|
||||
assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
|
||||
|
||||
assert_eq!(
|
||||
env.configure_source(bucket, &saved_config).await?.status,
|
||||
200,
|
||||
"restore from saved full configuration"
|
||||
);
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
let restored = env.get_config(bucket).await?;
|
||||
assert_eq!(restored.status, 200, "restored ODM configuration persists");
|
||||
assert_eq!(restored.json()?.get("config"), Some(&expected_config));
|
||||
env.wait_until_source_consulted(bucket).await?;
|
||||
assert_eq!(
|
||||
read_object(&env.client, bucket, "source-only", None).await?.1,
|
||||
b"source read after recovery"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
|
||||
@@ -429,3 +555,653 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Child-process environment shared by both bucket-configuration scenarios.
|
||||
///
|
||||
/// The replication target is an in-process fake bound to `127.0.0.1`, which
|
||||
/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and
|
||||
/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the
|
||||
/// server's outbound health check.
|
||||
fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> {
|
||||
let mut env = vec![
|
||||
(SSE_MASTER_KEY_ENV, SSE_MASTER_KEY),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// Shorten the scanner cycle so the bucket's usage becomes authoritative
|
||||
// in seconds; both `set-bucket-quota` and quota admission block on it.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
];
|
||||
env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env.extend(replication_fast_env());
|
||||
env
|
||||
}
|
||||
|
||||
/// Restart `env` in place on the same data directory using an explicit binary.
|
||||
///
|
||||
/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches
|
||||
/// the workspace build, which is the upgrade direction only. The rollback
|
||||
/// scenario needs the reverse: stop the current build and bring the pinned
|
||||
/// previous release up on the metadata that build just wrote.
|
||||
async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult {
|
||||
env.stop_server();
|
||||
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
|
||||
}
|
||||
|
||||
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
|
||||
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
|
||||
loop {
|
||||
let (status, response) =
|
||||
admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline {
|
||||
return Err(format!("setting the quota of {bucket} failed: {status} {response}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// PUT into a quota-enabled bucket, riding out the post-start quota-admission
|
||||
/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`].
|
||||
///
|
||||
/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that
|
||||
/// never ends, is a genuine regression and surfaces as an error.
|
||||
async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult {
|
||||
let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT;
|
||||
loop {
|
||||
let result = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.send()
|
||||
.await;
|
||||
let error = match result {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => error,
|
||||
};
|
||||
let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable");
|
||||
if !retryable || Instant::now() >= deadline {
|
||||
return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result<Option<u64>, BoxError> {
|
||||
let path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
|
||||
if status != StatusCode::OK {
|
||||
return Err(format!("reading the quota of {bucket} failed: {status} {response}").into());
|
||||
}
|
||||
let quota: serde_json::Value = serde_json::from_str(&response)?;
|
||||
Ok(quota.get("quota").and_then(serde_json::Value::as_u64))
|
||||
}
|
||||
|
||||
/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`.
|
||||
///
|
||||
/// Returns an error for any non-200, because rustfs#7172 made this endpoint
|
||||
/// fail closed on a `bucket-targets.json` blob the running build cannot parse.
|
||||
/// An upgrade that misreads a blob written by the previous release therefore
|
||||
/// shows up here as an error, and a silently dropped target shows up as an
|
||||
/// empty list — the caller must distinguish the two.
|
||||
async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result<Vec<serde_json::Value>, BoxError> {
|
||||
let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket));
|
||||
let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
|
||||
if status != StatusCode::OK {
|
||||
return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into());
|
||||
}
|
||||
Ok(serde_json::from_str(&response)?)
|
||||
}
|
||||
|
||||
/// Assert that `bucket` still carries exactly the replication target `arn`.
|
||||
async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult {
|
||||
let targets = list_remote_targets(env, bucket).await?;
|
||||
assert_eq!(
|
||||
targets.len(),
|
||||
1,
|
||||
"{context}: list-remote-targets must still report the single configured target, got {targets:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
targets[0].get("arn").and_then(serde_json::Value::as_str),
|
||||
Some(arn),
|
||||
"{context}: the target ARN changed across the restart: {targets:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Configure a replication target on `bucket` pointing at the in-process fake,
|
||||
/// then attach an enabled replication rule for it. Returns the target ARN.
|
||||
async fn configure_replication(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
) -> Result<String, BoxError> {
|
||||
let arn = set_replication_target_with_options(
|
||||
env,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(env, bucket, &arn).await?;
|
||||
Ok(arn)
|
||||
}
|
||||
|
||||
async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult {
|
||||
let configuration = ServerSideEncryptionConfiguration::builder()
|
||||
.rules(
|
||||
ServerSideEncryptionRule::builder()
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::Aes256)
|
||||
.build()?,
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()?;
|
||||
client
|
||||
.put_bucket_encryption()
|
||||
.bucket(bucket)
|
||||
.server_side_encryption_configuration(configuration)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult {
|
||||
let response = client.get_bucket_encryption().bucket(bucket).send().await?;
|
||||
let rules = response
|
||||
.server_side_encryption_configuration()
|
||||
.ok_or("GetBucketEncryption omitted the configuration")?
|
||||
.rules();
|
||||
assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}");
|
||||
assert_eq!(
|
||||
rules[0]
|
||||
.apply_server_side_encryption_by_default()
|
||||
.map(ServerSideEncryptionByDefault::sse_algorithm),
|
||||
Some(&ServerSideEncryption::Aes256),
|
||||
"{context}: the default encryption algorithm changed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult {
|
||||
let tagging = Tagging::builder()
|
||||
.tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?)
|
||||
.build()?;
|
||||
client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult {
|
||||
let tags = client.get_bucket_tagging().bucket(bucket).send().await?;
|
||||
let tag_set = tags.tag_set();
|
||||
assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}");
|
||||
assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed");
|
||||
assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult {
|
||||
let versioning = client.get_bucket_versioning().bucket(bucket).send().await?;
|
||||
assert_eq!(
|
||||
versioning.status(),
|
||||
Some(&BucketVersioningStatus::Enabled),
|
||||
"{context}: versioning is no longer Enabled on {bucket}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bucket_policy_document(bucket: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Sid": "UpgradePublicRead",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": ["*"] },
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/public/*")]
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
/// `GET .../on-demand-migration/{bucket}/status`.
|
||||
///
|
||||
/// The migration module defaults on from rustfs#7089, so a bucket that never
|
||||
/// configured a source must still answer `configured: false` rather than
|
||||
/// engaging the migration path.
|
||||
async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status");
|
||||
let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"the migration status endpoint must answer for an unconfigured bucket: {status} {response}"
|
||||
);
|
||||
let body: serde_json::Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(
|
||||
body.get("configured"),
|
||||
Some(&serde_json::Value::Bool(false)),
|
||||
"a bucket upgraded from the previous release must not look migration-configured: {body}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A GET for a key that was never written must be a plain `NoSuchKey`.
|
||||
///
|
||||
/// With the migration module on by default this is the cheap proof that an
|
||||
/// unconfigured bucket never consults a source: any migration engagement would
|
||||
/// surface as a different status or error code here.
|
||||
async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult {
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a key that was never written must not be readable");
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"a missing key must stay a 404 on a bucket with no migration configuration"
|
||||
);
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"a missing key must stay NoSuchKey on a bucket with no migration configuration"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bucket configuration written by the pinned previous release must survive an
|
||||
/// upgrade to the current build unchanged, and must keep working.
|
||||
///
|
||||
/// This pins the three on-disk surfaces the on-demand-migration series moved:
|
||||
///
|
||||
/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so
|
||||
/// every configuration read below decodes a 44-key blob on 46-key code.
|
||||
/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption /
|
||||
/// public-access-block / quota blob "present but unreadable" instead of
|
||||
/// silently defaulting, and made `list-remote-targets` fail closed on it. A
|
||||
/// replication target configured by the old release must therefore still be
|
||||
/// *listed*, not dropped and not an error.
|
||||
/// * rustfs#7183 made the object write path refuse a PUT when the bucket's
|
||||
/// encryption configuration cannot be read, so a misparsed SSE config would
|
||||
/// turn every PUT to that bucket into a 500.
|
||||
///
|
||||
/// Not covered on purpose: on-demand-migration configuration itself, which the
|
||||
/// previous release has no public API for — the reverse direction is asserted
|
||||
/// instead (an upgraded bucket reports `configured: false`).
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
|
||||
// In-process: the fake target outlives both server processes, so the
|
||||
// replication target stays reachable across the upgrade.
|
||||
let replication_target = FakeS3Target::start().await?;
|
||||
replication_target.create_bucket(CONFIG_REPLICA_BUCKET);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let server_env = bucket_config_server_env();
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?;
|
||||
env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?;
|
||||
old_client
|
||||
.create_bucket()
|
||||
.bucket(CONFIG_LOCKED_BUCKET)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Plain bucket: policy, tags, lifecycle, quota.
|
||||
let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET);
|
||||
old_client
|
||||
.put_bucket_policy()
|
||||
.bucket(CONFIG_PLAIN_BUCKET)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?;
|
||||
old_client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(CONFIG_PLAIN_BUCKET)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(
|
||||
LifecycleRule::builder()
|
||||
.id(LIFECYCLE_RULE_ID)
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build())
|
||||
.expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build())
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?;
|
||||
|
||||
// Encrypted bucket: SSE-S3 default encryption plus a fully restrictive
|
||||
// public access block, both of which rustfs#7172 now fails closed on.
|
||||
put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?;
|
||||
old_client
|
||||
.put_public_access_block()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.public_access_block_configuration(
|
||||
PublicAccessBlockConfiguration::builder()
|
||||
.block_public_acls(true)
|
||||
.ignore_public_acls(true)
|
||||
.block_public_policy(true)
|
||||
.restrict_public_buckets(true)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Replicated bucket: versioning, a validated remote target, a rule.
|
||||
enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?;
|
||||
let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?;
|
||||
assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?;
|
||||
|
||||
// Object-lock bucket: a default GOVERNANCE retention on a fresh bucket.
|
||||
old_client
|
||||
.put_object_lock_configuration()
|
||||
.bucket(CONFIG_LOCKED_BUCKET)
|
||||
.object_lock_configuration(
|
||||
ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.days(OBJECT_LOCK_DAYS)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let plain_key = "plain/written-by-previous";
|
||||
let plain_bytes = b"plain object written by the previous RustFS release";
|
||||
put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?;
|
||||
|
||||
let encrypted_key = "encrypted/written-by-previous";
|
||||
let encrypted_bytes = b"default-encrypted object written by the previous RustFS release";
|
||||
old_client
|
||||
.put_object()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.key(encrypted_key)
|
||||
.body(ByteStream::from_static(encrypted_bytes))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None)
|
||||
.await?
|
||||
.0,
|
||||
Some(ServerSideEncryption::Aes256),
|
||||
"the previous release must apply the bucket default encryption it just accepted"
|
||||
);
|
||||
|
||||
// The multipart object lives in the default-encrypted bucket so the
|
||||
// upgraded build has to reassemble parts *and* re-derive the object key.
|
||||
let multipart_key = "encrypted/multipart-written-by-previous";
|
||||
let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()];
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
let versioned_key = "versioned/written-by-previous";
|
||||
let versioned_bytes = b"versioned object written by the previous RustFS release";
|
||||
let versioned_id = old_client
|
||||
.put_object()
|
||||
.bucket(CONFIG_REPLICATED_BUCKET)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from_static(versioned_bytes))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let new_client = env.create_s3_client();
|
||||
|
||||
// Every configuration must read back unchanged on the upgraded build.
|
||||
let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?;
|
||||
let upgraded_policy: serde_json::Value =
|
||||
serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?;
|
||||
assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade");
|
||||
assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?;
|
||||
|
||||
let lifecycle = new_client
|
||||
.get_bucket_lifecycle_configuration()
|
||||
.bucket(CONFIG_PLAIN_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let rules = lifecycle.rules();
|
||||
assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}");
|
||||
assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID));
|
||||
assert_eq!(rules[0].status(), &ExpirationStatus::Enabled);
|
||||
assert_eq!(
|
||||
rules[0].expiration().and_then(LifecycleExpiration::days),
|
||||
Some(LIFECYCLE_DAYS),
|
||||
"the lifecycle expiration changed across the upgrade"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?,
|
||||
Some(BUCKET_QUOTA_BYTES),
|
||||
"the bucket quota changed across the upgrade"
|
||||
);
|
||||
|
||||
assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?;
|
||||
let public_access_block = new_client
|
||||
.get_public_access_block()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let public_access_block = public_access_block
|
||||
.public_access_block_configuration()
|
||||
.ok_or("GetPublicAccessBlock omitted the configuration")?;
|
||||
assert_eq!(public_access_block.block_public_acls(), Some(true));
|
||||
assert_eq!(public_access_block.ignore_public_acls(), Some(true));
|
||||
assert_eq!(public_access_block.block_public_policy(), Some(true));
|
||||
assert_eq!(public_access_block.restrict_public_buckets(), Some(true));
|
||||
|
||||
assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?;
|
||||
// rustfs#7172: neither an empty list nor an error is acceptable here.
|
||||
assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?;
|
||||
let replication = new_client
|
||||
.get_bucket_replication()
|
||||
.bucket(CONFIG_REPLICATED_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let replication_rules = replication
|
||||
.replication_configuration()
|
||||
.ok_or("GetBucketReplication omitted the configuration")?
|
||||
.rules();
|
||||
assert_eq!(
|
||||
replication_rules.len(),
|
||||
1,
|
||||
"the replication rule count changed across the upgrade: {replication_rules:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
replication_rules[0].destination().map(|destination| destination.bucket()),
|
||||
Some(target_arn.as_str()),
|
||||
"the replication rule no longer points at the configured target"
|
||||
);
|
||||
|
||||
let object_lock = new_client
|
||||
.get_object_lock_configuration()
|
||||
.bucket(CONFIG_LOCKED_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let object_lock = object_lock
|
||||
.object_lock_configuration()
|
||||
.ok_or("GetObjectLockConfiguration omitted the configuration")?;
|
||||
assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled));
|
||||
let retention = object_lock
|
||||
.rule()
|
||||
.and_then(ObjectLockRule::default_retention)
|
||||
.ok_or("the object lock configuration lost its default retention")?;
|
||||
assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance));
|
||||
assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS));
|
||||
|
||||
// rustfs#7183: a PUT into the default-encrypted bucket must still succeed
|
||||
// and still come back encrypted.
|
||||
let post_upgrade_encrypted_key = "encrypted/written-after-upgrade";
|
||||
let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build";
|
||||
new_client
|
||||
.put_object()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.key(post_upgrade_encrypted_key)
|
||||
.body(ByteStream::from_static(post_upgrade_encrypted_bytes))
|
||||
.send()
|
||||
.await?;
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?;
|
||||
assert_eq!(
|
||||
encryption,
|
||||
Some(ServerSideEncryption::Aes256),
|
||||
"a PUT after the upgrade lost the bucket default encryption"
|
||||
);
|
||||
assert_eq!(body, post_upgrade_encrypted_bytes);
|
||||
|
||||
let post_upgrade_plain_key = "plain/written-after-upgrade";
|
||||
let post_upgrade_plain_bytes = b"plain object written by the current RustFS build";
|
||||
put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?;
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?;
|
||||
assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT");
|
||||
assert_eq!(body, post_upgrade_plain_bytes);
|
||||
|
||||
// Every object written by the previous release reads back byte-identical.
|
||||
assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes);
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, encrypted_bytes);
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade");
|
||||
assert_eq!(
|
||||
read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id))
|
||||
.await?
|
||||
.1,
|
||||
versioned_bytes
|
||||
);
|
||||
|
||||
// rustfs#7089: the migration module is on by default, but a bucket that
|
||||
// never configured a source behaves exactly as before.
|
||||
assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?;
|
||||
assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?;
|
||||
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rolling back to the pinned previous release must still read the bucket
|
||||
/// metadata the current build wrote.
|
||||
///
|
||||
/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the
|
||||
/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON`
|
||||
/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder
|
||||
/// has to skip those two unknown keys instead of failing the whole blob. If it
|
||||
/// did not, every configuration read below would come back empty or error and
|
||||
/// the rollback would silently discard the bucket's configuration.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
|
||||
let replication_target = FakeS3Target::start().await?;
|
||||
replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let server_env = bucket_config_server_env();
|
||||
env.start_rustfs_server_with_env(vec![], &server_env).await?;
|
||||
let new_client = env.create_s3_client();
|
||||
|
||||
env.create_test_bucket(ROLLBACK_BUCKET).await?;
|
||||
enable_versioning(&new_client, ROLLBACK_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?;
|
||||
put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?;
|
||||
let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?;
|
||||
assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?;
|
||||
|
||||
let single_key = "rollback/single";
|
||||
let single_bytes = b"single-part object written by the current RustFS build";
|
||||
let single_version = new_client
|
||||
.put_object()
|
||||
.bucket(ROLLBACK_BUCKET)
|
||||
.key(single_key)
|
||||
.body(ByteStream::from_static(single_bytes))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let multipart_key = "rollback/multipart";
|
||||
let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()];
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
|
||||
assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
|
||||
assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
|
||||
assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?;
|
||||
|
||||
let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, single_bytes);
|
||||
let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
let post_rollback_bytes = b"object written by the previous RustFS release after the rollback";
|
||||
old_client
|
||||
.put_object()
|
||||
.bucket(ROLLBACK_BUCKET)
|
||||
.key(post_rollback_key)
|
||||
.body(ByteStream::from_static(post_rollback_bytes))
|
||||
.send()
|
||||
.await?;
|
||||
let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?;
|
||||
assert_eq!(
|
||||
encryption,
|
||||
Some(ServerSideEncryption::Aes256),
|
||||
"the rolled-back release lost the bucket default encryption"
|
||||
);
|
||||
assert_eq!(body, post_rollback_bytes);
|
||||
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
|
||||
# Compiles the controlled list-objects namespace-journal chaos injector into a
|
||||
# production binary (it is always available to tests). Off by default so the
|
||||
# RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
|
||||
@@ -212,10 +213,9 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
|
||||
parking_lot = { workspace = true }
|
||||
base64-simd.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
google-cloud-storage = { workspace = true }
|
||||
google-cloud-auth = { workspace = true }
|
||||
google-cloud-storage = { workspace = true, optional = true }
|
||||
google-cloud-auth = { workspace = true, optional = true }
|
||||
faster-hex = { workspace = true }
|
||||
quick-xml = { workspace = true }
|
||||
ratelimit = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
|
||||
|
||||
@@ -146,69 +146,23 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod on_demand_migration {
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
|
||||
Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
|
||||
LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
|
||||
OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
|
||||
PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
|
||||
SourceLatencySnapshot, source_backend_spec, source_client_spec,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
|
||||
ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig,
|
||||
Provider, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig,
|
||||
ValidationContext,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
|
||||
PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody,
|
||||
WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
|
||||
idle_guarded_body,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
|
||||
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
|
||||
MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
|
||||
decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub mod backfill {
|
||||
pub use crate::bucket::on_demand_migration::backfill::{
|
||||
BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
|
||||
BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
|
||||
BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
|
||||
BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
|
||||
BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
|
||||
StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
|
||||
read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
|
||||
};
|
||||
}
|
||||
pub mod source_client {
|
||||
pub use crate::bucket::on_demand_migration::source_client::{
|
||||
AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError,
|
||||
SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceProbe, SourceProvider, SourceSse,
|
||||
SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, resolve_path_style,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
|
||||
ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
|
||||
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
|
||||
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
|
||||
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
|
||||
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
|
||||
reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
|
||||
}
|
||||
|
||||
pub mod migration {
|
||||
@@ -251,7 +205,7 @@ pub mod bucket {
|
||||
pub mod remote_s3_client {
|
||||
pub use crate::bucket::remote_s3_client::{
|
||||
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_client,
|
||||
validate_remote_endpoint,
|
||||
build_remote_s3_config, validate_remote_endpoint, validate_target_ca_pem,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -497,9 +451,9 @@ pub mod object {
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
|
||||
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
|
||||
unregister_object_mutation_hook,
|
||||
ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
|
||||
@@ -489,28 +489,17 @@ impl BucketMetadata {
|
||||
!self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
///
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
/// follows the global durability mode); a parse failure is logged so a
|
||||
/// corrupted entry cannot silently change fsync behavior.
|
||||
/// Parsed on-demand migration config, if one is stored.
|
||||
///
|
||||
/// `Ok(None)` means no config (absent or cleared). A stored payload that
|
||||
/// does not parse is an error, never a default: the runtime must not
|
||||
/// pull from a source it cannot describe.
|
||||
pub fn on_demand_migration_config(
|
||||
&self,
|
||||
) -> std::result::Result<
|
||||
Option<super::on_demand_migration::OnDemandMigrationConfig>,
|
||||
super::on_demand_migration::OnDemandMigrationConfigError,
|
||||
> {
|
||||
if self.on_demand_migration_config_json.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
|
||||
/// Opaque application-owned configuration with its persisted update time.
|
||||
/// Empty bytes mean absent or cleared; decoding belongs to the consumer.
|
||||
pub fn on_demand_migration_config(&self) -> Option<(&[u8], OffsetDateTime)> {
|
||||
(!self.on_demand_migration_config_json.is_empty()).then_some((
|
||||
self.on_demand_migration_config_json.as_slice(),
|
||||
self.on_demand_migration_config_updated_at,
|
||||
))
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
/// Invalid payloads follow the global mode after logging a parse failure.
|
||||
pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
|
||||
if self.durability_config_json.is_empty() {
|
||||
return None;
|
||||
@@ -916,13 +905,6 @@ impl BucketMetadata {
|
||||
self.durability_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
|
||||
// Structural check only (shape, unknown fields); the
|
||||
// deployment-relative rules run in the admin handler with a
|
||||
// `ValidationContext`. A blob this build cannot read must not
|
||||
// be persisted for every later reader to trip over.
|
||||
if !data.is_empty() {
|
||||
super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
|
||||
}
|
||||
self.on_demand_migration_config_json = data;
|
||||
self.on_demand_migration_config_updated_at = updated;
|
||||
}
|
||||
@@ -1978,51 +1960,30 @@ mod test {
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
/// rustfs/backlog#2148: the on-demand migration config is a RustFS
|
||||
/// extension entry that round-trips through `update_config` and the
|
||||
/// msgpack codec, clears on delete, and never parses corruption into a
|
||||
/// default.
|
||||
/// The metadata codec preserves application-owned bytes and timestamps.
|
||||
#[test]
|
||||
fn on_demand_migration_config_round_trips_and_tracks_updates() {
|
||||
use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
|
||||
|
||||
let mut bm = BucketMetadata::new("odm-bucket");
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
|
||||
|
||||
let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
|
||||
assert_eq!(bm.on_demand_migration_config(), None, "fresh metadata carries no config");
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.expect("valid config is accepted");
|
||||
assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
|
||||
|
||||
.expect("opaque config is accepted");
|
||||
let stamped = bm.on_demand_migration_config_updated_at;
|
||||
assert_ne!(stamped, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Some((ODM_JSON, stamped)));
|
||||
let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
|
||||
assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
|
||||
assert_eq!(
|
||||
back.on_demand_migration_config_updated_at.unix_timestamp(),
|
||||
bm.on_demand_migration_config_updated_at.unix_timestamp()
|
||||
);
|
||||
assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
|
||||
|
||||
// A blob this build cannot read is rejected at the write boundary
|
||||
// rather than persisted for every reader to trip over.
|
||||
let before = bm.on_demand_migration_config_json.clone();
|
||||
assert!(
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
|
||||
|
||||
// Delete clears the entry.
|
||||
let stamped = bm.on_demand_migration_config_updated_at;
|
||||
assert_eq!(back.on_demand_migration_config_updated_at.unix_timestamp(), stamped.unix_timestamp());
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
|
||||
assert!(bm.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None));
|
||||
assert_eq!(bm.on_demand_migration_config(), None);
|
||||
assert!(bm.on_demand_migration_config_updated_at >= stamped);
|
||||
|
||||
// Corruption that bypassed `update_config` (disk, another writer)
|
||||
// is a typed error, never a default.
|
||||
bm.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, b"not-json".to_vec())
|
||||
.unwrap();
|
||||
let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
back.on_demand_migration_config_json, b"not-json",
|
||||
"metadata must not reinterpret application bytes"
|
||||
);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
|
||||
@@ -2034,7 +1995,7 @@ mod test {
|
||||
let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
|
||||
assert!(bm.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None));
|
||||
assert_eq!(bm.on_demand_migration_config(), None);
|
||||
|
||||
bm.default_timestamps();
|
||||
assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
|
||||
|
||||
@@ -19,7 +19,6 @@ use super::quota::BucketQuota;
|
||||
use super::target::BucketTargets;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
|
||||
use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
@@ -49,6 +48,11 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Opaque bucket configuration notifications for application-owned services.
|
||||
/// `None` withdraws a configuration; consumers validate nonempty bytes.
|
||||
pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime, Uuid)>) + Send + Sync>;
|
||||
pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
|
||||
|
||||
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
@@ -395,39 +399,21 @@ fn clear_bucket_durability(bucket: &str) {
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
}
|
||||
|
||||
/// Publish the bucket's on-demand migration config (or its absence) to the
|
||||
/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
|
||||
///
|
||||
/// Called from the same five cache-install paths as
|
||||
/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
|
||||
/// published as `None`: the runtime must stop pulling for that bucket rather
|
||||
/// than keep an older config or guess.
|
||||
/// Publish application-owned bytes on every cache install path.
|
||||
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
|
||||
return;
|
||||
};
|
||||
match bm.on_demand_migration_config() {
|
||||
Ok(config) => hook(bucket, config.as_ref()),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %bucket,
|
||||
config = "on_demand_migration",
|
||||
error = %err,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
hook(bucket, None);
|
||||
}
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(
|
||||
bucket,
|
||||
super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
|
||||
bm.on_demand_migration_config()
|
||||
.map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Withdraw a bucket's on-demand migration config when its metadata leaves
|
||||
/// the cache.
|
||||
fn clear_on_demand_migration(bucket: &str) {
|
||||
if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
|
||||
hook(bucket, None);
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1049,15 +1035,21 @@ pub async fn get_durability_config(
|
||||
}
|
||||
|
||||
/// The bucket's on-demand migration config with its update time, or
|
||||
/// `Ok(None)` when the bucket has none. A stored payload that does not parse
|
||||
/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
|
||||
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
|
||||
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_on_demand_migration_config(bucket).await
|
||||
}
|
||||
|
||||
/// Resolve opaque configuration from the store's own metadata system.
|
||||
pub async fn get_on_demand_migration_config_in(api: &ECStore, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
let sys = bucket_metadata_sys_of(&api.ctx)?;
|
||||
let lock = sys.read().await;
|
||||
lock.get_on_demand_migration_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -2579,29 +2571,27 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
/// See [`get_on_demand_migration_config`].
|
||||
pub async fn get_on_demand_migration_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
pub async fn get_on_demand_migration_config(&self, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
let config = bm.on_demand_migration_config().map_err(Error::other)?;
|
||||
Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
|
||||
Ok(bm
|
||||
.on_demand_migration_config()
|
||||
.map(|(bytes, updated_at)| (bytes.to_vec(), updated_at)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only fixture shared with sibling modules (e.g. the quota checker
|
||||
/// tests): a 4-disk `ECStore` on an isolated instance context, so tests
|
||||
/// exercising the metadata system never touch ambient process state.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_support {
|
||||
use super::*;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
|
||||
pub(crate) async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let mut dirs = Vec::with_capacity(4);
|
||||
let mut endpoints = Vec::with_capacity(4);
|
||||
for disk_idx in 0..4 {
|
||||
@@ -4385,19 +4375,26 @@ mod tests {
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
|
||||
type RecordedOdmHookCall = (String, RecordedOdmConfig);
|
||||
|
||||
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
|
||||
/// their own bucket name; the hook is process-wide and set once.
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
fn install_recording_odm_hook() {
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config| {
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
|
||||
BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config_file, config| {
|
||||
assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((
|
||||
bucket.to_string(),
|
||||
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
|
||||
));
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
|
||||
ODM_HOOK_CALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
@@ -4407,54 +4404,6 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
|
||||
/// stored payload it cannot parse as a typed error, never as a default
|
||||
/// and never as `ConfigNotFound`.
|
||||
#[tokio::test]
|
||||
async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
|
||||
use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
|
||||
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "odm-accessor";
|
||||
|
||||
sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
|
||||
assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
|
||||
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
|
||||
sys.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let err = sys
|
||||
.get_on_demand_migration_config(bucket)
|
||||
.await
|
||||
.expect_err("corrupt config must not read as a default");
|
||||
assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
|
||||
let typed = match &err {
|
||||
Error::Io(io) => io
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
|
||||
_ => None,
|
||||
};
|
||||
assert!(
|
||||
matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
|
||||
"typed parse error must survive the Result boundary, got: {err:?}"
|
||||
);
|
||||
|
||||
let mut valid = BucketMetadata::new(bucket);
|
||||
valid
|
||||
.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let stamped = valid.on_demand_migration_config_updated_at;
|
||||
sys.set(bucket.to_string(), Arc::new(valid)).await;
|
||||
let (config, updated_at) = sys
|
||||
.get_on_demand_migration_config(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("stored config is returned");
|
||||
assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
|
||||
assert_eq!(updated_at, stamped);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: the publish hook fires on every path that
|
||||
/// installs bucket metadata into the cache (set, initial load, peer
|
||||
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
|
||||
@@ -4468,15 +4417,22 @@ mod tests {
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
|
||||
}
|
||||
let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
let expect_publish = |before: usize, label: &str| {
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
|
||||
assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
Some(ODM_JSON),
|
||||
"{label} must publish the stored bytes"
|
||||
);
|
||||
assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
|
||||
};
|
||||
|
||||
// set (via persist_new_and_set, which installs through `set`).
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.bucket_incarnation_id = incarnation;
|
||||
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let writer = BucketMetadataSys::new(ecstore.clone());
|
||||
@@ -4518,14 +4474,18 @@ mod tests {
|
||||
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
|
||||
assert_eq!(calls.last().unwrap(), &None);
|
||||
|
||||
// A corrupt payload is withdrawn, never published as a config.
|
||||
// Opaque bytes reach the application even if they are not valid JSON.
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1);
|
||||
assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
Some(b"not-json".as_slice()),
|
||||
"the application validates opaque config bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -26,7 +26,6 @@ mod metadata_test;
|
||||
pub mod migration;
|
||||
mod msgp_decode;
|
||||
pub mod object_lock;
|
||||
pub mod on_demand_migration;
|
||||
pub mod policy_sys;
|
||||
pub mod quota;
|
||||
pub mod remote_s3_client;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,172 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! One contract every [`SourceBackend`] implementation must satisfy.
|
||||
//!
|
||||
//! The migration pipeline talks to a source only through the trait, so a new
|
||||
//! provider is correct exactly when it answers the same questions the same way:
|
||||
//! the same head fields, the same range semantics, the same page shape, the
|
||||
//! same error classes. Each backend supplies a fixture that answers this fixed
|
||||
//! corpus in its own dialect and then runs [`assert_backend_contract`], so a
|
||||
//! provider-specific mapping bug shows up as a contract failure rather than as
|
||||
//! a surprise in the pull pipeline.
|
||||
//!
|
||||
//! Backends differ in two documented ways, declared through
|
||||
//! [`BackendCapabilities`]: whether the provider's ETag is a content digest,
|
||||
//! and whether the provider can resume a listing from a key.
|
||||
|
||||
use super::source_client::{SourceBackend, SourceError, SourceListRequest};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// The single object every fixture serves.
|
||||
pub(super) const OBJECT_KEY: &str = "dir/a.txt";
|
||||
pub(super) const OBJECT_BODY: &[u8] = b"hello";
|
||||
/// MD5 of [`OBJECT_BODY`]; the ETag of the object on a digest provider.
|
||||
pub(super) const OBJECT_MD5: &str = "5d41402abc4b2a76b9719d911017c592";
|
||||
/// The second key the fixture's listing returns, on its second page.
|
||||
pub(super) const SECOND_KEY: &str = "dir/b.txt";
|
||||
pub(super) const COMMON_PREFIX: &str = "dir/sub/";
|
||||
pub(super) const LIST_CURSOR: &str = "cursor-1";
|
||||
/// A key the fixture answers with the provider's "no such object".
|
||||
pub(super) const MISSING_KEY: &str = "missing";
|
||||
/// A key the fixture answers with the provider's "not authorized".
|
||||
pub(super) const FORBIDDEN_KEY: &str = "secret";
|
||||
|
||||
/// Where backends are allowed to differ.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(super) struct BackendCapabilities {
|
||||
/// The provider's ETag is an opaque token, not a digest of the bytes.
|
||||
pub(super) etag_is_opaque: bool,
|
||||
/// The provider can resume a listing from a key rather than only from an
|
||||
/// opaque cursor.
|
||||
pub(super) supports_start_after: bool,
|
||||
/// The provider has an object-tagging concept at all. GCS does not, and
|
||||
/// answers with an empty map instead of failing a pull.
|
||||
pub(super) supports_tagging: bool,
|
||||
}
|
||||
|
||||
/// Drives `backend` through the shared corpus. Fixtures are scripted in
|
||||
/// request order, so the call order here is part of the contract.
|
||||
pub(super) async fn assert_backend_contract(backend: &dyn SourceBackend, caps: BackendCapabilities) {
|
||||
// 1. HEAD maps the object's shared fields.
|
||||
let head = backend.head(OBJECT_KEY).await.expect("HEAD of the fixture object");
|
||||
assert_eq!(head.size, OBJECT_BODY.len() as u64, "HEAD reports the object size");
|
||||
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
|
||||
assert_eq!(
|
||||
head.user_metadata,
|
||||
HashMap::from([("owner".to_string(), "alice".to_string())]),
|
||||
"user metadata is keyed without the provider prefix"
|
||||
);
|
||||
assert!(head.storage_class.is_some(), "the provider's tier is recorded");
|
||||
assert!(head.last_modified.is_some(), "the provider's timestamp is parsed");
|
||||
assert!(head.sse.is_none(), "the fixture object is not server-side encrypted");
|
||||
assert!(!head.is_multipart_etag);
|
||||
assert_eq!(head.etag_is_opaque, caps.etag_is_opaque);
|
||||
match caps.etag_is_opaque {
|
||||
false => assert_eq!(head.etag.as_deref(), Some(OBJECT_MD5), "a digest ETag is mapped verbatim"),
|
||||
true => assert!(head.etag.is_some(), "an opaque ETag is still recorded"),
|
||||
}
|
||||
|
||||
// 2. An unranged GET streams the whole object and reports no range.
|
||||
let got = backend.get(OBJECT_KEY, None).await.expect("unranged GET");
|
||||
assert_eq!(got.head.size, OBJECT_BODY.len() as u64);
|
||||
assert!(got.content_range.is_none(), "an unranged GET has no content-range");
|
||||
assert_eq!(got.head.etag_is_opaque, caps.etag_is_opaque, "GET and HEAD agree about the ETag");
|
||||
let body = got.body.collect().await.expect("body streams").into_bytes();
|
||||
assert_eq!(body.as_ref(), OBJECT_BODY);
|
||||
|
||||
// 3. A ranged GET returns exactly the requested interval, and `size` is
|
||||
// the length of the returned bytes rather than of the object.
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 1,
|
||||
end: 3,
|
||||
};
|
||||
let got = backend.get(OBJECT_KEY, Some(&range)).await.expect("ranged GET");
|
||||
assert_eq!(got.head.size, 3, "a ranged GET reports the range length");
|
||||
assert_eq!(got.content_range.as_deref(), Some("bytes 1-3/5"));
|
||||
let body = got.body.collect().await.expect("body streams").into_bytes();
|
||||
assert_eq!(body.as_ref(), &OBJECT_BODY[1..=3]);
|
||||
|
||||
// 4. A delimiter listing rolls prefixes up and hands back a cursor.
|
||||
let page = backend
|
||||
.list(&SourceListRequest {
|
||||
prefix: Some("dir/"),
|
||||
delimiter: Some("/"),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("first listing page");
|
||||
assert_eq!(page.objects.len(), 1, "the first page holds one object");
|
||||
assert_eq!(page.objects[0].key, OBJECT_KEY, "listing keys are in the source namespace");
|
||||
assert_eq!(page.objects[0].size, OBJECT_BODY.len() as u64);
|
||||
assert!(page.objects[0].last_modified.is_some());
|
||||
assert_eq!(page.common_prefixes, vec![COMMON_PREFIX.to_string()]);
|
||||
assert!(page.is_truncated);
|
||||
assert_eq!(page.next_continuation_token.as_deref(), Some(LIST_CURSOR));
|
||||
|
||||
// 5. The cursor is passed back verbatim and the last page ends the walk.
|
||||
let page = backend
|
||||
.list(&SourceListRequest {
|
||||
prefix: Some("dir/"),
|
||||
delimiter: Some("/"),
|
||||
continuation_token: Some(LIST_CURSOR),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("second listing page");
|
||||
assert_eq!(page.objects.len(), 1);
|
||||
assert_eq!(page.objects[0].key, SECOND_KEY);
|
||||
assert!(!page.is_truncated);
|
||||
assert!(page.next_continuation_token.is_none(), "a complete listing carries no cursor");
|
||||
|
||||
// 6. Tags come back as a flat map, empty on a provider without tags.
|
||||
let tags = backend.tagging(OBJECT_KEY).await.expect("object tags");
|
||||
match caps.supports_tagging {
|
||||
true => assert_eq!(tags, HashMap::from([("env".to_string(), "prod".to_string())])),
|
||||
false => assert!(tags.is_empty(), "a provider without tags reports none: {tags:?}"),
|
||||
}
|
||||
|
||||
// 7. The probe confirms the bucket or container answers.
|
||||
backend.probe().await.expect("probe of the fixture bucket");
|
||||
|
||||
// 8. A missing object is `NotFound`, and never retried.
|
||||
let err = backend.head(MISSING_KEY).await.expect_err("a missing object must fail");
|
||||
assert!(matches!(err, SourceError::NotFound), "{err:?}");
|
||||
assert_eq!(err.class_label(), "not_found");
|
||||
assert!(!err.is_retryable());
|
||||
|
||||
// 9. A denied object is `AccessDenied`, and never retried.
|
||||
let err = backend.head(FORBIDDEN_KEY).await.expect_err("a denied object must fail");
|
||||
assert!(matches!(err, SourceError::AccessDenied), "{err:?}");
|
||||
assert_eq!(err.class_label(), "access_denied");
|
||||
assert!(!err.is_retryable());
|
||||
|
||||
// 10. A provider without a key cursor must refuse one instead of listing
|
||||
// from the wrong position. This issues no request either way.
|
||||
if !caps.supports_start_after {
|
||||
let err = backend
|
||||
.list(&SourceListRequest {
|
||||
start_after: Some(OBJECT_KEY),
|
||||
max_keys: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("a backend without a key cursor must refuse start_after");
|
||||
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,367 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket three-state circuit breaker protecting an on-demand migration
|
||||
//! source (rustfs/backlog#2152).
|
||||
//!
|
||||
//! `Closed` lets every request through and counts consecutive failures
|
||||
//! inside a sliding window; reaching the threshold opens the breaker. `Open`
|
||||
//! rejects everything until the open duration elapses, then moves to
|
||||
//! `HalfOpen`, which admits a single probe: success closes the breaker,
|
||||
//! failure re-opens it. Timing uses `tokio::time::Instant` so tests can drive
|
||||
//! it with `tokio::time::pause`.
|
||||
//!
|
||||
//! Only transport-level failures count (`Throttled`, `Timeout`, `Connect`,
|
||||
//! `ServerError`). `NotFound` is a healthy answer and resets the failure
|
||||
//! streak; `AccessDenied`, `Unsupported` and `Other` are configuration or
|
||||
//! object problems that neither open nor close the breaker.
|
||||
|
||||
use super::source_client::SourceError;
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// Consecutive counted failures that open the breaker.
|
||||
pub const BREAKER_FAILURE_THRESHOLD: u32 = 5;
|
||||
/// Failures further apart than this do not accumulate.
|
||||
pub const BREAKER_FAILURE_WINDOW: Duration = Duration::from_secs(30);
|
||||
/// How long an open breaker rejects before admitting a probe.
|
||||
pub const BREAKER_OPEN_DURATION: Duration = Duration::from_secs(30);
|
||||
/// Probes admitted while half-open.
|
||||
pub const BREAKER_HALF_OPEN_MAX_PROBES: u32 = 1;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BreakerState {
|
||||
Closed,
|
||||
Open,
|
||||
HalfOpen,
|
||||
}
|
||||
|
||||
impl BreakerState {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
BreakerState::Closed => "closed",
|
||||
BreakerState::Open => "open",
|
||||
BreakerState::HalfOpen => "half_open",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A state change the caller may want to log.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct BreakerTransition {
|
||||
pub from: BreakerState,
|
||||
pub to: BreakerState,
|
||||
}
|
||||
|
||||
/// How a source result is scored by the breaker.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum BreakerVerdict {
|
||||
/// Resets the failure streak; closes a half-open breaker.
|
||||
Success,
|
||||
/// Counts toward the threshold; re-opens a half-open breaker.
|
||||
Failure,
|
||||
/// Leaves the breaker untouched.
|
||||
Neutral,
|
||||
}
|
||||
|
||||
impl BreakerVerdict {
|
||||
/// `None` is a successful source call.
|
||||
pub fn for_result(error: Option<&SourceError>) -> Self {
|
||||
match error {
|
||||
None | Some(SourceError::NotFound) => BreakerVerdict::Success,
|
||||
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
|
||||
BreakerVerdict::Failure
|
||||
}
|
||||
Some(
|
||||
SourceError::AccessDenied
|
||||
| SourceError::Unsupported(_)
|
||||
| SourceError::InvalidPagination(_)
|
||||
| SourceError::Other(_),
|
||||
) => BreakerVerdict::Neutral,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Inner {
|
||||
state: BreakerState,
|
||||
consecutive_failures: u32,
|
||||
last_failure_at: Option<Instant>,
|
||||
opened_at: Option<Instant>,
|
||||
half_open_probes: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Breaker {
|
||||
inner: Mutex<Inner>,
|
||||
}
|
||||
|
||||
impl Default for Breaker {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Breaker {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(Inner {
|
||||
state: BreakerState::Closed,
|
||||
consecutive_failures: 0,
|
||||
last_failure_at: None,
|
||||
opened_at: None,
|
||||
half_open_probes: 0,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Current state after applying the open-duration timeout.
|
||||
pub fn state(&self) -> BreakerState {
|
||||
let mut inner = self.inner.lock();
|
||||
Self::advance(&mut inner, Instant::now());
|
||||
inner.state
|
||||
}
|
||||
|
||||
/// Whether a request may reach the source right now. Consumes the
|
||||
/// half-open probe budget when it grants one.
|
||||
pub fn allow_request(&self) -> bool {
|
||||
let mut inner = self.inner.lock();
|
||||
Self::advance(&mut inner, Instant::now());
|
||||
match inner.state {
|
||||
BreakerState::Closed => true,
|
||||
BreakerState::Open => false,
|
||||
BreakerState::HalfOpen => {
|
||||
if inner.half_open_probes < BREAKER_HALF_OPEN_MAX_PROBES {
|
||||
inner.half_open_probes += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scores a source result; returns the transition it caused, if any.
|
||||
pub fn record(&self, verdict: BreakerVerdict) -> Option<BreakerTransition> {
|
||||
match verdict {
|
||||
BreakerVerdict::Success => self.record_success(),
|
||||
BreakerVerdict::Failure => self.record_failure(),
|
||||
BreakerVerdict::Neutral => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_success(&self) -> Option<BreakerTransition> {
|
||||
let mut inner = self.inner.lock();
|
||||
let now = Instant::now();
|
||||
Self::advance(&mut inner, now);
|
||||
inner.consecutive_failures = 0;
|
||||
inner.last_failure_at = None;
|
||||
match inner.state {
|
||||
BreakerState::Closed => None,
|
||||
// A success while open can only come from a request admitted
|
||||
// before the breaker opened; it says nothing about recovery.
|
||||
BreakerState::Open => None,
|
||||
BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Closed, now)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_failure(&self) -> Option<BreakerTransition> {
|
||||
let mut inner = self.inner.lock();
|
||||
let now = Instant::now();
|
||||
Self::advance(&mut inner, now);
|
||||
match inner.state {
|
||||
BreakerState::Closed => {
|
||||
let within_window = inner
|
||||
.last_failure_at
|
||||
.is_some_and(|last| now.saturating_duration_since(last) <= BREAKER_FAILURE_WINDOW);
|
||||
inner.consecutive_failures = if within_window { inner.consecutive_failures + 1 } else { 1 };
|
||||
inner.last_failure_at = Some(now);
|
||||
if inner.consecutive_failures >= BREAKER_FAILURE_THRESHOLD {
|
||||
Some(Self::transition(&mut inner, BreakerState::Open, now))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
BreakerState::Open => None,
|
||||
BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Open, now)),
|
||||
}
|
||||
}
|
||||
|
||||
fn advance(inner: &mut Inner, now: Instant) {
|
||||
if inner.state == BreakerState::Open
|
||||
&& inner
|
||||
.opened_at
|
||||
.is_some_and(|opened| now.saturating_duration_since(opened) >= BREAKER_OPEN_DURATION)
|
||||
{
|
||||
Self::transition(inner, BreakerState::HalfOpen, now);
|
||||
}
|
||||
}
|
||||
|
||||
fn transition(inner: &mut Inner, to: BreakerState, now: Instant) -> BreakerTransition {
|
||||
let from = inner.state;
|
||||
inner.state = to;
|
||||
match to {
|
||||
BreakerState::Open => {
|
||||
inner.opened_at = Some(now);
|
||||
inner.half_open_probes = 0;
|
||||
}
|
||||
BreakerState::HalfOpen => {
|
||||
inner.half_open_probes = 0;
|
||||
}
|
||||
BreakerState::Closed => {
|
||||
inner.opened_at = None;
|
||||
inner.half_open_probes = 0;
|
||||
inner.consecutive_failures = 0;
|
||||
inner.last_failure_at = None;
|
||||
}
|
||||
}
|
||||
BreakerTransition { from, to }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn server_error() -> SourceError {
|
||||
SourceError::ServerError(503)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn five_failures_open_then_half_open_after_timeout() {
|
||||
let breaker = Breaker::new();
|
||||
for i in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&server_error()))), None, "failure {i}");
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
assert_eq!(
|
||||
breaker.record(BreakerVerdict::for_result(Some(&server_error()))),
|
||||
Some(BreakerTransition {
|
||||
from: BreakerState::Closed,
|
||||
to: BreakerState::Open
|
||||
})
|
||||
);
|
||||
assert_eq!(breaker.state(), BreakerState::Open);
|
||||
assert!(!breaker.allow_request());
|
||||
|
||||
tokio::time::advance(BREAKER_OPEN_DURATION - Duration::from_secs(1)).await;
|
||||
assert!(!breaker.allow_request());
|
||||
assert_eq!(breaker.state(), BreakerState::Open);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(breaker.state(), BreakerState::HalfOpen);
|
||||
assert!(breaker.allow_request(), "one probe is admitted");
|
||||
assert!(!breaker.allow_request(), "second probe is rejected");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn half_open_probe_success_closes_and_failure_reopens() {
|
||||
let breaker = Breaker::new();
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD {
|
||||
breaker.record_failure();
|
||||
}
|
||||
tokio::time::advance(BREAKER_OPEN_DURATION).await;
|
||||
assert!(breaker.allow_request());
|
||||
assert_eq!(
|
||||
breaker.record_failure(),
|
||||
Some(BreakerTransition {
|
||||
from: BreakerState::HalfOpen,
|
||||
to: BreakerState::Open
|
||||
})
|
||||
);
|
||||
assert!(!breaker.allow_request());
|
||||
|
||||
tokio::time::advance(BREAKER_OPEN_DURATION).await;
|
||||
assert!(breaker.allow_request());
|
||||
assert_eq!(
|
||||
breaker.record_success(),
|
||||
Some(BreakerTransition {
|
||||
from: BreakerState::HalfOpen,
|
||||
to: BreakerState::Closed
|
||||
})
|
||||
);
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
assert!(breaker.allow_request());
|
||||
// The streak restarts from zero after closing.
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
assert_eq!(breaker.record_failure(), None);
|
||||
}
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn failures_outside_window_do_not_accumulate() {
|
||||
let breaker = Breaker::new();
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
breaker.record_failure();
|
||||
}
|
||||
tokio::time::advance(BREAKER_FAILURE_WINDOW + Duration::from_secs(1)).await;
|
||||
assert_eq!(breaker.record_failure(), None, "stale streak restarts at one");
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_found_and_access_denied_do_not_count() {
|
||||
let breaker = Breaker::new();
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
breaker.record(BreakerVerdict::for_result(Some(&server_error())));
|
||||
}
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::AccessDenied))), None);
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
// AccessDenied is neutral: the streak is still one short of opening.
|
||||
assert_eq!(
|
||||
breaker.record(BreakerVerdict::for_result(Some(&SourceError::Unsupported("sse-c".into())))),
|
||||
None
|
||||
);
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Other("x".into())))), None);
|
||||
// NotFound is a healthy answer and resets the streak entirely.
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::NotFound))), None);
|
||||
for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
|
||||
assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Timeout))), None);
|
||||
}
|
||||
assert_eq!(breaker.state(), BreakerState::Closed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdicts_cover_every_source_error_class() {
|
||||
assert_eq!(BreakerVerdict::for_result(None), BreakerVerdict::Success);
|
||||
assert_eq!(BreakerVerdict::for_result(Some(&SourceError::NotFound)), BreakerVerdict::Success);
|
||||
for failure in [
|
||||
SourceError::Throttled,
|
||||
SourceError::Timeout,
|
||||
SourceError::Connect("refused".into()),
|
||||
SourceError::ServerError(500),
|
||||
] {
|
||||
assert_eq!(BreakerVerdict::for_result(Some(&failure)), BreakerVerdict::Failure, "{failure:?}");
|
||||
}
|
||||
for neutral in [
|
||||
SourceError::AccessDenied,
|
||||
SourceError::Unsupported("sse-c".into()),
|
||||
SourceError::Other("x".into()),
|
||||
] {
|
||||
assert_eq!(BreakerVerdict::for_result(Some(&neutral)), BreakerVerdict::Neutral, "{neutral:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_labels_are_stable() {
|
||||
assert_eq!(BreakerState::Closed.as_str(), "closed");
|
||||
assert_eq!(BreakerState::Open.as_str(), "open");
|
||||
assert_eq!(BreakerState::HalfOpen.as_str(), "half_open");
|
||||
assert_eq!(serde_json::to_string(&BreakerState::HalfOpen).unwrap(), "\"half_open\"");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,506 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Native Google Cloud Storage source backend.
|
||||
//!
|
||||
//! The `gcs` provider already reaches GCS through its S3 interoperability API,
|
||||
//! which needs an HMAC key pair. This backend is the other half: it authorizes
|
||||
//! with a service-account key, the credential most GCS projects actually issue,
|
||||
//! by minting OAuth tokens through the shared `google-cloud-auth` credential
|
||||
//! machinery the tier layer already uses.
|
||||
//!
|
||||
//! Two GCS surfaces are involved, each for the half it describes best. The read
|
||||
//! path uses the XML API (`/{bucket}/{object}`), whose responses carry
|
||||
//! `x-goog-meta-*` user metadata and the `x-goog-hash` digest in one round trip.
|
||||
//! Listing uses the JSON API (`objects.list`), whose `pageToken` maps directly
|
||||
//! onto the shared page cursor and whose `prefixes` are the delimiter roll-up.
|
||||
//! Both accept the same bearer token.
|
||||
//!
|
||||
//! Every call this backend makes needs only `storage.objects.get` and
|
||||
//! `storage.objects.list`, the two permissions of the `objectViewer` role, so a
|
||||
//! key scoped to exactly the migration's needs works.
|
||||
//!
|
||||
//! `x-goog-hash` carries a base64 MD5 for every non-composite object; it is
|
||||
//! converted to hex and becomes the head's ETag, so a pulled object is checked
|
||||
//! against the digest GCS itself computed. A composite object has no MD5, and
|
||||
//! its ETag is then marked opaque rather than checked.
|
||||
|
||||
use super::native_http::{
|
||||
NativeHeadFields, NativeHttp, base64_md5_to_hex, header, native_source_head, parse_http_timestamp, read_text, response_body,
|
||||
};
|
||||
use super::source_client::{
|
||||
GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
SourceTimeouts, range_header_value,
|
||||
};
|
||||
use crate::bucket::remote_s3_client::RemoteS3ClientError;
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder};
|
||||
use google_cloud_auth::credentials::{CacheableResource, Credentials};
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use url::Url;
|
||||
|
||||
/// Read-only object scope: this backend never writes to the source.
|
||||
const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
|
||||
const METADATA_PREFIX: &str = "x-goog-meta-";
|
||||
/// GCS reports its error code in the response body, not a header; the shared
|
||||
/// transport takes a header name, so it is given one that never matches and
|
||||
/// classification falls back to the status.
|
||||
const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
|
||||
/// One `objects.list` page is small; refuse an unbounded document.
|
||||
const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
pub struct GcsNativeSourceBackend {
|
||||
http: NativeHttp,
|
||||
bucket: String,
|
||||
credentials: Credentials,
|
||||
}
|
||||
|
||||
impl GcsNativeSourceBackend {
|
||||
pub fn new(
|
||||
endpoint: &str,
|
||||
bucket: &str,
|
||||
spec: &GcsSourceSpec,
|
||||
timeouts: SourceTimeouts,
|
||||
skip_tls_verify: bool,
|
||||
ca_cert_pem: Option<&str>,
|
||||
) -> Result<Self, RemoteS3ClientError> {
|
||||
let key: serde_json::Value = serde_json::from_str(&spec.service_account_json)
|
||||
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not valid JSON"))?;
|
||||
let credentials = ServiceAccountBuilder::new(key)
|
||||
.with_access_specifier(AccessSpecifier::from_scopes([READ_ONLY_SCOPE]))
|
||||
.build()
|
||||
.map_err(|_| RemoteS3ClientError::Credentials("gcs service account key is not usable"))?;
|
||||
Ok(Self {
|
||||
http: NativeHttp::new(endpoint, timeouts, skip_tls_verify, ca_cert_pem)?,
|
||||
bucket: bucket.to_string(),
|
||||
credentials,
|
||||
})
|
||||
}
|
||||
|
||||
/// Authorization headers for one request. A credential failure is reported
|
||||
/// as `AccessDenied` with no message: the renderer of a credential error
|
||||
/// has the key material in scope, and the class is what callers act on.
|
||||
async fn auth_headers(&self) -> Result<HeaderMap, SourceError> {
|
||||
match self.credentials.headers(http::Extensions::new()).await {
|
||||
Ok(CacheableResource::New { data, .. }) => Ok(data),
|
||||
// Only returned when the caller passes an entity tag, which this
|
||||
// backend never does; an empty set is still the honest answer.
|
||||
Ok(CacheableResource::NotModified) => Ok(HeaderMap::new()),
|
||||
Err(_) => Err(SourceError::AccessDenied),
|
||||
}
|
||||
}
|
||||
|
||||
/// XML API URL of one object; `/` in the key stay path separators.
|
||||
fn object_url(&self, key: &str) -> Result<Url, SourceError> {
|
||||
self.http.url(std::iter::once(self.bucket.as_str()).chain(key.split('/')))
|
||||
}
|
||||
|
||||
/// JSON API URL of the bucket's object collection.
|
||||
fn objects_url(&self) -> Result<Url, SourceError> {
|
||||
self.http.url(["storage", "v1", "b", self.bucket.as_str(), "o"])
|
||||
}
|
||||
|
||||
async fn request(&self, method: Method, url: Url, mut headers: HeaderMap) -> Result<reqwest::Request, SourceError> {
|
||||
for (name, value) in self.auth_headers().await? {
|
||||
if let Some(name) = name {
|
||||
headers.insert(name, value);
|
||||
}
|
||||
}
|
||||
let mut request = reqwest::Request::new(method, url);
|
||||
*request.headers_mut() = headers;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
/// Shared mapping for the XML API's HEAD and GET responses.
|
||||
fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
|
||||
if header(headers, "x-goog-encryption-key-sha256").is_some() {
|
||||
return Err(SourceError::Unsupported(
|
||||
"source object uses a customer-supplied encryption key; customer-key sources are not supported".to_string(),
|
||||
));
|
||||
}
|
||||
// `x-goog-hash` lists digests as `name=base64`, comma separated, and may
|
||||
// repeat across header lines. Only the MD5 describes the whole object.
|
||||
let md5 = headers
|
||||
.get_all("x-goog-hash")
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.filter_map(|digest| digest.trim().strip_prefix("md5="))
|
||||
.find_map(base64_md5_to_hex);
|
||||
|
||||
let (etag, etag_is_opaque) = match md5 {
|
||||
Some(md5) => (Some(md5), false),
|
||||
// A composite object has no MD5; its ETag describes the composition
|
||||
// rather than the bytes, so it is provenance only.
|
||||
None => (header(headers, "etag").map(str::to_string), true),
|
||||
};
|
||||
native_source_head(
|
||||
headers,
|
||||
METADATA_PREFIX,
|
||||
NativeHeadFields {
|
||||
etag,
|
||||
etag_is_opaque,
|
||||
version_id: header(headers, "x-goog-generation").map(str::to_string),
|
||||
storage_class: header(headers, "x-goog-storage-class").map(str::to_string),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SourceBackend for GcsNativeSourceBackend {
|
||||
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
|
||||
let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?;
|
||||
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
|
||||
Self::head_from_response(response.headers())
|
||||
}
|
||||
|
||||
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(range) = range.map(range_header_value).transpose()? {
|
||||
headers.insert(
|
||||
http::header::RANGE,
|
||||
HeaderValue::from_str(&range).map_err(|_| SourceError::Other("invalid range header".to_string()))?,
|
||||
);
|
||||
}
|
||||
let request = self.request(Method::GET, self.object_url(key)?, headers).await?;
|
||||
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
|
||||
let head = Self::head_from_response(response.headers())?;
|
||||
let content_range = header(response.headers(), "content-range").map(str::to_string);
|
||||
Ok(SourceGet {
|
||||
head,
|
||||
body: response_body(response),
|
||||
content_range,
|
||||
})
|
||||
}
|
||||
|
||||
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
|
||||
// `objects.list` offers `startOffset`, which is inclusive, so it cannot
|
||||
// express "resume after this key" without silently repeating it.
|
||||
if request.start_after.is_some() {
|
||||
return Err(SourceError::Unsupported(
|
||||
"gcs sources cannot resume a listing from a key; use the continuation token".to_string(),
|
||||
));
|
||||
}
|
||||
let mut url = self.objects_url()?;
|
||||
{
|
||||
let mut query = url.query_pairs_mut();
|
||||
if let Some(prefix) = request.prefix.filter(|prefix| !prefix.is_empty()) {
|
||||
query.append_pair("prefix", prefix);
|
||||
}
|
||||
if let Some(delimiter) = request.delimiter.filter(|delimiter| !delimiter.is_empty()) {
|
||||
query.append_pair("delimiter", delimiter);
|
||||
}
|
||||
if let Some(token) = request.continuation_token.filter(|token| !token.is_empty()) {
|
||||
query.append_pair("pageToken", token);
|
||||
}
|
||||
if request.max_keys > 0 {
|
||||
query.append_pair("maxResults", &request.max_keys.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
|
||||
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
|
||||
let body = read_text(response, MAX_JSON_BYTES).await?;
|
||||
parse_objects_list(&body)
|
||||
}
|
||||
|
||||
/// GCS has no object tagging API; user metadata is already carried by the
|
||||
/// head mapping. An empty map keeps `policy.copy_tags` from failing a pull
|
||||
/// over a concept the provider does not have.
|
||||
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
|
||||
Ok(HashMap::new())
|
||||
}
|
||||
|
||||
/// A one-object listing, not `buckets.get`: the migration pipeline only
|
||||
/// ever needs `storage.objects.list` and `storage.objects.get`, and a key
|
||||
/// scoped to exactly those (the `objectViewer` role) cannot read the bucket
|
||||
/// resource. Probing with `buckets.get` would reject a correct key.
|
||||
async fn probe(&self) -> Result<(), SourceError> {
|
||||
let mut url = self.objects_url()?;
|
||||
url.query_pairs_mut().append_pair("maxResults", "1");
|
||||
let request = self.request(Method::GET, url, HeaderMap::new()).await?;
|
||||
let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
|
||||
read_text(response, MAX_JSON_BYTES)
|
||||
.await
|
||||
.and_then(|body| parse_objects_list(&body))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ObjectsList {
|
||||
#[serde(default)]
|
||||
items: Vec<ListedObject>,
|
||||
#[serde(default)]
|
||||
prefixes: Vec<String>,
|
||||
#[serde(default)]
|
||||
next_page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ListedObject {
|
||||
name: String,
|
||||
/// GCS renders the size as a decimal string, not a JSON number.
|
||||
#[serde(default)]
|
||||
size: Option<String>,
|
||||
#[serde(default)]
|
||||
updated: Option<String>,
|
||||
#[serde(default)]
|
||||
md5_hash: Option<String>,
|
||||
#[serde(default)]
|
||||
etag: Option<String>,
|
||||
#[serde(default)]
|
||||
storage_class: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
|
||||
let listing: ObjectsList =
|
||||
serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
|
||||
let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
|
||||
let objects = listing
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let etag = item
|
||||
.md5_hash
|
||||
.as_deref()
|
||||
.and_then(base64_md5_to_hex)
|
||||
.or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
|
||||
.filter(|etag| !etag.is_empty());
|
||||
SourceObject {
|
||||
key: item.name,
|
||||
etag,
|
||||
size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
|
||||
last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
|
||||
storage_class: item.storage_class,
|
||||
// GCS never encodes a part count in a digest or an ETag.
|
||||
is_multipart_etag: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(SourcePage {
|
||||
objects,
|
||||
common_prefixes: listing.prefixes,
|
||||
is_truncated: next_continuation_token.is_some(),
|
||||
next_continuation_token,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
|
||||
use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
|
||||
use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
|
||||
|
||||
const LIST_PAGE_ONE: &str = r#"{
|
||||
"kind": "storage#objects",
|
||||
"nextPageToken": "cursor-1",
|
||||
"prefixes": ["dir/sub/"],
|
||||
"items": [
|
||||
{
|
||||
"name": "dir/a.txt",
|
||||
"size": "5",
|
||||
"updated": "2015-10-21T07:28:00.000Z",
|
||||
"md5Hash": "XUFAKrxLKna5cZ2REBfFkg==",
|
||||
"etag": "CJizy9Wq0McCEAE=",
|
||||
"storageClass": "STANDARD"
|
||||
}
|
||||
]
|
||||
}"#;
|
||||
|
||||
const LIST_PAGE_TWO: &str = r#"{
|
||||
"kind": "storage#objects",
|
||||
"items": [
|
||||
{
|
||||
"name": "dir/b.txt",
|
||||
"size": "7",
|
||||
"updated": "2015-10-21T07:28:00.000Z",
|
||||
"etag": "\"CJizy9Wq0McCEAI=\""
|
||||
}
|
||||
]
|
||||
}"#;
|
||||
|
||||
fn backend(endpoint: &Url) -> GcsNativeSourceBackend {
|
||||
GcsNativeSourceBackend {
|
||||
http: NativeHttp::for_test(endpoint.clone()),
|
||||
bucket: "legacy".to_string(),
|
||||
// Anonymous credentials add no headers, so the fixture sees exactly
|
||||
// the request this backend builds.
|
||||
credentials: AnonymousBuilder::new().build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn object_headers() -> Vec<(&'static str, String)> {
|
||||
vec![
|
||||
("Content-Type", "text/plain".to_string()),
|
||||
("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
|
||||
("ETag", "\"CJizy9Wq0McCEAE=\"".to_string()),
|
||||
("x-goog-hash", "crc32c=AAAAAA==,md5=XUFAKrxLKna5cZ2REBfFkg==".to_string()),
|
||||
("x-goog-meta-owner", "alice".to_string()),
|
||||
("x-goog-storage-class", "STANDARD".to_string()),
|
||||
("x-goog-generation", "1445412480000000".to_string()),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn objects_list_maps_items_prefixes_and_the_page_token() {
|
||||
let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse");
|
||||
assert_eq!(page.common_prefixes, vec!["dir/sub/"]);
|
||||
assert!(page.is_truncated);
|
||||
assert_eq!(page.next_continuation_token.as_deref(), Some("cursor-1"));
|
||||
assert_eq!(page.objects.len(), 1);
|
||||
assert_eq!(page.objects[0].key, "dir/a.txt");
|
||||
assert_eq!(page.objects[0].size, 5, "the string size is parsed");
|
||||
assert_eq!(
|
||||
page.objects[0].etag.as_deref(),
|
||||
Some("5d41402abc4b2a76b9719d911017c592"),
|
||||
"the base64 md5Hash becomes a hex ETag"
|
||||
);
|
||||
assert_eq!(page.objects[0].storage_class.as_deref(), Some("STANDARD"));
|
||||
assert!(page.objects[0].last_modified.is_some(), "RFC 3339 `updated` is parsed");
|
||||
|
||||
let page = parse_objects_list(LIST_PAGE_TWO).expect("page should parse");
|
||||
assert!(!page.is_truncated);
|
||||
assert!(page.next_continuation_token.is_none());
|
||||
assert_eq!(
|
||||
page.objects[0].etag.as_deref(),
|
||||
Some("CJizy9Wq0McCEAI="),
|
||||
"without md5Hash the raw etag is carried"
|
||||
);
|
||||
|
||||
assert!(parse_objects_list("not json").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_prefers_the_goog_hash_md5_over_the_etag() {
|
||||
let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, object_headers(), String::new())]).await;
|
||||
let head = backend(&endpoint).head("dir/a b.txt").await.expect("HEAD should map");
|
||||
|
||||
let recorded = recorded.lock().expect("recorder lock").clone();
|
||||
assert_eq!(recorded[0].method, "HEAD");
|
||||
assert_eq!(recorded[0].target, "/legacy/dir/a%20b.txt", "the XML API addresses the object by path");
|
||||
assert_eq!(
|
||||
head.etag.as_deref(),
|
||||
Some("5d41402abc4b2a76b9719d911017c592"),
|
||||
"the x-goog-hash md5 is the content digest"
|
||||
);
|
||||
assert!(!head.etag_is_opaque, "a GCS md5 may be checked against the pulled bytes");
|
||||
assert_eq!(head.user_metadata, HashMap::from([("owner".to_string(), "alice".to_string())]));
|
||||
assert_eq!(head.version_id.as_deref(), Some("1445412480000000"));
|
||||
assert_eq!(head.storage_class.as_deref(), Some("STANDARD"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_composite_object_without_an_md5_keeps_an_opaque_etag() {
|
||||
let headers = object_headers()
|
||||
.into_iter()
|
||||
.map(|(name, value)| {
|
||||
if name == "x-goog-hash" {
|
||||
(name, "crc32c=AAAAAA==".to_string())
|
||||
} else {
|
||||
(name, value)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
|
||||
let head = backend(&endpoint).head("composed").await.expect("HEAD should map");
|
||||
assert_eq!(head.etag.as_deref(), Some("CJizy9Wq0McCEAE="));
|
||||
assert!(head.etag_is_opaque, "a composite ETag describes the composition, not the bytes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn customer_supplied_key_objects_are_refused() {
|
||||
let mut headers = object_headers();
|
||||
headers.push(("x-goog-encryption-key-sha256", "abc".to_string()));
|
||||
let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(200, headers, String::new())]).await;
|
||||
let err = backend(&endpoint)
|
||||
.head("a.txt")
|
||||
.await
|
||||
.expect_err("CSEK objects are unsupported");
|
||||
assert!(matches!(err, SourceError::Unsupported(_)), "{err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_and_probe_address_the_json_api() {
|
||||
let (endpoint, recorded) = scripted_server(vec![
|
||||
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
|
||||
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
|
||||
])
|
||||
.await;
|
||||
let backend = backend(&endpoint);
|
||||
|
||||
backend
|
||||
.list(&SourceListRequest {
|
||||
prefix: Some("dir/"),
|
||||
delimiter: Some("/"),
|
||||
continuation_token: Some("cursor-0"),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("listing should succeed");
|
||||
backend.probe().await.expect("probe should succeed");
|
||||
|
||||
let recorded = recorded.lock().expect("recorder lock").clone();
|
||||
assert!(recorded[0].target.starts_with("/storage/v1/b/legacy/o?"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("prefix=dir%2F"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("delimiter=%2F"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("pageToken=cursor-0"), "{}", recorded[0].target);
|
||||
assert!(recorded[0].target.contains("maxResults=2"), "{}", recorded[0].target);
|
||||
assert_eq!(
|
||||
recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1",
|
||||
"the probe uses the listing permission the pipeline already needs"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gcs_native_backend_satisfies_the_shared_backend_contract() {
|
||||
let mut ranged = object_headers();
|
||||
ranged.push(("Content-Range", "bytes 1-3/5".to_string()));
|
||||
// A HEAD reports the object size with no body, exactly as GCS does.
|
||||
let mut head_only = object_headers();
|
||||
head_only.push(("Content-Length", "5".to_string()));
|
||||
let (endpoint, _) = scripted_server(vec![
|
||||
ScriptedResponse::new(200, head_only, String::new()),
|
||||
ScriptedResponse::new(200, object_headers(), "hello".to_string()),
|
||||
ScriptedResponse::new(206, ranged, "ell".to_string()),
|
||||
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_ONE.to_string()),
|
||||
ScriptedResponse::new(200, Vec::new(), LIST_PAGE_TWO.to_string()),
|
||||
// GCS has no tagging call, so the contract's tag step issues no
|
||||
// request; the probe is the next one on the wire.
|
||||
ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
|
||||
ScriptedResponse::new(404, Vec::new(), String::new()),
|
||||
ScriptedResponse::new(403, Vec::new(), String::new()),
|
||||
])
|
||||
.await;
|
||||
|
||||
assert_backend_contract(
|
||||
&backend(&endpoint),
|
||||
BackendCapabilities {
|
||||
etag_is_opaque: false,
|
||||
supports_start_after: false,
|
||||
// GCS objects have no tags; the contract's tag step is skipped.
|
||||
supports_tagging: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,73 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! On-Demand Migration (ODM): a bucket can name an external S3-compatible
|
||||
//! source bucket; GET misses are served from that source and backfilled
|
||||
//! locally. This module owns the bucket-level configuration model
|
||||
//! (`on-demand-migration.json` in the bucket metadata file), the source
|
||||
//! client, and the per-node runtime (`sys`) that turns configs into live
|
||||
//! clients guarded by a breaker, a negative cache, singleflight and a pull
|
||||
//! concurrency limit (rustfs/backlog#2147).
|
||||
//!
|
||||
//! A source is reached through one `SourceBackend`: the S3 dialect for every
|
||||
//! S3-compatible provider, and a native backend for the providers that have no
|
||||
//! S3 API (`azure`, `gcs_native`).
|
||||
|
||||
pub mod azure;
|
||||
#[cfg(test)]
|
||||
mod backend_contract;
|
||||
pub mod backfill;
|
||||
pub mod breaker;
|
||||
pub mod config;
|
||||
pub mod gcs;
|
||||
pub mod list_through;
|
||||
mod native_http;
|
||||
pub mod negative_cache;
|
||||
pub mod pull;
|
||||
pub mod source_client;
|
||||
pub mod stats;
|
||||
pub mod sys;
|
||||
#[cfg(test)]
|
||||
mod test_http_fixture;
|
||||
|
||||
pub use breaker::{
|
||||
BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
|
||||
BreakerState, BreakerTransition, BreakerVerdict,
|
||||
};
|
||||
pub use config::{
|
||||
AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
|
||||
ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider,
|
||||
RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub use list_through::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
|
||||
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
|
||||
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
|
||||
decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
|
||||
pub use pull::{
|
||||
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion,
|
||||
PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody, WriteBackError,
|
||||
WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, idle_guarded_body,
|
||||
};
|
||||
pub use stats::{
|
||||
GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
|
||||
PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
|
||||
};
|
||||
pub use sys::{
|
||||
ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
|
||||
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
|
||||
source_client_spec,
|
||||
};
|
||||
@@ -1,415 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared HTTP transport for the on-demand migration source backends that do
|
||||
//! not speak S3 (Azure Blob, native GCS).
|
||||
//!
|
||||
//! The S3 backend rides the AWS SDK; these providers have no SigV4 dialect, so
|
||||
//! they talk plain HTTP through one `reqwest` client that carries the same
|
||||
//! connect/read timeouts and TLS policy the operator configured for the source.
|
||||
//! Redirects are refused: the endpoint passed the outbound policy gate once, and
|
||||
//! following a source-chosen `Location` would leave that gate behind.
|
||||
//!
|
||||
//! Errors never render the request URL. A SAS token lives in the query string,
|
||||
//! so a `reqwest` error rendered with its URL would print the credential into
|
||||
//! the log line and the admin response.
|
||||
|
||||
use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag};
|
||||
use crate::bucket::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use futures::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use std::collections::HashMap;
|
||||
use std::time::SystemTime;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::{Rfc2822, Rfc3339};
|
||||
use url::Url;
|
||||
|
||||
/// Origin the native backends are allowed to address, plus the HTTP client
|
||||
/// that reaches it.
|
||||
pub(super) struct NativeHttp {
|
||||
client: reqwest::Client,
|
||||
endpoint: Url,
|
||||
}
|
||||
|
||||
impl NativeHttp {
|
||||
/// `endpoint` must be a bare `scheme://host[:port]` origin; it is checked
|
||||
/// against the outbound policy exactly like an S3 source endpoint.
|
||||
pub(super) fn new(
|
||||
endpoint: &str,
|
||||
timeouts: SourceTimeouts,
|
||||
skip_tls_verify: bool,
|
||||
ca_cert_pem: Option<&str>,
|
||||
) -> Result<Self, RemoteS3ClientError> {
|
||||
let endpoint = Url::parse(endpoint.trim()).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if !matches!(endpoint.scheme(), "http" | "https") {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint(format!(
|
||||
"unsupported scheme {}; expected http or https",
|
||||
endpoint.scheme()
|
||||
)));
|
||||
}
|
||||
if endpoint.host_str().is_none_or(str::is_empty) {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint has no host".to_string()));
|
||||
}
|
||||
if !endpoint.username().is_empty() || endpoint.password().is_some() {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint("endpoint must not carry userinfo".to_string()));
|
||||
}
|
||||
if !matches!(endpoint.path(), "" | "/") || endpoint.query().is_some() || endpoint.fragment().is_some() {
|
||||
return Err(RemoteS3ClientError::InvalidEndpoint(
|
||||
"endpoint must be an origin without path, query or fragment".to_string(),
|
||||
));
|
||||
}
|
||||
validate_remote_endpoint(&endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
|
||||
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.connect_timeout(timeouts.connect)
|
||||
.read_timeout(timeouts.read)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent(USER_AGENT_SUFFIX);
|
||||
if skip_tls_verify {
|
||||
builder = builder.danger_accept_invalid_certs(true);
|
||||
} else if let Some(pem) = ca_cert_pem.map(str::trim).filter(|pem| !pem.is_empty()) {
|
||||
// Reject a malformed bundle the same way the S3 path does, so the
|
||||
// operator sees "invalid CA PEM" instead of a TLS handshake failure.
|
||||
validate_target_ca_pem(pem)?;
|
||||
let certificate = reqwest::Certificate::from_pem(pem.as_bytes())
|
||||
.map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
|
||||
builder = builder.add_root_certificate(certificate);
|
||||
}
|
||||
|
||||
let client = builder
|
||||
.build()
|
||||
.map_err(|err| RemoteS3ClientError::InvalidEndpoint(format!("http client cannot be built: {err}")))?;
|
||||
Ok(Self { client, endpoint })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn for_test(endpoint: Url) -> Self {
|
||||
Self {
|
||||
client: reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("test http client should build"),
|
||||
endpoint,
|
||||
}
|
||||
}
|
||||
|
||||
/// A URL under the endpoint origin. `segments` are percent-encoded as
|
||||
/// path segments, so a key containing `?`, `#` or a space cannot rewrite
|
||||
/// the request target.
|
||||
pub(super) fn url<'a>(&self, segments: impl IntoIterator<Item = &'a str>) -> Result<Url, SourceError> {
|
||||
let mut url = self.endpoint.clone();
|
||||
{
|
||||
let mut path = url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?;
|
||||
path.clear();
|
||||
path.extend(segments);
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
/// Sends the request and returns the response only for a 2xx status.
|
||||
/// Non-2xx statuses are classified from the status and the provider's own
|
||||
/// error-code header; response bodies are not read, so no provider message
|
||||
/// can smuggle credentials or markup into a log line.
|
||||
pub(super) async fn send(
|
||||
&self,
|
||||
request: reqwest::Request,
|
||||
error_code_header: &str,
|
||||
) -> Result<reqwest::Response, SourceError> {
|
||||
let response = self.client.execute(request).await.map_err(classify_transport_error)?;
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
let code = response
|
||||
.headers()
|
||||
.get(error_code_header)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
Err(classify_status(
|
||||
status.as_u16(),
|
||||
None,
|
||||
match &code {
|
||||
Some(code) => format!("source returned HTTP {status} ({code})"),
|
||||
None => format!("source returned HTTP {status}"),
|
||||
},
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Renders a transport failure without the request URL: a SAS token or a
|
||||
/// signed query would otherwise reach logs and admin responses.
|
||||
pub(super) fn classify_transport_error(err: reqwest::Error) -> SourceError {
|
||||
let is_timeout = err.is_timeout();
|
||||
let is_connect = err.is_connect();
|
||||
let message = err.without_url().to_string();
|
||||
if is_timeout {
|
||||
SourceError::Timeout
|
||||
} else if is_connect {
|
||||
SourceError::Connect(message)
|
||||
} else {
|
||||
SourceError::Other(message)
|
||||
}
|
||||
}
|
||||
|
||||
/// Streams the response body without buffering it.
|
||||
pub(super) fn response_body(response: reqwest::Response) -> ByteStream {
|
||||
let stream = response.bytes_stream().map(|chunk| {
|
||||
chunk
|
||||
.map(http_body::Frame::data)
|
||||
.map_err(|err| std::io::Error::other(err.without_url().to_string()))
|
||||
});
|
||||
ByteStream::new(SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream)))
|
||||
}
|
||||
|
||||
/// Reads a bounded response body as UTF-8, for the XML and JSON listings.
|
||||
pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) -> Result<String, SourceError> {
|
||||
let mut body = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(classify_transport_error)?;
|
||||
if body.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(SourceError::Other("source listing response exceeded the size limit".to_string()));
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
String::from_utf8(body).map_err(|_| SourceError::Other("source listing response is not valid UTF-8".to_string()))
|
||||
}
|
||||
|
||||
/// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
|
||||
/// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
|
||||
/// an MD5.
|
||||
pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
|
||||
let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
|
||||
(raw.len() == 16).then(|| faster_hex::hex_string(&raw))
|
||||
}
|
||||
|
||||
pub(super) fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
|
||||
headers.get(name).and_then(|value| value.to_str().ok()).map(str::trim)
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
header(headers, name).filter(|value| !value.is_empty()).map(str::to_string)
|
||||
}
|
||||
|
||||
/// `Last-Modified` and friends arrive as an HTTP date; the JSON dialects use
|
||||
/// RFC 3339 for the same field, so both are accepted.
|
||||
pub(super) fn parse_http_timestamp(value: &str) -> Option<SystemTime> {
|
||||
OffsetDateTime::parse(value, &Rfc2822)
|
||||
.or_else(|_| OffsetDateTime::parse(value, &Rfc3339))
|
||||
.ok()
|
||||
.map(SystemTime::from)
|
||||
}
|
||||
|
||||
/// Provider-specific fields the shared header mapping cannot infer.
|
||||
pub(super) struct NativeHeadFields {
|
||||
pub(super) etag: Option<String>,
|
||||
/// The ETag is an opaque token rather than a digest of the bytes.
|
||||
pub(super) etag_is_opaque: bool,
|
||||
pub(super) version_id: Option<String>,
|
||||
pub(super) storage_class: Option<String>,
|
||||
}
|
||||
|
||||
/// Maps a HEAD or GET response onto [`SourceHead`]. `metadata_prefix` is the
|
||||
/// provider's user-metadata header prefix (`x-ms-meta-`, `x-goog-meta-`); the
|
||||
/// stored shape drops it, matching the `x-amz-meta-` handling of the S3 path.
|
||||
pub(super) fn native_source_head(
|
||||
headers: &HeaderMap,
|
||||
metadata_prefix: &str,
|
||||
fields: NativeHeadFields,
|
||||
) -> Result<SourceHead, SourceError> {
|
||||
let size = header(headers, "content-length")
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.ok_or_else(|| SourceError::Other("source response has no valid content-length".to_string()))?;
|
||||
|
||||
let mut user_metadata = HashMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = name.as_str();
|
||||
if let Some(key) = name.strip_prefix(metadata_prefix)
|
||||
&& !key.is_empty()
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
user_metadata.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let etag = fields
|
||||
.etag
|
||||
.map(|etag| etag.trim().trim_matches('"').to_string())
|
||||
.filter(|etag| !etag.is_empty());
|
||||
// An opaque ETag never encodes a part count, so the multipart flag stays
|
||||
// false for it however the provider happens to spell the token.
|
||||
let is_multipart_etag = !fields.etag_is_opaque && etag.as_deref().is_some_and(is_multipart_etag);
|
||||
|
||||
Ok(SourceHead {
|
||||
etag,
|
||||
size,
|
||||
last_modified: header(headers, "last-modified").and_then(parse_http_timestamp),
|
||||
content_type: header_string(headers, "content-type"),
|
||||
content_encoding: header_string(headers, "content-encoding"),
|
||||
content_disposition: header_string(headers, "content-disposition"),
|
||||
content_language: header_string(headers, "content-language"),
|
||||
cache_control: header_string(headers, "cache-control"),
|
||||
expires: header_string(headers, "expires"),
|
||||
user_metadata,
|
||||
version_id: fields.version_id,
|
||||
storage_class: fields.storage_class,
|
||||
// Neither native provider hands back ciphertext: a customer-key object
|
||||
// is refused by the backend before it reaches this mapping, and the
|
||||
// service-managed encryption is transparent to the reader.
|
||||
sse: None,
|
||||
is_multipart_etag,
|
||||
etag_is_opaque: fields.etag_is_opaque,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::HeaderValue;
|
||||
|
||||
fn headers(pairs: &[(&str, &str)]) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in pairs {
|
||||
headers.insert(
|
||||
http::HeaderName::from_bytes(name.as_bytes()).expect("test header name"),
|
||||
HeaderValue::from_str(value).expect("test header value"),
|
||||
);
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
fn fields() -> NativeHeadFields {
|
||||
NativeHeadFields {
|
||||
etag: None,
|
||||
etag_is_opaque: false,
|
||||
version_id: None,
|
||||
storage_class: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_source_head_maps_content_headers_and_prefixed_metadata() {
|
||||
let headers = headers(&[
|
||||
("content-length", "1234"),
|
||||
("content-type", "text/plain"),
|
||||
("content-encoding", "gzip"),
|
||||
("content-language", "en"),
|
||||
("content-disposition", "attachment"),
|
||||
("cache-control", "max-age=60"),
|
||||
("expires", "Thu, 01 Jan 2026 00:00:00 GMT"),
|
||||
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"),
|
||||
("x-ms-meta-owner", "alice"),
|
||||
("x-goog-meta-owner", "not-mine"),
|
||||
]);
|
||||
let head = native_source_head(
|
||||
&headers,
|
||||
"x-ms-meta-",
|
||||
NativeHeadFields {
|
||||
etag: Some("\"0x8DCE1D2\"".to_string()),
|
||||
etag_is_opaque: true,
|
||||
version_id: Some("2026-01-01T00:00:00.0000000Z".to_string()),
|
||||
storage_class: Some("Hot".to_string()),
|
||||
},
|
||||
)
|
||||
.expect("head should map");
|
||||
|
||||
assert_eq!(head.size, 1234);
|
||||
assert_eq!(head.content_type.as_deref(), Some("text/plain"));
|
||||
assert_eq!(head.content_encoding.as_deref(), Some("gzip"));
|
||||
assert_eq!(head.content_language.as_deref(), Some("en"));
|
||||
assert_eq!(head.content_disposition.as_deref(), Some("attachment"));
|
||||
assert_eq!(head.cache_control.as_deref(), Some("max-age=60"));
|
||||
assert_eq!(head.expires.as_deref(), Some("Thu, 01 Jan 2026 00:00:00 GMT"));
|
||||
assert_eq!(
|
||||
head.last_modified,
|
||||
Some(SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_445_412_480)),
|
||||
"HTTP-date Last-Modified must parse"
|
||||
);
|
||||
assert_eq!(
|
||||
head.user_metadata,
|
||||
HashMap::from([("owner".to_string(), "alice".to_string())]),
|
||||
"only the provider's own metadata prefix is read"
|
||||
);
|
||||
assert_eq!(head.etag.as_deref(), Some("0x8DCE1D2"), "quotes are stripped, the token is kept");
|
||||
assert!(head.etag_is_opaque);
|
||||
assert!(!head.is_multipart_etag);
|
||||
assert_eq!(head.storage_class.as_deref(), Some("Hot"));
|
||||
assert!(head.sse.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_source_head_requires_a_content_length() {
|
||||
let err = native_source_head(&headers(&[("content-type", "text/plain")]), "x-ms-meta-", fields())
|
||||
.expect_err("a response without content-length is unusable");
|
||||
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_etag_never_reads_as_a_multipart_etag() {
|
||||
// A digest-shaped ETag keeps the S3 reading; the same string marked
|
||||
// opaque must not be split into "digest-partcount".
|
||||
for (opaque, expected) in [(false, true), (true, false)] {
|
||||
let head = native_source_head(
|
||||
&headers(&[("content-length", "1")]),
|
||||
"x-ms-meta-",
|
||||
NativeHeadFields {
|
||||
etag: Some("d41d8cd98f00b204e9800998ecf8427e-3".to_string()),
|
||||
etag_is_opaque: opaque,
|
||||
..fields()
|
||||
},
|
||||
)
|
||||
.expect("head should map");
|
||||
assert_eq!(head.is_multipart_etag, expected, "opaque = {opaque}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base64_md5_converts_only_sixteen_byte_digests() {
|
||||
assert_eq!(
|
||||
base64_md5_to_hex("1B2M2Y8AsgTpgAmY7PhCfg==").as_deref(),
|
||||
Some("d41d8cd98f00b204e9800998ecf8427e")
|
||||
);
|
||||
assert_eq!(base64_md5_to_hex("not base64!").as_deref(), None);
|
||||
// A CRC32C digest is four bytes: it must not pass as an MD5.
|
||||
assert_eq!(base64_md5_to_hex("AAAAAA==").as_deref(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_http_rejects_endpoints_that_are_not_bare_origins() {
|
||||
for bad in [
|
||||
"ftp://source.example.com",
|
||||
"https://user:pw@source.example.com",
|
||||
"https://source.example.com/container",
|
||||
"https://source.example.com/?x=1",
|
||||
"not a url",
|
||||
] {
|
||||
assert!(
|
||||
NativeHttp::new(bad, SourceTimeouts::default(), false, None).is_err(),
|
||||
"{bad} must be rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_http_percent_encodes_every_path_segment() {
|
||||
let http = NativeHttp::for_test(Url::parse("https://acct.blob.core.windows.net").expect("origin"));
|
||||
let url = http.url(["container", "dir", "a b?c#d.txt"]).expect("url should build");
|
||||
assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt");
|
||||
assert_eq!(url.query(), None, "a key with '?' must not become a query");
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket cache of keys the source answered 404 for
|
||||
//! (rustfs/backlog#2152). A hit short-circuits the source lookup for
|
||||
//! `policy.negative_cache_ttl_secs`; a TTL of zero disables the cache.
|
||||
//!
|
||||
//! Entries are never invalidated on a local PUT: once the object exists
|
||||
//! locally the handler never consults ODM for it, so a stale negative entry
|
||||
//! is harmless.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// Upper bound on remembered keys per bucket; LRU eviction beyond it.
|
||||
pub const NEGATIVE_CACHE_MAX_ENTRIES: u64 = 100_000;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct NegativeCache {
|
||||
cache: Option<moka::sync::Cache<String, ()>>,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl NegativeCache {
|
||||
/// `ttl == 0` builds a disabled cache that never records anything.
|
||||
pub fn new(ttl: Duration) -> Self {
|
||||
Self::with_capacity(ttl, NEGATIVE_CACHE_MAX_ENTRIES)
|
||||
}
|
||||
|
||||
pub fn with_capacity(ttl: Duration, max_entries: u64) -> Self {
|
||||
let cache = (!ttl.is_zero()).then(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(max_entries)
|
||||
.time_to_live(ttl)
|
||||
.build()
|
||||
});
|
||||
Self { cache, ttl }
|
||||
}
|
||||
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.cache.is_some()
|
||||
}
|
||||
|
||||
pub fn ttl(&self) -> Duration {
|
||||
self.ttl
|
||||
}
|
||||
|
||||
/// Whether `key` is currently remembered as absent on the source.
|
||||
pub fn contains(&self, key: &str) -> bool {
|
||||
self.cache.as_ref().is_some_and(|cache| cache.get(key).is_some())
|
||||
}
|
||||
|
||||
/// Remembers `key` as absent; no-op when disabled.
|
||||
pub fn insert(&self, key: &str) {
|
||||
if let Some(cache) = &self.cache {
|
||||
cache.insert(key.to_string(), ());
|
||||
}
|
||||
}
|
||||
|
||||
/// Forgets `key` (e.g. after an admin-triggered backfill found it).
|
||||
pub fn remove(&self, key: &str) {
|
||||
if let Some(cache) = &self.cache {
|
||||
cache.invalidate(key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Approximate live entry count, for status snapshots only.
|
||||
pub fn len(&self) -> u64 {
|
||||
self.cache.as_ref().map_or(0, |cache| {
|
||||
cache.run_pending_tasks();
|
||||
cache.entry_count()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn entry_expires_after_ttl() {
|
||||
let cache = NegativeCache::new(Duration::from_millis(80));
|
||||
assert!(cache.is_enabled());
|
||||
cache.insert("a/x");
|
||||
assert!(cache.contains("a/x"));
|
||||
assert!(!cache.contains("a/y"));
|
||||
std::thread::sleep(Duration::from_millis(160));
|
||||
assert!(!cache.contains("a/x"), "entry must expire after the TTL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_ttl_disables_the_cache() {
|
||||
let cache = NegativeCache::new(Duration::ZERO);
|
||||
assert!(!cache.is_enabled());
|
||||
cache.insert("a/x");
|
||||
assert!(!cache.contains("a/x"));
|
||||
assert!(cache.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_forgets_a_key() {
|
||||
let cache = NegativeCache::new(Duration::from_secs(30));
|
||||
cache.insert("a/x");
|
||||
cache.remove("a/x");
|
||||
assert!(!cache.contains("a/x"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capacity_bounds_entries() {
|
||||
let cache = NegativeCache::with_capacity(Duration::from_secs(30), 4);
|
||||
for i in 0..64 {
|
||||
cache.insert(&format!("k{i}"));
|
||||
}
|
||||
assert!(cache.len() <= 4, "len {} exceeds capacity", cache.len());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,531 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket on-demand migration counters (rustfs/backlog#2152).
|
||||
//!
|
||||
//! `OdmStats` is lock-free and survives config rebuilds; `snapshot()` turns
|
||||
//! it into the serializable `OdmStatsSnapshot` that the metrics collector
|
||||
//! and the admin status route (ODM-10/14/15) consume. Field names and label
|
||||
//! values are a wire contract: the golden JSON test below pins them.
|
||||
|
||||
use super::breaker::BreakerState;
|
||||
use super::source_client::SourceError;
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Request operations that can enter ODM.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OdmOp {
|
||||
Get,
|
||||
Head,
|
||||
}
|
||||
|
||||
impl OdmOp {
|
||||
pub const ALL: [OdmOp; 2] = [OdmOp::Get, OdmOp::Head];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
OdmOp::Get => "get",
|
||||
OdmOp::Head => "head",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How a request that entered ODM ended. `local_hit` is deliberately absent:
|
||||
/// requests served locally never reach the runtime.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OdmOutcome {
|
||||
SourceHit,
|
||||
SourceMiss,
|
||||
SourceError,
|
||||
BreakerOpen,
|
||||
NegativeCached,
|
||||
Filtered,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl OdmOutcome {
|
||||
pub const ALL: [OdmOutcome; 7] = [
|
||||
OdmOutcome::SourceHit,
|
||||
OdmOutcome::SourceMiss,
|
||||
OdmOutcome::SourceError,
|
||||
OdmOutcome::BreakerOpen,
|
||||
OdmOutcome::NegativeCached,
|
||||
OdmOutcome::Filtered,
|
||||
OdmOutcome::Unsupported,
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
OdmOutcome::SourceHit => "source_hit",
|
||||
OdmOutcome::SourceMiss => "source_miss",
|
||||
OdmOutcome::SourceError => "source_error",
|
||||
OdmOutcome::BreakerOpen => "breaker_open",
|
||||
OdmOutcome::NegativeCached => "negative_cached",
|
||||
OdmOutcome::Filtered => "filtered",
|
||||
OdmOutcome::Unsupported => "unsupported",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which pipeline stored a pulled object locally.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PullPath {
|
||||
/// Streamed to the client and written locally in one pass.
|
||||
Inline,
|
||||
/// Pulled by a background task after a partial/large read.
|
||||
Background,
|
||||
/// Pulled by the backfill job.
|
||||
Backfill,
|
||||
}
|
||||
|
||||
impl PullPath {
|
||||
pub const ALL: [PullPath; 3] = [PullPath::Inline, PullPath::Background, PullPath::Backfill];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
PullPath::Inline => "inline",
|
||||
PullPath::Background => "background",
|
||||
PullPath::Backfill => "backfill",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a pull did not produce a local object.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PullFailureReason {
|
||||
SourceNotFound,
|
||||
SourceAccessDenied,
|
||||
SourceThrottled,
|
||||
SourceTimeout,
|
||||
SourceConnect,
|
||||
SourceServerError,
|
||||
SourceUnsupported,
|
||||
SourceOther,
|
||||
/// Source bytes did not match the ETag advertised by HEAD/GET.
|
||||
EtagMismatch,
|
||||
/// The local write (internal PUT) failed.
|
||||
LocalWrite,
|
||||
/// The bucket quota rejected the write-back.
|
||||
Quota,
|
||||
/// The bucket state was removed or the process is shutting down.
|
||||
Canceled,
|
||||
/// The background pull queue was full.
|
||||
QueueFull,
|
||||
}
|
||||
|
||||
impl PullFailureReason {
|
||||
pub const ALL: [PullFailureReason; 13] = [
|
||||
PullFailureReason::SourceNotFound,
|
||||
PullFailureReason::SourceAccessDenied,
|
||||
PullFailureReason::SourceThrottled,
|
||||
PullFailureReason::SourceTimeout,
|
||||
PullFailureReason::SourceConnect,
|
||||
PullFailureReason::SourceServerError,
|
||||
PullFailureReason::SourceUnsupported,
|
||||
PullFailureReason::SourceOther,
|
||||
PullFailureReason::EtagMismatch,
|
||||
PullFailureReason::LocalWrite,
|
||||
PullFailureReason::Quota,
|
||||
PullFailureReason::Canceled,
|
||||
PullFailureReason::QueueFull,
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
PullFailureReason::SourceNotFound => "source_not_found",
|
||||
PullFailureReason::SourceAccessDenied => "source_access_denied",
|
||||
PullFailureReason::SourceThrottled => "source_throttled",
|
||||
PullFailureReason::SourceTimeout => "source_timeout",
|
||||
PullFailureReason::SourceConnect => "source_connect",
|
||||
PullFailureReason::SourceServerError => "source_server_error",
|
||||
PullFailureReason::SourceUnsupported => "source_unsupported",
|
||||
PullFailureReason::SourceOther => "source_other",
|
||||
PullFailureReason::EtagMismatch => "etag_mismatch",
|
||||
PullFailureReason::LocalWrite => "local_write",
|
||||
PullFailureReason::Quota => "quota",
|
||||
PullFailureReason::Canceled => "canceled",
|
||||
PullFailureReason::QueueFull => "queue_full",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&SourceError> for PullFailureReason {
|
||||
fn from(err: &SourceError) -> Self {
|
||||
match err {
|
||||
SourceError::NotFound => PullFailureReason::SourceNotFound,
|
||||
SourceError::AccessDenied => PullFailureReason::SourceAccessDenied,
|
||||
SourceError::Throttled => PullFailureReason::SourceThrottled,
|
||||
SourceError::Timeout => PullFailureReason::SourceTimeout,
|
||||
SourceError::Connect(_) => PullFailureReason::SourceConnect,
|
||||
SourceError::ServerError(_) => PullFailureReason::SourceServerError,
|
||||
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
|
||||
SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bounds (milliseconds) of the source latency histogram buckets; the
|
||||
/// implicit last bucket is unbounded. Roughly logarithmic from 5 ms to 60 s.
|
||||
pub const SOURCE_LATENCY_BUCKET_BOUNDS_MS: [u64; 14] = [
|
||||
5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000,
|
||||
];
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct LatencyHistogram {
|
||||
/// One counter per bound plus one for the overflow bucket.
|
||||
buckets: [AtomicU64; SOURCE_LATENCY_BUCKET_BOUNDS_MS.len() + 1],
|
||||
count: AtomicU64,
|
||||
sum_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl LatencyHistogram {
|
||||
fn observe(&self, latency: Duration) {
|
||||
let ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
|
||||
let index = SOURCE_LATENCY_BUCKET_BOUNDS_MS
|
||||
.iter()
|
||||
.position(|bound| ms <= *bound)
|
||||
.unwrap_or(SOURCE_LATENCY_BUCKET_BOUNDS_MS.len());
|
||||
self.buckets[index].fetch_add(1, Ordering::Relaxed);
|
||||
self.count.fetch_add(1, Ordering::Relaxed);
|
||||
self.sum_ms.fetch_add(ms, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> SourceLatencySnapshot {
|
||||
let mut cumulative = 0;
|
||||
let buckets = SOURCE_LATENCY_BUCKET_BOUNDS_MS
|
||||
.iter()
|
||||
.zip(self.buckets.iter())
|
||||
.map(|(bound, counter)| {
|
||||
cumulative += counter.load(Ordering::Relaxed);
|
||||
LatencyBucketSnapshot {
|
||||
le_ms: *bound,
|
||||
count: cumulative,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
SourceLatencySnapshot {
|
||||
buckets,
|
||||
count: self.count.load(Ordering::Relaxed),
|
||||
sum_ms: self.sum_ms.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The most recent source failure, kept for operators: class only, never the
|
||||
/// key or the message (which may echo attacker-controlled input).
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LastSourceError {
|
||||
pub class: String,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub at: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct OdmStats {
|
||||
requests_total: [[AtomicU64; OdmOutcome::ALL.len()]; OdmOp::ALL.len()],
|
||||
pulled_bytes_total: AtomicU64,
|
||||
pulled_objects_total: [AtomicU64; PullPath::ALL.len()],
|
||||
pull_failures_total: [AtomicU64; PullFailureReason::ALL.len()],
|
||||
inflight_pulls: AtomicU64,
|
||||
queue_depth: AtomicU64,
|
||||
source_latency: LatencyHistogram,
|
||||
last_source_error: Mutex<Option<LastSourceError>>,
|
||||
}
|
||||
|
||||
impl OdmStats {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn record_request(&self, op: OdmOp, outcome: OdmOutcome) {
|
||||
self.requests_total[op as usize][outcome as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_pulled_bytes(&self, bytes: u64) {
|
||||
self.pulled_bytes_total.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_pulled_object(&self, path: PullPath) {
|
||||
self.pulled_objects_total[path as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_pull_failure(&self, reason: PullFailureReason) {
|
||||
self.pull_failures_total[reason as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_source_latency(&self, latency: Duration) {
|
||||
self.source_latency.observe(latency);
|
||||
}
|
||||
|
||||
pub fn record_source_error(&self, err: &SourceError) {
|
||||
self.record_source_error_at(err, OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
pub fn record_source_error_at(&self, err: &SourceError, at: OffsetDateTime) {
|
||||
*self.last_source_error.lock() = Some(LastSourceError {
|
||||
class: err.class_label().to_string(),
|
||||
at,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn last_source_error(&self) -> Option<LastSourceError> {
|
||||
self.last_source_error.lock().clone()
|
||||
}
|
||||
|
||||
pub fn inflight_pulls(&self) -> u64 {
|
||||
self.inflight_pulls.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn queue_depth(&self) -> u64 {
|
||||
self.queue_depth.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// RAII increment of `inflight_pulls`.
|
||||
pub fn inflight_guard(self: &Arc<Self>) -> GaugeGuard {
|
||||
GaugeGuard::new(Arc::clone(self), OdmGauge::InflightPulls)
|
||||
}
|
||||
|
||||
/// RAII increment of `queue_depth`.
|
||||
pub fn queue_guard(self: &Arc<Self>) -> GaugeGuard {
|
||||
GaugeGuard::new(Arc::clone(self), OdmGauge::QueueDepth)
|
||||
}
|
||||
|
||||
fn gauge(&self, gauge: OdmGauge) -> &AtomicU64 {
|
||||
match gauge {
|
||||
OdmGauge::InflightPulls => &self.inflight_pulls,
|
||||
OdmGauge::QueueDepth => &self.queue_depth,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only, side-effect-free copy of every counter. The breaker lives
|
||||
/// next to the stats in the bucket state; its state is passed in so the
|
||||
/// snapshot stays a single document.
|
||||
pub fn snapshot(&self, breaker_state: BreakerState) -> OdmStatsSnapshot {
|
||||
let mut requests_total = BTreeMap::new();
|
||||
for op in OdmOp::ALL {
|
||||
let mut by_outcome = BTreeMap::new();
|
||||
for outcome in OdmOutcome::ALL {
|
||||
by_outcome.insert(
|
||||
outcome.as_str().to_string(),
|
||||
self.requests_total[op as usize][outcome as usize].load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
requests_total.insert(op.as_str().to_string(), by_outcome);
|
||||
}
|
||||
let pulled_objects_total = PullPath::ALL
|
||||
.iter()
|
||||
.map(|path| {
|
||||
(
|
||||
path.as_str().to_string(),
|
||||
self.pulled_objects_total[*path as usize].load(Ordering::Relaxed),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let pull_failures_total = PullFailureReason::ALL
|
||||
.iter()
|
||||
.map(|reason| {
|
||||
(
|
||||
reason.as_str().to_string(),
|
||||
self.pull_failures_total[*reason as usize].load(Ordering::Relaxed),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
OdmStatsSnapshot {
|
||||
requests_total,
|
||||
pulled_bytes_total: self.pulled_bytes_total.load(Ordering::Relaxed),
|
||||
pulled_objects_total,
|
||||
pull_failures_total,
|
||||
inflight_pulls: self.inflight_pulls(),
|
||||
queue_depth: self.queue_depth(),
|
||||
source_latency: self.source_latency.snapshot(),
|
||||
last_source_error: self.last_source_error(),
|
||||
breaker_state,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum OdmGauge {
|
||||
InflightPulls,
|
||||
QueueDepth,
|
||||
}
|
||||
|
||||
/// Increments a gauge on creation and decrements it on drop. Owns its
|
||||
/// `OdmStats` so it can live inside the pull slot handed to callers.
|
||||
#[derive(Debug)]
|
||||
pub struct GaugeGuard {
|
||||
stats: Arc<OdmStats>,
|
||||
gauge: OdmGauge,
|
||||
}
|
||||
|
||||
impl GaugeGuard {
|
||||
fn new(stats: Arc<OdmStats>, gauge: OdmGauge) -> Self {
|
||||
stats.gauge(gauge).fetch_add(1, Ordering::Relaxed);
|
||||
Self { stats, gauge }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GaugeGuard {
|
||||
fn drop(&mut self) {
|
||||
self.stats.gauge(self.gauge).fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LatencyBucketSnapshot {
|
||||
/// Upper bound of the bucket in milliseconds.
|
||||
pub le_ms: u64,
|
||||
/// Cumulative observations at or below `le_ms`.
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SourceLatencySnapshot {
|
||||
pub buckets: Vec<LatencyBucketSnapshot>,
|
||||
/// Total observations, including those above the last bound.
|
||||
pub count: u64,
|
||||
pub sum_ms: u64,
|
||||
}
|
||||
|
||||
/// Serializable copy of [`OdmStats`]. Every key is snake_case and every
|
||||
/// label set is fixed, so consumers can rely on the document shape.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OdmStatsSnapshot {
|
||||
/// `op -> outcome -> count`.
|
||||
pub requests_total: BTreeMap<String, BTreeMap<String, u64>>,
|
||||
pub pulled_bytes_total: u64,
|
||||
/// `path -> count`.
|
||||
pub pulled_objects_total: BTreeMap<String, u64>,
|
||||
/// `reason -> count`.
|
||||
pub pull_failures_total: BTreeMap<String, u64>,
|
||||
pub inflight_pulls: u64,
|
||||
pub queue_depth: u64,
|
||||
pub source_latency: SourceLatencySnapshot,
|
||||
pub last_source_error: Option<LastSourceError>,
|
||||
pub breaker_state: BreakerState,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
use time::macros::datetime;
|
||||
|
||||
#[test]
|
||||
fn snapshot_matches_golden_json() {
|
||||
let stats = Arc::new(OdmStats::new());
|
||||
stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
|
||||
stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
|
||||
stats.record_request(OdmOp::Head, OdmOutcome::NegativeCached);
|
||||
stats.record_pulled_bytes(4096);
|
||||
stats.record_pulled_object(PullPath::Inline);
|
||||
stats.record_pull_failure(PullFailureReason::from(&SourceError::Timeout));
|
||||
stats.record_source_latency(Duration::from_millis(3));
|
||||
stats.record_source_latency(Duration::from_millis(750));
|
||||
stats.record_source_latency(Duration::from_secs(90));
|
||||
stats.record_source_error_at(&SourceError::ServerError(502), datetime!(2026-09-02 10:00:00 UTC));
|
||||
let _inflight = stats.inflight_guard();
|
||||
let _queued = stats.queue_guard();
|
||||
|
||||
let snapshot = stats.snapshot(BreakerState::HalfOpen);
|
||||
let actual = serde_json::to_value(&snapshot).unwrap();
|
||||
let expected = json!({
|
||||
"requests_total": {
|
||||
"get": {
|
||||
"breaker_open": 0, "filtered": 0, "negative_cached": 0, "source_error": 0,
|
||||
"source_hit": 2, "source_miss": 0, "unsupported": 0
|
||||
},
|
||||
"head": {
|
||||
"breaker_open": 0, "filtered": 0, "negative_cached": 1, "source_error": 0,
|
||||
"source_hit": 0, "source_miss": 0, "unsupported": 0
|
||||
}
|
||||
},
|
||||
"pulled_bytes_total": 4096,
|
||||
"pulled_objects_total": { "backfill": 0, "background": 0, "inline": 1 },
|
||||
"pull_failures_total": {
|
||||
"canceled": 0, "etag_mismatch": 0, "local_write": 0, "queue_full": 0, "quota": 0,
|
||||
"source_access_denied": 0, "source_connect": 0, "source_not_found": 0, "source_other": 0,
|
||||
"source_server_error": 0, "source_throttled": 0, "source_timeout": 1, "source_unsupported": 0
|
||||
},
|
||||
"inflight_pulls": 1,
|
||||
"queue_depth": 1,
|
||||
"source_latency": {
|
||||
"buckets": [
|
||||
{ "le_ms": 5, "count": 1 }, { "le_ms": 10, "count": 1 }, { "le_ms": 20, "count": 1 },
|
||||
{ "le_ms": 50, "count": 1 }, { "le_ms": 100, "count": 1 }, { "le_ms": 200, "count": 1 },
|
||||
{ "le_ms": 500, "count": 1 }, { "le_ms": 1000, "count": 2 }, { "le_ms": 2000, "count": 2 },
|
||||
{ "le_ms": 5000, "count": 2 }, { "le_ms": 10000, "count": 2 }, { "le_ms": 20000, "count": 2 },
|
||||
{ "le_ms": 30000, "count": 2 }, { "le_ms": 60000, "count": 2 }
|
||||
],
|
||||
"count": 3,
|
||||
"sum_ms": 90753
|
||||
},
|
||||
"last_source_error": { "class": "server_error", "at": "2026-09-02T10:00:00Z" },
|
||||
"breaker_state": "half_open"
|
||||
});
|
||||
assert_eq!(actual, expected);
|
||||
|
||||
let round_trip: OdmStatsSnapshot = serde_json::from_value(actual).unwrap();
|
||||
assert_eq!(round_trip, snapshot);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gauges_return_to_zero_when_guards_drop() {
|
||||
let stats = Arc::new(OdmStats::new());
|
||||
{
|
||||
let _a = stats.inflight_guard();
|
||||
let _b = stats.inflight_guard();
|
||||
let _c = stats.queue_guard();
|
||||
assert_eq!(stats.inflight_pulls(), 2);
|
||||
assert_eq!(stats.queue_depth(), 1);
|
||||
}
|
||||
assert_eq!(stats.inflight_pulls(), 0);
|
||||
assert_eq!(stats.queue_depth(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pull_failure_reason_covers_every_source_error_class() {
|
||||
let cases = [
|
||||
(SourceError::NotFound, PullFailureReason::SourceNotFound),
|
||||
(SourceError::AccessDenied, PullFailureReason::SourceAccessDenied),
|
||||
(SourceError::Throttled, PullFailureReason::SourceThrottled),
|
||||
(SourceError::Timeout, PullFailureReason::SourceTimeout),
|
||||
(SourceError::Connect("x".into()), PullFailureReason::SourceConnect),
|
||||
(SourceError::ServerError(500), PullFailureReason::SourceServerError),
|
||||
(SourceError::Unsupported("x".into()), PullFailureReason::SourceUnsupported),
|
||||
(SourceError::Other("x".into()), PullFailureReason::SourceOther),
|
||||
];
|
||||
for (err, reason) in cases {
|
||||
assert_eq!(PullFailureReason::from(&err), reason, "{err:?}");
|
||||
assert_eq!(serde_json::to_string(&reason).unwrap(), format!("\"{}\"", reason.as_str()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_lists_are_exhaustive_and_unique() {
|
||||
let outcomes: std::collections::BTreeSet<_> = OdmOutcome::ALL.iter().map(|o| o.as_str()).collect();
|
||||
assert_eq!(outcomes.len(), OdmOutcome::ALL.len());
|
||||
let reasons: std::collections::BTreeSet<_> = PullFailureReason::ALL.iter().map(|r| r.as_str()).collect();
|
||||
assert_eq!(reasons.len(), PullFailureReason::ALL.len());
|
||||
let paths: std::collections::BTreeSet<_> = PullPath::ALL.iter().map(|p| p.as_str()).collect();
|
||||
assert_eq!(paths.len(), PullPath::ALL.len());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,120 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Scripted HTTP server for the native source backends' tests.
|
||||
//!
|
||||
//! The S3 backend can be driven through the SDK's own connector; the native
|
||||
//! backends talk to a real socket, so their tests need a server that answers a
|
||||
//! fixed script and records what it was asked. Every response closes its
|
||||
//! connection, which keeps one request on one socket and makes the script order
|
||||
//! exactly the request order.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use url::Url;
|
||||
|
||||
pub(super) struct ScriptedResponse {
|
||||
status: u16,
|
||||
headers: Vec<(&'static str, String)>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl ScriptedResponse {
|
||||
pub(super) fn new(status: u16, headers: Vec<(&'static str, String)>, body: String) -> Self {
|
||||
Self { status, headers, body }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct RecordedRequest {
|
||||
pub(super) method: String,
|
||||
/// Request target as it appeared on the wire: path plus query.
|
||||
pub(super) target: String,
|
||||
pub(super) headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl RecordedRequest {
|
||||
pub(super) fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;
|
||||
|
||||
/// Binds a loopback listener that answers `responses` in order and returns its
|
||||
/// origin plus the recorder. The task ends once the script is exhausted.
|
||||
pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind");
|
||||
let port = listener.local_addr().expect("fixture address").port();
|
||||
let recorder: Recorder = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&recorder);
|
||||
|
||||
tokio::spawn(async move {
|
||||
for response in responses {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 2048];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
match stream.read(&mut buffer).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(read) => request.extend_from_slice(&buffer[..read]),
|
||||
}
|
||||
}
|
||||
let text = String::from_utf8_lossy(&request).into_owned();
|
||||
let mut lines = text.lines();
|
||||
let start = lines.next().unwrap_or_default().to_string();
|
||||
let mut parts = start.split_whitespace();
|
||||
sink.lock().expect("recorder lock").push(RecordedRequest {
|
||||
method: parts.next().unwrap_or_default().to_string(),
|
||||
target: parts.next().unwrap_or_default().to_string(),
|
||||
headers: lines
|
||||
.take_while(|line| !line.is_empty())
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
|
||||
.collect(),
|
||||
});
|
||||
|
||||
// A scripted HEAD declares the object size in its own headers while
|
||||
// carrying no body, so an explicit `Content-Length` wins over the
|
||||
// body length.
|
||||
let declares_length = response
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("content-length"));
|
||||
let mut rendered = match declares_length {
|
||||
true => format!("HTTP/1.1 {} Scripted\r\nConnection: close\r\n", response.status),
|
||||
false => format!(
|
||||
"HTTP/1.1 {} Scripted\r\nContent-Length: {}\r\nConnection: close\r\n",
|
||||
response.status,
|
||||
response.body.len()
|
||||
),
|
||||
};
|
||||
for (name, value) in response.headers {
|
||||
rendered.push_str(&format!("{name}: {value}\r\n"));
|
||||
}
|
||||
rendered.push_str("\r\n");
|
||||
rendered.push_str(&response.body);
|
||||
let _ = stream.write_all(rendered.as_bytes()).await;
|
||||
let _ = stream.flush().await;
|
||||
}
|
||||
});
|
||||
|
||||
(Url::parse(&format!("http://127.0.0.1:{port}")).expect("fixture endpoint"), recorder)
|
||||
}
|
||||
@@ -180,6 +180,8 @@ impl RemoteS3EndpointSpec {
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RemoteS3ClientError {
|
||||
#[error("the {0} backend is not included in this build")]
|
||||
BackendNotCompiled(&'static str),
|
||||
#[error("remote endpoint requires credentials")]
|
||||
MissingCredentials,
|
||||
#[error("{0}")]
|
||||
@@ -281,9 +283,7 @@ impl Intercept for UserAgentSuffixInterceptor {
|
||||
|
||||
/// Builds the SDK config for `spec` without finalizing it, so callers can add
|
||||
/// interceptors or (in tests) swap the HTTP client before `build()`.
|
||||
pub(crate) async fn build_remote_s3_config(
|
||||
spec: &RemoteS3EndpointSpec,
|
||||
) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
|
||||
pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
|
||||
let Some(credentials) = &spec.credentials else {
|
||||
return Err(RemoteS3ClientError::MissingCredentials);
|
||||
};
|
||||
@@ -523,7 +523,7 @@ fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
|
||||
pub fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
|
||||
validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ pub(crate) mod tier_probe_intent;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
#[cfg(feature = "gcs")]
|
||||
pub mod warm_backend_gcs;
|
||||
pub mod warm_backend_huaweicloud;
|
||||
pub mod warm_backend_minio;
|
||||
|
||||
@@ -19,13 +19,14 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
#[cfg(feature = "gcs")]
|
||||
use crate::services::tier::warm_backend_gcs::WarmBackendGCS;
|
||||
use crate::services::tier::{
|
||||
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||
warm_backend_aliyun::WarmBackendAliyun,
|
||||
warm_backend_azure::WarmBackendAzure,
|
||||
warm_backend_gcs::WarmBackendGCS,
|
||||
warm_backend_huaweicloud::WarmBackendHuaweicloud,
|
||||
warm_backend_minio::WarmBackendMinIO,
|
||||
warm_backend_r2::WarmBackendR2,
|
||||
@@ -912,6 +913,15 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
});
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
TierType::GCS => {
|
||||
return Err(AdminError {
|
||||
code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(),
|
||||
message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(),
|
||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||
});
|
||||
}
|
||||
#[cfg(feature = "gcs")]
|
||||
TierType::GCS => {
|
||||
if let Some(gcs_config) = tier.gcs.as_ref() {
|
||||
let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
|
||||
@@ -1028,6 +1038,27 @@ mod tests {
|
||||
|
||||
const PROBE_VERSION: &str = "remote-v2";
|
||||
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
#[tokio::test]
|
||||
async fn gcs_backend_not_compiled_preserves_config() {
|
||||
let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#;
|
||||
let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend");
|
||||
assert_eq!(tier.tier_type, TierType::GCS);
|
||||
let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable");
|
||||
let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips");
|
||||
assert_eq!(restored.tier_type, TierType::GCS);
|
||||
let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved");
|
||||
assert_eq!(restored_gcs.bucket, "archive");
|
||||
assert_eq!(restored_gcs.creds, "secret");
|
||||
assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED");
|
||||
let error = match new_warm_backend(&tier, false).await {
|
||||
Ok(_) => panic!("an excluded GCS backend cannot be constructed"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code);
|
||||
assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
struct CountingBackend {
|
||||
put_result: fn() -> Result<String, std::io::Error>,
|
||||
removes: Arc<AtomicUsize>,
|
||||
|
||||
@@ -329,11 +329,11 @@ impl ECStore {
|
||||
/// reuse its result, which is sound because bucket deletion/recreation
|
||||
/// requires the lifecycle WRITE lock and therefore cannot have run while
|
||||
/// any read guard was continuously held.
|
||||
pub(crate) async fn acquire_bucket_incarnation_fence(
|
||||
pub async fn acquire_bucket_incarnation_fence(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
|
||||
) -> Result<super::BucketIncarnationFenceGuard> {
|
||||
let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let pieces = super::bucket_fence::FencePieces {
|
||||
registry: self.bucket_fence_registry.clone(),
|
||||
|
||||
@@ -150,7 +150,7 @@ impl BucketFenceRegistry {
|
||||
/// A held bucket lifecycle read lock plus its registration in the fence
|
||||
/// registry. Dropping the guard deregisters it; the memo is cleared when the
|
||||
/// last guard for the bucket drops (or a lost lock is observed).
|
||||
pub(crate) struct BucketIncarnationFenceGuard {
|
||||
pub struct BucketIncarnationFenceGuard {
|
||||
inner: Option<NamespaceLockGuard>,
|
||||
registry: Arc<BucketFenceRegistry>,
|
||||
bucket: String,
|
||||
@@ -158,6 +158,14 @@ pub(crate) struct BucketIncarnationFenceGuard {
|
||||
}
|
||||
|
||||
impl BucketIncarnationFenceGuard {
|
||||
/// Propagate lifecycle lock loss into the storage commit checks.
|
||||
/// The caller still owns this guard until the complete write tail drains.
|
||||
pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
|
||||
if let Some(guard) = self.namespace_lock_guard() {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
@@ -346,6 +354,36 @@ mod tests {
|
||||
first_pieces.abandon("b", first.token);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_options_inherit_bucket_fence_lock_loss() {
|
||||
let lock = NamespaceLock::new("bucket-fence-options".to_string(), Arc::new(LocalClient::new()));
|
||||
let inner = lock
|
||||
.acquire_guard(&lock_request("options"))
|
||||
.await
|
||||
.expect("acquire")
|
||||
.expect("quorum");
|
||||
let pieces = FencePieces {
|
||||
registry: Arc::default(),
|
||||
inner,
|
||||
};
|
||||
let registration = pieces.enter("b");
|
||||
let fence = pieces.into_guard("b", registration.token);
|
||||
let mut opts = crate::object_api::ObjectOptions::default();
|
||||
fence.attach_to_object_options(&mut opts);
|
||||
let inherited = opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("checkpoint inherits lifecycle guard");
|
||||
assert!(!inherited.is_lock_lost());
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
fence.namespace_lock_guard().expect("held guard").lock_lost_notified(),
|
||||
)
|
||||
.await
|
||||
.expect("distributed guard expires");
|
||||
assert!(inherited.is_lock_lost(), "the actual pre-rename options must observe lifecycle lock loss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buckets_are_isolated() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
|
||||
@@ -353,6 +353,11 @@ async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken)
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Shutdown token owned by this store instance.
|
||||
pub fn background_cancel_token(&self) -> Option<CancellationToken> {
|
||||
self.ctx.background_cancel_token()
|
||||
}
|
||||
|
||||
/// Validate topology and process storage-class overrides before any disk is opened.
|
||||
pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
|
||||
let drive_counts = startup_pool_drive_counts(endpoint_pools);
|
||||
|
||||
@@ -417,6 +417,7 @@ const MAX_UPLOADS_LIST: usize = 10000;
|
||||
mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
mod heal;
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
//! Wire types for `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}`,
|
||||
//! `GET .../status`, `POST .../backfill?op=start|cancel` and
|
||||
//! `GET .../backfill` (ODM-12), mirroring the server's config model
|
||||
//! (`crates/ecstore/src/bucket/on_demand_migration/config.rs`) and handler
|
||||
//! (`rustfs/src/on_demand_migration/config.rs`) and handler
|
||||
//! responses (`rustfs/src/admin/handlers/on_demand_migration.rs`). The SDK
|
||||
//! owns its own copies, madmin-go style; the fixtures under
|
||||
//! `fixtures/on_demand_migration/` are the contract both sides pin
|
||||
|
||||
@@ -38,3 +38,4 @@ pub(crate) use storage_api::metrics::{
|
||||
obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
|
||||
obs_transition_state_handle,
|
||||
};
|
||||
pub use storage_api::register_on_demand_migration_metrics_source;
|
||||
|
||||
@@ -17,13 +17,6 @@ use std::time::Duration;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
|
||||
use rustfs_ecstore::api::bucket::on_demand_migration::backfill::{
|
||||
BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::on_demand_migration::{
|
||||
BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
|
||||
OnDemandMigrationSys as SourceOnDemandMigrationSys,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::replication::{
|
||||
BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog,
|
||||
MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot,
|
||||
@@ -44,9 +37,7 @@ pub(crate) use rustfs_ecstore::api::runtime::{
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
use crate::metrics::collectors::{
|
||||
OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
|
||||
};
|
||||
use crate::metrics::collectors::{OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBucketStats};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
|
||||
@@ -465,70 +456,37 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
|
||||
buckets
|
||||
}
|
||||
|
||||
fn on_demand_migration_stats_from_snapshot(snapshot: SourceOdmBucketSnapshot) -> OnDemandMigrationBucketStats {
|
||||
let stats = snapshot.stats;
|
||||
OnDemandMigrationBucketStats {
|
||||
bucket: snapshot.bucket,
|
||||
requests_total: stats.requests_total,
|
||||
pulled_bytes_total: stats.pulled_bytes_total,
|
||||
pulled_objects_total: stats.pulled_objects_total,
|
||||
pull_failures_total: stats.pull_failures_total,
|
||||
inflight_pulls: stats.inflight_pulls,
|
||||
queue_depth: stats.queue_depth,
|
||||
source_latency_buckets: stats
|
||||
.source_latency
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.le_ms, bucket.count))
|
||||
.collect(),
|
||||
source_latency_count: stats.source_latency.count,
|
||||
source_latency_sum_ms: stats.source_latency.sum_ms,
|
||||
breaker_state: match stats.breaker_state {
|
||||
SourceOdmBreakerState::Closed => OnDemandMigrationBreakerState::Closed,
|
||||
SourceOdmBreakerState::HalfOpen => OnDemandMigrationBreakerState::HalfOpen,
|
||||
SourceOdmBreakerState::Open => OnDemandMigrationBreakerState::Open,
|
||||
},
|
||||
}
|
||||
struct OnDemandMigrationMetricsSource {
|
||||
snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
|
||||
backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
|
||||
}
|
||||
|
||||
/// Every bucket with live on-demand migration state on this node, sorted by
|
||||
/// name. Empty while the module switch is off.
|
||||
pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
|
||||
SourceOnDemandMigrationSys::get()
|
||||
.snapshot()
|
||||
.into_iter()
|
||||
.map(on_demand_migration_stats_from_snapshot)
|
||||
.collect()
|
||||
}
|
||||
static ON_DEMAND_MIGRATION_METRICS_SOURCE: std::sync::OnceLock<OnDemandMigrationMetricsSource> = std::sync::OnceLock::new();
|
||||
|
||||
fn on_demand_migration_backfill_stats_from_checkpoint(
|
||||
bucket: String,
|
||||
checkpoint: SourceBackfillCheckpoint,
|
||||
) -> OdmBackfillBucketStats {
|
||||
OdmBackfillBucketStats {
|
||||
bucket,
|
||||
state: checkpoint.state.as_str().to_string(),
|
||||
listed: checkpoint.listed,
|
||||
enqueued: checkpoint.enqueued,
|
||||
pulled: checkpoint.pulled,
|
||||
skipped_existing: checkpoint.skipped_existing,
|
||||
failed: checkpoint.failed,
|
||||
bytes: checkpoint.bytes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Backfill jobs running on this node, sorted by bucket. Empty until the
|
||||
/// runner is installed, and empty again once a job finishes: the series are
|
||||
/// per-node job progress, not a cluster-wide history.
|
||||
pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
|
||||
let buckets = source_global_backfill_runner()
|
||||
.map(|runner| {
|
||||
runner
|
||||
.local_job_snapshots()
|
||||
.into_iter()
|
||||
.map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
|
||||
.collect()
|
||||
/// Register the application-owned ODM snapshots before starting the collector.
|
||||
pub fn register_on_demand_migration_metrics_source(
|
||||
snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
|
||||
backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
|
||||
) -> bool {
|
||||
ON_DEMAND_MIGRATION_METRICS_SOURCE
|
||||
.set(OnDemandMigrationMetricsSource {
|
||||
snapshot,
|
||||
backfill_snapshot,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
|
||||
ON_DEMAND_MIGRATION_METRICS_SOURCE
|
||||
.get()
|
||||
.map(|source| (source.snapshot)())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
|
||||
let buckets = ON_DEMAND_MIGRATION_METRICS_SOURCE
|
||||
.get()
|
||||
.map(|source| (source.backfill_snapshot)())
|
||||
.unwrap_or_default();
|
||||
OdmBackfillRuntimeStats { server, buckets }
|
||||
}
|
||||
@@ -580,6 +538,31 @@ pub(crate) async fn obs_replication_site_stats_snapshot(current_data_transfer_ra
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn on_demand_migration_callbacks_supply_runtime_snapshots() {
|
||||
assert!(register_on_demand_migration_metrics_source(
|
||||
|| vec![OnDemandMigrationBucketStats {
|
||||
bucket: "configured".into(),
|
||||
pulled_bytes_total: 4096,
|
||||
..Default::default()
|
||||
}],
|
||||
|| vec![OdmBackfillBucketStats {
|
||||
bucket: "backfill".into(),
|
||||
pulled: 3,
|
||||
..Default::default()
|
||||
}],
|
||||
));
|
||||
let snapshot = obs_on_demand_migration_snapshot();
|
||||
assert_eq!(snapshot.len(), 1);
|
||||
assert_eq!(snapshot[0].bucket, "configured");
|
||||
assert_eq!(snapshot[0].pulled_bytes_total, 4096);
|
||||
let backfill = obs_on_demand_migration_backfill_snapshot("node-a".into());
|
||||
assert_eq!(backfill.server, "node-a");
|
||||
assert_eq!(backfill.buckets.len(), 1);
|
||||
assert_eq!(backfill.buckets[0].bucket, "backfill");
|
||||
assert_eq!(backfill.buckets[0].pulled, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn obs_replication_numeric_conversions_floor_negative_values() {
|
||||
assert_eq!(i64_to_u64_floor_zero(-1), 0);
|
||||
@@ -772,51 +755,6 @@ mod tests {
|
||||
assert_eq!(snapshot.mrf_last_flush_duration_millis, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_demand_migration_snapshot_projects_counters_and_breaker_state() {
|
||||
// Built from JSON: the snapshot's timestamps use `time`, which obs does not depend on.
|
||||
let snapshot: SourceOdmBucketSnapshot = serde_json::from_value(serde_json::json!({
|
||||
"bucket": "photos",
|
||||
"provider": "minio",
|
||||
"endpoint_host": "source.example.com",
|
||||
"applied_at": "2026-09-02T10:00:00Z",
|
||||
"client_error": null,
|
||||
"negative_cache_entries": 0,
|
||||
"inflight_keys": 1,
|
||||
"max_concurrent_pulls": 8,
|
||||
"stats": {
|
||||
"requests_total": {"get": {"source_hit": 2}},
|
||||
"pulled_bytes_total": 4096,
|
||||
"pulled_objects_total": {"inline": 1},
|
||||
"pull_failures_total": {"source_timeout": 1},
|
||||
"inflight_pulls": 1,
|
||||
"queue_depth": 2,
|
||||
"source_latency": {
|
||||
"buckets": [{"le_ms": 5, "count": 1}, {"le_ms": 10, "count": 2}],
|
||||
"count": 3,
|
||||
"sum_ms": 90753
|
||||
},
|
||||
"last_source_error": {"class": "server_error", "at": "2026-09-02T10:00:00Z"},
|
||||
"breaker_state": "open"
|
||||
}
|
||||
}))
|
||||
.expect("runtime snapshot decodes");
|
||||
|
||||
let stats = on_demand_migration_stats_from_snapshot(snapshot);
|
||||
|
||||
assert_eq!(stats.bucket, "photos");
|
||||
assert_eq!(stats.requests_total["get"]["source_hit"], 2);
|
||||
assert_eq!(stats.pulled_bytes_total, 4096);
|
||||
assert_eq!(stats.pulled_objects_total["inline"], 1);
|
||||
assert_eq!(stats.pull_failures_total["source_timeout"], 1);
|
||||
assert_eq!(stats.inflight_pulls, 1);
|
||||
assert_eq!(stats.queue_depth, 2);
|
||||
assert_eq!(stats.source_latency_buckets, vec![(5, 1), (10, 2)]);
|
||||
assert_eq!(stats.source_latency_count, 3);
|
||||
assert_eq!(stats.source_latency_sum_ms, 90_753);
|
||||
assert_eq!(stats.breaker_state, OnDemandMigrationBreakerState::Open);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
|
||||
Reference in New Issue
Block a user