mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 04:25:54 +00:00
feat(odm): merge the source listing into ListObjectsV2 (#7112)
* feat(odm): merge the source listing into ListObjectsV2 Adds policy.list_through: ListObjectsV2 merges the local and source listings into one ordered page so clients see the whole namespace during an on-demand migration. Local entries win a key both sides hold, CommonPrefixes are unioned under a delimiter, and the continuation token is an opaque versioned envelope carrying both cursors. A source listing failure or an open breaker follows policy.source_error: propagate answers 424, not_found answers from local state and marks the response x-rustfs-on-demand-migration-list: local_only. Source listings are capped at 10 per second per bucket. * test(odm): refresh the e2e-full darwin selection digest The list-through e2e module adds seven cases to the merge lane.
This commit is contained in:
Zhengchao An
@@ -109,6 +109,7 @@ pub struct OdmPolicy {
|
||||
pub head: String,
|
||||
pub range_get: String,
|
||||
pub source_error: String,
|
||||
pub list_through: bool,
|
||||
pub respect_local_delete_marker: bool,
|
||||
pub preserve_etag: bool,
|
||||
pub copy_tags: bool,
|
||||
@@ -136,6 +137,7 @@ impl Default for OdmPolicy {
|
||||
head: "proxy".to_string(),
|
||||
range_get: "serve_and_backfill".to_string(),
|
||||
source_error: "propagate".to_string(),
|
||||
list_through: false,
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
@@ -576,6 +578,23 @@ impl OdmTestEnv {
|
||||
}
|
||||
|
||||
/// Raw signed `GET /{bucket}/{key}` against the RustFS under test.
|
||||
/// Raw signed `ListObjectsV2` (`?list-type=2&<query>`) so a scenario can
|
||||
/// assert on the response headers and the raw XML, which the SDK hides.
|
||||
pub async fn raw_list_objects_v2(&self, bucket: &str, query: &str) -> Result<RawResponse, BoxError> {
|
||||
let url = format!(
|
||||
"{}/{bucket}?list-type=2{}{query}",
|
||||
self.rustfs.url,
|
||||
if query.is_empty() { "" } else { "&" }
|
||||
);
|
||||
let response =
|
||||
signed_request(http::Method::GET, &url, &self.rustfs.access_key, &self.rustfs.secret_key, None, None).await?;
|
||||
Ok(RawResponse {
|
||||
status: response.status().as_u16(),
|
||||
headers: response.headers().clone(),
|
||||
body: response.bytes().await?,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn raw_get(&self, bucket: &str, key: &str) -> Result<RawResponse, BoxError> {
|
||||
let url = format!("{}/{bucket}/{key}", self.rustfs.url);
|
||||
let response =
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Optional merged `ListObjectsV2` (`policy.list_through`, ODM-17,
|
||||
//! rustfs/backlog#2164): full pagination over a source and a local namespace,
|
||||
//! common-prefix union under a delimiter, the continuation-token contract, and
|
||||
//! the two `source_error` behaviours when the source listing fails.
|
||||
|
||||
use super::common::{BoxError, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env};
|
||||
use crate::fake_s3_target::{FaultAction, Operation};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
|
||||
type TestResult = Result<(), BoxError>;
|
||||
|
||||
const SOURCE_BUCKET: &str = "odm-list-source";
|
||||
const LIST_HEADER: &str = "x-rustfs-on-demand-migration-list";
|
||||
|
||||
/// Byte lengths that tell a local object from a source one in a listing.
|
||||
const SOURCE_BODY_LEN: usize = 3;
|
||||
const LOCAL_BODY_LEN: usize = 11;
|
||||
|
||||
fn body(len: usize) -> Bytes {
|
||||
vec![b'x'; len].into()
|
||||
}
|
||||
|
||||
/// RustFS migrating `bucket` from `SOURCE_BUCKET` with `list_through` on.
|
||||
async fn list_through_env(bucket: &str, adjust: impl FnOnce(&mut OdmSourceSpec)) -> Result<OdmTestEnv, BoxError> {
|
||||
start_configured_env(bucket, SOURCE_BUCKET, |spec| {
|
||||
spec.policy.list_through = true;
|
||||
adjust(spec);
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Every key the bucket lists, walked through the merged continuation token.
|
||||
/// Also returns the size each page reported per key and the page sizes, so a
|
||||
/// caller can assert who won a shared key and that no page exceeded `max_keys`.
|
||||
async fn walk_listing(
|
||||
env: &OdmTestEnv,
|
||||
bucket: &str,
|
||||
delimiter: Option<&str>,
|
||||
max_keys: i32,
|
||||
) -> Result<(Vec<(String, i64)>, Vec<String>, Vec<usize>), BoxError> {
|
||||
let mut objects = Vec::new();
|
||||
let mut prefixes = Vec::new();
|
||||
let mut page_sizes = Vec::new();
|
||||
let mut token: Option<String> = None;
|
||||
for _ in 0..1000 {
|
||||
let page = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(max_keys)
|
||||
.set_delimiter(delimiter.map(str::to_string))
|
||||
.set_continuation_token(token.take())
|
||||
.send()
|
||||
.await?;
|
||||
let listed = page.contents().len() + page.common_prefixes().len();
|
||||
page_sizes.push(listed);
|
||||
for object in page.contents() {
|
||||
objects.push((object.key().unwrap_or_default().to_string(), object.size().unwrap_or_default()));
|
||||
}
|
||||
for prefix in page.common_prefixes() {
|
||||
prefixes.push(prefix.prefix().unwrap_or_default().to_string());
|
||||
}
|
||||
if !page.is_truncated().unwrap_or(false) {
|
||||
return Ok((objects, prefixes, page_sizes));
|
||||
}
|
||||
token = Some(
|
||||
page.next_continuation_token()
|
||||
.ok_or("truncated page without a continuation token")?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
Err("merged listing did not terminate".into())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_merges_the_whole_namespace_across_full_pagination() -> TestResult {
|
||||
let bucket = "odm-list-merge";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
|
||||
// 2000 source keys, 80 of them also local, plus 10 local-only keys that
|
||||
// interleave between source keys ("obj-00010x" sorts after "obj-00010").
|
||||
let source_keys: Vec<String> = (0..2000).map(|index| format!("obj-{index:05}")).collect();
|
||||
let seeds: Vec<SeedObject> = source_keys
|
||||
.iter()
|
||||
.map(|key| SeedObject::new(key.clone(), body(SOURCE_BODY_LEN)))
|
||||
.collect();
|
||||
env.seed_source(SOURCE_BUCKET, &seeds);
|
||||
|
||||
let shared: Vec<String> = source_keys.iter().step_by(25).cloned().collect();
|
||||
let local_only: Vec<String> = (0..10).map(|index| format!("obj-{:05}x", index * 7)).collect();
|
||||
for key in shared.iter().chain(local_only.iter()) {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
let max_keys = 97;
|
||||
let (objects, prefixes, page_sizes) = walk_listing(&env, bucket, None, max_keys).await?;
|
||||
assert!(prefixes.is_empty(), "no delimiter means no common prefixes");
|
||||
|
||||
let mut expected: Vec<String> = source_keys.iter().chain(local_only.iter()).cloned().collect();
|
||||
expected.sort();
|
||||
expected.dedup();
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(listed, expected, "the merged listing is the sorted, deduplicated union");
|
||||
assert!(
|
||||
page_sizes.iter().all(|size| *size <= max_keys as usize),
|
||||
"no page may exceed max_keys: {page_sizes:?}"
|
||||
);
|
||||
|
||||
let shared_sizes: Vec<i64> = objects
|
||||
.iter()
|
||||
.filter(|(key, _)| shared.contains(key))
|
||||
.map(|(_, size)| *size)
|
||||
.collect();
|
||||
assert_eq!(shared_sizes.len(), shared.len(), "every shared key is listed exactly once");
|
||||
assert!(
|
||||
shared_sizes.iter().all(|size| *size == LOCAL_BODY_LEN as i64),
|
||||
"the local object wins a key both sides hold"
|
||||
);
|
||||
let source_sizes: Vec<i64> = objects
|
||||
.iter()
|
||||
.filter(|(key, _)| !shared.contains(key) && !local_only.contains(key))
|
||||
.map(|(_, size)| *size)
|
||||
.collect();
|
||||
assert!(
|
||||
source_sizes.iter().all(|size| *size == SOURCE_BODY_LEN as i64),
|
||||
"source-only keys report the source's own size"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_unions_common_prefixes_under_a_delimiter() -> TestResult {
|
||||
let bucket = "odm-list-delimiter";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("p1/a", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("p1/b", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("p2/a", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("top-s", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
for key in ["p1/c", "p3/a", "top-l"] {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
// A page size of two forces the prefix union to survive page boundaries.
|
||||
let (objects, prefixes, page_sizes) = walk_listing(&env, bucket, Some("/"), 2).await?;
|
||||
assert_eq!(prefixes, vec!["p1/", "p2/", "p3/"], "prefixes are unioned and deduplicated");
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(listed, vec!["top-l", "top-s"]);
|
||||
assert!(page_sizes.iter().all(|size| *size <= 2), "{page_sizes:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_propagates_a_source_listing_failure() -> TestResult {
|
||||
let bucket = "odm-list-propagate";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new("remote", body(SOURCE_BODY_LEN))]);
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("local")
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.source
|
||||
.inject(Operation::ListObjectsV2, FaultAction::ResponseStatus(503), 1);
|
||||
let failure = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("propagate must surface the source failure");
|
||||
let failure = failure.into_service_error();
|
||||
assert_eq!(failure.meta().code(), Some("SourceUnavailable"), "{failure:?}");
|
||||
|
||||
// The next listing sees a healthy source again and merges both sides.
|
||||
let (objects, _, _) = walk_listing(&env, bucket, None, 100).await?;
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(listed, vec!["local", "remote"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_degrades_to_local_only_under_the_not_found_policy() -> TestResult {
|
||||
let bucket = "odm-list-degrade";
|
||||
let env = list_through_env(bucket, |spec| spec.policy.source_error = "not_found".to_string()).await?;
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new("remote", body(SOURCE_BODY_LEN))]);
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("local")
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.source
|
||||
.inject(Operation::ListObjectsV2, FaultAction::ResponseStatus(503), 1);
|
||||
let degraded = env.raw_list_objects_v2(bucket, "max-keys=100").await?;
|
||||
assert_eq!(degraded.status, 200, "{}", String::from_utf8_lossy(°raded.body));
|
||||
assert_eq!(
|
||||
degraded.header(LIST_HEADER),
|
||||
Some("local_only"),
|
||||
"a degraded listing must say so in the response header"
|
||||
);
|
||||
let xml = String::from_utf8_lossy(°raded.body).to_string();
|
||||
assert!(xml.contains("<Key>local</Key>"), "{xml}");
|
||||
assert!(!xml.contains("<Key>remote</Key>"), "a degraded listing shows local state only: {xml}");
|
||||
|
||||
let healthy = env.raw_list_objects_v2(bucket, "max-keys=100").await?;
|
||||
assert_eq!(healthy.status, 200);
|
||||
assert_eq!(healthy.header(LIST_HEADER), None, "a healthy merge carries no degradation marker");
|
||||
assert!(String::from_utf8_lossy(&healthy.body).contains("<Key>remote</Key>"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
|
||||
let bucket = "odm-list-token";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("a", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("b", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("c", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
|
||||
let page = env.client.list_objects_v2().bucket(bucket).max_keys(1).send().await?;
|
||||
let token = page.next_continuation_token().ok_or("first page must be truncated")?;
|
||||
let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
|
||||
assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
|
||||
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
|
||||
let rejected = env
|
||||
.raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rejected.status,
|
||||
400,
|
||||
"a bumped token version is a client error: {}",
|
||||
String::from_utf8_lossy(&rejected.body)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_merged_token_keeps_paginating_after_list_through_is_turned_off() -> TestResult {
|
||||
let bucket = "odm-list-token-off";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("s1", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("s2", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
for key in ["l1", "l2"] {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
let page = env.client.list_objects_v2().bucket(bucket).max_keys(1).send().await?;
|
||||
assert_eq!(page.contents()[0].key(), Some("l1"));
|
||||
let token = page
|
||||
.next_continuation_token()
|
||||
.ok_or("first page must be truncated")?
|
||||
.to_string();
|
||||
|
||||
let mut spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
spec.policy.list_through = false;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
let resumed = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(10)
|
||||
.continuation_token(token)
|
||||
.send()
|
||||
.await?;
|
||||
let listed: Vec<&str> = resumed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed, vec!["l2"], "a merged token falls back to its local cursor");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_local_delete_marker_hides_the_source_key_from_a_merged_listing() -> TestResult {
|
||||
let bucket = "odm-list-delete-marker";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("kept", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("shadowed", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("shadowed")
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
env.client.delete_object().bucket(bucket).key("shadowed").send().await?;
|
||||
|
||||
let (objects, _, _) = walk_listing(&env, bucket, None, 100).await?;
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(
|
||||
listed,
|
||||
vec!["kept"],
|
||||
"a local delete marker shadows the source key the same way it does on GET"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -18,7 +18,8 @@
|
||||
//! fake S3 source, admin-API wrappers, seeding and local-state assertions.
|
||||
//! `harness_self_test` proves the harness itself; `get_basic_test` covers the
|
||||
//! GET read-through (rustfs/backlog#2156) and `backfill_test` the background
|
||||
//! backfill job (ODM-12, rustfs/backlog#2159). The fault, concurrency,
|
||||
//! backfill job (ODM-12, rustfs/backlog#2159); `list_through_test` covers the
|
||||
//! optional merged `ListObjectsV2` (ODM-17, rustfs/backlog#2164). The fault, concurrency,
|
||||
//! interaction and real-source matrix is rustfs/backlog#2158; its lane split
|
||||
//! lives in `.config/nextest.toml` (fault / concurrency / real source run
|
||||
//! nightly, the rest in the merge lane).
|
||||
@@ -31,4 +32,5 @@ mod fault_test;
|
||||
mod get_basic_test;
|
||||
mod harness_self_test;
|
||||
mod interaction_test;
|
||||
mod list_through_test;
|
||||
mod real_source_test;
|
||||
|
||||
@@ -166,6 +166,11 @@ pub mod bucket {
|
||||
WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
|
||||
idle_guarded_body,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
||||
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
||||
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub mod backfill {
|
||||
pub use crate::bucket::on_demand_migration::backfill::{
|
||||
BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
|
||||
@@ -179,8 +184,8 @@ pub mod bucket {
|
||||
}
|
||||
pub mod source_client {
|
||||
pub use crate::bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe,
|
||||
SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
|
||||
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
|
||||
resolve_path_style,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1717,6 +1717,7 @@ mod tests {
|
||||
objects,
|
||||
is_truncated,
|
||||
next_continuation_token: is_truncated.then(|| end.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -238,6 +238,11 @@ pub struct PolicyConfig {
|
||||
pub range_get: RangeGetPolicy,
|
||||
#[serde(default)]
|
||||
pub source_error: SourceErrorPolicy,
|
||||
/// Merge the source listing into `ListObjectsV2` so clients see the whole
|
||||
/// namespace during the migration (rustfs/backlog#2164). Off by default:
|
||||
/// it puts the source in the path of every listing.
|
||||
#[serde(default)]
|
||||
pub list_through: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub respect_local_delete_marker: bool,
|
||||
#[serde(default = "default_true")]
|
||||
@@ -268,6 +273,7 @@ impl Default for PolicyConfig {
|
||||
head: HeadPolicy::default(),
|
||||
range_get: RangeGetPolicy::default(),
|
||||
source_error: SourceErrorPolicy::default(),
|
||||
list_through: false,
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
@@ -615,6 +621,7 @@ mod tests {
|
||||
"head": "proxy",
|
||||
"range_get": "serve_and_backfill",
|
||||
"source_error": "propagate",
|
||||
"list_through": false,
|
||||
"respect_local_delete_marker": true,
|
||||
"preserve_etag": true,
|
||||
"copy_tags": false,
|
||||
|
||||
@@ -0,0 +1,824 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Optional `ListObjectsV2` list-through (`policy.list_through`,
|
||||
//! rustfs/backlog#2164): the local listing and the source listing are merged
|
||||
//! into one ordered page so clients see the whole namespace while a bucket is
|
||||
//! migrating.
|
||||
//!
|
||||
//! Everything here is pure. The handler owns the I/O and the payloads; this
|
||||
//! module owns the ordering, the page boundary, and the opaque continuation
|
||||
//! token that carries both cursors.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// The only continuation-token envelope version this build reads and writes.
|
||||
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
|
||||
|
||||
/// Envelope marker. A bucket that is *not* merging hands out the local
|
||||
/// listing's own marker, so the decoder needs a positive signal before it
|
||||
/// treats an opaque token as a merged one.
|
||||
const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
|
||||
|
||||
/// Pages fetched per side per request: the first page, plus at most one refill
|
||||
/// when the first one was mostly consumed by the previous page. Two pages of
|
||||
/// `max_keys` always cover a full merged page, so this is a bound, not a
|
||||
/// heuristic.
|
||||
pub const MAX_LIST_FETCHES_PER_SIDE: usize = 2;
|
||||
|
||||
/// Per-bucket ceiling on source `ListObjectsV2` calls, in calls per second.
|
||||
pub const SOURCE_LIST_RATE_PER_SEC: u32 = 10;
|
||||
|
||||
/// How long a listing may wait for a source rate-limit slot before it gives up
|
||||
/// and answers from local state alone.
|
||||
pub const SOURCE_LIST_MAX_RATE_WAIT: Duration = Duration::from_secs(1);
|
||||
|
||||
/// One listing entry as the merge orders it: an object key, or — under a
|
||||
/// delimiter — a rolled-up common prefix. Both sort by `name` alone, which is
|
||||
/// how S3 interleaves `Contents` and `CommonPrefixes` on the wire.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ListEntryKey {
|
||||
pub name: String,
|
||||
pub is_prefix: bool,
|
||||
}
|
||||
|
||||
impl ListEntryKey {
|
||||
pub fn object(name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
is_prefix: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prefix(name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
is_prefix: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MergeSide {
|
||||
Local,
|
||||
Source,
|
||||
}
|
||||
|
||||
/// One entry of the merged page: the side it came from and its index in that
|
||||
/// side's buffer, in push order. The caller keeps the payloads.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct MergePick {
|
||||
pub side: MergeSide,
|
||||
pub index: usize,
|
||||
}
|
||||
|
||||
/// The continuation-token envelope. Opaque to clients: it is serialized as
|
||||
/// JSON and then base64-encoded by the same helper that encodes a plain local
|
||||
/// marker, so the wire shape is `base64(json)`.
|
||||
///
|
||||
/// A `null` cursor with `done = false` means "list that side from the start";
|
||||
/// `done = true` means the side is finished and must not be listed again.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListThroughToken {
|
||||
/// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
|
||||
pub t: String,
|
||||
pub v: u32,
|
||||
#[serde(default)]
|
||||
pub local: Option<String>,
|
||||
#[serde(default)]
|
||||
pub local_done: bool,
|
||||
#[serde(default)]
|
||||
pub source: Option<String>,
|
||||
#[serde(default)]
|
||||
pub source_done: bool,
|
||||
/// Last entry the previous page consumed. A side whose page was only
|
||||
/// partially consumed is re-listed from the same cursor and everything at
|
||||
/// or below this key is dropped, which is delimiter-safe: a rolled-up
|
||||
/// common prefix compares as itself, never as its members.
|
||||
#[serde(default)]
|
||||
pub last_key: Option<String>,
|
||||
}
|
||||
|
||||
impl ListThroughToken {
|
||||
fn new(local: SideCursor, source: SideCursor, last_key: Option<String>) -> Self {
|
||||
Self {
|
||||
t: LIST_THROUGH_TOKEN_TAG.to_string(),
|
||||
v: LIST_THROUGH_TOKEN_VERSION,
|
||||
local: local.token,
|
||||
local_done: local.done,
|
||||
source: source.token,
|
||||
source_done: source.done,
|
||||
last_key,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> String {
|
||||
// The envelope is built here from owned strings, so serialization
|
||||
// cannot fail; the fallback keeps the signature infallible.
|
||||
serde_json::to_string(self).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// What a decoded (base64-stripped) continuation token turned out to be.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ListThroughCursor {
|
||||
/// A plain local listing marker: the bucket was not merging when the token
|
||||
/// was issued, or the client is paginating a non-merged listing.
|
||||
Local(String),
|
||||
Merged(Box<ListThroughToken>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ListThroughTokenError {
|
||||
#[error("continuation token version {0} is not supported")]
|
||||
UnsupportedVersion(u32),
|
||||
/// The message never echoes the token: it is client-controlled input.
|
||||
#[error("continuation token is malformed")]
|
||||
Malformed,
|
||||
}
|
||||
|
||||
/// Classifies an already base64-decoded continuation token.
|
||||
///
|
||||
/// Only a JSON object carrying the envelope marker is read as a merged token;
|
||||
/// anything else is a local marker, so a bucket that turns `list_through` off
|
||||
/// keeps paginating with the tokens it handed out. A token that *is* an
|
||||
/// envelope but was tampered with (unknown version, unknown field, truncated
|
||||
/// JSON) is an error, never a silent fallback.
|
||||
pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
|
||||
if !decoded.starts_with('{') {
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
}
|
||||
let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
|
||||
// Not JSON at all: an object key may legitimately start with '{'.
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
};
|
||||
if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
}
|
||||
match value.get("v").and_then(serde_json::Value::as_u64) {
|
||||
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {}
|
||||
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
|
||||
None => return Err(ListThroughTokenError::Malformed),
|
||||
}
|
||||
serde_json::from_value::<ListThroughToken>(value)
|
||||
.map(|token| ListThroughCursor::Merged(Box::new(token)))
|
||||
.map_err(|_| ListThroughTokenError::Malformed)
|
||||
}
|
||||
|
||||
/// How the source must be listed for a request, given `filter.prefix`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum SourceListPlan {
|
||||
/// The request prefix and `filter.prefix` are disjoint: the source holds
|
||||
/// nothing this listing could show.
|
||||
Skip,
|
||||
/// Ordinary paged listing under `prefix`, rolled up with the request's
|
||||
/// delimiter — the source's own roll-up boundary matches the request's.
|
||||
Page { prefix: String },
|
||||
/// `filter.prefix` reaches past a delimiter, so every key the source could
|
||||
/// contribute rolls into this one common prefix. One bounded probe listing
|
||||
/// decides whether it exists; there is nothing to paginate.
|
||||
Folded { probe_prefix: String, common_prefix: String },
|
||||
}
|
||||
|
||||
/// Intersects the request prefix with `filter.prefix` and decides how (or
|
||||
/// whether) the source is listed.
|
||||
pub fn source_list_plan(request_prefix: &str, filter_prefix: Option<&str>, delimiter: Option<&str>) -> SourceListPlan {
|
||||
let filter = filter_prefix.unwrap_or_default();
|
||||
let source_prefix = if filter.starts_with(request_prefix) {
|
||||
filter
|
||||
} else if request_prefix.starts_with(filter) {
|
||||
request_prefix
|
||||
} else {
|
||||
return SourceListPlan::Skip;
|
||||
};
|
||||
|
||||
let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty()) else {
|
||||
return SourceListPlan::Page {
|
||||
prefix: source_prefix.to_string(),
|
||||
};
|
||||
};
|
||||
|
||||
// `source_prefix` always starts with `request_prefix`, so this slice is on
|
||||
// a character boundary.
|
||||
let extra = &source_prefix[request_prefix.len()..];
|
||||
match extra.find(delimiter) {
|
||||
Some(at) => SourceListPlan::Folded {
|
||||
probe_prefix: source_prefix.to_string(),
|
||||
common_prefix: format!("{request_prefix}{}", &extra[..at + delimiter.len()]),
|
||||
},
|
||||
None => SourceListPlan::Page {
|
||||
prefix: source_prefix.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Where one side resumes.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SideCursor {
|
||||
pub token: Option<String>,
|
||||
pub done: bool,
|
||||
}
|
||||
|
||||
/// One page a side actually fetched this round.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct FetchedPage {
|
||||
/// Token it was fetched with; `None` means from the start of the listing.
|
||||
token: Option<String>,
|
||||
/// Entries it contributed to the buffer, after the `last_key` filter.
|
||||
count: usize,
|
||||
/// Cursor for the page after it, `None` when it was the last one.
|
||||
next_token: Option<String>,
|
||||
}
|
||||
|
||||
/// Where a side resumes after `consumed` of its buffered entries were taken.
|
||||
///
|
||||
/// A fully consumed page advances to its successor; a partially consumed one
|
||||
/// is re-listed from the same cursor next time and re-filtered by `last_key`.
|
||||
fn advance_cursor(pages: &[FetchedPage], consumed: usize) -> SideCursor {
|
||||
let mut remaining = consumed;
|
||||
let mut cursor = SideCursor { token: None, done: true };
|
||||
for page in pages {
|
||||
if remaining >= page.count {
|
||||
remaining -= page.count;
|
||||
cursor = match &page.next_token {
|
||||
Some(next) => SideCursor {
|
||||
token: Some(next.clone()),
|
||||
done: false,
|
||||
},
|
||||
None => SideCursor { token: None, done: true },
|
||||
};
|
||||
} else {
|
||||
cursor = SideCursor {
|
||||
token: page.token.clone(),
|
||||
done: false,
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
cursor
|
||||
}
|
||||
|
||||
/// A page the merge driver still needs.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct FetchRequest {
|
||||
pub side: MergeSide,
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SideState {
|
||||
start: SideCursor,
|
||||
pages: Vec<FetchedPage>,
|
||||
entries: Vec<ListEntryKey>,
|
||||
more: bool,
|
||||
disabled: bool,
|
||||
}
|
||||
|
||||
impl SideState {
|
||||
fn from_cursor(token: Option<String>, done: bool) -> Self {
|
||||
Self {
|
||||
start: SideCursor { token, done },
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn needs_page(&self, max_keys: usize) -> Option<Option<String>> {
|
||||
if self.disabled || self.start.done {
|
||||
return None;
|
||||
}
|
||||
match self.pages.last() {
|
||||
None => Some(self.start.token.clone()),
|
||||
Some(last) => {
|
||||
let room = self.entries.len() < max_keys;
|
||||
let capped = self.pages.len() >= MAX_LIST_FETCHES_PER_SIDE;
|
||||
(self.more && room && !capped).then(|| last.next_token.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The merged page, once both sides have handed over everything they will.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct MergeOutcome {
|
||||
/// Entries of the merged page, in wire order; indices point into each
|
||||
/// side's buffer in push order.
|
||||
pub picks: Vec<MergePick>,
|
||||
pub is_truncated: bool,
|
||||
/// `Some` exactly when `is_truncated`.
|
||||
pub next_token: Option<ListThroughToken>,
|
||||
}
|
||||
|
||||
/// Drives one merged page: the caller asks [`Self::next_fetch`] what to list,
|
||||
/// hands the page back with [`Self::push_page`], and finishes with
|
||||
/// [`Self::finish`]. Nothing here does I/O, so the same driver is exercised by
|
||||
/// the property test and by the handler.
|
||||
#[derive(Debug)]
|
||||
pub struct ListThroughMerger {
|
||||
max_keys: usize,
|
||||
last_key: Option<String>,
|
||||
local: SideState,
|
||||
source: SideState,
|
||||
}
|
||||
|
||||
impl ListThroughMerger {
|
||||
/// `token` is the envelope from the client's continuation token, absent on
|
||||
/// the first page of a listing.
|
||||
pub fn new(max_keys: usize, token: Option<&ListThroughToken>) -> Self {
|
||||
let (local, source, last_key) = match token {
|
||||
Some(token) => (
|
||||
SideState::from_cursor(token.local.clone(), token.local_done),
|
||||
SideState::from_cursor(token.source.clone(), token.source_done),
|
||||
token.last_key.clone(),
|
||||
),
|
||||
None => (SideState::default(), SideState::default(), None),
|
||||
};
|
||||
Self {
|
||||
max_keys,
|
||||
last_key,
|
||||
local,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an entry the listing returned still belongs to this page: a
|
||||
/// re-listed page repeats what the previous page already consumed.
|
||||
pub fn accepts(&self, name: &str) -> bool {
|
||||
self.last_key.as_deref().is_none_or(|bound| name > bound)
|
||||
}
|
||||
|
||||
/// The source contributes nothing to this page: it failed, is rate-limited,
|
||||
/// or `filter.prefix` excludes it.
|
||||
pub fn disable_source(&mut self) {
|
||||
self.source.disabled = true;
|
||||
}
|
||||
|
||||
pub fn next_fetch(&self) -> Option<FetchRequest> {
|
||||
for (side, state) in [(MergeSide::Local, &self.local), (MergeSide::Source, &self.source)] {
|
||||
if let Some(token) = state.needs_page(self.max_keys) {
|
||||
return Some(FetchRequest { side, token });
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Records one fetched page. `entries` must be sorted by `name` and already
|
||||
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads
|
||||
/// in the same order.
|
||||
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
|
||||
let state = match side {
|
||||
MergeSide::Local => &mut self.local,
|
||||
MergeSide::Source => &mut self.source,
|
||||
};
|
||||
let token = match state.pages.last() {
|
||||
Some(last) => last.next_token.clone(),
|
||||
None => state.start.token.clone(),
|
||||
};
|
||||
// A truncated page without a cursor cannot be continued; treating the
|
||||
// side as finished is the only alternative to looping on it forever.
|
||||
state.more = is_truncated && next_token.is_some();
|
||||
state.pages.push(FetchedPage {
|
||||
token,
|
||||
count: entries.len(),
|
||||
next_token: is_truncated.then_some(next_token).flatten(),
|
||||
});
|
||||
state.entries.extend(entries);
|
||||
}
|
||||
|
||||
pub fn finish(self) -> MergeOutcome {
|
||||
let Self {
|
||||
max_keys,
|
||||
last_key,
|
||||
local,
|
||||
source,
|
||||
} = self;
|
||||
|
||||
// A side with more pages behind it can only be trusted up to the last
|
||||
// key it handed over: past that horizon the other side's entries could
|
||||
// still be deduplicated by one we have not seen, which is what keeps
|
||||
// "local wins on equal keys" true across page boundaries.
|
||||
let horizon = [
|
||||
local
|
||||
.more
|
||||
.then(|| local.entries.last().map_or("", |entry| entry.name.as_str())),
|
||||
source
|
||||
.more
|
||||
.then(|| source.entries.last().map_or("", |entry| entry.name.as_str())),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min();
|
||||
|
||||
let mut picks = Vec::with_capacity(max_keys.min(local.entries.len() + source.entries.len()));
|
||||
let mut consumed_local = 0usize;
|
||||
let mut consumed_source = 0usize;
|
||||
let mut consumed_key: Option<String> = None;
|
||||
|
||||
while picks.len() < max_keys {
|
||||
let next_local = local.entries.get(consumed_local).map(|entry| entry.name.as_str());
|
||||
let next_source = source.entries.get(consumed_source).map(|entry| entry.name.as_str());
|
||||
let name = match (next_local, next_source) {
|
||||
(None, None) => break,
|
||||
(Some(name), None) | (None, Some(name)) => name,
|
||||
(Some(left), Some(right)) => left.min(right),
|
||||
};
|
||||
if horizon.is_some_and(|horizon| name > horizon) {
|
||||
break;
|
||||
}
|
||||
let take_local = next_local == Some(name);
|
||||
let take_source = next_source == Some(name);
|
||||
consumed_key = Some(name.to_string());
|
||||
if take_local {
|
||||
picks.push(MergePick {
|
||||
side: MergeSide::Local,
|
||||
index: consumed_local,
|
||||
});
|
||||
consumed_local += 1;
|
||||
} else {
|
||||
picks.push(MergePick {
|
||||
side: MergeSide::Source,
|
||||
index: consumed_source,
|
||||
});
|
||||
}
|
||||
if take_source {
|
||||
consumed_source += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let local_cursor = advance_cursor(&local.pages, consumed_local);
|
||||
let source_cursor = if source.disabled {
|
||||
// Keep the source where it was so a recovered source resumes there;
|
||||
// this page is answered from local state alone.
|
||||
source.start.clone()
|
||||
} else {
|
||||
advance_cursor(&source.pages, consumed_source)
|
||||
};
|
||||
let local_left = !local_cursor.done || consumed_local < local.entries.len();
|
||||
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
|
||||
let is_truncated = local_left || source_left;
|
||||
|
||||
let last_key = consumed_key.or(last_key);
|
||||
MergeOutcome {
|
||||
picks,
|
||||
is_truncated,
|
||||
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Token bucket capping source `ListObjectsV2` calls for one bucket.
|
||||
///
|
||||
/// A caller that cannot be served inside its budget is refused rather than
|
||||
/// queued: a listing degrades to local state instead of holding the request
|
||||
/// open behind other tenants' listings.
|
||||
#[derive(Debug)]
|
||||
pub struct SourceListRateLimiter {
|
||||
rate_per_sec: f64,
|
||||
burst: f64,
|
||||
state: Mutex<RateLimiterState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RateLimiterState {
|
||||
tokens: f64,
|
||||
updated_at: Instant,
|
||||
}
|
||||
|
||||
impl SourceListRateLimiter {
|
||||
pub fn new(rate_per_sec: u32) -> Self {
|
||||
let rate_per_sec = f64::from(rate_per_sec.max(1));
|
||||
Self {
|
||||
rate_per_sec,
|
||||
burst: rate_per_sec,
|
||||
state: Mutex::new(RateLimiterState {
|
||||
tokens: rate_per_sec,
|
||||
updated_at: Instant::now(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserves one call, returning how long the caller must wait before making
|
||||
/// it, or `None` when that wait would exceed `max_wait` (nothing is
|
||||
/// reserved then).
|
||||
pub fn reserve(&self, max_wait: Duration) -> Option<Duration> {
|
||||
self.reserve_at(Instant::now(), max_wait)
|
||||
}
|
||||
|
||||
pub fn reserve_at(&self, now: Instant, max_wait: Duration) -> Option<Duration> {
|
||||
let mut state = self.state.lock();
|
||||
let elapsed = now.saturating_duration_since(state.updated_at).as_secs_f64();
|
||||
state.tokens = (state.tokens + elapsed * self.rate_per_sec).min(self.burst);
|
||||
state.updated_at = now;
|
||||
if state.tokens >= 1.0 {
|
||||
state.tokens -= 1.0;
|
||||
return Some(Duration::ZERO);
|
||||
}
|
||||
let wait = Duration::from_secs_f64((1.0 - state.tokens) / self.rate_per_sec);
|
||||
if wait > max_wait {
|
||||
return None;
|
||||
}
|
||||
state.tokens -= 1.0;
|
||||
Some(wait)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SourceListRateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new(SOURCE_LIST_RATE_PER_SEC)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
/// One `ListObjectsV2` page over a sorted key set, with the S3 rules the
|
||||
/// merge relies on: delimiter roll-up, `max_keys`, and a continuation
|
||||
/// token that resumes after the last entry the page returned.
|
||||
fn reference_page(
|
||||
keys: &[String],
|
||||
prefix: &str,
|
||||
delimiter: Option<&str>,
|
||||
after: Option<&str>,
|
||||
max_keys: usize,
|
||||
) -> (Vec<ListEntryKey>, bool, Option<String>) {
|
||||
let mut entries: Vec<ListEntryKey> = Vec::new();
|
||||
for key in keys.iter().filter(|key| key.starts_with(prefix)) {
|
||||
let entry = match delimiter.and_then(|delimiter| key[prefix.len()..].find(delimiter).map(|at| (delimiter, at))) {
|
||||
Some((delimiter, at)) => ListEntryKey::prefix(&key[..prefix.len() + at + delimiter.len()]),
|
||||
None => ListEntryKey::object(key.clone()),
|
||||
};
|
||||
if entries.last().is_none_or(|last| last.name != entry.name) {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
if let Some(after) = after {
|
||||
entries.retain(|entry| entry.name.as_str() > after);
|
||||
}
|
||||
let truncated = entries.len() > max_keys;
|
||||
entries.truncate(max_keys);
|
||||
let next = truncated.then(|| entries.last().map(|entry| entry.name.clone())).flatten();
|
||||
(entries, truncated && next.is_some(), next)
|
||||
}
|
||||
|
||||
/// Full pagination through the merger, returning every entry it emitted and
|
||||
/// the page sizes it produced.
|
||||
fn walk(
|
||||
local: &[String],
|
||||
source: &[String],
|
||||
prefix: &str,
|
||||
delimiter: Option<&str>,
|
||||
max_keys: usize,
|
||||
) -> (Vec<(ListEntryKey, MergeSide)>, Vec<usize>) {
|
||||
let mut emitted = Vec::new();
|
||||
let mut page_sizes = Vec::new();
|
||||
let mut token: Option<ListThroughToken> = None;
|
||||
for _ in 0..10_000 {
|
||||
let mut merger = ListThroughMerger::new(max_keys, token.as_ref());
|
||||
let mut buffers = [Vec::<ListEntryKey>::new(), Vec::<ListEntryKey>::new()];
|
||||
while let Some(fetch) = merger.next_fetch() {
|
||||
let keys = match fetch.side {
|
||||
MergeSide::Local => local,
|
||||
MergeSide::Source => source,
|
||||
};
|
||||
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
|
||||
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
|
||||
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
|
||||
merger.push_page(fetch.side, kept, truncated, next);
|
||||
}
|
||||
let outcome = merger.finish();
|
||||
page_sizes.push(outcome.picks.len());
|
||||
for pick in &outcome.picks {
|
||||
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
|
||||
emitted.push((entry, pick.side));
|
||||
}
|
||||
if !outcome.is_truncated {
|
||||
return (emitted, page_sizes);
|
||||
}
|
||||
token = outcome.next_token;
|
||||
}
|
||||
panic!("merged pagination did not terminate");
|
||||
}
|
||||
|
||||
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
|
||||
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
|
||||
all.sort();
|
||||
all.dedup();
|
||||
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
|
||||
entries
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_page_rolls_up_and_paginates() {
|
||||
let keys = vec!["a/1".to_string(), "a/2".to_string(), "b".to_string(), "c/1".to_string()];
|
||||
let (entries, truncated, next) = reference_page(&keys, "", Some("/"), None, 2);
|
||||
assert_eq!(entries, vec![ListEntryKey::prefix("a/"), ListEntryKey::object("b")]);
|
||||
assert!(truncated);
|
||||
assert_eq!(next.as_deref(), Some("b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_pages_are_ordered_and_local_wins_on_equal_keys() {
|
||||
let local = vec!["a".to_string(), "c".to_string()];
|
||||
let source = vec!["b".to_string(), "c".to_string(), "d".to_string()];
|
||||
let (emitted, sizes) = walk(&local, &source, "", None, 2);
|
||||
let names: Vec<&str> = emitted.iter().map(|(entry, _)| entry.name.as_str()).collect();
|
||||
assert_eq!(names, vec!["a", "b", "c", "d"]);
|
||||
assert_eq!(emitted[2].1, MergeSide::Local, "the shared key must come from local");
|
||||
assert!(sizes.iter().all(|size| *size <= 2), "{sizes:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_only_listing_paginates_without_a_local_side() {
|
||||
let source: Vec<String> = (0..7).map(|index| format!("k{index}")).collect();
|
||||
let (emitted, _) = walk(&[], &source, "", None, 3);
|
||||
assert_eq!(emitted.len(), 7);
|
||||
assert!(emitted.iter().all(|(_, side)| *side == MergeSide::Source));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_disabled_source_answers_from_local_alone() {
|
||||
let mut merger = ListThroughMerger::new(10, None);
|
||||
merger.disable_source();
|
||||
assert_eq!(
|
||||
merger.next_fetch(),
|
||||
Some(FetchRequest {
|
||||
side: MergeSide::Local,
|
||||
token: None
|
||||
})
|
||||
);
|
||||
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
|
||||
assert_eq!(merger.next_fetch(), None);
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.picks.len(), 1);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_degraded_page_keeps_the_source_cursor_for_the_next_one() {
|
||||
let resume = ListThroughToken {
|
||||
t: LIST_THROUGH_TOKEN_TAG.to_string(),
|
||||
v: LIST_THROUGH_TOKEN_VERSION,
|
||||
local: Some("local-1".to_string()),
|
||||
local_done: false,
|
||||
source: Some("source-1".to_string()),
|
||||
source_done: false,
|
||||
last_key: Some("a".to_string()),
|
||||
};
|
||||
let mut merger = ListThroughMerger::new(1, Some(&resume));
|
||||
merger.disable_source();
|
||||
merger.push_page(
|
||||
MergeSide::Local,
|
||||
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
|
||||
true,
|
||||
Some("local-2".to_string()),
|
||||
);
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("truncated page carries a token");
|
||||
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
|
||||
assert!(!token.source_done);
|
||||
assert_eq!(token.last_key.as_deref(), Some("b"));
|
||||
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_round_trips_and_rejects_tampering() {
|
||||
let token = ListThroughToken::new(
|
||||
SideCursor {
|
||||
token: Some("l".to_string()),
|
||||
done: false,
|
||||
},
|
||||
SideCursor { token: None, done: true },
|
||||
Some("k".to_string()),
|
||||
);
|
||||
let encoded = token.encode();
|
||||
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
|
||||
|
||||
let bumped = encoded.replace("\"v\":1", "\"v\":2");
|
||||
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2)));
|
||||
|
||||
let extra = encoded.replace("{", "{\"x\":1,");
|
||||
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
|
||||
|
||||
let truncated = &encoded[..encoded.len() - 3];
|
||||
assert_eq!(decode_continuation_token(truncated), Ok(ListThroughCursor::Local(truncated.to_string())));
|
||||
|
||||
let no_version = "{\"t\":\"odm-list\"}";
|
||||
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_local_marker_stays_local() {
|
||||
assert_eq!(
|
||||
decode_continuation_token("photos/2024/01.jpg"),
|
||||
Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decode_continuation_token("{not json"),
|
||||
Ok(ListThroughCursor::Local("{not json".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decode_continuation_token("{\"t\":\"other\"}"),
|
||||
Ok(ListThroughCursor::Local("{\"t\":\"other\"}".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_list_plan_intersects_the_filter_prefix() {
|
||||
assert_eq!(source_list_plan("", None, None), SourceListPlan::Page { prefix: String::new() });
|
||||
assert_eq!(
|
||||
source_list_plan("photos/2024/", Some("photos/"), None),
|
||||
SourceListPlan::Page {
|
||||
prefix: "photos/2024/".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
source_list_plan("photos/", Some("photos/2024/"), None),
|
||||
SourceListPlan::Page {
|
||||
prefix: "photos/2024/".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(source_list_plan("videos/", Some("photos/"), None), SourceListPlan::Skip);
|
||||
assert_eq!(
|
||||
source_list_plan("", Some("photos/2024/"), Some("/")),
|
||||
SourceListPlan::Folded {
|
||||
probe_prefix: "photos/2024/".to_string(),
|
||||
common_prefix: "photos/".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
source_list_plan("pho", Some("photos"), Some("/")),
|
||||
SourceListPlan::Page {
|
||||
prefix: "photos".to_string()
|
||||
},
|
||||
"a filter prefix that adds no delimiter keeps the source's own roll-up"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limiter_spends_its_burst_then_paces_and_refuses() {
|
||||
let limiter = SourceListRateLimiter::new(10);
|
||||
let start = Instant::now();
|
||||
for _ in 0..10 {
|
||||
assert_eq!(limiter.reserve_at(start, Duration::from_secs(1)), Some(Duration::ZERO));
|
||||
}
|
||||
let paced = limiter.reserve_at(start, Duration::from_secs(1)).expect("within the budget");
|
||||
assert!(paced > Duration::ZERO && paced <= Duration::from_millis(101), "{paced:?}");
|
||||
assert_eq!(limiter.reserve_at(start, Duration::ZERO), None, "a zero budget refuses");
|
||||
// A full second of refill restores the whole burst.
|
||||
assert_eq!(limiter.reserve_at(start + Duration::from_secs(5), Duration::ZERO), Some(Duration::ZERO));
|
||||
}
|
||||
|
||||
fn key_set() -> impl Strategy<Value = Vec<String>> {
|
||||
proptest::collection::btree_set(
|
||||
proptest::sample::select(vec!["a", "a/", "a/1", "a/2", "a/b/1", "b", "b/1", "c", "c/1", "c/2", "d", "d/e/f"])
|
||||
.prop_map(str::to_string),
|
||||
0..=12,
|
||||
)
|
||||
.prop_map(|set: BTreeSet<String>| set.into_iter().collect())
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(256))]
|
||||
|
||||
/// Full pagination of a merged listing equals the sorted, deduplicated
|
||||
/// union of both sides, with every shared key served by local, and no
|
||||
/// page longer than `max_keys`.
|
||||
#[test]
|
||||
fn merged_pagination_equals_the_deduplicated_union(
|
||||
local in key_set(),
|
||||
source in key_set(),
|
||||
max_keys in 1usize..=5,
|
||||
with_delimiter in any::<bool>(),
|
||||
prefix in proptest::sample::select(vec!["", "a", "a/", "c/"]),
|
||||
) {
|
||||
let delimiter = with_delimiter.then_some("/");
|
||||
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
|
||||
let got: Vec<ListEntryKey> = emitted.iter().map(|(entry, _)| entry.clone()).collect();
|
||||
prop_assert_eq!(got, expected(&local, &source, prefix, delimiter));
|
||||
prop_assert!(sizes.iter().all(|size| *size <= max_keys), "{:?}", sizes);
|
||||
for (entry, side) in &emitted {
|
||||
if !entry.is_prefix && local.iter().any(|key| key == &entry.name) {
|
||||
prop_assert_eq!(*side, MergeSide::Local, "local must win for {}", entry.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@
|
||||
pub mod backfill;
|
||||
pub mod breaker;
|
||||
pub mod config;
|
||||
pub mod list_through;
|
||||
pub mod negative_cache;
|
||||
pub mod pull;
|
||||
pub mod source_client;
|
||||
@@ -38,6 +39,11 @@ pub use config::{
|
||||
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
|
||||
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub use list_through::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
||||
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
||||
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
|
||||
pub use pull::{
|
||||
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion,
|
||||
|
||||
@@ -511,10 +511,28 @@ pub struct SourceObject {
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SourcePage {
|
||||
pub objects: Vec<SourceObject>,
|
||||
/// Rolled-up prefixes, in the local namespace; always empty when the
|
||||
/// request carried no delimiter.
|
||||
pub common_prefixes: Vec<String>,
|
||||
pub is_truncated: bool,
|
||||
pub next_continuation_token: Option<String>,
|
||||
}
|
||||
|
||||
/// One `ListObjectsV2` page request against the source. Keys are given in the
|
||||
/// local namespace; `SourceClient` maps them through `source_prefix`.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SourceListRequest<'a> {
|
||||
pub prefix: Option<&'a str>,
|
||||
/// Rolls the source's own listing up the same way the local one is rolled
|
||||
/// up, so a page under a delimiter stays bounded.
|
||||
pub delimiter: Option<&'a str>,
|
||||
/// Ignored by S3 when `continuation_token` is set, so the caller must pass
|
||||
/// at most one of the two.
|
||||
pub start_after: Option<&'a str>,
|
||||
pub continuation_token: Option<&'a str>,
|
||||
pub max_keys: i32,
|
||||
}
|
||||
|
||||
/// Result of [`SourceClient::probe`]: the bucket answered HEAD and a
|
||||
/// one-key listing succeeded.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -669,13 +687,35 @@ impl SourceClient {
|
||||
continuation_token: Option<&str>,
|
||||
max_keys: i32,
|
||||
) -> Result<SourcePage, SourceError> {
|
||||
self.list_page(&SourceListRequest {
|
||||
prefix,
|
||||
continuation_token,
|
||||
max_keys,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::list_objects_v2`] with the delimiter and start-after the
|
||||
/// list-through merge needs (rustfs/backlog#2164).
|
||||
pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
|
||||
// `start_after` is silently ignored by S3 once a continuation token is
|
||||
// present; refuse the ambiguous pair rather than list from the wrong
|
||||
// position.
|
||||
if request.continuation_token.is_some() && request.start_after.is_some() {
|
||||
return Err(SourceError::Other(
|
||||
"source listing takes either a continuation token or start-after, not both".to_string(),
|
||||
));
|
||||
}
|
||||
let output = self
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket)
|
||||
.prefix(self.source_key(prefix.unwrap_or_default()))
|
||||
.set_continuation_token(continuation_token.map(str::to_string))
|
||||
.max_keys(max_keys)
|
||||
.prefix(self.source_key(request.prefix.unwrap_or_default()))
|
||||
.set_delimiter(request.delimiter.map(str::to_string))
|
||||
.set_start_after(request.start_after.map(|after| self.source_key(after)))
|
||||
.set_continuation_token(request.continuation_token.map(str::to_string))
|
||||
.max_keys(request.max_keys)
|
||||
.send()
|
||||
.await
|
||||
.map_err(classify_sdk_error)?;
|
||||
@@ -693,9 +733,16 @@ impl SourceClient {
|
||||
.into_iter()
|
||||
.filter_map(|object| self.source_object(object))
|
||||
.collect();
|
||||
let common_prefixes = output
|
||||
.common_prefixes
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
|
||||
.collect();
|
||||
|
||||
Ok(SourcePage {
|
||||
objects,
|
||||
common_prefixes,
|
||||
is_truncated,
|
||||
next_continuation_token,
|
||||
})
|
||||
|
||||
@@ -44,6 +44,7 @@ use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict};
|
||||
use super::config::{
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig,
|
||||
};
|
||||
use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
|
||||
use super::negative_cache::NegativeCache;
|
||||
use super::pull::{OdmWriteBack, PullQueue};
|
||||
use super::source_client::{SourceClient, SourceClientSpec, SourceError, SourceProvider, SourceTimeouts};
|
||||
@@ -280,6 +281,8 @@ pub struct BucketOdmState {
|
||||
write_back: Option<Arc<dyn OdmWriteBack>>,
|
||||
/// Started by `pull::BucketOdmState::pull_queue` on first enqueue.
|
||||
pub(super) pull_queue: OnceLock<Arc<PullQueue>>,
|
||||
/// Caps source listings for this bucket under `policy.list_through`.
|
||||
list_rate_limiter: SourceListRateLimiter,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BucketOdmState {
|
||||
@@ -328,6 +331,7 @@ impl BucketOdmState {
|
||||
last_source_error_logged_at: Mutex::new(None),
|
||||
write_back,
|
||||
pull_queue: OnceLock::new(),
|
||||
list_rate_limiter: SourceListRateLimiter::new(SOURCE_LIST_RATE_PER_SEC),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -360,6 +364,12 @@ impl BucketOdmState {
|
||||
&self.negative_cache
|
||||
}
|
||||
|
||||
/// Per-bucket rate limit on source `ListObjectsV2` calls, consulted by the
|
||||
/// list-through merge (rustfs/backlog#2164).
|
||||
pub fn list_rate_limiter(&self) -> &SourceListRateLimiter {
|
||||
&self.list_rate_limiter
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> &Arc<OdmStats> {
|
||||
&self.stats
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
|
||||
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
|
||||
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
|
||||
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
|
||||
|
||||
@@ -192,6 +192,9 @@ pub struct OnDemandMigrationPolicy {
|
||||
pub range_get: OnDemandMigrationRangeGetPolicy,
|
||||
#[serde(default)]
|
||||
pub source_error: OnDemandMigrationSourceErrorPolicy,
|
||||
/// Merge the source listing into `ListObjectsV2` (rustfs/backlog#2164).
|
||||
#[serde(default)]
|
||||
pub list_through: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub respect_local_delete_marker: bool,
|
||||
#[serde(default = "default_true")]
|
||||
@@ -222,6 +225,7 @@ impl Default for OnDemandMigrationPolicy {
|
||||
head: OnDemandMigrationHeadPolicy::default(),
|
||||
range_get: OnDemandMigrationRangeGetPolicy::default(),
|
||||
source_error: OnDemandMigrationSourceErrorPolicy::default(),
|
||||
list_through: false,
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
|
||||
Reference in New Issue
Block a user