mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
feat(odm): merge the source listing into ListObjectsV2 (#7112)
* feat(odm): merge the source listing into ListObjectsV2 Adds policy.list_through: ListObjectsV2 merges the local and source listings into one ordered page so clients see the whole namespace during an on-demand migration. Local entries win a key both sides hold, CommonPrefixes are unioned under a delimiter, and the continuation token is an opaque versioned envelope carrying both cursors. A source listing failure or an open breaker follows policy.source_error: propagate answers 424, not_found answers from local state and marks the response x-rustfs-on-demand-migration-list: local_only. Source listings are capped at 10 per second per bucket. * test(odm): refresh the e2e-full darwin selection digest The list-through e2e module adds seven cases to the merge lane.
This commit is contained in:
Zhengchao An
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=bb8c16cd63a94ff5e5e400d891ccc980710529dd5cb13a5a0c955c6c313bccd9
|
||||
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
|
||||
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
|
||||
|
||||
@@ -109,6 +109,7 @@ pub struct OdmPolicy {
|
||||
pub head: String,
|
||||
pub range_get: String,
|
||||
pub source_error: String,
|
||||
pub list_through: bool,
|
||||
pub respect_local_delete_marker: bool,
|
||||
pub preserve_etag: bool,
|
||||
pub copy_tags: bool,
|
||||
@@ -136,6 +137,7 @@ impl Default for OdmPolicy {
|
||||
head: "proxy".to_string(),
|
||||
range_get: "serve_and_backfill".to_string(),
|
||||
source_error: "propagate".to_string(),
|
||||
list_through: false,
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
@@ -576,6 +578,23 @@ impl OdmTestEnv {
|
||||
}
|
||||
|
||||
/// Raw signed `GET /{bucket}/{key}` against the RustFS under test.
|
||||
/// Raw signed `ListObjectsV2` (`?list-type=2&<query>`) so a scenario can
|
||||
/// assert on the response headers and the raw XML, which the SDK hides.
|
||||
pub async fn raw_list_objects_v2(&self, bucket: &str, query: &str) -> Result<RawResponse, BoxError> {
|
||||
let url = format!(
|
||||
"{}/{bucket}?list-type=2{}{query}",
|
||||
self.rustfs.url,
|
||||
if query.is_empty() { "" } else { "&" }
|
||||
);
|
||||
let response =
|
||||
signed_request(http::Method::GET, &url, &self.rustfs.access_key, &self.rustfs.secret_key, None, None).await?;
|
||||
Ok(RawResponse {
|
||||
status: response.status().as_u16(),
|
||||
headers: response.headers().clone(),
|
||||
body: response.bytes().await?,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn raw_get(&self, bucket: &str, key: &str) -> Result<RawResponse, BoxError> {
|
||||
let url = format!("{}/{bucket}/{key}", self.rustfs.url);
|
||||
let response =
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Optional merged `ListObjectsV2` (`policy.list_through`, ODM-17,
|
||||
//! rustfs/backlog#2164): full pagination over a source and a local namespace,
|
||||
//! common-prefix union under a delimiter, the continuation-token contract, and
|
||||
//! the two `source_error` behaviours when the source listing fails.
|
||||
|
||||
use super::common::{BoxError, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env};
|
||||
use crate::fake_s3_target::{FaultAction, Operation};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
|
||||
type TestResult = Result<(), BoxError>;
|
||||
|
||||
const SOURCE_BUCKET: &str = "odm-list-source";
|
||||
const LIST_HEADER: &str = "x-rustfs-on-demand-migration-list";
|
||||
|
||||
/// Byte lengths that tell a local object from a source one in a listing.
|
||||
const SOURCE_BODY_LEN: usize = 3;
|
||||
const LOCAL_BODY_LEN: usize = 11;
|
||||
|
||||
fn body(len: usize) -> Bytes {
|
||||
vec![b'x'; len].into()
|
||||
}
|
||||
|
||||
/// RustFS migrating `bucket` from `SOURCE_BUCKET` with `list_through` on.
|
||||
async fn list_through_env(bucket: &str, adjust: impl FnOnce(&mut OdmSourceSpec)) -> Result<OdmTestEnv, BoxError> {
|
||||
start_configured_env(bucket, SOURCE_BUCKET, |spec| {
|
||||
spec.policy.list_through = true;
|
||||
adjust(spec);
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Every key the bucket lists, walked through the merged continuation token.
|
||||
/// Also returns the size each page reported per key and the page sizes, so a
|
||||
/// caller can assert who won a shared key and that no page exceeded `max_keys`.
|
||||
async fn walk_listing(
|
||||
env: &OdmTestEnv,
|
||||
bucket: &str,
|
||||
delimiter: Option<&str>,
|
||||
max_keys: i32,
|
||||
) -> Result<(Vec<(String, i64)>, Vec<String>, Vec<usize>), BoxError> {
|
||||
let mut objects = Vec::new();
|
||||
let mut prefixes = Vec::new();
|
||||
let mut page_sizes = Vec::new();
|
||||
let mut token: Option<String> = None;
|
||||
for _ in 0..1000 {
|
||||
let page = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(max_keys)
|
||||
.set_delimiter(delimiter.map(str::to_string))
|
||||
.set_continuation_token(token.take())
|
||||
.send()
|
||||
.await?;
|
||||
let listed = page.contents().len() + page.common_prefixes().len();
|
||||
page_sizes.push(listed);
|
||||
for object in page.contents() {
|
||||
objects.push((object.key().unwrap_or_default().to_string(), object.size().unwrap_or_default()));
|
||||
}
|
||||
for prefix in page.common_prefixes() {
|
||||
prefixes.push(prefix.prefix().unwrap_or_default().to_string());
|
||||
}
|
||||
if !page.is_truncated().unwrap_or(false) {
|
||||
return Ok((objects, prefixes, page_sizes));
|
||||
}
|
||||
token = Some(
|
||||
page.next_continuation_token()
|
||||
.ok_or("truncated page without a continuation token")?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
Err("merged listing did not terminate".into())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_merges_the_whole_namespace_across_full_pagination() -> TestResult {
|
||||
let bucket = "odm-list-merge";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
|
||||
// 2000 source keys, 80 of them also local, plus 10 local-only keys that
|
||||
// interleave between source keys ("obj-00010x" sorts after "obj-00010").
|
||||
let source_keys: Vec<String> = (0..2000).map(|index| format!("obj-{index:05}")).collect();
|
||||
let seeds: Vec<SeedObject> = source_keys
|
||||
.iter()
|
||||
.map(|key| SeedObject::new(key.clone(), body(SOURCE_BODY_LEN)))
|
||||
.collect();
|
||||
env.seed_source(SOURCE_BUCKET, &seeds);
|
||||
|
||||
let shared: Vec<String> = source_keys.iter().step_by(25).cloned().collect();
|
||||
let local_only: Vec<String> = (0..10).map(|index| format!("obj-{:05}x", index * 7)).collect();
|
||||
for key in shared.iter().chain(local_only.iter()) {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
let max_keys = 97;
|
||||
let (objects, prefixes, page_sizes) = walk_listing(&env, bucket, None, max_keys).await?;
|
||||
assert!(prefixes.is_empty(), "no delimiter means no common prefixes");
|
||||
|
||||
let mut expected: Vec<String> = source_keys.iter().chain(local_only.iter()).cloned().collect();
|
||||
expected.sort();
|
||||
expected.dedup();
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(listed, expected, "the merged listing is the sorted, deduplicated union");
|
||||
assert!(
|
||||
page_sizes.iter().all(|size| *size <= max_keys as usize),
|
||||
"no page may exceed max_keys: {page_sizes:?}"
|
||||
);
|
||||
|
||||
let shared_sizes: Vec<i64> = objects
|
||||
.iter()
|
||||
.filter(|(key, _)| shared.contains(key))
|
||||
.map(|(_, size)| *size)
|
||||
.collect();
|
||||
assert_eq!(shared_sizes.len(), shared.len(), "every shared key is listed exactly once");
|
||||
assert!(
|
||||
shared_sizes.iter().all(|size| *size == LOCAL_BODY_LEN as i64),
|
||||
"the local object wins a key both sides hold"
|
||||
);
|
||||
let source_sizes: Vec<i64> = objects
|
||||
.iter()
|
||||
.filter(|(key, _)| !shared.contains(key) && !local_only.contains(key))
|
||||
.map(|(_, size)| *size)
|
||||
.collect();
|
||||
assert!(
|
||||
source_sizes.iter().all(|size| *size == SOURCE_BODY_LEN as i64),
|
||||
"source-only keys report the source's own size"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_unions_common_prefixes_under_a_delimiter() -> TestResult {
|
||||
let bucket = "odm-list-delimiter";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("p1/a", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("p1/b", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("p2/a", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("top-s", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
for key in ["p1/c", "p3/a", "top-l"] {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
// A page size of two forces the prefix union to survive page boundaries.
|
||||
let (objects, prefixes, page_sizes) = walk_listing(&env, bucket, Some("/"), 2).await?;
|
||||
assert_eq!(prefixes, vec!["p1/", "p2/", "p3/"], "prefixes are unioned and deduplicated");
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(listed, vec!["top-l", "top-s"]);
|
||||
assert!(page_sizes.iter().all(|size| *size <= 2), "{page_sizes:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_propagates_a_source_listing_failure() -> TestResult {
|
||||
let bucket = "odm-list-propagate";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new("remote", body(SOURCE_BODY_LEN))]);
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("local")
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.source
|
||||
.inject(Operation::ListObjectsV2, FaultAction::ResponseStatus(503), 1);
|
||||
let failure = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("propagate must surface the source failure");
|
||||
let failure = failure.into_service_error();
|
||||
assert_eq!(failure.meta().code(), Some("SourceUnavailable"), "{failure:?}");
|
||||
|
||||
// The next listing sees a healthy source again and merges both sides.
|
||||
let (objects, _, _) = walk_listing(&env, bucket, None, 100).await?;
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(listed, vec!["local", "remote"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_degrades_to_local_only_under_the_not_found_policy() -> TestResult {
|
||||
let bucket = "odm-list-degrade";
|
||||
let env = list_through_env(bucket, |spec| spec.policy.source_error = "not_found".to_string()).await?;
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new("remote", body(SOURCE_BODY_LEN))]);
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("local")
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.source
|
||||
.inject(Operation::ListObjectsV2, FaultAction::ResponseStatus(503), 1);
|
||||
let degraded = env.raw_list_objects_v2(bucket, "max-keys=100").await?;
|
||||
assert_eq!(degraded.status, 200, "{}", String::from_utf8_lossy(°raded.body));
|
||||
assert_eq!(
|
||||
degraded.header(LIST_HEADER),
|
||||
Some("local_only"),
|
||||
"a degraded listing must say so in the response header"
|
||||
);
|
||||
let xml = String::from_utf8_lossy(°raded.body).to_string();
|
||||
assert!(xml.contains("<Key>local</Key>"), "{xml}");
|
||||
assert!(!xml.contains("<Key>remote</Key>"), "a degraded listing shows local state only: {xml}");
|
||||
|
||||
let healthy = env.raw_list_objects_v2(bucket, "max-keys=100").await?;
|
||||
assert_eq!(healthy.status, 200);
|
||||
assert_eq!(healthy.header(LIST_HEADER), None, "a healthy merge carries no degradation marker");
|
||||
assert!(String::from_utf8_lossy(&healthy.body).contains("<Key>remote</Key>"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
|
||||
let bucket = "odm-list-token";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("a", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("b", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("c", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
|
||||
let page = env.client.list_objects_v2().bucket(bucket).max_keys(1).send().await?;
|
||||
let token = page.next_continuation_token().ok_or("first page must be truncated")?;
|
||||
let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
|
||||
assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
|
||||
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
|
||||
let rejected = env
|
||||
.raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rejected.status,
|
||||
400,
|
||||
"a bumped token version is a client error: {}",
|
||||
String::from_utf8_lossy(&rejected.body)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_merged_token_keeps_paginating_after_list_through_is_turned_off() -> TestResult {
|
||||
let bucket = "odm-list-token-off";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("s1", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("s2", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
for key in ["l1", "l2"] {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
let page = env.client.list_objects_v2().bucket(bucket).max_keys(1).send().await?;
|
||||
assert_eq!(page.contents()[0].key(), Some("l1"));
|
||||
let token = page
|
||||
.next_continuation_token()
|
||||
.ok_or("first page must be truncated")?
|
||||
.to_string();
|
||||
|
||||
let mut spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
spec.policy.list_through = false;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
let resumed = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(10)
|
||||
.continuation_token(token)
|
||||
.send()
|
||||
.await?;
|
||||
let listed: Vec<&str> = resumed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed, vec!["l2"], "a merged token falls back to its local cursor");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_local_delete_marker_hides_the_source_key_from_a_merged_listing() -> TestResult {
|
||||
let bucket = "odm-list-delete-marker";
|
||||
let env = list_through_env(bucket, |_| {}).await?;
|
||||
env.client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("kept", body(SOURCE_BODY_LEN)),
|
||||
SeedObject::new("shadowed", body(SOURCE_BODY_LEN)),
|
||||
],
|
||||
);
|
||||
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("shadowed")
|
||||
.body(body(LOCAL_BODY_LEN).into())
|
||||
.send()
|
||||
.await?;
|
||||
env.client.delete_object().bucket(bucket).key("shadowed").send().await?;
|
||||
|
||||
let (objects, _, _) = walk_listing(&env, bucket, None, 100).await?;
|
||||
let listed: Vec<String> = objects.iter().map(|(key, _)| key.clone()).collect();
|
||||
assert_eq!(
|
||||
listed,
|
||||
vec!["kept"],
|
||||
"a local delete marker shadows the source key the same way it does on GET"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -18,7 +18,8 @@
|
||||
//! fake S3 source, admin-API wrappers, seeding and local-state assertions.
|
||||
//! `harness_self_test` proves the harness itself; `get_basic_test` covers the
|
||||
//! GET read-through (rustfs/backlog#2156) and `backfill_test` the background
|
||||
//! backfill job (ODM-12, rustfs/backlog#2159). The fault, concurrency,
|
||||
//! backfill job (ODM-12, rustfs/backlog#2159); `list_through_test` covers the
|
||||
//! optional merged `ListObjectsV2` (ODM-17, rustfs/backlog#2164). The fault, concurrency,
|
||||
//! interaction and real-source matrix is rustfs/backlog#2158; its lane split
|
||||
//! lives in `.config/nextest.toml` (fault / concurrency / real source run
|
||||
//! nightly, the rest in the merge lane).
|
||||
@@ -31,4 +32,5 @@ mod fault_test;
|
||||
mod get_basic_test;
|
||||
mod harness_self_test;
|
||||
mod interaction_test;
|
||||
mod list_through_test;
|
||||
mod real_source_test;
|
||||
|
||||
@@ -166,6 +166,11 @@ pub mod bucket {
|
||||
WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
|
||||
idle_guarded_body,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
||||
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
||||
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub mod backfill {
|
||||
pub use crate::bucket::on_demand_migration::backfill::{
|
||||
BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
|
||||
@@ -179,8 +184,8 @@ pub mod bucket {
|
||||
}
|
||||
pub mod source_client {
|
||||
pub use crate::bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe,
|
||||
SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
|
||||
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
|
||||
resolve_path_style,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1717,6 +1717,7 @@ mod tests {
|
||||
objects,
|
||||
is_truncated,
|
||||
next_continuation_token: is_truncated.then(|| end.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -238,6 +238,11 @@ pub struct PolicyConfig {
|
||||
pub range_get: RangeGetPolicy,
|
||||
#[serde(default)]
|
||||
pub source_error: SourceErrorPolicy,
|
||||
/// Merge the source listing into `ListObjectsV2` so clients see the whole
|
||||
/// namespace during the migration (rustfs/backlog#2164). Off by default:
|
||||
/// it puts the source in the path of every listing.
|
||||
#[serde(default)]
|
||||
pub list_through: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub respect_local_delete_marker: bool,
|
||||
#[serde(default = "default_true")]
|
||||
@@ -268,6 +273,7 @@ impl Default for PolicyConfig {
|
||||
head: HeadPolicy::default(),
|
||||
range_get: RangeGetPolicy::default(),
|
||||
source_error: SourceErrorPolicy::default(),
|
||||
list_through: false,
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
@@ -615,6 +621,7 @@ mod tests {
|
||||
"head": "proxy",
|
||||
"range_get": "serve_and_backfill",
|
||||
"source_error": "propagate",
|
||||
"list_through": false,
|
||||
"respect_local_delete_marker": true,
|
||||
"preserve_etag": true,
|
||||
"copy_tags": false,
|
||||
|
||||
@@ -0,0 +1,824 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Optional `ListObjectsV2` list-through (`policy.list_through`,
|
||||
//! rustfs/backlog#2164): the local listing and the source listing are merged
|
||||
//! into one ordered page so clients see the whole namespace while a bucket is
|
||||
//! migrating.
|
||||
//!
|
||||
//! Everything here is pure. The handler owns the I/O and the payloads; this
|
||||
//! module owns the ordering, the page boundary, and the opaque continuation
|
||||
//! token that carries both cursors.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// The only continuation-token envelope version this build reads and writes.
|
||||
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
|
||||
|
||||
/// Envelope marker. A bucket that is *not* merging hands out the local
|
||||
/// listing's own marker, so the decoder needs a positive signal before it
|
||||
/// treats an opaque token as a merged one.
|
||||
const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
|
||||
|
||||
/// Pages fetched per side per request: the first page, plus at most one refill
|
||||
/// when the first one was mostly consumed by the previous page. Two pages of
|
||||
/// `max_keys` always cover a full merged page, so this is a bound, not a
|
||||
/// heuristic.
|
||||
pub const MAX_LIST_FETCHES_PER_SIDE: usize = 2;
|
||||
|
||||
/// Per-bucket ceiling on source `ListObjectsV2` calls, in calls per second.
|
||||
pub const SOURCE_LIST_RATE_PER_SEC: u32 = 10;
|
||||
|
||||
/// How long a listing may wait for a source rate-limit slot before it gives up
|
||||
/// and answers from local state alone.
|
||||
pub const SOURCE_LIST_MAX_RATE_WAIT: Duration = Duration::from_secs(1);
|
||||
|
||||
/// One listing entry as the merge orders it: an object key, or — under a
|
||||
/// delimiter — a rolled-up common prefix. Both sort by `name` alone, which is
|
||||
/// how S3 interleaves `Contents` and `CommonPrefixes` on the wire.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ListEntryKey {
|
||||
pub name: String,
|
||||
pub is_prefix: bool,
|
||||
}
|
||||
|
||||
impl ListEntryKey {
|
||||
pub fn object(name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
is_prefix: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn prefix(name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
is_prefix: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MergeSide {
|
||||
Local,
|
||||
Source,
|
||||
}
|
||||
|
||||
/// One entry of the merged page: the side it came from and its index in that
|
||||
/// side's buffer, in push order. The caller keeps the payloads.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct MergePick {
|
||||
pub side: MergeSide,
|
||||
pub index: usize,
|
||||
}
|
||||
|
||||
/// The continuation-token envelope. Opaque to clients: it is serialized as
|
||||
/// JSON and then base64-encoded by the same helper that encodes a plain local
|
||||
/// marker, so the wire shape is `base64(json)`.
|
||||
///
|
||||
/// A `null` cursor with `done = false` means "list that side from the start";
|
||||
/// `done = true` means the side is finished and must not be listed again.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListThroughToken {
|
||||
/// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
|
||||
pub t: String,
|
||||
pub v: u32,
|
||||
#[serde(default)]
|
||||
pub local: Option<String>,
|
||||
#[serde(default)]
|
||||
pub local_done: bool,
|
||||
#[serde(default)]
|
||||
pub source: Option<String>,
|
||||
#[serde(default)]
|
||||
pub source_done: bool,
|
||||
/// Last entry the previous page consumed. A side whose page was only
|
||||
/// partially consumed is re-listed from the same cursor and everything at
|
||||
/// or below this key is dropped, which is delimiter-safe: a rolled-up
|
||||
/// common prefix compares as itself, never as its members.
|
||||
#[serde(default)]
|
||||
pub last_key: Option<String>,
|
||||
}
|
||||
|
||||
impl ListThroughToken {
|
||||
fn new(local: SideCursor, source: SideCursor, last_key: Option<String>) -> Self {
|
||||
Self {
|
||||
t: LIST_THROUGH_TOKEN_TAG.to_string(),
|
||||
v: LIST_THROUGH_TOKEN_VERSION,
|
||||
local: local.token,
|
||||
local_done: local.done,
|
||||
source: source.token,
|
||||
source_done: source.done,
|
||||
last_key,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> String {
|
||||
// The envelope is built here from owned strings, so serialization
|
||||
// cannot fail; the fallback keeps the signature infallible.
|
||||
serde_json::to_string(self).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// What a decoded (base64-stripped) continuation token turned out to be.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ListThroughCursor {
|
||||
/// A plain local listing marker: the bucket was not merging when the token
|
||||
/// was issued, or the client is paginating a non-merged listing.
|
||||
Local(String),
|
||||
Merged(Box<ListThroughToken>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ListThroughTokenError {
|
||||
#[error("continuation token version {0} is not supported")]
|
||||
UnsupportedVersion(u32),
|
||||
/// The message never echoes the token: it is client-controlled input.
|
||||
#[error("continuation token is malformed")]
|
||||
Malformed,
|
||||
}
|
||||
|
||||
/// Classifies an already base64-decoded continuation token.
|
||||
///
|
||||
/// Only a JSON object carrying the envelope marker is read as a merged token;
|
||||
/// anything else is a local marker, so a bucket that turns `list_through` off
|
||||
/// keeps paginating with the tokens it handed out. A token that *is* an
|
||||
/// envelope but was tampered with (unknown version, unknown field, truncated
|
||||
/// JSON) is an error, never a silent fallback.
|
||||
pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
|
||||
if !decoded.starts_with('{') {
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
}
|
||||
let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
|
||||
// Not JSON at all: an object key may legitimately start with '{'.
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
};
|
||||
if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
}
|
||||
match value.get("v").and_then(serde_json::Value::as_u64) {
|
||||
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {}
|
||||
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
|
||||
None => return Err(ListThroughTokenError::Malformed),
|
||||
}
|
||||
serde_json::from_value::<ListThroughToken>(value)
|
||||
.map(|token| ListThroughCursor::Merged(Box::new(token)))
|
||||
.map_err(|_| ListThroughTokenError::Malformed)
|
||||
}
|
||||
|
||||
/// How the source must be listed for a request, given `filter.prefix`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum SourceListPlan {
|
||||
/// The request prefix and `filter.prefix` are disjoint: the source holds
|
||||
/// nothing this listing could show.
|
||||
Skip,
|
||||
/// Ordinary paged listing under `prefix`, rolled up with the request's
|
||||
/// delimiter — the source's own roll-up boundary matches the request's.
|
||||
Page { prefix: String },
|
||||
/// `filter.prefix` reaches past a delimiter, so every key the source could
|
||||
/// contribute rolls into this one common prefix. One bounded probe listing
|
||||
/// decides whether it exists; there is nothing to paginate.
|
||||
Folded { probe_prefix: String, common_prefix: String },
|
||||
}
|
||||
|
||||
/// Intersects the request prefix with `filter.prefix` and decides how (or
|
||||
/// whether) the source is listed.
|
||||
pub fn source_list_plan(request_prefix: &str, filter_prefix: Option<&str>, delimiter: Option<&str>) -> SourceListPlan {
|
||||
let filter = filter_prefix.unwrap_or_default();
|
||||
let source_prefix = if filter.starts_with(request_prefix) {
|
||||
filter
|
||||
} else if request_prefix.starts_with(filter) {
|
||||
request_prefix
|
||||
} else {
|
||||
return SourceListPlan::Skip;
|
||||
};
|
||||
|
||||
let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty()) else {
|
||||
return SourceListPlan::Page {
|
||||
prefix: source_prefix.to_string(),
|
||||
};
|
||||
};
|
||||
|
||||
// `source_prefix` always starts with `request_prefix`, so this slice is on
|
||||
// a character boundary.
|
||||
let extra = &source_prefix[request_prefix.len()..];
|
||||
match extra.find(delimiter) {
|
||||
Some(at) => SourceListPlan::Folded {
|
||||
probe_prefix: source_prefix.to_string(),
|
||||
common_prefix: format!("{request_prefix}{}", &extra[..at + delimiter.len()]),
|
||||
},
|
||||
None => SourceListPlan::Page {
|
||||
prefix: source_prefix.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Where one side resumes.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SideCursor {
|
||||
pub token: Option<String>,
|
||||
pub done: bool,
|
||||
}
|
||||
|
||||
/// One page a side actually fetched this round.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct FetchedPage {
|
||||
/// Token it was fetched with; `None` means from the start of the listing.
|
||||
token: Option<String>,
|
||||
/// Entries it contributed to the buffer, after the `last_key` filter.
|
||||
count: usize,
|
||||
/// Cursor for the page after it, `None` when it was the last one.
|
||||
next_token: Option<String>,
|
||||
}
|
||||
|
||||
/// Where a side resumes after `consumed` of its buffered entries were taken.
|
||||
///
|
||||
/// A fully consumed page advances to its successor; a partially consumed one
|
||||
/// is re-listed from the same cursor next time and re-filtered by `last_key`.
|
||||
fn advance_cursor(pages: &[FetchedPage], consumed: usize) -> SideCursor {
|
||||
let mut remaining = consumed;
|
||||
let mut cursor = SideCursor { token: None, done: true };
|
||||
for page in pages {
|
||||
if remaining >= page.count {
|
||||
remaining -= page.count;
|
||||
cursor = match &page.next_token {
|
||||
Some(next) => SideCursor {
|
||||
token: Some(next.clone()),
|
||||
done: false,
|
||||
},
|
||||
None => SideCursor { token: None, done: true },
|
||||
};
|
||||
} else {
|
||||
cursor = SideCursor {
|
||||
token: page.token.clone(),
|
||||
done: false,
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
cursor
|
||||
}
|
||||
|
||||
/// A page the merge driver still needs.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct FetchRequest {
|
||||
pub side: MergeSide,
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SideState {
|
||||
start: SideCursor,
|
||||
pages: Vec<FetchedPage>,
|
||||
entries: Vec<ListEntryKey>,
|
||||
more: bool,
|
||||
disabled: bool,
|
||||
}
|
||||
|
||||
impl SideState {
|
||||
fn from_cursor(token: Option<String>, done: bool) -> Self {
|
||||
Self {
|
||||
start: SideCursor { token, done },
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn needs_page(&self, max_keys: usize) -> Option<Option<String>> {
|
||||
if self.disabled || self.start.done {
|
||||
return None;
|
||||
}
|
||||
match self.pages.last() {
|
||||
None => Some(self.start.token.clone()),
|
||||
Some(last) => {
|
||||
let room = self.entries.len() < max_keys;
|
||||
let capped = self.pages.len() >= MAX_LIST_FETCHES_PER_SIDE;
|
||||
(self.more && room && !capped).then(|| last.next_token.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The merged page, once both sides have handed over everything they will.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct MergeOutcome {
|
||||
/// Entries of the merged page, in wire order; indices point into each
|
||||
/// side's buffer in push order.
|
||||
pub picks: Vec<MergePick>,
|
||||
pub is_truncated: bool,
|
||||
/// `Some` exactly when `is_truncated`.
|
||||
pub next_token: Option<ListThroughToken>,
|
||||
}
|
||||
|
||||
/// Drives one merged page: the caller asks [`Self::next_fetch`] what to list,
|
||||
/// hands the page back with [`Self::push_page`], and finishes with
|
||||
/// [`Self::finish`]. Nothing here does I/O, so the same driver is exercised by
|
||||
/// the property test and by the handler.
|
||||
#[derive(Debug)]
|
||||
pub struct ListThroughMerger {
|
||||
max_keys: usize,
|
||||
last_key: Option<String>,
|
||||
local: SideState,
|
||||
source: SideState,
|
||||
}
|
||||
|
||||
impl ListThroughMerger {
|
||||
/// `token` is the envelope from the client's continuation token, absent on
|
||||
/// the first page of a listing.
|
||||
pub fn new(max_keys: usize, token: Option<&ListThroughToken>) -> Self {
|
||||
let (local, source, last_key) = match token {
|
||||
Some(token) => (
|
||||
SideState::from_cursor(token.local.clone(), token.local_done),
|
||||
SideState::from_cursor(token.source.clone(), token.source_done),
|
||||
token.last_key.clone(),
|
||||
),
|
||||
None => (SideState::default(), SideState::default(), None),
|
||||
};
|
||||
Self {
|
||||
max_keys,
|
||||
last_key,
|
||||
local,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an entry the listing returned still belongs to this page: a
|
||||
/// re-listed page repeats what the previous page already consumed.
|
||||
pub fn accepts(&self, name: &str) -> bool {
|
||||
self.last_key.as_deref().is_none_or(|bound| name > bound)
|
||||
}
|
||||
|
||||
/// The source contributes nothing to this page: it failed, is rate-limited,
|
||||
/// or `filter.prefix` excludes it.
|
||||
pub fn disable_source(&mut self) {
|
||||
self.source.disabled = true;
|
||||
}
|
||||
|
||||
pub fn next_fetch(&self) -> Option<FetchRequest> {
|
||||
for (side, state) in [(MergeSide::Local, &self.local), (MergeSide::Source, &self.source)] {
|
||||
if let Some(token) = state.needs_page(self.max_keys) {
|
||||
return Some(FetchRequest { side, token });
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Records one fetched page. `entries` must be sorted by `name` and already
|
||||
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads
|
||||
/// in the same order.
|
||||
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
|
||||
let state = match side {
|
||||
MergeSide::Local => &mut self.local,
|
||||
MergeSide::Source => &mut self.source,
|
||||
};
|
||||
let token = match state.pages.last() {
|
||||
Some(last) => last.next_token.clone(),
|
||||
None => state.start.token.clone(),
|
||||
};
|
||||
// A truncated page without a cursor cannot be continued; treating the
|
||||
// side as finished is the only alternative to looping on it forever.
|
||||
state.more = is_truncated && next_token.is_some();
|
||||
state.pages.push(FetchedPage {
|
||||
token,
|
||||
count: entries.len(),
|
||||
next_token: is_truncated.then_some(next_token).flatten(),
|
||||
});
|
||||
state.entries.extend(entries);
|
||||
}
|
||||
|
||||
pub fn finish(self) -> MergeOutcome {
|
||||
let Self {
|
||||
max_keys,
|
||||
last_key,
|
||||
local,
|
||||
source,
|
||||
} = self;
|
||||
|
||||
// A side with more pages behind it can only be trusted up to the last
|
||||
// key it handed over: past that horizon the other side's entries could
|
||||
// still be deduplicated by one we have not seen, which is what keeps
|
||||
// "local wins on equal keys" true across page boundaries.
|
||||
let horizon = [
|
||||
local
|
||||
.more
|
||||
.then(|| local.entries.last().map_or("", |entry| entry.name.as_str())),
|
||||
source
|
||||
.more
|
||||
.then(|| source.entries.last().map_or("", |entry| entry.name.as_str())),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min();
|
||||
|
||||
let mut picks = Vec::with_capacity(max_keys.min(local.entries.len() + source.entries.len()));
|
||||
let mut consumed_local = 0usize;
|
||||
let mut consumed_source = 0usize;
|
||||
let mut consumed_key: Option<String> = None;
|
||||
|
||||
while picks.len() < max_keys {
|
||||
let next_local = local.entries.get(consumed_local).map(|entry| entry.name.as_str());
|
||||
let next_source = source.entries.get(consumed_source).map(|entry| entry.name.as_str());
|
||||
let name = match (next_local, next_source) {
|
||||
(None, None) => break,
|
||||
(Some(name), None) | (None, Some(name)) => name,
|
||||
(Some(left), Some(right)) => left.min(right),
|
||||
};
|
||||
if horizon.is_some_and(|horizon| name > horizon) {
|
||||
break;
|
||||
}
|
||||
let take_local = next_local == Some(name);
|
||||
let take_source = next_source == Some(name);
|
||||
consumed_key = Some(name.to_string());
|
||||
if take_local {
|
||||
picks.push(MergePick {
|
||||
side: MergeSide::Local,
|
||||
index: consumed_local,
|
||||
});
|
||||
consumed_local += 1;
|
||||
} else {
|
||||
picks.push(MergePick {
|
||||
side: MergeSide::Source,
|
||||
index: consumed_source,
|
||||
});
|
||||
}
|
||||
if take_source {
|
||||
consumed_source += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let local_cursor = advance_cursor(&local.pages, consumed_local);
|
||||
let source_cursor = if source.disabled {
|
||||
// Keep the source where it was so a recovered source resumes there;
|
||||
// this page is answered from local state alone.
|
||||
source.start.clone()
|
||||
} else {
|
||||
advance_cursor(&source.pages, consumed_source)
|
||||
};
|
||||
let local_left = !local_cursor.done || consumed_local < local.entries.len();
|
||||
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
|
||||
let is_truncated = local_left || source_left;
|
||||
|
||||
let last_key = consumed_key.or(last_key);
|
||||
MergeOutcome {
|
||||
picks,
|
||||
is_truncated,
|
||||
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Token bucket capping source `ListObjectsV2` calls for one bucket.
|
||||
///
|
||||
/// A caller that cannot be served inside its budget is refused rather than
|
||||
/// queued: a listing degrades to local state instead of holding the request
|
||||
/// open behind other tenants' listings.
|
||||
#[derive(Debug)]
|
||||
pub struct SourceListRateLimiter {
|
||||
rate_per_sec: f64,
|
||||
burst: f64,
|
||||
state: Mutex<RateLimiterState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RateLimiterState {
|
||||
tokens: f64,
|
||||
updated_at: Instant,
|
||||
}
|
||||
|
||||
impl SourceListRateLimiter {
|
||||
pub fn new(rate_per_sec: u32) -> Self {
|
||||
let rate_per_sec = f64::from(rate_per_sec.max(1));
|
||||
Self {
|
||||
rate_per_sec,
|
||||
burst: rate_per_sec,
|
||||
state: Mutex::new(RateLimiterState {
|
||||
tokens: rate_per_sec,
|
||||
updated_at: Instant::now(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserves one call, returning how long the caller must wait before making
|
||||
/// it, or `None` when that wait would exceed `max_wait` (nothing is
|
||||
/// reserved then).
|
||||
pub fn reserve(&self, max_wait: Duration) -> Option<Duration> {
|
||||
self.reserve_at(Instant::now(), max_wait)
|
||||
}
|
||||
|
||||
pub fn reserve_at(&self, now: Instant, max_wait: Duration) -> Option<Duration> {
|
||||
let mut state = self.state.lock();
|
||||
let elapsed = now.saturating_duration_since(state.updated_at).as_secs_f64();
|
||||
state.tokens = (state.tokens + elapsed * self.rate_per_sec).min(self.burst);
|
||||
state.updated_at = now;
|
||||
if state.tokens >= 1.0 {
|
||||
state.tokens -= 1.0;
|
||||
return Some(Duration::ZERO);
|
||||
}
|
||||
let wait = Duration::from_secs_f64((1.0 - state.tokens) / self.rate_per_sec);
|
||||
if wait > max_wait {
|
||||
return None;
|
||||
}
|
||||
state.tokens -= 1.0;
|
||||
Some(wait)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SourceListRateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new(SOURCE_LIST_RATE_PER_SEC)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
/// One `ListObjectsV2` page over a sorted key set, with the S3 rules the
|
||||
/// merge relies on: delimiter roll-up, `max_keys`, and a continuation
|
||||
/// token that resumes after the last entry the page returned.
|
||||
fn reference_page(
|
||||
keys: &[String],
|
||||
prefix: &str,
|
||||
delimiter: Option<&str>,
|
||||
after: Option<&str>,
|
||||
max_keys: usize,
|
||||
) -> (Vec<ListEntryKey>, bool, Option<String>) {
|
||||
let mut entries: Vec<ListEntryKey> = Vec::new();
|
||||
for key in keys.iter().filter(|key| key.starts_with(prefix)) {
|
||||
let entry = match delimiter.and_then(|delimiter| key[prefix.len()..].find(delimiter).map(|at| (delimiter, at))) {
|
||||
Some((delimiter, at)) => ListEntryKey::prefix(&key[..prefix.len() + at + delimiter.len()]),
|
||||
None => ListEntryKey::object(key.clone()),
|
||||
};
|
||||
if entries.last().is_none_or(|last| last.name != entry.name) {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
if let Some(after) = after {
|
||||
entries.retain(|entry| entry.name.as_str() > after);
|
||||
}
|
||||
let truncated = entries.len() > max_keys;
|
||||
entries.truncate(max_keys);
|
||||
let next = truncated.then(|| entries.last().map(|entry| entry.name.clone())).flatten();
|
||||
(entries, truncated && next.is_some(), next)
|
||||
}
|
||||
|
||||
/// Full pagination through the merger, returning every entry it emitted and
|
||||
/// the page sizes it produced.
|
||||
fn walk(
|
||||
local: &[String],
|
||||
source: &[String],
|
||||
prefix: &str,
|
||||
delimiter: Option<&str>,
|
||||
max_keys: usize,
|
||||
) -> (Vec<(ListEntryKey, MergeSide)>, Vec<usize>) {
|
||||
let mut emitted = Vec::new();
|
||||
let mut page_sizes = Vec::new();
|
||||
let mut token: Option<ListThroughToken> = None;
|
||||
for _ in 0..10_000 {
|
||||
let mut merger = ListThroughMerger::new(max_keys, token.as_ref());
|
||||
let mut buffers = [Vec::<ListEntryKey>::new(), Vec::<ListEntryKey>::new()];
|
||||
while let Some(fetch) = merger.next_fetch() {
|
||||
let keys = match fetch.side {
|
||||
MergeSide::Local => local,
|
||||
MergeSide::Source => source,
|
||||
};
|
||||
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
|
||||
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
|
||||
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
|
||||
merger.push_page(fetch.side, kept, truncated, next);
|
||||
}
|
||||
let outcome = merger.finish();
|
||||
page_sizes.push(outcome.picks.len());
|
||||
for pick in &outcome.picks {
|
||||
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
|
||||
emitted.push((entry, pick.side));
|
||||
}
|
||||
if !outcome.is_truncated {
|
||||
return (emitted, page_sizes);
|
||||
}
|
||||
token = outcome.next_token;
|
||||
}
|
||||
panic!("merged pagination did not terminate");
|
||||
}
|
||||
|
||||
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
|
||||
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
|
||||
all.sort();
|
||||
all.dedup();
|
||||
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
|
||||
entries
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_page_rolls_up_and_paginates() {
|
||||
let keys = vec!["a/1".to_string(), "a/2".to_string(), "b".to_string(), "c/1".to_string()];
|
||||
let (entries, truncated, next) = reference_page(&keys, "", Some("/"), None, 2);
|
||||
assert_eq!(entries, vec![ListEntryKey::prefix("a/"), ListEntryKey::object("b")]);
|
||||
assert!(truncated);
|
||||
assert_eq!(next.as_deref(), Some("b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_pages_are_ordered_and_local_wins_on_equal_keys() {
|
||||
let local = vec!["a".to_string(), "c".to_string()];
|
||||
let source = vec!["b".to_string(), "c".to_string(), "d".to_string()];
|
||||
let (emitted, sizes) = walk(&local, &source, "", None, 2);
|
||||
let names: Vec<&str> = emitted.iter().map(|(entry, _)| entry.name.as_str()).collect();
|
||||
assert_eq!(names, vec!["a", "b", "c", "d"]);
|
||||
assert_eq!(emitted[2].1, MergeSide::Local, "the shared key must come from local");
|
||||
assert!(sizes.iter().all(|size| *size <= 2), "{sizes:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_only_listing_paginates_without_a_local_side() {
|
||||
let source: Vec<String> = (0..7).map(|index| format!("k{index}")).collect();
|
||||
let (emitted, _) = walk(&[], &source, "", None, 3);
|
||||
assert_eq!(emitted.len(), 7);
|
||||
assert!(emitted.iter().all(|(_, side)| *side == MergeSide::Source));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_disabled_source_answers_from_local_alone() {
|
||||
let mut merger = ListThroughMerger::new(10, None);
|
||||
merger.disable_source();
|
||||
assert_eq!(
|
||||
merger.next_fetch(),
|
||||
Some(FetchRequest {
|
||||
side: MergeSide::Local,
|
||||
token: None
|
||||
})
|
||||
);
|
||||
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
|
||||
assert_eq!(merger.next_fetch(), None);
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.picks.len(), 1);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_degraded_page_keeps_the_source_cursor_for_the_next_one() {
|
||||
let resume = ListThroughToken {
|
||||
t: LIST_THROUGH_TOKEN_TAG.to_string(),
|
||||
v: LIST_THROUGH_TOKEN_VERSION,
|
||||
local: Some("local-1".to_string()),
|
||||
local_done: false,
|
||||
source: Some("source-1".to_string()),
|
||||
source_done: false,
|
||||
last_key: Some("a".to_string()),
|
||||
};
|
||||
let mut merger = ListThroughMerger::new(1, Some(&resume));
|
||||
merger.disable_source();
|
||||
merger.push_page(
|
||||
MergeSide::Local,
|
||||
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
|
||||
true,
|
||||
Some("local-2".to_string()),
|
||||
);
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("truncated page carries a token");
|
||||
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
|
||||
assert!(!token.source_done);
|
||||
assert_eq!(token.last_key.as_deref(), Some("b"));
|
||||
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_round_trips_and_rejects_tampering() {
|
||||
let token = ListThroughToken::new(
|
||||
SideCursor {
|
||||
token: Some("l".to_string()),
|
||||
done: false,
|
||||
},
|
||||
SideCursor { token: None, done: true },
|
||||
Some("k".to_string()),
|
||||
);
|
||||
let encoded = token.encode();
|
||||
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
|
||||
|
||||
let bumped = encoded.replace("\"v\":1", "\"v\":2");
|
||||
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2)));
|
||||
|
||||
let extra = encoded.replace("{", "{\"x\":1,");
|
||||
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
|
||||
|
||||
let truncated = &encoded[..encoded.len() - 3];
|
||||
assert_eq!(decode_continuation_token(truncated), Ok(ListThroughCursor::Local(truncated.to_string())));
|
||||
|
||||
let no_version = "{\"t\":\"odm-list\"}";
|
||||
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_local_marker_stays_local() {
|
||||
assert_eq!(
|
||||
decode_continuation_token("photos/2024/01.jpg"),
|
||||
Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decode_continuation_token("{not json"),
|
||||
Ok(ListThroughCursor::Local("{not json".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decode_continuation_token("{\"t\":\"other\"}"),
|
||||
Ok(ListThroughCursor::Local("{\"t\":\"other\"}".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_list_plan_intersects_the_filter_prefix() {
|
||||
assert_eq!(source_list_plan("", None, None), SourceListPlan::Page { prefix: String::new() });
|
||||
assert_eq!(
|
||||
source_list_plan("photos/2024/", Some("photos/"), None),
|
||||
SourceListPlan::Page {
|
||||
prefix: "photos/2024/".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
source_list_plan("photos/", Some("photos/2024/"), None),
|
||||
SourceListPlan::Page {
|
||||
prefix: "photos/2024/".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(source_list_plan("videos/", Some("photos/"), None), SourceListPlan::Skip);
|
||||
assert_eq!(
|
||||
source_list_plan("", Some("photos/2024/"), Some("/")),
|
||||
SourceListPlan::Folded {
|
||||
probe_prefix: "photos/2024/".to_string(),
|
||||
common_prefix: "photos/".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
source_list_plan("pho", Some("photos"), Some("/")),
|
||||
SourceListPlan::Page {
|
||||
prefix: "photos".to_string()
|
||||
},
|
||||
"a filter prefix that adds no delimiter keeps the source's own roll-up"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limiter_spends_its_burst_then_paces_and_refuses() {
|
||||
let limiter = SourceListRateLimiter::new(10);
|
||||
let start = Instant::now();
|
||||
for _ in 0..10 {
|
||||
assert_eq!(limiter.reserve_at(start, Duration::from_secs(1)), Some(Duration::ZERO));
|
||||
}
|
||||
let paced = limiter.reserve_at(start, Duration::from_secs(1)).expect("within the budget");
|
||||
assert!(paced > Duration::ZERO && paced <= Duration::from_millis(101), "{paced:?}");
|
||||
assert_eq!(limiter.reserve_at(start, Duration::ZERO), None, "a zero budget refuses");
|
||||
// A full second of refill restores the whole burst.
|
||||
assert_eq!(limiter.reserve_at(start + Duration::from_secs(5), Duration::ZERO), Some(Duration::ZERO));
|
||||
}
|
||||
|
||||
fn key_set() -> impl Strategy<Value = Vec<String>> {
|
||||
proptest::collection::btree_set(
|
||||
proptest::sample::select(vec!["a", "a/", "a/1", "a/2", "a/b/1", "b", "b/1", "c", "c/1", "c/2", "d", "d/e/f"])
|
||||
.prop_map(str::to_string),
|
||||
0..=12,
|
||||
)
|
||||
.prop_map(|set: BTreeSet<String>| set.into_iter().collect())
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(256))]
|
||||
|
||||
/// Full pagination of a merged listing equals the sorted, deduplicated
|
||||
/// union of both sides, with every shared key served by local, and no
|
||||
/// page longer than `max_keys`.
|
||||
#[test]
|
||||
fn merged_pagination_equals_the_deduplicated_union(
|
||||
local in key_set(),
|
||||
source in key_set(),
|
||||
max_keys in 1usize..=5,
|
||||
with_delimiter in any::<bool>(),
|
||||
prefix in proptest::sample::select(vec!["", "a", "a/", "c/"]),
|
||||
) {
|
||||
let delimiter = with_delimiter.then_some("/");
|
||||
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
|
||||
let got: Vec<ListEntryKey> = emitted.iter().map(|(entry, _)| entry.clone()).collect();
|
||||
prop_assert_eq!(got, expected(&local, &source, prefix, delimiter));
|
||||
prop_assert!(sizes.iter().all(|size| *size <= max_keys), "{:?}", sizes);
|
||||
for (entry, side) in &emitted {
|
||||
if !entry.is_prefix && local.iter().any(|key| key == &entry.name) {
|
||||
prop_assert_eq!(*side, MergeSide::Local, "local must win for {}", entry.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@
|
||||
pub mod backfill;
|
||||
pub mod breaker;
|
||||
pub mod config;
|
||||
pub mod list_through;
|
||||
pub mod negative_cache;
|
||||
pub mod pull;
|
||||
pub mod source_client;
|
||||
@@ -38,6 +39,11 @@ pub use config::{
|
||||
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
|
||||
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub use list_through::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
||||
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
||||
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
|
||||
pub use pull::{
|
||||
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion,
|
||||
|
||||
@@ -511,10 +511,28 @@ pub struct SourceObject {
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SourcePage {
|
||||
pub objects: Vec<SourceObject>,
|
||||
/// Rolled-up prefixes, in the local namespace; always empty when the
|
||||
/// request carried no delimiter.
|
||||
pub common_prefixes: Vec<String>,
|
||||
pub is_truncated: bool,
|
||||
pub next_continuation_token: Option<String>,
|
||||
}
|
||||
|
||||
/// One `ListObjectsV2` page request against the source. Keys are given in the
|
||||
/// local namespace; `SourceClient` maps them through `source_prefix`.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SourceListRequest<'a> {
|
||||
pub prefix: Option<&'a str>,
|
||||
/// Rolls the source's own listing up the same way the local one is rolled
|
||||
/// up, so a page under a delimiter stays bounded.
|
||||
pub delimiter: Option<&'a str>,
|
||||
/// Ignored by S3 when `continuation_token` is set, so the caller must pass
|
||||
/// at most one of the two.
|
||||
pub start_after: Option<&'a str>,
|
||||
pub continuation_token: Option<&'a str>,
|
||||
pub max_keys: i32,
|
||||
}
|
||||
|
||||
/// Result of [`SourceClient::probe`]: the bucket answered HEAD and a
|
||||
/// one-key listing succeeded.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -669,13 +687,35 @@ impl SourceClient {
|
||||
continuation_token: Option<&str>,
|
||||
max_keys: i32,
|
||||
) -> Result<SourcePage, SourceError> {
|
||||
self.list_page(&SourceListRequest {
|
||||
prefix,
|
||||
continuation_token,
|
||||
max_keys,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::list_objects_v2`] with the delimiter and start-after the
|
||||
/// list-through merge needs (rustfs/backlog#2164).
|
||||
pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
|
||||
// `start_after` is silently ignored by S3 once a continuation token is
|
||||
// present; refuse the ambiguous pair rather than list from the wrong
|
||||
// position.
|
||||
if request.continuation_token.is_some() && request.start_after.is_some() {
|
||||
return Err(SourceError::Other(
|
||||
"source listing takes either a continuation token or start-after, not both".to_string(),
|
||||
));
|
||||
}
|
||||
let output = self
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket)
|
||||
.prefix(self.source_key(prefix.unwrap_or_default()))
|
||||
.set_continuation_token(continuation_token.map(str::to_string))
|
||||
.max_keys(max_keys)
|
||||
.prefix(self.source_key(request.prefix.unwrap_or_default()))
|
||||
.set_delimiter(request.delimiter.map(str::to_string))
|
||||
.set_start_after(request.start_after.map(|after| self.source_key(after)))
|
||||
.set_continuation_token(request.continuation_token.map(str::to_string))
|
||||
.max_keys(request.max_keys)
|
||||
.send()
|
||||
.await
|
||||
.map_err(classify_sdk_error)?;
|
||||
@@ -693,9 +733,16 @@ impl SourceClient {
|
||||
.into_iter()
|
||||
.filter_map(|object| self.source_object(object))
|
||||
.collect();
|
||||
let common_prefixes = output
|
||||
.common_prefixes
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
|
||||
.collect();
|
||||
|
||||
Ok(SourcePage {
|
||||
objects,
|
||||
common_prefixes,
|
||||
is_truncated,
|
||||
next_continuation_token,
|
||||
})
|
||||
|
||||
@@ -44,6 +44,7 @@ use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict};
|
||||
use super::config::{
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig,
|
||||
};
|
||||
use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
|
||||
use super::negative_cache::NegativeCache;
|
||||
use super::pull::{OdmWriteBack, PullQueue};
|
||||
use super::source_client::{SourceClient, SourceClientSpec, SourceError, SourceProvider, SourceTimeouts};
|
||||
@@ -280,6 +281,8 @@ pub struct BucketOdmState {
|
||||
write_back: Option<Arc<dyn OdmWriteBack>>,
|
||||
/// Started by `pull::BucketOdmState::pull_queue` on first enqueue.
|
||||
pub(super) pull_queue: OnceLock<Arc<PullQueue>>,
|
||||
/// Caps source listings for this bucket under `policy.list_through`.
|
||||
list_rate_limiter: SourceListRateLimiter,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BucketOdmState {
|
||||
@@ -328,6 +331,7 @@ impl BucketOdmState {
|
||||
last_source_error_logged_at: Mutex::new(None),
|
||||
write_back,
|
||||
pull_queue: OnceLock::new(),
|
||||
list_rate_limiter: SourceListRateLimiter::new(SOURCE_LIST_RATE_PER_SEC),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -360,6 +364,12 @@ impl BucketOdmState {
|
||||
&self.negative_cache
|
||||
}
|
||||
|
||||
/// Per-bucket rate limit on source `ListObjectsV2` calls, consulted by the
|
||||
/// list-through merge (rustfs/backlog#2164).
|
||||
pub fn list_rate_limiter(&self) -> &SourceListRateLimiter {
|
||||
&self.list_rate_limiter
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> &Arc<OdmStats> {
|
||||
&self.stats
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
|
||||
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
|
||||
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
|
||||
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
|
||||
|
||||
@@ -192,6 +192,9 @@ pub struct OnDemandMigrationPolicy {
|
||||
pub range_get: OnDemandMigrationRangeGetPolicy,
|
||||
#[serde(default)]
|
||||
pub source_error: OnDemandMigrationSourceErrorPolicy,
|
||||
/// Merge the source listing into `ListObjectsV2` (rustfs/backlog#2164).
|
||||
#[serde(default)]
|
||||
pub list_through: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub respect_local_delete_marker: bool,
|
||||
#[serde(default = "default_true")]
|
||||
@@ -222,6 +225,7 @@ impl Default for OnDemandMigrationPolicy {
|
||||
head: OnDemandMigrationHeadPolicy::default(),
|
||||
range_get: OnDemandMigrationRangeGetPolicy::default(),
|
||||
source_error: OnDemandMigrationSourceErrorPolicy::default(),
|
||||
list_through: false,
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
|
||||
@@ -103,7 +103,8 @@ The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unkno
|
||||
| `filter.source_prefix` | string \| null | `null` | Null or non-empty. Prepended to the local key to form the source key |
|
||||
| `policy.head` | `proxy` \| `local_only` | `proxy` | `local_only` answers a HEAD miss with 404 and no source traffic |
|
||||
| `policy.range_get` | `serve_and_backfill` \| `serve_only` | `serve_and_backfill` | Whether a Range GET also queues a background pull of the whole object |
|
||||
| `policy.source_error` | `propagate` \| `not_found` | `propagate` | `propagate` answers 424 `SourceUnavailable`; `not_found` degrades to 404 |
|
||||
| `policy.source_error` | `propagate` \| `not_found` | `propagate` | `propagate` answers 424 `SourceUnavailable`; `not_found` degrades to 404, and for a merged listing to local state only |
|
||||
| `policy.list_through` | bool | `false` | Merges the source listing into `ListObjectsV2` so clients see the whole namespace during the migration. Off by default: it puts the source in the path of every listing |
|
||||
| `policy.respect_local_delete_marker` | bool | `true` | A local delete marker is the final answer; only a versioned bucket can produce one |
|
||||
| `policy.preserve_etag` | bool | `true` | Keeps the source ETag on the stored object unless the bucket encrypts by default |
|
||||
| `policy.copy_tags` | bool | `false` | Copies source object tags; needs `s3:GetObjectTagging` and costs one extra source call per inline pull |
|
||||
@@ -137,7 +138,7 @@ Azure Blob has no preset; a native provider is deferred (rustfs/backlog#2166).
|
||||
|
||||
The credentials only ever need read access to the source bucket:
|
||||
|
||||
- `s3:ListBucket` on the bucket — used by the admin probe and by the backfill listing.
|
||||
- `s3:ListBucket` on the bucket — used by the admin probe, by the backfill listing, and by every merged listing under `policy.list_through`.
|
||||
- `s3:GetObject` on `<bucket>/*` — used by every HEAD and GET against the source.
|
||||
- `s3:GetObjectTagging` on `<bucket>/*` — only when `policy.copy_tags` is `true`.
|
||||
|
||||
@@ -155,7 +156,12 @@ Behaviour a client can observe. The "Test" column names the case that pins it: `
|
||||
| Client disconnects mid-stream on an inline pull | The write-back keeps draining the source and still stores the whole object | `get.rs::odm_get_inline_client_disconnect_still_stores_the_whole_object` |
|
||||
| Concurrent GET misses of one key | Singleflight: one leader tees, followers wait up to `first_byte_ms` and then re-read locally; on leader failure or timeout they stream through without queueing | `concurrency_test.rs::test_odm_concurrent_misses_on_one_key_coalesce`, `get.rs::odm_get_follower_rereads_local_after_the_leader_commits` |
|
||||
| HEAD miss | Proxied to the source, nothing is written locally; `local_only` answers 404 without source traffic | `head.rs::odm_head_source_hit_returns_output_and_writes_nothing_back`, `head.rs::odm_head_local_only_policy_is_404_without_source_traffic`, `real_source_test.rs::test_odm_rustfs_source_serves_pull_head_range_and_prefixes_real_single_node` |
|
||||
| LIST | Local only. A key that exists on the source but was never pulled is not listed, even while a GET of it succeeds | `interaction_test.rs::test_odm_write_back_respects_the_bucket_quota` |
|
||||
| LIST, `policy.list_through = false` (default) | Local only. A key that exists on the source but was never pulled is not listed, even while a GET of it succeeds | `interaction_test.rs::test_odm_write_back_respects_the_bucket_quota` |
|
||||
| `ListObjectsV2` with `policy.list_through = true` | The local and source listings are merged into one ordered page: byte-wise key order, local wins a key both sides hold, `CommonPrefixes` unioned and deduplicated under a delimiter. A source entry reports the source's own ETag, size and last-modified, storage class `STANDARD` and this bucket's owner. The continuation token is opaque and carries both cursors | `list_through_test.rs::list_through_merges_the_whole_namespace_across_full_pagination`, `list_through_test.rs::list_through_unions_common_prefixes_under_a_delimiter`, `list_through.rs::merged_pagination_equals_the_deduplicated_union` |
|
||||
| `ListObjects` (v1) and `ListObjectVersions`, whatever `list_through` says | Local only. v1 has no opaque continuation token that could carry two cursors, and a version listing has no meaning for a source whose versions were never pulled | `list_through_test.rs::a_merged_token_keeps_paginating_after_list_through_is_turned_off` |
|
||||
| Merged listing, source listing fails or the breaker is open | `propagate` answers 424 `SourceUnavailable`; `not_found` answers from local state alone and marks the response `x-rustfs-on-demand-migration-list: local_only` | `list_through_test.rs::list_through_propagates_a_source_listing_failure`, `list_through_test.rs::list_through_degrades_to_local_only_under_the_not_found_policy` |
|
||||
| Merged listing, tampered continuation token | 400 `InvalidArgument`; a token issued while `list_through` was on keeps paginating the local side after it is turned off | `list_through_test.rs::list_through_rejects_a_tampered_continuation_token`, `list_through.rs::token_round_trips_and_rejects_tampering` |
|
||||
| Merged listing, versioned bucket with a local delete marker | The shadowed key is dropped from the page, matching `respect_local_delete_marker` on GET. The local listing never returns delete markers, so such a bucket costs one extra local metadata read per source-only key in the page | `list_through_test.rs::a_local_delete_marker_hides_the_source_key_from_a_merged_listing` |
|
||||
| PUT / DELETE | Never touch the source. A local PUT shadows the source key for good | `interaction_test.rs::test_odm_delete_marker_shadows_the_source_but_a_plain_delete_does_not` |
|
||||
| Versioned bucket, local delete marker | With `respect_local_delete_marker` (default) the marker is the final answer and the source is not consulted | `interaction_test.rs::test_odm_delete_marker_shadows_the_source_but_a_plain_delete_does_not`, `head.rs::odm_head_verdict_respects_local_delete_marker_by_policy` |
|
||||
| Unversioned bucket, object deleted locally | The key becomes an ordinary miss and is pulled from the source again | `interaction_test.rs::test_odm_delete_marker_shadows_the_source_but_a_plain_delete_does_not` |
|
||||
@@ -306,7 +312,9 @@ sum by (bucket, reason) (rate(rustfs_on_demand_migration_pull_failures_total[5m]
|
||||
- **SSE-C source objects are not supported.** They are rejected with 424 `unsupported`; migrate them by another route.
|
||||
- **Anonymous (credential-less) sources are not supported yet.** `source.credentials: null` parses and passes structural validation, but the client builder has no anonymous mode, so the admin `PUT` refuses it and the runtime would treat such a bucket as unavailable. A public source still needs a key pair.
|
||||
- **Azure Blob is not a supported source** (rustfs/backlog#2166). GCS is supported only through its XML interoperability API with HMAC keys.
|
||||
- **LIST does not merge the source** (rustfs/backlog#2164). Only local objects are listed, so a client that lists before reading will not see un-migrated keys.
|
||||
- **LIST merges the source only when asked, and only for v2.** With the default `policy.list_through = false` a client that lists before reading will not see un-migrated keys. Turning it on merges `ListObjectsV2` alone; `ListObjects` (v1) and `ListObjectVersions` stay local.
|
||||
- **A merged listing costs up to two local listings and two source listings per page** (one per side, plus a refill when the previous page consumed most of what that side had buffered). Walking N merged keys at `max-keys=K` therefore costs ceil(N/K) requests and between ceil(N/K) and 2*ceil(N/K) source listings. Source listings are capped at 10 per second per bucket (a compile-time constant); a listing that cannot get a slot inside one second is treated like a source failure and follows `policy.source_error`.
|
||||
- **A degraded merged page loses the source keys in its window.** Under `source_error = not_found` the page is answered locally and the source cursor is left where it was, so the keys the source would have contributed between the previous page's last key and this one are not shown again once pagination moves on. The `x-rustfs-on-demand-migration-list: local_only` header marks every page this happened on.
|
||||
- **Write-through is undecided** (rustfs/backlog#2165). PUT and DELETE never reach the source in this version.
|
||||
- **`pull_failures_total` counts abandoned pulls, not attempts.** A pull that failed twice and then succeeded contributes nothing; attempt-level failure needs a new counter.
|
||||
- **The breaker's 30 s open window is a compile-time constant** with no environment override, which is why breaker-related tests have to wait it out.
|
||||
|
||||
@@ -0,0 +1,527 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! `ListObjectsV2` list-through for on-demand migration (rustfs/backlog#2164):
|
||||
//! when a bucket sets `policy.list_through`, the local listing and the source
|
||||
//! listing are merged into one ordered page so clients see the whole namespace
|
||||
//! while the migration runs.
|
||||
//!
|
||||
//! Only `ListObjectsV2` merges. `ListObjects` (v1) and `ListObjectVersions`
|
||||
//! stay local: v1 has no opaque continuation token to carry two cursors, and a
|
||||
//! version listing has no meaning for a source whose versions were never
|
||||
//! pulled.
|
||||
|
||||
use super::storage_api::bucket_usecase::ECStore;
|
||||
use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
|
||||
use super::storage_api::bucket_usecase::StorageObjectOptions;
|
||||
use super::storage_api::bucket_usecase::bucket::on_demand_migration::{
|
||||
BucketOdmState, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
|
||||
OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
|
||||
SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
|
||||
use super::storage_api::bucket_usecase::contract::list::{ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _};
|
||||
use super::storage_api::bucket_usecase::contract::object::ObjectOperations as _;
|
||||
use super::storage_api::bucket_usecase::s3::{S3Error, S3ErrorCode, S3Result};
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::ListObjectsV2Params;
|
||||
use crate::app::object::shared::{odm_source_unavailable_error, odm_state_error_class};
|
||||
use crate::error::ApiError;
|
||||
use futures::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::http::{SUFFIX_SOURCE_PROXY_REQUEST, get_header};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::debug;
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
|
||||
/// Storage class every source entry is reported with: the object is not local
|
||||
/// yet, so the only class RustFS can vouch for is the default one.
|
||||
const SOURCE_STORAGE_CLASS: &str = "STANDARD";
|
||||
|
||||
/// Concurrent local metadata probes when a versioned bucket has to check
|
||||
/// source-only keys for a shadowing delete marker.
|
||||
const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32;
|
||||
|
||||
/// Where the local side of a listing resumes.
|
||||
pub(crate) enum LocalListCursor {
|
||||
/// Continuation token for the local store, `None` for the first page.
|
||||
Token(Option<String>),
|
||||
/// The local side of a merged listing was already exhausted, so a listing
|
||||
/// that no longer merges has nothing left to return.
|
||||
Exhausted,
|
||||
}
|
||||
|
||||
/// Reads the (already base64-decoded) continuation token.
|
||||
///
|
||||
/// This runs whether or not the bucket merges: a token handed out while
|
||||
/// `list_through` was on must keep paginating the local side after it is
|
||||
/// turned off, and a tampered envelope must be rejected either way.
|
||||
pub(crate) fn decode_list_cursor(decoded: Option<&str>) -> S3Result<Option<ListThroughToken>> {
|
||||
match decoded.map(decode_continuation_token).transpose() {
|
||||
Ok(Some(ListThroughCursor::Merged(token))) => Ok(Some(*token)),
|
||||
Ok(_) => Ok(None),
|
||||
Err(err) => Err(invalid_continuation_token(&err)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The local cursor to use when the request is answered locally, given the
|
||||
/// decoded token.
|
||||
pub(crate) fn local_cursor(decoded: Option<&str>, merged: Option<&ListThroughToken>) -> LocalListCursor {
|
||||
match merged {
|
||||
Some(token) if token.local_done => LocalListCursor::Exhausted,
|
||||
Some(token) => LocalListCursor::Token(token.local.clone()),
|
||||
None => LocalListCursor::Token(decoded.map(str::to_string)),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_continuation_token(err: &ListThroughTokenError) -> S3Error {
|
||||
debug!(error = %err, "rejected an on-demand migration list continuation token");
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid continuation token".to_string())
|
||||
}
|
||||
|
||||
/// The bucket's live migration state when this request must merge the source.
|
||||
///
|
||||
/// `None` keeps the caller on the plain local listing: the module is off, the
|
||||
/// bucket has no source, `list_through` is off, or the request carries the
|
||||
/// `source-proxy-request` anti-loop marker and therefore comes from a peer
|
||||
/// that must be answered locally.
|
||||
pub(crate) fn list_through_state(bucket: &str, headers: &HeaderMap) -> Option<Arc<BucketOdmState>> {
|
||||
if get_header(headers, SUFFIX_SOURCE_PROXY_REQUEST).is_some() {
|
||||
return None;
|
||||
}
|
||||
let sys = OnDemandMigrationSys::get();
|
||||
if !sys.is_module_enabled() {
|
||||
return None;
|
||||
}
|
||||
let state = sys.state(bucket)?;
|
||||
state.config().policy.list_through.then_some(state)
|
||||
}
|
||||
|
||||
/// A merged page plus whether the source had to be left out of it.
|
||||
pub(crate) struct ListThroughOutcome {
|
||||
pub(crate) info: ListObjectsV2Info,
|
||||
/// The source could not be consulted; the answer is local state only and
|
||||
/// carries `x-rustfs-on-demand-migration-list: local_only`.
|
||||
pub(crate) degraded: bool,
|
||||
}
|
||||
|
||||
/// One entry of either side's buffer, in listing order.
|
||||
enum SideEntry {
|
||||
Object(Box<ObjectInfo>),
|
||||
Prefix(String),
|
||||
}
|
||||
|
||||
impl SideEntry {
|
||||
fn key(&self) -> ListEntryKey {
|
||||
match self {
|
||||
SideEntry::Object(info) => ListEntryKey::object(info.name.clone()),
|
||||
SideEntry::Prefix(prefix) => ListEntryKey::prefix(prefix.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Interleaves a listing page's objects and common prefixes into the single
|
||||
/// ordered sequence S3 paginates over. Both inputs are already sorted.
|
||||
fn interleave(objects: Vec<ObjectInfo>, prefixes: Vec<String>) -> Vec<SideEntry> {
|
||||
let mut merged = Vec::with_capacity(objects.len() + prefixes.len());
|
||||
let mut objects = objects.into_iter().peekable();
|
||||
let mut prefixes = prefixes.into_iter().peekable();
|
||||
loop {
|
||||
let take_object = match (objects.peek(), prefixes.peek()) {
|
||||
(None, None) => break,
|
||||
(Some(_), None) => true,
|
||||
(None, Some(_)) => false,
|
||||
(Some(object), Some(prefix)) => object.name.as_str() < prefix.as_str(),
|
||||
};
|
||||
if take_object {
|
||||
merged.push(SideEntry::Object(Box::new(objects.next().expect("peeked"))));
|
||||
} else {
|
||||
merged.push(SideEntry::Prefix(prefixes.next().expect("peeked")));
|
||||
}
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
/// One source listing entry in the local namespace. The ETag, size and
|
||||
/// last-modified are the source's own; the storage class is `STANDARD` and the
|
||||
/// owner (added by the output builder) is this bucket's, because the object is
|
||||
/// not local yet and RustFS can vouch for nothing else.
|
||||
fn source_object_info(bucket: &str, object: SourceObject) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
bucket: bucket.to_string(),
|
||||
name: object.key,
|
||||
mod_time: object.last_modified.map(OffsetDateTime::from),
|
||||
size: i64::try_from(object.size).unwrap_or(i64::MAX),
|
||||
etag: object.etag,
|
||||
storage_class: Some(SOURCE_STORAGE_CLASS.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn source_page_entries(bucket: &str, page: SourcePage) -> Vec<SideEntry> {
|
||||
let objects = page
|
||||
.objects
|
||||
.into_iter()
|
||||
.filter(|object| !object.key.is_empty())
|
||||
.map(|object| source_object_info(bucket, object))
|
||||
.collect();
|
||||
interleave(objects, page.common_prefixes)
|
||||
}
|
||||
|
||||
/// Runs one merged `ListObjectsV2` page.
|
||||
///
|
||||
/// Cost: at most two listings per side per request — the first page of each
|
||||
/// side, plus one refill when the previous page had already consumed most of
|
||||
/// what that side buffered. A full walk of N merged keys at `max_keys = K`
|
||||
/// therefore costs ceil(N/K) requests and between ceil(N/K) and 2*ceil(N/K)
|
||||
/// source listings.
|
||||
pub(crate) async fn merged_list_objects_v2(
|
||||
store: &Arc<ECStore>,
|
||||
state: &Arc<BucketOdmState>,
|
||||
bucket: &str,
|
||||
params: &ListObjectsV2Params,
|
||||
fetch_owner: bool,
|
||||
incl_deleted: bool,
|
||||
token: Option<&ListThroughToken>,
|
||||
) -> S3Result<ListThroughOutcome> {
|
||||
let policy = &state.config().policy;
|
||||
let max_keys = usize::try_from(params.max_keys).unwrap_or(0);
|
||||
let mut merger = ListThroughMerger::new(max_keys, token);
|
||||
let mut buffers: [Vec<Option<SideEntry>>; 2] = [Vec::new(), Vec::new()];
|
||||
let mut degraded = false;
|
||||
|
||||
let plan = source_list_plan(¶ms.prefix, state.config().filter.prefix.as_deref(), params.delimiter.as_deref());
|
||||
let mut client: Option<Arc<SourceClient>> = None;
|
||||
match &plan {
|
||||
// Nothing the source holds can appear under this prefix; that is a
|
||||
// filter decision, not a degradation.
|
||||
SourceListPlan::Skip => merger.disable_source(),
|
||||
_ => match state.client() {
|
||||
Ok(ready) if state.breaker().allow_request() => client = Some(Arc::clone(ready)),
|
||||
Ok(_) => degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "breaker_open")?,
|
||||
Err(error) => degrade_or_fail(&mut merger, &mut degraded, policy.source_error, odm_state_error_class(error))?,
|
||||
},
|
||||
}
|
||||
|
||||
while let Some(fetch) = merger.next_fetch() {
|
||||
let page = match fetch.side {
|
||||
MergeSide::Local => {
|
||||
let info = Arc::clone(store)
|
||||
.list_objects_v2(
|
||||
bucket,
|
||||
¶ms.prefix,
|
||||
fetch.token.clone(),
|
||||
params.delimiter.clone(),
|
||||
params.max_keys,
|
||||
fetch_owner,
|
||||
params.start_after_for_query.clone(),
|
||||
incl_deleted,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let objects = info.objects.into_iter().filter(|object| !object.name.is_empty()).collect();
|
||||
(interleave(objects, info.prefixes), info.is_truncated, info.next_continuation_token)
|
||||
}
|
||||
MergeSide::Source => {
|
||||
let client = client.as_ref().expect("the source side is disabled without a client");
|
||||
match fetch_source_page(state, client, bucket, params, &plan, fetch.token.as_deref()).await {
|
||||
Ok(page) => page,
|
||||
Err(class) => {
|
||||
degrade_or_fail(&mut merger, &mut degraded, policy.source_error, class)?;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let (entries, is_truncated, next_token) = page;
|
||||
let kept: Vec<SideEntry> = entries
|
||||
.into_iter()
|
||||
.filter(|entry| merger.accepts(&entry.key().name))
|
||||
.collect();
|
||||
let keys: Vec<ListEntryKey> = kept.iter().map(SideEntry::key).collect();
|
||||
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
|
||||
merger.push_page(fetch.side, keys, is_truncated, next_token);
|
||||
}
|
||||
|
||||
let outcome = merger.finish();
|
||||
let mut objects = Vec::with_capacity(outcome.picks.len());
|
||||
let mut prefixes = Vec::new();
|
||||
let mut source_only_keys = Vec::new();
|
||||
for pick in &outcome.picks {
|
||||
let buffer = &mut buffers[usize::from(pick.side == MergeSide::Source)];
|
||||
// Every pick names a distinct buffered entry, so taking it is safe.
|
||||
match buffer[pick.index].take().expect("a merged pick names a buffered entry") {
|
||||
SideEntry::Object(info) => {
|
||||
if pick.side == MergeSide::Source {
|
||||
source_only_keys.push(info.name.clone());
|
||||
}
|
||||
objects.push(*info);
|
||||
}
|
||||
SideEntry::Prefix(prefix) => prefixes.push(prefix),
|
||||
}
|
||||
}
|
||||
|
||||
if !source_only_keys.is_empty() && policy.respect_local_delete_marker && bucket_keeps_delete_markers(bucket).await {
|
||||
let shadowed = local_delete_markers(store, bucket, &source_only_keys).await;
|
||||
objects.retain(|object| !shadowed.contains(&object.name));
|
||||
}
|
||||
|
||||
Ok(ListThroughOutcome {
|
||||
info: ListObjectsV2Info {
|
||||
is_truncated: outcome.is_truncated,
|
||||
continuation_token: None,
|
||||
next_continuation_token: outcome.next_token.map(|token| token.encode()),
|
||||
objects,
|
||||
prefixes,
|
||||
},
|
||||
degraded,
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies `policy.source_error` to a source failure: `propagate` fails the
|
||||
/// listing with 424, `not_found` answers from local state alone.
|
||||
fn degrade_or_fail(
|
||||
merger: &mut ListThroughMerger,
|
||||
degraded: &mut bool,
|
||||
policy: SourceErrorPolicy,
|
||||
class: &'static str,
|
||||
) -> S3Result<()> {
|
||||
match policy {
|
||||
SourceErrorPolicy::Propagate => Err(odm_source_unavailable_error(class)),
|
||||
SourceErrorPolicy::NotFound => {
|
||||
merger.disable_source();
|
||||
*degraded = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type SourceFetch = (Vec<SideEntry>, bool, Option<String>);
|
||||
|
||||
/// One source listing, rate-limited per bucket. The error is the class label
|
||||
/// for the caller's `source_error` decision; the source's own message is never
|
||||
/// surfaced.
|
||||
async fn fetch_source_page(
|
||||
state: &Arc<BucketOdmState>,
|
||||
client: &SourceClient,
|
||||
bucket: &str,
|
||||
params: &ListObjectsV2Params,
|
||||
plan: &SourceListPlan,
|
||||
token: Option<&str>,
|
||||
) -> Result<SourceFetch, &'static str> {
|
||||
let Some(wait) = state.list_rate_limiter().reserve(SOURCE_LIST_MAX_RATE_WAIT) else {
|
||||
return Err("rate_limited");
|
||||
};
|
||||
if !wait.is_zero() {
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
|
||||
let request = match plan {
|
||||
SourceListPlan::Skip => return Ok((Vec::new(), false, None)),
|
||||
SourceListPlan::Page { prefix } => SourceListRequest {
|
||||
prefix: Some(prefix.as_str()),
|
||||
delimiter: params.delimiter.as_deref(),
|
||||
// S3 ignores start-after once a continuation token is present, so
|
||||
// the client's own start-after only applies to the first page.
|
||||
start_after: token.is_none().then(|| params.start_after_for_query.as_deref()).flatten(),
|
||||
continuation_token: token,
|
||||
max_keys: params.max_keys,
|
||||
},
|
||||
// Everything under `filter.prefix` rolls into one common prefix, so a
|
||||
// single bounded listing settles whether it exists.
|
||||
SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest {
|
||||
prefix: Some(probe_prefix.as_str()),
|
||||
max_keys: 1,
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
|
||||
let started = Instant::now();
|
||||
let result = client.list_page(&request).await;
|
||||
// A 404 on a listing is the bucket, not a key: keep it out of the negative
|
||||
// cache but let the breaker see everything else (same rule as the backfill).
|
||||
match &result {
|
||||
Err(SourceError::NotFound) => {}
|
||||
other => state.observe_source(started.elapsed(), "", other.as_ref().err()),
|
||||
}
|
||||
let page = result.map_err(|err| err.class_label())?;
|
||||
|
||||
match plan {
|
||||
SourceListPlan::Folded { common_prefix, .. } => {
|
||||
let exists = !page.objects.is_empty() || !page.common_prefixes.is_empty();
|
||||
Ok((
|
||||
exists
|
||||
.then(|| vec![SideEntry::Prefix(common_prefix.clone())])
|
||||
.unwrap_or_default(),
|
||||
false,
|
||||
None,
|
||||
))
|
||||
}
|
||||
_ => {
|
||||
let is_truncated = page.is_truncated;
|
||||
let next = page.next_continuation_token.clone();
|
||||
Ok((source_page_entries(bucket, page), is_truncated, next))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a delete on this bucket leaves a marker behind. Only such a bucket
|
||||
/// can shadow a source key the way the read path does.
|
||||
async fn bucket_keeps_delete_markers(bucket: &str) -> bool {
|
||||
BucketVersioningSys::enabled(bucket).await || BucketVersioningSys::suspended(bucket).await
|
||||
}
|
||||
|
||||
/// The subset of `keys` whose latest local version is a delete marker.
|
||||
///
|
||||
/// The local `ListObjectsV2` never returns delete markers, so a versioned
|
||||
/// bucket has to probe the source-only keys of the page: at most `max_keys`
|
||||
/// metadata reads, run with bounded concurrency and without the namespace lock
|
||||
/// — a listing is not a linearizable read, and the alternative is one lock per
|
||||
/// key per page. A lookup that fails for any other reason leaves the key
|
||||
/// visible rather than hiding data on a transient error.
|
||||
async fn local_delete_markers(store: &Arc<ECStore>, bucket: &str, keys: &[String]) -> std::collections::HashSet<String> {
|
||||
futures::stream::iter(keys.iter().cloned())
|
||||
.map(|key| {
|
||||
let store = Arc::clone(store);
|
||||
let bucket = bucket.to_string();
|
||||
async move {
|
||||
let options = StorageObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let info = store.get_object_info(&bucket, &key, &options).await;
|
||||
matches!(info, Ok(info) if info.delete_marker).then_some(key)
|
||||
}
|
||||
})
|
||||
.buffer_unordered(DELETE_MARKER_PROBE_CONCURRENCY)
|
||||
.filter_map(|shadowed| async move { shadowed })
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn token(local: Option<&str>, local_done: bool) -> ListThroughToken {
|
||||
ListThroughToken {
|
||||
t: "odm-list".to_string(),
|
||||
v: 1,
|
||||
local: local.map(str::to_string),
|
||||
local_done,
|
||||
source: Some("source-2".to_string()),
|
||||
source_done: false,
|
||||
last_key: Some("k".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn info(name: &str) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
name: name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn names(entries: &[SideEntry]) -> Vec<String> {
|
||||
entries.iter().map(|entry| entry.key().name).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interleave_orders_objects_and_prefixes_as_one_sequence() {
|
||||
let entries = interleave(vec![info("a"), info("b0"), info("c")], vec!["a/".to_string(), "b/".to_string()]);
|
||||
assert_eq!(names(&entries), vec!["a", "a/", "b/", "b0", "c"]);
|
||||
assert!(matches!(entries[1], SideEntry::Prefix(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_entries_report_the_source_facts_with_the_default_storage_class() {
|
||||
let page = SourcePage {
|
||||
objects: vec![SourceObject {
|
||||
key: "photos/a.jpg".to_string(),
|
||||
etag: Some("abc".to_string()),
|
||||
size: 7,
|
||||
last_modified: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||
storage_class: Some("GLACIER".to_string()),
|
||||
is_multipart_etag: false,
|
||||
}],
|
||||
common_prefixes: vec!["photos/2024/".to_string()],
|
||||
is_truncated: false,
|
||||
next_continuation_token: None,
|
||||
};
|
||||
let entries = source_page_entries("photos", page);
|
||||
assert_eq!(names(&entries), vec!["photos/2024/", "photos/a.jpg"]);
|
||||
let SideEntry::Object(object) = &entries[1] else {
|
||||
panic!("expected an object entry");
|
||||
};
|
||||
assert_eq!(object.etag.as_deref(), Some("abc"));
|
||||
assert_eq!(object.size, 7);
|
||||
assert_eq!(object.mod_time, Some(OffsetDateTime::UNIX_EPOCH));
|
||||
assert_eq!(
|
||||
object.storage_class.as_deref(),
|
||||
Some(SOURCE_STORAGE_CLASS),
|
||||
"a source storage class is never echoed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_merged_token_survives_list_through_being_turned_off() {
|
||||
let resume = token(Some("local-2"), false);
|
||||
let encoded = resume.encode();
|
||||
let decoded = decode_list_cursor(Some(&encoded)).expect("a valid envelope decodes");
|
||||
assert_eq!(decoded.as_ref(), Some(&resume));
|
||||
assert!(matches!(
|
||||
local_cursor(Some(&encoded), decoded.as_ref()),
|
||||
LocalListCursor::Token(Some(local)) if local == "local-2"
|
||||
));
|
||||
|
||||
let encoded = token(None, true).encode();
|
||||
let decoded = decode_list_cursor(Some(&encoded)).expect("a valid envelope decodes");
|
||||
assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() {
|
||||
assert!(
|
||||
decode_list_cursor(Some("photos/a.jpg"))
|
||||
.expect("plain markers decode")
|
||||
.is_none()
|
||||
);
|
||||
assert!(matches!(
|
||||
local_cursor(Some("photos/a.jpg"), None),
|
||||
LocalListCursor::Token(Some(local)) if local == "photos/a.jpg"
|
||||
));
|
||||
|
||||
let tampered = token(Some("local-2"), false).encode().replace("\"v\":1", "\"v\":9");
|
||||
let err = decode_list_cursor(Some(&tampered)).expect_err("a bumped version is rejected");
|
||||
assert_eq!(*err.code(), S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degrade_or_fail_follows_the_source_error_policy() {
|
||||
let mut merger = ListThroughMerger::new(10, None);
|
||||
let mut degraded = false;
|
||||
assert!(
|
||||
degrade_or_fail(&mut merger, &mut degraded, SourceErrorPolicy::Propagate, "server_error").is_err(),
|
||||
"propagate must surface the failure"
|
||||
);
|
||||
assert!(!degraded);
|
||||
|
||||
degrade_or_fail(&mut merger, &mut degraded, SourceErrorPolicy::NotFound, "server_error")
|
||||
.expect("not_found degrades instead of failing");
|
||||
assert!(degraded);
|
||||
assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local));
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,8 @@ use super::storage_api::bucket_usecase::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
request_context, validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use crate::app::bucket_list_through as list_through;
|
||||
use crate::app::object::shared::mark_on_demand_migration_list_local_only;
|
||||
use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete;
|
||||
use crate::app::runtime_sources::{
|
||||
AppContext, current_app_context, current_encryption_service, current_notification_system,
|
||||
@@ -2719,19 +2721,47 @@ impl DefaultBucketUsecase {
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
let object_infos = store
|
||||
.list_objects_v2(
|
||||
&bucket,
|
||||
¶ms.prefix,
|
||||
params.decoded_continuation_token.clone(),
|
||||
params.delimiter.clone(),
|
||||
params.max_keys,
|
||||
fetch_owner.unwrap_or_default(),
|
||||
params.start_after_for_query.clone(),
|
||||
incl_deleted,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
// The on-demand migration envelope is decoded whether or not this
|
||||
// bucket still merges: a token handed out under `list_through` must keep
|
||||
// paginating after the policy is turned off (rustfs/backlog#2164).
|
||||
let merged_token = list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?;
|
||||
let (object_infos, degraded) = match list_through::list_through_state(&bucket, &req.headers) {
|
||||
Some(state) => {
|
||||
let outcome = list_through::merged_list_objects_v2(
|
||||
&store,
|
||||
&state,
|
||||
&bucket,
|
||||
¶ms,
|
||||
fetch_owner.unwrap_or_default(),
|
||||
incl_deleted,
|
||||
merged_token.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
(outcome.info, outcome.degraded)
|
||||
}
|
||||
None => {
|
||||
let cursor = list_through::local_cursor(params.decoded_continuation_token.as_deref(), merged_token.as_ref());
|
||||
match cursor {
|
||||
list_through::LocalListCursor::Exhausted => (StorageListObjectsV2Info::default(), false),
|
||||
list_through::LocalListCursor::Token(token) => {
|
||||
let infos = store
|
||||
.list_objects_v2(
|
||||
&bucket,
|
||||
¶ms.prefix,
|
||||
token,
|
||||
params.delimiter.clone(),
|
||||
params.max_keys,
|
||||
fetch_owner.unwrap_or_default(),
|
||||
params.start_after_for_query.clone(),
|
||||
incl_deleted,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
(infos, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let output = build_list_objects_v2_output(
|
||||
object_infos,
|
||||
@@ -2745,7 +2775,11 @@ impl DefaultBucketUsecase {
|
||||
params.response_start_after,
|
||||
);
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
let mut response = S3Response::new(output);
|
||||
if degraded {
|
||||
mark_on_demand_migration_list_local_only(&mut response.headers);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_list_objects_v2m(
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
//! Concrete use-case modules will be introduced incrementally in Phase 3.
|
||||
|
||||
pub mod admin_usecase;
|
||||
pub(crate) mod bucket_list_through;
|
||||
pub mod bucket_usecase;
|
||||
pub mod context;
|
||||
pub(crate) mod metadata_route;
|
||||
|
||||
@@ -212,7 +212,7 @@ mod internal_put;
|
||||
mod on_demand_migration_put;
|
||||
mod put;
|
||||
mod restore;
|
||||
mod shared;
|
||||
pub(crate) mod shared;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
|
||||
|
||||
@@ -799,6 +799,12 @@ pub(super) fn object_lock_checks_required_for_state(state: &metadata_sys::Object
|
||||
pub(crate) const ON_DEMAND_MIGRATION_HEADER: http::HeaderName = http::HeaderName::from_static("x-rustfs-on-demand-migration");
|
||||
pub(crate) const ON_DEMAND_MIGRATION_SOURCE: HeaderValue = HeaderValue::from_static("source");
|
||||
|
||||
/// Response header marking a `ListObjectsV2` that could not consult the source
|
||||
/// and was answered from local state alone (rustfs/backlog#2164).
|
||||
pub(crate) const ON_DEMAND_MIGRATION_LIST_HEADER: http::HeaderName =
|
||||
http::HeaderName::from_static("x-rustfs-on-demand-migration-list");
|
||||
pub(crate) const ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY: HeaderValue = HeaderValue::from_static("local_only");
|
||||
|
||||
/// Custom S3 error code for a source failure surfaced under
|
||||
/// `policy.source_error = propagate`; carried on HTTP 424.
|
||||
pub(crate) const ODM_SOURCE_UNAVAILABLE_CODE: &str = "SourceUnavailable";
|
||||
@@ -872,6 +878,12 @@ pub(crate) fn mark_on_demand_migration_response(headers: &mut HeaderMap) {
|
||||
headers.insert(ON_DEMAND_MIGRATION_HEADER, ON_DEMAND_MIGRATION_SOURCE);
|
||||
}
|
||||
|
||||
/// Marks a merged listing that fell back to local state because the source
|
||||
/// could not be consulted (rustfs/backlog#2164).
|
||||
pub(crate) fn mark_on_demand_migration_list_local_only(headers: &mut HeaderMap) {
|
||||
headers.insert(ON_DEMAND_MIGRATION_LIST_HEADER, ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY);
|
||||
}
|
||||
|
||||
/// Evaluates the request's conditional headers (`If-Match`, `If-None-Match`,
|
||||
/// `If-Modified-Since`, `If-Unmodified-Since`) against the source's view of
|
||||
/// the object, with the same S3 semantics the local path applies to
|
||||
|
||||
@@ -33,6 +33,7 @@ pub(crate) mod s3 {
|
||||
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
|
||||
};
|
||||
pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
}
|
||||
|
||||
pub(crate) mod admin {
|
||||
@@ -626,7 +627,7 @@ pub(crate) mod bucket {
|
||||
|
||||
pub(crate) mod on_demand_migration {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceError, SourceGet, SourceHead,
|
||||
SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
|
||||
@@ -638,6 +639,10 @@ pub(crate) mod bucket {
|
||||
PullError, PullLeader, PullOutcome, PullReason, PullSlot, RangeGetPolicy, SourceBody, SourceErrorPolicy,
|
||||
commit_inline, idle_guarded_body,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
|
||||
ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
|
||||
SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod policy_sys {
|
||||
@@ -1157,12 +1162,16 @@ pub(crate) mod bucket_usecase {
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::storage_contracts::{ListObjectVersionsInfo, ListObjectsV2Info, ListOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::super::super::storage_contracts::ObjectOperations;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3_api};
|
||||
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3, s3_api};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations, validate_list_object_unordered_with_delimiter,
|
||||
ECStore, StorageObjectInfo, StorageObjectOptions, get_validated_store, process_lambda_configurations,
|
||||
process_queue_configurations, process_topic_configurations, validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user