mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 12:35:54 +00:00
Zhengchao An
7dfc2ee5f0
* feat(odm): merge the source listing into ListObjectsV2 Adds policy.list_through: ListObjectsV2 merges the local and source listings into one ordered page so clients see the whole namespace during an on-demand migration. Local entries win a key both sides hold, CommonPrefixes are unioned under a delimiter, and the continuation token is an opaque versioned envelope carrying both cursors. A source listing failure or an open breaker follows policy.source_error: propagate answers 424, not_found answers from local state and marks the response x-rustfs-on-demand-migration-list: local_only. Source listings are capped at 10 per second per bucket. * test(odm): refresh the e2e-full darwin selection digest The list-through e2e module adds seven cases to the merge lane.
61 lines
3.0 KiB
Rust
61 lines
3.0 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! On-Demand Migration (ODM): a bucket can name an external S3-compatible
|
|
//! source bucket; GET misses are served from that source and backfilled
|
|
//! locally. This module owns the bucket-level configuration model
|
|
//! (`on-demand-migration.json` in the bucket metadata file), the source
|
|
//! client, and the per-node runtime (`sys`) that turns configs into live
|
|
//! clients guarded by a breaker, a negative cache, singleflight and a pull
|
|
//! concurrency limit (rustfs/backlog#2147).
|
|
|
|
pub mod backfill;
|
|
pub mod breaker;
|
|
pub mod config;
|
|
pub mod list_through;
|
|
pub mod negative_cache;
|
|
pub mod pull;
|
|
pub mod source_client;
|
|
pub mod stats;
|
|
pub mod sys;
|
|
|
|
pub use breaker::{
|
|
BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
|
|
BreakerState, BreakerTransition, BreakerVerdict,
|
|
};
|
|
pub use config::{
|
|
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
|
|
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
|
|
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
|
};
|
|
pub use list_through::{
|
|
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
|
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
|
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
|
};
|
|
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
|
|
pub use pull::{
|
|
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS, PullCompletion,
|
|
PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody, WriteBackError,
|
|
WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, idle_guarded_body,
|
|
};
|
|
pub use stats::{
|
|
GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
|
|
PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
|
|
};
|
|
pub use sys::{
|
|
ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
|
|
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec,
|
|
};
|