mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 10:17:55 +00:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 23b17c2d5a | |||
| 89e2513205 | |||
| d091554ffe | |||
| c04ee41cf0 | |||
| a9691b6797 | |||
| 7db3882777 | |||
| 3f3e3f4f05 | |||
| 01e0af6312 | |||
| 3377688dab | |||
| 890ddea94b | |||
| 9f02ca6c36 | |||
| 33cd11472a | |||
| 3ff250f1cd | |||
| d795729585 | |||
| 9e6e02ea09 | |||
| 39274fc37c | |||
| 33eff4c3c4 | |||
| a2f16aa066 | |||
| 4c8b9f87e1 | |||
| 3272730c13 | |||
| 1862112d0c | |||
| cd0ac02879 | |||
| 6cf9cf7bb5 | |||
| f1f86ee9d0 | |||
| 1eef0de003 |
Generated
+5
@@ -9825,14 +9825,19 @@ name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"humantime",
|
||||
"hyper",
|
||||
"jiff",
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rustfs-signer",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use base64_simd::STANDARD;
|
||||
|
||||
|
||||
@@ -38,7 +38,10 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||
|
||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||
|
||||
@@ -476,13 +476,19 @@ impl Checksum for Xxhash64 {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
struct Md5 {
|
||||
hasher: md5::Md5,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
||||
)]
|
||||
impl Md5 {
|
||||
fn update(&mut self, bytes: &[u8]) {
|
||||
use md5::Digest;
|
||||
|
||||
@@ -373,8 +373,8 @@ pub mod error {
|
||||
|
||||
pub mod erasure {
|
||||
pub use crate::erasure::coding::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
||||
calc_shard_size, calc_shard_size_legacy,
|
||||
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
||||
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -82,7 +82,6 @@ use tracing::warn;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
|
||||
@@ -126,11 +126,23 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
|
||||
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
|
||||
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
|
||||
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
|
||||
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
|
||||
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub type TraceFn =
|
||||
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||
@@ -140,9 +152,21 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
|
||||
|
||||
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
||||
@@ -280,6 +304,10 @@ impl LifecycleSys {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn trace(oi: &ObjectInfo) -> TraceFn {
|
||||
let bucket = oi.bucket.clone();
|
||||
let name = oi.name.clone();
|
||||
@@ -570,6 +598,10 @@ async fn delete_free_version_remote_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_free_version_remote_object_then<T, F, Fut>(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
@@ -2868,6 +2900,10 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
|
||||
initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
|
||||
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
|
||||
}
|
||||
@@ -3352,6 +3388,10 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
|
||||
if remote_delete_succeeded {
|
||||
opts.skip_decommissioned = true;
|
||||
@@ -4339,6 +4379,10 @@ pub async fn expire_transitioned_object(
|
||||
Ok(dobj)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
let us = Uuid::new_v4().to_string();
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -4373,6 +4417,10 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||
let tier = _tier.to_string();
|
||||
Arc::new(move || {
|
||||
@@ -4391,6 +4439,10 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn get_transitioned_object_reader(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -5145,6 +5197,10 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res
|
||||
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||
let mut success = false;
|
||||
match event.action {
|
||||
@@ -7422,6 +7478,10 @@ mod tests {
|
||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||
// SAFETY: keep this note adjacent to the allowance for the repository guard.
|
||||
#[allow(unsafe_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)"
|
||||
)]
|
||||
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
|
||||
@@ -759,6 +759,10 @@ pub struct ManualTransitionWorkerResultRecord {
|
||||
}
|
||||
|
||||
impl ManualTransitionWorkerResultRecord {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||
Self::new_with_reason(job_id, task_key, result, None)
|
||||
}
|
||||
@@ -1257,6 +1261,10 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_task_record(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1320,6 +1328,10 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn load_manual_transition_worker_result_stats(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
@@ -1455,6 +1467,10 @@ async fn scan_manual_transition_worker_result_journal(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn reconcile_manual_transition_worker_results(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
|
||||
@@ -15,25 +15,35 @@
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||
pub(crate) use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||
};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
use crate::storage_api_contracts::object::DeletedObject;
|
||||
|
||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
|
||||
obj.version_purge_status.is_pending()
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
|
||||
replication_status_blocks_lifecycle(&obj.replication_status)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
|
||||
has_pending_object_replication(obj) || has_pending_version_purge(obj)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,10 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||
}
|
||||
|
||||
@@ -331,6 +331,10 @@ where
|
||||
persist_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
|
||||
@@ -148,6 +148,10 @@ struct RecoveryCursor {
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn recover_tier_free_versions(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
|
||||
@@ -385,6 +385,10 @@ impl ExpiryOp for Jentry {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
|
||||
if let Err(err) = &result
|
||||
@@ -395,6 +399,10 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
#[cfg(test)]
|
||||
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
|
||||
@@ -405,6 +413,10 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
|
||||
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -485,6 +497,10 @@ pub enum RemoteTierDeleteOutcome {
|
||||
AlreadyRemoved,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_object_from_remote_tier_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
|
||||
@@ -50,8 +50,16 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionTransactionError {
|
||||
#[error("transition transaction already exists")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
AlreadyExists,
|
||||
#[error("transition transaction is not found")]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
NotFound,
|
||||
#[error("transition transaction is corrupt: {0}")]
|
||||
Corrupt(&'static str),
|
||||
|
||||
@@ -60,12 +60,14 @@ struct ConfigWriteLockProbeState {
|
||||
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub struct ConfigWriteLockProbe {
|
||||
state: Arc<ConfigWriteLockProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl ConfigWriteLockProbe {
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub fn install(bucket: &str) -> Self {
|
||||
let state = Arc::new(ConfigWriteLockProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -84,6 +86,7 @@ impl ConfigWriteLockProbe {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
|
||||
pub async fn wait_until_attempted(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
@@ -890,6 +893,10 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
|
||||
bucket_meta_sys.get_bucket_policy_raw(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1104,6 +1111,10 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
|
||||
bucket_meta_sys.get_config_from_disk(bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1617,6 +1628,7 @@ impl BucketMetadataSys {
|
||||
/// [`Self::update`], with the payload computed from the loaded metadata
|
||||
/// instead of supplied up front. Loads through this system's own store so
|
||||
/// the read and the persisted write target the same instance.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
@@ -1721,6 +1733,7 @@ impl BucketMetadataSys {
|
||||
/// A miss is never published as an authoritative default, and a snapshot
|
||||
/// read before delete plus same-name recreation cannot replace the new
|
||||
/// generation.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
|
||||
if is_meta_bucketname(bucket) {
|
||||
return Err(Error::other("errInvalidArgument"));
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: bucket subsystems still contain staged ECStore migration code.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
|
||||
@@ -136,6 +136,7 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
|
||||
|
||||
/// Check if an object has legal hold enabled.
|
||||
/// Returns true if legal hold is ON.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
|
||||
let lhold = objectlock::get_object_legalhold_meta(user_defined);
|
||||
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
|
||||
@@ -151,6 +152,7 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
|
||||
/// # Returns
|
||||
/// * `true` if the object is locked (cannot be deleted/modified)
|
||||
/// * `false` if the object is not locked
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
|
||||
// Delete markers are never locked
|
||||
if is_delete_marker {
|
||||
|
||||
@@ -193,6 +193,7 @@ pub enum QuotaError {
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub struct QuotaErrorResponse {
|
||||
#[serde(rename = "Code")]
|
||||
pub code: String,
|
||||
@@ -208,6 +209,7 @@ pub struct QuotaErrorResponse {
|
||||
}
|
||||
|
||||
impl QuotaErrorResponse {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
|
||||
match quota_error {
|
||||
QuotaError::QuotaExceeded { .. } => Self {
|
||||
|
||||
@@ -899,6 +899,7 @@ async fn save_ledger_locked(
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn fail_next_quota_ledger_save_for_test() {
|
||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ pub use replication_filemeta_boundary::{
|
||||
pub(crate) use replication_filemeta_boundary::{
|
||||
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||
};
|
||||
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
|
||||
@@ -37,6 +37,10 @@ impl ReplicationConfigStore {
|
||||
com::read_config_limited(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
|
||||
@@ -24,15 +24,27 @@ use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn has_pending_version_purge(
|
||||
config: &ReplicationLifecycleConfig,
|
||||
object_name: &str,
|
||||
@@ -45,6 +57,10 @@ impl ReplicationLifecycleBridge {
|
||||
.is_some_and(|config| config.has_active_rules(object_name, true))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn check_delete_replication(
|
||||
bucket: &str,
|
||||
object: &ObjectToDelete,
|
||||
@@ -54,6 +70,10 @@ impl ReplicationLifecycleBridge {
|
||||
check_replicate_delete(bucket, object, source, opts, None).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
|
||||
let pending_status = decision.pending_status();
|
||||
ReplicationState {
|
||||
|
||||
@@ -19,17 +19,33 @@ use time::OffsetDateTime;
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::msgp_decode;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) struct ReplicationMsgpCodec;
|
||||
|
||||
impl ReplicationMsgpCodec {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
|
||||
msgp_decode::read_msgp_ext8_time(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
|
||||
msgp_decode::skip_msgp_value(rd)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
|
||||
msgp_decode::write_msgp_time(wr, time)
|
||||
}
|
||||
|
||||
@@ -77,6 +77,10 @@ impl ReplicationObjectBridge {
|
||||
load_delete_request_config_in(ctx, bucket).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn delete_config_snapshot_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -231,6 +231,10 @@ pub(crate) async fn load_delete_replication_config(
|
||||
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn load_delete_replication_config_in(
|
||||
ctx: &ReplicationInstanceContext,
|
||||
bucket: &str,
|
||||
|
||||
@@ -217,6 +217,10 @@ impl DurableMrfBacklogTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
|
||||
let mut tracker = DurableMrfBacklogTracker {
|
||||
available: true,
|
||||
@@ -712,6 +716,10 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
|
||||
// MRF worker lifecycle
|
||||
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
mrf_stop_tx: Sender<()>,
|
||||
|
||||
// Worker size tracking
|
||||
@@ -940,6 +948,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Resizes worker priority and counts
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn resize_worker_priority(
|
||||
&self,
|
||||
pri: ReplicationPriority,
|
||||
@@ -1180,6 +1192,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Queues an MRF save operation
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||
}
|
||||
@@ -1651,6 +1667,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling regular replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1664,6 +1684,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling large object replication operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_large_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1678,6 +1702,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling MRF (Most Recent Failures) operations
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
async fn add_mrf_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
@@ -1691,6 +1719,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Delete resync metadata from replication resync state in memory
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn delete_resync_metadata(&self, bucket: &str) {
|
||||
let mut status_map = self.resyncer.status_map.write().await;
|
||||
status_map.remove(bucket);
|
||||
|
||||
@@ -21,11 +21,31 @@ pub(crate) use rustfs_replication::{
|
||||
should_count_head_proxy_failure,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||
|
||||
fn map_replication_error(err: rustfs_replication::Error) -> Error {
|
||||
|
||||
@@ -122,6 +122,10 @@ const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"tcp connect error",
|
||||
];
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
|
||||
|
||||
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
|
||||
@@ -328,6 +332,10 @@ fn bounded_resync_max_jobs(value: usize) -> usize {
|
||||
#[derive(Debug)]
|
||||
pub struct ReplicationResyncer {
|
||||
pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub worker_size: usize,
|
||||
pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>,
|
||||
resync_admission: Arc<Semaphore>,
|
||||
@@ -544,6 +552,10 @@ impl ReplicationResyncer {
|
||||
.is_some_and(|status| status.failed_count > 0)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
|
||||
@@ -340,6 +340,10 @@ impl ReplicationStats {
|
||||
}
|
||||
|
||||
/// Site replication update replica statistics
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn sr_update_replica_stat(&self, size: i64) {
|
||||
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
|
||||
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
@@ -59,6 +59,10 @@ impl fmt::Debug for Credentials {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub enum ServiceType {
|
||||
#[default]
|
||||
Replication,
|
||||
|
||||
@@ -73,23 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
|
||||
if object_name.len() > 1024 {
|
||||
return Err(Error::other("Object name cannot be longer than 1024 characters"));
|
||||
}
|
||||
if !object_name.is_ascii() {
|
||||
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
|
||||
if object_name.trim().is_empty() {
|
||||
return Err(Error::other("Object name cannot be empty"));
|
||||
}
|
||||
check_valid_object_name_prefix(object_name)
|
||||
}
|
||||
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
@@ -100,6 +83,10 @@ where
|
||||
Ok(ans)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
@@ -186,15 +173,27 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
||||
/// Client-facing reason attached to rejections of object keys that Win32/NTFS
|
||||
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
|
||||
/// accept the full S3 key character set.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
|
||||
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
|
||||
|
||||
/// Client-facing reason for path segments Windows can store but not address
|
||||
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
|
||||
|
||||
/// Reserved DOS device names that shadow regular files on Windows, even when
|
||||
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
|
||||
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
|
||||
@@ -204,6 +203,10 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
|
||||
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a
|
||||
/// dot or a space, and reserved DOS device names — bare or with an extension
|
||||
/// (`NUL.txt`), matching classic Win32 path resolution semantics.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
|
||||
)]
|
||||
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
|
||||
object.split(['/', '\\']).any(|segment| {
|
||||
if segment.ends_with('.') || segment.ends_with(' ') {
|
||||
|
||||
@@ -90,6 +90,10 @@ impl BucketVersioningSys {
|
||||
/// caller's own instance context so a second in-process store never
|
||||
/// answers with the first instance's versioning state; falls back to the
|
||||
/// ambient system when the instance cell is not initialized.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
|
||||
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
|
||||
return Ok(VersioningConfiguration::default());
|
||||
|
||||
@@ -637,14 +637,23 @@ impl Default for DiskOperationMetrics {
|
||||
}
|
||||
|
||||
impl DiskOperationMetrics {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||
)]
|
||||
fn record_call(&mut self) {
|
||||
self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
|
||||
)]
|
||||
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
|
||||
self.record_latency_atomic(now_sec, elapsed);
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")]
|
||||
fn record(&mut self, now_sec: u64, elapsed: Duration) {
|
||||
self.record_call();
|
||||
self.record_latency(now_sec, elapsed);
|
||||
@@ -770,6 +779,7 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
/// Set disk as faulty
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn set_faulty(&self) {
|
||||
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
}
|
||||
@@ -850,6 +860,7 @@ impl DiskHealthTracker {
|
||||
became_offline
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
let current = self.runtime_state();
|
||||
if current == RuntimeDriveHealthState::Offline {
|
||||
@@ -980,11 +991,13 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
/// Get waiting operations count
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn waiting_count(&self) -> u32 {
|
||||
self.waiting.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Get last success timestamp
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn last_success(&self) -> i64 {
|
||||
self.last_success.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -1026,21 +1039,6 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
/// Health check context key for tracking disk operations
|
||||
#[derive(Debug, Clone)]
|
||||
struct HealthDiskCtxKey;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct HealthDiskCtxValue {
|
||||
last_success: Arc<AtomicI64>,
|
||||
}
|
||||
|
||||
impl HealthDiskCtxValue {
|
||||
fn log_success(&self) {
|
||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1072,10 +1070,6 @@ impl LocalDiskWrapper {
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_health(disk: Arc<LocalDisk>, health_check: bool, health: Arc<DiskHealthTracker>) -> Self {
|
||||
Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default()))
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_reconnect_state(
|
||||
disk: Arc<LocalDisk>,
|
||||
health_check: bool,
|
||||
@@ -1438,20 +1432,6 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_id(&self, want_id: Option<Uuid>) -> Result<()> {
|
||||
if want_id.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let stored_disk_id = self.disk.get_disk_id().await?;
|
||||
|
||||
if stored_disk_id != want_id {
|
||||
return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if disk ID is stale
|
||||
async fn check_disk_stale(&self) -> Result<()> {
|
||||
let Some(current_disk_id) = *self.disk_id.read().await else {
|
||||
|
||||
@@ -48,6 +48,7 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
|
||||
match io_err.kind() {
|
||||
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
|
||||
|
||||
@@ -178,6 +178,7 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
match fs::remove_file(path.as_ref()).await {
|
||||
|
||||
@@ -665,6 +665,7 @@ async fn remove_empty_directory_tree_under_mount_lease(
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn remove_empty_directory_tree_with(
|
||||
root: &Path,
|
||||
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
@@ -1016,13 +1017,29 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str,
|
||||
/// When enabled, shard reads bypass the page cache using O_DIRECT flag.
|
||||
/// Requires aligned buffers (typically 512 bytes or 4096 bytes).
|
||||
/// Default: false (uses page cache via mmap/pread).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
|
||||
|
||||
/// Minimum shard size threshold for O_DIRECT reads.
|
||||
/// Only shards larger than this threshold will use O_DIRECT.
|
||||
/// Default: 4MB.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
|
||||
@@ -1036,7 +1053,15 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
|
||||
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
|
||||
/// back to buffered writes for the whole disk. Non-Linux always falls back.
|
||||
/// Default: false (buffered writes via the page cache, as before).
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
||||
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
||||
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
||||
@@ -1095,12 +1120,14 @@ macro_rules! cached_read_env {
|
||||
|
||||
cached_read_env! {
|
||||
/// Check if O_DIRECT reads are enabled.
|
||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||
fn is_direct_io_read_enabled() -> bool =
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
|
||||
}
|
||||
|
||||
cached_read_env! {
|
||||
/// Check if O_DIRECT shard/part data writes are enabled.
|
||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||
fn is_direct_io_write_enabled() -> bool =
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
|
||||
}
|
||||
@@ -1456,6 +1483,7 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
||||
|
||||
cached_read_env! {
|
||||
/// Get the O_DIRECT read threshold size.
|
||||
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
|
||||
fn get_direct_io_read_threshold() -> usize =
|
||||
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
|
||||
}
|
||||
@@ -1673,12 +1701,20 @@ impl DirectIoWriteState {
|
||||
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
|
||||
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks
|
||||
/// into one positioned write to keep the syscall count low.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
|
||||
|
||||
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
|
||||
/// size rounded up to a whole multiple of `align` so the buffer address, every
|
||||
/// flushed batch length, and every write offset stay alignment-correct.
|
||||
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
fn direct_write_staging_capacity(align: usize) -> usize {
|
||||
debug_assert!(align.is_power_of_two() && align >= 512);
|
||||
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
|
||||
@@ -1687,6 +1723,10 @@ fn direct_write_staging_capacity(align: usize) -> usize {
|
||||
/// Split `filled` staged bytes into the alignment-sized prefix written with
|
||||
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
|
||||
/// so the tail-boundary math is unit-tested on any host.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
|
||||
let aligned = filled - (filled % align);
|
||||
(aligned, filled - aligned)
|
||||
@@ -2142,6 +2182,7 @@ fn set_delete_version_fail_after_data_staged(path: &str) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
|
||||
DELETE_VERSION_FAIL_AFTER_COMMIT
|
||||
.lock()
|
||||
@@ -2447,6 +2488,10 @@ enum SyncMode {
|
||||
FileOnly,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||
)]
|
||||
struct FileCacheReclaimWriter {
|
||||
inner: File,
|
||||
reclaim_len: usize,
|
||||
@@ -2454,6 +2499,10 @@ struct FileCacheReclaimWriter {
|
||||
reclaimed: bool,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
|
||||
)]
|
||||
struct FileCacheReclaimReader {
|
||||
inner: File,
|
||||
reclaim_offset: u64,
|
||||
@@ -2519,6 +2568,10 @@ impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)"
|
||||
)]
|
||||
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
|
||||
// Runs per read-stream page-cache reclaim window; skip the whole emission
|
||||
// (three metric-key constructions) when general metrics are disabled.
|
||||
@@ -3071,6 +3124,7 @@ impl LocalIoBackend for StdBackend {
|
||||
use memmap2::MmapOptions;
|
||||
use std::time::{Duration as StdDuration, Instant as StdInstant};
|
||||
|
||||
#[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")]
|
||||
struct MmapCopyReadResult {
|
||||
bytes: Bytes,
|
||||
access_check_duration: StdDuration,
|
||||
@@ -4704,6 +4758,10 @@ fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
|
||||
Arc::new(StdBackend::new(root))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)"
|
||||
)]
|
||||
pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
@@ -5490,6 +5548,7 @@ impl LocalDisk {
|
||||
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
|
||||
let path_ref = path.as_ref();
|
||||
let path_str = path_ref.to_string_lossy();
|
||||
@@ -5567,15 +5626,24 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
// Check if a path is valid
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||
check_local_disk_valid_path(self.io_root(), path)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
fn reject_symlink_components(&self, path: &Path) -> Result<()> {
|
||||
reject_local_disk_symlink_components(self.io_root(), path)
|
||||
}
|
||||
|
||||
// Batch path generation with single lock acquisition
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
|
||||
let mut results = Vec::with_capacity(requests.len());
|
||||
let mut cache_misses = Vec::new();
|
||||
@@ -6488,6 +6556,7 @@ impl LocalDisk {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
||||
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
||||
Ok(f)
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod disk_store;
|
||||
pub mod endpoint;
|
||||
@@ -1114,6 +1113,10 @@ pub struct DiskInfo {
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)"
|
||||
)]
|
||||
pub struct Info {
|
||||
pub total: u64,
|
||||
pub free: u64,
|
||||
@@ -1372,6 +1375,7 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn has_part_err(part_errs: &[usize]) -> bool {
|
||||
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
|
||||
}
|
||||
|
||||
@@ -571,6 +571,10 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
|
||||
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
||||
/// itself.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
||||
)]
|
||||
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
for entry in std::fs::read_dir(dir.as_ref())? {
|
||||
let entry = entry?;
|
||||
@@ -583,6 +587,7 @@ pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
||||
/// concurrently, bounded both per directory and process-wide.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
|
||||
}
|
||||
@@ -1809,10 +1814,6 @@ impl RenameCommitGuard {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result<RenameDestinationPathGuard> {
|
||||
self.destination_directory_guard(directory, false)
|
||||
}
|
||||
|
||||
pub(crate) fn create_destination_directory_for_path_access(
|
||||
&self,
|
||||
directory: &Path,
|
||||
@@ -2858,13 +2859,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if a file exists.
|
||||
/// Returns true if the file exists, false otherwise.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn file_exists(path: impl AsRef<Path>) -> bool {
|
||||
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether an [`io::Error`] means "the directory is not empty".
|
||||
///
|
||||
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either
|
||||
|
||||
@@ -820,10 +820,263 @@ impl BitrotWriterWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
|
||||
//
|
||||
// A broken hash implementation (bad SIMD feature combination, platform drift, a
|
||||
// key-handling regression) fails silently: every shard reads back "corrupt",
|
||||
// heal rewrites data that was fine, and cross-platform clusters disagree about
|
||||
// which copy is healthy. The self-test below pins the algorithms the moment a
|
||||
// process starts, so a drifted build announces itself instead of quietly
|
||||
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
|
||||
// analysis-2026-08-16.md §6 HS-11.
|
||||
|
||||
/// Length of the deterministic self-test payload.
|
||||
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
|
||||
|
||||
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
|
||||
/// (the production default). Pinned so any platform or build where the
|
||||
/// implementation drifts fails startup instead of mis-hashing shards.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
|
||||
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
|
||||
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
|
||||
];
|
||||
|
||||
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
|
||||
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
|
||||
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
|
||||
];
|
||||
|
||||
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
|
||||
/// Highway digests above this one is externally verifiable, so it guards the
|
||||
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
|
||||
/// self-computed constants.
|
||||
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
|
||||
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
|
||||
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
|
||||
];
|
||||
|
||||
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
|
||||
/// platform and every run hashes the same 4096 bytes.
|
||||
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
|
||||
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
|
||||
for byte in payload.iter_mut() {
|
||||
state ^= state >> 12;
|
||||
state ^= state << 25;
|
||||
state ^= state >> 27;
|
||||
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
|
||||
}
|
||||
payload
|
||||
}
|
||||
|
||||
/// Why a bitrot self-test failed.
|
||||
#[derive(Debug)]
|
||||
pub enum BitrotSelfTestError {
|
||||
/// A known-answer digest mismatched the pinned constant.
|
||||
KnownAnswerMismatch {
|
||||
algorithm: &'static str,
|
||||
got: String,
|
||||
want: String,
|
||||
},
|
||||
/// A freshly encoded shard failed `bitrot_verify`.
|
||||
RoundtripVerify { algorithm: &'static str, detail: String },
|
||||
/// A verified roundtrip read back different bytes than were written.
|
||||
RoundtripReadback { algorithm: &'static str },
|
||||
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
|
||||
TamperNotRejected {
|
||||
algorithm: &'static str,
|
||||
tampered: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BitrotSelfTestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::KnownAnswerMismatch { algorithm, got, want } => {
|
||||
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
|
||||
}
|
||||
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
|
||||
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
|
||||
Self::TamperNotRejected { algorithm, tampered } => {
|
||||
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BitrotSelfTestError {}
|
||||
|
||||
fn self_test_hex(bytes: &[u8]) -> String {
|
||||
rustfs_utils::hex(bytes)
|
||||
}
|
||||
|
||||
// (kept as a named one-liner so every KAT failure site reads the same; the
|
||||
// underlying formatter is the shared `rustfs_utils::hex`)
|
||||
|
||||
/// Compare a digest against its pinned constant. Split out so a test can drive
|
||||
/// it with a wrong constant and prove the mismatch path fires.
|
||||
fn bitrot_kat_check(
|
||||
algorithm: &'static str,
|
||||
algo: &HashAlgorithm,
|
||||
payload: &[u8],
|
||||
expected: &[u8; 32],
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let digest = algo.hash_encode(payload);
|
||||
let digest = digest.as_ref();
|
||||
if digest.len() != expected.len() || digest != expected.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm,
|
||||
got: self_test_hex(digest),
|
||||
want: self_test_hex(expected),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
|
||||
/// every block back through `BitrotReader` comparing bytes.
|
||||
async fn bitrot_roundtrip_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encode failed: {err}"),
|
||||
})?;
|
||||
}
|
||||
let encoded = writer.into_inner().into_inner();
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
if encoded.len() != on_disk {
|
||||
return Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
|
||||
});
|
||||
}
|
||||
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: err.to_string(),
|
||||
})?;
|
||||
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
|
||||
let mut offset = 0usize;
|
||||
while offset < payload.len() {
|
||||
let want = shard_size.min(payload.len() - offset);
|
||||
let mut buf = vec![0u8; want];
|
||||
let read = reader
|
||||
.read(&mut buf)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("read back failed at offset {offset}: {err}"),
|
||||
})?;
|
||||
if read != want || buf[..read] != payload[offset..offset + read] {
|
||||
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
|
||||
}
|
||||
offset += read;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flip one byte and require `bitrot_verify` to reject the result.
|
||||
async fn bitrot_tamper_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
tampered: &'static str,
|
||||
flip_at: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("self-test encode should not fail");
|
||||
}
|
||||
let mut corrupt = writer.into_inner().into_inner();
|
||||
let flip_index = flip_at % corrupt.len();
|
||||
corrupt[flip_index] ^= 0x80;
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
|
||||
// The flipped byte must be rejected as a hash mismatch specifically, not
|
||||
// by any incidental read error: an in-memory cursor cannot fail reads,
|
||||
// so accepting any other failure here would mask a verify path that
|
||||
// errors out before it ever compares hashes.
|
||||
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
|
||||
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
|
||||
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("tampered shard rejected with an unexpected error: {err}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify every bitrot algorithm this crate can write or verify in production:
|
||||
/// both streaming Highway variants roundtrip end to end (encode → size formula
|
||||
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
|
||||
/// and the leading hash, while all three hashed algorithms reproduce their
|
||||
/// pinned known-answer digests.
|
||||
///
|
||||
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
|
||||
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
|
||||
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
|
||||
let payload = bitrot_self_test_payload();
|
||||
|
||||
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
|
||||
// itself, before any self-pinned constants are consulted.
|
||||
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
|
||||
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm: "SHA256",
|
||||
got: self_test_hex(abc.as_ref()),
|
||||
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
|
||||
});
|
||||
}
|
||||
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256S",
|
||||
&HashAlgorithm::HighwayHash256S,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
|
||||
)?;
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256SLegacy",
|
||||
&HashAlgorithm::HighwayHash256SLegacy,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
|
||||
)?;
|
||||
|
||||
for (algorithm, algo) in [
|
||||
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
|
||||
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
|
||||
] {
|
||||
// Full blocks plus a partial tail, exactly like a real part stripe.
|
||||
let tail_len = 2 * 1024 + 333;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
|
||||
// One flipped byte in the final data block, one in the first leading
|
||||
// hash: both must fail verification.
|
||||
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
|
||||
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test,
|
||||
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
};
|
||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||
use bytes::Bytes;
|
||||
@@ -1090,6 +1343,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_payload_is_deterministic() {
|
||||
// Two independent builds of the payload must agree byte for byte, or
|
||||
// the pinned known-answer digests below would be meaningless.
|
||||
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
|
||||
let payload = bitrot_self_test_payload();
|
||||
let wrong = [0u8; 32];
|
||||
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
|
||||
.expect_err("a zeroed digest must never match");
|
||||
match err {
|
||||
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
|
||||
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_self_test_passes() {
|
||||
bitrot_self_test()
|
||||
.await
|
||||
.expect("the pinned digests and roundtrip checks must all pass on this platform");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
||||
let mut counting = VectoredCountingWriter::default();
|
||||
@@ -1189,7 +1468,7 @@ mod tests {
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x80;
|
||||
let err = bitrot_verify(
|
||||
Cursor::new(corrupt),
|
||||
std::io::Cursor::new(corrupt),
|
||||
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
||||
data.len(),
|
||||
algo,
|
||||
@@ -1282,7 +1561,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
||||
let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut out = [0u8; 5];
|
||||
let err = reader
|
||||
.read(&mut out)
|
||||
@@ -1407,7 +1686,7 @@ mod tests {
|
||||
(HashAlgorithm::HighwayHash256, true),
|
||||
] {
|
||||
let label = format!("{algo:?}");
|
||||
let writer = Cursor::new(Vec::<u8>::new());
|
||||
let writer = std::io::Cursor::new(Vec::<u8>::new());
|
||||
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
||||
w.write(&[7u8; 16]).await.unwrap();
|
||||
let written = w.into_inner().into_inner();
|
||||
@@ -1492,7 +1771,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
w.write(payload).await.unwrap();
|
||||
w.into_inner().into_inner()
|
||||
}
|
||||
@@ -1600,7 +1879,7 @@ mod tests {
|
||||
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
||||
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
||||
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
w.write(chunk).await.unwrap();
|
||||
}
|
||||
@@ -1674,14 +1953,14 @@ mod tests {
|
||||
w.write(&data).await.expect("write shard");
|
||||
|
||||
let mut via_read = vec![0u8; SHARD];
|
||||
let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
.read(&mut via_read)
|
||||
.await
|
||||
.expect("read");
|
||||
|
||||
// A buffer with only capacity — no initialized bytes at all.
|
||||
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut via_append, SHARD)
|
||||
.await
|
||||
.expect("read_appending");
|
||||
@@ -1706,7 +1985,7 @@ mod tests {
|
||||
encoded.truncate(encoded.len() - 1);
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a truncated shard must not succeed");
|
||||
@@ -1732,7 +2011,7 @@ mod tests {
|
||||
encoded[last] ^= 0xff;
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a corrupt shard must not verify");
|
||||
@@ -1844,7 +2123,7 @@ mod tests {
|
||||
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
||||
);
|
||||
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
||||
let mut streamed = Cursor::new(encoded.clone());
|
||||
let mut streamed = std::io::Cursor::new(encoded.clone());
|
||||
assert!(
|
||||
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
||||
"a non-Bytes source must stay on the streaming path"
|
||||
@@ -1872,7 +2151,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut via_stream, SHARD)
|
||||
.await
|
||||
.expect("streaming read");
|
||||
|
||||
@@ -704,6 +704,7 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn create_deferred_bitrot_reader(
|
||||
inline_data: Option<Bytes>,
|
||||
disk: Option<DiskStore>,
|
||||
|
||||
@@ -180,11 +180,13 @@ pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome {
|
||||
Fallback(GetCodecStreamingFallbackReason),
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) struct MultipartCodecStreamingReader {
|
||||
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl MultipartCodecStreamingReader {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
|
||||
Self {
|
||||
readers: VecDeque::from(readers),
|
||||
@@ -666,6 +668,60 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
parts_metadata: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<&'static str> {
|
||||
if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) {
|
||||
return Some(reason);
|
||||
}
|
||||
|
||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||
candidate.erasure.data_blocks,
|
||||
candidate.erasure.parity_blocks,
|
||||
candidate.erasure.block_size,
|
||||
candidate.uses_legacy_checksum,
|
||||
) else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||
};
|
||||
let data_files =
|
||||
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
}) {
|
||||
Ok(data_files) => data_files,
|
||||
Err(reason) => return Some(reason),
|
||||
};
|
||||
|
||||
let Some(part) = candidate.parts.first() else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||
};
|
||||
let Ok(object_size) = usize::try_from(candidate.size) else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||
};
|
||||
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
||||
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||
HashAlgorithm::HighwayHash256SLegacy
|
||||
} else {
|
||||
checksum_info.algorithm
|
||||
};
|
||||
let read_length = inline_erasure_shard_file_offset(
|
||||
0,
|
||||
object_size,
|
||||
object_size,
|
||||
candidate.erasure.block_size,
|
||||
erasure.data_shards,
|
||||
candidate.uses_legacy_checksum,
|
||||
);
|
||||
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum);
|
||||
let Ok(mut readers) =
|
||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||
else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
|
||||
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
||||
Some(body) if body.len() == object_size => None,
|
||||
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||
}
|
||||
}
|
||||
|
||||
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
|
||||
// `inline_data` excludes remote objects; this diagnostic reports them separately.
|
||||
if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE);
|
||||
@@ -703,51 +759,7 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
|
||||
}
|
||||
|
||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||
candidate.erasure.data_blocks,
|
||||
candidate.erasure.parity_blocks,
|
||||
candidate.erasure.block_size,
|
||||
candidate.uses_legacy_checksum,
|
||||
) else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||
};
|
||||
let data_files =
|
||||
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
}) {
|
||||
Ok(data_files) => data_files,
|
||||
Err(reason) => return Some(reason),
|
||||
};
|
||||
|
||||
let Some(part) = candidate.parts.first() else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||
};
|
||||
let checksum_info = candidate.erasure.get_checksum_info(part.number);
|
||||
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||
HashAlgorithm::HighwayHash256SLegacy
|
||||
} else {
|
||||
checksum_info.algorithm
|
||||
};
|
||||
let read_length = inline_erasure_shard_file_offset(
|
||||
0,
|
||||
object_size,
|
||||
object_size,
|
||||
candidate.erasure.block_size,
|
||||
erasure.data_shards,
|
||||
candidate.uses_legacy_checksum,
|
||||
);
|
||||
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum);
|
||||
let Ok(mut readers) =
|
||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||
else {
|
||||
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
|
||||
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
||||
Some(body) if body.len() == object_size => None,
|
||||
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn data_read_inline_missing_shards_are_pending(
|
||||
@@ -1836,6 +1848,7 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards(
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
@@ -2126,6 +2139,7 @@ pub(in crate::set_disk) async fn create_data_block_bitrot_readers(
|
||||
setup
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
@@ -2442,6 +2456,7 @@ impl SetDisks {
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||
let disk = disk.clone();
|
||||
let task_opts = opts;
|
||||
@@ -2449,10 +2464,14 @@ impl SetDisks {
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
tokio::spawn(async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
@@ -2548,6 +2567,7 @@ impl SetDisks {
|
||||
let mut scheduled_count = 0usize;
|
||||
let mut force_full_wait = false;
|
||||
let mut final_miss_reason_override = None;
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
let spawn_read_version =
|
||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||
let task_opts = opts;
|
||||
@@ -2555,6 +2575,7 @@ impl SetDisks {
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
join_set.spawn(async move {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
@@ -2563,6 +2584,9 @@ impl SetDisks {
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
@@ -2596,6 +2620,14 @@ impl SetDisks {
|
||||
Ok(file_info) => {
|
||||
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
||||
accumulator.observe_file_info(&file_info);
|
||||
if bounded_fanout
|
||||
&& read_data
|
||||
&& !force_full_wait
|
||||
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
|
||||
{
|
||||
force_full_wait = true;
|
||||
final_miss_reason_override.get_or_insert(reason);
|
||||
}
|
||||
if let Some(slot) = ress.get_mut(index) {
|
||||
*slot = file_info;
|
||||
}
|
||||
@@ -2955,6 +2987,7 @@ impl SetDisks {
|
||||
(meta_file_infos, errs)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) async fn read_multiple_files(
|
||||
disks: &[Option<DiskStore>],
|
||||
req: ReadMultipleReq,
|
||||
@@ -2988,14 +3021,11 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||
let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||
Ok(collected) => collected,
|
||||
Err(()) => return empty_quorum_result(),
|
||||
};
|
||||
|
||||
// debug!("ReadMultipleResp ress {:?}", ress);
|
||||
// debug!("ReadMultipleResp errors {:?}", errors);
|
||||
|
||||
let mut ret = Vec::with_capacity(req.files.len());
|
||||
|
||||
for want in req.files.iter() {
|
||||
@@ -3134,6 +3164,7 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
type RenameDataLegacyTuple = (
|
||||
Vec<Option<DiskStore>>,
|
||||
RenameConvergence,
|
||||
@@ -3143,6 +3174,7 @@ type RenameDataLegacyTuple = (
|
||||
);
|
||||
|
||||
impl RenameDataCommit {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
|
||||
(
|
||||
self.online_disks,
|
||||
@@ -3261,6 +3293,7 @@ impl SetDisks {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||
#[allow(clippy::type_complexity)]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) async fn rename_data(
|
||||
disks: &[Option<DiskStore>],
|
||||
src_bucket: &str,
|
||||
@@ -5073,6 +5106,7 @@ fn is_cleanup_not_found(e: &DiskError) -> bool {
|
||||
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
|
||||
/// must not be silently swallowed as an ignorable error (fixes the historical
|
||||
/// `Unexpected`/`DiskNotFound` misclassification).
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
|
||||
match joined {
|
||||
Ok(res) => res,
|
||||
@@ -5297,6 +5331,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
|
||||
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
||||
pub const CLEANUP: &str = "cleanup";
|
||||
/// The per-disk `read_version` phase of metadata read fan-out.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub const READ_VERSION: &str = "read_version";
|
||||
}
|
||||
|
||||
@@ -5734,6 +5769,130 @@ mod tests {
|
||||
(dirs, disks)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
|| {
|
||||
assert_eq!(
|
||||
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
|
||||
Some(Duration::from_millis(25))
|
||||
);
|
||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
|
||||
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
|
||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
|
||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
|
||||
],
|
||||
|| {
|
||||
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "metadata-slowtail-fault-bucket";
|
||||
let object = "objects/metadata-slowtail-fault-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let read_without_data =
|
||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
|
||||
tokio::time::timeout(Duration::from_millis(100), read_without_data)
|
||||
.await
|
||||
.expect("non-data metadata fanout must not be delayed by the data-read slowtail hook")
|
||||
.expect("metadata fanout without read_data should resolve");
|
||||
|
||||
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||
.await
|
||||
.is_err(),
|
||||
"data-read metadata fanout must wait for the injected slow read_version response"
|
||||
);
|
||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||
.await
|
||||
.expect("injected slowtail should eventually complete")
|
||||
.expect("data-read metadata fanout should resolve");
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "metadata-slowtail-early-stop-bucket";
|
||||
let object = "objects/metadata-slowtail-early-stop-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||
.await
|
||||
.is_err(),
|
||||
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
|
||||
);
|
||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||
.await
|
||||
.expect("injected early-stop slowtail should eventually complete")
|
||||
.expect("early-stop metadata fanout should resolve");
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
||||
///
|
||||
/// The metadata fan-out issues each `read_version` inside its own
|
||||
@@ -7081,7 +7240,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() {
|
||||
async fn bounded_non_inline_data_get_immediately_forces_full_fanout() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "bounded-data-get-hedge-bucket";
|
||||
let object = "bounded-data-get-hedge-object";
|
||||
@@ -7112,7 +7271,7 @@ mod tests {
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded data-read fanout should hedge by starting the spare disk");
|
||||
.expect("bounded non-inline data-read fanout should immediately schedule the spare disk");
|
||||
|
||||
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||
assert!(
|
||||
@@ -7128,7 +7287,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
DISKS as u64,
|
||||
"bounded data-read fanout should issue the paused disk plus one spare hedge"
|
||||
"bounded non-inline data-read fanout should issue the paused disk plus the remaining spare"
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
@@ -7155,16 +7314,42 @@ mod tests {
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||
],
|
||||
async {
|
||||
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let (parts_metadata, errs, diagnostics) =
|
||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
|
||||
let disks_for_read = disks.clone();
|
||||
let mut read = tokio::spawn(async move {
|
||||
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
|
||||
.await
|
||||
.expect("default data-read metadata should resolve");
|
||||
});
|
||||
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("default bounded non-inline read should schedule the paused metadata task");
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect(
|
||||
"default bounded non-inline read should immediately force full fanout after the first non-inline response",
|
||||
);
|
||||
|
||||
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||
assert!(
|
||||
pending.is_err(),
|
||||
"default non-inline data reads must not return before the paused metadata response"
|
||||
);
|
||||
barrier.release();
|
||||
let (parts_metadata, errs, diagnostics) = read
|
||||
.await
|
||||
.expect("metadata read task should not panic")
|
||||
.expect("default data-read metadata should resolve");
|
||||
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
DISKS as u64,
|
||||
"default non-inline GET data-read metadata must keep full fanout for read-failure tolerance"
|
||||
"default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first"
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
|
||||
@@ -42,12 +42,20 @@ impl<'a> SetDisksCtx<'a> {
|
||||
}
|
||||
|
||||
/// The borrowed core, for state not yet fronted by a typed accessor.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn core(&self) -> &'a SetDisks {
|
||||
self.core
|
||||
}
|
||||
|
||||
// --- Immutable topology / config (fixed after construction) ---
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn set_index(&self) -> usize {
|
||||
self.core.set_index
|
||||
}
|
||||
@@ -56,14 +64,26 @@ impl<'a> SetDisksCtx<'a> {
|
||||
self.core.pool_index
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn set_drive_count(&self) -> usize {
|
||||
self.core.set_drive_count
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn default_parity_count(&self) -> usize {
|
||||
self.core.default_parity_count
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
|
||||
&self.core.set_endpoints
|
||||
}
|
||||
@@ -72,6 +92,10 @@ impl<'a> SetDisksCtx<'a> {
|
||||
&self.core.format
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn locker_owner(&self) -> &'a str {
|
||||
&self.core.locker_owner
|
||||
}
|
||||
@@ -84,6 +108,10 @@ impl<'a> SetDisksCtx<'a> {
|
||||
|
||||
// --- Locker trio ---
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
|
||||
&self.core.lockers
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@
|
||||
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
||||
|
||||
// #730: SetDisks still hosts staged read/heal/write migration helpers.
|
||||
#![allow(dead_code)]
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
|
||||
@@ -103,9 +102,7 @@ use crate::storage_api_contracts::{
|
||||
};
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
use crate::{
|
||||
bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts,
|
||||
},
|
||||
bucket::lifecycle::bucket_lifecycle_ops::{LifecycleOps, get_transitioned_object_reader_with_tier_manager, put_restore_opts},
|
||||
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
|
||||
config::storageclass,
|
||||
disk::{
|
||||
@@ -177,15 +174,14 @@ use std::future::Future;
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::mem::{self};
|
||||
use std::pin::Pin;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
io::{Cursor, Write},
|
||||
path::Path,
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
@@ -624,7 +620,9 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
|
||||
// Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout.
|
||||
// ============================================================================
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
||||
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
|
||||
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
|
||||
@@ -698,7 +696,15 @@ const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EAR
|
||||
// the env var to `false` to fall back to full-wait metadata fanout.
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||
)]
|
||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100;
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE";
|
||||
@@ -708,7 +714,12 @@ const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_ME
|
||||
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true;
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false;
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true;
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS";
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET";
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX";
|
||||
|
||||
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
|
||||
|
||||
@@ -910,6 +921,10 @@ mod prepared_get_object_metadata_tests {
|
||||
.expect("test should find an object whose initial fanout covers both data shards")
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
|
||||
)]
|
||||
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||
.get(3)
|
||||
@@ -1110,7 +1125,10 @@ mod prepared_get_object_metadata_tests {
|
||||
|
||||
assert_eq!(object_size, payload.len() as i64);
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(calls_total, 4, "default production GET should eagerly schedule the full metadata fanout");
|
||||
assert_eq!(
|
||||
calls_total, 4,
|
||||
"default production inline GET should schedule the initial bounded quorum plus one hedge"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_scheduled",
|
||||
@@ -1684,6 +1702,95 @@ fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GetMetadataSlowtailFaultConfig {
|
||||
delay: Duration,
|
||||
disks: Arc<[usize]>,
|
||||
bucket: Option<String>,
|
||||
object_prefix: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct GetMetadataSlowtailFaultRequest {
|
||||
delay: Duration,
|
||||
disks: Arc<[usize]>,
|
||||
}
|
||||
|
||||
impl GetMetadataSlowtailFaultRequest {
|
||||
fn delay_for_disk(&self, disk_index: usize) -> Option<Duration> {
|
||||
self.disks.contains(&disk_index).then_some(self.delay)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_get_metadata_slowtail_fault_disks(raw: &str) -> Option<Vec<usize>> {
|
||||
let mut disks = Vec::new();
|
||||
for item in raw.split(',').map(str::trim).filter(|item| !item.is_empty()) {
|
||||
let Ok(index) = item.parse::<usize>() else {
|
||||
return None;
|
||||
};
|
||||
if !disks.contains(&index) {
|
||||
disks.push(index);
|
||||
}
|
||||
}
|
||||
(!disks.is_empty()).then_some(disks)
|
||||
}
|
||||
|
||||
fn load_get_metadata_slowtail_fault_config() -> Option<GetMetadataSlowtailFaultConfig> {
|
||||
let delay_ms = rustfs_utils::get_env_u64(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, 0);
|
||||
if delay_ms == 0 {
|
||||
return None;
|
||||
}
|
||||
let disks = parse_get_metadata_slowtail_fault_disks(&std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS).ok()?)?;
|
||||
let bucket = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET)
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty());
|
||||
let object_prefix = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX)
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty());
|
||||
Some(GetMetadataSlowtailFaultConfig {
|
||||
delay: Duration::from_millis(delay_ms),
|
||||
disks: Arc::from(disks.into_boxed_slice()),
|
||||
bucket,
|
||||
object_prefix,
|
||||
})
|
||||
}
|
||||
|
||||
fn get_metadata_slowtail_fault_request(bucket: &str, object: &str, read_data: bool) -> Option<GetMetadataSlowtailFaultRequest> {
|
||||
if !read_data {
|
||||
return None;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let config = load_get_metadata_slowtail_fault_config();
|
||||
#[cfg(test)]
|
||||
let config = config.as_ref()?;
|
||||
#[cfg(not(test))]
|
||||
let config = ({
|
||||
static CACHED: OnceLock<Option<GetMetadataSlowtailFaultConfig>> = OnceLock::new();
|
||||
CACHED.get_or_init(load_get_metadata_slowtail_fault_config).as_ref()
|
||||
})?;
|
||||
|
||||
if let Some(expected_bucket) = &config.bucket
|
||||
&& expected_bucket != bucket
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if let Some(expected_prefix) = &config.object_prefix
|
||||
&& !object.starts_with(expected_prefix)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(GetMetadataSlowtailFaultRequest {
|
||||
delay: config.delay,
|
||||
disks: config.disks.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn get_metadata_slowtail_fault_delay(bucket: &str, object: &str, disk_index: usize, read_data: bool) -> Option<Duration> {
|
||||
get_metadata_slowtail_fault_request(bucket, object, read_data)?.delay_for_disk(disk_index)
|
||||
}
|
||||
|
||||
/// Check if multipart reads prefetch the next part's bitrot reader setup
|
||||
/// while the current part decodes (backlog#870).
|
||||
///
|
||||
@@ -1709,6 +1816,10 @@ fn is_multipart_reader_setup_prefetch_enabled() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||
)]
|
||||
fn get_metadata_early_stop_rollout_pct() -> u32 {
|
||||
static CACHED: OnceLock<u32> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| {
|
||||
@@ -1748,6 +1859,10 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj
|
||||
}
|
||||
|
||||
/// Should this specific request use metadata early-stop?
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
|
||||
)]
|
||||
pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool {
|
||||
let base = is_get_metadata_early_stop_enabled();
|
||||
let pct = get_metadata_early_stop_rollout_pct();
|
||||
@@ -2181,6 +2296,7 @@ fn classify_get_codec_streaming_object_class(
|
||||
GetCodecStreamingObjectClass::PlainSinglePart
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn is_get_small_object_direct_memory_eligible_with_threshold(
|
||||
range: &Option<HTTPRangeSpec>,
|
||||
object_info: &ObjectInfo,
|
||||
@@ -2786,6 +2902,7 @@ pub struct SetDisks {
|
||||
/// Stable namespace shared by every object lock created for this set.
|
||||
set_lock_namespace: Arc<str>,
|
||||
pub format: FormatV3,
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
||||
@@ -3061,11 +3178,13 @@ struct GetObjectMetadataCacheEntry {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct DiskHealthEntry {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
last_check: Instant,
|
||||
online: bool,
|
||||
}
|
||||
|
||||
impl DiskHealthEntry {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn cached_value(&self) -> Option<bool> {
|
||||
if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL {
|
||||
Some(self.online)
|
||||
@@ -3659,6 +3778,7 @@ fn multipart_put_large_batch_min_size_bytes() -> usize {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
|
||||
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
|
||||
SmallWritePath::Inline
|
||||
@@ -4237,6 +4357,7 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj
|
||||
///
|
||||
/// Fail closed: when bucket metadata cannot be resolved the check stays on, so
|
||||
/// object-lock protection is never skipped because of a metadata lookup miss.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool {
|
||||
bucket_meta.is_none_or(|meta| meta.object_locking())
|
||||
}
|
||||
@@ -4512,15 +4633,6 @@ impl Hash for ObjProps {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Debug)]
|
||||
pub struct HealEntryResult {
|
||||
pub bytes: usize,
|
||||
pub success: bool,
|
||||
pub skipped: bool,
|
||||
pub entry_done: bool,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
fn is_object_dangling(
|
||||
meta_arr: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
@@ -5297,6 +5409,7 @@ pub fn is_valid_storage_class(storage_class: &str) -> bool {
|
||||
}
|
||||
|
||||
/// Returns true if the storage class is a cold storage tier that requires special handling
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
||||
matches!(
|
||||
storage_class,
|
||||
@@ -5305,6 +5418,7 @@ pub fn is_cold_storage_class(storage_class: &str) -> bool {
|
||||
}
|
||||
|
||||
/// Returns true if the storage class is an infrequent access tier
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_infrequent_access_class(storage_class: &str) -> bool {
|
||||
matches!(
|
||||
storage_class,
|
||||
|
||||
@@ -453,7 +453,9 @@ impl SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let write_lock_guard = if !opts.no_lock {
|
||||
// Bound, not `_`: this guard must live to the end of the scope. A bare
|
||||
// `_` would drop it here and release the namespace write lock.
|
||||
let _write_lock_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(
|
||||
ns_lock
|
||||
@@ -996,7 +998,7 @@ impl SetDisks {
|
||||
readers.push(None);
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
Err(_e) => {
|
||||
readers.push(None);
|
||||
continue;
|
||||
}
|
||||
@@ -1545,6 +1547,9 @@ impl SetDisks {
|
||||
|
||||
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
|
||||
for (disk_index, disk) in disks.iter().enumerate() {
|
||||
// Only the #[cfg(test)] fault-injection branch below reads this.
|
||||
#[cfg(not(test))]
|
||||
let _ = disk_index;
|
||||
let Some(disk) = disk else {
|
||||
return Ok(DanglingDeleteSafety::UnsafeToDelete);
|
||||
};
|
||||
@@ -1716,6 +1721,10 @@ impl SetDisks {
|
||||
Ok((result, None))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "lock-taking wrapper over the live heal_object_dir_locked; only comments reference it (backlog#1823)"
|
||||
)]
|
||||
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
|
||||
pub(in crate::set_disk) async fn heal_object_dir(
|
||||
&self,
|
||||
|
||||
@@ -66,6 +66,7 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||
match err {
|
||||
LockResult::Timeout => {
|
||||
@@ -79,6 +80,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn format_lock_error_from_error(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -143,6 +145,7 @@ impl SetDisks {
|
||||
disks
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
|
||||
let snapshot = self.drive_membership_snapshot().await;
|
||||
let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
|
||||
@@ -153,6 +156,10 @@ impl SetDisks {
|
||||
disks
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "local-only sibling of the test-covered get_online_disks; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
|
||||
let snapshot = self.drive_membership_snapshot().await;
|
||||
let mut disks = snapshot
|
||||
@@ -432,6 +439,10 @@ impl SetDisks {
|
||||
Ok((disk, fm))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity healing-disk accessor with no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(in crate::set_disk) async fn get_online_disk_with_healing(
|
||||
&self,
|
||||
incl_healing: bool,
|
||||
@@ -440,6 +451,10 @@ impl SetDisks {
|
||||
Ok((new_disks, healing > 0))
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reached only from get_online_disk_with_healing, itself uncalled in this port (backlog#1823)"
|
||||
)]
|
||||
pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info(
|
||||
&self,
|
||||
incl_healing: bool,
|
||||
|
||||
@@ -415,6 +415,7 @@ fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize
|
||||
/// never returned, but flips `is_truncated` to `true` and yields a
|
||||
/// `next_upload_id_marker` pointing at the last returned upload so the caller can
|
||||
/// resume paging.
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) {
|
||||
let is_truncated = remaining.len() > max_uploads;
|
||||
let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect();
|
||||
@@ -557,6 +558,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub(super) async fn check_upload_id_exists(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1398,7 +1400,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let mut count = max_parts;
|
||||
|
||||
for (i, part) in object_parts.iter().enumerate() {
|
||||
for part in object_parts.iter() {
|
||||
if let Some(err) = &part.error {
|
||||
warn!("list_object_parts part error: {:?}", &err);
|
||||
}
|
||||
@@ -2041,8 +2043,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
|
||||
{
|
||||
error!(
|
||||
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
|
||||
p.part_num, bucket, object
|
||||
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}, err={}",
|
||||
p.part_num, bucket, object, err
|
||||
);
|
||||
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
||||
}
|
||||
@@ -2087,8 +2089,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
} else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) {
|
||||
error!(
|
||||
"complete_multipart_upload checksum matches failed want={}, got={}",
|
||||
wtcs.encoded, checksum.encoded
|
||||
"complete_multipart_upload checksum matches failed want={}, got={}, err={}",
|
||||
wtcs.encoded, checksum.encoded, err
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"complete_multipart_upload checksum matches failed want={}, got={}",
|
||||
|
||||
@@ -3507,6 +3507,10 @@ struct TransitionUploadedSaveProbeState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
struct TransitionUploadedSaveProbe {
|
||||
state: Arc<TransitionUploadedSaveProbeState>,
|
||||
}
|
||||
@@ -3517,6 +3521,10 @@ static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Opti
|
||||
|
||||
#[cfg(test)]
|
||||
impl TransitionUploadedSaveProbe {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionUploadedSaveProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -3533,6 +3541,10 @@ impl TransitionUploadedSaveProbe {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
fn attempts(&self) -> usize {
|
||||
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
@@ -3738,6 +3750,10 @@ struct TransitionCommitBarrierState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
struct TransitionCommitBarrier {
|
||||
state: Arc<TransitionCommitBarrierState>,
|
||||
}
|
||||
@@ -3748,14 +3764,26 @@ static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Ar
|
||||
|
||||
#[cfg(test)]
|
||||
impl TransitionCommitBarrier {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self {
|
||||
Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
fn install_after_lease_check(bucket: &str, object: &str) -> Self {
|
||||
Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation)
|
||||
}
|
||||
@@ -3778,12 +3806,20 @@ impl TransitionCommitBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("transition should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
@@ -5620,7 +5656,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// TODO: Lifecycle
|
||||
|
||||
let mut version_found = true;
|
||||
let (mut goi, write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
|
||||
// delete_object_version below derives its own majority quorum from the
|
||||
// disk array, so the object-derived quorum here is unused.
|
||||
let (mut goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
|
||||
if let Some(err) = &gerr
|
||||
&& goi.name.is_empty()
|
||||
{
|
||||
@@ -6374,7 +6412,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
|
||||
for disk in disks.iter() {
|
||||
if let Some(disk) = disk {
|
||||
if disk.is_some() {
|
||||
continue;
|
||||
}
|
||||
let _ = self
|
||||
@@ -10170,6 +10208,288 @@ mod transition_upload_integrity_tests {
|
||||
assert!(backend.contains(remote_object).await, "committed remote object should remain available");
|
||||
}
|
||||
|
||||
/// Compresses `plaintext` with the codec the PUT path uses, so the stored
|
||||
/// bytes round-trip through the read path's decompressor.
|
||||
async fn compress_for_storage(plaintext: &[u8]) -> Vec<u8> {
|
||||
let mut reader = crate::io_support::rio::compression_reader(
|
||||
Cursor::new(plaintext.to_vec()),
|
||||
rustfs_utils::CompressionAlgorithm::default(),
|
||||
false,
|
||||
);
|
||||
let mut compressed = Vec::new();
|
||||
reader.read_to_end(&mut compressed).await.expect("plaintext should compress");
|
||||
assert!(compressed.len() < plaintext.len(), "test payload must actually compress");
|
||||
compressed
|
||||
}
|
||||
|
||||
/// Writes a genuinely compressed object: stored data is `compressed`, and the
|
||||
/// metadata marks it compressed with the plaintext length as its actual size,
|
||||
/// exactly as the app-layer compress path records it.
|
||||
async fn write_compressed_source(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
disk_stores: &[DiskStore],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
plaintext: &[u8],
|
||||
compressed: &[u8],
|
||||
) -> ObjectInfo {
|
||||
for disk in disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let mut user_defined = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
|
||||
);
|
||||
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
|
||||
let stream = crate::io_support::rio::HashReader::from_stream(
|
||||
Cursor::new(compressed.to_vec()),
|
||||
compressed.len() as i64,
|
||||
plaintext.len() as i64,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("hash reader over compressed bytes");
|
||||
let mut reader = PutObjReader::new(stream);
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("compressed object should be written")
|
||||
}
|
||||
|
||||
async fn read_transitioned(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
) -> (Vec<u8>, i64) {
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
|
||||
.await
|
||||
.expect("transitioned object reader should open");
|
||||
let published_size = reader.object_info.size;
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("transitioned body should drain");
|
||||
(body, published_size)
|
||||
}
|
||||
|
||||
/// Transition uploads the object's STORED bytes, so a tiered read has to
|
||||
/// apply the same transform an erasure read would. #6107 routed this path
|
||||
/// through `ReadPlan` to stop serving an encrypted object's ciphertext;
|
||||
/// compression rides the same plan, and nothing pinned it (backlog#1851).
|
||||
/// Without the transform this GET returns the compressed bytes under the
|
||||
/// compressed size — silent corruption for every client of a compressed
|
||||
/// object that ILM has moved to a warm tier.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn transitioned_compressed_object_get_returns_plaintext() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transitioned-compressed-get-bucket";
|
||||
let object = "object.txt";
|
||||
let plaintext = b"transitioned compressed objects must decompress on read ".repeat(20_000);
|
||||
let compressed = compress_for_storage(&plaintext).await;
|
||||
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
||||
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (local_body, local_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
||||
assert_eq!(local_body, plaintext, "control: the pre-transition read must decompress");
|
||||
assert_eq!(
|
||||
local_size,
|
||||
plaintext.len() as i64,
|
||||
"control: the pre-transition read publishes the plaintext size"
|
||||
);
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("transition should commit");
|
||||
|
||||
let put_versions = backend.put_versions().await;
|
||||
assert_eq!(put_versions.len(), 1, "transition should upload one remote candidate");
|
||||
let remote_bytes = backend
|
||||
.bytes(&put_versions[0].0)
|
||||
.await
|
||||
.expect("remote candidate should be stored");
|
||||
assert_eq!(
|
||||
remote_bytes, compressed,
|
||||
"transition uploads the stored representation; the read side is what has to decode it"
|
||||
);
|
||||
|
||||
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
||||
assert_eq!(body, plaintext, "a tiered read must return the object's content, not its stored bytes");
|
||||
assert_eq!(
|
||||
published_size,
|
||||
plaintext.len() as i64,
|
||||
"a tiered read must publish the plaintext size, not the compressed one"
|
||||
);
|
||||
}
|
||||
|
||||
/// A ranged tiered read is expressed in plaintext coordinates, so the plan
|
||||
/// has to translate it into the remote copy's compressed extent and skip
|
||||
/// into the decompressed stream — the same translation the erasure path does.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn transitioned_compressed_object_range_get_returns_plaintext_slice() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transitioned-compressed-range-bucket";
|
||||
let object = "object.txt";
|
||||
let plaintext = b"ranged reads of transitioned compressed objects must land in plaintext ".repeat(20_000);
|
||||
let compressed = compress_for_storage(&plaintext).await;
|
||||
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("transition should commit");
|
||||
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
// Deliberately past the compressed size, so a range still measured in
|
||||
// stored coordinates could not produce this slice.
|
||||
let start = compressed.len() as i64 + 4096;
|
||||
let end = start + 511;
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start,
|
||||
end,
|
||||
};
|
||||
let (body, published_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
|
||||
|
||||
let expected = &plaintext[start as usize..=end as usize];
|
||||
assert_eq!(body, expected, "a ranged tiered read must return that plaintext slice");
|
||||
assert_eq!(published_size, expected.len() as i64, "a ranged tiered read publishes the slice length");
|
||||
}
|
||||
|
||||
/// The restore copy-back re-writes the object under its original metadata,
|
||||
/// which still says "compressed". It therefore has to keep receiving the
|
||||
/// STORED bytes: `restore_request_active` holds it on the plan's `Plain`
|
||||
/// branch, and decompressing there would write plaintext under compressed
|
||||
/// metadata.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn restore_read_of_transitioned_compressed_object_keeps_stored_bytes() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transitioned-compressed-restore-bucket";
|
||||
let object = "object.txt";
|
||||
let plaintext = b"restore copy-back must keep the stored representation intact ".repeat(20_000);
|
||||
let compressed = compress_for_storage(&plaintext).await;
|
||||
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("transition should commit");
|
||||
|
||||
let oi = set_disks
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("transitioned metadata should resolve");
|
||||
let restore_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
part_number: Some(1),
|
||||
transition: TransitionOptions {
|
||||
restore_request: s3s::dto::RestoreRequest {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
||||
bucket,
|
||||
object,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&oi,
|
||||
&restore_opts,
|
||||
&set_disks.ctx.tier_config_mgr(),
|
||||
set_disks.ctx.object_encryption_resolver(),
|
||||
)
|
||||
.await
|
||||
.expect("restore read of the tiered copy should open");
|
||||
let published_size = reader.object_info.size;
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("restore body should drain");
|
||||
|
||||
assert_eq!(body, compressed, "a restore read must copy the stored bytes back verbatim");
|
||||
assert_eq!(
|
||||
published_size,
|
||||
compressed.len() as i64,
|
||||
"a restore read must keep publishing the stored size"
|
||||
);
|
||||
}
|
||||
|
||||
/// Plain objects must keep streaming the remote bytes through untouched:
|
||||
/// their plan is `Plain`, so the tiered read stays byte-identical.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn transitioned_plain_object_get_is_unchanged() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transitioned-plain-get-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"plain transitioned objects must keep reading back byte-identical ".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("transition should commit");
|
||||
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
|
||||
assert_eq!(body, payload);
|
||||
assert_eq!(published_size, payload.len() as i64);
|
||||
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 100,
|
||||
end: 611,
|
||||
};
|
||||
let (ranged_body, ranged_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
|
||||
assert_eq!(ranged_body, &payload[100..=611]);
|
||||
assert_eq!(ranged_size, payload.len() as i64, "a plain ranged read keeps publishing the object size");
|
||||
}
|
||||
|
||||
async fn corrupt_beyond_read_quorum(
|
||||
temp_dirs: &[tempfile::TempDir],
|
||||
bucket: &str,
|
||||
|
||||
@@ -116,6 +116,7 @@ impl SetDisks {
|
||||
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||
MetadataCacheLookup::Hit(entry) => Some(entry),
|
||||
@@ -1826,6 +1827,7 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp
|
||||
.then_some(GET_METADATA_CACHE_REASON_META_BUCKET)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool {
|
||||
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
|
||||
}
|
||||
@@ -3935,7 +3937,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_early_stop_bounded_fanout_defaults_to_disabled() {
|
||||
fn metadata_early_stop_bounded_fanout_defaults_to_enabled() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
@@ -3944,20 +3946,20 @@ mod tests {
|
||||
],
|
||||
|| {
|
||||
assert!(is_get_metadata_data_read_early_stop_enabled());
|
||||
assert!(!is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||
},
|
||||
);
|
||||
temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true"))], || {
|
||||
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||
temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false"))], || {
|
||||
assert!(!is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||
});
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
|
||||
],
|
||||
|| {
|
||||
assert!(!is_get_metadata_data_read_early_stop_enabled());
|
||||
assert!(!is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -487,22 +487,21 @@ pub fn record_get_object_completion(total_duration_secs: f64, response_size_byte
|
||||
|
||||
/// Record the streaming strategy chosen for a GetObject response body.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_stream_strategy(strategy: &str, buffer_size_bytes: usize, response_size_bytes: i64) {
|
||||
pub fn record_get_object_stream_strategy(strategy: &'static str, buffer_size_bytes: usize, response_size_bytes: i64) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy.to_string()).increment(1);
|
||||
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy.to_string())
|
||||
.record(usize_to_f64(buffer_size_bytes));
|
||||
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy.to_string())
|
||||
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy).increment(1);
|
||||
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy).record(usize_to_f64(buffer_size_bytes));
|
||||
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy)
|
||||
.record(i64_non_negative_to_f64(response_size_bytes));
|
||||
}
|
||||
|
||||
/// Record the response-body handoff shape from a GetObject reader into the S3 streaming body.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_response_handoff(
|
||||
strategy: &str,
|
||||
buffer_source: &str,
|
||||
strategy: &'static str,
|
||||
buffer_source: &'static str,
|
||||
buffer_size_bytes: usize,
|
||||
response_size_bytes: i64,
|
||||
duration_secs: f64,
|
||||
@@ -512,26 +511,26 @@ pub fn record_get_object_response_handoff(
|
||||
}
|
||||
counter!(
|
||||
"rustfs_io_get_object_response_handoff_total",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string()
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source
|
||||
)
|
||||
.increment(1);
|
||||
histogram!(
|
||||
"rustfs_io_get_object_response_handoff_buffer_size_bytes",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string()
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source
|
||||
)
|
||||
.record(usize_to_f64(buffer_size_bytes));
|
||||
histogram!(
|
||||
"rustfs_io_get_object_response_handoff_response_size_bytes",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string()
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source
|
||||
)
|
||||
.record(i64_non_negative_to_f64(response_size_bytes));
|
||||
histogram!(
|
||||
"rustfs_io_get_object_response_handoff_duration_seconds",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string()
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source
|
||||
)
|
||||
.record(duration_secs);
|
||||
record_get_object_response_handoff_duration("s3_handler", duration_secs);
|
||||
@@ -539,14 +538,18 @@ pub fn record_get_object_response_handoff(
|
||||
|
||||
/// Record ReaderStream capacity chosen for GetObject handoff.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source: &str, buffer_size_bytes: usize) {
|
||||
pub fn record_get_object_reader_stream_buffer_size(
|
||||
strategy: &'static str,
|
||||
buffer_source: &'static str,
|
||||
buffer_size_bytes: usize,
|
||||
) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
histogram!(
|
||||
"rustfs_io_get_object_reader_stream_buffer_size_bytes",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string()
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source
|
||||
)
|
||||
.record(usize_to_f64(buffer_size_bytes));
|
||||
}
|
||||
@@ -554,8 +557,8 @@ pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source
|
||||
/// Record ReaderStream poll outcomes for GetObject handoff attribution.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_reader_stream_poll(
|
||||
strategy: &str,
|
||||
buffer_source: &str,
|
||||
strategy: &'static str,
|
||||
buffer_source: &'static str,
|
||||
outcome: &'static str,
|
||||
remaining_before: usize,
|
||||
bytes: usize,
|
||||
@@ -567,36 +570,36 @@ pub fn record_get_object_reader_stream_poll(
|
||||
let bytes = u64::try_from(bytes).unwrap_or(u64::MAX);
|
||||
counter!(
|
||||
"rustfs_io_get_object_reader_stream_poll_total",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string(),
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source,
|
||||
"outcome" => outcome
|
||||
)
|
||||
.increment(1);
|
||||
counter!(
|
||||
"rustfs_io_get_object_reader_stream_poll_bytes_total",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string(),
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source,
|
||||
"outcome" => outcome
|
||||
)
|
||||
.increment(bytes);
|
||||
histogram!(
|
||||
"rustfs_io_get_object_reader_stream_poll_remaining_bytes",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string(),
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source,
|
||||
"outcome" => outcome
|
||||
)
|
||||
.record(usize_to_f64(remaining_before));
|
||||
histogram!(
|
||||
"rustfs_io_get_object_reader_stream_poll_bytes",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string(),
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source,
|
||||
"outcome" => outcome
|
||||
)
|
||||
.record(usize_to_f64(bytes as usize));
|
||||
histogram!(
|
||||
"rustfs_io_get_object_reader_stream_poll_duration_seconds",
|
||||
"strategy" => strategy.to_string(),
|
||||
"buffer_source" => buffer_source.to_string(),
|
||||
"strategy" => strategy,
|
||||
"buffer_source" => buffer_source,
|
||||
"outcome" => outcome
|
||||
)
|
||||
.record(duration_secs);
|
||||
|
||||
@@ -31,7 +31,10 @@ pub struct KeystoneClient {
|
||||
admin_password: Option<String>,
|
||||
admin_project: Option<String>,
|
||||
admin_domain: String,
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "TLS verification flag parsed from config; the reqwest client is built before it is consulted, so nothing reads it back (backlog#1823)"
|
||||
)]
|
||||
verify_ssl: bool,
|
||||
/// Request timeout applied to the underlying HTTP client.
|
||||
timeout: std::time::Duration,
|
||||
|
||||
@@ -20,7 +20,10 @@ use tracing::{debug, info};
|
||||
|
||||
/// Maps Keystone identities to RustFS concepts
|
||||
pub struct KeystoneIdentityMapper {
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "keeps the Keystone client alive for the mapper's lifetime; the mapping paths do not call through it yet (backlog#1823)"
|
||||
)]
|
||||
client: Arc<KeystoneClient>,
|
||||
role_policy_map: HashMap<String, String>,
|
||||
enable_tenant_prefix: bool,
|
||||
|
||||
@@ -293,6 +293,15 @@ enum StrictVaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
Kubernetes {
|
||||
role: String,
|
||||
#[serde(default)]
|
||||
mount: Option<String>,
|
||||
#[serde(default)]
|
||||
jwt_path: Option<std::path::PathBuf>,
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
TokenFile {
|
||||
path: std::path::PathBuf,
|
||||
#[serde(default)]
|
||||
@@ -319,6 +328,17 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => Self::Kubernetes {
|
||||
role,
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
|
||||
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -901,6 +923,42 @@ mod tests {
|
||||
assert!(request.to_kms_config().validate().is_ok());
|
||||
}
|
||||
|
||||
/// The admin API reaches Kubernetes auth with the role alone; the mount and
|
||||
/// the projected token path fall back to the cluster defaults, so a Tenant
|
||||
/// manifest carries no credential and no cluster-specific paths.
|
||||
#[test]
|
||||
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
|
||||
let raw = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"mount_path": "rustfs",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs" } }
|
||||
});
|
||||
|
||||
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
|
||||
let config = request.to_kms_config();
|
||||
config.validate().expect("kubernetes auth must validate");
|
||||
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
|
||||
let unknown_field = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
|
||||
});
|
||||
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
|
||||
.expect_err("an unknown auth field must be rejected rather than silently dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
||||
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
||||
|
||||
@@ -550,6 +550,7 @@ impl VaultKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin {
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
|
||||
/// projected ServiceAccount token for a lease-bound Vault token.
|
||||
///
|
||||
/// The JWT is re-read on every login because the kubelet rotates a projected
|
||||
/// token well inside the pod's lifetime; caching it would strand the source on
|
||||
/// an expired assertion once the current Vault token can no longer be renewed.
|
||||
///
|
||||
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
|
||||
/// the projected token and mounts it world-readable by default, so rejecting
|
||||
/// group/other bits would refuse every standard pod rather than catch a
|
||||
/// deployment error.
|
||||
pub(crate) struct KubernetesLogin {
|
||||
/// Unauthenticated client used only for the login exchange.
|
||||
login_client: VaultClient,
|
||||
mount: String,
|
||||
role: String,
|
||||
jwt_path: PathBuf,
|
||||
}
|
||||
|
||||
impl KubernetesLogin {
|
||||
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
|
||||
Ok(Self {
|
||||
login_client: settings.build_login_client()?,
|
||||
mount,
|
||||
role,
|
||||
jwt_path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the ServiceAccount token for one login attempt.
|
||||
///
|
||||
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
|
||||
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
|
||||
/// has not projected yet heals the source without a restart.
|
||||
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
|
||||
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
|
||||
.await
|
||||
.map_err(|error| AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Failed to read Kubernetes ServiceAccount token {}: {error}",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
})?;
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
raw.zeroize();
|
||||
return Err(AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Kubernetes ServiceAccount token {} is empty",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
});
|
||||
}
|
||||
let jwt = SecretString::new(trimmed.to_string());
|
||||
raw.zeroize();
|
||||
Ok(jwt)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TokenSource for KubernetesLogin {
|
||||
async fn acquire(&self) -> AttemptResult<TokenLease> {
|
||||
let jwt = self.resolve_jwt().await?;
|
||||
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
|
||||
.await
|
||||
.map_err(|error| attempt_error("Kubernetes login", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
|
||||
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
|
||||
let auth = vaultrs::token::renew_self(client, None)
|
||||
.await
|
||||
.map_err(|error| attempt_error("token renewal", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for KubernetesLogin {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// The login client embeds Vault client settings and must stay out of
|
||||
// Debug output; the role name is not a secret, and the JWT is never held.
|
||||
f.debug_struct("KubernetesLogin")
|
||||
.field("mount", &self.mount)
|
||||
.field("role", &self.role)
|
||||
.field("jwt_path", &self.jwt_path)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
||||
/// token file (for example a Vault Agent auto-auth sink).
|
||||
///
|
||||
@@ -464,6 +555,9 @@ pub(crate) fn token_source_for(
|
||||
secret_id.clone(),
|
||||
secret_id_file.clone(),
|
||||
)?)),
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings {
|
||||
pub(crate) namespace: Option<String>,
|
||||
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
||||
pub(crate) attempt_timeout: Duration,
|
||||
/// Whether to accept an unverified Vault server certificate. Gated on
|
||||
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
|
||||
pub(crate) skip_tls_verify: bool,
|
||||
}
|
||||
|
||||
impl VaultConnectionSettings {
|
||||
@@ -499,6 +596,11 @@ impl VaultConnectionSettings {
|
||||
// operation-level retry policy.
|
||||
settings_builder.timeout(Some(self.attempt_timeout));
|
||||
settings_builder.token(token);
|
||||
// Always set explicitly: left unset, vaultrs derives this from its own
|
||||
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
|
||||
// disable certificate verification behind the KMS configuration and its
|
||||
// insecure-defaults gate.
|
||||
settings_builder.verify(!self.skip_tls_verify);
|
||||
|
||||
if let Some(namespace) = &self.namespace {
|
||||
settings_builder.namespace(Some(namespace.clone()));
|
||||
@@ -551,6 +653,10 @@ impl VaultCredentialPolicy {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::Kubernetes {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::TokenFile {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
@@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle {
|
||||
|
||||
impl VaultClientHandle {
|
||||
/// Absolute expiry of this generation's token.
|
||||
///
|
||||
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
|
||||
/// a value too large to add to `issued_at` would panic on the bare `+`. A
|
||||
/// TTL that cannot be represented is indistinguishable from no expiry, so it
|
||||
/// collapses to `None` — the same answer already given for the zero-lease
|
||||
/// tokens Vault issues, which keeps the token in use and still fully
|
||||
/// validated by Vault on every call.
|
||||
fn expires_at(&self) -> Option<Instant> {
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl)
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
|
||||
}
|
||||
|
||||
/// When the renewal task should refresh this generation: half the TTL,
|
||||
/// leaving the second half as budget for retries before the fail-closed
|
||||
/// window is reached.
|
||||
///
|
||||
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
|
||||
/// leaving a token that never expires with nothing to renew.
|
||||
fn renew_at(&self) -> Option<Instant> {
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -662,7 +778,7 @@ impl VaultCredentialProvider {
|
||||
let handle = self.current.load_full();
|
||||
if let Some(expires_at) = handle.expires_at() {
|
||||
let now = Instant::now();
|
||||
if now + self.policy.safety_window >= expires_at {
|
||||
if self.inside_safety_window(now, expires_at) {
|
||||
return Err(KmsError::credentials_unavailable(format!(
|
||||
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
||||
handle.generation, self.policy.safety_window
|
||||
@@ -672,6 +788,18 @@ impl VaultCredentialProvider {
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Whether the token expiring at `expires_at` is close enough to refuse.
|
||||
///
|
||||
/// `safety_window` reaches here from persisted configuration, so it is not
|
||||
/// guaranteed to have passed this version's validation: a window too large
|
||||
/// to add to the current instant would panic on the bare `+`. Such a window
|
||||
/// means every token is always inside it, so saturating to "refuse" is both
|
||||
/// the fail-closed answer and the one the arithmetic was reaching for.
|
||||
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
|
||||
now.checked_add(self.policy.safety_window)
|
||||
.is_none_or(|deadline| deadline >= expires_at)
|
||||
}
|
||||
|
||||
/// Publish the credential gauges for the generation currently installed.
|
||||
///
|
||||
/// The fail-closed gauge re-evaluates the very gate
|
||||
@@ -683,7 +811,7 @@ impl VaultCredentialProvider {
|
||||
let fail_closed = match handle.expires_at() {
|
||||
Some(expires_at) => {
|
||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||
now + self.policy.safety_window >= expires_at
|
||||
self.inside_safety_window(now, expires_at)
|
||||
}
|
||||
// A generation without an expiry has no remaining TTL to report
|
||||
// and can never lapse, so it can never fail closed either.
|
||||
@@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::REDACTED_SECRET;
|
||||
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
|
||||
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
||||
@@ -871,6 +999,7 @@ mod tests {
|
||||
address: "http://127.0.0.1:8200".to_string(),
|
||||
namespace: Some("team-namespace".to_string()),
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1057,6 +1186,143 @@ mod tests {
|
||||
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_auth_method_maps_to_login_source() {
|
||||
let settings = test_settings();
|
||||
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
|
||||
.expect("kubernetes auth must map to a login source");
|
||||
|
||||
assert!(format!("{source:?}").contains("KubernetesLogin"));
|
||||
}
|
||||
|
||||
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
|
||||
/// path from persisted configuration, so the fail-closed comparison must
|
||||
/// survive a window too large to add to the current instant. Before the
|
||||
/// checked arithmetic this panicked with "overflow when adding duration to
|
||||
/// instant" on the first request after a lease-bearing login.
|
||||
#[tokio::test]
|
||||
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = provider
|
||||
.current()
|
||||
.expect_err("a window wider than any lease must refuse the token");
|
||||
assert!(
|
||||
matches!(error, KmsError::CredentialsUnavailable { .. }),
|
||||
"expected CredentialsUnavailable, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `lease_duration` is a bare u64 straight off the Vault response and forms
|
||||
/// the other side of the same comparison, so an absurd one must not panic
|
||||
/// either. It is indistinguishable from a non-expiring token, which is how
|
||||
/// the zero-lease case already behaves.
|
||||
#[tokio::test]
|
||||
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(u64::MAX),
|
||||
true,
|
||||
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
provider
|
||||
.current()
|
||||
.expect("a token whose expiry cannot be represented must stay usable");
|
||||
}
|
||||
|
||||
/// The configured flag has to reach the HTTP client, not just the config
|
||||
/// struct: every generation (authenticated and login) builds its own client,
|
||||
/// and a Vault with a self-signed certificate fails the handshake unless
|
||||
/// each one carries the setting.
|
||||
#[test]
|
||||
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
|
||||
for skip_tls_verify in [false, true] {
|
||||
let settings = VaultConnectionSettings {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
namespace: None,
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify,
|
||||
};
|
||||
|
||||
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
|
||||
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
|
||||
|
||||
let login = settings.build_login_client().expect("login client must build");
|
||||
assert_eq!(login.settings.verify, !skip_tls_verify);
|
||||
}
|
||||
}
|
||||
|
||||
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
|
||||
/// builder leaves it unset, which would disable certificate verification
|
||||
/// without passing the KMS insecure-defaults gate.
|
||||
#[test]
|
||||
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
|
||||
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
|
||||
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
|
||||
assert!(
|
||||
client.settings.verify,
|
||||
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// The projected token is read fresh per login attempt and trimmed, so a
|
||||
/// kubelet rotation is picked up without a restart and a trailing newline
|
||||
/// does not corrupt the assertion sent to Vault.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("token");
|
||||
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
|
||||
|
||||
let login = KubernetesLogin::new(
|
||||
&test_settings(),
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
"rustfs".to_string(),
|
||||
path.clone(),
|
||||
)
|
||||
.expect("login source must build");
|
||||
|
||||
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
|
||||
|
||||
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
|
||||
assert_eq!(
|
||||
login.resolve_jwt().await.expect("second read").expose(),
|
||||
"rotated-jwt",
|
||||
"a rotated projected token must be picked up without a restart"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ServiceAccount token is re-read per attempt, so an unreadable or
|
||||
/// empty one fails that attempt without reaching Vault; the refresh loop
|
||||
/// keeps retrying, which is what lets a late projection heal the source.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let missing = dir.path().join("absent-token");
|
||||
let empty = dir.path().join("empty-token");
|
||||
tokio::fs::write(&empty, " \n").await.expect("write empty token");
|
||||
|
||||
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
|
||||
let login =
|
||||
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
|
||||
.expect("login source must build");
|
||||
|
||||
let error = login
|
||||
.acquire()
|
||||
.await
|
||||
.expect_err("an unusable ServiceAccount token must fail the attempt");
|
||||
assert!(matches!(error.class, ErrorClass::Fatal));
|
||||
assert!(error.error.to_string().contains(expected), "got {}", error.error);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_renewal_task_renews_at_half_ttl() {
|
||||
let (provider, state) = scripted_provider(
|
||||
|
||||
@@ -415,6 +415,7 @@ impl VaultTransitKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -450,6 +450,10 @@ impl VaultRestoreClient {
|
||||
address: target.address.clone(),
|
||||
namespace: target.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
// A restore target carries no TLS settings, so certificates are
|
||||
// always verified: recovery is the last path that should accept an
|
||||
// unauthenticated Vault.
|
||||
skip_tls_verify: false,
|
||||
};
|
||||
let source = token_source_for(&target.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
+295
-54
@@ -25,6 +25,10 @@ use url::Url;
|
||||
|
||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
|
||||
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
|
||||
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
|
||||
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
|
||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||
@@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
|
||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
||||
@@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
|
||||
/// Where the kubelet projects a pod's ServiceAccount token by default.
|
||||
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||
|
||||
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
||||
///
|
||||
@@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String {
|
||||
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_mount() -> String {
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_jwt_path() -> PathBuf {
|
||||
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
|
||||
}
|
||||
|
||||
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
||||
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
||||
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
||||
@@ -490,6 +508,23 @@ pub enum VaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
|
||||
/// for a lease-bound Vault token that is renewed in the background.
|
||||
Kubernetes {
|
||||
/// Vault role bound to this ServiceAccount.
|
||||
role: String,
|
||||
/// Kubernetes auth engine mount path.
|
||||
#[serde(default = "default_vault_kubernetes_mount")]
|
||||
mount: String,
|
||||
/// Projected ServiceAccount token to present. Re-read on every login so
|
||||
/// a token the kubelet rotates is picked up without a restart.
|
||||
#[serde(default = "default_vault_kubernetes_jwt_path")]
|
||||
jwt_path: PathBuf,
|
||||
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
|
||||
/// per-attempt timeout.
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
||||
/// the token is read from `path` and re-read periodically so a token
|
||||
/// rotated by the agent is picked up without a restart.
|
||||
@@ -520,6 +555,16 @@ impl VaultAuthMethod {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kubernetes authentication with the default mount and projected token path.
|
||||
pub fn kubernetes(role: String) -> Self {
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount: default_vault_kubernetes_mount(),
|
||||
jwt_path: default_vault_kubernetes_jwt_path(),
|
||||
refresh_safety_window_secs: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Agent-managed token file with the default poll interval.
|
||||
pub fn token_file(path: PathBuf) -> Self {
|
||||
Self::TokenFile {
|
||||
@@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod {
|
||||
.field("mount", mount)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
// No redaction: the role and mount name a Vault binding, and the
|
||||
// ServiceAccount token itself is never held on this type.
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => f
|
||||
.debug_struct("Kubernetes")
|
||||
.field("role", role)
|
||||
.field("mount", mount)
|
||||
.field("jwt_path", jwt_path)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
Self::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1028,50 +1087,12 @@ impl KmsConfig {
|
||||
});
|
||||
}
|
||||
KmsBackend::VaultKv2 => {
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
config.backend_config =
|
||||
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
|
||||
}
|
||||
KmsBackend::VaultTransit => {
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
|
||||
metadata_kv_mount: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
),
|
||||
metadata_key_prefix: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
|
||||
),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
config.backend_config =
|
||||
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
|
||||
}
|
||||
KmsBackend::Static => {
|
||||
// Read from file first, then fall back to direct env var
|
||||
@@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
path.starts_with(std::env::temp_dir())
|
||||
}
|
||||
|
||||
/// Command-line values that take precedence over the matching environment
|
||||
/// variables when assembling a Vault backend configuration.
|
||||
///
|
||||
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
|
||||
/// reads, so these are only set when the operator passed an explicit flag.
|
||||
///
|
||||
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
|
||||
/// redacting `Debug` impls elsewhere in this module exist because a derived one
|
||||
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
|
||||
/// error instead of a leak.
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct VaultCliOverrides<'a> {
|
||||
pub address: Option<&'a str>,
|
||||
pub token: Option<&'a str>,
|
||||
pub mount_path: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Assemble the Vault KV2 backend configuration from the environment.
|
||||
///
|
||||
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
|
||||
/// so both resolve the same auth method, namespace, TLS and mount settings.
|
||||
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
|
||||
let mount_path = match overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
|
||||
{
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
Ok(VaultConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Assemble the Vault Transit backend configuration from the environment.
|
||||
///
|
||||
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
|
||||
/// points share it.
|
||||
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
|
||||
Ok(VaultTransitConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path: overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
|
||||
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
|
||||
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||
override_value
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
|
||||
}
|
||||
|
||||
/// Resolve the Vault auth method from environment variables.
|
||||
///
|
||||
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
||||
@@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
||||
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
||||
/// file taking precedence. Without a role id the legacy token flow applies.
|
||||
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
|
||||
///
|
||||
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
|
||||
/// presents the pod's projected ServiceAccount token.
|
||||
///
|
||||
/// `token_override` carries a token supplied on the command line; it stands in
|
||||
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
|
||||
/// so a flag and the variable it mirrors select the same method.
|
||||
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
|
||||
let token = token_override
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
|
||||
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
|
||||
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
|
||||
|
||||
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
||||
// A token file names one authoritative credential source; combining it
|
||||
// with another one would leave the effective identity ambiguous, so
|
||||
// that is a configuration error rather than a precedence rule.
|
||||
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
|
||||
)));
|
||||
for (name, configured) in [
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
|
||||
(ENV_KMS_VAULT_TOKEN, token.is_some()),
|
||||
] {
|
||||
if configured {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
}
|
||||
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
||||
}
|
||||
|
||||
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
|
||||
if let Some(role) = kubernetes_role {
|
||||
// Unlike a leftover static token, a second login method is never a
|
||||
// stale remnant: both were configured deliberately and neither can be
|
||||
// ranked over the other.
|
||||
if role_id.is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
return Ok(VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
|
||||
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
|
||||
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
|
||||
refresh_safety_window_secs: None,
|
||||
});
|
||||
}
|
||||
|
||||
let Some(role_id) = role_id else {
|
||||
return Ok(VaultAuthMethod::Token {
|
||||
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
|
||||
token: token.unwrap_or_else(|| "dev-token".to_string()),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => {
|
||||
if role.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
|
||||
}
|
||||
if mount.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
|
||||
}
|
||||
if jwt_path.as_os_str().is_empty() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1976,6 +2117,106 @@ mod tests {
|
||||
.expect("well-formed token file auth must validate");
|
||||
}
|
||||
|
||||
/// A Kubernetes role alone configures the method: the credential is the
|
||||
/// pod's projected ServiceAccount token, so nothing secret is in the
|
||||
/// environment and the mount and token path fall back to the cluster
|
||||
/// defaults.
|
||||
#[test]
|
||||
fn test_from_env_selects_kubernetes() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
|
||||
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
|
||||
],
|
||||
|| {
|
||||
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!(
|
||||
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||
vault.auth_method
|
||||
);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
assert_eq!(refresh_safety_window_secs, &None);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
],
|
||||
|| {
|
||||
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_bad_kubernetes_settings() {
|
||||
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
|
||||
backend: KmsBackend::VaultTransit,
|
||||
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
auth_method,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes role must be rejected");
|
||||
assert!(error.to_string().contains("role"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: String::new(),
|
||||
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes mount must be rejected");
|
||||
assert!(error.to_string().contains("mount"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
jwt_path: PathBuf::new(),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty ServiceAccount token path must be rejected");
|
||||
assert!(error.to_string().contains("token path"), "got {error}");
|
||||
|
||||
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
|
||||
.validate()
|
||||
.expect("well-formed kubernetes auth must validate");
|
||||
}
|
||||
|
||||
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
||||
/// empty one names a path no Vault engine answers. The Transit backend
|
||||
/// already rejects its own empty mounts; this closes the same gap on the
|
||||
|
||||
@@ -18,8 +18,6 @@
|
||||
//! data encryption keys using master keys. It abstracts the encryption
|
||||
//! operations so that different backends can share the same encryption logic.
|
||||
|
||||
#![allow(dead_code)] // Trait methods may be used by implementations
|
||||
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -37,7 +37,11 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
humantime.workspace = true
|
||||
http.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
rustfs-signer.workspace = true
|
||||
s3s.workspace = true
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
@@ -49,3 +53,4 @@ doctest = false
|
||||
|
||||
[dev-dependencies]
|
||||
rmp-serde.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] }
|
||||
|
||||
@@ -0,0 +1,851 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869).
|
||||
//!
|
||||
//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4
|
||||
//! request signing (the same scheme the server's admin router authenticates),
|
||||
//! so `mc`-style tooling and automation can drive heal start/query/cancel and
|
||||
//! read background-heal / scanner status without hand-rolling HTTP.
|
||||
//!
|
||||
//! Wire structs in this module mirror the server-side shapes
|
||||
//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`,
|
||||
//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where
|
||||
//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply
|
||||
//! nested status payloads that the server composes from runtime types are
|
||||
//! carried through as `serde_json::Value` and flattened maps rather than
|
||||
//! duplicated field-for-field, so the client cannot silently drift on fields
|
||||
//! it never interprets.
|
||||
|
||||
use crate::heal_commands::HealResultItem;
|
||||
use http::Method;
|
||||
use serde::{Deserialize, Serialize, de};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Default admin API path prefix on a RustFS endpoint.
|
||||
pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin";
|
||||
/// Default SigV4 region when the server has no explicit region configured.
|
||||
pub const DEFAULT_REGION: &str = "us-east-1";
|
||||
|
||||
/// Scan mode for a heal request, mirroring the server's numeric-or-name wire
|
||||
/// encoding (`0` unknown/default, `1` normal, `2` deep).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealScanMode {
|
||||
/// Server default; behaves as [`HealScanMode::Normal`].
|
||||
#[default]
|
||||
Unknown,
|
||||
/// Metadata-level checks only.
|
||||
Normal,
|
||||
/// Full bitrot verification while healing.
|
||||
Deep,
|
||||
}
|
||||
|
||||
impl HealScanMode {
|
||||
fn wire_number(self) -> u8 {
|
||||
match self {
|
||||
Self::Unknown => 0,
|
||||
Self::Normal => 1,
|
||||
Self::Deep => 2,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_wire_number(value: u8) -> Option<Self> {
|
||||
match value {
|
||||
0 => Some(Self::Unknown),
|
||||
1 => Some(Self::Normal),
|
||||
2 => Some(Self::Deep),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_wire_name(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"normal" => Some(Self::Normal),
|
||||
"deep" => Some(Self::Deep),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for HealScanMode {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_u8(self.wire_number())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for HealScanMode {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct HealScanModeVisitor;
|
||||
|
||||
impl de::Visitor<'_> for HealScanModeVisitor {
|
||||
type Value = HealScanMode;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("a heal scan mode number or name")
|
||||
}
|
||||
|
||||
fn visit_u64<E: de::Error>(self, value: u64) -> Result<Self::Value, E> {
|
||||
u8::try_from(value)
|
||||
.ok()
|
||||
.and_then(HealScanMode::from_wire_number)
|
||||
.ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}")))
|
||||
}
|
||||
|
||||
fn visit_str<E: de::Error>(self, value: &str) -> Result<Self::Value, E> {
|
||||
HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}")))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(HealScanModeVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal options for an admin heal request (mirror of the server body type).
|
||||
/// Fields default on decode: a client should tolerate a server response whose
|
||||
/// settings object omits fields it never set.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct HealOpts {
|
||||
#[serde(default)]
|
||||
pub recursive: bool,
|
||||
#[serde(rename = "dryRun", default)]
|
||||
pub dry_run: bool,
|
||||
#[serde(default)]
|
||||
pub remove: bool,
|
||||
#[serde(default)]
|
||||
pub recreate: bool,
|
||||
#[serde(rename = "scanMode", default)]
|
||||
pub scan_mode: HealScanMode,
|
||||
#[serde(rename = "updateParity", default)]
|
||||
pub update_parity: bool,
|
||||
#[serde(rename = "nolock", default)]
|
||||
pub no_lock: bool,
|
||||
#[serde(rename = "pool", default)]
|
||||
pub pool: Option<usize>,
|
||||
#[serde(rename = "set", default)]
|
||||
pub set: Option<usize>,
|
||||
}
|
||||
|
||||
/// Successful heal start / path-scoped cancel response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealStartSuccess {
|
||||
pub client_token: String,
|
||||
pub client_address: String,
|
||||
#[serde(default)]
|
||||
pub start_time: String,
|
||||
}
|
||||
|
||||
/// Heal task status response (query, cancel-with-token, start-then-poll).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealTaskStatus {
|
||||
/// `running` | `finished` | `stopped` | `notFound`.
|
||||
pub summary: String,
|
||||
/// Failure detail for stopped tasks; empty otherwise.
|
||||
#[serde(rename = "detail", default)]
|
||||
pub failure_detail: String,
|
||||
#[serde(default)]
|
||||
pub start_time: String,
|
||||
#[serde(default)]
|
||||
pub settings: HealOpts,
|
||||
#[serde(default)]
|
||||
pub items: Vec<HealResultItem>,
|
||||
#[serde(default)]
|
||||
pub truncated: bool,
|
||||
/// Live progress snapshot; the exact shape is owned by the heal runtime.
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/background-heal/status` response. Known top-level fields are
|
||||
/// typed; the flattened heal info and operations matrix pass through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackgroundHealStatus {
|
||||
/// `disabled` | `uninitialized` | `idle` | `active` | `degraded`.
|
||||
pub state: String,
|
||||
#[serde(default)]
|
||||
pub heal_queue_length: u64,
|
||||
#[serde(default)]
|
||||
pub heal_active_tasks: u64,
|
||||
#[serde(default)]
|
||||
pub cluster_status_complete: bool,
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
|
||||
/// priority-by-source operations matrix), carried verbatim.
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `GET /v3/scanner/status` response, typed at the fields operators branch
|
||||
/// on; everything else passes through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScannerStatus {
|
||||
pub enabled: bool,
|
||||
/// `fresh` | `stale` | `unknown`; absent when the scanner never completed
|
||||
/// a cycle.
|
||||
#[serde(default)]
|
||||
pub freshness: Option<ScannerFreshness>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Freshness block of the scanner status response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScannerFreshness {
|
||||
/// `fresh` | `stale` | `unknown`.
|
||||
pub state: String,
|
||||
}
|
||||
|
||||
impl ScannerStatus {
|
||||
/// Convenience accessor for the freshness state string.
|
||||
pub fn freshness(&self) -> &str {
|
||||
self.freshness
|
||||
.as_ref()
|
||||
.map(|freshness| freshness.state.as_str())
|
||||
.unwrap_or("unknown")
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything that can go wrong in an admin client call.
|
||||
#[derive(Debug)]
|
||||
pub enum AdminClientError {
|
||||
/// The endpoint URL could not be parsed.
|
||||
InvalidEndpoint(String),
|
||||
/// Request build/send failed (DNS, connect, timeout, body read).
|
||||
Transport(reqwest::Error),
|
||||
/// The server answered a non-2xx status.
|
||||
HttpStatus { status: u16, body: String },
|
||||
/// The response body did not decode into the expected shape.
|
||||
Decode { message: String },
|
||||
}
|
||||
|
||||
impl std::fmt::Display for AdminClientError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"),
|
||||
Self::Transport(err) => write!(f, "admin request transport failure: {err}"),
|
||||
Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"),
|
||||
Self::Decode { message } => write!(f, "admin response decode failure: {message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AdminClientError {}
|
||||
|
||||
impl From<reqwest::Error> for AdminClientError {
|
||||
fn from(err: reqwest::Error) -> Self {
|
||||
Self::Transport(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed client for a RustFS admin API.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdminClient {
|
||||
endpoint: reqwest::Url,
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
session_token: String,
|
||||
region: String,
|
||||
api_prefix: String,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl AdminClient {
|
||||
/// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root
|
||||
/// or admin credentials. Requests are SigV4-signed with the same scheme
|
||||
/// the server's admin router authenticates.
|
||||
pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result<Self, AdminClientError> {
|
||||
let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if url.host_str().is_none() {
|
||||
return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string()));
|
||||
}
|
||||
let http = reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(AdminClientError::Transport)?;
|
||||
Ok(Self {
|
||||
endpoint: url,
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
session_token: String::new(),
|
||||
region: DEFAULT_REGION.to_string(),
|
||||
api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(),
|
||||
http,
|
||||
})
|
||||
}
|
||||
|
||||
/// Attach an STS session token (signed as `x-amz-security-token`).
|
||||
pub fn with_session_token(mut self, session_token: impl Into<String>) -> Self {
|
||||
self.session_token = session_token.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the SigV4 region (defaults to `us-east-1`, matching a
|
||||
/// region-less RustFS deployment).
|
||||
pub fn with_region(mut self, region: impl Into<String>) -> Self {
|
||||
self.region = region.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the admin API path prefix (defaults to `/rustfs/admin`).
|
||||
pub fn with_api_prefix(mut self, prefix: impl Into<String>) -> Self {
|
||||
self.api_prefix = prefix.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Start a heal. `bucket` empty and `prefix` empty heals the whole
|
||||
/// deployment (requires `recursive` or a `pool`/`set` pair in `opts`,
|
||||
/// enforced server-side); a bucket alone heals the bucket (the server
|
||||
/// forces `recursive` for bucket heals).
|
||||
pub async fn heal_start(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
force_start: bool,
|
||||
) -> Result<HealStartSuccess, AdminClientError> {
|
||||
let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
let mut query = Vec::new();
|
||||
if force_start {
|
||||
query.push(("forceStart", "true".to_string()));
|
||||
}
|
||||
self.post_json(&heal_path(bucket, prefix), &query, body).await
|
||||
}
|
||||
|
||||
/// Query the status of the heal identified by `client_token` (the token
|
||||
/// returned by [`Self::heal_start`]) at the path it was started on.
|
||||
pub async fn heal_status(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stop a heal: with a `client_token` only that task is cancelled and its
|
||||
/// final status returned; without one, every heal task at the path is
|
||||
/// cancelled (the server answers with a start-success-shaped receipt).
|
||||
pub async fn heal_stop(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: Option<&str>,
|
||||
) -> Result<HealStopOutcome, AdminClientError> {
|
||||
let mut query = vec![("forceStop", "true".to_string())];
|
||||
if let Some(token) = client_token {
|
||||
query.push(("clientToken", token.to_string()));
|
||||
}
|
||||
match client_token {
|
||||
Some(_) => {
|
||||
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::Stopped(status))
|
||||
}
|
||||
None => {
|
||||
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::PathStopped(success))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cluster-aggregated background heal status.
|
||||
pub async fn background_heal_status(&self) -> Result<BackgroundHealStatus, AdminClientError> {
|
||||
self.get_json("/v3/background-heal/status").await
|
||||
}
|
||||
|
||||
/// Data scanner status (enabled state, freshness, runtime config).
|
||||
pub async fn scanner_status(&self) -> Result<ScannerStatus, AdminClientError> {
|
||||
self.get_json("/v3/scanner/status").await
|
||||
}
|
||||
|
||||
/// ILM expiry worker status. The payload is owned by the expiry
|
||||
/// subsystem and still evolving; returned verbatim.
|
||||
pub async fn ilm_expiry_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
self.get_json("/v3/ilm/expiry/status").await
|
||||
}
|
||||
|
||||
/// Durable replacement-recovery status (admin v4). The payload is owned
|
||||
/// by the heal runtime; returned verbatim.
|
||||
pub async fn replacement_recovery_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
self.get_json("/v4/heal/replacement-recovery").await
|
||||
}
|
||||
|
||||
/// Signed GET returning a decoded JSON body; escape hatch for endpoints
|
||||
/// this client does not wrap yet.
|
||||
pub async fn get_json<T: for<'de> Deserialize<'de>>(&self, path: &str) -> Result<T, AdminClientError> {
|
||||
let url = self.url_for(path, &[])?;
|
||||
let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?;
|
||||
self.execute(request).await
|
||||
}
|
||||
|
||||
/// Signed POST returning a decoded JSON body.
|
||||
async fn post_json<T: for<'de> Deserialize<'de>>(
|
||||
&self,
|
||||
path: &str,
|
||||
query: &[(&str, String)],
|
||||
body: Vec<u8>,
|
||||
) -> Result<T, AdminClientError> {
|
||||
let content_type = if body.is_empty() { None } else { Some("application/json") };
|
||||
let url = self.url_for(path, query)?;
|
||||
let request = self.sign_and_build(Method::POST, url, body, content_type).await?;
|
||||
self.execute(request).await
|
||||
}
|
||||
|
||||
fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result<reqwest::Url, AdminClientError> {
|
||||
let mut url = self
|
||||
.endpoint
|
||||
.join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path))
|
||||
.map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if !query.is_empty() {
|
||||
let mut pairs = url.query_pairs_mut();
|
||||
for (key, value) in query {
|
||||
pairs.append_pair(key, value);
|
||||
}
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
/// Build a SigV4-signed request via the same signer the server trusts,
|
||||
/// then hand the signed headers to the HTTP client. The signature covers
|
||||
/// method, path, query, and an unsigned-payload marker — the same shape
|
||||
/// RustFS itself sends for peer admin calls.
|
||||
async fn sign_and_build(
|
||||
&self,
|
||||
method: Method,
|
||||
url: reqwest::Url,
|
||||
body: Vec<u8>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<reqwest::Request, AdminClientError> {
|
||||
let authority = match (url.host_str(), url.port_or_known_default()) {
|
||||
(Some(host), Some(port)) => format!("{host}:{port}"),
|
||||
_ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())),
|
||||
};
|
||||
let mut builder = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(url.as_str())
|
||||
.header(http::header::HOST, &authority)
|
||||
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
builder = builder.header(http::header::CONTENT_TYPE, content_type);
|
||||
}
|
||||
let unsigned = builder
|
||||
.body(s3s::Body::empty())
|
||||
.map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?;
|
||||
let signed = rustfs_signer::sign_v4(
|
||||
unsigned,
|
||||
body.len() as i64,
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
&self.session_token,
|
||||
&self.region,
|
||||
);
|
||||
|
||||
let mut request = self
|
||||
.http
|
||||
.request(method, url)
|
||||
.body(body)
|
||||
.build()
|
||||
.map_err(AdminClientError::Transport)?;
|
||||
let headers = request.headers_mut();
|
||||
for (name, value) in signed.headers().iter() {
|
||||
// HOST is owned by the HTTP client; the signed value above was
|
||||
// built from the same URL authority, so they always agree.
|
||||
if name == http::header::HOST {
|
||||
continue;
|
||||
}
|
||||
headers.insert(name, value.clone());
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
async fn execute<T: for<'de> Deserialize<'de>>(&self, request: reqwest::Request) -> Result<T, AdminClientError> {
|
||||
let response = self.http.execute(request).await?;
|
||||
let status = response.status();
|
||||
let bytes = response.bytes().await?;
|
||||
if !status.is_success() {
|
||||
return Err(AdminClientError::HttpStatus {
|
||||
status: status.as_u16(),
|
||||
body: String::from_utf8_lossy(&bytes).into_owned(),
|
||||
});
|
||||
}
|
||||
serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task
|
||||
/// answers with that task's status, cancelling a whole path answers with a
|
||||
/// start-success-shaped receipt.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealStopOutcome {
|
||||
Stopped(HealTaskStatus),
|
||||
PathStopped(HealStartSuccess),
|
||||
}
|
||||
|
||||
fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String {
|
||||
match (bucket, prefix) {
|
||||
(Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => {
|
||||
format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix))
|
||||
}
|
||||
(Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => {
|
||||
format!("/v3/heal/{}", percent_encode_path_segment(bucket))
|
||||
}
|
||||
_ => "/v3/heal/".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a single path segment (slashes are content, not separators, inside
|
||||
/// bucket/prefix path params).
|
||||
fn percent_encode_path_segment(segment: &str) -> String {
|
||||
let mut out = String::with_capacity(segment.len());
|
||||
for byte in segment.bytes() {
|
||||
match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char),
|
||||
_ => out.push_str(&format!("%{byte:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus,
|
||||
ScannerStatus, heal_path, percent_encode_path_segment,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[test]
|
||||
fn heal_paths_cover_root_bucket_and_prefix() {
|
||||
assert_eq!(heal_path(None, None), "/v3/heal/");
|
||||
assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/");
|
||||
assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket");
|
||||
assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_segments_percent_encode_reserved_characters() {
|
||||
assert_eq!(percent_encode_path_segment("a b"), "a%20b");
|
||||
assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb");
|
||||
assert_eq!(percent_encode_path_segment("ü"), "%C3%BC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_opts_round_trip_through_the_server_wire_shape() {
|
||||
let opts = HealOpts {
|
||||
recursive: true,
|
||||
dry_run: false,
|
||||
remove: true,
|
||||
recreate: false,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: false,
|
||||
pool: Some(1),
|
||||
set: Some(2),
|
||||
};
|
||||
let wire = serde_json::to_value(&opts).unwrap();
|
||||
assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number");
|
||||
let back: HealOpts = serde_json::from_value(wire).unwrap();
|
||||
assert_eq!(back.scan_mode, HealScanMode::Deep);
|
||||
assert_eq!(back.pool, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_scan_mode_accepts_both_wire_encodings() {
|
||||
assert_eq!(serde_json::from_value::<HealScanMode>(json!(1)).unwrap(), HealScanMode::Normal);
|
||||
assert_eq!(serde_json::from_value::<HealScanMode>(json!("deep")).unwrap(), HealScanMode::Deep);
|
||||
assert!(serde_json::from_value::<HealScanMode>(json!(9)).is_err());
|
||||
assert!(serde_json::from_value::<HealScanMode>(json!("sideways")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_task_status_decodes_the_server_response_shape() {
|
||||
let raw = json!({
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-08-17T00:00:00Z",
|
||||
"settings": {"recursive": false, "scanMode": 1},
|
||||
"items": [{
|
||||
"resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "",
|
||||
"parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1,
|
||||
"before": {"drives": []}, "after": {"drives": []}, "objectSize": 128
|
||||
}],
|
||||
"truncated": false
|
||||
});
|
||||
let status: HealTaskStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.summary, "finished");
|
||||
assert_eq!(status.items.len(), 1);
|
||||
assert_eq!(status.settings.scan_mode, HealScanMode::Normal);
|
||||
assert!(status.progress.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
|
||||
let raw = json!({
|
||||
"state": "active",
|
||||
"bitrotStartTime": "t",
|
||||
"healQueueLength": 3,
|
||||
"healActiveTasks": 1,
|
||||
"healOperations": {"queueLength": 3},
|
||||
"clusterStatusComplete": true
|
||||
});
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.state, "active");
|
||||
assert_eq!(status.heal_queue_length, 3);
|
||||
assert!(status.cluster_status_complete);
|
||||
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_status_defaults_freshness_to_unknown() {
|
||||
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
|
||||
let status: ScannerStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.freshness(), "stale");
|
||||
let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap();
|
||||
assert_eq!(bare.freshness(), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_endpoint_is_rejected_without_io() {
|
||||
let err = AdminClient::new("not a url", "ak", "sk").unwrap_err();
|
||||
assert!(matches!(err, AdminClientError::InvalidEndpoint(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn signed_requests_carry_sigv4_authorization_and_correct_target() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin")
|
||||
.expect("client builds against the test server");
|
||||
|
||||
let start: HealStartSuccess = client
|
||||
.heal_start(
|
||||
Some("bucket"),
|
||||
None,
|
||||
&HealOpts {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("signed heal start decodes");
|
||||
|
||||
assert_eq!(start.client_token, "token-1");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
|
||||
assert!(!request.query.contains("forceStart"), "absent flags must not be sent");
|
||||
let auth = request.header("authorization").expect("request must be signed");
|
||||
assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}");
|
||||
assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header");
|
||||
assert_eq!(
|
||||
request.header("x-amz-content-sha256").as_deref(),
|
||||
Some("UNSIGNED-PAYLOAD"),
|
||||
"the client signs the same payload marker RustFS peer calls use"
|
||||
);
|
||||
assert_eq!(request.header("content-type").as_deref(), Some("application/json"));
|
||||
assert!(request.body.contains("\"recursive\":true"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_sends_client_token_on_the_same_path() {
|
||||
let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let status = client
|
||||
.heal_status(Some("bucket"), None, "token-1")
|
||||
.await
|
||||
.expect("status decodes");
|
||||
assert_eq!(status.summary, "running");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
|
||||
assert!(request.query.contains("clientToken=token-1"));
|
||||
assert!(!request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_without_token_takes_the_path_cancel_branch() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes");
|
||||
assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_)));
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("forceStop=true"));
|
||||
assert!(!request.query.contains("clientToken"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_status_maps_to_a_typed_error_with_body() {
|
||||
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
let err = client.scanner_status().await.unwrap_err();
|
||||
match err {
|
||||
AdminClientError::HttpStatus { status, body } => {
|
||||
assert_eq!(status, 403);
|
||||
assert!(body.contains("AccessDenied"));
|
||||
}
|
||||
other => panic!("expected HttpStatus, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_success_body_maps_to_a_decode_error() {
|
||||
let server = TestServer::spawn("not json", 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. }));
|
||||
}
|
||||
|
||||
/// One recorded request, parsed off the wire with the minimum needed for
|
||||
/// assertions: method, path, query, headers, body.
|
||||
#[derive(Debug, Clone)]
|
||||
struct RecordedRequest {
|
||||
method: String,
|
||||
path: String,
|
||||
query: String,
|
||||
headers: Vec<(String, String)>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl RecordedRequest {
|
||||
fn header(&self, name: &str) -> Option<String> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal HTTP/1.1 server: one canned response per connection, every
|
||||
/// request recorded behind an `Arc<Mutex>`. Deliberately dependency-free —
|
||||
/// the assertions only need the raw request bytes.
|
||||
struct TestServer {
|
||||
addr: std::net::SocketAddr,
|
||||
requests: Arc<Mutex<Vec<RecordedRequest>>>,
|
||||
}
|
||||
|
||||
impl TestServer {
|
||||
async fn spawn(response_body: &'static str, status: u16) -> Self {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind ephemeral port");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let requests: Arc<Mutex<Vec<RecordedRequest>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let recorded = requests.clone();
|
||||
tokio::spawn(async move {
|
||||
let reason = if status == 200 { "OK" } else { "Forbidden" };
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}",
|
||||
response_body.len()
|
||||
);
|
||||
// Each request is a fresh connection (connection: close); a
|
||||
// bounded loop serves every call a test makes while letting
|
||||
// the task exit instead of lingering for the whole process.
|
||||
for _ in 0..16 {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
break;
|
||||
};
|
||||
let mut buffer = Vec::with_capacity(2048);
|
||||
let mut chunk = [0u8; 2048];
|
||||
// Read headers plus content-length body, or stop on close.
|
||||
loop {
|
||||
if let Some(end) = find_header_end(&buffer) {
|
||||
let content_length = extract_content_length(&buffer[..end]);
|
||||
if buffer.len() >= end + content_length {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let n = match stream.read(&mut chunk).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
};
|
||||
buffer.extend_from_slice(&chunk[..n]);
|
||||
if buffer.len() > 64 * 1024 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(request) = parse_request(&buffer) {
|
||||
recorded.lock().expect("recorded lock").push(request);
|
||||
}
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
let _ = stream.shutdown().await;
|
||||
}
|
||||
});
|
||||
|
||||
Self { addr, requests }
|
||||
}
|
||||
|
||||
fn recorded(&self) -> RecordedRequest {
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("recorded lock")
|
||||
.last()
|
||||
.cloned()
|
||||
.expect("the client call must have produced one recorded request")
|
||||
}
|
||||
}
|
||||
|
||||
fn find_header_end(buffer: &[u8]) -> Option<usize> {
|
||||
buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4)
|
||||
}
|
||||
|
||||
fn extract_content_length(headers: &[u8]) -> usize {
|
||||
let text = String::from_utf8_lossy(headers).to_ascii_lowercase();
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("content-length:"))
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn parse_request(raw: &[u8]) -> Option<RecordedRequest> {
|
||||
let end = find_header_end(raw)?;
|
||||
let head = String::from_utf8_lossy(&raw[..end]);
|
||||
let body = String::from_utf8_lossy(&raw[end..]).into_owned();
|
||||
let mut lines = head.lines();
|
||||
let request_line = lines.next()?;
|
||||
let mut parts = request_line.split_whitespace();
|
||||
let method = parts.next()?.to_string();
|
||||
let target = parts.next()?.to_string();
|
||||
let (path, query) = match target.split_once('?') {
|
||||
Some((path, query)) => (path.to_string(), query.to_string()),
|
||||
None => (target, String::new()),
|
||||
};
|
||||
let headers = lines
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
|
||||
.collect();
|
||||
Some(RecordedRequest {
|
||||
method,
|
||||
path,
|
||||
query,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod client;
|
||||
pub mod group;
|
||||
pub mod heal_commands;
|
||||
pub mod health;
|
||||
@@ -25,6 +26,7 @@ pub mod trace;
|
||||
pub mod user;
|
||||
pub mod utils;
|
||||
|
||||
pub use client::*;
|
||||
pub use group::*;
|
||||
pub use info_commands::*;
|
||||
pub use policy::*;
|
||||
|
||||
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRIAMUser {
|
||||
#[serde(rename = "accessKey", default)]
|
||||
pub access_key: String,
|
||||
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRGroupInfo {
|
||||
#[serde(rename = "updateReq", default)]
|
||||
pub update_req: GroupAddRemove,
|
||||
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRIAMItem {
|
||||
#[serde(default)]
|
||||
pub r#type: String,
|
||||
|
||||
@@ -40,7 +40,10 @@ impl RuleEvents for RuleView {
|
||||
#[derive(Debug)]
|
||||
struct CompiledRules {
|
||||
// Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase)
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "speculative retention: the comment above keeps it for richer snapshot-time judgements that no code performs yet (backlog#1823)"
|
||||
)]
|
||||
rules_map: RulesMap,
|
||||
// for RulesContainer::iter_rules
|
||||
rule_views: Vec<RuleView>,
|
||||
|
||||
@@ -187,7 +187,6 @@ impl RulesMap {
|
||||
/// # Parameters
|
||||
/// * `event_name` - The EventName from which to remove the rule.
|
||||
/// * `pattern` - The pattern of the rule to be removed.
|
||||
#[allow(dead_code)]
|
||||
pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) {
|
||||
let mut remove_event = false;
|
||||
|
||||
@@ -209,7 +208,6 @@ impl RulesMap {
|
||||
///
|
||||
/// # Parameters
|
||||
/// * `event_names` - A slice of EventNames to be removed.
|
||||
#[allow(dead_code)]
|
||||
pub fn remove_rules(&mut self, event_names: &[EventName]) {
|
||||
for event_name in event_names {
|
||||
self.map.remove(event_name);
|
||||
@@ -223,7 +221,6 @@ impl RulesMap {
|
||||
/// * `event_name` - The EventName to update.
|
||||
/// * `pattern` - The pattern of the rule to be updated.
|
||||
/// * `target_id` - The TargetID to be added.
|
||||
#[allow(dead_code)]
|
||||
pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) {
|
||||
self.map.entry(event_name).or_default().add(pattern, target_id);
|
||||
self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask
|
||||
|
||||
@@ -18,12 +18,6 @@ use rustfs_targets::arn::TargetID;
|
||||
/// TargetIDSet - A collection representation of TargetID.
|
||||
pub type TargetIdSet = HashSet<TargetID>;
|
||||
|
||||
/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed)
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn new_target_id_set(target_ids: Vec<TargetID>) -> TargetIdSet {
|
||||
target_ids.into_iter().collect()
|
||||
}
|
||||
|
||||
// HashSet has built-in clone, union, difference and other operations.
|
||||
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
|
||||
// If you need to exactly match Go's API style, you can add wrapper functions.
|
||||
|
||||
@@ -17,7 +17,6 @@ use std::time::Duration;
|
||||
/// Environment variable key for the global default metrics interval (seconds).
|
||||
pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC";
|
||||
/// Default interval for metrics collection if not specified otherwise.
|
||||
#[allow(dead_code)]
|
||||
pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Environment variable key for cluster metrics interval (seconds).
|
||||
|
||||
@@ -145,21 +145,18 @@ impl PrometheusMetric {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[allow(dead_code)]
|
||||
pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self {
|
||||
self.labels.push((key, value.into()));
|
||||
self
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[allow(dead_code)]
|
||||
pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self {
|
||||
self.labels.push((key, Cow::Owned(value)));
|
||||
self
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[allow(dead_code)]
|
||||
pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self {
|
||||
self.labels = labels;
|
||||
self
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType};
|
||||
use std::collections::HashSet;
|
||||
|
||||
/// MetricDescriptor - Metric descriptors
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MetricDescriptor {
|
||||
pub name: MetricName,
|
||||
@@ -52,7 +51,6 @@ impl MetricDescriptor {
|
||||
}
|
||||
|
||||
/// Get the full metric name in Prometheus style: <namespace>_<subsystem>_<name>
|
||||
#[allow(dead_code)]
|
||||
pub fn get_full_metric_name(&self) -> String {
|
||||
let namespace = self.namespace.as_str();
|
||||
let formatted_subsystem = self.subsystem.as_str();
|
||||
@@ -61,7 +59,6 @@ impl MetricDescriptor {
|
||||
}
|
||||
|
||||
/// check whether the label is in the label set
|
||||
#[allow(dead_code)]
|
||||
pub fn has_label(&mut self, label: &str) -> bool {
|
||||
self.get_label_set().contains(label)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
/// The metric name is the individual name of the metric
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum MetricName {
|
||||
// The generic metric name
|
||||
@@ -443,7 +442,6 @@ pub enum MetricName {
|
||||
}
|
||||
|
||||
impl MetricName {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> String {
|
||||
match self {
|
||||
Self::AuthTotal => "auth_total".to_string(),
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
/// MetricType - Indicates the type of indicator
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MetricType {
|
||||
Counter,
|
||||
@@ -23,7 +22,6 @@ pub enum MetricType {
|
||||
|
||||
impl MetricType {
|
||||
/// convert the metric type to a string representation
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Counter => "counter",
|
||||
@@ -34,7 +32,6 @@ impl MetricType {
|
||||
|
||||
/// Convert the metric type to the Prometheus value type
|
||||
/// In a Rust implementation, this might return the corresponding Prometheus Rust client type
|
||||
#[allow(dead_code)]
|
||||
pub fn as_prom(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Counter => "counter.",
|
||||
|
||||
@@ -56,7 +56,6 @@ pub fn new_gauge_md(
|
||||
}
|
||||
|
||||
/// create a new histogram indicator descriptor
|
||||
#[allow(dead_code)]
|
||||
pub fn new_histogram_md(
|
||||
name: impl Into<MetricName>,
|
||||
help: impl Into<String>,
|
||||
|
||||
@@ -19,7 +19,6 @@ pub enum MetricNamespace {
|
||||
}
|
||||
|
||||
impl MetricNamespace {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::RustFS => "rustfs",
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
/// Format the path to the metric name format
|
||||
/// Replace '/' and '-' with '_'
|
||||
#[allow(dead_code)]
|
||||
pub fn format_path_to_metric_name(path: &str) -> String {
|
||||
path.trim_start_matches('/').replace(['/', '-'], "_")
|
||||
}
|
||||
|
||||
@@ -102,7 +102,6 @@ impl MetricSubsystem {
|
||||
}
|
||||
|
||||
/// Get the formatted metric name format string
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> String {
|
||||
format_path_to_metric_name(self.path())
|
||||
}
|
||||
@@ -151,7 +150,6 @@ impl MetricSubsystem {
|
||||
}
|
||||
|
||||
/// A convenient way to create custom subsystems directly
|
||||
#[allow(dead_code)]
|
||||
pub fn new(path: impl Into<String>) -> Self {
|
||||
Self::Custom(path.into())
|
||||
}
|
||||
@@ -176,7 +174,6 @@ impl std::fmt::Display for MetricSubsystem {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub mod subsystems {
|
||||
use super::MetricSubsystem;
|
||||
|
||||
|
||||
@@ -38,7 +38,10 @@ pub enum Rotation {
|
||||
Minutely,
|
||||
Hourly,
|
||||
Daily,
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "constructed only by this file's rolling-appender tests; the lib target cannot see them (backlog#1823)"
|
||||
)]
|
||||
Never,
|
||||
}
|
||||
|
||||
|
||||
@@ -219,10 +219,6 @@ impl PartialEq for Functions {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct Value;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::policy::Functions;
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn is_simple_match<P, N>(pattern: P, name: N) -> bool
|
||||
where
|
||||
P: AsRef<str>,
|
||||
@@ -29,7 +28,10 @@ where
|
||||
inner_match(pattern, name, false)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "prefix-matcher asserted by this file's tests; no production caller yet (backlog#1823)"
|
||||
)]
|
||||
pub fn is_match_as_pattern_prefix<P, N>(pattern: P, text: N) -> bool
|
||||
where
|
||||
P: AsRef<str>,
|
||||
|
||||
@@ -38,7 +38,6 @@ use std::collections::HashMap;
|
||||
/// - Account format is invalid
|
||||
/// - Credentials don't contain project_id
|
||||
/// - Account project_id doesn't match credentials project_id
|
||||
#[allow(dead_code)] // Used by Swift implementation
|
||||
pub fn validate_account_access(account: &str, credentials: &Credentials) -> SwiftResult<String> {
|
||||
// Extract project_id from account (strip "AUTH_" prefix)
|
||||
let account_project_id = account
|
||||
@@ -70,7 +69,6 @@ pub fn validate_account_access(account: &str, credentials: &Credentials) -> Swif
|
||||
///
|
||||
/// Admin users (with "admin" or "reseller_admin" roles) can perform
|
||||
/// cross-tenant operations and administrative tasks.
|
||||
#[allow(dead_code)] // Used by Swift implementation
|
||||
pub fn is_admin_user(credentials: &Credentials) -> bool {
|
||||
credentials
|
||||
.claims
|
||||
|
||||
@@ -144,7 +144,6 @@ impl ContainerMapper {
|
||||
/// - S3 bucket name compatible (only uses [a-z0-9-])
|
||||
/// - Deterministic mapping (same input always produces same bucket name)
|
||||
/// - Fixed-length prefix (16 hex chars = 8 bytes)
|
||||
#[allow(dead_code)] // Used in: create/delete container operations
|
||||
pub fn swift_to_s3_bucket(&self, container: &str, project_id: &str) -> String {
|
||||
if self.config.tenant_prefix_enabled {
|
||||
let hash = self.hash_project_id(project_id);
|
||||
@@ -216,7 +215,6 @@ pub fn bucket_info_to_container(info: &BucketInfo, mapper: &ContainerMapper, pro
|
||||
/// 2. Lists all S3 buckets
|
||||
/// 3. Filters to buckets belonging to this tenant (using tenant prefix)
|
||||
/// 4. Converts BucketInfo to Swift Container format
|
||||
#[allow(dead_code)] // Used by handler: list containers
|
||||
pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftResult<Vec<Container>> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -279,7 +277,6 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR
|
||||
/// - Returns 201 Created on success
|
||||
/// - Returns 202 Accepted if container already exists
|
||||
/// - Returns 400 Bad Request for invalid container names
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn create_container(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<bool> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -348,7 +345,6 @@ fn validate_container_name(container: &str) -> SwiftResult<()> {
|
||||
}
|
||||
|
||||
/// Container metadata for HEAD response
|
||||
#[allow(dead_code)] // TODO: Remove once Swift API integration is complete
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ContainerMetadata {
|
||||
/// Number of objects in container
|
||||
@@ -411,7 +407,6 @@ pub(crate) async fn get_container_custom_metadata(
|
||||
/// - HEAD /v1/{account}/{container} returns container metadata
|
||||
/// - Returns 204 No Content on success with headers
|
||||
/// - Returns 404 Not Found if container doesn't exist
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn get_container_metadata(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<ContainerMetadata> {
|
||||
let (bucket_name, bucket_info, custom_metadata) = get_container_metadata_base(account, container, credentials).await?;
|
||||
|
||||
@@ -448,7 +443,6 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials:
|
||||
/// - The update is additive: items the request does not name keep their stored
|
||||
/// value, and removal is explicit, via `X-Remove-Container-Meta-{name}` or an
|
||||
/// empty value
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn update_container_metadata(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -520,7 +514,6 @@ pub async fn update_container_metadata(
|
||||
/// - Returns 204 No Content on success
|
||||
/// - Returns 404 Not Found if container doesn't exist
|
||||
/// - Returns 409 Conflict if container is not empty
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn delete_container(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -603,7 +596,6 @@ pub async fn delete_container(account: &str, container: &str, credentials: &Cred
|
||||
/// - Account validation fails
|
||||
/// - Container doesn't exist
|
||||
/// - Storage layer errors occur
|
||||
#[allow(dead_code)] // Handler integration: GET container
|
||||
pub async fn list_objects(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -706,7 +698,6 @@ pub async fn list_objects(
|
||||
/// Versioning configuration is stored as an S3 bucket tag:
|
||||
/// - Tag key: `swift-versions-location`
|
||||
/// - Tag value: archive container name
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn enable_versioning(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -795,7 +786,6 @@ pub async fn enable_versioning(
|
||||
/// * `account` - Account identifier
|
||||
/// * `container` - Container name to disable versioning on
|
||||
/// * `credentials` - Keystone credentials
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn disable_versioning(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// Validate account access
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -855,7 +845,6 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
|
||||
/// # Returns
|
||||
/// - Some(archive_container_name) if versioning is enabled
|
||||
/// - None if versioning is not enabled
|
||||
#[allow(dead_code)] // Used by handler and object.rs
|
||||
pub async fn get_versions_location(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<Option<String>> {
|
||||
// Validate account access
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -918,7 +907,6 @@ pub async fn get_versions_location(account: &str, container: &str, credentials:
|
||||
/// &credentials
|
||||
/// ).await?;
|
||||
/// ```
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn set_container_acl(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -1022,7 +1010,6 @@ pub async fn set_container_acl(
|
||||
/// println!("Container is publicly readable");
|
||||
/// }
|
||||
/// ```
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn get_container_acl(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -1083,7 +1070,6 @@ pub async fn get_container_acl(
|
||||
///
|
||||
/// # Returns
|
||||
/// Ok(()) if ACLs were deleted successfully
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn delete_container_acl(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// Setting both ACLs to None removes them
|
||||
set_container_acl(account, container, None, None, credentials).await
|
||||
|
||||
@@ -20,7 +20,6 @@ use std::fmt;
|
||||
|
||||
/// Swift-specific error type
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)] // Error variants used by Swift implementation
|
||||
pub enum SwiftError {
|
||||
/// 400 Bad Request
|
||||
BadRequest(String),
|
||||
|
||||
@@ -122,12 +122,10 @@ fn swift_user_metadata(headers: &HeaderMap) -> Option<HashMap<String, String>> {
|
||||
///
|
||||
/// Handles URL encoding/decoding and path normalization for Swift object keys.
|
||||
/// Swift object names can contain any UTF-8 characters except null bytes.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub struct ObjectKeyMapper;
|
||||
|
||||
impl ObjectKeyMapper {
|
||||
/// Create a new object key mapper
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
@@ -140,7 +138,6 @@ impl ObjectKeyMapper {
|
||||
/// - Not contain null bytes
|
||||
/// - Not contain '..' path segments (directory traversal)
|
||||
/// - Not start with '/' (leading slash handled by routing)
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn validate_object_name(object: &str) -> SwiftResult<()> {
|
||||
if object.is_empty() {
|
||||
return Err(SwiftError::BadRequest("Object name cannot be empty".to_string()));
|
||||
@@ -183,7 +180,6 @@ impl ObjectKeyMapper {
|
||||
/// Example:
|
||||
/// - Swift: "photos/vacation/beach photo.jpg"
|
||||
/// - S3: "photos/vacation/beach photo.jpg"
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn swift_to_s3_key(object: &str) -> SwiftResult<String> {
|
||||
Self::validate_object_name(object)?;
|
||||
Ok(object.to_string())
|
||||
@@ -193,7 +189,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// This is essentially an identity transformation since we store
|
||||
/// Swift object names as-is in S3.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn s3_to_swift_name(key: &str) -> String {
|
||||
key.to_string()
|
||||
}
|
||||
@@ -208,7 +203,6 @@ impl ObjectKeyMapper {
|
||||
/// - Object: "vacation/beach.jpg"
|
||||
/// - Bucket: "abc123:photos"
|
||||
/// - Key: "vacation/beach.jpg"
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn build_s3_key(object: &str) -> SwiftResult<String> {
|
||||
Self::swift_to_s3_key(object)
|
||||
}
|
||||
@@ -220,7 +214,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// Example URL: /v1/AUTH_abc/container/path%2Fto%2Ffile.txt
|
||||
/// Decoded: "path/to/file.txt"
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn decode_object_from_url(encoded: &str) -> SwiftResult<String> {
|
||||
// Decode percent-encoding
|
||||
let decoded = urlencoding::decode(encoded).map_err(|e| SwiftError::BadRequest(format!("Invalid URL encoding: {}", e)))?;
|
||||
@@ -233,7 +226,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// When constructing URLs (e.g., for redirect responses), we need to
|
||||
/// percent-encode object names.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn encode_object_for_url(object: &str) -> String {
|
||||
urlencoding::encode(object).to_string()
|
||||
}
|
||||
@@ -241,7 +233,6 @@ impl ObjectKeyMapper {
|
||||
/// Check if object name represents a directory (pseudo-directory)
|
||||
///
|
||||
/// In Swift, objects ending with '/' are treated as directory markers.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn is_directory_marker(object: &str) -> bool {
|
||||
object.ends_with('/')
|
||||
}
|
||||
@@ -250,7 +241,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// Removes redundant slashes and normalizes the path while preserving
|
||||
/// trailing slashes for directory markers.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn normalize_path(object: &str) -> String {
|
||||
// Split by '/', filter out empty segments (except if it's the end)
|
||||
let has_trailing_slash = object.ends_with('/');
|
||||
@@ -324,7 +314,6 @@ fn sanitize_storage_error<E: std::fmt::Display>(operation: &str, error: E) -> Sw
|
||||
/// # Returns
|
||||
/// * `Ok(etag)` - Object ETag on success
|
||||
/// * `Err(SwiftError)` - Error if validation fails or upload fails
|
||||
#[allow(dead_code)] // Handler integration: PUT object
|
||||
pub async fn put_object<R>(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -445,7 +434,6 @@ where
|
||||
///
|
||||
/// Similar to put_object, but allows directly specifying metadata instead of extracting from headers.
|
||||
/// This is used internally for storing SLO manifests and marker objects.
|
||||
#[allow(dead_code)] // Used by SLO implementation
|
||||
pub async fn put_object_with_metadata<R>(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -549,7 +537,6 @@ where
|
||||
/// - `bytes=1000-1999` - Bytes 1000-1999
|
||||
/// - `bytes=1000-` - From byte 1000 to end
|
||||
/// - `bytes=-500` - Last 500 bytes
|
||||
#[allow(dead_code)] // Handler integration: GET object
|
||||
pub async fn get_object(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -608,7 +595,6 @@ pub async fn get_object(
|
||||
/// # Returns
|
||||
/// * `Ok(object_info)` - Object metadata (ObjectInfo)
|
||||
/// * `Err(SwiftError)` - Error if validation fails or object not found
|
||||
#[allow(dead_code)] // Handler integration: HEAD object
|
||||
pub async fn head_object(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -671,7 +657,6 @@ pub async fn head_object(
|
||||
/// # Returns
|
||||
/// * `Ok(())` - Object deleted successfully (or didn't exist)
|
||||
/// * `Err(SwiftError)` - Error if validation fails or deletion fails
|
||||
#[allow(dead_code)] // Handler integration: DELETE object
|
||||
pub async fn delete_object(account: &str, container: &str, object: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// 1. Validate account access and get project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -732,7 +717,6 @@ pub async fn delete_object(account: &str, container: &str, object: &str, credent
|
||||
/// # Returns
|
||||
/// * `Ok(())` - Metadata updated successfully
|
||||
/// * `Err(SwiftError)` - Error if validation fails, object not found, or update fails
|
||||
#[allow(dead_code)] // Handler integration: POST object
|
||||
pub async fn update_object_metadata(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -846,7 +830,6 @@ pub async fn update_object_metadata(
|
||||
/// # Handler Integration Note
|
||||
/// The current handler architecture needs to be updated to pass headers through
|
||||
/// to support COPY method and X-Copy-From header detection. See handler.rs for details.
|
||||
#[allow(dead_code)] // Handler integration: COPY object
|
||||
#[allow(clippy::too_many_arguments)] // Necessary for full copy functionality
|
||||
pub async fn copy_object(
|
||||
src_account: &str,
|
||||
@@ -979,7 +962,6 @@ pub async fn copy_object(
|
||||
/// assert_eq!(container, "my-container");
|
||||
/// assert_eq!(object, "path/to/file.txt");
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: COPY method
|
||||
pub fn parse_destination_header(destination: &str) -> SwiftResult<(String, String)> {
|
||||
let destination = destination.trim_start_matches('/');
|
||||
let parts: Vec<&str> = destination.splitn(2, '/').collect();
|
||||
@@ -1013,7 +995,6 @@ pub fn parse_destination_header(destination: &str) -> SwiftResult<(String, Strin
|
||||
/// # Returns
|
||||
/// * `Ok((container, object))` - Parsed container and object names
|
||||
/// * `Err(SwiftError)` - Error if format is invalid
|
||||
#[allow(dead_code)] // Handler integration: X-Copy-From
|
||||
pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)> {
|
||||
// Same parsing logic as Destination header
|
||||
parse_destination_header(copy_from)
|
||||
@@ -1042,7 +1023,6 @@ pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)>
|
||||
/// assert_eq!(range.start, 0);
|
||||
/// assert_eq!(range.end, 1023);
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: Range header
|
||||
pub fn parse_range_header(range_str: &str) -> SwiftResult<HTTPRangeSpec> {
|
||||
if !range_str.starts_with("bytes=") {
|
||||
return Err(SwiftError::BadRequest("Range header must start with 'bytes='".to_string()));
|
||||
@@ -1124,7 +1104,6 @@ pub fn parse_range_header(range_str: &str) -> SwiftResult<HTTPRangeSpec> {
|
||||
/// let header = format_content_range(0, 1023, 5000);
|
||||
/// assert_eq!(header, "bytes 0-1023/5000");
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: Range header
|
||||
pub fn format_content_range(start: i64, end: i64, total: i64) -> String {
|
||||
format!("bytes {}-{}/{}", start, end, total)
|
||||
}
|
||||
|
||||
@@ -50,7 +50,6 @@ pub enum SwiftRoute {
|
||||
|
||||
impl SwiftRoute {
|
||||
/// Get the account identifier from the route
|
||||
#[allow(dead_code)] // Public API for future use
|
||||
pub fn account(&self) -> &str {
|
||||
match self {
|
||||
SwiftRoute::Account { account, .. } => account,
|
||||
@@ -60,7 +59,6 @@ impl SwiftRoute {
|
||||
}
|
||||
|
||||
/// Extract project_id from account string (removes AUTH_ prefix)
|
||||
#[allow(dead_code)] // Public API for future use
|
||||
pub fn project_id(&self) -> Option<&str> {
|
||||
let account = self.account();
|
||||
ACCOUNT_PATTERN
|
||||
|
||||
@@ -19,7 +19,6 @@ use std::collections::HashMap;
|
||||
|
||||
/// Swift container metadata
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[allow(dead_code)] // Used in container listing operations
|
||||
pub struct Container {
|
||||
/// Container name
|
||||
pub name: String,
|
||||
@@ -34,7 +33,6 @@ pub struct Container {
|
||||
|
||||
/// Swift object metadata
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[allow(dead_code)] // Used in object listing operations
|
||||
pub struct Object {
|
||||
/// Object name (key)
|
||||
pub name: String,
|
||||
@@ -50,7 +48,6 @@ pub struct Object {
|
||||
|
||||
/// Swift metadata extracted from headers
|
||||
#[derive(Debug, Clone, Default)]
|
||||
#[allow(dead_code)] // Used by Swift implementation
|
||||
pub struct SwiftMetadata {
|
||||
/// Custom metadata key-value pairs (from X-Container-Meta-* or X-Object-Meta-*)
|
||||
pub metadata: HashMap<String, String>,
|
||||
|
||||
@@ -6,9 +6,18 @@
|
||||
#
|
||||
# The MinIO release is pinned so the captured fixture format is reproducible;
|
||||
# this is the release the interop tests were validated against.
|
||||
FROM minio/minio:RELEASE.2025-09-07T16-13-09Z AS minio
|
||||
#
|
||||
# Both base images are build args so a network that cannot reach Docker Hub can
|
||||
# point them at a mirror carrying the same content — quay.io publishes the MinIO
|
||||
# releases, and public.ecr.aws mirrors the official Python images. CI keeps the
|
||||
# Docker Hub defaults. Override with:
|
||||
# --build-arg MINIO_IMAGE=quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z \
|
||||
# --build-arg PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim
|
||||
ARG MINIO_IMAGE=minio/minio:RELEASE.2025-09-07T16-13-09Z
|
||||
ARG PYTHON_IMAGE=python:3.12-slim
|
||||
FROM ${MINIO_IMAGE} AS minio
|
||||
|
||||
FROM python:3.12-slim
|
||||
FROM ${PYTHON_IMAGE}
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends openssl ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -22,6 +22,18 @@ Use the automated path when you want the lab to:
|
||||
- upload a predefined SSE fixture case
|
||||
- export the generated backend tree into the lab layout
|
||||
|
||||
## Networks without Docker Hub access
|
||||
|
||||
`capture_via_docker.sh` pulls its two base images from Docker Hub by default. Where that registry is unreachable, point the build at mirrors carrying the same content — quay.io publishes the MinIO releases and public.ecr.aws mirrors the official Python images:
|
||||
|
||||
```bash
|
||||
MINIO_LAB_MINIO_IMAGE=quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z \
|
||||
MINIO_LAB_PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim \
|
||||
./capture_via_docker.sh
|
||||
```
|
||||
|
||||
Pin the MinIO tag to the same release the Dockerfile names; an unpinned `:latest` captures whatever format that day's build writes, which is not what the interop tests were validated against.
|
||||
|
||||
## Layout
|
||||
|
||||
The default root is `artifacts/minio-fixture-lab`, which is already ignored by the repository.
|
||||
|
||||
@@ -34,8 +34,19 @@ if [ "${cases[0]}" != "all" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# Base images are overridable so a network without Docker Hub access can point
|
||||
# them at a mirror (see the Dockerfile header). Unset by default, which keeps the
|
||||
# Dockerfile's Docker Hub defaults for CI.
|
||||
build_args=()
|
||||
if [ -n "${MINIO_LAB_MINIO_IMAGE:-}" ]; then
|
||||
build_args+=(--build-arg "MINIO_IMAGE=${MINIO_LAB_MINIO_IMAGE}")
|
||||
fi
|
||||
if [ -n "${MINIO_LAB_PYTHON_IMAGE:-}" ]; then
|
||||
build_args+=(--build-arg "PYTHON_IMAGE=${MINIO_LAB_PYTHON_IMAGE}")
|
||||
fi
|
||||
|
||||
echo ">> building ${IMAGE}"
|
||||
docker build -f "${SCRIPT_DIR}/Dockerfile" -t "${IMAGE}" "${SCRIPT_DIR}"
|
||||
docker build -f "${SCRIPT_DIR}/Dockerfile" -t "${IMAGE}" "${build_args[@]}" "${SCRIPT_DIR}"
|
||||
|
||||
echo ">> capturing fixtures into ${FIXTURE_REL}"
|
||||
docker run --rm -v "${REPO_ROOT}:/repo" "${IMAGE}" \
|
||||
|
||||
@@ -4621,6 +4621,8 @@ mod tests {
|
||||
let _subscriber_guard = tracing::subscriber::set_default(subscriber);
|
||||
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
// Canonicalize for the "drive" field comparison (scanner resolves symlinks).
|
||||
let canonical_temp_dir = std::fs::canonicalize(&temp_dir).unwrap_or_else(|_| temp_dir.clone());
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
|
||||
let object_dir = temp_dir.join("bucket").join("object");
|
||||
@@ -4689,7 +4691,7 @@ mod tests {
|
||||
let fields = &events[0]["fields"];
|
||||
assert_eq!(fields["component"], LOG_COMPONENT_SCANNER);
|
||||
assert_eq!(fields["subsystem"], LOG_SUBSYSTEM_FOLDER);
|
||||
assert_eq!(fields["drive"], temp_dir.to_string_lossy().as_ref());
|
||||
assert_eq!(fields["drive"], canonical_temp_dir.to_string_lossy().as_ref());
|
||||
assert_eq!(fields["bucket"], "bucket");
|
||||
assert_eq!(fields["object"], "object");
|
||||
assert_eq!(fields["metadata_path"], metadata_path.to_string_lossy().as_ref());
|
||||
|
||||
@@ -27,6 +27,10 @@ use crate::CloudMetadataFetcher;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AwsMetadataFetcher {
|
||||
client: Client,
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "IMDS endpoint retained beside the client it configures; requests build their own URLs (backlog#1823)"
|
||||
)]
|
||||
metadata_endpoint: String,
|
||||
}
|
||||
|
||||
@@ -46,55 +50,6 @@ impl AwsMetadataFetcher {
|
||||
metadata_endpoint: "http://169.254.169.254".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieves an IMDSv2 token for secure metadata access.
|
||||
#[allow(dead_code)]
|
||||
async fn get_metadata_token(&self) -> Result<String, AppError> {
|
||||
let url = format!("{}/latest/api/token", self.metadata_endpoint);
|
||||
|
||||
match self
|
||||
.client
|
||||
.put(&url)
|
||||
.header("X-aws-ec2-metadata-token-ttl-seconds", "21600")
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => {
|
||||
if response.status().is_success() {
|
||||
let token = response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| AppError::cloud(format!("Failed to read IMDSv2 token: {}", e)))?;
|
||||
Ok(token)
|
||||
} else {
|
||||
debug!(
|
||||
event = "trusted_proxies.cloud_metadata",
|
||||
component = "trusted_proxies",
|
||||
subsystem = "aws_metadata",
|
||||
provider = "aws",
|
||||
operation = "imdsv2_token",
|
||||
result = "http_error",
|
||||
status = %response.status(),
|
||||
"trusted proxy cloud metadata request failed"
|
||||
);
|
||||
Err(AppError::cloud("Failed to obtain IMDSv2 token"))
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
event = "trusted_proxies.cloud_metadata",
|
||||
component = "trusted_proxies",
|
||||
subsystem = "aws_metadata",
|
||||
provider = "aws",
|
||||
operation = "imdsv2_token",
|
||||
result = "request_failed",
|
||||
error = %e,
|
||||
"trusted proxy cloud metadata request failed"
|
||||
);
|
||||
Err(AppError::cloud(format!("IMDSv2 request failed: {}", e)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -68,7 +68,6 @@ pub fn is_env_set(key: &str) -> bool {
|
||||
}
|
||||
|
||||
/// Returns a list of all proxy-related environment variables and their current values.
|
||||
#[allow(dead_code)]
|
||||
pub fn get_all_proxy_env_vars() -> Vec<(String, String)> {
|
||||
let vars = [
|
||||
ENV_TRUSTED_PROXY_ENABLED,
|
||||
|
||||
@@ -68,7 +68,6 @@ pub async fn read_full_or_eof<R: AsyncRead + Send + Sync + Unpin>(
|
||||
|
||||
/// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before any bytes are read.
|
||||
/// Like Go's io.ReadFull.
|
||||
#[allow(dead_code)]
|
||||
pub async fn read_full<R: AsyncRead + Send + Sync + Unpin>(reader: R, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
match read_full_or_eof(reader, buf).await? {
|
||||
Some(n) => Ok(n),
|
||||
|
||||
@@ -431,7 +431,6 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result<SocketAddr>
|
||||
Ok(resolved_addr)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = Result<Bytes, E>> + Send + 'static
|
||||
where
|
||||
S: Stream<Item = Result<Bytes, E>> + Send + 'static,
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
///
|
||||
/// The table follows Linux `include/uapi/linux/magic.h`; filesystem magic
|
||||
/// values without a stable Linux uapi source stay `UNKNOWN`.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn get_fs_type(fs_type: u64) -> &'static str {
|
||||
// Magic numbers for various filesystems.
|
||||
match fs_type {
|
||||
|
||||
@@ -70,7 +70,6 @@ pub fn is_dir_object(object: &str) -> bool {
|
||||
///
|
||||
/// If the object name ends with `GLOBAL_DIR_SUFFIX`, it is replaced with a slash.
|
||||
/// Otherwise, the name is returned as is.
|
||||
#[allow(dead_code)]
|
||||
pub fn decode_dir_object(object: &str) -> String {
|
||||
if has_suffix(object, GLOBAL_DIR_SUFFIX) {
|
||||
format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
|
||||
|
||||
For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
|
||||
For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
|
||||
|
||||
## Backend comparison
|
||||
|
||||
|
||||
@@ -8,9 +8,12 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments |
|
||||
| AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar |
|
||||
| Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute |
|
||||
| Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth |
|
||||
|
||||
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous.
|
||||
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity.
|
||||
|
||||
All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`.
|
||||
|
||||
The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification.
|
||||
|
||||
@@ -56,7 +59,44 @@ Deliver the SecretID out of band — a secrets-manager-mounted file, an init-con
|
||||
|
||||
The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login.
|
||||
|
||||
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart.
|
||||
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart.
|
||||
|
||||
## Kubernetes authentication
|
||||
|
||||
On Kubernetes this is the method to prefer: the pod's own ServiceAccount is the identity, so there is no credential to distribute, rotate, or leak into a Secret.
|
||||
|
||||
### Vault-side setup
|
||||
|
||||
```shell
|
||||
vault auth enable kubernetes
|
||||
|
||||
vault write auth/kubernetes/config \
|
||||
kubernetes_host="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT"
|
||||
|
||||
vault write auth/kubernetes/role/rustfs \
|
||||
bound_service_account_names=rustfs \
|
||||
bound_service_account_namespaces=rustfs \
|
||||
token_policies=rustfs-kms \
|
||||
token_ttl=1h
|
||||
```
|
||||
|
||||
As with AppRole, keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s).
|
||||
|
||||
### RustFS configuration
|
||||
|
||||
```shell
|
||||
RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend
|
||||
RUSTFS_KMS_VAULT_ADDRESS=https://vault.vault.svc.cluster.local:8200
|
||||
RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs
|
||||
# Optional, defaults to "kubernetes":
|
||||
# RUSTFS_KMS_VAULT_KUBERNETES_MOUNT=kubernetes
|
||||
# Optional, defaults to the kubelet's projected token path:
|
||||
# RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
```
|
||||
|
||||
RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart.
|
||||
|
||||
A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own.
|
||||
|
||||
## Vault Agent token file
|
||||
|
||||
@@ -101,13 +141,13 @@ If the agent stops refreshing the file that is fine — RustFS re-reads the same
|
||||
|
||||
## Fail-closed window
|
||||
|
||||
For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
|
||||
For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
|
||||
|
||||
- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it.
|
||||
- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration.
|
||||
- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration.
|
||||
- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise.
|
||||
|
||||
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file).
|
||||
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file).
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
@@ -117,6 +157,8 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r
|
||||
| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file |
|
||||
| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider |
|
||||
| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider |
|
||||
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file |
|
||||
| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server |
|
||||
| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider |
|
||||
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file |
|
||||
|
||||
When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault.
|
||||
|
||||
@@ -286,6 +286,7 @@ fn auth_method_kind(auth: &VaultAuthMethod) -> String {
|
||||
match auth {
|
||||
VaultAuthMethod::Token { .. } => "token",
|
||||
VaultAuthMethod::AppRole { .. } => "approle",
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes",
|
||||
VaultAuthMethod::TokenFile { .. } => "token-file",
|
||||
}
|
||||
.to_string()
|
||||
@@ -484,7 +485,10 @@ fn business_trust_root_secrets(config: &KmsConfig) -> Vec<Zeroizing<String>> {
|
||||
secrets.push(Zeroizing::new(role_id.clone()));
|
||||
secrets.push(Zeroizing::new(secret_id.clone()));
|
||||
}
|
||||
VaultAuthMethod::TokenFile { .. } => {}
|
||||
// Kubernetes and TokenFile hold no inline plaintext credential: the
|
||||
// ServiceAccount token and the agent-managed token live in files, and
|
||||
// the role names a Vault binding rather than half a credential pair.
|
||||
VaultAuthMethod::Kubernetes { .. } | VaultAuthMethod::TokenFile { .. } => {}
|
||||
};
|
||||
|
||||
match &config.backend_config {
|
||||
|
||||
@@ -16,8 +16,8 @@ use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::current_scanner_metrics_report;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env};
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use crate::startup_background::{ENV_SCANNER_ENABLED, scanner_enabled_from_env};
|
||||
use chrono::Utc;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -417,6 +417,13 @@ struct SystemAdminDiscovery {
|
||||
struct ServerInfoResponse {
|
||||
info: InfoMessage,
|
||||
admin_discovery: SystemAdminDiscovery,
|
||||
/// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873):
|
||||
/// `passed` (algorithms verified at boot), `failed` (a drifted hash
|
||||
/// implementation — the process is serving with degraded integrity
|
||||
/// checking unless `RUSTFS_BITROT_SELFTEST_STRICT` aborted it), or
|
||||
/// `unknown` (not yet run or disabled).
|
||||
#[serde(rename = "bitrotSelftest")]
|
||||
bitrot_selftest: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -433,6 +440,14 @@ fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery
|
||||
}
|
||||
}
|
||||
|
||||
fn bitrot_selftest_status_str() -> &'static str {
|
||||
match crate::bitrot_selftest::bitrot_selftest_passed() {
|
||||
Some(true) => "passed",
|
||||
Some(false) => "failed",
|
||||
None => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for ServerInfoHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
@@ -464,6 +479,7 @@ impl Operation for ServerInfoHandler {
|
||||
let response = ServerInfoResponse {
|
||||
info,
|
||||
admin_discovery: system_admin_discovery(&usecase),
|
||||
bitrot_selftest: bitrot_selftest_status_str(),
|
||||
};
|
||||
|
||||
let data = serde_json::to_vec(&response).map_err(|e| {
|
||||
@@ -1535,6 +1551,18 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The startup bitrot self-test outcome must surface in server info as one
|
||||
/// of three closed-set strings, never an internal enum or a null
|
||||
/// (rustfs/backlog#1873). This test pins the string mapping; whether the
|
||||
/// process-global cell holds Some(true)/Some(false)/None is owned by
|
||||
/// `crate::bitrot_selftest`'s own tests.
|
||||
#[test]
|
||||
fn bitrot_selftest_status_str_is_a_closed_set_of_operators_strings() {
|
||||
let rendered = super::bitrot_selftest_status_str();
|
||||
assert!(matches!(rendered, "passed" | "failed" | "unknown"));
|
||||
assert_eq!(super::bitrot_selftest_status_str(), rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_info_response_exposes_admin_discovery_paths() {
|
||||
let usecase = DefaultAdminUsecase::without_context();
|
||||
@@ -1556,6 +1584,7 @@ mod tests {
|
||||
pools: None,
|
||||
},
|
||||
admin_discovery: system_admin_discovery(&usecase),
|
||||
bitrot_selftest: super::bitrot_selftest_status_str(),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(response).expect("server info response should serialize");
|
||||
|
||||
@@ -2995,9 +2995,9 @@ fn table_entry_from_create_table_request(
|
||||
let CreateTableRequest {
|
||||
name,
|
||||
location,
|
||||
schema,
|
||||
partition_spec,
|
||||
write_order,
|
||||
mut schema,
|
||||
mut partition_spec,
|
||||
mut write_order,
|
||||
stage_create,
|
||||
mut properties,
|
||||
} = request;
|
||||
@@ -3031,6 +3031,9 @@ fn table_entry_from_create_table_request(
|
||||
let metadata_location =
|
||||
crate::table_catalog::default_table_metadata_file_path(namespace, &table, &next_metadata_file_name(1, &table_id));
|
||||
|
||||
crate::table_catalog::assign_fresh_create_schema_ids(&mut schema, partition_spec.as_mut(), write_order.as_mut())
|
||||
.map_err(catalog_store_error)?;
|
||||
|
||||
let entry = crate::table_catalog::TableEntry {
|
||||
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
|
||||
table_bucket: bucket.to_string(),
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user