mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 16:46:55 +00:00
fix(data-usage): make thin usage-cache types a read-only projection (#5981)
Delete the dead ecstore save_data_usage_cache and the thin DataUsageCache::marshal_msg it was the only caller of, drop the Serialize derive (and the dead DataUsageCacheStorage trait with its save path) from the thin projection types so no write path can exist outside the scanner's canonical map-encoded writer, and pin the persisted .usage-cache.bin wire bytes with cross-crate fixture tests on both the scanner writer and the thin reader. Refs rustfs/backlog#1828 (T1-T3).
This commit is contained in:
Generated
-1
@@ -9345,7 +9345,6 @@ dependencies = [
|
|||||||
name = "rustfs-data-usage"
|
name = "rustfs-data-usage"
|
||||||
version = "1.0.0-rc.1"
|
version = "1.0.0-rc.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
|
||||||
"hotpath",
|
"hotpath",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rustfs-filemeta",
|
"rustfs-filemeta",
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
|
|||||||
hotpath.workspace = true
|
hotpath.workspace = true
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
rmp-serde = { workspace = true }
|
rmp-serde = { workspace = true }
|
||||||
async-trait = { workspace = true }
|
|
||||||
rustfs-filemeta = { workspace = true }
|
rustfs-filemeta = { workspace = true }
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
|
|||||||
@@ -846,8 +846,15 @@ impl DataUsageEntry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Data usage cache info
|
/// Read-only projection of the scanner's `.usage-cache.bin` info block.
|
||||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
///
|
||||||
|
/// The canonical wire format is written by the hand-written map-encoded
|
||||||
|
/// `Serialize` on the scanner-side `DataUsageCacheInfo`
|
||||||
|
/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields.
|
||||||
|
/// This type decodes only the shared subset and is deliberately not
|
||||||
|
/// `Serialize`: a derived (array) encoding of this 6-field subset would
|
||||||
|
/// corrupt the cache for scanner readers, so no write path may exist here.
|
||||||
|
#[derive(Clone, Debug, Default, Deserialize)]
|
||||||
pub struct DataUsageCacheInfo {
|
pub struct DataUsageCacheInfo {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
pub next_cycle: u64,
|
pub next_cycle: u64,
|
||||||
@@ -863,8 +870,12 @@ pub struct DataUsageCacheInfo {
|
|||||||
pub snapshot_complete: bool,
|
pub snapshot_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Data usage cache
|
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
///
|
||||||
|
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||||
|
/// owns the persisted format; this type only decodes it (see
|
||||||
|
/// [`DataUsageCacheInfo`]) and must never grow a serialization path.
|
||||||
|
#[derive(Clone, Debug, Default, Deserialize)]
|
||||||
pub struct DataUsageCache {
|
pub struct DataUsageCache {
|
||||||
pub info: DataUsageCacheInfo,
|
pub info: DataUsageCacheInfo,
|
||||||
pub cache: HashMap<String, DataUsageEntry>,
|
pub cache: HashMap<String, DataUsageEntry>,
|
||||||
@@ -1186,31 +1197,10 @@ impl DataUsageCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn marshal_msg(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
let mut buf = Vec::new();
|
|
||||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
|
||||||
Ok(buf)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let t: Self = rmp_serde::from_slice(buf)?;
|
let t: Self = rmp_serde::from_slice(buf)?;
|
||||||
Ok(t)
|
Ok(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Note: load and save methods are storage-specific and should be implemented
|
|
||||||
// in the ecstore crate where storage access is available
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Trait for storage-specific operations on DataUsageCache
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
pub trait DataUsageCacheStorage {
|
|
||||||
/// Load data usage cache from backend storage
|
|
||||||
async fn load(store: &dyn std::any::Any, name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>>
|
|
||||||
where
|
|
||||||
Self: Sized;
|
|
||||||
|
|
||||||
/// Save data usage cache to backend storage
|
|
||||||
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper structs and functions for cache operations
|
// Helper structs and functions for cache operations
|
||||||
@@ -1832,6 +1822,82 @@ mod tests {
|
|||||||
assert!(decoded.all_tier_stats.is_none());
|
assert!(decoded.all_tier_stats.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Scanner-written `.usage-cache.bin` bytes: a 2-element array of the
|
||||||
|
/// canonical 16-field map-encoded info block and one map-encoded entry.
|
||||||
|
/// Captured from the canonical writer's `marshal_msg` — see
|
||||||
|
/// `usage_cache_wire_format_is_pinned` in
|
||||||
|
/// `crates/scanner/src/data_usage_define.rs`, which pins these exact
|
||||||
|
/// bytes and documents regeneration. Hardcoded here because a
|
||||||
|
/// dev-dependency on rustfs-scanner would pull the whole ecstore tree
|
||||||
|
/// into this crate's test build, and a fixture generated at test runtime
|
||||||
|
/// could not detect writer drift anyway.
|
||||||
|
const SCANNER_USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
||||||
|
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||||
|
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
||||||
|
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
||||||
|
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
||||||
|
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
||||||
|
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
||||||
|
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
||||||
|
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
||||||
|
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
||||||
|
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
||||||
|
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
||||||
|
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
||||||
|
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
||||||
|
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
||||||
|
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
||||||
|
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
||||||
|
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
||||||
|
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
||||||
|
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||||
|
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
||||||
|
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
||||||
|
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
||||||
|
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
||||||
|
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
||||||
|
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
||||||
|
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
||||||
|
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
||||||
|
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
||||||
|
];
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn thin_usage_cache_decodes_scanner_wire_fixture() {
|
||||||
|
let decoded =
|
||||||
|
DataUsageCache::unmarshal(SCANNER_USAGE_CACHE_WIRE_FIXTURE).expect("thin projection decodes a scanner-written cache");
|
||||||
|
|
||||||
|
// The six fields shared with the scanner's 16-field info block; the
|
||||||
|
// remaining ten (lifecycle, replication, checkpoint, heals, ...) must
|
||||||
|
// be skipped, not error.
|
||||||
|
assert_eq!(decoded.info.name, "wire-bucket");
|
||||||
|
assert_eq!(decoded.info.next_cycle, 7);
|
||||||
|
assert_eq!(
|
||||||
|
decoded.info.last_update,
|
||||||
|
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
||||||
|
);
|
||||||
|
assert!(decoded.info.skip_healing);
|
||||||
|
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
||||||
|
assert!(decoded.info.snapshot_complete);
|
||||||
|
|
||||||
|
// Entries use the shared canonical map-encoded type end to end.
|
||||||
|
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
||||||
|
assert_eq!(entry.size, 4096);
|
||||||
|
assert_eq!(entry.objects, 3);
|
||||||
|
assert_eq!(entry.versions, 5);
|
||||||
|
assert_eq!(entry.delete_markers, 1);
|
||||||
|
assert!(entry.compacted);
|
||||||
|
assert_eq!(entry.failed_objects, 2);
|
||||||
|
assert_eq!(
|
||||||
|
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
||||||
|
Some(&TierStats {
|
||||||
|
total_size: 2048,
|
||||||
|
num_versions: 2,
|
||||||
|
num_objects: 1,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hash_path_uses_portable_slash_semantics() {
|
fn hash_path_uses_portable_slash_semantics() {
|
||||||
for (input, expected) in [
|
for (input, expected) in [
|
||||||
|
|||||||
@@ -2150,30 +2150,6 @@ pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str
|
|||||||
Ok(d)
|
Ok(d)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(cache))]
|
|
||||||
pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate::error::Result<()> {
|
|
||||||
use crate::config::com::save_config;
|
|
||||||
use crate::disk::BUCKET_META_PREFIX;
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
let Some(store) = runtime_sources::object_store_handle() else {
|
|
||||||
return Err(Error::other("errServerNotInitialized"));
|
|
||||||
};
|
|
||||||
let buf = cache.marshal_msg().map_err(Error::other)?;
|
|
||||||
let buf_clone = buf.clone();
|
|
||||||
|
|
||||||
let store_clone = store.clone();
|
|
||||||
|
|
||||||
let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string();
|
|
||||||
|
|
||||||
let name_clone = name.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let _ = save_config(store_clone, &format!("{}{}", name_clone, ".bkp"), buf_clone).await;
|
|
||||||
});
|
|
||||||
save_config(store, &name, buf).await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Persist the current in-memory compression total to the backend.
|
/// Persist the current in-memory compression total to the backend.
|
||||||
/// Resets the debounce counter so the next auto-persist won't fire
|
/// Resets the debounce counter so the next auto-persist won't fire
|
||||||
/// immediately after this manual flush (intended for shutdown paths).
|
/// immediately after this manual flush (intended for shutdown paths).
|
||||||
|
|||||||
@@ -2403,6 +2403,150 @@ mod tests {
|
|||||||
assert_eq!(decoded.cache.get("bucket").map(|entry| entry.objects), Some(3));
|
assert_eq!(decoded.cache.get("bucket").map(|entry| entry.objects), Some(3));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Deterministic, fully populated cache used to pin the persisted
|
||||||
|
/// `.usage-cache.bin` wire bytes. Every map/set holds at most one element
|
||||||
|
/// so the map-encoded `marshal_msg` output is byte-stable.
|
||||||
|
fn wire_fixture_cache() -> DataUsageCache {
|
||||||
|
let mut entry = DataUsageEntry {
|
||||||
|
size: 4096,
|
||||||
|
objects: 3,
|
||||||
|
versions: 5,
|
||||||
|
delete_markers: 1,
|
||||||
|
compacted: true,
|
||||||
|
failed_objects: 2,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
entry.add_tier_sizes(&HashMap::from([(
|
||||||
|
"WARM".to_string(),
|
||||||
|
TierStats {
|
||||||
|
total_size: 2048,
|
||||||
|
num_versions: 2,
|
||||||
|
num_objects: 1,
|
||||||
|
},
|
||||||
|
)]));
|
||||||
|
let mut cache = DataUsageCache {
|
||||||
|
info: DataUsageCacheInfo {
|
||||||
|
name: "wire-bucket".to_string(),
|
||||||
|
next_cycle: 7,
|
||||||
|
leader_epoch: 9,
|
||||||
|
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000)),
|
||||||
|
skip_healing: true,
|
||||||
|
failed_objects: HashMap::from([("wire-bucket/lost".to_string(), 11)]),
|
||||||
|
scan_resume_after: Some("wire-bucket/resume".to_string()),
|
||||||
|
pending_heals: vec![PendingScannerHeal {
|
||||||
|
kind: PendingScannerHealKind::Object,
|
||||||
|
bucket: "wire-bucket".to_string(),
|
||||||
|
object: Some("broken".to_string()),
|
||||||
|
version_id: None,
|
||||||
|
scan_mode: HealScanMode::Normal,
|
||||||
|
first_seen: 100,
|
||||||
|
last_attempt: 200,
|
||||||
|
attempts: 3,
|
||||||
|
last_admission_result: "deferred".to_string(),
|
||||||
|
last_admission_reason: "budget".to_string(),
|
||||||
|
}],
|
||||||
|
source: Some(DataUsageCacheSource::new(1, 2)),
|
||||||
|
snapshot_complete: true,
|
||||||
|
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||||
|
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
cache.replace("wire-bucket", "", entry);
|
||||||
|
cache
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persisted `.usage-cache.bin` bytes produced by [`wire_fixture_cache`]
|
||||||
|
/// via `DataUsageCache::marshal_msg`: a 2-element array of the 16-field
|
||||||
|
/// map-encoded info block and the map of map-encoded entries.
|
||||||
|
///
|
||||||
|
/// The thin read-only projection in `crates/data-usage` decodes a copy of
|
||||||
|
/// this fixture (`thin_usage_cache_decodes_scanner_wire_fixture`); when
|
||||||
|
/// the encoding legitimately changes, regenerate both copies from
|
||||||
|
/// `wire_fixture_cache().marshal_msg()` and re-verify old readers.
|
||||||
|
const USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
||||||
|
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||||
|
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
||||||
|
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
||||||
|
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
||||||
|
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
||||||
|
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
||||||
|
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
||||||
|
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
||||||
|
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
||||||
|
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
||||||
|
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
||||||
|
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
||||||
|
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
||||||
|
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
||||||
|
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
||||||
|
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
||||||
|
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
||||||
|
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
||||||
|
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||||
|
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
||||||
|
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
||||||
|
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
||||||
|
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
||||||
|
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
||||||
|
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
||||||
|
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
||||||
|
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
||||||
|
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
||||||
|
];
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn usage_cache_wire_format_is_pinned() {
|
||||||
|
// Writer: the canonical map-encoded serializer must reproduce the
|
||||||
|
// pinned bytes. Round-trip tests cannot see format drift, so any
|
||||||
|
// encoding change (field rename/reorder, map->array switch) fails
|
||||||
|
// here and forces re-verifying old readers and the thin projection
|
||||||
|
// in crates/data-usage before the fixture is regenerated.
|
||||||
|
let encoded = wire_fixture_cache().marshal_msg().expect("marshal fixture cache");
|
||||||
|
assert_eq!(
|
||||||
|
encoded.as_slice(),
|
||||||
|
USAGE_CACHE_WIRE_FIXTURE,
|
||||||
|
"persisted .usage-cache.bin encoding drifted from the pinned fixture"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Reader: the pinned bytes decode with every field intact.
|
||||||
|
let decoded = DataUsageCache::unmarshal(USAGE_CACHE_WIRE_FIXTURE).expect("decode pinned fixture");
|
||||||
|
assert_eq!(decoded.info.name, "wire-bucket");
|
||||||
|
assert_eq!(decoded.info.next_cycle, 7);
|
||||||
|
assert_eq!(decoded.info.leader_epoch, 9);
|
||||||
|
assert_eq!(
|
||||||
|
decoded.info.last_update,
|
||||||
|
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
||||||
|
);
|
||||||
|
assert!(decoded.info.skip_healing);
|
||||||
|
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
||||||
|
assert_eq!(decoded.info.scan_resume_after.as_deref(), Some("wire-bucket/resume"));
|
||||||
|
assert_eq!(decoded.info.pending_heals.len(), 1);
|
||||||
|
assert_eq!(decoded.info.pending_heals[0].kind, PendingScannerHealKind::Object);
|
||||||
|
assert_eq!(decoded.info.pending_heals[0].object.as_deref(), Some("broken"));
|
||||||
|
assert_eq!(decoded.info.source, Some(DataUsageCacheSource::new(1, 2)));
|
||||||
|
assert!(decoded.info.snapshot_complete);
|
||||||
|
assert_eq!(decoded.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
|
||||||
|
assert_eq!(decoded.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||||
|
|
||||||
|
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
||||||
|
assert_eq!(entry.size, 4096);
|
||||||
|
assert_eq!(entry.objects, 3);
|
||||||
|
assert_eq!(entry.versions, 5);
|
||||||
|
assert_eq!(entry.delete_markers, 1);
|
||||||
|
assert!(entry.compacted);
|
||||||
|
assert_eq!(entry.failed_objects, 2);
|
||||||
|
assert_eq!(
|
||||||
|
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
||||||
|
Some(&TierStats {
|
||||||
|
total_size: 2048,
|
||||||
|
num_versions: 2,
|
||||||
|
num_objects: 1,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn data_usage_cache_prepare_for_scan_rejects_unscoped_distributed_cache() {
|
fn data_usage_cache_prepare_for_scan_rejects_unscoped_distributed_cache() {
|
||||||
let mut cache = DataUsageCache {
|
let mut cache = DataUsageCache {
|
||||||
|
|||||||
Reference in New Issue
Block a user