mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
houseme
25c6bdf490
* chore(perf): harden amd64 profiling benchmark flow * fix(profiling): isolate bench buckets and map protobuf conflict * perf: avoid blocking owned local writes * style: format profile admin handler * docs: clarify observability trace validation * perf: reduce mkdir overhead on local writes * perf: add rename_data meta microbenchmark * perf(filemeta): fast-path data_dir decode in version meta * perf(filemeta): collapse data-dir lookup into one scan * perf(filemeta): reduce scan allocs and refresh meta bench * perf(ecstore): skip mkdir path on read-only open * perf(filemeta): single-pass unshared data-dir scan * perf(filemeta): add two-key inline remove fast path * perf(filemeta): compare remove-two keys by bytes first * bench(ecstore): add remove_two-only micro benchmark * bench(ecstore): stabilize rename_data meta benchmark timing * bench(ecstore): align rename_data path with remove_two * perf(filemeta): avoid uuid string alloc in remove_two * perf(filemeta): add fast-path for empty inline data * perf(filemeta): streamline add_version match branch * perf(filemeta): fast-return remove_key on miss * perf(filemeta): speed up add_version insertion lookup * style(ecstore): normalize formatting in perf-tuning files * refactor(filemeta): unify inline data removal paths
412 lines
12 KiB
Rust
412 lines
12 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use crate::error::{Error, Result};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::io::{Cursor, Read};
|
|
use uuid::Uuid;
|
|
|
|
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
|
pub struct InlineData(Vec<u8>);
|
|
|
|
const INLINE_DATA_VER: u8 = 1;
|
|
|
|
impl InlineData {
|
|
fn contains_key_by<F>(&self, mut should_remove: F) -> Result<bool>
|
|
where
|
|
F: FnMut(&[u8]) -> bool,
|
|
{
|
|
let buf = self.after_version();
|
|
if buf.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut scan_cur = Cursor::new(buf);
|
|
let mut scan_fields_len = rmp::decode::read_map_len(&mut scan_cur)? as usize;
|
|
|
|
while scan_fields_len > 0 {
|
|
scan_fields_len -= 1;
|
|
|
|
let str_len = rmp::decode::read_str_len(&mut scan_cur)? as usize;
|
|
let key_start = scan_cur.position() as usize;
|
|
let key_end = key_start + str_len;
|
|
scan_cur.set_position(key_end as u64);
|
|
|
|
let bin_len = rmp::decode::read_bin_len(&mut scan_cur)? as usize;
|
|
let value_start = scan_cur.position() as usize;
|
|
let value_end = value_start + bin_len;
|
|
scan_cur.set_position(value_end as u64);
|
|
|
|
if should_remove(&buf[key_start..key_end]) {
|
|
return Ok(true);
|
|
}
|
|
}
|
|
|
|
Ok(false)
|
|
}
|
|
|
|
fn remove_keys_by<F>(&mut self, mut should_remove: F) -> Result<bool>
|
|
where
|
|
F: FnMut(&[u8]) -> bool,
|
|
{
|
|
let buf = self.after_version();
|
|
if buf.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut cur = Cursor::new(buf);
|
|
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
|
|
let mut keys = Vec::with_capacity(fields_len);
|
|
let mut values = Vec::with_capacity(fields_len);
|
|
let mut found = false;
|
|
|
|
while fields_len > 0 {
|
|
fields_len -= 1;
|
|
|
|
let str_len = rmp::decode::read_str_len(&mut cur)? as usize;
|
|
let mut field_buf = vec![0u8; str_len];
|
|
cur.read_exact(&mut field_buf)?;
|
|
|
|
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
|
let start = cur.position() as usize;
|
|
let end = start + bin_len;
|
|
cur.set_position(end as u64);
|
|
|
|
if should_remove(field_buf.as_slice()) {
|
|
found = true;
|
|
continue;
|
|
}
|
|
|
|
keys.push(String::from_utf8(field_buf)?);
|
|
values.push(buf[start..end].to_vec());
|
|
}
|
|
|
|
if !found {
|
|
return Ok(false);
|
|
}
|
|
|
|
if keys.is_empty() {
|
|
self.0 = Vec::new();
|
|
return Ok(true);
|
|
}
|
|
|
|
self.serialize(keys, values)?;
|
|
Ok(true)
|
|
}
|
|
|
|
fn remove_two_keys_by_bytes(&mut self, first_key: &[u8], second_key: &[u8]) -> Result<bool> {
|
|
let buf = self.after_version();
|
|
if buf.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
|
|
let same = first_key == second_key;
|
|
let mut cur = Cursor::new(buf);
|
|
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
|
|
let mut keys = Vec::with_capacity(fields_len + 1);
|
|
let mut values = Vec::with_capacity(fields_len + 1);
|
|
let mut found = false;
|
|
|
|
while fields_len > 0 {
|
|
fields_len -= 1;
|
|
|
|
let str_len = rmp::decode::read_str_len(&mut cur)? as usize;
|
|
let mut field_buf = vec![0u8; str_len];
|
|
cur.read_exact(&mut field_buf)?;
|
|
|
|
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
|
let start = cur.position() as usize;
|
|
let end = start + bin_len;
|
|
cur.set_position(end as u64);
|
|
|
|
let should_remove = if same {
|
|
field_buf.as_slice() == first_key
|
|
} else {
|
|
field_buf.as_slice() == first_key || field_buf.as_slice() == second_key
|
|
};
|
|
|
|
if should_remove {
|
|
found = true;
|
|
continue;
|
|
}
|
|
|
|
keys.push(String::from_utf8(field_buf)?);
|
|
values.push(buf[start..end].to_vec());
|
|
}
|
|
|
|
if !found {
|
|
return Ok(false);
|
|
}
|
|
|
|
if keys.is_empty() {
|
|
self.0 = Vec::new();
|
|
return Ok(true);
|
|
}
|
|
|
|
self.serialize(keys, values)?;
|
|
Ok(true)
|
|
}
|
|
|
|
pub fn new() -> Self {
|
|
Self(Vec::new())
|
|
}
|
|
pub fn update(&mut self, buf: &[u8]) {
|
|
self.0 = buf.to_vec()
|
|
}
|
|
pub fn as_slice(&self) -> &[u8] {
|
|
self.0.as_slice()
|
|
}
|
|
pub fn version_ok(&self) -> bool {
|
|
if self.0.is_empty() {
|
|
return true;
|
|
}
|
|
|
|
self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
|
|
}
|
|
|
|
pub fn after_version(&self) -> &[u8] {
|
|
if self.0.is_empty() { &self.0 } else { &self.0[1..] }
|
|
}
|
|
|
|
pub fn entries(&self) -> Result<usize> {
|
|
if self.0.is_empty() || !self.version_ok() {
|
|
return Ok(0);
|
|
}
|
|
|
|
let buf = self.after_version();
|
|
|
|
let mut cur = Cursor::new(buf);
|
|
|
|
let fields_len = rmp::decode::read_map_len(&mut cur)?;
|
|
|
|
Ok(fields_len as usize)
|
|
}
|
|
|
|
pub fn find(&self, key: &str) -> Result<Option<Vec<u8>>> {
|
|
if self.0.is_empty() || !self.version_ok() {
|
|
return Ok(None);
|
|
}
|
|
|
|
let buf = self.after_version();
|
|
|
|
let mut cur = Cursor::new(buf);
|
|
|
|
let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
|
|
|
|
while fields_len > 0 {
|
|
fields_len -= 1;
|
|
|
|
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
|
|
|
let mut field_buff = vec![0u8; str_len as usize];
|
|
|
|
cur.read_exact(&mut field_buff)?;
|
|
|
|
let field = String::from_utf8(field_buff)?;
|
|
|
|
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
|
let start = cur.position() as usize;
|
|
let end = start + bin_len;
|
|
cur.set_position(end as u64);
|
|
|
|
if field.as_str() == key {
|
|
let buf = &buf[start..end];
|
|
return Ok(Some(buf.to_vec()));
|
|
}
|
|
}
|
|
|
|
Ok(None)
|
|
}
|
|
|
|
pub fn validate(&self) -> Result<()> {
|
|
if self.0.is_empty() {
|
|
return Ok(());
|
|
}
|
|
|
|
let mut cur = Cursor::new(self.after_version());
|
|
|
|
let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
|
|
|
|
while fields_len > 0 {
|
|
fields_len -= 1;
|
|
|
|
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
|
|
|
let mut field_buff = vec![0u8; str_len as usize];
|
|
|
|
cur.read_exact(&mut field_buff)?;
|
|
|
|
let field = String::from_utf8(field_buff)?;
|
|
if field.is_empty() {
|
|
return Err(Error::other("InlineData key empty"));
|
|
}
|
|
|
|
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
|
let start = cur.position() as usize;
|
|
let end = start + bin_len;
|
|
cur.set_position(end as u64);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub fn replace(&mut self, key: &str, value: Vec<u8>) -> Result<()> {
|
|
if self.after_version().is_empty() {
|
|
let mut keys = Vec::with_capacity(1);
|
|
let mut values = Vec::with_capacity(1);
|
|
|
|
keys.push(key.to_owned());
|
|
values.push(value);
|
|
|
|
return self.serialize(keys, values);
|
|
}
|
|
|
|
let buf = self.after_version();
|
|
let mut cur = Cursor::new(buf);
|
|
|
|
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
|
|
let mut keys = Vec::with_capacity(fields_len + 1);
|
|
let mut values = Vec::with_capacity(fields_len + 1);
|
|
|
|
let mut replaced = false;
|
|
|
|
while fields_len > 0 {
|
|
fields_len -= 1;
|
|
|
|
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
|
|
|
let mut field_buff = vec![0u8; str_len as usize];
|
|
|
|
cur.read_exact(&mut field_buff)?;
|
|
|
|
let find_key = String::from_utf8(field_buff)?;
|
|
|
|
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
|
let start = cur.position() as usize;
|
|
let end = start + bin_len;
|
|
cur.set_position(end as u64);
|
|
|
|
let find_value = &buf[start..end];
|
|
|
|
if find_key.as_str() == key {
|
|
values.push(value.clone());
|
|
replaced = true
|
|
} else {
|
|
values.push(find_value.to_vec());
|
|
}
|
|
|
|
keys.push(find_key);
|
|
}
|
|
|
|
if !replaced {
|
|
keys.push(key.to_owned());
|
|
values.push(value);
|
|
}
|
|
|
|
self.serialize(keys, values)
|
|
}
|
|
|
|
pub fn remove_key(&mut self, key: &str) -> Result<bool> {
|
|
let key_bytes = key.as_bytes();
|
|
if !self.contains_key_by(|candidate| candidate == key_bytes)? {
|
|
return Ok(false);
|
|
}
|
|
self.remove_keys_by(|candidate| candidate == key_bytes)
|
|
}
|
|
|
|
pub fn remove(&mut self, remove_keys: Vec<Uuid>) -> Result<bool> {
|
|
let mut encoded_keys = Vec::with_capacity(remove_keys.len());
|
|
for key in remove_keys {
|
|
let mut buf = Uuid::encode_buffer();
|
|
encoded_keys.push(key.hyphenated().encode_lower(&mut buf).to_string().into_bytes());
|
|
}
|
|
|
|
self.remove_keys_by(|candidate| encoded_keys.iter().any(|key| candidate == key.as_slice()))
|
|
}
|
|
|
|
pub fn remove_two(&mut self, first: Uuid, second: Uuid) -> Result<bool> {
|
|
let mut first_buf = Uuid::encode_buffer();
|
|
let mut second_buf = Uuid::encode_buffer();
|
|
let first_key = first.hyphenated().encode_lower(&mut first_buf).as_bytes();
|
|
let second_key = second.hyphenated().encode_lower(&mut second_buf).as_bytes();
|
|
self.remove_two_keys_by_bytes(first_key, second_key)
|
|
}
|
|
fn serialize(&mut self, keys: Vec<String>, values: Vec<Vec<u8>>) -> Result<()> {
|
|
assert_eq!(keys.len(), values.len(), "InlineData serialize: keys/values not match");
|
|
|
|
if keys.is_empty() {
|
|
self.0 = Vec::new();
|
|
return Ok(());
|
|
}
|
|
|
|
let mut wr = Vec::new();
|
|
|
|
wr.push(INLINE_DATA_VER);
|
|
|
|
let map_len = keys.len();
|
|
|
|
rmp::encode::write_map_len(&mut wr, map_len as u32)?;
|
|
|
|
for i in 0..map_len {
|
|
rmp::encode::write_str(&mut wr, keys[i].as_str())?;
|
|
rmp::encode::write_bin(&mut wr, values[i].as_slice())?;
|
|
}
|
|
|
|
self.0 = wr;
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn remove_key_miss_keeps_inline_data_unchanged() {
|
|
let mut data = InlineData::new();
|
|
data.replace("keep", b"value".to_vec()).expect("seed inline data");
|
|
let before = data.as_slice().to_vec();
|
|
|
|
let removed = data.remove_key("missing").expect("remove_key should succeed");
|
|
|
|
assert!(!removed);
|
|
assert_eq!(data.as_slice(), before.as_slice());
|
|
}
|
|
|
|
#[test]
|
|
fn remove_two_removes_only_matching_keys() {
|
|
let first = Uuid::new_v4();
|
|
let second = Uuid::new_v4();
|
|
let keep = Uuid::new_v4();
|
|
let mut data = InlineData::new();
|
|
data.replace(first.hyphenated().to_string().as_str(), b"first".to_vec())
|
|
.expect("seed first key");
|
|
data.replace(second.hyphenated().to_string().as_str(), b"second".to_vec())
|
|
.expect("seed second key");
|
|
data.replace(keep.hyphenated().to_string().as_str(), b"keep".to_vec())
|
|
.expect("seed keep key");
|
|
|
|
let removed = data.remove_two(first, second).expect("remove_two should succeed");
|
|
|
|
assert!(removed);
|
|
assert_eq!(data.find(first.hyphenated().to_string().as_str()).expect("find first"), None);
|
|
assert_eq!(data.find(second.hyphenated().to_string().as_str()).expect("find second"), None);
|
|
assert_eq!(
|
|
data.find(keep.hyphenated().to_string().as_str()).expect("find keep"),
|
|
Some(b"keep".to_vec())
|
|
);
|
|
}
|
|
}
|