gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

heal admin api(1)

Browse Source 
Signed-off-by: mujunxiang <1948535941@qq.com>
This commit is contained in:
mujunxiang
2024-11-21 20:59:54 +08:00
parent 98ed93133b
commit a421b5be2b
9 changed files with 1709 additions and 530 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/protos/src/generated/flatbuffers_generated/models.rs
+103 -104
View File
@@ -1,9 +1,10 @@
// automatically generated by the FlatBuffers compiler, do not modify
// @generated
use core::cmp::Ordering;
use core::mem;
use core::cmp::Ordering;
extern crate flatbuffers;
use self::flatbuffers::{EndianScalar, Follow};
@@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow};
#[allow(unused_imports, dead_code)]
pub mod models {
use core::cmp::Ordering;
use core::mem;
use core::mem;
use core::cmp::Ordering;
extern crate flatbuffers;
use self::flatbuffers::{EndianScalar, Follow};
extern crate flatbuffers;
use self::flatbuffers::{EndianScalar, Follow};
pub enum PingBodyOffset {}
#[derive(Copy, Clone, PartialEq)]
pub enum PingBodyOffset {}
#[derive(Copy, Clone, PartialEq)]
pub struct PingBody<'a> {
pub _tab: flatbuffers::Table<'a>,
pub struct PingBody<'a> {
pub _tab: flatbuffers::Table<'a>,
}
impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
type Inner = PingBody<'a>;
#[inline]
unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
Self { _tab: flatbuffers::Table::new(buf, loc) }
}
}
impl<'a> PingBody<'a> {
pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
pub const fn get_fully_qualified_name() -> &'static str {
"models.PingBody"
}
#[inline]
pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
PingBody { _tab: table }
}
#[allow(unused_mut)]
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
args: &'args PingBodyArgs<'args>
) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
let mut builder = PingBodyBuilder::new(_fbb);
if let Some(x) = args.payload { builder.add_payload(x); }
builder.finish()
}
#[inline]
pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
// Safety:
// Created from valid Table for this object
// which contains a valid value in this slot
unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)}
}
}
impl flatbuffers::Verifiable for PingBody<'_> {
#[inline]
fn run_verifier(
v: &mut flatbuffers::Verifier, pos: usize
) -> Result<(), flatbuffers::InvalidFlatbuffer> {
use self::flatbuffers::Verifiable;
v.visit_table(pos)?
.visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
.finish();
Ok(())
}
}
pub struct PingBodyArgs<'a> {
pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
}
impl<'a> Default for PingBodyArgs<'a> {
#[inline]
fn default() -> Self {
PingBodyArgs {
payload: None,
}
}
}
impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
type Inner = PingBody<'a>;
#[inline]
unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
Self {
_tab: flatbuffers::Table::new(buf, loc),
}
}
pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
}
impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
#[inline]
pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
}
#[inline]
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
let start = _fbb.start_table();
PingBodyBuilder {
fbb_: _fbb,
start_: start,
}
}
#[inline]
pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
let o = self.fbb_.end_table(self.start_);
flatbuffers::WIPOffset::new(o.value())
}
}
impl<'a> PingBody<'a> {
pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
impl core::fmt::Debug for PingBody<'_> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
let mut ds = f.debug_struct("PingBody");
ds.field("payload", &self.payload());
ds.finish()
}
}
} // pub mod models
pub const fn get_fully_qualified_name() -> &'static str {
"models.PingBody"
}
#[inline]
pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
PingBody { _tab: table }
}
#[allow(unused_mut)]
pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
_fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
args: &'args PingBodyArgs<'args>,
) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
let mut builder = PingBodyBuilder::new(_fbb);
if let Some(x) = args.payload {
builder.add_payload(x);
}
builder.finish()
}
#[inline]
pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
// Safety:
// Created from valid Table for this object
// which contains a valid value in this slot
unsafe {
self._tab
.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)
}
}
}
impl flatbuffers::Verifiable for PingBody<'_> {
#[inline]
fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> {
use self::flatbuffers::Verifiable;
v.visit_table(pos)?
.visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
.finish();
Ok(())
}
}
pub struct PingBodyArgs<'a> {
pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
}
impl<'a> Default for PingBodyArgs<'a> {
#[inline]
fn default() -> Self {
PingBodyArgs { payload: None }
}
}
pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
}
impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
#[inline]
pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b, u8>>) {
self.fbb_
.push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
}
#[inline]
pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
let start = _fbb.start_table();
PingBodyBuilder {
fbb_: _fbb,
start_: start,
}
}
#[inline]
pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
let o = self.fbb_.end_table(self.start_);
flatbuffers::WIPOffset::new(o.value())
}
}
impl core::fmt::Debug for PingBody<'_> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
let mut ds = f.debug_struct("PingBody");
ds.field("payload", &self.payload());
ds.finish()
}
}
} // pub mod models
common/protos/src/generated/proto_gen/node_service.rs
+1548 -398
View File
File diff suppressed because it is too large Load Diff
ecstore/src/cache_value/metacache_set.rs
+1 -1
View File
@@ -139,7 +139,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
});
}
let errs: Vec<Option<Error>> = Vec::with_capacity(readers.len());
let errs: Vec<Option<Error>> = vec![None; readers.len()];
loop {
let mut current = MetaCacheEntry::default();
let (mut at_eof, mut has_err, mut agree) = (0, 0, 0);
ecstore/src/heal/background_heal_ops.rs
+1
View File
@@ -37,6 +37,7 @@ pub async fn init_auto_heal() {
init_background_healing().await;
if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
if v == "on" {
info!("start monitor local disks and heal");
GLOBAL_BackgroundHealState
.write()
.await
ecstore/src/heal/data_scanner.rs
+5 -8
View File
@@ -511,9 +511,7 @@ impl FolderScanner {
let this_hash = hash_path(&folder.name);
let was_compacted = into.compacted;
// do not really loop
let mut times = 1;
while times > 0 {
'outer: {
let mut abandoned_children: DataUsageHashMap = if !into.compacted {
self.old_cache.find_children_copy(this_hash.clone())
} else {
@@ -622,7 +620,7 @@ impl FolderScanner {
}
if found_objects && *GLOBAL_IsErasure.read().await {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
break;
break 'outer;
}
let should_compact = self.new_cache.info.name != folder.name
@@ -714,11 +712,11 @@ impl FolderScanner {
// Scan for healing
if abandoned_children.is_empty() || !self.should_heal().await {
break;
break 'outer;
}
if self.disks.is_empty() || self.disks_quorum == 0 {
break;
break 'outer;
}
let (bg_seq, found) = GLOBAL_BackgroundHealState
@@ -727,7 +725,7 @@ impl FolderScanner {
.get_heal_sequence_by_token(BG_HEALING_UUID)
.await;
if !found {
break;
break 'outer;
}
let bg_seq = bg_seq.unwrap();
@@ -901,7 +899,6 @@ impl FolderScanner {
scan(&this, into, self).await;
}
}
times -= 1;
}
if !was_compacted {
self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), into);
ecstore/src/heal/heal_ops.rs
+28 -4
View File
@@ -7,6 +7,8 @@ use super::{
HEAL_ITEM_BUCKET_METADATA,
},
};
use crate::heal::heal_commands::{HEAL_DEEP_SCAN, HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT};
use crate::store_api::StorageAPI;
use crate::{
config::common::CONFIG_PREFIX,
disk::RUSTFS_META_BUCKET,
@@ -25,10 +27,6 @@ use crate::{
new_object_layer_fn,
utils::path::has_profix,
};
use crate::{
heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT},
store_api::StorageAPI,
};
use lazy_static::lazy_static;
use std::{
collections::HashMap,
@@ -140,6 +138,32 @@ pub fn new_bg_heal_sequence() -> HealSequence {
}
}
pub fn new_heal_sequence(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> HealSequence {
let client_token = Uuid::new_v4().to_string();
let (tx, rx) = mpsc::channel(10);
HealSequence {
bucket: bucket.to_string(),
object: obj_prefix.to_string(),
report_progress: true,
start_time: SystemTime::now(),
client_token,
client_address: client_addr.to_string(),
force_started: force_start,
setting: hs.clone(),
current_status: Arc::new(RwLock::new(HealSequenceStatus {
summary: HEAL_NOT_STARTED_STATUS.to_string(),
heal_setting: hs,
..Default::default()
})),
traverse_and_heal_done_tx: Arc::new(RwLock::new(tx)),
traverse_and_heal_done_rx: Arc::new(RwLock::new(rx)),
scanned_items_map: HashMap::new(),
healed_items_map: HashMap::new(),
heal_failed_items_map: HashMap::new(),
..Default::default()
}
}
impl Default for HealSequence {
fn default() -> Self {
let (h_tx, h_rx) = mpsc::channel(1);
ecstore/src/store.rs
+1 -1
View File
@@ -2173,7 +2173,7 @@ async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, po
*GLOBAL_Local_Node_Name.write().await = peer_set[0].clone();
}
fn is_valid_object_prefix(object: &str) -> bool {
pub fn is_valid_object_prefix(object: &str) -> bool {
// Implement object prefix validation
!object.is_empty() // Placeholder
}
rustfs/src/admin/mod.rs
+16 -12
View File
@@ -2,13 +2,14 @@ pub mod handlers;
pub mod router;
use common::error::Result;
use ecstore::global::{is_dist_erasure, is_erasure};
use hyper::Method;
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
const ADMIN_PREFIX: &str = "/rustfs/admin";
pub fn make_admin_route() -> Result<impl S3Route> {
pub async fn make_admin_route() -> Result<impl S3Route> {
let mut r = S3Router::new();
r.insert(Method::POST, "/", AdminOperation(&handlers::AssumeRoleHandle {}))?;
@@ -53,17 +54,6 @@ pub fn make_admin_route() -> Result<impl S3Route> {
AdminOperation(&handlers::MetricsHandler {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{prefix}").as_str(),
AdminOperation(&handlers::HealHandler {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/background-heal/status").as_str(),
AdminOperation(&handlers::BackgroundHealStatusHandler {}),
)?;
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/pools/list").as_str(),
@@ -101,5 +91,19 @@ pub fn make_admin_route() -> Result<impl S3Route> {
AdminOperation(&handlers::RebalanceStop {}),
)?;
// Some APIs are only available in EC mode
// if is_dist_erasure().await || is_erasure().await {
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{prefix}").as_str(),
AdminOperation(&handlers::HealHandler {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/background-heal/status").as_str(),
AdminOperation(&handlers::BackgroundHealStatusHandler {}),
)?;
// }
Ok(r)
}
rustfs/src/main.rs
+6 -2
View File
@@ -10,7 +10,7 @@ use clap::Parser;
use common::error::{Error, Result};
use ecstore::{
endpoints::EndpointServerPools,
heal::data_scanner::init_data_scanner,
heal::{background_heal_ops::init_auto_heal, data_scanner::init_data_scanner},
set_global_endpoints,
store::{init_local_disks, ECStore},
update_erasure_type,
@@ -192,7 +192,11 @@ async fn run(opt: config::Opt) -> Result<()> {
})?;
warn!(" init store success!");
// init scanner
init_data_scanner().await;
// init_data_scanner().await;
// // init auto heal
// init_auto_heal().await;
info!("server was started");
tokio::select! {
_ = tokio::signal::ctrl_c() => {