mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
refactor: Restructure project layout and clean up dependencies (#30)
This commit introduces a significant reorganization of the project structure to improve maintainability and clarity. Key changes include: - Adjusted the directory layout for a more logical module organization. - Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times. - Updated import paths and configuration files to reflect the structural changes.
This commit is contained in:
houseme
@@ -0,0 +1,43 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::lifecycle;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub enum LcEventSrc {
|
||||
#[default]
|
||||
None,
|
||||
Heal,
|
||||
Scanner,
|
||||
Decom,
|
||||
Rebal,
|
||||
S3HeadObject,
|
||||
S3GetObject,
|
||||
S3ListObjects,
|
||||
S3PutObject,
|
||||
S3CopyObject,
|
||||
S3CompleteMultipartUpload,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct LcAuditEvent {
|
||||
pub event: lifecycle::Event,
|
||||
pub source: LcEventSrc,
|
||||
}
|
||||
|
||||
impl LcAuditEvent {
|
||||
pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
|
||||
Self { event, source }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,844 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
|
||||
use futures::Future;
|
||||
use http::HeaderMap;
|
||||
use lazy_static::lazy_static;
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::io::Write;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::select;
|
||||
use tokio::sync::mpsc::{Receiver, Sender};
|
||||
use tokio::sync::{RwLock, mpsc};
|
||||
use tracing::{error, info};
|
||||
use uuid::Uuid;
|
||||
use xxhash_rust::xxh64;
|
||||
|
||||
//use rustfs_notify::{BucketNotificationConfig, Event, EventName, LogLevel, NotificationError, init_logger};
|
||||
//use rustfs_notify::{initialize, notification_system};
|
||||
use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
|
||||
use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
|
||||
use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
|
||||
use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
|
||||
use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
|
||||
use crate::client::object_api_utils::new_getobjectreader;
|
||||
use crate::error::Error;
|
||||
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
|
||||
use crate::event::name::EventName;
|
||||
use crate::event_notification::{EventArgs, send_event};
|
||||
use crate::global::GLOBAL_LocalNodeName;
|
||||
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
|
||||
use crate::heal::{
|
||||
data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
|
||||
data_scanner_metric::ScannerMetrics,
|
||||
data_usage_cache::TierStats,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::StorageAPI;
|
||||
use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
|
||||
use crate::tier::warm_backend::WarmBackendGetOpts;
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
|
||||
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||
pub type TraceFn =
|
||||
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
|
||||
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||
|
||||
static XXHASH_SEED: u64 = 0;
|
||||
|
||||
const _DISABLED: &str = "Disabled";
|
||||
|
||||
//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class.";
|
||||
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
|
||||
|
||||
lazy_static! {
|
||||
pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
|
||||
pub static ref GLOBAL_TransitionState: Arc<TransitionState> = TransitionState::new();
|
||||
}
|
||||
|
||||
pub struct LifecycleSys;
|
||||
|
||||
impl LifecycleSys {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self)
|
||||
}
|
||||
|
||||
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0;
|
||||
Some(lc)
|
||||
}
|
||||
|
||||
pub fn trace(_oi: &ObjectInfo) -> TraceFn {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
struct ExpiryTask {
|
||||
obj_info: ObjectInfo,
|
||||
event: lifecycle::Event,
|
||||
src: LcEventSrc,
|
||||
}
|
||||
|
||||
impl ExpiryOp for ExpiryTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
|
||||
let _ = hasher.write(format!("{}", self.obj_info.name).as_bytes());
|
||||
hasher.flush();
|
||||
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
struct ExpiryStats {
|
||||
missed_expiry_tasks: AtomicI64,
|
||||
missed_freevers_tasks: AtomicI64,
|
||||
missed_tier_journal_tasks: AtomicI64,
|
||||
workers: AtomicI64,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ExpiryStats {
|
||||
pub fn missed_tasks(&self) -> i64 {
|
||||
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn missed_free_vers_tasks(&self) -> i64 {
|
||||
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn missed_tier_journal_tasks(&self) -> i64 {
|
||||
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn num_workers(&self) -> i64 {
|
||||
self.workers.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ExpiryOp: 'static {
|
||||
fn op_hash(&self) -> u64;
|
||||
fn as_any(&self) -> &dyn Any;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct TransitionedObject {
|
||||
pub name: String,
|
||||
pub version_id: String,
|
||||
pub tier: String,
|
||||
pub free_version: bool,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
struct FreeVersionTask(ObjectInfo);
|
||||
|
||||
impl ExpiryOp for FreeVersionTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
let _ = hasher.write(format!("{}", self.0.transitioned_object.tier).as_bytes());
|
||||
let _ = hasher.write(format!("{}", self.0.transitioned_object.name).as_bytes());
|
||||
hasher.flush();
|
||||
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
struct NewerNoncurrentTask {
|
||||
bucket: String,
|
||||
versions: Vec<ObjectToDelete>,
|
||||
event: lifecycle::Event,
|
||||
}
|
||||
|
||||
impl ExpiryOp for NewerNoncurrentTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
let _ = hasher.write(format!("{}", self.bucket).as_bytes());
|
||||
let _ = hasher.write(format!("{}", self.versions[0].object_name).as_bytes());
|
||||
hasher.flush();
|
||||
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ExpiryState {
|
||||
tasks_tx: Vec<Sender<Option<ExpiryOpType>>>,
|
||||
tasks_rx: Vec<Arc<tokio::sync::Mutex<Receiver<Option<ExpiryOpType>>>>>,
|
||||
stats: Option<ExpiryStats>,
|
||||
}
|
||||
|
||||
impl ExpiryState {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
pub fn new() -> Arc<RwLock<Self>> {
|
||||
Arc::new(RwLock::new(Self {
|
||||
tasks_tx: vec![],
|
||||
tasks_rx: vec![],
|
||||
stats: Some(ExpiryStats {
|
||||
missed_expiry_tasks: AtomicI64::new(0),
|
||||
missed_freevers_tasks: AtomicI64::new(0),
|
||||
missed_tier_journal_tasks: AtomicI64::new(0),
|
||||
workers: AtomicI64::new(0),
|
||||
}),
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn pending_tasks(&self) -> usize {
|
||||
let rxs = &self.tasks_rx;
|
||||
if rxs.len() == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut tasks = 0;
|
||||
for rx in rxs.iter() {
|
||||
tasks += rx.lock().await.len();
|
||||
}
|
||||
tasks
|
||||
}
|
||||
|
||||
pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
|
||||
let wrkr = self.get_worker_ch(je.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
|
||||
}
|
||||
let wrkr = wrkr.expect("err");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => ()
|
||||
_ = wrkr.send(Some(Box::new(je.clone()))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
|
||||
let task = FreeVersionTask(oi);
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
|
||||
return;
|
||||
}
|
||||
let wrkr = wrkr.expect("err!");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => {}
|
||||
_ = wrkr.send(Some(Box::new(task))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
|
||||
let task = ExpiryTask {
|
||||
obj_info: oi.clone(),
|
||||
event: event.clone(),
|
||||
src: src.clone(),
|
||||
};
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
return;
|
||||
}
|
||||
let wrkr = wrkr.expect("err!");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => {}
|
||||
_ = wrkr.send(Some(Box::new(task))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
|
||||
if versions.len() == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let task = NewerNoncurrentTask {
|
||||
bucket: String::from(bucket),
|
||||
versions,
|
||||
event: lc_event,
|
||||
};
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
return;
|
||||
}
|
||||
let wrkr = wrkr.expect("err!");
|
||||
select! {
|
||||
//_ -> GlobalContext.Done() => {}
|
||||
_ = wrkr.send(Some(Box::new(task))) => (),
|
||||
else => {
|
||||
*self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
|
||||
if self.tasks_tx.len() == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
|
||||
}
|
||||
|
||||
pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
|
||||
if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut state = GLOBAL_ExpiryState.write().await;
|
||||
|
||||
while state.tasks_tx.len() < n {
|
||||
let (tx, rx) = mpsc::channel(10000);
|
||||
let api = api.clone();
|
||||
let rx = Arc::new(tokio::sync::Mutex::new(rx));
|
||||
state.tasks_tx.push(tx);
|
||||
state.tasks_rx.push(rx.clone());
|
||||
*state.stats.as_mut().expect("err").workers.get_mut() += 1;
|
||||
tokio::spawn(async move {
|
||||
let mut rx = rx.lock().await;
|
||||
//let mut expiry_state = GLOBAL_ExpiryState.read().await;
|
||||
ExpiryState::worker(&mut *rx, api).await;
|
||||
});
|
||||
}
|
||||
|
||||
let mut l = state.tasks_tx.len();
|
||||
while l > n {
|
||||
let worker = state.tasks_tx[l - 1].clone();
|
||||
worker.send(None).await.unwrap_or(());
|
||||
state.tasks_tx.remove(l - 1);
|
||||
state.tasks_rx.remove(l - 1);
|
||||
*state.stats.as_mut().expect("err").workers.get_mut() -= 1;
|
||||
l -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
|
||||
loop {
|
||||
select! {
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!("got ctrl+c, exits");
|
||||
break;
|
||||
}
|
||||
v = rx.recv() => {
|
||||
if v.is_none() {
|
||||
break;
|
||||
}
|
||||
let v = v.expect("err!");
|
||||
if v.is_none() {
|
||||
//rx.close();
|
||||
//drop(rx);
|
||||
let _ = rx;
|
||||
return;
|
||||
}
|
||||
let v = v.expect("err!");
|
||||
if v.as_any().is::<ExpiryTask>() {
|
||||
let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
|
||||
if v.obj_info.transitioned_object.status != "" {
|
||||
apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
|
||||
} else {
|
||||
apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
|
||||
}
|
||||
}
|
||||
else if v.as_any().is::<NewerNoncurrentTask>() {
|
||||
let _v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
|
||||
//delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
|
||||
}
|
||||
else if v.as_any().is::<Jentry>() {
|
||||
//transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
|
||||
}
|
||||
else if v.as_any().is::<FreeVersionTask>() {
|
||||
let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
|
||||
let _oi = v.0.clone();
|
||||
|
||||
}
|
||||
else {
|
||||
//info!("Invalid work type - {:?}", v);
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TransitionTask {
|
||||
obj_info: ObjectInfo,
|
||||
src: LcEventSrc,
|
||||
event: lifecycle::Event,
|
||||
}
|
||||
|
||||
impl ExpiryOp for TransitionTask {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
|
||||
//let _ = hasher.write(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
|
||||
hasher.flush();
|
||||
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionState {
|
||||
transition_tx: A_Sender<Option<TransitionTask>>,
|
||||
transition_rx: A_Receiver<Option<TransitionTask>>,
|
||||
pub num_workers: AtomicI64,
|
||||
kill_tx: A_Sender<()>,
|
||||
kill_rx: A_Receiver<()>,
|
||||
active_tasks: AtomicI64,
|
||||
missed_immediate_tasks: AtomicI64,
|
||||
last_day_stats: Arc<Mutex<HashMap<String, LastDayTierStats>>>,
|
||||
}
|
||||
|
||||
impl TransitionState {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
pub fn new() -> Arc<Self> {
|
||||
let (tx1, rx1) = bounded(100000);
|
||||
let (tx2, rx2) = bounded(1);
|
||||
Arc::new(Self {
|
||||
transition_tx: tx1,
|
||||
transition_rx: rx1,
|
||||
num_workers: AtomicI64::new(0),
|
||||
kill_tx: tx2,
|
||||
kill_rx: rx2,
|
||||
active_tasks: AtomicI64::new(0),
|
||||
missed_immediate_tasks: AtomicI64::new(0),
|
||||
last_day_stats: Arc::new(Mutex::new(HashMap::new())),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
|
||||
let task = TransitionTask {
|
||||
obj_info: oi.clone(),
|
||||
src: src.clone(),
|
||||
event: event.clone(),
|
||||
};
|
||||
select! {
|
||||
//_ -> t.ctx.Done() => (),
|
||||
_ = self.transition_tx.send(Some(task)) => (),
|
||||
else => {
|
||||
match src {
|
||||
LcEventSrc::S3PutObject | LcEventSrc::S3CopyObject | LcEventSrc::S3CompleteMultipartUpload => {
|
||||
self.missed_immediate_tasks.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
_ => ()
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init(api: Arc<ECStore>) {
|
||||
let mut n = 10; //globalAPIConfig.getTransitionWorkers();
|
||||
let tw = 10; //globalILMConfig.getTransitionWorkers();
|
||||
if tw > 0 {
|
||||
n = tw;
|
||||
}
|
||||
|
||||
//let mut transition_state = GLOBAL_TransitionState.write().await;
|
||||
//self.objAPI = objAPI
|
||||
Self::update_workers(api, n).await;
|
||||
}
|
||||
|
||||
pub fn pending_tasks(&self) -> usize {
|
||||
//let transition_rx = GLOBAL_TransitionState.transition_rx.lock().unwrap();
|
||||
let transition_rx = &GLOBAL_TransitionState.transition_rx;
|
||||
transition_rx.len()
|
||||
}
|
||||
|
||||
pub fn active_tasks(&self) -> i64 {
|
||||
self.active_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn missed_immediate_tasks(&self) -> i64 {
|
||||
self.missed_immediate_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub async fn worker(api: Arc<ECStore>) {
|
||||
loop {
|
||||
select! {
|
||||
_ = GLOBAL_TransitionState.kill_rx.recv() => {
|
||||
return;
|
||||
}
|
||||
task = GLOBAL_TransitionState.transition_rx.recv() => {
|
||||
if task.is_err() {
|
||||
break;
|
||||
}
|
||||
let task = task.expect("err!");
|
||||
if task.is_none() {
|
||||
//self.transition_rx.close();
|
||||
//drop(self.transition_rx);
|
||||
return;
|
||||
}
|
||||
let task = task.expect("err!");
|
||||
if task.as_any().is::<TransitionTask>() {
|
||||
let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
|
||||
|
||||
GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
|
||||
if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
||||
error!("Transition to {} failed for {}/{} version:{} with {}",
|
||||
task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string());
|
||||
}
|
||||
} else {
|
||||
let mut ts = TierStats {
|
||||
total_size: task.obj_info.size as u64,
|
||||
num_versions: 1,
|
||||
..Default::default()
|
||||
};
|
||||
if task.obj_info.is_latest {
|
||||
ts.num_objects = 1;
|
||||
}
|
||||
GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
|
||||
}
|
||||
GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
else => ()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
|
||||
let mut tier_stats = self.last_day_stats.lock().unwrap();
|
||||
tier_stats
|
||||
.entry(tier.to_string())
|
||||
.and_modify(|e| e.add_stats(ts))
|
||||
.or_insert(LastDayTierStats::default());
|
||||
}
|
||||
|
||||
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
|
||||
let tier_stats = self.last_day_stats.lock().unwrap();
|
||||
let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
|
||||
for (tier, st) in tier_stats.iter() {
|
||||
res.insert(tier.clone(), st.clone());
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub async fn update_workers(api: Arc<ECStore>, n: i64) {
|
||||
Self::update_workers_inner(api, n).await;
|
||||
}
|
||||
|
||||
pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
|
||||
let mut n = n;
|
||||
if n == 0 {
|
||||
n = 100;
|
||||
}
|
||||
|
||||
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
|
||||
while num_workers < n {
|
||||
let clone_api = api.clone();
|
||||
tokio::spawn(async move {
|
||||
TransitionState::worker(clone_api).await;
|
||||
});
|
||||
num_workers = num_workers + 1;
|
||||
GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
|
||||
while num_workers > n {
|
||||
let worker = GLOBAL_TransitionState.kill_tx.clone();
|
||||
worker.send(()).await;
|
||||
num_workers = num_workers - 1;
|
||||
GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let mut workers = num_cpus::get() / 2;
|
||||
//globalILMConfig.getExpirationWorkers()
|
||||
if let Ok(env_expiration_workers) = env::var("_RUSTFS_EXPIRATION_WORKERS") {
|
||||
if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
|
||||
workers = num_expirations;
|
||||
}
|
||||
}
|
||||
|
||||
if workers == 0 {
|
||||
workers = 100;
|
||||
}
|
||||
|
||||
//let expiry_state = GLOBAL_ExpiryStSate.write().await;
|
||||
ExpiryState::resize_workers(workers, api).await;
|
||||
}
|
||||
|
||||
pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
|
||||
for rule in &lc.rules {
|
||||
if let Some(transitions) = &rule.transitions {
|
||||
for transition in transitions {
|
||||
if let Some(storage_class) = &transition.storage_class {
|
||||
if storage_class.as_str() != "" {
|
||||
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
|
||||
if !valid {
|
||||
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
|
||||
for noncurrent_version_transition in noncurrent_version_transitions {
|
||||
if let Some(storage_class) = &noncurrent_version_transition.storage_class {
|
||||
if storage_class.as_str() != "" {
|
||||
let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
|
||||
if !valid {
|
||||
return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await;
|
||||
if !lc.is_none() {
|
||||
let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
|
||||
match event.action {
|
||||
lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
|
||||
if oi.delete_marker || oi.is_dir {
|
||||
return;
|
||||
}
|
||||
GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn expire_transitioned_object(
|
||||
api: Arc<ECStore>,
|
||||
oi: &ObjectInfo,
|
||||
lc_event: &lifecycle::Event,
|
||||
_src: &LcEventSrc,
|
||||
) -> Result<ObjectInfo, std::io::Error> {
|
||||
//let traceFn = GLOBAL_LifecycleSys.trace(oi);
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
expiration: ExpirationOptions { expire: true },
|
||||
..Default::default()
|
||||
};
|
||||
if lc_event.action == IlmAction::DeleteVersionAction {
|
||||
opts.version_id = oi.version_id.map(|id| id.to_string());
|
||||
}
|
||||
//let tags = LcAuditEvent::new(src, lcEvent).Tags();
|
||||
if lc_event.action == IlmAction::DeleteRestoredAction {
|
||||
opts.transition.expire_restored = true;
|
||||
match api.delete_object(&oi.bucket, &oi.name, opts).await {
|
||||
Ok(dobj) => {
|
||||
//audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
|
||||
return Ok(dobj);
|
||||
}
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
let ret = delete_object_from_remote_tier(
|
||||
&oi.transitioned_object.name,
|
||||
&oi.transitioned_object.version_id,
|
||||
&oi.transitioned_object.tier,
|
||||
)
|
||||
.await;
|
||||
if ret.is_ok() {
|
||||
opts.skip_decommissioned = true;
|
||||
} else {
|
||||
//transitionLogIf(ctx, err);
|
||||
}
|
||||
|
||||
let dobj = api.delete_object(&oi.bucket, &oi.name, opts).await?;
|
||||
|
||||
//defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
|
||||
|
||||
let mut event_name = EventName::ObjectRemovedDelete;
|
||||
if oi.delete_marker {
|
||||
event_name = EventName::ObjectRemovedDeleteMarkerCreated;
|
||||
}
|
||||
let obj_info = ObjectInfo {
|
||||
name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
delete_marker: oi.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.as_ref().to_string(),
|
||||
bucket_name: obj_info.bucket.clone(),
|
||||
object: obj_info,
|
||||
user_agent: "Internal: [ILM-Expiry]".to_string(),
|
||||
host: GLOBAL_LocalNodeName.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
/*let system = match notification_system() {
|
||||
Some(sys) => sys,
|
||||
None => {
|
||||
let config = Config::new();
|
||||
initialize(config).await?;
|
||||
notification_system().expect("Failed to initialize notification system")
|
||||
}
|
||||
};
|
||||
let event = Arc::new(Event::new_test_event("my-bucket", "document.pdf", EventName::ObjectCreatedPut));
|
||||
system.send_event(event).await;*/
|
||||
|
||||
Ok(dobj)
|
||||
}
|
||||
|
||||
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
let us = Uuid::new_v4().to_string();
|
||||
let mut hasher = Sha256::new();
|
||||
let _ = hasher.write(format!("{}/{}", get_global_deployment_id().unwrap_or_default(), bucket).as_bytes());
|
||||
hasher.flush();
|
||||
let hash = rustfs_utils::crypto::hex(hasher.clone().finalize().as_slice());
|
||||
let obj = format!("{}/{}/{}/{}", &hash[0..16], &us[0..2], &us[2..4], &us);
|
||||
Ok(obj)
|
||||
}
|
||||
|
||||
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
|
||||
let time_ilm = ScannerMetrics::time_ilm(lae.event.action);
|
||||
|
||||
let opts = ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: lifecycle::TRANSITION_PENDING.to_string(),
|
||||
tier: lae.event.storage_class,
|
||||
etag: oi.etag.clone().expect("err").to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
//lifecycle_audit_event: lae,
|
||||
version_id: Some(oi.version_id.expect("err").to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
mod_time: oi.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
time_ilm(1);
|
||||
api.transition_object(&oi.bucket, &oi.name, &opts).await
|
||||
}
|
||||
|
||||
pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub async fn get_transitioned_object_reader(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
rs: HTTPRangeSpec,
|
||||
h: HeaderMap,
|
||||
oi: ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
|
||||
let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
|
||||
Ok(d) => d,
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
|
||||
let ret = new_getobjectreader(rs, &oi, opts, &h);
|
||||
if let Err(err) = ret {
|
||||
return Err(error_resp_to_object_err(err, vec![bucket, object]));
|
||||
}
|
||||
let (get_fn, off, length) = ret.expect("err");
|
||||
let mut gopts = WarmBackendGetOpts::default();
|
||||
|
||||
if off >= 0 && length >= 0 {
|
||||
gopts.start_offset = off;
|
||||
gopts.length = length;
|
||||
}
|
||||
|
||||
//return Ok(HttpFileReader::new(rs, &oi, opts, &h));
|
||||
//timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
|
||||
let reader = tgt_client
|
||||
.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
|
||||
.await?;
|
||||
Ok(get_fn(reader, h))
|
||||
}
|
||||
|
||||
pub fn post_restore_opts(_r: http::Request<Body>, _bucket: &str, _object: &str) -> Result<ObjectOptions, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn put_restore_opts(_bucket: &str, _object: &str, _rreq: &RestoreObjectRequest, _oi: &ObjectInfo) -> ObjectOptions {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub trait LifecycleOps {
|
||||
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts;
|
||||
}
|
||||
|
||||
impl LifecycleOps for ObjectInfo {
|
||||
fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
|
||||
lifecycle::ObjectOpts {
|
||||
name: self.name.clone(),
|
||||
user_tags: self.user_tags.clone(),
|
||||
version_id: self.version_id.expect("err").to_string(),
|
||||
mod_time: self.mod_time,
|
||||
size: self.size as usize,
|
||||
is_latest: self.is_latest,
|
||||
num_versions: self.num_versions,
|
||||
delete_marker: self.delete_marker,
|
||||
successor_mod_time: self.successor_mod_time,
|
||||
//restore_ongoing: self.restore_ongoing,
|
||||
//restore_expires: self.restore_expires,
|
||||
transition_status: self.transitioned_object.status.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct S3Location {
|
||||
pub bucketname: String,
|
||||
//pub encryption: Encryption,
|
||||
pub prefix: String,
|
||||
pub storage_class: String,
|
||||
//pub tagging: Tags,
|
||||
pub user_metadata: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct OutputLocation(pub S3Location);
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct RestoreObjectRequest {
|
||||
pub days: i64,
|
||||
pub ror_type: String,
|
||||
pub tier: String,
|
||||
pub description: String,
|
||||
//pub select_parameters: SelectParameters,
|
||||
pub output_location: OutputLocation,
|
||||
}
|
||||
|
||||
const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
|
||||
@@ -0,0 +1,729 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, Transition,
|
||||
};
|
||||
use std::cmp::Ordering;
|
||||
use std::env;
|
||||
use std::fmt::Display;
|
||||
use time::macros::{datetime, offset};
|
||||
use time::{self, Duration, OffsetDateTime};
|
||||
|
||||
use crate::bucket::lifecycle::rule::TransitionOps;
|
||||
|
||||
use super::bucket_lifecycle_ops::RestoreObjectRequest;
|
||||
|
||||
pub const TRANSITION_COMPLETE: &str = "complete";
|
||||
pub const TRANSITION_PENDING: &str = "pending";
|
||||
|
||||
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules";
|
||||
const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
|
||||
const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
|
||||
const _ERR_XML_NOT_WELL_FORMED: &str =
|
||||
"The XML you provided was not well-formed or did not validate against our published schema";
|
||||
const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
|
||||
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum IlmAction {
|
||||
NoneAction = 0,
|
||||
DeleteAction,
|
||||
DeleteVersionAction,
|
||||
TransitionAction,
|
||||
TransitionVersionAction,
|
||||
DeleteRestoredAction,
|
||||
DeleteRestoredVersionAction,
|
||||
DeleteAllVersionsAction,
|
||||
DelMarkerDeleteAllVersionsAction,
|
||||
ActionCount,
|
||||
}
|
||||
|
||||
impl IlmAction {
|
||||
pub fn delete_restored(&self) -> bool {
|
||||
*self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
|
||||
}
|
||||
|
||||
pub fn delete_versioned(&self) -> bool {
|
||||
*self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
|
||||
}
|
||||
|
||||
pub fn delete_all(&self) -> bool {
|
||||
*self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
|
||||
}
|
||||
|
||||
pub fn delete(&self) -> bool {
|
||||
if self.delete_restored() {
|
||||
return true;
|
||||
}
|
||||
*self == Self::DeleteVersionAction
|
||||
|| *self == Self::DeleteAction
|
||||
|| *self == Self::DeleteAllVersionsAction
|
||||
|| *self == Self::DelMarkerDeleteAllVersionsAction
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for IlmAction {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait RuleValidate {
|
||||
fn validate(&self) -> Result<(), std::io::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl RuleValidate for LifecycleRule {
|
||||
/*fn validate_id(&self) -> Result<()> {
|
||||
if self.id.len() > 255 {
|
||||
return errInvalidRuleID;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_status(&self) -> Result<()> {
|
||||
if self.Status.len() == 0 {
|
||||
return errEmptyRuleStatus;
|
||||
}
|
||||
|
||||
if self.Status != Enabled && self.Status != Disabled {
|
||||
return errInvalidRuleStatus;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_expiration(&self) -> Result<()> {
|
||||
self.Expiration.Validate();
|
||||
}
|
||||
|
||||
fn validate_noncurrent_expiration(&self) -> Result<()> {
|
||||
self.NoncurrentVersionExpiration.Validate()
|
||||
}
|
||||
|
||||
fn validate_prefix_and_filter(&self) -> Result<()> {
|
||||
if !self.Prefix.set && self.Filter.IsEmpty() || self.Prefix.set && !self.Filter.IsEmpty() {
|
||||
return errXMLNotWellFormed;
|
||||
}
|
||||
if !self.Prefix.set {
|
||||
return self.Filter.Validate();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_transition(&self) -> Result<()> {
|
||||
self.Transition.Validate()
|
||||
}
|
||||
|
||||
fn validate_noncurrent_transition(&self) -> Result<()> {
|
||||
self.NoncurrentVersionTransition.Validate()
|
||||
}
|
||||
|
||||
fn get_prefix(&self) -> String {
|
||||
if p := self.Prefix.String(); p != "" {
|
||||
return p
|
||||
}
|
||||
if p := self.Filter.Prefix.String(); p != "" {
|
||||
return p
|
||||
}
|
||||
if p := self.Filter.And.Prefix.String(); p != "" {
|
||||
return p
|
||||
}
|
||||
"".to_string()
|
||||
}*/
|
||||
|
||||
fn validate(&self) -> Result<(), std::io::Error> {
|
||||
/*self.validate_id()?;
|
||||
self.validate_status()?;
|
||||
self.validate_expiration()?;
|
||||
self.validate_noncurrent_expiration()?;
|
||||
self.validate_prefix_and_filter()?;
|
||||
self.validate_transition()?;
|
||||
self.validate_noncurrent_transition()?;
|
||||
if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() {
|
||||
return errInvalidRuleDelMarkerExpiration
|
||||
}
|
||||
if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() {
|
||||
return errXMLNotWellFormed
|
||||
}*/
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Lifecycle {
|
||||
async fn has_transition(&self) -> bool;
|
||||
fn has_expiry(&self) -> bool;
|
||||
async fn has_active_rules(&self, prefix: &str) -> bool;
|
||||
async fn validate(&self, lr_retention: bool) -> Result<(), std::io::Error>;
|
||||
async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
|
||||
async fn eval(&self, obj: &ObjectOpts) -> Event;
|
||||
async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event;
|
||||
//fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
|
||||
async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Lifecycle for BucketLifecycleConfiguration {
|
||||
async fn has_transition(&self) -> bool {
|
||||
for rule in self.rules.iter() {
|
||||
if !rule.transitions.is_none() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn has_expiry(&self) -> bool {
|
||||
for rule in self.rules.iter() {
|
||||
if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn has_active_rules(&self, prefix: &str) -> bool {
|
||||
if self.rules.len() == 0 {
|
||||
return false;
|
||||
}
|
||||
for rule in self.rules.iter() {
|
||||
if rule.status.as_str() == ExpirationStatus::DISABLED {
|
||||
continue;
|
||||
}
|
||||
|
||||
let rule_prefix = rule.prefix.as_ref().expect("err!");
|
||||
if prefix.len() > 0 && rule_prefix.len() > 0 && !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!");
|
||||
if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 {
|
||||
return true;
|
||||
}
|
||||
if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 {
|
||||
return true;
|
||||
}
|
||||
if !rule.noncurrent_version_transitions.is_none() {
|
||||
return true;
|
||||
}
|
||||
let rule_expiration = rule.expiration.as_ref().expect("err!");
|
||||
if !rule_expiration.date.is_none()
|
||||
&& OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
|
||||
< OffsetDateTime::now_utc().unix_timestamp()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if !rule_expiration.date.is_none() {
|
||||
return true;
|
||||
}
|
||||
if rule_expiration.expired_object_delete_marker.expect("err!") {
|
||||
return true;
|
||||
}
|
||||
let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
|
||||
let rule_transitions_0 = rule_transitions[0].clone();
|
||||
if !rule_transitions_0.date.is_none()
|
||||
&& OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
|
||||
< OffsetDateTime::now_utc().unix_timestamp()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if !rule.transitions.is_none() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn validate(&self, lr_retention: bool) -> Result<(), std::io::Error> {
|
||||
if self.rules.len() > 1000 {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES));
|
||||
}
|
||||
if self.rules.len() == 0 {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE));
|
||||
}
|
||||
|
||||
for r in &self.rules {
|
||||
r.validate()?;
|
||||
if let Some(expiration) = r.expiration.as_ref() {
|
||||
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
|
||||
if lr_retention && (!expired_object_delete_marker) {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (i, _) in self.rules.iter().enumerate() {
|
||||
if i == self.rules.len() - 1 {
|
||||
break;
|
||||
}
|
||||
let other_rules = &self.rules[i + 1..];
|
||||
for other_rule in other_rules {
|
||||
if self.rules[i].id == other_rule.id {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>> {
|
||||
if obj.name == "" {
|
||||
return None;
|
||||
}
|
||||
let mut rules = Vec::<LifecycleRule>::new();
|
||||
for rule in self.rules.iter() {
|
||||
if rule.status.as_str() == ExpirationStatus::DISABLED {
|
||||
continue;
|
||||
}
|
||||
if let Some(prefix) = rule.prefix.clone() {
|
||||
if !obj.name.starts_with(prefix.as_str()) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
/*if !rule.filter.test_tags(obj.user_tags) {
|
||||
continue;
|
||||
}*/
|
||||
//if !obj.delete_marker && !rule.filter.BySize(obj.size) {
|
||||
if !obj.delete_marker && false {
|
||||
continue;
|
||||
}
|
||||
rules.push(rule.clone());
|
||||
}
|
||||
Some(rules)
|
||||
}
|
||||
|
||||
async fn eval(&self, obj: &ObjectOpts) -> Event {
|
||||
self.eval_inner(obj, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event {
|
||||
let mut events = Vec::<Event>::new();
|
||||
if obj.mod_time.expect("err").unix_timestamp() == 0 {
|
||||
return Event::default();
|
||||
}
|
||||
|
||||
if let Some(restore_expires) = obj.restore_expires {
|
||||
if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() {
|
||||
let mut action = IlmAction::DeleteRestoredAction;
|
||||
if !obj.is_latest {
|
||||
action = IlmAction::DeleteRestoredVersionAction;
|
||||
}
|
||||
|
||||
events.push(Event {
|
||||
action,
|
||||
due: Some(now),
|
||||
rule_id: "".into(),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref lc_rules) = self.filter_rules(obj).await {
|
||||
for rule in lc_rules.iter() {
|
||||
if obj.expired_object_deletemarker() {
|
||||
if let Some(expiration) = rule.expiration.as_ref() {
|
||||
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
|
||||
events.push(Event {
|
||||
action: IlmAction::DeleteVersionAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(now),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(expiration) = rule.expiration.as_ref() {
|
||||
if let Some(days) = expiration.days {
|
||||
let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/);
|
||||
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
|
||||
events.push(Event {
|
||||
action: IlmAction::DeleteVersionAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(expected_expiry),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if obj.is_latest {
|
||||
if let Some(ref expiration) = rule.expiration {
|
||||
if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
|
||||
if obj.delete_marker && expired_object_delete_marker {
|
||||
let due = expiration.next_due(obj);
|
||||
if let Some(due) = due {
|
||||
if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() {
|
||||
events.push(Event {
|
||||
action: IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(due),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !obj.is_latest {
|
||||
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
|
||||
if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
|
||||
if newer_noncurrent_versions > 0 {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !obj.is_latest {
|
||||
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
|
||||
if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days {
|
||||
if noncurrent_days != 0 {
|
||||
if let Some(successor_mod_time) = obj.successor_mod_time {
|
||||
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
|
||||
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
|
||||
events.push(Event {
|
||||
action: IlmAction::DeleteVersionAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(expected_expiry),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !obj.is_latest {
|
||||
if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions {
|
||||
if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class {
|
||||
if storage_class.as_str() != "" && !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE
|
||||
{
|
||||
let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
|
||||
if due.is_some()
|
||||
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp())
|
||||
{
|
||||
events.push(Event {
|
||||
action: IlmAction::TransitionVersionAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due,
|
||||
storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
|
||||
.storage_class
|
||||
.clone()
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if obj.is_latest && !obj.delete_marker {
|
||||
if let Some(ref expiration) = rule.expiration {
|
||||
if let Some(ref date) = expiration.date {
|
||||
let date0 = OffsetDateTime::from(date.clone());
|
||||
if date0.unix_timestamp() != 0
|
||||
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp())
|
||||
{
|
||||
events.push(Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(date0),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
}
|
||||
} else if let Some(days) = expiration.days {
|
||||
if days != 0 {
|
||||
let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
|
||||
if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
|
||||
let mut event = Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(expected_expiry),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
};
|
||||
/*if rule.expiration.expect("err!").delete_all.val {
|
||||
event.action = IlmAction::DeleteAllVersionsAction
|
||||
}*/
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if obj.transition_status != TRANSITION_COMPLETE {
|
||||
if let Some(ref transitions) = rule.transitions {
|
||||
let due = transitions[0].next_due(obj);
|
||||
if let Some(due) = due {
|
||||
if due.unix_timestamp() > 0
|
||||
&& (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp())
|
||||
{
|
||||
events.push(Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
rule_id: rule.id.clone().expect("err!"),
|
||||
due: Some(due),
|
||||
storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if events.len() > 0 {
|
||||
events.sort_by(|a, b| {
|
||||
if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
|
||||
&& now.unix_timestamp() > b.due.expect("err").unix_timestamp()
|
||||
|| a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
|
||||
{
|
||||
match a.action {
|
||||
IlmAction::DeleteAllVersionsAction
|
||||
| IlmAction::DelMarkerDeleteAllVersionsAction
|
||||
| IlmAction::DeleteAction
|
||||
| IlmAction::DeleteVersionAction => {
|
||||
return Ordering::Less;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
match b.action {
|
||||
IlmAction::DeleteAllVersionsAction
|
||||
| IlmAction::DelMarkerDeleteAllVersionsAction
|
||||
| IlmAction::DeleteAction
|
||||
| IlmAction::DeleteVersionAction => {
|
||||
return Ordering::Greater;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
return Ordering::Less;
|
||||
}
|
||||
|
||||
if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() {
|
||||
return Ordering::Less;
|
||||
}
|
||||
return Ordering::Greater;
|
||||
});
|
||||
return events[0].clone();
|
||||
}
|
||||
|
||||
Event::default()
|
||||
}
|
||||
|
||||
async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event {
|
||||
if let Some(filter_rules) = self.filter_rules(obj).await {
|
||||
for rule in filter_rules.iter() {
|
||||
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
|
||||
if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
|
||||
if newer_noncurrent_versions == 0 {
|
||||
continue;
|
||||
}
|
||||
return Event {
|
||||
action: IlmAction::DeleteVersionAction,
|
||||
rule_id: rule.id.clone().expect("err"),
|
||||
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
|
||||
newer_noncurrent_versions: newer_noncurrent_versions as usize,
|
||||
due: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
storage_class: "".into(),
|
||||
};
|
||||
} else {
|
||||
return Event {
|
||||
action: IlmAction::DeleteVersionAction,
|
||||
rule_id: rule.id.clone().expect("err"),
|
||||
noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
|
||||
newer_noncurrent_versions: 0,
|
||||
due: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
storage_class: "".into(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Event::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait LifecycleCalculate {
|
||||
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LifecycleCalculate for LifecycleExpiration {
|
||||
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
|
||||
if !obj.is_latest || !obj.delete_marker {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LifecycleCalculate for NoncurrentVersionTransition {
|
||||
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
|
||||
if obj.is_latest || self.storage_class.is_none() {
|
||||
return None;
|
||||
}
|
||||
if self.noncurrent_days.is_none() {
|
||||
return obj.successor_mod_time;
|
||||
}
|
||||
Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LifecycleCalculate for Transition {
|
||||
fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
|
||||
if !obj.is_latest || self.days.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(date) = self.date.clone() {
|
||||
return Some(date.into());
|
||||
}
|
||||
|
||||
if self.days.is_none() {
|
||||
return obj.mod_time;
|
||||
}
|
||||
Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
|
||||
if days == 0 {
|
||||
return mod_time;
|
||||
}
|
||||
let t = mod_time
|
||||
.to_offset(offset!(-0:00:00))
|
||||
.saturating_add(Duration::days(0 /*days as i64*/)); //debug
|
||||
let mut hour = 3600;
|
||||
if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
|
||||
if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
|
||||
hour = num_hour;
|
||||
}
|
||||
}
|
||||
//t.Truncate(24 * hour)
|
||||
t
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ObjectOpts {
|
||||
pub name: String,
|
||||
pub user_tags: String,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
pub size: usize,
|
||||
pub version_id: String,
|
||||
pub is_latest: bool,
|
||||
pub delete_marker: bool,
|
||||
pub num_versions: usize,
|
||||
pub successor_mod_time: Option<OffsetDateTime>,
|
||||
pub transition_status: String,
|
||||
pub restore_ongoing: bool,
|
||||
pub restore_expires: Option<OffsetDateTime>,
|
||||
pub versioned: bool,
|
||||
pub version_suspended: bool,
|
||||
}
|
||||
|
||||
impl ObjectOpts {
|
||||
pub fn expired_object_deletemarker(&self) -> bool {
|
||||
self.delete_marker && self.num_versions == 1
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Event {
|
||||
pub action: IlmAction,
|
||||
pub rule_id: String,
|
||||
pub due: Option<OffsetDateTime>,
|
||||
pub noncurrent_days: u32,
|
||||
pub newer_noncurrent_versions: usize,
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
impl Default for Event {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
action: IlmAction::NoneAction,
|
||||
rule_id: "".into(),
|
||||
due: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ExpirationOptions {
|
||||
pub expire: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TransitionOptions {
|
||||
pub status: String,
|
||||
pub tier: String,
|
||||
pub etag: String,
|
||||
pub restore_request: RestoreObjectRequest,
|
||||
pub restore_expiry: OffsetDateTime,
|
||||
pub expire_restored: bool,
|
||||
}
|
||||
|
||||
impl Default for TransitionOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
status: Default::default(),
|
||||
tier: Default::default(),
|
||||
etag: Default::default(),
|
||||
restore_request: Default::default(),
|
||||
restore_expiry: OffsetDateTime::now_utc(),
|
||||
expire_restored: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod bucket_lifecycle_audit;
|
||||
pub mod bucket_lifecycle_ops;
|
||||
pub mod lifecycle;
|
||||
pub mod rule;
|
||||
pub mod tier_last_day_stats;
|
||||
pub mod tier_sweeper;
|
||||
@@ -0,0 +1,69 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use s3s::dto::{LifecycleRuleFilter, Transition};
|
||||
|
||||
const _ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
|
||||
const _ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
|
||||
const ERR_TRANSITION_INVALID: &str =
|
||||
"Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
|
||||
const _ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
|
||||
|
||||
pub trait Filter {
|
||||
fn test_tags(&self, user_tags: &str) -> bool;
|
||||
fn by_size(&self, sz: i64) -> bool;
|
||||
}
|
||||
|
||||
impl Filter for LifecycleRuleFilter {
|
||||
fn test_tags(&self, user_tags: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn by_size(&self, sz: i64) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
pub trait TransitionOps {
|
||||
fn validate(&self) -> Result<(), std::io::Error>;
|
||||
}
|
||||
|
||||
impl TransitionOps for Transition {
|
||||
fn validate(&self) -> Result<(), std::io::Error> {
|
||||
if !self.date.is_none() && self.days.expect("err!") > 0 {
|
||||
return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
|
||||
}
|
||||
|
||||
if self.storage_class.is_none() {
|
||||
return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rule() {
|
||||
//assert!(skip_access_checks(p.to_str().unwrap()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use sha2::Sha256;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Sub;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{error, warn};
|
||||
|
||||
use crate::heal::data_usage_cache::TierStats;
|
||||
|
||||
pub type DailyAllTierStats = HashMap<String, LastDayTierStats>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LastDayTierStats {
|
||||
bins: [TierStats; 24],
|
||||
updated_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
impl Default for LastDayTierStats {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
bins: Default::default(),
|
||||
updated_at: OffsetDateTime::now_utc(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LastDayTierStats {
|
||||
pub fn add_stats(&mut self, ts: TierStats) {
|
||||
let mut now = OffsetDateTime::now_utc();
|
||||
self.forward_to(&mut now);
|
||||
|
||||
let now_idx = now.hour() as usize;
|
||||
self.bins[now_idx] = self.bins[now_idx].add(&ts);
|
||||
}
|
||||
|
||||
fn forward_to(&mut self, t: &mut OffsetDateTime) {
|
||||
if t.unix_timestamp() == 0 {
|
||||
*t = OffsetDateTime::now_utc();
|
||||
}
|
||||
|
||||
let since = t.sub(self.updated_at).whole_hours();
|
||||
if since < 1 {
|
||||
return;
|
||||
}
|
||||
|
||||
let (idx, mut last_idx) = (t.hour(), self.updated_at.hour());
|
||||
|
||||
self.updated_at = *t;
|
||||
|
||||
if since >= 24 {
|
||||
self.bins = [TierStats::default(); 24];
|
||||
return;
|
||||
}
|
||||
|
||||
while last_idx != idx {
|
||||
last_idx = (last_idx + 1) % 24;
|
||||
self.bins[last_idx as usize] = TierStats::default();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||
let mut cl = self.clone();
|
||||
let mut cm = m.clone();
|
||||
let mut merged = LastDayTierStats::default();
|
||||
|
||||
if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {
|
||||
cm.forward_to(&mut cl.updated_at);
|
||||
merged.updated_at = cl.updated_at;
|
||||
} else {
|
||||
cl.forward_to(&mut cm.updated_at);
|
||||
merged.updated_at = cm.updated_at;
|
||||
}
|
||||
|
||||
for (i, _) in cl.bins.iter().enumerate() {
|
||||
merged.bins[i] = cl.bins[i].add(&cm.bins[i]);
|
||||
}
|
||||
|
||||
merged
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {}
|
||||
@@ -0,0 +1,152 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::io::{Cursor, Write};
|
||||
use xxhash_rust::xxh64;
|
||||
|
||||
use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
|
||||
use super::lifecycle::{self, ObjectOpts};
|
||||
use crate::global::GLOBAL_TierConfigMgr;
|
||||
|
||||
static XXHASH_SEED: u64 = 0;
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
struct ObjSweeper {
|
||||
object: String,
|
||||
bucket: String,
|
||||
version_id: String,
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
transition_status: String,
|
||||
transition_tier: String,
|
||||
transition_version_id: String,
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ObjSweeper {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||
Ok(Self {
|
||||
object: object.into(),
|
||||
bucket: bucket.into(),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn with_version(&mut self, vid: String) -> &Self {
|
||||
self.version_id = vid;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||
self.versioned = versioned;
|
||||
self.suspended = suspended;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.suspended,
|
||||
..Default::default()
|
||||
};
|
||||
if self.suspended && self.version_id == "" {
|
||||
opts.version_id = String::from("");
|
||||
}
|
||||
opts
|
||||
}
|
||||
|
||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||
self.transition_tier = info.tier;
|
||||
self.transition_status = info.status;
|
||||
self.remote_object = info.name;
|
||||
self.transition_version_id = info.version_id;
|
||||
}
|
||||
|
||||
pub fn should_remove_remote_object(&self) -> Option<Jentry> {
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut del_tier = false;
|
||||
if !self.versioned || self.suspended {
|
||||
// 1, 2.a, 2.b
|
||||
del_tier = true;
|
||||
} else if self.versioned && self.version_id != "" {
|
||||
// 3.a
|
||||
del_tier = true;
|
||||
}
|
||||
if del_tier {
|
||||
return Some(Jentry {
|
||||
obj_name: self.remote_object.clone(),
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn sweep(&self) {
|
||||
let je = self.should_remove_remote_object();
|
||||
if !je.is_none() {
|
||||
let mut expiry_state = GLOBAL_ExpiryState.write().await;
|
||||
expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(unused_assignments)]
|
||||
pub struct Jentry {
|
||||
obj_name: String,
|
||||
version_id: String,
|
||||
tier_name: String,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
let _ = hasher.write(format!("{}", self.tier_name).as_bytes());
|
||||
let _ = hasher.write(format!("{}", self.obj_name).as_bytes());
|
||||
hasher.flush();
|
||||
xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
|
||||
let mut config_mgr = GLOBAL_TierConfigMgr.write().await;
|
||||
let w = match config_mgr.get_driver(tier_name).await {
|
||||
Ok(w) => w,
|
||||
Err(e) => return Err(std::io::Error::other(e)),
|
||||
};
|
||||
w.remove(obj_name, rv_id).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {}
|
||||
Reference in New Issue
Block a user