fix(ecstore): repair lifecycle transition and restore flows (#2240)

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: weisd <im@weisd.in>
This commit is contained in:
cxymds
2026-03-23 12:29:13 +08:00
committed by GitHub
parent 2e7abfbd63
commit 236142a682
16 changed files with 1209 additions and 94 deletions
+83 -1
View File
@@ -28,7 +28,7 @@ use http::StatusCode;
use metrics::counter;
use rustfs_config::RUSTFS_REGION;
use rustfs_ecstore::bucket::{
lifecycle::bucket_lifecycle_ops::validate_transition_tier,
lifecycle::bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, validate_transition_tier},
metadata::{
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG,
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG,
@@ -126,6 +126,27 @@ fn validate_lifecycle_rule_status(rules: &[LifecycleRule]) -> Result<(), &'stati
Ok(())
}
fn lifecycle_has_transition_rules(config: &BucketLifecycleConfiguration) -> bool {
config.rules.iter().any(|rule| {
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
&& (rule.transitions.as_ref().is_some_and(|transitions| {
transitions.iter().any(|transition| {
transition
.storage_class
.as_ref()
.is_some_and(|storage_class| !storage_class.as_str().is_empty())
})
}) || rule.noncurrent_version_transitions.as_ref().is_some_and(|transitions| {
transitions.iter().any(|transition| {
transition
.storage_class
.as_ref()
.is_some_and(|storage_class| !storage_class.as_str().is_empty())
})
}))
})
}
#[derive(Clone, Default)]
pub struct DefaultBucketUsecase {
context: Option<Arc<AppContext>>,
@@ -1051,6 +1072,17 @@ impl DefaultBucketUsecase {
.await
.map_err(ApiError::from)?;
if lifecycle_has_transition_rules(&input_cfg)
&& let Some(store) = new_object_layer_fn()
{
let bucket_name = bucket.clone();
tokio::spawn(async move {
if let Err(err) = enqueue_transition_for_existing_objects(store, &bucket_name).await {
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue transition for existing objects");
}
});
}
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
}
@@ -1902,6 +1934,56 @@ mod tests {
assert_eq!(validate_lifecycle_rule_status(&rules).unwrap_err(), ERR_LIFECYCLE_RULE_STATUS);
}
#[test]
fn lifecycle_has_transition_rules_ignores_disabled_rules() {
let config = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::DISABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("disabled-transition".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]),
}],
};
assert!(!lifecycle_has_transition_rules(&config));
}
#[test]
fn lifecycle_has_transition_rules_accepts_enabled_noncurrent_transitions() {
let config = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("enabled-noncurrent-transition".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]),
prefix: None,
transitions: None,
}],
};
assert!(lifecycle_has_transition_rules(&config));
}
#[tokio::test]
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
let input = ListBucketsInput::builder().build().unwrap();
@@ -0,0 +1,439 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
use crate::storage::ecfs::FS;
use bytes::Bytes;
use futures::stream;
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_ecstore::{
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
bucket::metadata_sys,
client::object_api_utils::to_s3s_etag,
client::transition_api::{ReadCloser, ReaderImpl},
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
global::GLOBAL_TierConfigMgr,
store::ECStore,
store_api::{
BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions,
PutObjReader,
},
tier::{
tier_config::{TierConfig, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts},
},
};
use s3s::{S3Request, dto::*};
use serial_test::serial;
use std::{
collections::HashMap,
convert::Infallible,
io::Cursor,
path::PathBuf,
sync::{Arc, Once, OnceLock},
time::Duration,
};
use tokio::fs;
use tokio::io::AsyncReadExt;
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
static INIT: Once = Once::new();
const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
fn init_tracing() {
INIT.call_once(|| {});
}
async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
init_tracing();
if let Some((paths, ecstore)) = GLOBAL_ENV.get() {
return (paths.clone(), ecstore.clone());
}
let test_base_dir = format!("/tmp/rustfs_app_lifecycle_test_{}", Uuid::new_v4());
let temp_dir = PathBuf::from(&test_base_dir);
if temp_dir.exists() {
fs::remove_dir_all(&temp_dir).await.ok();
}
fs::create_dir_all(&temp_dir).await.unwrap();
let disk_paths = vec![
temp_dir.join("disk1"),
temp_dir.join("disk2"),
temp_dir.join("disk3"),
temp_dir.join("disk4"),
];
for disk_path in &disk_paths {
fs::create_dir_all(disk_path).await.unwrap();
}
let mut endpoints = Vec::new();
for (i, disk_path) in disk_paths.iter().enumerate() {
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
endpoints.push(endpoint);
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "test".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:9003".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.unwrap();
let buckets_list = ecstore
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
(disk_paths, ecstore)
}
async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
(**ecstore)
.make_bucket(
bucket_name,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("Failed to create test bucket");
}
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
let mut reader = PutObjReader::from_vec(data.to_vec());
(**ecstore)
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("Failed to upload test object");
}
async fn set_bucket_lifecycle_transition_with_tier(
bucket_name: &str,
storage_class: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let lifecycle_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<LifecycleConfiguration>
<Rule>
<ID>test-rule</ID>
<Status>Enabled</Status>
<Filter>
<Prefix>test/</Prefix>
</Filter>
<Transition>
<Days>0</Days>
<StorageClass>{storage_class}</StorageClass>
</Transition>
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
#[derive(Clone, Default)]
struct MockWarmBackend {
objects: Arc<Mutex<HashMap<String, Vec<u8>>>>,
}
impl MockWarmBackend {
async fn put_bytes(&self, object: &str, bytes: Vec<u8>) -> String {
self.objects.lock().await.insert(object.to_string(), bytes);
Uuid::new_v4().to_string()
}
async fn read_bytes(&self, reader: ReaderImpl) -> Result<Vec<u8>, std::io::Error> {
match reader {
ReaderImpl::Body(bytes) => Ok(bytes.to_vec()),
ReaderImpl::ObjectBody(mut reader) => {
let mut buf = Vec::new();
reader.stream.read_to_end(&mut buf).await?;
Ok(buf)
}
}
}
}
#[async_trait::async_trait]
impl WarmBackend for MockWarmBackend {
async fn put(&self, object: &str, r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
let bytes = self.read_bytes(r).await?;
Ok(self.put_bytes(object, bytes).await)
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
_length: i64,
_meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
let bytes = self.read_bytes(r).await?;
Ok(self.put_bytes(object, bytes).await)
}
async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let objects = self.objects.lock().await;
let Some(bytes) = objects.get(object) else {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
};
let start = opts.start_offset.max(0) as usize;
let end = if opts.length > 0 {
start.saturating_add(opts.length as usize).min(bytes.len())
} else {
bytes.len()
};
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
}
async fn remove(&self, object: &str, _rv: &str) -> Result<(), std::io::Error> {
self.objects.lock().await.remove(object);
Ok(())
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
}
async fn register_mock_tier(tier_name: &str) -> MockWarmBackend {
let backend = MockWarmBackend::default();
let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
tier_config_mgr.tiers.insert(
tier_name.to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::MinIO,
name: tier_name.to_string(),
..Default::default()
},
);
tier_config_mgr
.driver_cache
.insert(tier_name.to_string(), Box::new(backend.clone()));
backend
}
async fn wait_for_transition(
ecstore: &Arc<ECStore>,
bucket: &str,
object: &str,
timeout: Duration,
) -> Option<rustfs_ecstore::store_api::ObjectInfo> {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if let Ok(info) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await
&& info.transitioned_object.status == "complete"
{
return Some(info);
}
if tokio::time::Instant::now() >= deadline {
return None;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
input,
method,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
fn streaming_blob_from_bytes(data: &[u8]) -> StreamingBlob {
let body = Bytes::copy_from_slice(data);
StreamingBlob::wrap::<_, Infallible>(stream::once(async move { Ok(body) }))
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "requires isolated global object layer state"]
async fn put_and_copy_object_transition_immediately_via_usecases() {
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let usecase = DefaultObjectUsecase::without_context();
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&tier_name).await;
let put_bucket = format!("test-api-put-{}", &Uuid::new_v4().simple().to_string()[..8]);
let put_object = "test/object.txt";
let put_payload = b"Hello, immediate transition through put API!";
create_test_bucket(&ecstore, put_bucket.as_str()).await;
set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name)
.await
.expect("Failed to set lifecycle configuration");
let put_input = PutObjectInput::builder()
.bucket(put_bucket.clone())
.key(put_object.to_string())
.body(Some(streaming_blob_from_bytes(put_payload)))
.content_length(Some(put_payload.len() as i64))
.build()
.unwrap();
usecase
.execute_put_object(&fs, build_request(put_input, Method::PUT))
.await
.expect("Failed to put object through usecase");
let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("object should transition immediately after put usecase");
assert_eq!(put_info.transitioned_object.status, "complete");
assert_eq!(put_info.transitioned_object.tier, tier_name);
assert!(backend.objects.lock().await.contains_key(&put_info.transitioned_object.name));
let src_bucket = format!("test-api-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
let dst_bucket = format!("test-api-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
let src_object = "test/source.txt";
let dst_object = "test/copied.txt";
let copy_payload = b"copy object immediate transition through copy API";
create_test_bucket(&ecstore, src_bucket.as_str()).await;
create_test_bucket(&ecstore, dst_bucket.as_str()).await;
set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
.await
.expect("Failed to set destination lifecycle configuration");
upload_test_object(&ecstore, src_bucket.as_str(), src_object, copy_payload).await;
let copy_input = CopyObjectInput::builder()
.copy_source(CopySource::Bucket {
bucket: src_bucket.clone().into(),
key: src_object.to_string().into(),
version_id: None,
})
.bucket(dst_bucket.clone())
.key(dst_object.to_string())
.build()
.unwrap();
usecase
.execute_copy_object(build_request(copy_input, Method::PUT))
.await
.expect("Failed to copy object through usecase");
let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("copied object should transition immediately after copy usecase");
assert_eq!(copy_info.transitioned_object.status, "complete");
assert_eq!(copy_info.transitioned_object.tier, tier_name);
assert!(backend.objects.lock().await.contains_key(&copy_info.transitioned_object.name));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "requires isolated global object layer state"]
async fn complete_multipart_upload_transitions_immediately_via_usecase() {
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultMultipartUsecase::without_context();
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&tier_name).await;
let bucket = format!("test-api-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "test/multipart.txt";
let payload = b"multipart immediate transition through complete API";
create_test_bucket(&ecstore, bucket.as_str()).await;
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
.await
.expect("Failed to set lifecycle configuration");
let upload = ecstore
.new_multipart_upload(bucket.as_str(), object, &ObjectOptions::default())
.await
.expect("Failed to create multipart upload");
let mut reader = PutObjReader::from_vec(payload.to_vec());
let uploaded_part = ecstore
.put_object_part(bucket.as_str(), object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
.await
.expect("Failed to upload multipart part");
let complete_input = CompleteMultipartUploadInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.upload_id(upload.upload_id.clone())
.multipart_upload(Some(CompletedMultipartUpload {
parts: Some(vec![CompletedPart {
part_number: Some(1),
e_tag: uploaded_part.etag.clone().map(|etag| to_s3s_etag(&etag)),
..Default::default()
}]),
}))
.build()
.unwrap();
usecase
.execute_complete_multipart_upload(build_request(complete_input, Method::POST))
.await
.expect("Failed to complete multipart upload through usecase");
let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
.await
.expect("multipart object should transition immediately after complete usecase");
assert_eq!(info.transitioned_object.status, "complete");
assert_eq!(info.transitioned_object.tier, tier_name);
assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name));
}
+3
View File
@@ -20,3 +20,6 @@ pub mod bucket_usecase;
pub mod context;
pub mod multipart_usecase;
pub mod object_usecase;
#[cfg(test)]
mod lifecycle_transition_api_test;
+7
View File
@@ -30,6 +30,7 @@ use futures::StreamExt;
use http::{HeaderMap, Uri};
use rustfs_ecstore::bucket::quota::checker::QuotaChecker;
use rustfs_ecstore::bucket::{
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
metadata_sys,
quota::QuotaOperation,
replication::{get_must_replicate_options, must_replicate, schedule_replication},
@@ -60,6 +61,10 @@ use tokio_util::io::StreamReader;
use tracing::{info, instrument, warn};
use urlencoding::encode;
async fn maybe_enqueue_transition_immediate(obj_info: &rustfs_ecstore::store_api::ObjectInfo, src: LcEventSrc) {
enqueue_transition_immediate(obj_info, src).await;
}
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_size: i64) -> S3Result<()> {
@@ -356,6 +361,8 @@ impl DefaultMultipartUsecase {
}
}
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
// Invalidate cache for the completed multipart object
let manager = get_concurrency_manager();
let mpu_bucket = bucket.clone();
+12 -1
View File
@@ -42,7 +42,8 @@ use metrics::{counter, histogram};
use rustfs_ecstore::bucket::quota::checker::QuotaChecker;
use rustfs_ecstore::bucket::{
lifecycle::{
bucket_lifecycle_ops::{RestoreRequestOps, post_restore_opts},
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts},
lifecycle::{self, Lifecycle, TransitionOptions},
},
metadata::{BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG},
@@ -116,6 +117,10 @@ use tokio_util::io::{ReaderStream, StreamReader};
use tracing::{debug, error, info, instrument, warn};
use uuid::Uuid;
async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) {
enqueue_transition_immediate(obj_info, src).await;
}
/// Extract trailing-header checksum values, overriding the corresponding input fields.
fn apply_trailing_checksums(
algorithm: Option<&str>,
@@ -509,6 +514,8 @@ impl DefaultObjectUsecase {
.await
.map_err(ApiError::from)?;
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
// Fast in-memory update for immediate quota consistency
rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await;
@@ -2307,6 +2314,8 @@ impl DefaultObjectUsecase {
.await
.map_err(ApiError::from)?;
maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await;
// Update quota tracking after successful copy
if has_bucket_metadata {
rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, oi.size as u64).await;
@@ -3607,6 +3616,8 @@ impl DefaultObjectUsecase {
.await
.map_err(ApiError::from)?;
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
let manager = get_concurrency_manager();
let fpath_clone = fpath.clone();
let bucket_clone = bucket.clone();