gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 16:46:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/ilm (#721)

Browse Source 
* fix tip remote tier error
* fix transitioned_object
* fix filemeta
* add GCS R2
* add aliyun tencent huaweicloud azure gcs r2 backend tier
* fix signer
* change azure to s3
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>
This commit is contained in:
likewu
2025-10-27 20:23:50 +08:00
committed by GitHub
parent 6f3d2885cd
commit 2aca1f77af
38 changed files with 4054 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/tier.rs
+18
View File
@@ -116,6 +116,24 @@ impl Operation for AddTier {
TierType::MinIO => {
args.name = args.minio.clone().unwrap().name;
}
TierType::Aliyun => {
args.name = args.aliyun.clone().unwrap().name;
}
TierType::Tencent => {
args.name = args.tencent.clone().unwrap().name;
}
TierType::Huaweicloud => {
args.name = args.huaweicloud.clone().unwrap().name;
}
TierType::Azure => {
args.name = args.azure.clone().unwrap().name;
}
TierType::GCS => {
args.name = args.gcs.clone().unwrap().name;
}
TierType::R2 => {
args.name = args.r2.clone().unwrap().name;
}
_ => (),
}
debug!("add tier args {:?}", args);
rustfs/src/storage/ecfs.rs
+154 -72
View File
@@ -33,7 +33,10 @@ use http::{HeaderMap, StatusCode};
use metrics::counter;
use rustfs_ecstore::{
bucket::{
lifecycle::{bucket_lifecycle_ops::validate_transition_tier, lifecycle::Lifecycle},
lifecycle::{
bucket_lifecycle_ops::{RestoreRequestOps, post_restore_opts, validate_transition_tier},
lifecycle::{self, Lifecycle, TransitionOptions},
},
metadata::{
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_REPLICATION_CONFIG,
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
@@ -74,7 +77,8 @@ use rustfs_ecstore::{
},
};
use rustfs_filemeta::REPLICATE_INCOMING_DELETE;
use rustfs_filemeta::{ReplicationStatusType, ReplicationType, VersionPurgeStatusType, fileinfo::ObjectPartInfo};
use rustfs_filemeta::fileinfo::{ObjectPartInfo, RestoreStatusOps};
use rustfs_filemeta::{ReplicationStatusType, ReplicationType, VersionPurgeStatusType};
use rustfs_kms::{
DataKey,
service_manager::get_global_encryption_service,
@@ -103,11 +107,15 @@ use rustfs_utils::{
CompressionAlgorithm,
http::{
AMZ_BUCKET_REPLICATION_STATUS,
headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER},
headers::{
AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE,
RESERVED_METADATA_PREFIX_LOWER,
},
},
path::{is_dir_object, path_join_buf},
};
use rustfs_zip::CompressionFormat;
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
use std::{
collections::HashMap,
@@ -846,118 +854,180 @@ impl S3 for FS {
Ok(S3Response::new(output))
}
async fn restore_object(&self, _req: S3Request<RestoreObjectInput>) -> S3Result<S3Response<RestoreObjectOutput>> {
Err(s3_error!(NotImplemented, "RestoreObject is not implemented yet"))
/*
let bucket = params.bucket;
if let Err(e) = un_escape_path(params.object) {
async fn restore_object(&self, req: S3Request<RestoreObjectInput>) -> S3Result<S3Response<RestoreObjectOutput>> {
let RestoreObjectInput {
bucket,
key: object,
restore_request: rreq,
version_id,
..
} = req.input.clone();
let rreq = rreq.unwrap();
/*if let Err(e) = un_escape_path(object) {
warn!("post restore object failed, e: {:?}", e);
return Err(S3Error::with_message(S3ErrorCode::Custom("PostRestoreObjectFailed".into()), "post restore object failed"));
}
}*/
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
if Err(err) = check_request_auth_type(req, policy::RestoreObjectAction, bucket, object) {
/*if Err(err) = check_request_auth_type(req, policy::RestoreObjectAction, bucket, object) {
return Err(S3Error::with_message(S3ErrorCode::Custom("PostRestoreObjectFailed".into()), "post restore object failed"));
}
}*/
if req.content_length <= 0 {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed"));
}
let Some(opts) = post_restore_opts(req, bucket, object) else {
/*if req.content_length <= 0 {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed"));
}*/
let Ok(opts) = post_restore_opts(&version_id.unwrap(), &bucket, &object).await else {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrEmptyRequestBody".into()),
"post restore object failed",
));
};
let Some(obj_info) = store.get_object_info(bucket, object, opts) else {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed"));
let Ok(mut obj_info) = store.get_object_info(&bucket, &object, &opts).await else {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrEmptyRequestBody".into()),
"post restore object failed",
));
};
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed"));
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrEmptyRequestBody".into()),
"post restore object failed",
));
}
let mut api_err;
let Some(rreq) = parse_restore_request(req.body(), req.content_length) else {
let api_err = errorCodes.ToAPIErr(ErrMalformedXML);
api_err.description = err.Error()
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed"));
};
let mut status_code = http::StatusCode::OK;
//let mut api_err;
let mut _status_code = http::StatusCode::OK;
let mut already_restored = false;
if Err(err) = rreq.validate(store) {
api_err = errorCodes.ToAPIErr(ErrMalformedXML)
api_err.description = err.Error()
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed"));
if let Err(_err) = rreq.validate(store.clone()) {
//api_err = to_api_err(ErrMalformedXML);
//api_err.description = err.to_string();
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrEmptyRequestBody".into()),
"post restore object failed",
));
} else {
if obj_info.restore_ongoing && rreq.Type != "SELECT" {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrObjectRestoreAlreadyInProgress".into()), "post restore object failed"));
if obj_info.restore_ongoing && (rreq.type_.is_none() || rreq.type_.as_ref().unwrap().as_str() != "SELECT") {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrObjectRestoreAlreadyInProgress".into()),
"post restore object failed",
));
}
if !obj_info.restore_ongoing && !obj_info.restore_expires.unix_timestamp() == 0 {
status_code = http::StatusCode::Accepted;
if !obj_info.restore_ongoing && obj_info.restore_expires.unwrap().unix_timestamp() != 0 {
_status_code = http::StatusCode::ACCEPTED;
already_restored = true;
}
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days);
let mut metadata = clone_mss(obj_info.user_defined);
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), *rreq.days.as_ref().unwrap());
let mut metadata = obj_info.user_defined.clone();
if rreq.type != "SELECT" {
obj_info.metadataOnly = true;
metadata[xhttp.AmzRestoreExpiryDays] = rreq.days;
metadata[xhttp.AmzRestoreRequestDate] = OffsetDateTime::now_utc().format(http::TimeFormat);
let mut header = HeaderMap::new();
let obj_info_ = obj_info.clone();
if rreq.type_.is_none() || rreq.type_.as_ref().unwrap().as_str() != "SELECT" {
obj_info.metadata_only = true;
metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap().to_string());
metadata.insert(AMZ_RESTORE_REQUEST_DATE.to_string(), OffsetDateTime::now_utc().format(&Rfc3339).unwrap());
if already_restored {
metadata[AmzRestore] = completed_restore_obj(restore_expiry).String()
metadata.insert(
X_AMZ_RESTORE.as_str().to_string(),
RestoreStatus {
is_restore_in_progress: Some(false),
restore_expiry_date: Some(Timestamp::from(restore_expiry)),
}
.to_string(),
);
} else {
metadata[AmzRestore] = ongoing_restore_obj().to_string()
metadata.insert(
X_AMZ_RESTORE.as_str().to_string(),
RestoreStatus {
is_restore_in_progress: Some(true),
restore_expiry_date: Some(Timestamp::from(OffsetDateTime::now_utc())),
}
.to_string(),
);
}
obj_info.user_defined = metadata;
if let Err(err) = store.copy_object(bucket, object, bucket, object, obj_info, ObjectOptions {
version_id: obj_info.version_id,
}, ObjectOptions {
version_id: obj_info.version_id,
m_time: obj_info.mod_time,
}) {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "post restore object failed"));
if let Err(_err) = store
.clone()
.copy_object(
&bucket,
&object,
&bucket,
&object,
&mut obj_info,
&ObjectOptions {
version_id: obj_info_.version_id.map(|e| e.to_string()),
..Default::default()
},
&ObjectOptions {
version_id: obj_info_.version_id.map(|e| e.to_string()),
mod_time: obj_info_.mod_time,
..Default::default()
},
)
.await
{
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrInvalidObjectState".into()),
"post restore object failed",
));
}
if already_restored {
return Ok(());
let output = RestoreObjectOutput {
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
restore_output_path: None,
};
return Ok(S3Response::new(output));
}
}
let restore_object = must_get_uuid();
if rreq.output_location.s3.bucket_name != "" {
w.Header()[AmzRestoreOutputPath] = []string{pathJoin(rreq.OutputLocation.S3.BucketName, rreq.OutputLocation.S3.Prefix, restore_object)}
let restore_object = Uuid::new_v4().to_string();
//if let Some(rreq) = rreq {
if let Some(output_location) = &rreq.output_location {
if let Some(s3) = &output_location.s3 {
if !s3.bucket_name.is_empty() {
header.insert(
X_AMZ_RESTORE_OUTPUT_PATH,
format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(),
);
}
}
}
w.WriteHeader(status_code)
send_event(EventArgs {
//}
/*send_event(EventArgs {
event_name: event::ObjectRestorePost,
bucket_name: bucket,
object: obj_info,
req_params: extract_req_params(r),
user_agent: req.user_agent(),
host: handlers::get_source_ip(r),
});
});*/
tokio::spawn(async move {
if !rreq.SelectParameters.IsEmpty() {
let actual_size = obj_info.get_actual_size();
/*if rreq.select_parameters.is_some() {
let actual_size = obj_info_.get_actual_size();
if actual_size.is_err() {
return Err(S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "post restore object failed"));
}
let object_rsc = s3select.NewObjectReadSeekCloser(
|offset int64| -> (io.ReadCloser, error) {
let object_rsc = s3select.new_object_read_seek_closer(
|offset: i64| -> (ReadCloser, error) {
rs := &HTTPRangeSpec{
IsSuffixLength: false,
Start: offset,
End: -1,
}
return getTransitionedObjectReader(bucket, object, rs, r.Header,
obj_info, ObjectOptions {version_id: obj_info.version_id});
return get_transitioned_object_reader(bucket, object, rs, r.Header,
obj_info, ObjectOptions {version_id: obj_info_.version_id});
},
actual_size.unwrap(),
);
if err = rreq.SelectParameters.Open(objectRSC); err != nil {
if err = rreq.select_parameters.open(object_rsc); err != nil {
if serr, ok := err.(s3select.SelectError); ok {
let encoded_error_response = encodeResponse(APIErrorResponse {
code: serr.ErrorCode(),
@@ -982,29 +1052,41 @@ impl S3 for FS {
rreq.select_parameters.evaluate(rw);
rreq.select_parameters.Close();
return Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header));
}
}*/
let opts = ObjectOptions {
transition: TransitionOptions {
restore_request: rreq,
restore_expiry: restore_expiry,
restore_expiry,
..Default::default()
},
version_id: objInfo.version_id,
}
if Err(err) = store.restore_transitioned_object(bucket, object, opts) {
format!(format!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err.to_string()));
return Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header));
version_id: obj_info_.version_id.map(|e| e.to_string()),
..Default::default()
};
if let Err(err) = store.clone().restore_transitioned_object(&bucket, &object, &opts).await {
warn!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err.to_string());
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrRestoreTransitionedObject".into()),
format!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err),
));
}
send_event(EventArgs {
/*send_event(EventArgs {
EventName: event.ObjectRestoreCompleted,
BucketName: bucket,
Object: objInfo,
ReqParams: extractReqParams(r),
UserAgent: r.UserAgent(),
Host: handlers.GetSourceIP(r),
});
});*/
Ok(())
});
*/
let output = RestoreObjectOutput {
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
restore_output_path: None,
};
return Ok(S3Response::with_headers(output, header));
}
/// Delete a bucket