Fix/ilm (#721)

* fix tip remote tier error
* fix transitioned_object
* fix filemeta
* add GCS R2
* add aliyun tencent huaweicloud azure gcs r2 backend tier
* fix signer
* change azure to s3
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>

crates/filemeta/src/fileinfo.rs

@@ -12,16 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use super::filemeta::TRANSITION_COMPLETE;
 use crate::error::{Error, Result};
 use crate::{ReplicationState, ReplicationStatusType, VersionPurgeStatusType};
 use bytes::Bytes;
 use rmp_serde::Serializer;
 use rustfs_utils::HashAlgorithm;
 use rustfs_utils::http::headers::{RESERVED_METADATA_PREFIX_LOWER, RUSTFS_HEALING};
+use s3s::dto::{RestoreStatus, Timestamp};
+use s3s::header::X_AMZ_RESTORE;
 use serde::Deserialize;
 use serde::Serialize;
 use std::collections::HashMap;
-use time::OffsetDateTime;
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use uuid::Uuid;
 
 pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
@@ -35,6 +38,8 @@ pub const TIER_FV_ID: &str = "tier-free-versionID";
 pub const TIER_FV_MARKER: &str = "tier-free-marker";
 pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
 
+const ERR_RESTORE_HDR_MALFORMED: &str = "x-amz-restore header malformed";
+
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
 pub struct ObjectPartInfo {
     pub etag: String,
@@ -394,7 +399,10 @@ impl FileInfo {
 
     /// Check if the object is remote (transitioned to another tier)
     pub fn is_remote(&self) -> bool {
-        !self.transition_tier.is_empty()
+        if self.transition_status != TRANSITION_COMPLETE {
+            return false;
+        }
+        !is_restored_object_on_disk(&self.metadata)
     }
 
     /// Get the data directory for this object
@@ -535,3 +543,101 @@ pub struct FilesInfo {
     pub files: Vec<FileInfo>,
     pub is_truncated: bool,
 }
+
+pub trait RestoreStatusOps {
+    fn expiry(&self) -> Option<OffsetDateTime>;
+    fn on_going(&self) -> bool;
+    fn on_disk(&self) -> bool;
+    fn to_string(&self) -> String;
+}
+
+impl RestoreStatusOps for RestoreStatus {
+    fn expiry(&self) -> Option<OffsetDateTime> {
+        if self.on_going() {
+            return None;
+        }
+        self.restore_expiry_date.clone().map(OffsetDateTime::from)
+    }
+
+    fn on_going(&self) -> bool {
+        if let Some(on_going) = self.is_restore_in_progress {
+            return on_going;
+        }
+        false
+    }
+
+    fn on_disk(&self) -> bool {
+        let expiry = self.expiry();
+        if let Some(expiry0) = expiry
+            && OffsetDateTime::now_utc().unix_timestamp() < expiry0.unix_timestamp()
+        {
+            return true;
+        }
+        false
+    }
+
+    fn to_string(&self) -> String {
+        if self.on_going() {
+            return "ongoing-request=\"true\"".to_string();
+        }
+        format!(
+            "ongoing-request=\"false\", expiry-date=\"{}\"",
+            OffsetDateTime::from(self.restore_expiry_date.clone().unwrap())
+                .format(&Rfc3339)
+                .unwrap()
+        )
+    }
+}
+
+fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
+    let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect();
+    let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect();
+    if progress_tokens.len() != 2 {
+        return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
+    }
+    if progress_tokens[0].trim() != "ongoing-request" {
+        return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
+    }
+
+    match progress_tokens[1] {
+        "true" | "\"true\"" => {
+            if tokens.len() == 1 {
+                return Ok(RestoreStatus {
+                    is_restore_in_progress: Some(true),
+                    ..Default::default()
+                });
+            }
+        }
+        "false" | "\"false\"" => {
+            if tokens.len() != 2 {
+                return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
+            }
+            let expiry_tokens: Vec<&str> = tokens[1].splitn(2, "=").collect();
+            if expiry_tokens.len() != 2 {
+                return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
+            }
+            if expiry_tokens[0].trim() != "expiry-date" {
+                return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
+            }
+            let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339).unwrap();
+            /*if err != nil {
+                return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
+            }*/
+            return Ok(RestoreStatus {
+                is_restore_in_progress: Some(false),
+                restore_expiry_date: Some(Timestamp::from(expiry)),
+            });
+        }
+        _ => (),
+    }
+    Err(Error::other(ERR_RESTORE_HDR_MALFORMED))
+}
+
+pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
+    if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str()) {
+        if let Ok(restore_status) = parse_restore_obj_status(restore_hdr) {
+            return restore_status.on_disk();
+        }
+    }
+    false
+}

crates/filemeta/src/filemeta.rs

@@ -22,8 +22,9 @@ use byteorder::ByteOrder;
 use bytes::Bytes;
 use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
 use rustfs_utils::http::headers::{
-    self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX,
-    RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
+    self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS,
+    AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER,
+    VERSION_PURGE_STATUS_KEY,
 };
 use s3s::header::X_AMZ_RESTORE;
 use serde::{Deserialize, Serialize};
@@ -68,9 +69,6 @@ pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
 pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
 pub const TRANSITION_TIER: &str = "transition-tier";
 
-const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days";
-const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date";
-
 // type ScanHeaderVersionFn = Box<dyn Fn(usize, &[u8], &[u8]) -> Result<()>>;
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -693,11 +691,6 @@ impl FileMeta {
             }
         }
 
-        // ???
-        if fi.transition_status == TRANSITION_COMPLETE {
-            update_version = false;
-        }
-
         for (i, ver) in self.versions.iter().enumerate() {
             if ver.header.version_id != fi.version_id {
                 continue;
@@ -1088,13 +1081,24 @@ impl FileMeta {
 
     /// Count shared data directories
     pub fn shared_data_dir_count(&self, version_id: Option<Uuid>, data_dir: Option<Uuid>) -> usize {
+        if self.data.entries().unwrap_or_default() > 0
+            && version_id.is_some()
+            && self
+                .data
+                .find(version_id.unwrap().to_string().as_str())
+                .unwrap_or_default()
+                .is_some()
+        {
+            return 0;
+        }
+
         self.versions
             .iter()
             .filter(|v| {
                 v.header.version_type == VersionType::Object && v.header.version_id != version_id && v.header.user_data_dir()
             })
-            .filter_map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).ok().flatten())
-            .filter(|&dir| Some(dir) == data_dir)
+            .filter_map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).ok())
+            .filter(|&dir| dir == data_dir)
             .count()
     }
 
@@ -1838,8 +1842,8 @@ impl MetaObject {
 
     pub fn remove_restore_hdrs(&mut self) {
         self.meta_user.remove(X_AMZ_RESTORE.as_str());
-        self.meta_user.remove(X_AMZ_RESTORE_EXPIRY_DAYS);
-        self.meta_user.remove(X_AMZ_RESTORE_REQUEST_DATE);
+        self.meta_user.remove(AMZ_RESTORE_EXPIRY_DAYS);
+        self.meta_user.remove(AMZ_RESTORE_REQUEST_DATE);
     }
 
     pub fn uses_data_dir(&self) -> bool {

crates/filemeta/src/filemeta_inline.rs

@@ -44,6 +44,20 @@ impl InlineData {
         if self.0.is_empty() { &self.0 } else { &self.0[1..] }
     }
 
+    pub fn entries(&self) -> Result<usize> {
+        if self.0.is_empty() || !self.version_ok() {
+            return Ok(0);
+        }
+
+        let buf = self.after_version();
+
+        let mut cur = Cursor::new(buf);
+
+        let fields_len = rmp::decode::read_map_len(&mut cur)?;
+
+        Ok(fields_len as usize)
+    }
+
     pub fn find(&self, key: &str) -> Result<Option<Vec<u8>>> {
         if self.0.is_empty() || !self.version_ok() {
             return Ok(None);