feat(table-catalog): internalize catalog backing paths (#3295)

* feat(table-catalog): internalize catalog backing paths

* fix(table-catalog): clean internal catalog on bucket delete

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>

crates/ecstore/src/bucket/metadata.rs

@@ -30,6 +30,7 @@ use s3s::dto::{
     ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
 };
 use serde::Serializer;
+use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::io::{Read, Write};
 use std::sync::Arc;
@@ -246,6 +247,28 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
 pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
 pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
 pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
+pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
+pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
+
+pub fn table_catalog_path_hash(value: &str) -> String {
+    let digest = Sha256::digest(value.as_bytes());
+    let mut output = String::with_capacity(digest.len() * 2);
+    const HEX: &[u8; 16] = b"0123456789abcdef";
+    for byte in digest {
+        output.push(char::from(HEX[usize::from(byte >> 4)]));
+        output.push(char::from(HEX[usize::from(byte & 0x0f)]));
+    }
+    output
+}
+
+pub fn table_bucket_catalog_metadata_prefix(bucket: &str) -> String {
+    format!(
+        "{}/{}/{}",
+        BUCKET_TABLE_CATALOG_META_PREFIX,
+        BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX,
+        table_catalog_path_hash(bucket)
+    )
+}
 
 #[derive(Debug, Clone)]
 pub struct BucketMetadata {

crates/ecstore/src/store/bucket.rs

@@ -13,7 +13,10 @@
 // limitations under the License.
 
 use super::*;
-use crate::bucket::{metadata::BUCKET_TABLE_RESERVED_PREFIX, utils::is_meta_bucketname};
+use crate::bucket::{
+    metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
+    utils::is_meta_bucketname,
+};
 use crate::global::get_global_bucket_monitor;
 use crate::set_disk::get_lock_acquire_timeout;
 
@@ -56,6 +59,13 @@ async fn validate_table_bucket_delete_guard(bucket: &str) -> Result<()> {
     Ok(())
 }
 
+fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
+    [
+        table_bucket_catalog_metadata_prefix(bucket),
+        format!("{BUCKET_META_PREFIX}/{bucket}"),
+    ]
+}
+
 impl ECStore {
     #[instrument(skip(self))]
     pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
@@ -229,9 +239,11 @@ impl ECStore {
 
         // TODO: replication opts.srdelete_op
 
-        // Delete the metadata
-        self.delete_all(RUSTFS_META_BUCKET, format!("{BUCKET_META_PREFIX}/{bucket}").as_str())
-            .await?;
+        // Delete internal metadata after the bucket is gone so stale catalog records cannot be reused
+        // if the same bucket name is created again.
+        for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
+            self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()).await?;
+        }
         if let Some(monitor) = get_global_bucket_monitor() {
             monitor.delete_bucket(bucket);
         }
@@ -241,7 +253,10 @@ impl ECStore {
 
 #[cfg(test)]
 mod tests {
-    use super::{should_override_created_from_metadata, validate_table_bucket_delete_allowed};
+    use super::{
+        bucket_delete_metadata_cleanup_prefixes, should_override_created_from_metadata, validate_table_bucket_delete_allowed,
+    };
+    use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
     use crate::error::StorageError;
     use time::OffsetDateTime;
 
@@ -264,4 +279,12 @@ mod tests {
         assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
         assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
     }
+
+    #[test]
+    fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
+        let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
+
+        assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
+        assert!(prefixes.contains(&"buckets/analytics".to_string()));
+    }
 }