diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index effc04dc3..b3424cdb2 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -1100,6 +1100,98 @@ where mod test { use super::*; + /// Decode a whitespace-tolerant hex fixture into bytes. + fn decode_hex(s: &str) -> Vec { + let s: String = s.chars().filter(|c| !c.is_whitespace()).collect(); + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("valid hex fixture")) + .collect() + } + + /// backlog#580: prove RustFS parses a real MinIO-written bucket `.metadata.bin` + /// blob without loss. The fixture is the raw `.metadata.bin` object body + /// (4-byte `format|version` header + msgpack) carved from the + /// `.minio.sys/buckets/interop/.metadata.bin` object written by MinIO + /// `RELEASE.2025-07-23` — see `tests/fixtures/minio/README.md`. + #[test] + fn parses_real_minio_bucket_metadata_blob_without_loss() { + let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")); + + // Same 4-byte format|version header (1|1) and msgpack layout as MinIO. + BucketMetadata::check_header(&blob).expect("valid .metadata.bin header"); + let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata"); + + // Raw config fields survive the msgpack decode (PascalCase MinIO field names). + assert_eq!(bm.name, "interop"); + assert!(!bm.policy_config_json.is_empty(), "policy JSON present"); + assert!(!bm.lifecycle_config_xml.is_empty(), "lifecycle XML present"); + assert!(!bm.object_lock_config_xml.is_empty(), "object-lock XML present"); + assert!(!bm.versioning_config_xml.is_empty(), "versioning XML present"); + assert!(!bm.tagging_config_xml.is_empty(), "tagging XML present"); + assert!(!bm.quota_config_json.is_empty(), "quota JSON present"); + + // Typed parse of each stored config must succeed. `parse_all_configs` + // logs+skips on error, so a None here means a real MinIO-compat parse gap. + bm.parse_all_configs().expect("parse_all_configs"); + assert!(bm.policy_config.is_some(), "policy parsed"); + assert!(bm.versioning_config.is_some(), "versioning parsed"); + assert!(bm.object_lock_config.is_some(), "object-lock parsed"); + assert!(bm.tagging_config.is_some(), "tagging parsed"); + assert!(bm.quota_config.is_some(), "quota parsed"); + assert!( + bm.lifecycle_config.is_some(), + "lifecycle parsed (MinIO writes an extension element)" + ); + assert!(bm.notification_config.is_some(), "notification parsed"); + assert!(bm.sse_config.is_some(), "encryption (SSE) parsed"); + assert!(bm.replication_config.is_some(), "replication parsed"); + + // Object lock is expressed through the parsed config, not the legacy + // `LockEnabled` flag (MinIO leaves that false for config-based locks). + assert!(bm.object_locking(), "object lock active via parsed config"); + } + + /// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's + /// inline-data extraction does not yet recover the object body from a + /// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data` + /// returns bytes that are not the `.metadata.bin` blob (no `format|version` + /// header). Kept as an ignored, documented reproduction until the MinIO + /// inline-data framing is handled on the read path. + /// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata + /// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]` + /// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that + /// bitrot prefix, not a format incompatibility). Running the raw inline shard + /// through RustFS's `BitrotReader` with the default `HighwayHash256S` must + /// verify the checksum and yield the exact `.metadata.bin` blob. + #[tokio::test] + async fn reads_minio_inline_bucket_metadata_via_bitrot() { + use crate::erasure::coding::BitrotReader; + use rustfs_utils::HashAlgorithm; + + let xlmeta = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata_full.xlmeta.hex")); + let fm = rustfs_filemeta::FileMeta::load(&xlmeta).expect("parse MinIO xl.meta"); + let fi = fm + .into_fileinfo("interop", ".metadata.bin", "", true, false, false) + .expect("into_fileinfo"); + // The raw inline shard is `[HighwayHash256 (32B)][object body]`. + let inline = fi.data.expect("inline shard present"); + let algo = HashAlgorithm::HighwayHash256S; + let body_len = inline.len() - algo.size(); + + let mut reader = BitrotReader::new(std::io::Cursor::new(inline.to_vec()), body_len, algo, false); + let mut body = vec![0u8; body_len]; + let read = reader.read(&mut body).await.expect("bitrot verify + read MinIO inline shard"); + assert_eq!(read, body_len); + + // The verified body is exactly the `.metadata.bin` blob, and it parses. + BucketMetadata::check_header(&body).expect("recovered body is a valid .metadata.bin"); + let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob"); + assert_eq!(bm.name, "interop"); + bm.parse_all_configs().expect("parse recovered configs"); + assert!(bm.lifecycle_config.is_some()); + } + #[tokio::test] async fn marshal_msg() { // write_time(OffsetDateTime::UNIX_EPOCH).unwrap(); diff --git a/crates/ecstore/src/bucket/migration.rs b/crates/ecstore/src/bucket/migration.rs index a2c6692b1..ead8dee73 100644 --- a/crates/ecstore/src/bucket/migration.rs +++ b/crates/ecstore/src/bucket/migration.rs @@ -517,4 +517,122 @@ mod tests { assert_eq!(decoded.id, 123); assert_eq!(decoded.targets_map["arn:replication::1:dest"].resync_id, "reset-1"); } + + fn decode_hex(s: &str) -> Vec { + let s: String = s.chars().filter(|c| !c.is_whitespace()).collect(); + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("valid hex fixture")) + .collect() + } + + /// backlog#580 Phase 2/4: end-to-end proof that `try_migrate_bucket_metadata` + /// pulls a real MinIO-written bucket-metadata blob from a `.minio.sys` layout + /// into `.rustfs.sys`, and that the migrated blob loads every config. Uses a + /// throwaway 4-drive local ECStore. Fixture: `tests/fixtures/minio/README.md`. + #[tokio::test] + async fn migrates_real_minio_bucket_metadata_end_to_end() { + use crate::bucket::metadata::{BUCKET_METADATA_FILE, BucketMetadata}; + use crate::config::com::read_config; + use crate::disk::endpoint::Endpoint; + use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; + use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, BucketOptions, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations}; + use crate::store::{ECStore, init_local_disks}; + use rustfs_utils::path::SLASH_SEPARATOR; + use tokio::fs; + use tokio_util::sync::CancellationToken; + use uuid::Uuid; + + let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")); + + // --- Stand up a throwaway 4-drive local ECStore. --- + let base = std::path::PathBuf::from(format!("/tmp/rustfs_minio_migrate_test_{}", Uuid::new_v4())); + let disk_paths: Vec<_> = (1..=4).map(|i| base.join(format!("disk{i}"))).collect(); + for p in &disk_paths { + fs::create_dir_all(p).await.unwrap(); + } + let mut endpoints = Vec::new(); + for (i, p) in disk_paths.iter().enumerate() { + let mut ep = Endpoint::try_from(p.to_str().unwrap()).unwrap(); + ep.set_pool_index(0); + ep.set_set_index(0); + ep.set_disk_index(i); + endpoints.push(ep); + } + let endpoint_pools = EndpointServerPools(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: "minio-migrate-test".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }]); + init_local_disks(endpoint_pools.clone()).await.unwrap(); + let ecstore = ECStore::new("127.0.0.1:0".parse().unwrap(), endpoint_pools, CancellationToken::new()) + .await + .unwrap(); + let existing: Vec = ecstore + .list_bucket(&BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .unwrap() + .into_iter() + .map(|b| b.name) + .collect(); + crate::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), existing).await; + + let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}interop{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}"); + let put_opts = ObjectOptions::default(); + + // --- Arrange the pre-migration state a MinIO import starts from: the --- + // bucket exists and its config lives under `.minio.sys`, while `.rustfs.sys` + // has no metadata for it yet. + ecstore.make_bucket("interop", &MakeBucketOptions::default()).await.unwrap(); + let _ = ecstore + .delete_object(RUSTFS_META_BUCKET, &meta_path, ObjectOptions::default()) + .await; + // MinIO leaves its `.minio.sys` meta volume on every drive; recreate it so + // the source object can be seeded through the object layer. + for p in &disk_paths { + fs::create_dir_all(p.join(MIGRATING_META_BUCKET)).await.ok(); + } + let mut src = PutObjReader::from_vec(blob.clone()); + ecstore + .put_object(MIGRATING_META_BUCKET, &meta_path, &mut src, &put_opts) + .await + .expect("seed .minio.sys bucket metadata"); + + // --- Run the real startup migration. --- + super::try_migrate_bucket_metadata(ecstore.clone()).await; + + // --- The migrated `.rustfs.sys` blob must carry every MinIO config, --- + // byte-identical to the source (typed XML/JSON parsing of these fields is + // covered by the bucket-metadata parse-parity test). + let migrated = read_config(ecstore.clone(), &meta_path) + .await + .expect("read migrated bucket metadata"); + BucketMetadata::check_header(&migrated).expect("migrated blob has a valid header"); + let bm = BucketMetadata::unmarshal(&migrated[4..]).expect("unmarshal migrated bucket metadata"); + let src = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal source bucket metadata"); + + assert_eq!(bm.name, "interop"); + assert_eq!(bm.policy_config_json, src.policy_config_json, "policy migrated intact"); + assert_eq!(bm.lifecycle_config_xml, src.lifecycle_config_xml, "lifecycle migrated intact"); + assert_eq!(bm.object_lock_config_xml, src.object_lock_config_xml, "object-lock migrated intact"); + assert_eq!(bm.versioning_config_xml, src.versioning_config_xml, "versioning migrated intact"); + assert_eq!(bm.tagging_config_xml, src.tagging_config_xml, "tagging migrated intact"); + assert_eq!(bm.quota_config_json, src.quota_config_json, "quota migrated intact"); + assert_eq!(bm.notification_config_xml, src.notification_config_xml, "notification migrated intact"); + assert_eq!(bm.encryption_config_xml, src.encryption_config_xml, "encryption migrated intact"); + assert_eq!(bm.replication_config_xml, src.replication_config_xml, "replication migrated intact"); + assert!(!bm.lifecycle_config_xml.is_empty(), "lifecycle present"); + assert!(!bm.replication_config_xml.is_empty(), "replication present"); + + fs::remove_dir_all(&base).await.ok(); + } } diff --git a/crates/ecstore/tests/fixtures/minio/README.md b/crates/ecstore/tests/fixtures/minio/README.md new file mode 100644 index 000000000..e326e7ad1 --- /dev/null +++ b/crates/ecstore/tests/fixtures/minio/README.md @@ -0,0 +1,21 @@ +# Real MinIO on-disk fixtures (backlog#580 interop) + +Generated by MinIO `RELEASE.2025-07-23T15-54-02Z` (single-drive, local SSE-S3 +KMS + webhook notify target enabled) writing a bucket configured with +**versioning, object-lock (GOVERNANCE default), lifecycle, tagging, quota, a +public-download policy, SSE-S3 encryption, a webhook notification target, and a +replication rule** (which also seeds a bucket-targets blob), plus inline / +versioned / multipart objects and a delete marker. + +- `bucket_metadata_full.xlmeta.hex` — the `.minio.sys/buckets/interop/.metadata.bin` + object (xl.meta wrapper + inline `.metadata.bin` blob). +- `bucket_metadata.blob.hex` — the raw `.metadata.bin` object body (4-byte + `format|version` header + msgpack), carved from the object above. + +Unencrypted object `xl.meta` fixtures (clean plaintext sizes) live under +`crates/filemeta/tests/fixtures/minio/`. + +These prove RustFS reads and migrates MinIO-written metadata without loss. +Not captured here (tooling/scope): CORS, public-access-block, and bucket ACL +(the SNSD test binary + `mc` did not expose CORS; bucket-targets credentials are +stored KMS-encrypted and are a documented partial). diff --git a/crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex b/crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex new file mode 100644 index 000000000..8fc7d0802 --- /dev/null +++ b/crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex @@ -0,0 +1 @@ +01000100de0019a44e616d65a7696e7465726f70a743726561746564c70c05000000006a4d22410c8f35c0ab4c6f636b456e61626c6564c2b0506f6c696379436f6e6669674a534f4ec501167b2256657273696f6e223a22323031322d31302d3137222c2253746174656d656e74223a5b7b22456666656374223a22416c6c6f77222c225072696e636970616c223a7b22415753223a5b222a225d7d2c22416374696f6e223a5b2273333a4765744275636b65744c6f636174696f6e222c2273333a4c6973744275636b6574225d2c225265736f75726365223a5b2261726e3a6177733a73333a3a3a696e7465726f70225d7d2c7b22456666656374223a22416c6c6f77222c225072696e636970616c223a7b22415753223a5b222a225d7d2c22416374696f6e223a5b2273333a4765744f626a656374225d2c225265736f75726365223a5b2261726e3a6177733a73333a3a3a696e7465726f702f2a225d7d5d7db54e6f74696669636174696f6e436f6e666967584d4cc4e73c4e6f74696669636174696f6e436f6e66696775726174696f6e20786d6c6e733d22687474703a2f2f73332e616d617a6f6e6177732e636f6d2f646f632f323030362d30332d30312f223e3c5175657565436f6e66696775726174696f6e3e3c49643e3c2f49643e3c4576656e743e73333a4f626a656374437265617465643a2a3c2f4576656e743e3c51756575653e61726e3a6d696e696f3a7371733a3a7072696d6172793a776562686f6f6b3c2f51756575653e3c2f5175657565436f6e66696775726174696f6e3e3c2f4e6f74696669636174696f6e436f6e66696775726174696f6e3eb24c6966656379636c65436f6e666967584d4cc501583c4c6966656379636c65436f6e66696775726174696f6e3e3c52756c653e3c49443e64393669346738396b38683236613935737436303c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c46696c7465723e3c5072656669783e3c2f5072656669783e3c2f46696c7465723e3c45787069726174696f6e3e3c446179733e33303c2f446179733e3c2f45787069726174696f6e3e3c4e6f6e63757272656e7456657273696f6e45787069726174696f6e3e3c4e6f6e63757272656e74446179733e373c2f4e6f6e63757272656e74446179733e3c2f4e6f6e63757272656e7456657273696f6e45787069726174696f6e3e3c2f52756c653e3c4578706972795570646174656441743e323032362d30372d30375431353a35383a35372e3333373331355a3c2f4578706972795570646174656441743e3c2f4c6966656379636c65436f6e66696775726174696f6e3eb34f626a6563744c6f636b436f6e666967584d4cc4b83c4f626a6563744c6f636b436f6e66696775726174696f6e3e3c4f626a6563744c6f636b456e61626c65643e456e61626c65643c2f4f626a6563744c6f636b456e61626c65643e3c52756c653e3c44656661756c74526574656e74696f6e3e3c4d6f64653e474f5645524e414e43453c2f4d6f64653e3c446179733e373c2f446179733e3c2f44656661756c74526574656e74696f6e3e3c2f52756c653e3c2f4f626a6563744c6f636b436f6e66696775726174696f6e3eb356657273696f6e696e67436f6e666967584d4cc47b3c56657273696f6e696e67436f6e66696775726174696f6e20786d6c6e733d22687474703a2f2f73332e616d617a6f6e6177732e636f6d2f646f632f323030362d30332d30312f223e3c5374617475733e456e61626c65643c2f5374617475733e3c2f56657273696f6e696e67436f6e66696775726174696f6e3eb3456e6372797074696f6e436f6e666967584d4cc4f03c53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e20786d6c6e733d22687474703a2f2f73332e616d617a6f6e6177732e636f6d2f646f632f323030362d30332d30312f223e3c52756c653e3c4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c535345416c676f726974686d3e4145533235363c2f535345416c676f726974686d3e3c2f4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c2f52756c653e3c2f53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3eb054616767696e67436f6e666967584d4cc4803c54616767696e673e3c5461675365743e3c5461673e3c4b65793e7465616d3c2f4b65793e3c56616c75653e73746f726167653c2f56616c75653e3c2f5461673e3c5461673e3c4b65793e656e763c2f4b65793e3c56616c75653e70726f643c2f56616c75653e3c2f5461673e3c2f5461675365743e3c2f54616767696e673eaf51756f7461436f6e6669674a534f4ec4477b2271756f7461223a31303733373431383234302c2273697a65223a302c2272617465223a302c227265717565737473223a302c2271756f746174797065223a2268617264227db45265706c69636174696f6e436f6e666967584d4cc5027f3c5265706c69636174696f6e436f6e66696775726174696f6e3e3c52756c653e3c49443e64393669346d30396b386832766c6469666b61673c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c5072696f726974793e313c2f5072696f726974793e3c44656c6574654d61726b65725265706c69636174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f44656c6574654d61726b65725265706c69636174696f6e3e3c44656c6574655265706c69636174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f44656c6574655265706c69636174696f6e3e3c44657374696e6174696f6e3e3c4275636b65743e61726e3a6d696e696f3a7265706c69636174696f6e3a3a65663538353961662d313230612d343231382d393462352d6265323334373066336336303a696e7465726f702d64723c2f4275636b65743e3c2f44657374696e6174696f6e3e3c536f7572636553656c656374696f6e43726974657269613e3c5265706c6963614d6f64696669636174696f6e733e3c5374617475733e456e61626c65643c2f5374617475733e3c2f5265706c6963614d6f64696669636174696f6e733e3c2f536f7572636553656c656374696f6e43726974657269613e3c46696c7465723e3c5072656669783e3c2f5072656669783e3c2f46696c7465723e3c4578697374696e674f626a6563745265706c69636174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f4578697374696e674f626a6563745265706c69636174696f6e3e3c2f52756c653e3c526f6c653e3c2f526f6c653e3c2f5265706c69636174696f6e436f6e66696775726174696f6e3eb74275636b657454617267657473436f6e6669674a534f4ec5029320017202928fc0a787c7c4cd4ad0e4fea39d969029ff12b9872f2d555ace5e5f9e3e6267de9939051ed910cc9a244ff99624e6d67cb0c19f1eec1f8250992f9a095bdab238a1d294aecb456b873d0162ca6aed18ed79844c6ef5a4a357b192d5e9b507a6933cebca12ea2ab2cc9f61ffb864784aaa37009471e13c8b35db6990a30f1cff0416afe1e1da8b275d35f510f0daedd0f18bb58d1495dac6fd2e00d380e52e0356709e0ca450177b1a1d4f5d4b8c4486ce000df06b1d758e1e04b4627d42225e2851576ea79cedf1e02010c44e053eaccac748a7c0ce9168ef99fb12e68a41fce1dc7a92a598a0555a90d50ca86a2c3e7e93e48e209d80cba5d6860ac2159d15780f6cdf8b5728d3407cfab2e77cc6af17fc4592ef1d5c7640a6600782d5215e6038e05de7f589946cb130204c074cf23950c308d81226f3cb8897f478492342cb3c2807e99cd4130507c75a49b15e431f48fea70d8e75128b0857891791348738327e2c82911ef4785f533f44c7b14f4e2fd5e6dd7fd69ebcfa7214db0d936ff66174c329a145aa6c213733091a196fbac46518f1d32a23186fbe5ee497fe02aacb79191e2fe9b741e078be4b117bc6a9be33471d4f2fe1b1f69baba6173bbde1e50bccf8f8218daf0e473377cd11c0be59dd1ebf7e0b1e13865a333d2af5a5d3d834002d9c0df610beb2b7c36ebe3cdecae1a6264a1c61757190425b43e57a51af72ab40b9aa30a70b7670079831dd8dca3864c0a320c7a53c03c593159db1284da8a5cb75263c37df90ac1e7a4259d45804572e557216d7e51ca0501ed4b6df5de62b5e32a79c981763b5d449d3dad02933e45cdc00c44906e85d91a790d4a6a9ec6805c8266693a76bd6fd172deed5e45eedfd67b17b59601c44c6c8a9c3f10cd6e61dd1f41176b4b8a1968467bb4275636b657454617267657473436f6e6669674d6574614a534f4ec5022c7b22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d4976223a226f704a6a4e4a524163523050433241694d39687a4f2b6669396a5a37537963504a624a463538673657396b3d222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d53332d4b6d732d4b65792d4964223a226d792d6d696e696f2d6b6579222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d53332d4b6d732d5365616c65642d4b6579223a2272573564656946486558456a31494a6d754844314f4967754e4f475355302f4176707278464f37446c4b2f532b6869316d3035524445713570735a6c495441716b4d53577176546842514139716541556f7578782f626a4278697a45366a742f4662593036513d3d222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d53332d5365616c65642d4b6579223a2249414566414e527242456462762f58657944307930614a384c66536b53637748645234366b426d7949567647554352734a724d6d4f557872496e396371666874305979413332777077446e59494b496c3951317148513d3d222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d5365616c2d416c676f726974686d223a224441524576322d484d41432d534841323536227db5506f6c696379436f6e666967557064617465644174c70c05000000006a4d22411cbe42d8b94f626a6563744c6f636b436f6e666967557064617465644174c70c05000000006a4d2241117b72d8b9456e6372797074696f6e436f6e666967557064617465644174c70c05000000006a4d2241210723a0b654616767696e67436f6e666967557064617465644174c70c05000000006a4d2241168a85e8b451756f7461436f6e666967557064617465644174c70c05000000006a4d2241196776b8ba5265706c69636174696f6e436f6e666967557064617465644174c70c05000000006a4d225808da53b0b956657273696f6e696e67436f6e666967557064617465644174c70c05000000006a4d22410f256548b84c6966656379636c65436f6e666967557064617465644174c70c05000000006a4d2241141f4a78bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c05000000006a4d2241249f7148bc4275636b657454617267657473436f6e666967557064617465644174c70c05000000006a4d22580855f8e0d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c05000000006a4d22580855f8e0 \ No newline at end of file diff --git a/crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex b/crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex new file mode 100644 index 000000000..c418ed5ca --- /dev/null +++ b/crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex @@ -0,0 +1 @@ +584c322001000300c600000178030301c42697c41000000000000000000000000000000000d318c00d31d6e9e600c40435de5ded01060001c5014a83a45479706501a556324f626ade0011a24944c41000000000000000000000000000000000a444446972c410865bf145dc2f45ff928860f185aa6768a64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a657391d11074aa506172744153697a657391d11074a453697a65d11074a54d54696d65d318c00d31d6e9e600a74d65746153797381bc782d6d696e696f2d696e7465726e616c2d696e6c696e652d64617461c40474727565a74d65746155737282a465746167d9206536623463393031346635396637383166626333356531376236633931326639ac636f6e74656e742d74797065b86170706c69636174696f6e2f6f637465742d73747265616da176ce6881059ace67b248600181a46e756c6cc510946774fc294177a90e63fb44df5a31ab2110a761b35bfc9042e3217df5e8b5eebf01000100de0019a44e616d65a7696e7465726f70a743726561746564c70c05000000006a4d22410c8f35c0ab4c6f636b456e61626c6564c2b0506f6c696379436f6e6669674a534f4ec501167b2256657273696f6e223a22323031322d31302d3137222c2253746174656d656e74223a5b7b22456666656374223a22416c6c6f77222c225072696e636970616c223a7b22415753223a5b222a225d7d2c22416374696f6e223a5b2273333a4765744275636b65744c6f636174696f6e222c2273333a4c6973744275636b6574225d2c225265736f75726365223a5b2261726e3a6177733a73333a3a3a696e7465726f70225d7d2c7b22456666656374223a22416c6c6f77222c225072696e636970616c223a7b22415753223a5b222a225d7d2c22416374696f6e223a5b2273333a4765744f626a656374225d2c225265736f75726365223a5b2261726e3a6177733a73333a3a3a696e7465726f702f2a225d7d5d7db54e6f74696669636174696f6e436f6e666967584d4cc4e73c4e6f74696669636174696f6e436f6e66696775726174696f6e20786d6c6e733d22687474703a2f2f73332e616d617a6f6e6177732e636f6d2f646f632f323030362d30332d30312f223e3c5175657565436f6e66696775726174696f6e3e3c49643e3c2f49643e3c4576656e743e73333a4f626a656374437265617465643a2a3c2f4576656e743e3c51756575653e61726e3a6d696e696f3a7371733a3a7072696d6172793a776562686f6f6b3c2f51756575653e3c2f5175657565436f6e66696775726174696f6e3e3c2f4e6f74696669636174696f6e436f6e66696775726174696f6e3eb24c6966656379636c65436f6e666967584d4cc501583c4c6966656379636c65436f6e66696775726174696f6e3e3c52756c653e3c49443e64393669346738396b38683236613935737436303c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c46696c7465723e3c5072656669783e3c2f5072656669783e3c2f46696c7465723e3c45787069726174696f6e3e3c446179733e33303c2f446179733e3c2f45787069726174696f6e3e3c4e6f6e63757272656e7456657273696f6e45787069726174696f6e3e3c4e6f6e63757272656e74446179733e373c2f4e6f6e63757272656e74446179733e3c2f4e6f6e63757272656e7456657273696f6e45787069726174696f6e3e3c2f52756c653e3c4578706972795570646174656441743e323032362d30372d30375431353a35383a35372e3333373331355a3c2f4578706972795570646174656441743e3c2f4c6966656379636c65436f6e66696775726174696f6e3eb34f626a6563744c6f636b436f6e666967584d4cc4b83c4f626a6563744c6f636b436f6e66696775726174696f6e3e3c4f626a6563744c6f636b456e61626c65643e456e61626c65643c2f4f626a6563744c6f636b456e61626c65643e3c52756c653e3c44656661756c74526574656e74696f6e3e3c4d6f64653e474f5645524e414e43453c2f4d6f64653e3c446179733e373c2f446179733e3c2f44656661756c74526574656e74696f6e3e3c2f52756c653e3c2f4f626a6563744c6f636b436f6e66696775726174696f6e3eb356657273696f6e696e67436f6e666967584d4cc47b3c56657273696f6e696e67436f6e66696775726174696f6e20786d6c6e733d22687474703a2f2f73332e616d617a6f6e6177732e636f6d2f646f632f323030362d30332d30312f223e3c5374617475733e456e61626c65643c2f5374617475733e3c2f56657273696f6e696e67436f6e66696775726174696f6e3eb3456e6372797074696f6e436f6e666967584d4cc4f03c53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e20786d6c6e733d22687474703a2f2f73332e616d617a6f6e6177732e636f6d2f646f632f323030362d30332d30312f223e3c52756c653e3c4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c535345416c676f726974686d3e4145533235363c2f535345416c676f726974686d3e3c2f4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c2f52756c653e3c2f53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3eb054616767696e67436f6e666967584d4cc4803c54616767696e673e3c5461675365743e3c5461673e3c4b65793e7465616d3c2f4b65793e3c56616c75653e73746f726167653c2f56616c75653e3c2f5461673e3c5461673e3c4b65793e656e763c2f4b65793e3c56616c75653e70726f643c2f56616c75653e3c2f5461673e3c2f5461675365743e3c2f54616767696e673eaf51756f7461436f6e6669674a534f4ec4477b2271756f7461223a31303733373431383234302c2273697a65223a302c2272617465223a302c227265717565737473223a302c2271756f746174797065223a2268617264227db45265706c69636174696f6e436f6e666967584d4cc5027f3c5265706c69636174696f6e436f6e66696775726174696f6e3e3c52756c653e3c49443e64393669346d30396b386832766c6469666b61673c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c5072696f726974793e313c2f5072696f726974793e3c44656c6574654d61726b65725265706c69636174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f44656c6574654d61726b65725265706c69636174696f6e3e3c44656c6574655265706c69636174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f44656c6574655265706c69636174696f6e3e3c44657374696e6174696f6e3e3c4275636b65743e61726e3a6d696e696f3a7265706c69636174696f6e3a3a65663538353961662d313230612d343231382d393462352d6265323334373066336336303a696e7465726f702d64723c2f4275636b65743e3c2f44657374696e6174696f6e3e3c536f7572636553656c656374696f6e43726974657269613e3c5265706c6963614d6f64696669636174696f6e733e3c5374617475733e456e61626c65643c2f5374617475733e3c2f5265706c6963614d6f64696669636174696f6e733e3c2f536f7572636553656c656374696f6e43726974657269613e3c46696c7465723e3c5072656669783e3c2f5072656669783e3c2f46696c7465723e3c4578697374696e674f626a6563745265706c69636174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f4578697374696e674f626a6563745265706c69636174696f6e3e3c2f52756c653e3c526f6c653e3c2f526f6c653e3c2f5265706c69636174696f6e436f6e66696775726174696f6e3eb74275636b657454617267657473436f6e6669674a534f4ec5029320017202928fc0a787c7c4cd4ad0e4fea39d969029ff12b9872f2d555ace5e5f9e3e6267de9939051ed910cc9a244ff99624e6d67cb0c19f1eec1f8250992f9a095bdab238a1d294aecb456b873d0162ca6aed18ed79844c6ef5a4a357b192d5e9b507a6933cebca12ea2ab2cc9f61ffb864784aaa37009471e13c8b35db6990a30f1cff0416afe1e1da8b275d35f510f0daedd0f18bb58d1495dac6fd2e00d380e52e0356709e0ca450177b1a1d4f5d4b8c4486ce000df06b1d758e1e04b4627d42225e2851576ea79cedf1e02010c44e053eaccac748a7c0ce9168ef99fb12e68a41fce1dc7a92a598a0555a90d50ca86a2c3e7e93e48e209d80cba5d6860ac2159d15780f6cdf8b5728d3407cfab2e77cc6af17fc4592ef1d5c7640a6600782d5215e6038e05de7f589946cb130204c074cf23950c308d81226f3cb8897f478492342cb3c2807e99cd4130507c75a49b15e431f48fea70d8e75128b0857891791348738327e2c82911ef4785f533f44c7b14f4e2fd5e6dd7fd69ebcfa7214db0d936ff66174c329a145aa6c213733091a196fbac46518f1d32a23186fbe5ee497fe02aacb79191e2fe9b741e078be4b117bc6a9be33471d4f2fe1b1f69baba6173bbde1e50bccf8f8218daf0e473377cd11c0be59dd1ebf7e0b1e13865a333d2af5a5d3d834002d9c0df610beb2b7c36ebe3cdecae1a6264a1c61757190425b43e57a51af72ab40b9aa30a70b7670079831dd8dca3864c0a320c7a53c03c593159db1284da8a5cb75263c37df90ac1e7a4259d45804572e557216d7e51ca0501ed4b6df5de62b5e32a79c981763b5d449d3dad02933e45cdc00c44906e85d91a790d4a6a9ec6805c8266693a76bd6fd172deed5e45eedfd67b17b59601c44c6c8a9c3f10cd6e61dd1f41176b4b8a1968467bb4275636b657454617267657473436f6e6669674d6574614a534f4ec5022c7b22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d4976223a226f704a6a4e4a524163523050433241694d39687a4f2b6669396a5a37537963504a624a463538673657396b3d222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d53332d4b6d732d4b65792d4964223a226d792d6d696e696f2d6b6579222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d53332d4b6d732d5365616c65642d4b6579223a2272573564656946486558456a31494a6d754844314f4967754e4f475355302f4176707278464f37446c4b2f532b6869316d3035524445713570735a6c495441716b4d53577176546842514139716541556f7578782f626a4278697a45366a742f4662593036513d3d222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d53332d5365616c65642d4b6579223a2249414566414e527242456462762f58657944307930614a384c66536b53637748645234366b426d7949567647554352734a724d6d4f557872496e396371666874305979413332777077446e59494b496c3951317148513d3d222c22582d4d696e696f2d496e7465726e616c2d5365727665722d536964652d456e6372797074696f6e2d5365616c2d416c676f726974686d223a224441524576322d484d41432d534841323536227db5506f6c696379436f6e666967557064617465644174c70c05000000006a4d22411cbe42d8b94f626a6563744c6f636b436f6e666967557064617465644174c70c05000000006a4d2241117b72d8b9456e6372797074696f6e436f6e666967557064617465644174c70c05000000006a4d2241210723a0b654616767696e67436f6e666967557064617465644174c70c05000000006a4d2241168a85e8b451756f7461436f6e666967557064617465644174c70c05000000006a4d2241196776b8ba5265706c69636174696f6e436f6e666967557064617465644174c70c05000000006a4d225808da53b0b956657273696f6e696e67436f6e666967557064617465644174c70c05000000006a4d22410f256548b84c6966656379636c65436f6e666967557064617465644174c70c05000000006a4d2241141f4a78bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c05000000006a4d2241249f7148bc4275636b657454617267657473436f6e666967557064617465644174c70c05000000006a4d22580855f8e0d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c05000000006a4d22580855f8e0 \ No newline at end of file diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 3638391f7..a2e9981ce 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -920,6 +920,52 @@ mod test { use proptest::collection::vec; use proptest::prelude::*; + /// backlog#580: RustFS parses real MinIO-written object xl.meta (inline, + /// versioned, and multipart) into equivalent `FileInfo`. Object metadata is + /// the strong part of MinIO interop; this pins it against real fixtures. + #[test] + fn parses_real_minio_object_xlmeta() { + // Small inlined object. + let small = create_minio_small_object_xlmeta().expect("load small fixture"); + let (major, _minor, _hdr, meta_ver) = FileMeta::read_format_versions(&small).unwrap(); + assert_eq!(major, 1); + assert_eq!(meta_ver, FileMeta::load(&small).unwrap().meta_ver); + let fm = FileMeta::load(&small).expect("parse small MinIO xl.meta"); + assert_eq!(fm.versions.len(), 1); + let fi = fm + .into_fileinfo("interop", "small.txt", "", false, false, true) + .expect("small fileinfo"); + assert_eq!(fi.size, 19, "small.txt size"); + assert_eq!(fi.num_versions, 1); + assert!(fi.is_latest); + + // Versioned object: two object versions plus a delete marker (latest). + let versioned = create_minio_versioned_object_xlmeta().expect("load versioned fixture"); + let fm = FileMeta::load(&versioned).expect("parse versioned MinIO xl.meta"); + assert_eq!(fm.versions.len(), 3, "two object versions + one delete marker"); + let delete_markers = fm + .versions + .iter() + .filter(|v| v.header.version_type == VersionType::Delete) + .count(); + assert_eq!(delete_markers, 1, "delete marker parsed from MinIO xl.meta"); + let fi = fm + .into_fileinfo("interop", "versioned.txt", "", false, false, true) + .expect("versioned fileinfo"); + assert_eq!(fi.num_versions, 3); + assert!(fi.version_id.is_some(), "versioned object carries a version id"); + + // Larger object stored as an erasure-coded part (not inlined). + let large = create_minio_large_object_xlmeta().expect("load large fixture"); + let fm = FileMeta::load(&large).expect("parse large MinIO xl.meta"); + assert_eq!(fm.versions.len(), 1); + let fi = fm + .into_fileinfo("interop", "large.bin", "", false, false, true) + .expect("large fileinfo"); + assert_eq!(fi.size, 300_000, "large.bin size"); + assert!(!fi.parts.is_empty(), "multipart/part layout present"); + } + /// Wraps a raw meta block in a valid XL2 container (header, bin32 length /// prefix, and CRC trailer) so decode tests exercise the meta parsing /// itself rather than the envelope checks. diff --git a/crates/filemeta/src/test_data.rs b/crates/filemeta/src/test_data.rs index e5cb73982..310346292 100644 --- a/crates/filemeta/src/test_data.rs +++ b/crates/filemeta/src/test_data.rs @@ -151,6 +151,26 @@ pub fn create_issue_2434_legacy_meta_v2_pool_xlmeta() -> Result> { decode_hex_fixture(include_str!("../tests/fixtures/issue_2434_legacy_meta_v2_pool.hex")) } +/// Real MinIO-written xl.meta for a small inlined object (`small.txt`, 19 bytes), +/// captured from MinIO `RELEASE.2025-07-23` — see `tests/fixtures/minio/README.md` +/// (backlog#580). +pub fn create_minio_small_object_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/minio/object_small_txt.xlmeta.hex")) +} + +/// Real MinIO-written xl.meta for a versioned object (`versioned.txt`, two +/// versions) — see `tests/fixtures/minio/README.md` (backlog#580). +pub fn create_minio_versioned_object_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/minio/object_versioned_txt.xlmeta.hex")) +} + +/// Real MinIO-written xl.meta for a larger, non-inlined object (`large.bin`, +/// 300 KiB, stored as `part.1`) — see `tests/fixtures/minio/README.md` +/// (backlog#580). +pub fn create_minio_large_object_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/minio/object_large_bin.xlmeta.hex")) +} + fn write_legacy_time(wr: &mut Vec, ts: OffsetDateTime) { wr.push(MSGPACK_EXT8); wr.push(12); diff --git a/crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex b/crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex new file mode 100644 index 000000000..2cfa16503 --- /dev/null +++ b/crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex @@ -0,0 +1 @@ +584c322001000300c6000001a4030301c42697c410727d23b40d2944e98e80d34405582863d318c00d6f34711ea0c404ad77932901020001c5017683a45479706501a556324f626ade0011a24944c410727d23b40d2944e98e80d34405582863a444446972c41059f847a8f7844db9b905a8352a9906bfa64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a657391d2000493e0aa506172744153697a657391d2000493e0a453697a65d2000493e0a54d54696d65d318c00d6f34711ea0a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c406333030303030b4782d6d696e696f2d696e7465726e616c2d637263c40bd001bbcd55bc019325344da74d65746155737282a465746167d92231326365653861623034393337646333333236666364326163633538626265352d31ac636f6e74656e742d74797065b86170706c69636174696f6e2f6f637465742d73747265616da176ce6881059aceb9aa266a \ No newline at end of file diff --git a/crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex b/crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex new file mode 100644 index 000000000..ef0c19531 --- /dev/null +++ b/crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex @@ -0,0 +1 @@ +584c322001000300c600000186030301c42697c410746dde29b59243efac7e37d408079143d318c00d6f26446968c4047761ed8301020001c5015883a45479706501a556324f626ade0011a24944c410746dde29b59243efac7e37d408079143a444446972c41019286b7785854468b7e9a8efddf329d7a64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a65739113aa506172744153697a65739113a453697a6513a54d54696d65d318c00d6f26446968a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c4023139b4782d6d696e696f2d696e7465726e616c2d637263c40bd0012cd2c3a001f50b8e4ba74d65746155737282ac636f6e74656e742d74797065aa746578742f706c61696ea465746167d92261396238633039663133663333663537306430326264346462316662356439372d31a176ce6881059ace755b0e49 \ No newline at end of file diff --git a/crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex b/crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex new file mode 100644 index 000000000..826153cd1 --- /dev/null +++ b/crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex @@ -0,0 +1 @@ +584c322001000300c600000375030303c42697c4106178c538372e4d93a122e55fc1bed128d318c00d6f3a44dd58c40439c29d5902000000c44383a45479706502a644656c4f626a83a24944c4106178c538372e4d93a122e55fc1bed128a54d54696d65d318c00d6f3a44dd58a74d65746153797380a176ce6881059ac42697c410065f0d6b0c1b4f94bfe1e98e8235fcf1d318c00d6f2fe16138c4049e93c2e701020001c5015883a45479706501a556324f626ade0011a24944c410065f0d6b0c1b4f94bfe1e98e8235fcf1a444446972c4108b76b5d6063245b6a4da67a01789d62ca64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a6573910caa506172744153697a6573910ca453697a650ca54d54696d65d318c00d6f2fe16138a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c4023132b4782d6d696e696f2d696e7465726e616c2d637263c40bd0014d39f0a401ff13e583a74d65746155737282ac636f6e74656e742d74797065aa746578742f706c61696ea465746167d92263363239643133613630313832303136356237333638373331313430613738632d31a176ce6881059ac42697c410b12dfda0210b405b89b1eaafd1964008d318c00d6f2b5d4848c404fc20d11a01020001c5015783a45479706501a556324f626ade0011a24944c410b12dfda0210b405b89b1eaafd1964008a444446972c41001dd843548214e4abbdfc472b35ec475a64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a65739102aa506172744153697a65739102a453697a6502a54d54696d65d318c00d6f2b5d4848a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c40132b4782d6d696e696f2d696e7465726e616c2d637263c40bd001e1792ab801e57f38d5a74d65746155737282ac636f6e74656e742d74797065aa746578742f706c61696ea465746167d92239306262653039353930353265626361656630623038323733666235313063372d31a176ce6881059ace638dab49 \ No newline at end of file diff --git a/docs/architecture/minio-file-format-compat.md b/docs/architecture/minio-file-format-compat.md index 00aa64c88..42dafc0bc 100644 --- a/docs/architecture/minio-file-format-compat.md +++ b/docs/architecture/minio-file-format-compat.md @@ -237,6 +237,52 @@ closing the two partial encodings — not a rewrite. - Exit criterion: a CI job that fails if a real MinIO-written object or bucket blob cannot be read. +#### Phase 1 status — first fixtures landed (verified 2026-07-07) + +A real MinIO `RELEASE.2025-07-23` single-drive instance wrote a bucket with +versioning, object-lock (GOVERNANCE default), lifecycle, tagging, quota, and a +public-download policy, plus inline / versioned / multipart objects. The on-disk +`xl.meta` blobs are captured as hex fixtures +(`crates/filemeta/tests/fixtures/minio/`, `crates/ecstore/tests/fixtures/minio/`). + +Proven by regression tests: + +- **Object `xl.meta` read parity** — `parses_real_minio_object_xlmeta` + (`crates/filemeta/src/filemeta.rs`): small inline, two-object-version + delete + marker, and multipart objects all parse to the expected `FileInfo`. +- **Bucket-metadata parse parity** — `parses_real_minio_bucket_metadata_blob_without_loss` + (`crates/ecstore/src/bucket/metadata.rs`): the msgpack blob decodes via the + PascalCase MinIO field names, and `parse_all_configs` loads **all ten** config + types present in the corpus without loss — policy, lifecycle (**including + MinIO's `` extension**), object-lock, versioning, tagging, + quota, notification, encryption (SSE-S3), and replication (**including the + `DeleteMarkerReplication` / `ExistingObjectReplication` MinIO extensions**). +- **Inline bucket-metadata read parity** — `reads_minio_inline_bucket_metadata_via_bitrot` + (`crates/ecstore/src/bucket/metadata.rs`): MinIO stores an inlined object body + as `[HighwayHash256 (32B)][body]`. The "`inline_data` 前缀不同" that weisd + raised on 2026-03-06 is exactly that bitrot prefix — **not** a format + incompatibility. Feeding the raw inline shard through RustFS's `BitrotReader` + with the default `HighwayHash256S` verifies the checksum (confirming RustFS's + hash matches MinIO's) and yields the exact `.metadata.bin` blob, which then + parses. So the object-layer inline read is compatible; the earlier "extract + `fi.data` directly" concern was reading the shard before the bitrot layer + strips its prefix. +- **End-to-end migration** — `migrates_real_minio_bucket_metadata_end_to_end` + (`crates/ecstore/src/bucket/migration.rs`): on a throwaway 4-drive local + `ECStore`, a real MinIO `.metadata.bin` seeded under a `.minio.sys` layout is + migrated by `try_migrate_bucket_metadata` into `.rustfs.sys`, and the migrated + blob carries every config (policy / lifecycle / object-lock / versioning / + tagging / quota / notification / encryption / replication) byte-identical to + the source. This exercises the Phase 2 source adapter + (`MIGRATING_META_BUCKET = ".minio.sys"`) end-to-end through the object layer — + proven, not just present. + +Still to broaden: transitioned `xl.meta`; CORS, public-access-block, and bucket +ACL configs (the SNSD test binary/`mc` did not expose these); and bucket-targets +credentials, which MinIO stores KMS-encrypted (a documented partial). These run +as ordinary crate tests, so they already execute in the normal `cargo +test`/nextest CI jobs. + ### Phase 2 — MinIO source adapter for migration - Generalize the importer in `crates/ecstore/src/bucket/migration.rs` so the @@ -256,14 +302,26 @@ closing the two partial encodings — not a rewrite. scope; if not, keep the blob round-trippable but document the enforcement limit (already reflected in the router compatibility matrix). -### Phase 4 — Round-trip / write-back parity (stretch) +### Phase 4 — Round-trip / write-back parity (non-goal for migration) -- Prove a MinIO binary can re-read a RustFS-written `xl.meta` and - `.metadata.bin` (the reverse direction). This is a stretch goal because - RustFS writes meta_ver 3 and MinIO's acceptance of specific v3 features must - be validated per feature. -- Exit criterion: a MinIO instance mounted on a RustFS-written drive set serves - objects and bucket configs without repair. +Proving a MinIO binary can re-read a *RustFS-written drive set* (the reverse +direction) is **out of scope for the migration use case**, which is one-way +MinIO → RustFS: + +- RustFS's meta bucket is `.rustfs.sys` (`crates/ecstore/src/disk/mod.rs:29`); + MinIO looks for `.minio.sys`. A MinIO binary pointed at a RustFS drive set + does not find `format.json` or bucket configs and refuses the set — this is a + set-level divergence, not an object-format one. +- The object-level `xl.meta` format *does* match (proven above), so the reverse + direction is limited by drive-set discovery, not by per-object encoding. +- The supported flow is one-way: `try_migrate_bucket_metadata` / + `try_migrate_iam_config` / `format.json` migration import a MinIO layout into + RustFS. There is no requirement to keep a live MinIO able to serve + RustFS-written drives. + +If a true bidirectional round-trip is ever needed, it would require RustFS to +optionally write the `.minio.sys` set layout — a separate feature, not part of +the interop/migration story tracked here. ---