mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
test(interop): add real MinIO read and migration parity tests (#4377)
test(interop): real-MinIO read + migration parity, Phase 1/2 (backlog#580) Capture authentic on-disk fixtures from MinIO RELEASE.2025-07-23 (a bucket with versioning, object-lock, lifecycle, tagging, quota, a public policy, SSE-S3 encryption, a webhook notification target, and a replication rule, plus inline / versioned / multipart objects and a delete marker) and prove RustFS reads and migrates them losslessly: - filemeta parses_real_minio_object_xlmeta: small inline, two-object-version + delete marker, and multipart object xl.meta parse to the expected FileInfo. - ecstore parses_real_minio_bucket_metadata_blob_without_loss: the MinIO .metadata.bin msgpack decodes via the PascalCase field names and parse_all_configs loads all ten config types present (policy, lifecycle incl. <ExpiryUpdatedAt>, object-lock, versioning, tagging, quota, notification, encryption/SSE-S3, replication incl. DeleteMarkerReplication / ExistingObjectReplication) without loss. - ecstore reads_minio_inline_bucket_metadata_via_bitrot: MinIO inlines an object body as [HighwayHash256 32B][body]; RustFS's BitrotReader with HighwayHash256S verifies and yields the exact blob (the "inline_data 前缀不同" is that prefix). - ecstore migrates_real_minio_bucket_metadata_end_to_end: on a throwaway 4-drive local ECStore, a real MinIO .metadata.bin seeded under .minio.sys is migrated into .rustfs.sys byte-identically for every config, exercising the Phase 2 source adapter (MIGRATING_META_BUCKET = ".minio.sys") through the object layer. All four run as ordinary crate tests (nextest CI). Phase 4 (MinIO re-reading a RustFS drive) is documented as out of scope for one-way migration. Refs rustfs/backlog#580
This commit is contained in:
Zhengchao An
@@ -1100,6 +1100,98 @@ where
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
/// Decode a whitespace-tolerant hex fixture into bytes.
|
||||
fn decode_hex(s: &str) -> Vec<u8> {
|
||||
let s: String = s.chars().filter(|c| !c.is_whitespace()).collect();
|
||||
(0..s.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("valid hex fixture"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// backlog#580: prove RustFS parses a real MinIO-written bucket `.metadata.bin`
|
||||
/// blob without loss. The fixture is the raw `.metadata.bin` object body
|
||||
/// (4-byte `format|version` header + msgpack) carved from the
|
||||
/// `.minio.sys/buckets/interop/.metadata.bin` object written by MinIO
|
||||
/// `RELEASE.2025-07-23` — see `tests/fixtures/minio/README.md`.
|
||||
#[test]
|
||||
fn parses_real_minio_bucket_metadata_blob_without_loss() {
|
||||
let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex"));
|
||||
|
||||
// Same 4-byte format|version header (1|1) and msgpack layout as MinIO.
|
||||
BucketMetadata::check_header(&blob).expect("valid .metadata.bin header");
|
||||
let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
|
||||
|
||||
// Raw config fields survive the msgpack decode (PascalCase MinIO field names).
|
||||
assert_eq!(bm.name, "interop");
|
||||
assert!(!bm.policy_config_json.is_empty(), "policy JSON present");
|
||||
assert!(!bm.lifecycle_config_xml.is_empty(), "lifecycle XML present");
|
||||
assert!(!bm.object_lock_config_xml.is_empty(), "object-lock XML present");
|
||||
assert!(!bm.versioning_config_xml.is_empty(), "versioning XML present");
|
||||
assert!(!bm.tagging_config_xml.is_empty(), "tagging XML present");
|
||||
assert!(!bm.quota_config_json.is_empty(), "quota JSON present");
|
||||
|
||||
// Typed parse of each stored config must succeed. `parse_all_configs`
|
||||
// logs+skips on error, so a None here means a real MinIO-compat parse gap.
|
||||
bm.parse_all_configs().expect("parse_all_configs");
|
||||
assert!(bm.policy_config.is_some(), "policy parsed");
|
||||
assert!(bm.versioning_config.is_some(), "versioning parsed");
|
||||
assert!(bm.object_lock_config.is_some(), "object-lock parsed");
|
||||
assert!(bm.tagging_config.is_some(), "tagging parsed");
|
||||
assert!(bm.quota_config.is_some(), "quota parsed");
|
||||
assert!(
|
||||
bm.lifecycle_config.is_some(),
|
||||
"lifecycle parsed (MinIO writes an <ExpiryUpdatedAt> extension element)"
|
||||
);
|
||||
assert!(bm.notification_config.is_some(), "notification parsed");
|
||||
assert!(bm.sse_config.is_some(), "encryption (SSE) parsed");
|
||||
assert!(bm.replication_config.is_some(), "replication parsed");
|
||||
|
||||
// Object lock is expressed through the parsed config, not the legacy
|
||||
// `LockEnabled` flag (MinIO leaves that false for config-based locks).
|
||||
assert!(bm.object_locking(), "object lock active via parsed config");
|
||||
}
|
||||
|
||||
/// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's
|
||||
/// inline-data extraction does not yet recover the object body from a
|
||||
/// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data`
|
||||
/// returns bytes that are not the `.metadata.bin` blob (no `format|version`
|
||||
/// header). Kept as an ignored, documented reproduction until the MinIO
|
||||
/// inline-data framing is handled on the read path.
|
||||
/// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata
|
||||
/// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]`
|
||||
/// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that
|
||||
/// bitrot prefix, not a format incompatibility). Running the raw inline shard
|
||||
/// through RustFS's `BitrotReader` with the default `HighwayHash256S` must
|
||||
/// verify the checksum and yield the exact `.metadata.bin` blob.
|
||||
#[tokio::test]
|
||||
async fn reads_minio_inline_bucket_metadata_via_bitrot() {
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
|
||||
let xlmeta = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata_full.xlmeta.hex"));
|
||||
let fm = rustfs_filemeta::FileMeta::load(&xlmeta).expect("parse MinIO xl.meta");
|
||||
let fi = fm
|
||||
.into_fileinfo("interop", ".metadata.bin", "", true, false, false)
|
||||
.expect("into_fileinfo");
|
||||
// The raw inline shard is `[HighwayHash256 (32B)][object body]`.
|
||||
let inline = fi.data.expect("inline shard present");
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let body_len = inline.len() - algo.size();
|
||||
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(inline.to_vec()), body_len, algo, false);
|
||||
let mut body = vec![0u8; body_len];
|
||||
let read = reader.read(&mut body).await.expect("bitrot verify + read MinIO inline shard");
|
||||
assert_eq!(read, body_len);
|
||||
|
||||
// The verified body is exactly the `.metadata.bin` blob, and it parses.
|
||||
BucketMetadata::check_header(&body).expect("recovered body is a valid .metadata.bin");
|
||||
let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
|
||||
assert_eq!(bm.name, "interop");
|
||||
bm.parse_all_configs().expect("parse recovered configs");
|
||||
assert!(bm.lifecycle_config.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn marshal_msg() {
|
||||
// write_time(OffsetDateTime::UNIX_EPOCH).unwrap();
|
||||
|
||||
@@ -517,4 +517,122 @@ mod tests {
|
||||
assert_eq!(decoded.id, 123);
|
||||
assert_eq!(decoded.targets_map["arn:replication::1:dest"].resync_id, "reset-1");
|
||||
}
|
||||
|
||||
fn decode_hex(s: &str) -> Vec<u8> {
|
||||
let s: String = s.chars().filter(|c| !c.is_whitespace()).collect();
|
||||
(0..s.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("valid hex fixture"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// backlog#580 Phase 2/4: end-to-end proof that `try_migrate_bucket_metadata`
|
||||
/// pulls a real MinIO-written bucket-metadata blob from a `.minio.sys` layout
|
||||
/// into `.rustfs.sys`, and that the migrated blob loads every config. Uses a
|
||||
/// throwaway 4-drive local ECStore. Fixture: `tests/fixtures/minio/README.md`.
|
||||
#[tokio::test]
|
||||
async fn migrates_real_minio_bucket_metadata_end_to_end() {
|
||||
use crate::bucket::metadata::{BUCKET_METADATA_FILE, BucketMetadata};
|
||||
use crate::config::com::read_config;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, BucketOptions, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations};
|
||||
use crate::store::{ECStore, init_local_disks};
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex"));
|
||||
|
||||
// --- Stand up a throwaway 4-drive local ECStore. ---
|
||||
let base = std::path::PathBuf::from(format!("/tmp/rustfs_minio_migrate_test_{}", Uuid::new_v4()));
|
||||
let disk_paths: Vec<_> = (1..=4).map(|i| base.join(format!("disk{i}"))).collect();
|
||||
for p in &disk_paths {
|
||||
fs::create_dir_all(p).await.unwrap();
|
||||
}
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, p) in disk_paths.iter().enumerate() {
|
||||
let mut ep = Endpoint::try_from(p.to_str().unwrap()).unwrap();
|
||||
ep.set_pool_index(0);
|
||||
ep.set_set_index(0);
|
||||
ep.set_disk_index(i);
|
||||
endpoints.push(ep);
|
||||
}
|
||||
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "minio-migrate-test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
}]);
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
let ecstore = ECStore::new("127.0.0.1:0".parse().unwrap(), endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
let existing: Vec<String> = ecstore
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|b| b.name)
|
||||
.collect();
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), existing).await;
|
||||
|
||||
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}interop{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
|
||||
let put_opts = ObjectOptions::default();
|
||||
|
||||
// --- Arrange the pre-migration state a MinIO import starts from: the ---
|
||||
// bucket exists and its config lives under `.minio.sys`, while `.rustfs.sys`
|
||||
// has no metadata for it yet.
|
||||
ecstore.make_bucket("interop", &MakeBucketOptions::default()).await.unwrap();
|
||||
let _ = ecstore
|
||||
.delete_object(RUSTFS_META_BUCKET, &meta_path, ObjectOptions::default())
|
||||
.await;
|
||||
// MinIO leaves its `.minio.sys` meta volume on every drive; recreate it so
|
||||
// the source object can be seeded through the object layer.
|
||||
for p in &disk_paths {
|
||||
fs::create_dir_all(p.join(MIGRATING_META_BUCKET)).await.ok();
|
||||
}
|
||||
let mut src = PutObjReader::from_vec(blob.clone());
|
||||
ecstore
|
||||
.put_object(MIGRATING_META_BUCKET, &meta_path, &mut src, &put_opts)
|
||||
.await
|
||||
.expect("seed .minio.sys bucket metadata");
|
||||
|
||||
// --- Run the real startup migration. ---
|
||||
super::try_migrate_bucket_metadata(ecstore.clone()).await;
|
||||
|
||||
// --- The migrated `.rustfs.sys` blob must carry every MinIO config, ---
|
||||
// byte-identical to the source (typed XML/JSON parsing of these fields is
|
||||
// covered by the bucket-metadata parse-parity test).
|
||||
let migrated = read_config(ecstore.clone(), &meta_path)
|
||||
.await
|
||||
.expect("read migrated bucket metadata");
|
||||
BucketMetadata::check_header(&migrated).expect("migrated blob has a valid header");
|
||||
let bm = BucketMetadata::unmarshal(&migrated[4..]).expect("unmarshal migrated bucket metadata");
|
||||
let src = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal source bucket metadata");
|
||||
|
||||
assert_eq!(bm.name, "interop");
|
||||
assert_eq!(bm.policy_config_json, src.policy_config_json, "policy migrated intact");
|
||||
assert_eq!(bm.lifecycle_config_xml, src.lifecycle_config_xml, "lifecycle migrated intact");
|
||||
assert_eq!(bm.object_lock_config_xml, src.object_lock_config_xml, "object-lock migrated intact");
|
||||
assert_eq!(bm.versioning_config_xml, src.versioning_config_xml, "versioning migrated intact");
|
||||
assert_eq!(bm.tagging_config_xml, src.tagging_config_xml, "tagging migrated intact");
|
||||
assert_eq!(bm.quota_config_json, src.quota_config_json, "quota migrated intact");
|
||||
assert_eq!(bm.notification_config_xml, src.notification_config_xml, "notification migrated intact");
|
||||
assert_eq!(bm.encryption_config_xml, src.encryption_config_xml, "encryption migrated intact");
|
||||
assert_eq!(bm.replication_config_xml, src.replication_config_xml, "replication migrated intact");
|
||||
assert!(!bm.lifecycle_config_xml.is_empty(), "lifecycle present");
|
||||
assert!(!bm.replication_config_xml.is_empty(), "replication present");
|
||||
|
||||
fs::remove_dir_all(&base).await.ok();
|
||||
}
|
||||
}
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# Real MinIO on-disk fixtures (backlog#580 interop)
|
||||
|
||||
Generated by MinIO `RELEASE.2025-07-23T15-54-02Z` (single-drive, local SSE-S3
|
||||
KMS + webhook notify target enabled) writing a bucket configured with
|
||||
**versioning, object-lock (GOVERNANCE default), lifecycle, tagging, quota, a
|
||||
public-download policy, SSE-S3 encryption, a webhook notification target, and a
|
||||
replication rule** (which also seeds a bucket-targets blob), plus inline /
|
||||
versioned / multipart objects and a delete marker.
|
||||
|
||||
- `bucket_metadata_full.xlmeta.hex` — the `.minio.sys/buckets/interop/.metadata.bin`
|
||||
object (xl.meta wrapper + inline `.metadata.bin` blob).
|
||||
- `bucket_metadata.blob.hex` — the raw `.metadata.bin` object body (4-byte
|
||||
`format|version` header + msgpack), carved from the object above.
|
||||
|
||||
Unencrypted object `xl.meta` fixtures (clean plaintext sizes) live under
|
||||
`crates/filemeta/tests/fixtures/minio/`.
|
||||
|
||||
These prove RustFS reads and migrates MinIO-written metadata without loss.
|
||||
Not captured here (tooling/scope): CORS, public-access-block, and bucket ACL
|
||||
(the SNSD test binary + `mc` did not expose CORS; bucket-targets credentials are
|
||||
stored KMS-encrypted and are a documented partial).
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -920,6 +920,52 @@ mod test {
|
||||
use proptest::collection::vec;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// backlog#580: RustFS parses real MinIO-written object xl.meta (inline,
|
||||
/// versioned, and multipart) into equivalent `FileInfo`. Object metadata is
|
||||
/// the strong part of MinIO interop; this pins it against real fixtures.
|
||||
#[test]
|
||||
fn parses_real_minio_object_xlmeta() {
|
||||
// Small inlined object.
|
||||
let small = create_minio_small_object_xlmeta().expect("load small fixture");
|
||||
let (major, _minor, _hdr, meta_ver) = FileMeta::read_format_versions(&small).unwrap();
|
||||
assert_eq!(major, 1);
|
||||
assert_eq!(meta_ver, FileMeta::load(&small).unwrap().meta_ver);
|
||||
let fm = FileMeta::load(&small).expect("parse small MinIO xl.meta");
|
||||
assert_eq!(fm.versions.len(), 1);
|
||||
let fi = fm
|
||||
.into_fileinfo("interop", "small.txt", "", false, false, true)
|
||||
.expect("small fileinfo");
|
||||
assert_eq!(fi.size, 19, "small.txt size");
|
||||
assert_eq!(fi.num_versions, 1);
|
||||
assert!(fi.is_latest);
|
||||
|
||||
// Versioned object: two object versions plus a delete marker (latest).
|
||||
let versioned = create_minio_versioned_object_xlmeta().expect("load versioned fixture");
|
||||
let fm = FileMeta::load(&versioned).expect("parse versioned MinIO xl.meta");
|
||||
assert_eq!(fm.versions.len(), 3, "two object versions + one delete marker");
|
||||
let delete_markers = fm
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|v| v.header.version_type == VersionType::Delete)
|
||||
.count();
|
||||
assert_eq!(delete_markers, 1, "delete marker parsed from MinIO xl.meta");
|
||||
let fi = fm
|
||||
.into_fileinfo("interop", "versioned.txt", "", false, false, true)
|
||||
.expect("versioned fileinfo");
|
||||
assert_eq!(fi.num_versions, 3);
|
||||
assert!(fi.version_id.is_some(), "versioned object carries a version id");
|
||||
|
||||
// Larger object stored as an erasure-coded part (not inlined).
|
||||
let large = create_minio_large_object_xlmeta().expect("load large fixture");
|
||||
let fm = FileMeta::load(&large).expect("parse large MinIO xl.meta");
|
||||
assert_eq!(fm.versions.len(), 1);
|
||||
let fi = fm
|
||||
.into_fileinfo("interop", "large.bin", "", false, false, true)
|
||||
.expect("large fileinfo");
|
||||
assert_eq!(fi.size, 300_000, "large.bin size");
|
||||
assert!(!fi.parts.is_empty(), "multipart/part layout present");
|
||||
}
|
||||
|
||||
/// Wraps a raw meta block in a valid XL2 container (header, bin32 length
|
||||
/// prefix, and CRC trailer) so decode tests exercise the meta parsing
|
||||
/// itself rather than the envelope checks.
|
||||
|
||||
@@ -151,6 +151,26 @@ pub fn create_issue_2434_legacy_meta_v2_pool_xlmeta() -> Result<Vec<u8>> {
|
||||
decode_hex_fixture(include_str!("../tests/fixtures/issue_2434_legacy_meta_v2_pool.hex"))
|
||||
}
|
||||
|
||||
/// Real MinIO-written xl.meta for a small inlined object (`small.txt`, 19 bytes),
|
||||
/// captured from MinIO `RELEASE.2025-07-23` — see `tests/fixtures/minio/README.md`
|
||||
/// (backlog#580).
|
||||
pub fn create_minio_small_object_xlmeta() -> Result<Vec<u8>> {
|
||||
decode_hex_fixture(include_str!("../tests/fixtures/minio/object_small_txt.xlmeta.hex"))
|
||||
}
|
||||
|
||||
/// Real MinIO-written xl.meta for a versioned object (`versioned.txt`, two
|
||||
/// versions) — see `tests/fixtures/minio/README.md` (backlog#580).
|
||||
pub fn create_minio_versioned_object_xlmeta() -> Result<Vec<u8>> {
|
||||
decode_hex_fixture(include_str!("../tests/fixtures/minio/object_versioned_txt.xlmeta.hex"))
|
||||
}
|
||||
|
||||
/// Real MinIO-written xl.meta for a larger, non-inlined object (`large.bin`,
|
||||
/// 300 KiB, stored as `part.1`) — see `tests/fixtures/minio/README.md`
|
||||
/// (backlog#580).
|
||||
pub fn create_minio_large_object_xlmeta() -> Result<Vec<u8>> {
|
||||
decode_hex_fixture(include_str!("../tests/fixtures/minio/object_large_bin.xlmeta.hex"))
|
||||
}
|
||||
|
||||
fn write_legacy_time(wr: &mut Vec<u8>, ts: OffsetDateTime) {
|
||||
wr.push(MSGPACK_EXT8);
|
||||
wr.push(12);
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
584c322001000300c6000001a4030301c42697c410727d23b40d2944e98e80d34405582863d318c00d6f34711ea0c404ad77932901020001c5017683a45479706501a556324f626ade0011a24944c410727d23b40d2944e98e80d34405582863a444446972c41059f847a8f7844db9b905a8352a9906bfa64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a657391d2000493e0aa506172744153697a657391d2000493e0a453697a65d2000493e0a54d54696d65d318c00d6f34711ea0a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c406333030303030b4782d6d696e696f2d696e7465726e616c2d637263c40bd001bbcd55bc019325344da74d65746155737282a465746167d92231326365653861623034393337646333333236666364326163633538626265352d31ac636f6e74656e742d74797065b86170706c69636174696f6e2f6f637465742d73747265616da176ce6881059aceb9aa266a
|
||||
@@ -0,0 +1 @@
|
||||
584c322001000300c600000186030301c42697c410746dde29b59243efac7e37d408079143d318c00d6f26446968c4047761ed8301020001c5015883a45479706501a556324f626ade0011a24944c410746dde29b59243efac7e37d408079143a444446972c41019286b7785854468b7e9a8efddf329d7a64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a65739113aa506172744153697a65739113a453697a6513a54d54696d65d318c00d6f26446968a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c4023139b4782d6d696e696f2d696e7465726e616c2d637263c40bd0012cd2c3a001f50b8e4ba74d65746155737282ac636f6e74656e742d74797065aa746578742f706c61696ea465746167d92261396238633039663133663333663537306430326264346462316662356439372d31a176ce6881059ace755b0e49
|
||||
@@ -0,0 +1 @@
|
||||
584c322001000300c600000375030303c42697c4106178c538372e4d93a122e55fc1bed128d318c00d6f3a44dd58c40439c29d5902000000c44383a45479706502a644656c4f626a83a24944c4106178c538372e4d93a122e55fc1bed128a54d54696d65d318c00d6f3a44dd58a74d65746153797380a176ce6881059ac42697c410065f0d6b0c1b4f94bfe1e98e8235fcf1d318c00d6f2fe16138c4049e93c2e701020001c5015883a45479706501a556324f626ade0011a24944c410065f0d6b0c1b4f94bfe1e98e8235fcf1a444446972c4108b76b5d6063245b6a4da67a01789d62ca64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a6573910caa506172744153697a6573910ca453697a650ca54d54696d65d318c00d6f2fe16138a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c4023132b4782d6d696e696f2d696e7465726e616c2d637263c40bd0014d39f0a401ff13e583a74d65746155737282ac636f6e74656e742d74797065aa746578742f706c61696ea465746167d92263363239643133613630313832303136356237333638373331313430613738632d31a176ce6881059ac42697c410b12dfda0210b405b89b1eaafd1964008d318c00d6f2b5d4848c404fc20d11a01020001c5015783a45479706501a556324f626ade0011a24944c410b12dfda0210b405b89b1eaafd1964008a444446972c41001dd843548214e4abbdfc472b35ec475a64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a65739102aa506172744153697a65739102a453697a6502a54d54696d65d318c00d6f2b5d4848a74d65746153797382bc582d4d696e696f2d496e7465726e616c2d61637475616c2d73697a65c40132b4782d6d696e696f2d696e7465726e616c2d637263c40bd001e1792ab801e57f38d5a74d65746155737282ac636f6e74656e742d74797065aa746578742f706c61696ea465746167d92239306262653039353930353265626361656630623038323733666235313063372d31a176ce6881059ace638dab49
|
||||
@@ -237,6 +237,52 @@ closing the two partial encodings — not a rewrite.
|
||||
- Exit criterion: a CI job that fails if a real MinIO-written object or bucket
|
||||
blob cannot be read.
|
||||
|
||||
#### Phase 1 status — first fixtures landed (verified 2026-07-07)
|
||||
|
||||
A real MinIO `RELEASE.2025-07-23` single-drive instance wrote a bucket with
|
||||
versioning, object-lock (GOVERNANCE default), lifecycle, tagging, quota, and a
|
||||
public-download policy, plus inline / versioned / multipart objects. The on-disk
|
||||
`xl.meta` blobs are captured as hex fixtures
|
||||
(`crates/filemeta/tests/fixtures/minio/`, `crates/ecstore/tests/fixtures/minio/`).
|
||||
|
||||
Proven by regression tests:
|
||||
|
||||
- **Object `xl.meta` read parity** — `parses_real_minio_object_xlmeta`
|
||||
(`crates/filemeta/src/filemeta.rs`): small inline, two-object-version + delete
|
||||
marker, and multipart objects all parse to the expected `FileInfo`.
|
||||
- **Bucket-metadata parse parity** — `parses_real_minio_bucket_metadata_blob_without_loss`
|
||||
(`crates/ecstore/src/bucket/metadata.rs`): the msgpack blob decodes via the
|
||||
PascalCase MinIO field names, and `parse_all_configs` loads **all ten** config
|
||||
types present in the corpus without loss — policy, lifecycle (**including
|
||||
MinIO's `<ExpiryUpdatedAt>` extension**), object-lock, versioning, tagging,
|
||||
quota, notification, encryption (SSE-S3), and replication (**including the
|
||||
`DeleteMarkerReplication` / `ExistingObjectReplication` MinIO extensions**).
|
||||
- **Inline bucket-metadata read parity** — `reads_minio_inline_bucket_metadata_via_bitrot`
|
||||
(`crates/ecstore/src/bucket/metadata.rs`): MinIO stores an inlined object body
|
||||
as `[HighwayHash256 (32B)][body]`. The "`inline_data` 前缀不同" that weisd
|
||||
raised on 2026-03-06 is exactly that bitrot prefix — **not** a format
|
||||
incompatibility. Feeding the raw inline shard through RustFS's `BitrotReader`
|
||||
with the default `HighwayHash256S` verifies the checksum (confirming RustFS's
|
||||
hash matches MinIO's) and yields the exact `.metadata.bin` blob, which then
|
||||
parses. So the object-layer inline read is compatible; the earlier "extract
|
||||
`fi.data` directly" concern was reading the shard before the bitrot layer
|
||||
strips its prefix.
|
||||
- **End-to-end migration** — `migrates_real_minio_bucket_metadata_end_to_end`
|
||||
(`crates/ecstore/src/bucket/migration.rs`): on a throwaway 4-drive local
|
||||
`ECStore`, a real MinIO `.metadata.bin` seeded under a `.minio.sys` layout is
|
||||
migrated by `try_migrate_bucket_metadata` into `.rustfs.sys`, and the migrated
|
||||
blob carries every config (policy / lifecycle / object-lock / versioning /
|
||||
tagging / quota / notification / encryption / replication) byte-identical to
|
||||
the source. This exercises the Phase 2 source adapter
|
||||
(`MIGRATING_META_BUCKET = ".minio.sys"`) end-to-end through the object layer —
|
||||
proven, not just present.
|
||||
|
||||
Still to broaden: transitioned `xl.meta`; CORS, public-access-block, and bucket
|
||||
ACL configs (the SNSD test binary/`mc` did not expose these); and bucket-targets
|
||||
credentials, which MinIO stores KMS-encrypted (a documented partial). These run
|
||||
as ordinary crate tests, so they already execute in the normal `cargo
|
||||
test`/nextest CI jobs.
|
||||
|
||||
### Phase 2 — MinIO source adapter for migration
|
||||
|
||||
- Generalize the importer in `crates/ecstore/src/bucket/migration.rs` so the
|
||||
@@ -256,14 +302,26 @@ closing the two partial encodings — not a rewrite.
|
||||
scope; if not, keep the blob round-trippable but document the enforcement
|
||||
limit (already reflected in the router compatibility matrix).
|
||||
|
||||
### Phase 4 — Round-trip / write-back parity (stretch)
|
||||
### Phase 4 — Round-trip / write-back parity (non-goal for migration)
|
||||
|
||||
- Prove a MinIO binary can re-read a RustFS-written `xl.meta` and
|
||||
`.metadata.bin` (the reverse direction). This is a stretch goal because
|
||||
RustFS writes meta_ver 3 and MinIO's acceptance of specific v3 features must
|
||||
be validated per feature.
|
||||
- Exit criterion: a MinIO instance mounted on a RustFS-written drive set serves
|
||||
objects and bucket configs without repair.
|
||||
Proving a MinIO binary can re-read a *RustFS-written drive set* (the reverse
|
||||
direction) is **out of scope for the migration use case**, which is one-way
|
||||
MinIO → RustFS:
|
||||
|
||||
- RustFS's meta bucket is `.rustfs.sys` (`crates/ecstore/src/disk/mod.rs:29`);
|
||||
MinIO looks for `.minio.sys`. A MinIO binary pointed at a RustFS drive set
|
||||
does not find `format.json` or bucket configs and refuses the set — this is a
|
||||
set-level divergence, not an object-format one.
|
||||
- The object-level `xl.meta` format *does* match (proven above), so the reverse
|
||||
direction is limited by drive-set discovery, not by per-object encoding.
|
||||
- The supported flow is one-way: `try_migrate_bucket_metadata` /
|
||||
`try_migrate_iam_config` / `format.json` migration import a MinIO layout into
|
||||
RustFS. There is no requirement to keep a live MinIO able to serve
|
||||
RustFS-written drives.
|
||||
|
||||
If a true bidirectional round-trip is ever needed, it would require RustFS to
|
||||
optionally write the `.minio.sys` set layout — a separate feature, not part of
|
||||
the interop/migration story tracked here.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user