// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Integration test: verify create_bitrot_reader with HighwayHash256SLegacy reads main-version objects. //! //! Supports both EC (part files) and inline objects. Reads xl.meta to compute params. //! Uses create_bitrot_reader only (no ECStore/get_object_reader) to isolate bitrot compatibility. //! //! Run from workspace root: //! cargo test -p rustfs-ecstore test_legacy_bitrot_read -- --nocapture //! //! Environment: //! RUSTFS_LEGACY_TEST_ROOT - Test data root (default: workspace root) //! RUSTFS_LEGACY_TEST_DISK - Disk name under root (default: "test1" for rustfs, "test" for minio) //! RUSTFS_SKIP_LEGACY_TEST - Set to 1 to skip //! //! For MinIO data: RUSTFS_LEGACY_TEST_ROOT=/path/to/minio (disk "test" and .minio.sys auto-detected). mod storage_api; use rustfs_filemeta::{FileInfoOpts, get_file_info}; use rustfs_utils::HashAlgorithm; use std::path::{Path, PathBuf}; use storage_api::legacy_bitrot_read::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, create_bitrot_reader, new_disk}; use tokio::fs; fn workspace_root() -> PathBuf { let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into()); PathBuf::from(&manifest) .ancestors() .nth(2) .unwrap_or_else(|| std::path::Path::new(".")) .to_path_buf() } fn legacy_test_root() -> PathBuf { std::env::var_os("RUSTFS_LEGACY_TEST_ROOT") .map(PathBuf::from) .unwrap_or_else(workspace_root) } fn legacy_test_disk(root: &Path) -> String { if let Some(disk) = std::env::var_os("RUSTFS_LEGACY_TEST_DISK") { return disk.to_string_lossy().into_owned(); } if root.join("test/.minio.sys/format.json").exists() { "test".to_string() } else { "test1".to_string() } } fn legacy_format_exists(root: &Path, disk_name: &str) -> bool { let disk_root = root.join(disk_name); disk_root.join(".rustfs.sys/format.json").exists() || disk_root.join(".minio.sys/format.json").exists() } fn legacy_object_meta_exists(root: &Path, disk_name: &str, bucket: &str, object: &str) -> bool { root.join(disk_name) .join(bucket) .join(object) .join(STORAGE_FORMAT_FILE) .exists() } fn legacy_test_data_exists() -> bool { if std::env::var("RUSTFS_SKIP_LEGACY_TEST").unwrap_or_default() == "1" { return false; } let root = legacy_test_root(); let disk_name = legacy_test_disk(&root); let bucket = "vvvv"; legacy_format_exists(&root, &disk_name) && (legacy_object_meta_exists(&root, &disk_name, bucket, "ktvzip.tar.gz") || legacy_object_meta_exists(&root, &disk_name, bucket, "path_traversal.md")) } async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &str, bucket: &str, object: &str) -> bool { let xl_meta_path = root.join(disk_name).join(bucket).join(object).join(STORAGE_FORMAT_FILE); if !xl_meta_path.exists() { eprintln!("xl_meta_path not found: {:?}", xl_meta_path); return false; } let buf = match fs::read(&xl_meta_path).await { Ok(b) => b, Err(_) => { eprintln!("Failed to read xl_meta_path: {:?}", xl_meta_path); return false; } }; let fi = match get_file_info( &buf, bucket, object, "", FileInfoOpts { data: true, // need inline data for inline objects include_free_versions: false, }, ) { Ok(f) => f, Err(_) => { eprintln!("Failed to get file info: {:?}", xl_meta_path); return false; } }; if fi.deleted || fi.parts.is_empty() { eprintln!("File is deleted or has no parts: {:?}", xl_meta_path); return false; } let shard_size = fi.erasure.shard_size(); let part_number = 1; let checksum_info = fi.erasure.get_checksum_info(part_number); let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { HashAlgorithm::HighwayHash256SLegacy } else { checksum_info.algorithm }; eprintln!("fi.inline_data(): {:?}", fi.inline_data()); eprintln!("fi.is_remote(): {:?}", fi.metadata); // Inline path: use fi.data if fi.inline_data() { let inline_bytes = match &fi.data { Some(b) if !b.is_empty() => b.as_ref(), _ => { eprintln!("Inline data is empty: {:?}", xl_meta_path); return false; } }; let read_length = fi.size as usize; if read_length == 0 { eprintln!("Read length is 0: {:?}", xl_meta_path); return false; } let mut reader = match create_bitrot_reader( Some(inline_bytes), None, bucket, "", 0, read_length, shard_size, checksum_algo.clone(), false, false, // use_zero_copy ) .await { Ok(Some(r)) => r, _ => { eprintln!("Failed to create bitrot reader for inline data: {:?}", xl_meta_path); return false; } }; let mut buf = vec![0u8; shard_size]; match reader.read(&mut buf).await { Ok(n) if n > 0 => { eprintln!("Successfully read {} bytes (inline) via create_bitrot_reader with {:?}", n, checksum_algo); return true; } _ => { eprintln!("Failed to read inline data: {:?}", xl_meta_path); return false; } } } // EC path: use disk + part file let data_dir = match fi.data_dir { Some(d) => d, None => { eprintln!("Data dir is empty: {:?}", xl_meta_path); return false; } }; let path = format!("{object}/{data_dir}/part.{}", part_number); let part_path = root.join(disk_name).join(bucket).join(&path); if !part_path.exists() { eprintln!("Part file not found: {:?}", part_path); return false; } let disk_path = root.join(disk_name); let path_str = disk_path.to_str().expect("path"); let mut ep = Endpoint::try_from(path_str).expect("endpoint"); ep.set_pool_index(0); ep.set_set_index(0); ep.set_disk_index(0); let opt = DiskOption { cleanup: false, health_check: false, }; let disk = match new_disk(&ep, &opt).await { Ok(d) => d, Err(_) => { eprintln!("Failed to create disk: {:?}", disk_path); return false; } }; let read_length = shard_size; let mut reader = match create_bitrot_reader( None, Some(&disk), bucket, &path, 0, read_length, shard_size, checksum_algo.clone(), false, false, ) // use_zero_copy .await { Ok(Some(r)) => r, _ => { eprintln!("Failed to create bitrot reader for EC part: {:?}", part_path); return false; } }; let mut buf = vec![0u8; shard_size]; match reader.read(&mut buf).await { Ok(n) if n > 0 => { eprintln!( "Successfully read {} bytes (EC part) via create_bitrot_reader with {:?}", n, checksum_algo ); true } _ => { eprintln!("Failed to read EC part: {:?}", part_path); false } } } #[tokio::test] async fn test_legacy_bitrot_read() { if !legacy_test_data_exists() { eprintln!("Skipping legacy bitrot test: legacy fixture xl.meta not found or RUSTFS_SKIP_LEGACY_TEST=1"); return; } let root = legacy_test_root(); let disk_name = legacy_test_disk(&root); let bucket = "vvvv"; let mut checked = 0usize; for object in ["ktvzip.tar.gz", "path_traversal.md"] { if !legacy_object_meta_exists(&root, &disk_name, bucket, object) { eprintln!("Skipping missing legacy object fixture: {object}"); continue; } checked += 1; let ok = run_legacy_bitrot_test_for_object(&root, &disk_name, bucket, object).await; assert!(ok, "create_bitrot_reader failed for {object}"); } assert!(checked > 0, "legacy bitrot fixture preflight found no readable objects"); }