mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 15:16:56 +00:00
test(ecstore): pin persisted metadata key literals and bucket config goldens (#5904)
This commit is contained in:
@@ -453,6 +453,11 @@ pub mod rpc {
|
||||
pub mod set_disk {
|
||||
pub use crate::set_disk::{DEFAULT_READ_BUFFER_SIZE, SetDisks, get_lock_acquire_timeout, is_valid_storage_class};
|
||||
|
||||
/// Return the canonical object-metadata identity used for read-quorum grouping.
|
||||
pub fn file_info_quorum_hash(meta: &rustfs_filemeta::FileInfo) -> [u8; 32] {
|
||||
crate::set_disk::SetDisks::file_info_quorum_hash(meta)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
@@ -200,6 +200,29 @@ mod tests {
|
||||
assert!(retention.retain_until_date.is_some());
|
||||
}
|
||||
|
||||
/// backlog#1733 g-key-002: the persisted literal keys must still be read
|
||||
/// through the current header constants, or WORM metadata fails open.
|
||||
#[test]
|
||||
fn persisted_compliance_lock_metadata_remains_effective() {
|
||||
let mut meta = HashMap::new();
|
||||
meta.insert("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string());
|
||||
meta.insert("x-amz-object-lock-retain-until-date".to_string(), "9999-01-01T00:00:00Z".to_string());
|
||||
meta.insert("x-amz-object-lock-legal-hold".to_string(), "ON".to_string());
|
||||
|
||||
let retention = get_object_retention_meta(&meta);
|
||||
assert_eq!(
|
||||
retention.mode.as_ref().map(|mode| mode.as_str()),
|
||||
Some(ObjectLockRetentionMode::COMPLIANCE)
|
||||
);
|
||||
assert!(retention.retain_until_date.is_some(), "persisted retention date must remain readable");
|
||||
|
||||
let legal_hold = get_object_legalhold_meta(&meta);
|
||||
assert_eq!(
|
||||
legal_hold.status.as_ref().map(|status| status.as_str()),
|
||||
Some(ObjectLockLegalHoldStatus::ON)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_object_legalhold_meta_empty() {
|
||||
let meta = HashMap::new();
|
||||
|
||||
@@ -580,7 +580,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
pub(crate) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
Self::update_file_info_quorum_hash(&mut hasher, meta);
|
||||
let digest = hasher.finalize();
|
||||
|
||||
@@ -1016,6 +1016,83 @@ mod test {
|
||||
use proptest::collection::vec;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// A restore header meaning "restored copy is on disk until far in the future".
|
||||
/// Format produced by `RestoreStatusOps::to_string` and consumed by
|
||||
/// `parse_restore_obj_status` (fileinfo.rs).
|
||||
const RESTORED_ON_DISK: &str = "ongoing-request=\"false\", expiry-date=\"9999-01-01T00:00:00Z\"";
|
||||
|
||||
/// backlog#1733 (P9-01 §4.3/§7.6, g-key-001): pin the five `s3s::header`
|
||||
/// constants that double as **persisted metadata map keys**. They are not
|
||||
/// just HTTP header names — they are stored inside xl.meta (`meta_user`)
|
||||
/// and read back by fail-open code, so a silent drift produces zero
|
||||
/// HTTP-visible errors while:
|
||||
///
|
||||
/// 1. **WORM silently dissolves** — `get_object_retention_meta`
|
||||
/// (ecstore objectlock.rs) returns an empty retention when the lock keys
|
||||
/// are unreadable, making every compliance-locked object deletable.
|
||||
/// 2. **Live data dirs can be reclaimed** — `MetaObject::uses_data_dir`
|
||||
/// falls back to `is_restored_object_on_disk`, which returns `false`
|
||||
/// when `x-amz-restore` is unreadable, so a restored object's data dir
|
||||
/// is judged unused.
|
||||
///
|
||||
/// Any migration replacing these constants must keep the literals byte-stable.
|
||||
#[test]
|
||||
fn persisted_metadata_keys_are_byte_stable() {
|
||||
use s3s::header::{
|
||||
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
|
||||
X_AMZ_SERVER_SIDE_ENCRYPTION,
|
||||
};
|
||||
assert_eq!(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(), "x-amz-object-lock-legal-hold");
|
||||
assert_eq!(X_AMZ_OBJECT_LOCK_MODE.as_str(), "x-amz-object-lock-mode");
|
||||
assert_eq!(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(), "x-amz-object-lock-retain-until-date");
|
||||
assert_eq!(X_AMZ_RESTORE.as_str(), "x-amz-restore");
|
||||
assert_eq!(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str(), "x-amz-server-side-encryption");
|
||||
}
|
||||
|
||||
/// backlog#1733 g-key-003: a restored-to-local object must keep its data
|
||||
/// dir. The restore marker lives under the pinned `x-amz-restore` key; if
|
||||
/// the key ever drifts this flips to `false` and the data dir becomes
|
||||
/// eligible for reclamation while the restored copy is still being served.
|
||||
#[test]
|
||||
fn restored_object_keeps_using_data_dir() {
|
||||
let mut obj = MetaObject::default();
|
||||
obj.meta_user
|
||||
.insert("x-amz-restore".to_string(), RESTORED_ON_DISK.to_string());
|
||||
assert!(obj.uses_data_dir(), "restored object's data dir must be considered in use");
|
||||
|
||||
// The same fail-open shape the pin protects against: without the marker
|
||||
// the data dir is judged unused — exactly what a key drift would cause.
|
||||
let bare = MetaObject::default();
|
||||
assert!(!bare.uses_data_dir(), "object without restore marker reports data dir unused");
|
||||
}
|
||||
|
||||
/// backlog#1733 g-key-004: a transition-complete object short-circuits to
|
||||
/// `false` even when the restore marker is present — the existing
|
||||
/// precedence must not change.
|
||||
#[test]
|
||||
fn transition_complete_object_does_not_use_data_dir() {
|
||||
use rustfs_utils::http::{SUFFIX_TRANSITION_STATUS, insert_bytes};
|
||||
let mut obj = MetaObject::default();
|
||||
obj.meta_user
|
||||
.insert("x-amz-restore".to_string(), RESTORED_ON_DISK.to_string());
|
||||
insert_bytes(&mut obj.meta_sys, SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.as_bytes().to_vec());
|
||||
assert!(!obj.uses_data_dir(), "transition-complete short-circuit must win over the restore marker");
|
||||
}
|
||||
|
||||
/// The restore-header parser and the pinned key literal must agree: the
|
||||
/// marker written under `x-amz-restore` is only meaningful if the parser
|
||||
/// accepts it.
|
||||
#[test]
|
||||
fn restore_marker_roundtrips_through_parser() {
|
||||
let mut meta = HashMap::new();
|
||||
meta.insert(X_AMZ_RESTORE.as_str().to_string(), RESTORED_ON_DISK.to_string());
|
||||
assert!(crate::is_restored_object_on_disk(&meta));
|
||||
|
||||
// An in-progress restore is not "on disk".
|
||||
meta.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"true\"".to_string());
|
||||
assert!(!crate::is_restored_object_on_disk(&meta));
|
||||
}
|
||||
|
||||
/// backlog#580: RustFS parses real MinIO-written object xl.meta (inline,
|
||||
/// versioned, and multipart) into equivalent `FileInfo`. Object metadata is
|
||||
/// the strong part of MinIO interop; this pins it against real fixtures.
|
||||
|
||||
@@ -1855,6 +1855,7 @@ impl From<MetaObjectV1ChecksumInfo> for ChecksumInfo {
|
||||
"highwayhash256" => HashAlgorithm::HighwayHash256,
|
||||
"highwayhash256S" => HashAlgorithm::HighwayHash256S,
|
||||
"blake2b" | "blake2b512" => HashAlgorithm::BLAKE2b512,
|
||||
"md5" => HashAlgorithm::Md5,
|
||||
_ => HashAlgorithm::HighwayHash256S,
|
||||
},
|
||||
hash: Bytes::from(value.hash),
|
||||
|
||||
@@ -189,7 +189,14 @@ fn encode_legacy_v1_header(version_id: Uuid, mod_time: OffsetDateTime) -> Vec<u8
|
||||
wr
|
||||
}
|
||||
|
||||
fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateTime) -> Vec<u8> {
|
||||
fn encode_legacy_v1_body(
|
||||
version_id: Uuid,
|
||||
data_dir: Uuid,
|
||||
mod_time: OffsetDateTime,
|
||||
erasure_index: usize,
|
||||
checksum: Option<(&str, &[u8])>,
|
||||
object_size: usize,
|
||||
) -> Vec<u8> {
|
||||
let mut wr = Vec::new();
|
||||
|
||||
rmp::encode::write_map_len(&mut wr, 3).unwrap();
|
||||
@@ -208,7 +215,7 @@ fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateT
|
||||
rmp::encode::write_str(&mut wr, "Stat").unwrap();
|
||||
rmp::encode::write_map_len(&mut wr, 5).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "Size").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 11).unwrap();
|
||||
rmp::encode::write_sint(&mut wr, object_size as i64).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "ModTime").unwrap();
|
||||
write_legacy_time(&mut wr, mod_time);
|
||||
rmp::encode::write_str(&mut wr, "Name").unwrap();
|
||||
@@ -229,14 +236,23 @@ fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateT
|
||||
rmp::encode::write_str(&mut wr, "BlockSize").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 1_048_576).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "Index").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 1).unwrap();
|
||||
rmp::encode::write_sint(&mut wr, erasure_index as i64).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "Distribution").unwrap();
|
||||
rmp::encode::write_array_len(&mut wr, 6).unwrap();
|
||||
for value in 1..=6 {
|
||||
rmp::encode::write_sint(&mut wr, value).unwrap();
|
||||
}
|
||||
rmp::encode::write_str(&mut wr, "Checksums").unwrap();
|
||||
rmp::encode::write_array_len(&mut wr, 0).unwrap();
|
||||
rmp::encode::write_array_len(&mut wr, u32::from(checksum.is_some())).unwrap();
|
||||
if let Some((algorithm, hash)) = checksum {
|
||||
rmp::encode::write_map_len(&mut wr, 3).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "PartNumber").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 1).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "Algorithm").unwrap();
|
||||
rmp::encode::write_str(&mut wr, algorithm).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "Hash").unwrap();
|
||||
rmp::encode::write_bin(&mut wr, hash).unwrap();
|
||||
}
|
||||
|
||||
rmp::encode::write_str(&mut wr, "Meta").unwrap();
|
||||
rmp::encode::write_map_len(&mut wr, 1).unwrap();
|
||||
@@ -251,9 +267,9 @@ fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateT
|
||||
rmp::encode::write_str(&mut wr, "n").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 1).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "s").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 11).unwrap();
|
||||
rmp::encode::write_sint(&mut wr, object_size as i64).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "as").unwrap();
|
||||
rmp::encode::write_sint(&mut wr, 11).unwrap();
|
||||
rmp::encode::write_sint(&mut wr, object_size as i64).unwrap();
|
||||
rmp::encode::write_str(&mut wr, "mt").unwrap();
|
||||
write_legacy_time(&mut wr, mod_time);
|
||||
|
||||
@@ -275,8 +291,29 @@ pub fn create_legacy_v1_object_xlmeta() -> Result<Vec<u8>> {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789)?;
|
||||
|
||||
let header = encode_legacy_v1_header(version_id, mod_time);
|
||||
let body = encode_legacy_v1_body(version_id, data_dir, mod_time);
|
||||
let body = encode_legacy_v1_body(version_id, data_dir, mod_time, 1, None, 11);
|
||||
|
||||
encode_legacy_v1_xlmeta(header, body)
|
||||
}
|
||||
|
||||
/// Legacy V1 xl.meta fixture with a per-drive whole-file bitrot checksum.
|
||||
pub fn create_legacy_v1_object_xlmeta_with_checksum(
|
||||
erasure_index: usize,
|
||||
algorithm: &str,
|
||||
hash: &[u8],
|
||||
object_size: usize,
|
||||
) -> Result<Vec<u8>> {
|
||||
let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?;
|
||||
let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?;
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789)?;
|
||||
|
||||
let header = encode_legacy_v1_header(version_id, mod_time);
|
||||
let body = encode_legacy_v1_body(version_id, data_dir, mod_time, erasure_index, Some((algorithm, hash)), object_size);
|
||||
|
||||
encode_legacy_v1_xlmeta(header, body)
|
||||
}
|
||||
|
||||
fn encode_legacy_v1_xlmeta(header: Vec<u8>, body: Vec<u8>) -> Result<Vec<u8>> {
|
||||
let mut wr = Vec::new();
|
||||
wr.extend_from_slice(b"XL2 ");
|
||||
wr.extend_from_slice(&1u16.to_le_bytes());
|
||||
|
||||
@@ -35,6 +35,10 @@ path = "src/lib.rs"
|
||||
name = "rustfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "rustfs-cli"
|
||||
path = "src/bin/rustfs-cli.rs"
|
||||
|
||||
[features]
|
||||
default = ["ftps", "webdav"]
|
||||
metrics-gpu = ["rustfs-obs/gpu"]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Offline and diagnostic RustFS command-line entry point.
|
||||
//!
|
||||
//! This binary shares RustFS's existing subcommand dispatcher and provides the
|
||||
//! documented entry point for offline tooling such as `inspect bucket-meta`.
|
||||
|
||||
fn main() {
|
||||
rustfs::startup_entrypoint::run_process();
|
||||
}
|
||||
@@ -16,7 +16,7 @@
|
||||
//!
|
||||
//! This module contains the command-line interface definitions including:
|
||||
//! - `Cli`: Main CLI parser
|
||||
//! - `Commands`: Subcommands (Server, Info, Tls)
|
||||
//! - `Commands`: Top-level server and diagnostic subcommands
|
||||
//! - `ServerOpts`: Server subcommand options
|
||||
//! - `InfoOpts`: Info subcommand options
|
||||
//! - `TlsOpts`: TLS diagnostic subcommand options
|
||||
@@ -56,7 +56,7 @@ pub(super) const LONG_VERSION: &str = concat!(
|
||||
);
|
||||
|
||||
/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand.
|
||||
pub const KNOWN_SUBCOMMANDS: &[&str] = &["server", "info", "tls", "diagnose"];
|
||||
pub const KNOWN_SUBCOMMANDS: &[&str] = &["server", "info", "tls", "diagnose", "inspect"];
|
||||
|
||||
/// Preprocess argv for legacy compatibility: `rustfs <volume>` and `rustfs --address ...` are
|
||||
/// treated as `rustfs server <volume>` and `rustfs server --address ...` respectively.
|
||||
@@ -116,6 +116,48 @@ pub enum Commands {
|
||||
Tls(TlsOpts),
|
||||
/// Analyze RustFS log files and report probable failure causes
|
||||
Diagnose(DiagnoseOpts),
|
||||
/// Offline, read-only inspection of on-disk data (no server required)
|
||||
Inspect(InspectOpts),
|
||||
}
|
||||
|
||||
/// Offline inspection subcommand options
|
||||
#[derive(Args, Clone)]
|
||||
pub struct InspectOpts {
|
||||
#[command(subcommand)]
|
||||
pub command: InspectCommands,
|
||||
}
|
||||
|
||||
/// Offline inspection subcommands
|
||||
#[derive(Subcommand, Clone)]
|
||||
pub enum InspectCommands {
|
||||
/// Export a bucket's persisted configuration bytes straight from drive roots
|
||||
/// (works even when the config XML no longer parses)
|
||||
BucketMeta(InspectBucketMetaOpts),
|
||||
}
|
||||
|
||||
/// `inspect bucket-meta` options
|
||||
#[derive(Args, Clone)]
|
||||
#[command(
|
||||
after_help = "IMPORTANT: Mount every source drive read-only for forensic use. Read-only application calls cannot prevent filesystem atime updates or path replacement races on writable mounts."
|
||||
)]
|
||||
pub struct InspectBucketMetaOpts {
|
||||
/// Drive root path(s). Repeat for multi-drive nodes: erasure-coded metadata
|
||||
/// needs enough drives for write quorum. Source media must be mounted
|
||||
/// read-only for strict forensic use.
|
||||
#[arg(long = "path", required = true, value_parser = NonEmptyStringValueParser::new())]
|
||||
pub paths: Vec<String>,
|
||||
|
||||
/// Bucket whose metadata to inspect
|
||||
#[arg(long, value_parser = NonEmptyStringValueParser::new())]
|
||||
pub bucket: String,
|
||||
|
||||
/// Write the raw `.metadata.bin` blob and each stored config's exact bytes to --out
|
||||
#[arg(long)]
|
||||
pub raw: bool,
|
||||
|
||||
/// New output directory for --raw (default: ./bucket-meta-<bucket>)
|
||||
#[arg(long, requires = "raw")]
|
||||
pub out: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
/// Diagnose report output format
|
||||
@@ -368,6 +410,8 @@ pub enum CommandResult {
|
||||
Tls(TlsOpts),
|
||||
/// Diagnose command with options
|
||||
Diagnose(DiagnoseOpts),
|
||||
/// Inspect command with options
|
||||
Inspect(InspectOpts),
|
||||
}
|
||||
|
||||
/// Create default ServerOpts from environment variables
|
||||
@@ -407,7 +451,7 @@ pub fn default_server_opts() -> ServerOpts {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Cli, preprocess_args_for_legacy};
|
||||
use super::{Cli, Commands, InspectCommands, preprocess_args_for_legacy};
|
||||
use clap::Parser;
|
||||
use clap::error::ErrorKind;
|
||||
|
||||
@@ -423,4 +467,52 @@ mod tests {
|
||||
};
|
||||
assert_eq!(err.kind(), ErrorKind::DisplayHelp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_bucket_meta_parses_repeated_drive_paths() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"rustfs",
|
||||
"inspect",
|
||||
"bucket-meta",
|
||||
"--path",
|
||||
"/data/drive-1",
|
||||
"--path",
|
||||
"/data/drive-2",
|
||||
"--bucket",
|
||||
"example-bucket",
|
||||
"--raw",
|
||||
"--out",
|
||||
"/tmp/export",
|
||||
])
|
||||
.expect("inspect arguments should parse");
|
||||
|
||||
let Some(Commands::Inspect(inspect)) = cli.command else {
|
||||
panic!("inspect command expected");
|
||||
};
|
||||
let InspectCommands::BucketMeta(opts) = inspect.command;
|
||||
assert_eq!(opts.paths, ["/data/drive-1", "/data/drive-2"]);
|
||||
assert_eq!(opts.bucket, "example-bucket");
|
||||
assert!(opts.raw);
|
||||
assert_eq!(opts.out.as_deref(), Some(std::path::Path::new("/tmp/export")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_bucket_meta_rejects_out_without_raw() {
|
||||
let err = match Cli::try_parse_from([
|
||||
"rustfs",
|
||||
"inspect",
|
||||
"bucket-meta",
|
||||
"--path",
|
||||
"/data/drive-1",
|
||||
"--bucket",
|
||||
"example-bucket",
|
||||
"--out",
|
||||
"/tmp/export",
|
||||
]) {
|
||||
Ok(_) => panic!("--out without --raw must be rejected"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), ErrorKind::MissingRequiredArgument);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#[cfg(test)]
|
||||
#[allow(unsafe_op_in_unsafe_fn)]
|
||||
mod tests {
|
||||
use crate::config::cli::default_server_opts;
|
||||
use crate::config::cli::{InspectCommands, default_server_opts};
|
||||
use crate::config::{CommandResult, Config, Opt, TlsCommands};
|
||||
use crate::storage_api::config_test::DisksLayout;
|
||||
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
|
||||
@@ -101,6 +101,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_inspect_subcommand_survives_legacy_preprocessing() {
|
||||
let result = Opt::parse_command([
|
||||
"rustfs",
|
||||
"inspect",
|
||||
"bucket-meta",
|
||||
"--path",
|
||||
"/data/drive-1",
|
||||
"--bucket",
|
||||
"example-bucket",
|
||||
])
|
||||
.expect("inspect command should survive legacy preprocessing");
|
||||
|
||||
match result {
|
||||
CommandResult::Inspect(opts) => match opts.command {
|
||||
InspectCommands::BucketMeta(opts) => {
|
||||
assert_eq!(opts.paths, ["/data/drive-1"]);
|
||||
assert_eq!(opts.bucket, "example-bucket");
|
||||
}
|
||||
},
|
||||
_ => panic!("expected inspect command result"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_parse_from_non_server_commands_falls_back_without_panicking() {
|
||||
|
||||
@@ -52,6 +52,7 @@ mod config_test;
|
||||
// Re-export public types
|
||||
pub use cli::{CommandResult, InfoOpts, InfoType};
|
||||
pub use cli::{DiagnoseFormat, DiagnoseOpts};
|
||||
pub use cli::{InspectBucketMetaOpts, InspectCommands, InspectOpts};
|
||||
pub use cli::{TlsCommands, TlsInspectOpts, TlsOpts};
|
||||
pub use config_struct::Config;
|
||||
pub use info::execute_info;
|
||||
|
||||
@@ -98,7 +98,7 @@ impl Opt {
|
||||
let cli = Cli::parse_from(args);
|
||||
match cli.command {
|
||||
Some(Commands::Server(opts)) => Self::from_server_opts(*opts),
|
||||
Some(Commands::Info(_)) | Some(Commands::Tls(_)) | Some(Commands::Diagnose(_)) => {
|
||||
Some(Commands::Info(_)) | Some(Commands::Tls(_)) | Some(Commands::Diagnose(_)) | Some(Commands::Inspect(_)) => {
|
||||
Self::from_server_opts(default_server_opts())
|
||||
}
|
||||
None => {
|
||||
@@ -134,6 +134,7 @@ impl Opt {
|
||||
Some(Commands::Info(opts)) => Ok(CommandResult::Info(opts)),
|
||||
Some(Commands::Tls(opts)) => Ok(CommandResult::Tls(opts)),
|
||||
Some(Commands::Diagnose(opts)) => Ok(CommandResult::Diagnose(opts)),
|
||||
Some(Commands::Inspect(opts)) => Ok(CommandResult::Inspect(opts)),
|
||||
Some(Commands::Server(opts)) => Self::server_command_result(Self::from_server_opts(*opts)),
|
||||
None => {
|
||||
// Default to server with empty volumes (will be filled from env)
|
||||
@@ -162,7 +163,7 @@ impl Opt {
|
||||
let cli = Cli::try_parse_from(args)?;
|
||||
match cli.command {
|
||||
Some(Commands::Server(opts)) => Ok(Self::from_server_opts(*opts)),
|
||||
Some(Commands::Info(_)) | Some(Commands::Tls(_)) | Some(Commands::Diagnose(_)) => {
|
||||
Some(Commands::Info(_)) | Some(Commands::Tls(_)) | Some(Commands::Diagnose(_)) | Some(Commands::Inspect(_)) => {
|
||||
Err(clap::Error::new(clap::error::ErrorKind::DisplayHelp))
|
||||
}
|
||||
None => {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -84,6 +84,7 @@ pub mod diagnose;
|
||||
pub mod embedded;
|
||||
pub mod error;
|
||||
pub mod init;
|
||||
pub mod inspect;
|
||||
pub(crate) mod kms_deletion_gate;
|
||||
pub mod license;
|
||||
pub mod memory_observability;
|
||||
|
||||
@@ -87,6 +87,9 @@ async fn async_main() -> Result<()> {
|
||||
// Diagnose short-circuits before observability init on purpose:
|
||||
// the report goes to stdout and must not be wrapped by the JSON logger.
|
||||
CommandResult::Diagnose(opts) => return crate::diagnose::execute_diagnose(&opts),
|
||||
// Inspect is offline like diagnose: read-only against drive paths, output
|
||||
// to stdout/--out, and must run before any observability/storage init.
|
||||
CommandResult::Inspect(opts) => return crate::inspect::execute_inspect(&opts).await,
|
||||
CommandResult::Server(config) => config,
|
||||
};
|
||||
|
||||
|
||||
@@ -543,7 +543,15 @@ pub(crate) mod ecstore_test_support {
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_set_disk {
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::{
|
||||
DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class,
|
||||
};
|
||||
}
|
||||
|
||||
/// Offline erasure primitives for `rustfs inspect bucket-meta` (backlog#1733):
|
||||
/// shard bitrot verification and reconstruction without a running store.
|
||||
pub(crate) mod ecstore_erasure {
|
||||
pub(crate) use rustfs_ecstore::api::erasure::{BitrotReader, Erasure};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_storage {
|
||||
|
||||
@@ -23,6 +23,16 @@ pub(crate) mod capacity {
|
||||
}
|
||||
}
|
||||
|
||||
/// Offline bucket-metadata inspection (`rustfs inspect bucket-meta`,
|
||||
/// backlog#1733): read-only shard verification and reconstruction without a
|
||||
/// running store.
|
||||
pub(crate) mod inspect {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata as bucket_metadata;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::utils::check_valid_bucket_name_strict;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_erasure::{BitrotReader, Erasure};
|
||||
pub(crate) use crate::storage::storage_api::ecstore_set_disk::file_info_quorum_hash;
|
||||
}
|
||||
|
||||
pub(crate) mod cluster {
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod capability {
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! End-to-end coverage for the offline inspect command dispatcher.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
|
||||
|
||||
fn decode_hex(source: &str) -> Vec<u8> {
|
||||
let digits = source
|
||||
.chars()
|
||||
.filter(|character| character.is_ascii_hexdigit())
|
||||
.collect::<String>();
|
||||
digits
|
||||
.as_bytes()
|
||||
.chunks_exact(2)
|
||||
.map(|pair| u8::from_str_radix(std::str::from_utf8(pair).expect("hex pair"), 16).expect("fixture hex byte"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_subcommand_reaches_the_offline_executor() {
|
||||
let drive = tempfile::tempdir().expect("drive tempdir");
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_rustfs-cli"))
|
||||
.args([
|
||||
"inspect",
|
||||
"bucket-meta",
|
||||
"--path",
|
||||
drive.path().to_string_lossy().as_ref(),
|
||||
"--bucket",
|
||||
"interop",
|
||||
])
|
||||
.output()
|
||||
.expect("run rustfs-cli inspect");
|
||||
|
||||
assert!(!output.status.success(), "missing metadata must reach the executor and fail");
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(stderr.contains("mount source drives read-only"), "executor warning missing: {stderr}");
|
||||
assert!(stderr.contains("no drive yielded a readable shard"), "executor error missing: {stderr}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_success_reports_persisted_header_and_all_config_timestamps() {
|
||||
let drive = tempfile::tempdir().expect("drive tempdir");
|
||||
let export = tempfile::tempdir().expect("export tempdir");
|
||||
let out = export.path().join("raw");
|
||||
let object_dir = drive.path().join(".rustfs.sys/buckets/interop/.metadata.bin");
|
||||
std::fs::create_dir_all(&object_dir).expect("create metadata object directory");
|
||||
let fixture = decode_hex(include_str!("../../crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex"));
|
||||
std::fs::write(object_dir.join("xl.meta"), fixture).expect("write metadata fixture");
|
||||
let drive_path = drive.path().to_string_lossy().into_owned();
|
||||
let out_path = out.to_string_lossy().into_owned();
|
||||
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_rustfs-cli"))
|
||||
.args([
|
||||
"inspect",
|
||||
"bucket-meta",
|
||||
"--path",
|
||||
drive_path.as_str(),
|
||||
"--bucket",
|
||||
"interop",
|
||||
"--raw",
|
||||
"--out",
|
||||
out_path.as_str(),
|
||||
])
|
||||
.output()
|
||||
.expect("run rustfs-cli inspect");
|
||||
|
||||
assert!(output.status.success(), "valid metadata inspection failed");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let expected_lines = [
|
||||
"bucket : interop",
|
||||
"format : 1",
|
||||
"version : 1",
|
||||
"created : 2026-07-07T15:58:57.210712Z",
|
||||
"notification.xml 231 2026-07-07T15:58:57.614429Z",
|
||||
"lifecycle.xml 344 2026-07-07T15:58:57.337595Z",
|
||||
"object-lock.xml 184 2026-07-07T15:58:57.293303Z",
|
||||
"versioning.xml 123 2026-07-07T15:58:57.254109Z",
|
||||
"bucket-encryption.xml 240 2026-07-07T15:58:57.554116Z",
|
||||
"tagging.xml 128 2026-07-07T15:58:57.378177Z",
|
||||
"replication.xml 639 2026-07-07T15:59:20.148526Z",
|
||||
"cors.xml 0 -",
|
||||
"logging.xml 0 -",
|
||||
"website.xml 0 -",
|
||||
"accelerate.xml 0 -",
|
||||
"request-payment.xml 0 -",
|
||||
"public-access-block.xml 0 -",
|
||||
];
|
||||
for expected in expected_lines {
|
||||
assert!(stdout.lines().any(|line| line == expected), "missing stdout line {expected:?}:\n{stdout}");
|
||||
}
|
||||
|
||||
let expected_blob = decode_hex(include_str!("../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex"));
|
||||
assert_eq!(std::fs::read_dir(&out).expect("read raw output").count(), 14);
|
||||
assert_eq!(std::fs::read(out.join(".metadata.bin")).expect("read raw metadata"), expected_blob);
|
||||
let metadata = BucketMetadata::unmarshal(&expected_blob[4..]).expect("unmarshal expected metadata");
|
||||
let expected_configs = [
|
||||
("notification.xml", metadata.notification_config_xml.as_slice()),
|
||||
("lifecycle.xml", metadata.lifecycle_config_xml.as_slice()),
|
||||
("object-lock.xml", metadata.object_lock_config_xml.as_slice()),
|
||||
("versioning.xml", metadata.versioning_config_xml.as_slice()),
|
||||
("bucket-encryption.xml", metadata.encryption_config_xml.as_slice()),
|
||||
("tagging.xml", metadata.tagging_config_xml.as_slice()),
|
||||
("replication.xml", metadata.replication_config_xml.as_slice()),
|
||||
("cors.xml", metadata.cors_config_xml.as_slice()),
|
||||
("logging.xml", metadata.logging_config_xml.as_slice()),
|
||||
("website.xml", metadata.website_config_xml.as_slice()),
|
||||
("accelerate.xml", metadata.accelerate_config_xml.as_slice()),
|
||||
("request-payment.xml", metadata.request_payment_config_xml.as_slice()),
|
||||
("public-access-block.xml", metadata.public_access_block_config_xml.as_slice()),
|
||||
];
|
||||
for (name, expected_bytes) in expected_configs {
|
||||
assert_eq!(
|
||||
std::fs::read(out.join(name)).unwrap_or_else(|error| panic!("read {name}: {error}")),
|
||||
expected_bytes,
|
||||
"{name} must preserve exact persisted bytes"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user