mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
perf(server): lighten internode data-plane stack (#3735)
* refactor(server): split internode dispatch scaffold
* test(server): cover internode dispatch prefix split
* refactor(server): name internode stack boundaries
* perf(server): skip internode request logging layer
* perf(server): skip internode trace layer
* perf(server): use lite internode request context
* feat(metrics): track internode rpc duration
* feat(ecstore): add put object stage summary logs
* test(metrics): update internode descriptor expectations
* fix(server): tighten internode path matching
* fix(pr): address review follow-up comments
* style(ecstore): simplify commit tail duration field
* refactor(ecstore): group put stage summary fields
* refactor(ecstore): inline put stage summary log
* fix(s3): return storage class for object attributes
* merge: sync latest main and resolve object attributes conflict
* fmt
* fix(server): remove duplicate rpc imports
* build(deps): bump memmap2 for RUSTSEC-2026-0186
* fix(s3select): align object_store with datafusion
* chore(deps): prune workspace dependencies
* perf(fuzz): optimize CI runtime with build/run split and matrix parallelization
Separate fuzz harness compilation from execution to eliminate redundant
builds across targets. Introduce matrix-based parallel execution for
PR smoke and nightly fuzz jobs.
Changes:
- Split CI workflow into `fuzz-build` (compile once) and matrix run jobs
(`pr-fuzz-smoke`, `nightly-fuzz-corpus`) that execute targets in parallel
- Add `BUILD_ONLY` mode to run_ci_targets.sh / run_nightly_targets.sh
- Add run_single_target.sh for matrix jobs (no build phase)
- Optimize `local_metadata` fuzz target: reduce prefix iterations from
8-10 (4 functions each) to 5 critical prefixes (parser-only), cutting
per-iteration cost by ~3-5x
- Move archive path validation (`validate_extract_relative_path`,
`normalize_extract_entry_key`) from `rustfs` to `rustfs-utils::path`,
eliminating `rustfs` binary crate dependency from fuzz harness
- Remove `rustfs` from fuzz/Cargo.toml (drops significant transitive deps)
- Add unit tests for archive path validation in rustfs-utils
- Update fuzz/README.md with new workflow and script documentation
Expected CI improvement: PR smoke wall-clock from ~120min (frequent
timeout) to ~40min; nightly from ~180min to ~60min.
* refactor(fuzz): consolidate scripts and fix prefix test alignment
Replace three duplicated shell scripts (run_ci_targets.sh,
run_nightly_targets.sh, run_single_target.sh) with a single
parameterized run.sh that supports BUILD_ONLY, SKIP_BUILD, and
MAX_TOTAL_TIME environment variables.
Fix local_metadata fuzz target prefix testing: replace always-true
'len > 0' guard with lengths aligned to xl.meta binary layout
(4/5/8/12 bytes for magic+version+header fields). Remove redundant
empty-slice test.
Hoist RUSTFLAGS to workflow top-level env to eliminate per-job
duplication. Update README with unified script documentation.
Net: -118 lines, zero functionality loss.
* perf(fuzz): optimize CI runtime with build/run split and matrix parallelization
Restructure fuzz CI workflow to eliminate redundant compilation and
run targets in parallel via matrix strategy.
Workflow changes:
- Split into fuzz-build (compile once) and matrix run jobs
- PR smoke: 3 targets parallel, 60s each, timeout 30min (was 120min)
- Nightly: 3 targets parallel, 300s each, timeout 60min (was 180min)
- Pass compiled harness via actions/artifact between jobs
- Hoist RUSTFLAGS to workflow top-level env
Script consolidation:
- Replace 3 duplicated scripts with single parameterized run.sh
- Supports BUILD_ONLY, SKIP_BUILD, MAX_TOTAL_TIME env vars
Target optimizations:
- Remove rustfs binary crate from fuzz dependencies (was pulling
979 transitive deps); move archive path validation to rustfs-utils
- Optimize local_metadata: reduce prefix iterations from 8-10x4
calls to 5 prefixes with parser-only (no decompress), aligned
with xl.meta binary layout (4/5/8/12 bytes)
- Add unit tests for archive path validation in rustfs-utils
- Update fuzz/README.md with unified script documentation
Expected: PR smoke wall-clock from ~120min (frequent timeout)
to ~40min; nightly from ~180min to ~60min.
* fix(rpc): resolve internode metrics via app context
* fmt
* ci: speed up fuzz smoke artifact restore
---------
Signed-off-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::path::Component;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -550,6 +551,52 @@ pub fn trim_etag(etag: &str) -> String {
|
||||
etag.trim_matches('"').to_string()
|
||||
}
|
||||
|
||||
/// Returns `true` if `path` contains a `..` component (parent directory traversal).
|
||||
fn contains_parent_dir_component(path: &str) -> bool {
|
||||
path.split(['/', '\\']).any(|component| component == "..")
|
||||
}
|
||||
|
||||
/// Validates that an archive entry path does not escape the target bucket.
|
||||
///
|
||||
/// Rejects paths containing `..` components, root prefixes, or device/prefix
|
||||
/// components that would allow path traversal outside the extraction root.
|
||||
///
|
||||
/// Returns `Ok(())` if the path is safe, or `Err(message)` describing the violation.
|
||||
pub fn validate_extract_relative_path(path: &str) -> Result<(), String> {
|
||||
let p = Path::new(path);
|
||||
if p.components()
|
||||
.any(|c| matches!(c, Component::Prefix(_) | Component::RootDir | Component::ParentDir))
|
||||
|| contains_parent_dir_component(p.to_string_lossy().as_ref())
|
||||
{
|
||||
return Err("archive entry path must stay within the target bucket".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Normalizes an archive entry key by applying a prefix, trimming slashes,
|
||||
/// and ensuring directory entries end with `/`.
|
||||
///
|
||||
/// Validates both the raw path and the final key against path traversal.
|
||||
///
|
||||
/// Returns `Ok(normalized_key)` or `Err(message)` if the path is unsafe.
|
||||
pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: bool) -> Result<String, String> {
|
||||
validate_extract_relative_path(path)?;
|
||||
let path = path.trim_matches('/');
|
||||
let mut key = match prefix {
|
||||
Some(prefix) if !path.is_empty() => format!("{prefix}/{path}"),
|
||||
Some(prefix) => prefix.to_string(),
|
||||
None => path.to_string(),
|
||||
};
|
||||
|
||||
if is_dir && !key.ends_with('/') {
|
||||
key.push('/');
|
||||
}
|
||||
|
||||
validate_extract_relative_path(&key)?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -918,4 +965,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_extract_relative_path_accepts_safe_paths() {
|
||||
assert!(validate_extract_relative_path("file.txt").is_ok());
|
||||
assert!(validate_extract_relative_path("dir/file.txt").is_ok());
|
||||
assert!(validate_extract_relative_path("a/b/c").is_ok());
|
||||
assert!(validate_extract_relative_path("").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_extract_relative_path_rejects_traversal() {
|
||||
assert!(validate_extract_relative_path("../escape.txt").is_err());
|
||||
assert!(validate_extract_relative_path("dir/../../../escape.txt").is_err());
|
||||
assert!(validate_extract_relative_path("/absolute/path").is_err());
|
||||
assert!(validate_extract_relative_path("dir/..").is_err());
|
||||
assert!(validate_extract_relative_path("..").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_extract_entry_key_basic() {
|
||||
assert_eq!(normalize_extract_entry_key("file.txt", None, false).unwrap(), "file.txt");
|
||||
assert_eq!(normalize_extract_entry_key("file.txt", Some("prefix"), false).unwrap(), "prefix/file.txt");
|
||||
assert_eq!(normalize_extract_entry_key("dir", None, true).unwrap(), "dir/");
|
||||
assert_eq!(normalize_extract_entry_key("", Some("prefix"), false).unwrap(), "prefix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_extract_entry_key_rejects_traversal() {
|
||||
assert!(normalize_extract_entry_key("../escape.txt", None, false).is_err());
|
||||
assert!(normalize_extract_entry_key("file.txt", Some("../bad"), false).is_err());
|
||||
assert!(normalize_extract_entry_key("/absolute", None, false).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user