mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 21:07:43 +00:00
feat(internode): harden data transport baseline
This commit is contained in:
@@ -3,6 +3,7 @@ set -euo pipefail
|
||||
|
||||
# Internode transport baseline runner.
|
||||
# Reuses scripts/run_object_batch_bench.sh and exports reproducible artifacts:
|
||||
# - run manifest with scenario/tool metadata and git revision
|
||||
# - object benchmark summaries per scenario/workload/concurrency
|
||||
# - optional internode operation metric deltas from a Prometheus text endpoint
|
||||
|
||||
@@ -61,6 +62,7 @@ Optional:
|
||||
|
||||
Notes:
|
||||
- This baseline covers S3 PUT/GET workloads and records internode metric deltas when --metrics-url is set.
|
||||
- The run manifest intentionally omits access keys, secret keys, and extra args to avoid writing credentials to artifacts.
|
||||
- Healing/replication-specific workloads should be run separately and appended to the same artifact directory.
|
||||
USAGE
|
||||
}
|
||||
@@ -127,6 +129,41 @@ setup_output() {
|
||||
echo "scenario,workload,concurrency,size,metric,operation,before,after,delta" > "${OUT_DIR}/internode_metric_deltas.csv"
|
||||
}
|
||||
|
||||
write_run_manifest() {
|
||||
local manifest="${OUT_DIR}/run_manifest.txt"
|
||||
local git_commit git_dirty rustc_version
|
||||
|
||||
git_commit="$(git -C "${PROJECT_ROOT}" rev-parse HEAD 2>/dev/null || echo "unknown")"
|
||||
if [[ -n "$(git -C "${PROJECT_ROOT}" status --porcelain 2>/dev/null || true)" ]]; then
|
||||
git_dirty="true"
|
||||
else
|
||||
git_dirty="false"
|
||||
fi
|
||||
rustc_version="$(rustc --version 2>/dev/null || echo "unknown")"
|
||||
|
||||
{
|
||||
echo "created_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "git_commit=${git_commit}"
|
||||
echo "git_dirty=${git_dirty}"
|
||||
echo "rustc_version=${rustc_version}"
|
||||
echo "kernel=$(uname -srvmo 2>/dev/null || echo "unknown")"
|
||||
echo "tool=${TOOL}"
|
||||
echo "region=${REGION}"
|
||||
echo "bucket_prefix=${BUCKET_PREFIX}"
|
||||
echo "scenarios=${SCENARIOS}"
|
||||
echo "sizes=${SIZES}"
|
||||
echo "concurrencies=${CONCURRENCIES}"
|
||||
echo "duration=${DURATION}"
|
||||
echo "samples=${SAMPLES}"
|
||||
echo "insecure=${INSECURE}"
|
||||
echo "metrics_url=${INTERNODE_METRICS_URL:-N/A}"
|
||||
echo "out_dir=${OUT_DIR}"
|
||||
echo "extra_args_present=$([[ -n "${EXTRA_ARGS}" ]] && echo true || echo false)"
|
||||
echo "access_key=REDACTED"
|
||||
echo "secret_key=REDACTED"
|
||||
} > "${manifest}"
|
||||
}
|
||||
|
||||
collect_internode_snapshot() {
|
||||
local snapshot_file="$1"
|
||||
if [[ -z "${INTERNODE_METRICS_URL}" ]]; then
|
||||
@@ -300,17 +337,20 @@ main() {
|
||||
parse_args "$@"
|
||||
validate_args
|
||||
require_cmd awk
|
||||
require_cmd curl
|
||||
require_cmd rg
|
||||
if [[ -n "${INTERNODE_METRICS_URL}" && "${DRY_RUN}" != "true" ]]; then
|
||||
require_cmd curl
|
||||
fi
|
||||
if [[ ! -x "${OBJECT_BENCH_SCRIPT}" ]]; then
|
||||
echo "ERROR: benchmark script not executable: ${OBJECT_BENCH_SCRIPT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
setup_output
|
||||
write_run_manifest
|
||||
run_all
|
||||
|
||||
echo "Artifacts:"
|
||||
echo " ${OUT_DIR}/run_manifest.txt"
|
||||
echo " ${OUT_DIR}/summary.csv"
|
||||
echo " ${OUT_DIR}/internode_metric_deltas.csv"
|
||||
}
|
||||
|
||||
@@ -74,6 +74,34 @@ require_cmd() {
|
||||
fi
|
||||
}
|
||||
|
||||
print_dry_run_command() {
|
||||
local redact_next=false
|
||||
local arg
|
||||
|
||||
printf '[DRY-RUN]'
|
||||
for arg in "$@"; do
|
||||
if [[ "${redact_next}" == "true" ]]; then
|
||||
printf ' %q' "REDACTED"
|
||||
redact_next=false
|
||||
continue
|
||||
fi
|
||||
|
||||
case "${arg}" in
|
||||
--access-key|--secret-key)
|
||||
printf ' %q' "${arg}"
|
||||
redact_next=true
|
||||
;;
|
||||
-accessKey=*|-secretKey=*)
|
||||
printf ' %q' "${arg%%=*}=REDACTED"
|
||||
;;
|
||||
*)
|
||||
printf ' %q' "${arg}"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
normalize_warp_host() {
|
||||
local raw="$1"
|
||||
raw="${raw#http://}"
|
||||
@@ -200,8 +228,7 @@ run_one() {
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
printf '\n'
|
||||
print_dry_run_command "${cmd[@]}"
|
||||
echo "size=$size tool=$TOOL dry_run" > "$log_file"
|
||||
else
|
||||
if ! "${cmd[@]}" 2>&1 | tee "$log_file"; then
|
||||
@@ -228,8 +255,7 @@ run_one() {
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
printf '\n'
|
||||
print_dry_run_command "${cmd[@]}"
|
||||
echo "size=$size tool=$TOOL dry_run" > "$log_file"
|
||||
else
|
||||
if ! "${cmd[@]}" 2>&1 | tee "$log_file"; then
|
||||
@@ -251,10 +277,12 @@ main() {
|
||||
parse_args "$@"
|
||||
validate_args
|
||||
require_cmd rg
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
require_cmd "$WARP_BIN"
|
||||
else
|
||||
require_cmd "$S3BENCH_BIN"
|
||||
if [[ "$DRY_RUN" != "true" ]]; then
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
require_cmd "$WARP_BIN"
|
||||
else
|
||||
require_cmd "$S3BENCH_BIN"
|
||||
fi
|
||||
fi
|
||||
|
||||
setup_output
|
||||
|
||||
Reference in New Issue
Block a user