mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
test(ecstore): add validation suite coverage gates (#4378)
This commit is contained in:
Zhengchao An
Executable
+541
@@ -0,0 +1,541 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROFILE="quick"
|
||||
OUT_DIR=""
|
||||
DRY_RUN=false
|
||||
SKIP_E2E=false
|
||||
SKIP_S3_TESTS=false
|
||||
SKIP_COVERAGE=false
|
||||
REQUIRE_FIXTURES=false
|
||||
UNIT_COVERAGE_MIN="95"
|
||||
UNIT_COVERAGE_TARGET="100"
|
||||
UNIT_COVERAGE_SCOPE="ec-critical"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_ecstore_validation_suite.sh [options]
|
||||
|
||||
Profiles:
|
||||
quick Core EC read/write/recovery checks for PR smoke (default)
|
||||
full quick + s3-tests subset + fixture tests + coverage
|
||||
destructive full + cluster/concurrency/destructive recovery checks
|
||||
fuzz bounded fuzz smoke for malformed storage inputs
|
||||
|
||||
Options:
|
||||
--profile <quick|full|destructive|fuzz>
|
||||
--out-dir <path> Default: target/ecstore-validation/<timestamp>
|
||||
--skip-e2e Skip e2e_test commands
|
||||
--skip-s3-tests Skip scripts/s3-tests/run.sh in full/destructive
|
||||
--skip-coverage Explicitly skip the mandatory unit coverage gate
|
||||
--unit-coverage-min <percent>
|
||||
Minimum rustfs-ecstore unit line coverage (default: 95)
|
||||
--unit-coverage-scope <ec-critical|crate>
|
||||
Coverage scope for the hard gate (default: ec-critical)
|
||||
--require-fixtures Fail when optional EC fixture tests cannot run
|
||||
--dry-run Print commands and write metadata without executing
|
||||
-h, --help
|
||||
|
||||
Artifacts:
|
||||
<out-dir>/run-metadata.env
|
||||
<out-dir>/summary.tsv
|
||||
<out-dir>/blackbox-matrix.tsv
|
||||
<out-dir>/logs/*.log
|
||||
USAGE
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$value"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--profile) PROFILE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--skip-e2e) SKIP_E2E=true; shift ;;
|
||||
--skip-s3-tests) SKIP_S3_TESTS=true; shift ;;
|
||||
--skip-coverage) SKIP_COVERAGE=true; shift ;;
|
||||
--unit-coverage-min) UNIT_COVERAGE_MIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--unit-coverage-scope) UNIT_COVERAGE_SCOPE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--require-fixtures) REQUIRE_FIXTURES=true; shift ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate_args() {
|
||||
case "$PROFILE" in
|
||||
quick|full|destructive|fuzz) ;;
|
||||
*)
|
||||
echo "ERROR: --profile must be quick, full, destructive, or fuzz" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! [[ "$UNIT_COVERAGE_MIN" =~ ^([0-9]|[1-9][0-9]|100)(\.[0-9]+)?$ ]]; then
|
||||
echo "ERROR: --unit-coverage-min must be between 0 and 100" >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$UNIT_COVERAGE_SCOPE" in
|
||||
ec-critical|crate) ;;
|
||||
*)
|
||||
echo "ERROR: --unit-coverage-scope must be ec-critical or crate" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
setup_workspace() {
|
||||
local root
|
||||
root="$(git rev-parse --show-toplevel)"
|
||||
cd "$root"
|
||||
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="target/ecstore-validation/$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR/logs"
|
||||
|
||||
SUMMARY="$OUT_DIR/summary.tsv"
|
||||
BLACKBOX_MATRIX="$OUT_DIR/blackbox-matrix.tsv"
|
||||
printf 'status\tlabel\tlog\n' >"$SUMMARY"
|
||||
}
|
||||
|
||||
write_metadata() {
|
||||
{
|
||||
printf 'profile=%s\n' "$PROFILE"
|
||||
printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
printf 'dry_run=%s\n' "$DRY_RUN"
|
||||
printf 'skip_e2e=%s\n' "$SKIP_E2E"
|
||||
printf 'skip_s3_tests=%s\n' "$SKIP_S3_TESTS"
|
||||
printf 'skip_coverage=%s\n' "$SKIP_COVERAGE"
|
||||
printf 'unit_coverage_min=%s\n' "$UNIT_COVERAGE_MIN"
|
||||
printf 'unit_coverage_target=%s\n' "$UNIT_COVERAGE_TARGET"
|
||||
printf 'unit_coverage_scope=%s\n' "$UNIT_COVERAGE_SCOPE"
|
||||
printf 'require_fixtures=%s\n' "$REQUIRE_FIXTURES"
|
||||
printf 'git_commit=%s\n' "$(git rev-parse HEAD 2>/dev/null || true)"
|
||||
printf 'git_branch=%s\n' "$(git branch --show-current 2>/dev/null || true)"
|
||||
printf 'rustc=%s\n' "$(rustc --version 2>/dev/null || true)"
|
||||
printf 'cargo=%s\n' "$(cargo --version 2>/dev/null || true)"
|
||||
} >"$OUT_DIR/run-metadata.env"
|
||||
}
|
||||
|
||||
safe_label() {
|
||||
printf '%s\n' "$1" | tr -c 'A-Za-z0-9_.-' '_'
|
||||
}
|
||||
|
||||
quote_command() {
|
||||
local quoted=()
|
||||
local arg
|
||||
for arg in "$@"; do
|
||||
quoted+=("$(printf '%q' "$arg")")
|
||||
done
|
||||
printf '%s ' "${quoted[@]}"
|
||||
}
|
||||
|
||||
record_skip() {
|
||||
local label="$1"
|
||||
local reason="$2"
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
printf 'SKIP: %s\n' "$reason" | tee "$log"
|
||||
printf 'SKIP\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
}
|
||||
|
||||
run_step() {
|
||||
local label="$1"
|
||||
shift
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
|
||||
{
|
||||
printf 'label=%s\n' "$label"
|
||||
printf 'command='
|
||||
quote_command "$@"
|
||||
printf '\n'
|
||||
} >"$log"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf 'DRY-RUN: '
|
||||
quote_command "$@"
|
||||
printf '\n'
|
||||
printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf 'RUN: %s\n' "$label"
|
||||
if "$@" 2>&1 | tee -a "$log"; then
|
||||
printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
else
|
||||
local status=$?
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return "$status"
|
||||
fi
|
||||
}
|
||||
|
||||
run_core_unit_steps() {
|
||||
run_step "filemeta-lib" cargo test -p rustfs-filemeta --lib
|
||||
run_step "ecstore-erasure-lib" cargo test -p rustfs-ecstore --lib erasure
|
||||
run_step "ecstore-set-disk-read-lib" cargo test -p rustfs-ecstore --lib set_disk::read
|
||||
run_step "ecstore-io-primitives-lib" cargo test -p rustfs-ecstore --lib set_disk::core::io_primitives
|
||||
run_step "ecstore-rename-rollback" \
|
||||
cargo test -p rustfs-ecstore --lib set_disk::tests::test_rename_data_quorum_failure_rolls_back_destination_object
|
||||
run_step "ecstore-disk-local-lib" cargo test -p rustfs-ecstore --lib disk::local
|
||||
run_step "ecstore-lib-all" cargo test -p rustfs-ecstore --lib -- --test-threads=1
|
||||
}
|
||||
|
||||
run_quick_e2e_steps() {
|
||||
if [[ "$SKIP_E2E" == "true" ]]; then
|
||||
record_skip "e2e-quick" "--skip-e2e was set"
|
||||
return
|
||||
fi
|
||||
|
||||
run_step "e2e-reliability-disk-fault" cargo test --package e2e_test reliability_disk_fault_test -- --nocapture
|
||||
run_step "e2e-heal-erasure-disk-rebuild" cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture
|
||||
run_step "e2e-namespace-lock-quorum" cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture
|
||||
}
|
||||
|
||||
run_quick_profile() {
|
||||
run_core_unit_steps
|
||||
run_quick_e2e_steps
|
||||
}
|
||||
|
||||
fixture_available() {
|
||||
[[ -n "${RUSTFS_MINIO_FIXTURE_ROOT:-}" && -d "${RUSTFS_MINIO_FIXTURE_ROOT}" && -n "${RUSTFS_MINIO_STATIC_KMS_KEY_B64:-}" ]]
|
||||
}
|
||||
|
||||
legacy_fixture_available() {
|
||||
local root="${RUSTFS_LEGACY_TEST_ROOT:-$(pwd)}"
|
||||
local disk="${RUSTFS_LEGACY_TEST_DISK:-}"
|
||||
if [[ -z "$disk" ]]; then
|
||||
if [[ -f "$root/test/.minio.sys/format.json" ]]; then
|
||||
disk="test"
|
||||
else
|
||||
disk="test1"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -f "$root/$disk/.rustfs.sys/format.json" && ! -f "$root/$disk/.minio.sys/format.json" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ -f "$root/$disk/vvvv/ktvzip.tar.gz/xl.meta" || -f "$root/$disk/vvvv/path_traversal.md/xl.meta" ]]
|
||||
}
|
||||
|
||||
minio_fixture_missing_reason() {
|
||||
if [[ -z "${RUSTFS_MINIO_FIXTURE_ROOT:-}" ]]; then
|
||||
printf 'RUSTFS_MINIO_FIXTURE_ROOT is not set\n'
|
||||
elif [[ ! -d "${RUSTFS_MINIO_FIXTURE_ROOT}" ]]; then
|
||||
printf 'RUSTFS_MINIO_FIXTURE_ROOT does not exist: %s\n' "$RUSTFS_MINIO_FIXTURE_ROOT"
|
||||
elif [[ -z "${RUSTFS_MINIO_STATIC_KMS_KEY_B64:-}" ]]; then
|
||||
printf 'RUSTFS_MINIO_STATIC_KMS_KEY_B64 is not set\n'
|
||||
else
|
||||
printf 'unknown MinIO fixture gate failure\n'
|
||||
fi
|
||||
}
|
||||
|
||||
write_blackbox_matrix() {
|
||||
local e2e_status="enabled"
|
||||
local s3_status="enabled"
|
||||
local destructive_status="enabled"
|
||||
local legacy_status="missing-optional"
|
||||
local minio_status="missing-optional"
|
||||
if [[ "$SKIP_E2E" == "true" ]]; then
|
||||
e2e_status="disabled"
|
||||
destructive_status="disabled"
|
||||
fi
|
||||
if [[ "$SKIP_S3_TESTS" == "true" ]]; then
|
||||
s3_status="disabled"
|
||||
fi
|
||||
if legacy_fixture_available; then
|
||||
legacy_status="enabled"
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
legacy_status="missing-required"
|
||||
fi
|
||||
if fixture_available; then
|
||||
minio_status="enabled"
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
minio_status="missing-required"
|
||||
fi
|
||||
|
||||
{
|
||||
printf 'profile\tscenario\tgate\tcommand\tfixture_env\tstatus\n'
|
||||
printf 'quick\tsingle-node disk fault read/write\tblack-box\tcargo test --package e2e_test reliability_disk_fault_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status"
|
||||
printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
|
||||
printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status"
|
||||
printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
printf 'destructive\tdelete marker migration semantics\tblack-box\tcargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
} >"$BLACKBOX_MATRIX"
|
||||
}
|
||||
|
||||
run_fixture_gate() {
|
||||
local label="ecstore-fixture-gate"
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
local legacy_ok=false
|
||||
local minio_ok=false
|
||||
if legacy_fixture_available; then
|
||||
legacy_ok=true
|
||||
fi
|
||||
if fixture_available; then
|
||||
minio_ok=true
|
||||
fi
|
||||
|
||||
{
|
||||
printf 'label=%s\n' "$label"
|
||||
printf 'legacy_fixture_available=%s\n' "$legacy_ok"
|
||||
printf 'minio_fixture_available=%s\n' "$minio_ok"
|
||||
printf 'require_fixtures=%s\n' "$REQUIRE_FIXTURES"
|
||||
if [[ "$minio_ok" != "true" ]]; then
|
||||
printf 'minio_fixture_reason=%s\n' "$(minio_fixture_missing_reason)"
|
||||
fi
|
||||
} | tee "$log"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$legacy_ok" == "true" && "$minio_ok" == "true" ]]; then
|
||||
printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf 'SKIP\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
}
|
||||
|
||||
run_fixture_steps() {
|
||||
if legacy_fixture_available; then
|
||||
run_step "ecstore-legacy-bitrot-read-fixture" cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
echo "ERROR: RUSTFS_LEGACY_TEST_ROOT/RUSTFS_LEGACY_TEST_DISK is required for legacy bitrot fixture tests" >&2
|
||||
exit 1
|
||||
else
|
||||
record_skip "ecstore-legacy-bitrot-read-fixture" "legacy bitrot fixture is not present"
|
||||
fi
|
||||
|
||||
if fixture_available; then
|
||||
run_step "ecstore-minio-generated-read-fixture" \
|
||||
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
echo "ERROR: $(minio_fixture_missing_reason)" >&2
|
||||
exit 1
|
||||
else
|
||||
record_skip "ecstore-minio-generated-read-fixture" "$(minio_fixture_missing_reason)"
|
||||
fi
|
||||
}
|
||||
|
||||
run_s3_subset() {
|
||||
if [[ "$SKIP_S3_TESTS" == "true" ]]; then
|
||||
record_skip "s3-tests-ec-subset" "--skip-s3-tests was set"
|
||||
return
|
||||
fi
|
||||
|
||||
run_step "s3-tests-ec-subset" \
|
||||
env TESTEXPR="multipart or range or versioning or delete" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh
|
||||
}
|
||||
|
||||
write_coverage_reports() {
|
||||
local lcov_path="$1"
|
||||
local coverage_summary="$2"
|
||||
local coverage_files="$3"
|
||||
local root="$4"
|
||||
|
||||
awk -v root="$root" -v scope="$UNIT_COVERAGE_SCOPE" -v summary="$coverage_summary" -v files="$coverage_files" '
|
||||
function is_crate_file(f) {
|
||||
return index(f, root "/crates/ecstore/src/") == 1
|
||||
}
|
||||
function is_ec_critical_file(f) {
|
||||
return index(f, root "/crates/ecstore/src/erasure/") == 1 ||
|
||||
f == root "/crates/ecstore/src/set_disk/read.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/shard_source.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/metadata.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/ops/object.rs" ||
|
||||
f == root "/crates/ecstore/src/set_disk/core/io_primitives.rs" ||
|
||||
f == root "/crates/ecstore/src/disk/local.rs"
|
||||
}
|
||||
function is_gate_file(f) {
|
||||
return scope == "crate" ? is_crate_file(f) : is_ec_critical_file(f)
|
||||
}
|
||||
function pct(hit, found) {
|
||||
return found == 0 ? 0 : (hit / found) * 100
|
||||
}
|
||||
function flush() {
|
||||
if (file == "") {
|
||||
return
|
||||
}
|
||||
if (is_crate_file(file)) {
|
||||
crate_hit += hit
|
||||
crate_found += found
|
||||
}
|
||||
if (is_ec_critical_file(file)) {
|
||||
ec_hit += hit
|
||||
ec_found += found
|
||||
}
|
||||
if (is_gate_file(file)) {
|
||||
gate_hit += hit
|
||||
gate_found += found
|
||||
}
|
||||
if (is_crate_file(file)) {
|
||||
printf "%s\t%.2f\t%d\t%d\n", file, pct(hit, found), hit, found >> files
|
||||
}
|
||||
}
|
||||
BEGIN {
|
||||
print "file\tline_coverage\thit\tfound" > files
|
||||
}
|
||||
/^SF:/ {
|
||||
flush()
|
||||
file = substr($0, 4)
|
||||
hit = 0
|
||||
found = 0
|
||||
next
|
||||
}
|
||||
/^LH:/ { hit += substr($0, 4); next }
|
||||
/^LF:/ { found += substr($0, 4); next }
|
||||
END {
|
||||
flush()
|
||||
if (gate_found == 0) {
|
||||
exit 2
|
||||
}
|
||||
print "metric\tactual\tmin\ttarget\tscope" > summary
|
||||
printf "gate_line_coverage\t%.2f\t%s\t%s\t%s\n", pct(gate_hit, gate_found), min, target, scope >> summary
|
||||
printf "ec_critical_line_coverage\t%.2f\t%s\t%s\tec-critical\n", pct(ec_hit, ec_found), min, target >> summary
|
||||
printf "crate_line_coverage\t%.2f\t%s\t%s\tcrate\n", pct(crate_hit, crate_found), min, target >> summary
|
||||
}
|
||||
' min="$UNIT_COVERAGE_MIN" target="$UNIT_COVERAGE_TARGET" "$lcov_path"
|
||||
}
|
||||
|
||||
run_coverage() {
|
||||
if [[ "$SKIP_COVERAGE" == "true" ]]; then
|
||||
record_skip "ecstore-coverage" "--skip-coverage was set"
|
||||
return
|
||||
fi
|
||||
|
||||
local label="ecstore-unit-coverage"
|
||||
local log="$OUT_DIR/logs/$(safe_label "$label").log"
|
||||
local coverage_dir="$OUT_DIR/coverage/ecstore"
|
||||
local lcov_path="$coverage_dir/lcov.info"
|
||||
local coverage_summary="$coverage_dir/summary.tsv"
|
||||
local coverage_files="$coverage_dir/files.tsv"
|
||||
local cmd=(cargo llvm-cov -p rustfs-ecstore --lib --lcov --output-path "$lcov_path" -- --test-threads=1)
|
||||
local root
|
||||
root="$(pwd)"
|
||||
mkdir -p "$coverage_dir"
|
||||
|
||||
{
|
||||
printf 'label=%s\n' "$label"
|
||||
printf 'unit_coverage_min=%s\n' "$UNIT_COVERAGE_MIN"
|
||||
printf 'unit_coverage_target=%s\n' "$UNIT_COVERAGE_TARGET"
|
||||
printf 'unit_coverage_scope=%s\n' "$UNIT_COVERAGE_SCOPE"
|
||||
printf 'command='
|
||||
quote_command "${cmd[@]}"
|
||||
printf '\n'
|
||||
} >"$log"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf 'DRY-RUN: '
|
||||
quote_command "${cmd[@]}"
|
||||
printf '\n'
|
||||
printf 'DRY_RUN\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! command -v cargo-llvm-cov >/dev/null 2>&1; then
|
||||
printf 'ERROR: cargo-llvm-cov is required for the ECStore unit coverage gate\n' | tee -a "$log"
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 127
|
||||
fi
|
||||
|
||||
printf 'RUN: %s\n' "$label"
|
||||
if "${cmd[@]}" 2>&1 | tee -a "$log"; then
|
||||
:
|
||||
else
|
||||
local status=$?
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return "$status"
|
||||
fi
|
||||
|
||||
if ! write_coverage_reports "$lcov_path" "$coverage_summary" "$coverage_files" "$root"; then
|
||||
printf 'ERROR: unable to compute scoped coverage from %s\n' "$lcov_path" | tee -a "$log"
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local line_coverage
|
||||
line_coverage="$(awk -F '\t' '$1 == "gate_line_coverage" { print $2 }' "$coverage_summary")"
|
||||
printf 'gate_line_coverage=%s%% scope=%s min=%s%% target=%s%%\n' \
|
||||
"$line_coverage" "$UNIT_COVERAGE_SCOPE" "$UNIT_COVERAGE_MIN" "$UNIT_COVERAGE_TARGET" | tee -a "$log"
|
||||
printf 'coverage_summary=%s\ncoverage_files=%s\n' "$coverage_summary" "$coverage_files" | tee -a "$log"
|
||||
|
||||
if awk -v actual="$line_coverage" -v min="$UNIT_COVERAGE_MIN" 'BEGIN { exit !(actual + 0 >= min + 0) }'; then
|
||||
printf 'PASS\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
else
|
||||
printf 'FAIL: %s coverage %s%% is below %s%%\n' "$UNIT_COVERAGE_SCOPE" "$line_coverage" "$UNIT_COVERAGE_MIN" | tee -a "$log"
|
||||
printf 'FAIL\t%s\t%s\n' "$label" "$log" >>"$SUMMARY"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_full_profile() {
|
||||
run_quick_profile
|
||||
run_fixture_gate
|
||||
run_fixture_steps
|
||||
run_s3_subset
|
||||
run_coverage
|
||||
}
|
||||
|
||||
run_destructive_profile() {
|
||||
run_full_profile
|
||||
if [[ "$SKIP_E2E" == "true" ]]; then
|
||||
record_skip "e2e-destructive" "--skip-e2e was set"
|
||||
return
|
||||
fi
|
||||
|
||||
run_step "e2e-cluster-concurrency" cargo test --package e2e_test cluster_concurrency_test -- --nocapture
|
||||
run_step "e2e-stale-multipart-cleanup-cluster" cargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture
|
||||
run_step "e2e-delete-marker-migration-semantics" cargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture
|
||||
}
|
||||
|
||||
run_fuzz_profile() {
|
||||
local max_total_time="${MAX_TOTAL_TIME:-60}"
|
||||
run_step "fuzz-storage-inputs" env MAX_TOTAL_TIME="$max_total_time" ./scripts/fuzz/run.sh
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate_args
|
||||
setup_workspace
|
||||
write_metadata
|
||||
write_blackbox_matrix
|
||||
|
||||
case "$PROFILE" in
|
||||
quick) run_quick_profile ;;
|
||||
full) run_full_profile ;;
|
||||
destructive) run_destructive_profile ;;
|
||||
fuzz) run_fuzz_profile ;;
|
||||
esac
|
||||
|
||||
printf 'summary=%s\n' "$SUMMARY"
|
||||
printf 'artifacts=%s\n' "$OUT_DIR"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user