Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue 12db8f3cfa fix: resolve clippy warnings in ecstore
- Remove unused mut on accounting variable in store/object.rs
- Use sort_by_key with Reverse instead of sort_by in rebalance/support.rs
2026-08-23 02:21:22 +08:00
8 changed files with 32 additions and 353 deletions
-20
View File
@@ -1,20 +0,0 @@
# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
# Update counts only with a linked coverage run and a reviewed explanation.
phase = "report-only"
allowed_drop_percentage_points = 1.0
[crates."crates/iam"]
covered = 5149
count = 8131
[crates."crates/kms"]
covered = 2950
count = 4200
[crates."crates/policy"]
covered = 4636
count = 5464
[crates."crates/crypto"]
covered = 469
count = 494
-1
View File
@@ -36,7 +36,6 @@ script-tests: ## Run shell script tests
./scripts/test_manual_transition_runbooks.sh
./scripts/check_embedded_secrets.sh --self-test
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_security_coverage.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
+12 -28
View File
@@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Workspace line-coverage baseline and security-crate calibration
# (backlog#1153 infra-5/infra-6).
# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
#
# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
# trend, while relevant pull requests run a report-only security-crate
# comparison. Neither job is a required check during calibration.
# NON-BLOCKING by design: this workflow only runs on schedule and manual
# dispatch, so it never attaches a status to a PR and must never be made a
# required check. It exists to give coverage a visible baseline and trend
# (per-crate table in the job summary, lcov artifact kept 90 days) — the
# per-crate ratchet for the security-critical crates builds on it later
# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
#
# Measurement scope matches the PR test gate (ci.yml "Run tests"):
# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -29,17 +31,6 @@
name: coverage
on:
pull_request:
branches: [main]
paths:
- "crates/iam/**"
- "crates/kms/**"
- "crates/policy/**"
- "crates/crypto/**"
- ".config/coverage-baselines.toml"
- "scripts/coverage_per_crate.py"
- "scripts/check_security_coverage.py"
- ".github/workflows/coverage.yml"
workflow_dispatch:
schedule:
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -48,10 +39,6 @@ on:
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
- cron: "43 7 * * 0"
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
permissions:
@@ -59,13 +46,12 @@ permissions:
jobs:
coverage:
name: Workspace line coverage
name: Workspace coverage (weekly)
runs-on: sm-standard-4
# The instrumented build cannot reuse the regular CI cache (different
# RUSTFLAGS), so a cold run rebuilds the workspace before running the
# full suite. Exact-head run 32573798257 needed 119m42s including reports
# and artifact upload, so keep a bounded 30-minute publication margin.
timeout-minutes: 150
# RUSTFLAGS), so a cold week rebuilds the workspace before running the
# full suite; give it double the test job's 60-minute budget.
timeout-minutes: 120
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
# Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -105,9 +91,7 @@ jobs:
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
- name: Write per-crate summary
run: |
python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
- name: Upload coverage artifact
if: always()
+1 -1
View File
@@ -3194,7 +3194,7 @@ impl ECStore {
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut accounting = vec![None; objects.len()];
let accounting = vec![None; objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
@@ -271,7 +271,7 @@ pub(super) fn resolve_latest_object_info_candidates(
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
.collect::<Vec<_>>();
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
latest_candidates.sort_by_key(|right| std::cmp::Reverse(right.idx));
let Some(winner) = latest_candidates.first() else {
return Err(Error::ErasureReadQuorum);
+4 -11
View File
@@ -158,11 +158,10 @@ added by backlog#1153 infra-4.
## Coverage
Workspace line coverage is measured weekly. Pull requests that touch iam, kms,
policy, or crypto also run a non-required, report-only comparison against
`.config/coverage-baselines.toml`. During calibration, a regression is recorded
in the job summary without failing the job; missing or malformed coverage
evidence still fails closed (backlog#1153 infra-6).
Line coverage is measured **weekly, not per-PR**, and is non-blocking: it
exists for visibility and trend, never as a required check. Per-crate ratchets
for the security-critical crates (iam / kms / policy / crypto) build on this
baseline later (backlog#1153 infra-6, report-only first).
- **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual
dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the
@@ -175,12 +174,6 @@ evidence still fails closed (backlog#1153 infra-6).
plus the full suite). It prints the same per-crate table via
`scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and
`coverage.json`.
- **Security-critical ratchet**: relevant pull requests compare iam / kms /
policy / crypto line coverage with the versioned baseline. Drops greater than
the configured one-percentage-point calibration threshold are marked
`REGRESSION (report-only)`. The weekly summary runs the same comparison so
calibration continues even when no relevant pull request is open. Baseline
changes require a linked coverage run and a reviewed explanation.
- **Trend comparison**: each run's job summary is the weekly per-crate
snapshot — open two runs from the Actions history (workflow "coverage") and
compare their tables. For line-level diffs, download the two runs'
-264
View File
@@ -1,264 +0,0 @@
#!/usr/bin/env python3
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compare security-critical crate line coverage with the report-only baseline."""
import argparse
import json
import math
import os
import sys
import tempfile
import tomllib
from pathlib import Path
from coverage_per_crate import fmt_pct, load_coverage
SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto")
def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]:
with open(path, "rb") as fh:
config = tomllib.load(fh)
if config.get("phase") != "report-only":
raise ValueError("coverage baseline phase must be report-only")
raw_allowed_drop = config["allowed_drop_percentage_points"]
if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)):
raise ValueError("allowed_drop_percentage_points must be a number")
allowed_drop = float(raw_allowed_drop)
if not math.isfinite(allowed_drop) or allowed_drop < 0:
raise ValueError("allowed_drop_percentage_points must be finite and non-negative")
baselines: dict[str, tuple[int, int]] = {}
for crate, values in config["crates"].items():
covered = values["covered"]
count = values["count"]
if type(covered) is not int or type(count) is not int:
raise ValueError(f"invalid baseline for {crate}: covered and count must be integers")
if covered < 0 or count <= 0 or covered > count:
raise ValueError(f"invalid baseline for {crate}: {covered}/{count}")
baselines[crate] = (covered, count)
missing = [crate for crate in SECURITY_CRATES if crate not in baselines]
unexpected = sorted(set(baselines).difference(SECURITY_CRATES))
if missing or unexpected:
raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}")
return allowed_drop, baselines
def compare(
current: dict[str, list[int]],
baselines: dict[str, tuple[int, int]],
allowed_drop: float,
) -> list[tuple[str, int, int, int, int, float, bool]]:
rows = []
for crate, (baseline_covered, baseline_count) in baselines.items():
if crate not in current:
raise ValueError(f"coverage report is missing {crate}")
covered, count = current[crate]
if type(covered) is not int or type(count) is not int:
raise ValueError(f"invalid coverage for {crate}: covered and count must be integers")
if covered < 0 or count <= 0 or covered > count:
raise ValueError(f"invalid coverage for {crate}: {covered}/{count}")
current_pct = 100.0 * covered / count
baseline_pct = 100.0 * baseline_covered / baseline_count
delta = current_pct - baseline_pct
rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop))
return rows
def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None:
print("## Security-critical coverage ratchet (report-only)")
print()
print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.")
print()
print("| Crate | Current | Baseline | Delta | Status |")
print("|---|---:|---:|---:|---|")
for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows:
status = "REGRESSION (report-only)" if regressed else "OK"
print(
f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) "
f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) "
f"| {delta:+.2f} pp | {status} |"
)
print()
print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.")
def self_test() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
coverage = root / "coverage.json"
baseline = root / "baseline.toml"
coverage_data = {
"data": [
{
"files": [
{
"filename": str(root / "crates/iam/src/lib.rs"),
"summary": {"lines": {"covered": 80, "count": 100}},
},
{
"filename": str(root / "crates/kms/src/lib.rs"),
"summary": {"lines": {"covered": 90, "count": 100}},
},
{
"filename": str(root / "crates/policy/src/lib.rs"),
"summary": {"lines": {"covered": 90, "count": 100}},
},
{
"filename": str(root / "crates/crypto/src/lib.rs"),
"summary": {"lines": {"covered": 90, "count": 100}},
},
],
"totals": {"lines": {"covered": 350, "count": 400}},
}
]
}
coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
baseline_text = """phase = "report-only"
allowed_drop_percentage_points = 1.0
[crates."crates/iam"]
covered = 90
count = 100
[crates."crates/kms"]
covered = 85
count = 100
[crates."crates/policy"]
covered = 90
count = 100
[crates."crates/crypto"]
covered = 90
count = 100
"""
baseline.write_text(baseline_text, encoding="utf-8")
current, _ = load_coverage(str(coverage), str(root))
allowed_drop, baselines = load_baselines(str(baseline))
rows = compare(current, baselines, allowed_drop)
assert [row[-1] for row in rows] == [True, False, False, False]
try:
compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop)
except ValueError as error:
assert str(error) == "coverage report is missing crates/kms"
else:
raise AssertionError("missing crate must fail closed")
try:
compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop)
except ValueError as error:
assert str(error) == "invalid coverage for crates/iam: 101/100"
else:
raise AssertionError("invalid coverage must fail closed")
for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"):
baseline.write_text(
baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"),
encoding="utf-8",
)
try:
load_baselines(str(baseline))
except ValueError:
pass
else:
raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed")
for field, invalid_values in (
("covered", ("true", '"90"', "90.0", "90.5")),
("count", ("true", '"100"', "100.0", "100.5")),
):
for invalid_value in invalid_values:
baseline.write_text(
baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1),
encoding="utf-8",
)
try:
load_baselines(str(baseline))
except ValueError:
pass
else:
raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed")
for covered, count in (
(True, 100),
(80, True),
(80.0, 100),
(80, 100.0),
(float("nan"), 100),
(80, float("inf")),
):
try:
compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop)
except ValueError:
pass
else:
raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed")
lines = coverage_data["data"][0]["files"][0]["summary"]["lines"]
for field, invalid_values in (
("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))),
("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))),
):
original = lines[field]
for invalid_value in invalid_values:
lines[field] = invalid_value
coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
try:
load_coverage(str(coverage), str(root))
except ValueError:
pass
else:
raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed")
lines[field] = original
baseline.write_text(
baseline_text.replace(
'[crates."crates/crypto"]\ncovered = 90\ncount = 100\n',
"",
),
encoding="utf-8",
)
try:
load_baselines(str(baseline))
except ValueError:
pass
else:
raise AssertionError("missing security-crate baseline must fail closed")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("coverage_json", nargs="?")
parser.add_argument("--baseline", default=".config/coverage-baselines.toml")
parser.add_argument("--repo-root", default=os.getcwd())
parser.add_argument("--self-test", action="store_true")
args = parser.parse_args()
if args.self_test:
self_test()
print("security coverage self-test passed")
return 0
if not args.coverage_json:
parser.error("coverage_json is required unless --self-test is used")
try:
current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root))
allowed_drop, baselines = load_baselines(args.baseline)
rows = compare(current, baselines, allowed_drop)
except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
print_report(rows, allowed_drop)
return 0
if __name__ == "__main__":
sys.exit(main())
+14 -27
View File
@@ -47,31 +47,6 @@ def fmt_pct(covered: int, count: int) -> str:
return f"{100.0 * covered / count:.2f}%" if count else ""
def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]:
covered = lines["covered"]
count = lines["count"]
if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count:
raise ValueError(f"invalid line coverage for {source}: {covered}/{count}")
return covered, count
def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]:
with open(path, encoding="utf-8") as fh:
export = json.load(fh)
data = export["data"][0]
files = data["files"]
total_covered, total_count = _line_counts(data["totals"]["lines"], "totals")
crates: dict[str, list[int]] = {}
for f in files:
covered, count = _line_counts(f["summary"]["lines"], f["filename"])
acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
acc[0] += covered
acc[1] += count
return crates, {"covered": total_covered, "count": total_count}
def main() -> int:
if len(sys.argv) < 2 or len(sys.argv) > 3:
print(__doc__.strip(), file=sys.stderr)
@@ -79,12 +54,24 @@ def main() -> int:
path = sys.argv[1]
root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd())
with open(path, encoding="utf-8") as fh:
export = json.load(fh)
try:
crates, totals = load_coverage(path, root)
except (KeyError, IndexError, ValueError) as exc:
data = export["data"][0]
files = data["files"]
totals = data["totals"]["lines"]
except (KeyError, IndexError) as exc:
print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr)
return 1
crates: dict[str, list[int]] = {}
for f in files:
lines = f["summary"]["lines"]
acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
acc[0] += lines["covered"]
acc[1] += lines["count"]
rows = sorted(
crates.items(),
key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,