mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 20:11:31 +00:00
shankar0123
135b271197
Phase 10 of the deploy-hardening I master bundle. Mirrors the
production-hardening-II Phase 8 OCSP-counter pattern. Per frozen
decision 0.9, the metric naming convention is
`certctl_deploy_<area>_total` with target_type + sub-label.
internal/service/deploy_counters.go:
- DeployCounters struct with sync.Map of per-target-type buckets
(apache, nginx, etc.). Lock-free fast path via sync/atomic
Uint64 counters; LoadOrStore on first tick.
- 8 sub-counters per target-type bucket:
- attemptsSuccess / attemptsFailure
- validateFailures (PreCommit returned error)
- reloadFailures (PostCommit returned error → rollback ran)
- postVerifyFails (post-deploy TLS handshake failed)
- rollbackRestored (rollback succeeded)
- rollbackAlsoFail (operator-actionable escalation)
- idempotentSkips (SHA-256 match → no-op deploy)
- Snapshot returns []DeploySnapshot for the Prometheus exposer.
internal/service/deploy_counters_test.go:
- 5 tests: zero-state, per-target-type tick isolation, race-detector
smoke under concurrent ticks, cross-target bucket isolation,
snapshot-mutation-doesn't-affect-counter.
internal/api/handler/metrics.go:
- New DeployCounterSnapshotter interface (mirrors CounterSnapshotter
for the OCSP counters but uses the per-target-type tuple shape).
- New DeploySnapshotEntry struct copying the service-layer shape;
avoids importing the service package directly so the handler
stays dependency-light.
- New SetDeployCounters setter on MetricsHandler (mirrors
SetOCSPCounters wiring).
- Prometheus exposer extended with 6 new metric blocks per frozen
decision 0.9:
- certctl_deploy_attempts_total{target_type, result}
- certctl_deploy_validate_failures_total{target_type}
- certctl_deploy_reload_failures_total{target_type}
- certctl_deploy_post_verify_failures_total{target_type}
- certctl_deploy_rollback_total{target_type, outcome}
- certctl_deploy_idempotent_skip_total{target_type}
- Output sorted by target_type for stable diffs across requests.
The agent-side wire-up (cmd/agent/main.go ticking counters in the
DeployCertificate dispatch site) is intentionally deferred to a
follow-up commit — Phase 10's load-bearing change is the
infrastructure; per-connector tick wiring is a mechanical follow-on.
Build + go vet clean. go test -count=1 green for service +
handler packages.
Phase 11 next: cross-cutting integration tests at deploy/test/.
107 lines
2.9 KiB
Go
107 lines
2.9 KiB
Go
package service
|
|
|
|
import (
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
// Phase 10 of the deploy-hardening I master bundle — DeployCounters
|
|
// unit tests. Mirrors ocsp_counters_test.go.
|
|
|
|
func TestDeployCounters_NewIsZero(t *testing.T) {
|
|
c := NewDeployCounters()
|
|
if got := c.Snapshot(); len(got) != 0 {
|
|
t.Errorf("snapshot at zero state = %d entries, want 0", len(got))
|
|
}
|
|
}
|
|
|
|
func TestDeployCounters_IncTicksTargetTypeBucket(t *testing.T) {
|
|
c := NewDeployCounters()
|
|
c.IncAttemptSuccess("nginx")
|
|
c.IncAttemptSuccess("nginx")
|
|
c.IncAttemptSuccess("apache")
|
|
c.IncAttemptFailure("nginx")
|
|
c.IncValidateFailure("nginx")
|
|
c.IncReloadFailure("nginx")
|
|
c.IncPostVerifyFailure("nginx")
|
|
c.IncRollbackRestored("nginx")
|
|
c.IncRollbackAlsoFailed("nginx")
|
|
c.IncIdempotentSkip("nginx")
|
|
|
|
snap := c.Snapshot()
|
|
if len(snap) != 2 {
|
|
t.Fatalf("snapshot len = %d, want 2 (nginx + apache)", len(snap))
|
|
}
|
|
got := map[string]DeploySnapshot{}
|
|
for _, s := range snap {
|
|
got[s.TargetType] = s
|
|
}
|
|
n := got["nginx"]
|
|
if n.AttemptsSuccess != 2 {
|
|
t.Errorf("nginx success = %d, want 2", n.AttemptsSuccess)
|
|
}
|
|
if n.AttemptsFailure != 1 {
|
|
t.Errorf("nginx failure = %d, want 1", n.AttemptsFailure)
|
|
}
|
|
if n.ValidateFailures != 1 || n.ReloadFailures != 1 || n.PostVerifyFails != 1 ||
|
|
n.RollbackRestored != 1 || n.RollbackAlsoFail != 1 || n.IdempotentSkips != 1 {
|
|
t.Errorf("nginx sub-counter mismatch: %+v", n)
|
|
}
|
|
a := got["apache"]
|
|
if a.AttemptsSuccess != 1 {
|
|
t.Errorf("apache success = %d, want 1", a.AttemptsSuccess)
|
|
}
|
|
}
|
|
|
|
func TestDeployCounters_ConcurrentTicks(t *testing.T) {
|
|
c := NewDeployCounters()
|
|
const goroutines = 10
|
|
const ticks = 100
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < goroutines; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := 0; j < ticks; j++ {
|
|
c.IncAttemptSuccess("nginx")
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
for _, s := range c.Snapshot() {
|
|
if s.TargetType == "nginx" && s.AttemptsSuccess != goroutines*ticks {
|
|
t.Errorf("nginx success = %d, want %d", s.AttemptsSuccess, goroutines*ticks)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDeployCounters_BucketsIsolatedAcrossTargetTypes(t *testing.T) {
|
|
c := NewDeployCounters()
|
|
c.IncAttemptSuccess("nginx")
|
|
c.IncReloadFailure("apache")
|
|
snap := c.Snapshot()
|
|
got := map[string]DeploySnapshot{}
|
|
for _, s := range snap {
|
|
got[s.TargetType] = s
|
|
}
|
|
if got["nginx"].ReloadFailures != 0 {
|
|
t.Errorf("nginx ReloadFailures bled across: got %d", got["nginx"].ReloadFailures)
|
|
}
|
|
if got["apache"].AttemptsSuccess != 0 {
|
|
t.Errorf("apache AttemptsSuccess bled across: got %d", got["apache"].AttemptsSuccess)
|
|
}
|
|
}
|
|
|
|
func TestDeployCounters_StableSnapshot(t *testing.T) {
|
|
// Snapshot read returns a copy — mutating the returned slice
|
|
// must NOT affect the underlying counters.
|
|
c := NewDeployCounters()
|
|
c.IncAttemptSuccess("nginx")
|
|
snap := c.Snapshot()
|
|
snap[0].AttemptsSuccess = 999
|
|
again := c.Snapshot()
|
|
if again[0].AttemptsSuccess != 1 {
|
|
t.Errorf("counter mutated through snapshot: got %d", again[0].AttemptsSuccess)
|
|
}
|
|
}
|