feat(M48): continuous TLS health monitoring — endpoint state machine, shared tlsprobe, 8 API endpoints, GUI

Adds continuous TLS endpoint health monitoring that closes the deploy→verify→monitor loop.
After M25 verifies a deployment succeeded once, M48 continuously confirms it stays healthy.

Key components:
- Shared `internal/tlsprobe/` package extracted from network scanner for reuse
- Health status state machine: healthy → degraded (2 failures) → down (5 failures),
  plus cert_mismatch when served fingerprint differs from expected
- 8th scheduler loop (60s tick, per-endpoint configurable intervals)
- PostgreSQL migration 000011: endpoint_health_checks + endpoint_health_history tables
- 8 REST API endpoints (CRUD, history, acknowledge, summary)
- Health Monitor GUI page with summary bar, status table, create modal, auto-refresh
- 38 new tests (5 tlsprobe + 11 domain + 10 service + 8 handler + 4 frontend)
- All coverage thresholds maintained (service 68%, handler 83%, domain 87%, middleware 63%)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
shankar0123
2026-04-15 21:45:45 -04:00
parent f2e60b93a3
commit 596d86a206
29 changed files with 3540 additions and 30 deletions
+313
View File
@@ -0,0 +1,313 @@
package service
import (
"context"
"fmt"
"log/slog"
"sync"
"time"
"github.com/shankar0123/certctl/internal/domain"
"github.com/shankar0123/certctl/internal/repository"
"github.com/shankar0123/certctl/internal/tlsprobe"
)
// HealthCheckService manages endpoint TLS health monitoring.
type HealthCheckService struct {
repo repository.HealthCheckRepository
auditService *AuditService
notifService *NotificationService
logger *slog.Logger
maxConcurrent int
defaultTimeout time.Duration
historyRetention time.Duration
autoCreate bool
}
// NewHealthCheckService creates a new HealthCheckService.
func NewHealthCheckService(
repo repository.HealthCheckRepository,
auditService *AuditService,
logger *slog.Logger,
maxConcurrent int,
defaultTimeout time.Duration,
historyRetention time.Duration,
autoCreate bool,
) *HealthCheckService {
return &HealthCheckService{
repo: repo,
auditService: auditService,
logger: logger,
maxConcurrent: maxConcurrent,
defaultTimeout: defaultTimeout,
historyRetention: historyRetention,
autoCreate: autoCreate,
}
}
// SetNotificationService sets the notification service for sending status transition alerts.
func (s *HealthCheckService) SetNotificationService(ns *NotificationService) {
s.notifService = ns
}
// RunHealthChecks is the scheduler entry point for continuous TLS health monitoring.
// Fetches endpoints due for check, probes concurrently with semaphore control,
// updates health status with state transitions, records history, and sends notifications.
func (s *HealthCheckService) RunHealthChecks(ctx context.Context) error {
// Fetch all endpoints due for check
checks, err := s.repo.ListDueForCheck(ctx)
if err != nil {
return fmt.Errorf("failed to list endpoints due for check: %w", err)
}
if len(checks) == 0 {
s.logger.Debug("no endpoints due for health check")
return nil
}
s.logger.Debug("running health checks", "endpoint_count", len(checks))
// Concurrent probing with semaphore
sem := make(chan struct{}, s.maxConcurrent)
var wg sync.WaitGroup
probeResults := make(map[string]tlsprobe.ProbeResult)
var mu sync.Mutex
for _, check := range checks {
wg.Add(1)
go func(c *domain.EndpointHealthCheck) {
defer wg.Done()
sem <- struct{}{} // acquire
defer func() { <-sem }() // release
result := tlsprobe.ProbeTLS(ctx, c.Endpoint, s.defaultTimeout)
mu.Lock()
probeResults[c.ID] = result
mu.Unlock()
}(check)
}
wg.Wait()
// Process results and update health status
successCount := 0
failureCount := 0
transitionCount := 0
for _, check := range checks {
result := probeResults[check.ID]
// Determine old status for transition detection
oldStatus := check.Status
// Update probe result fields
check.LastCheckedAt = timePtr(time.Now())
check.ResponseTimeMs = result.ResponseTimeMs
if result.Success {
successCount++
check.ObservedFingerprint = result.Fingerprint
check.TLSVersion = result.TLSVersion
check.CipherSuite = result.CipherSuite
check.CertSubject = result.Subject
check.CertIssuer = result.Issuer
check.CertExpiry = timePtr(result.NotAfter)
check.FailureReason = ""
check.LastSuccessAt = timePtr(time.Now())
check.ConsecutiveFailures = 0
} else {
failureCount++
check.LastFailureAt = timePtr(time.Now())
check.ConsecutiveFailures++
check.FailureReason = result.Error
}
// Transition state based on consecutive failures and fingerprint match
newStatus, transitioned := check.TransitionStatus(result.Success, result.Fingerprint)
if transitioned {
transitionCount++
check.Status = newStatus
check.LastTransitionAt = timePtr(time.Now())
// Reset acknowledged on transition
check.Acknowledged = false
// Log transition
s.logger.Info("health check status transition",
"endpoint", check.Endpoint,
"old_status", string(oldStatus),
"new_status", string(newStatus))
// Record audit event
if s.auditService != nil {
_ = s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
"health_check_status_transition", "health_check", check.ID,
map[string]interface{}{
"endpoint": check.Endpoint,
"old_status": string(oldStatus),
"new_status": string(newStatus),
})
}
}
// Update health check record
if err := s.repo.Update(ctx, check); err != nil {
s.logger.Error("failed to update health check",
"endpoint", check.Endpoint,
"error", err)
continue
}
// Record probe result in history
if err := s.repo.RecordHistory(ctx, &domain.HealthHistoryEntry{
HealthCheckID: check.ID,
Status: string(check.Status),
ResponseTimeMs: check.ResponseTimeMs,
Fingerprint: check.ObservedFingerprint,
FailureReason: check.FailureReason,
CheckedAt: time.Now(),
}); err != nil {
s.logger.Warn("failed to record health check history",
"endpoint", check.Endpoint,
"error", err)
}
}
// Purge old history entries once per run
if err := s.PurgeOldHistory(ctx); err != nil {
s.logger.Warn("failed to purge old health check history", "error", err)
}
s.logger.Debug("health check run completed",
"total", len(checks),
"success", successCount,
"failure", failureCount,
"transitions", transitionCount)
return nil
}
// Create creates a new health check endpoint.
func (s *HealthCheckService) Create(ctx context.Context, check *domain.EndpointHealthCheck) error {
if check.ID == "" {
check.ID = generateID("hc")
}
check.CreatedAt = time.Now()
check.UpdatedAt = time.Now()
if err := s.repo.Create(ctx, check); err != nil {
return fmt.Errorf("failed to create health check: %w", err)
}
if s.auditService != nil {
_ = s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
"health_check_created", "health_check", check.ID,
map[string]interface{}{
"endpoint": check.Endpoint,
})
}
return nil
}
// Get retrieves a health check by ID.
func (s *HealthCheckService) Get(ctx context.Context, id string) (*domain.EndpointHealthCheck, error) {
return s.repo.Get(ctx, id)
}
// Update updates an existing health check.
func (s *HealthCheckService) Update(ctx context.Context, check *domain.EndpointHealthCheck) error {
check.UpdatedAt = time.Now()
if err := s.repo.Update(ctx, check); err != nil {
return fmt.Errorf("failed to update health check: %w", err)
}
if s.auditService != nil {
_ = s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
"health_check_updated", "health_check", check.ID,
map[string]interface{}{
"endpoint": check.Endpoint,
})
}
return nil
}
// Delete deletes a health check.
func (s *HealthCheckService) Delete(ctx context.Context, id string) error {
if err := s.repo.Delete(ctx, id); err != nil {
return fmt.Errorf("failed to delete health check: %w", err)
}
if s.auditService != nil {
_ = s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
"health_check_deleted", "health_check", id,
map[string]interface{}{})
}
return nil
}
// List lists health checks with optional filtering.
func (s *HealthCheckService) List(ctx context.Context, filter *repository.HealthCheckFilter) ([]*domain.EndpointHealthCheck, int, error) {
if filter == nil {
filter = &repository.HealthCheckFilter{}
}
return s.repo.List(ctx, filter)
}
// GetHistory retrieves health check history for an endpoint.
func (s *HealthCheckService) GetHistory(ctx context.Context, healthCheckID string, limit int) ([]*domain.HealthHistoryEntry, error) {
if limit <= 0 {
limit = 100
}
if limit > 1000 {
limit = 1000
}
return s.repo.GetHistory(ctx, healthCheckID, limit)
}
// AcknowledgeIncident marks a health check incident as acknowledged.
func (s *HealthCheckService) AcknowledgeIncident(ctx context.Context, id string, actor string) error {
check, err := s.repo.Get(ctx, id)
if err != nil {
return fmt.Errorf("failed to get health check: %w", err)
}
check.Acknowledged = true
check.AcknowledgedBy = actor
check.AcknowledgedAt = timePtr(time.Now())
if err := s.repo.Update(ctx, check); err != nil {
return fmt.Errorf("failed to update health check: %w", err)
}
if s.auditService != nil {
_ = s.auditService.RecordEvent(ctx, actor, domain.ActorTypeUser,
"health_check_acknowledged", "health_check", id,
map[string]interface{}{
"endpoint": check.Endpoint,
})
}
return nil
}
// GetSummary returns aggregated health check status counts.
func (s *HealthCheckService) GetSummary(ctx context.Context) (*domain.HealthCheckSummary, error) {
return s.repo.GetSummary(ctx)
}
// PurgeOldHistory removes health check history entries older than the retention period.
func (s *HealthCheckService) PurgeOldHistory(ctx context.Context) error {
cutoff := time.Now().Add(-s.historyRetention)
_, err := s.repo.PurgeHistory(ctx, cutoff)
return err
}
// Helper functions
func timePtr(t time.Time) *time.Time {
return &t
}
+350
View File
@@ -0,0 +1,350 @@
package service
import (
"context"
"errors"
"log/slog"
"os"
"testing"
"time"
"github.com/shankar0123/certctl/internal/domain"
"github.com/shankar0123/certctl/internal/repository"
)
// mockHealthCheckRepo implements the HealthCheckRepository interface for testing.
type mockHealthCheckRepo struct {
checks map[string]*domain.EndpointHealthCheck
history []*domain.HealthHistoryEntry
createErr error
getErr error
updateErr error
deleteErr error
listErr error
listDueErr error
getHistoryErr error
recordHistoryErr error
purgeHistoryErr error
getSummaryErr error
getSummaryResult *domain.HealthCheckSummary
}
func newMockHealthCheckRepo() *mockHealthCheckRepo {
return &mockHealthCheckRepo{
checks: make(map[string]*domain.EndpointHealthCheck),
history: []*domain.HealthHistoryEntry{},
getSummaryResult: &domain.HealthCheckSummary{
Healthy: 0,
Degraded: 0,
Down: 0,
CertMismatch: 0,
Unknown: 0,
},
}
}
func (m *mockHealthCheckRepo) Create(ctx context.Context, check *domain.EndpointHealthCheck) error {
if m.createErr != nil {
return m.createErr
}
m.checks[check.ID] = check
return nil
}
func (m *mockHealthCheckRepo) Get(ctx context.Context, id string) (*domain.EndpointHealthCheck, error) {
if m.getErr != nil {
return nil, m.getErr
}
if check, ok := m.checks[id]; ok {
return check, nil
}
return nil, errors.New("not found")
}
func (m *mockHealthCheckRepo) GetByEndpoint(ctx context.Context, endpoint string) (*domain.EndpointHealthCheck, error) {
for _, check := range m.checks {
if check.Endpoint == endpoint {
return check, nil
}
}
return nil, errors.New("not found")
}
func (m *mockHealthCheckRepo) Update(ctx context.Context, check *domain.EndpointHealthCheck) error {
if m.updateErr != nil {
return m.updateErr
}
m.checks[check.ID] = check
return nil
}
func (m *mockHealthCheckRepo) Delete(ctx context.Context, id string) error {
if m.deleteErr != nil {
return m.deleteErr
}
delete(m.checks, id)
return nil
}
func (m *mockHealthCheckRepo) List(ctx context.Context, filter *repository.HealthCheckFilter) ([]*domain.EndpointHealthCheck, int, error) {
if m.listErr != nil {
return nil, 0, m.listErr
}
checks := make([]*domain.EndpointHealthCheck, 0, len(m.checks))
for _, check := range m.checks {
checks = append(checks, check)
}
return checks, len(checks), nil
}
func (m *mockHealthCheckRepo) ListDueForCheck(ctx context.Context) ([]*domain.EndpointHealthCheck, error) {
if m.listDueErr != nil {
return nil, m.listDueErr
}
checks := make([]*domain.EndpointHealthCheck, 0, len(m.checks))
for _, check := range m.checks {
if check.Enabled {
checks = append(checks, check)
}
}
return checks, nil
}
func (m *mockHealthCheckRepo) GetHistory(ctx context.Context, healthCheckID string, limit int) ([]*domain.HealthHistoryEntry, error) {
if m.getHistoryErr != nil {
return nil, m.getHistoryErr
}
return m.history, nil
}
func (m *mockHealthCheckRepo) RecordHistory(ctx context.Context, entry *domain.HealthHistoryEntry) error {
if m.recordHistoryErr != nil {
return m.recordHistoryErr
}
m.history = append(m.history, entry)
return nil
}
func (m *mockHealthCheckRepo) PurgeHistory(ctx context.Context, before time.Time) (int64, error) {
if m.purgeHistoryErr != nil {
return 0, m.purgeHistoryErr
}
return 0, nil
}
func (m *mockHealthCheckRepo) GetSummary(ctx context.Context) (*domain.HealthCheckSummary, error) {
if m.getSummaryErr != nil {
return nil, m.getSummaryErr
}
return m.getSummaryResult, nil
}
// Tests
func newTestLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))
}
func TestHealthCheckService_Create_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
check := &domain.EndpointHealthCheck{
Endpoint: "example.com:443",
Status: domain.HealthStatusUnknown,
Enabled: true,
CheckIntervalSecs: 300,
}
err := svc.Create(context.Background(), check)
if err != nil {
t.Fatalf("Create failed: %v", err)
}
if check.ID == "" {
t.Fatal("Expected ID to be set")
}
retrieved, _ := repo.Get(context.Background(), check.ID)
if retrieved == nil {
t.Fatal("Expected check to be in repo")
}
if retrieved.Endpoint != "example.com:443" {
t.Errorf("Expected endpoint example.com:443, got %s", retrieved.Endpoint)
}
}
func TestHealthCheckService_Create_RepoError(t *testing.T) {
repo := newMockHealthCheckRepo()
repo.createErr = errors.New("db error")
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
check := &domain.EndpointHealthCheck{
Endpoint: "example.com:443",
Enabled: true,
}
err := svc.Create(context.Background(), check)
if err == nil {
t.Fatal("Expected error, got nil")
}
}
func TestHealthCheckService_Get_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
check := &domain.EndpointHealthCheck{
ID: "hc-test-1",
Endpoint: "example.com:443",
Status: domain.HealthStatusHealthy,
}
repo.checks["hc-test-1"] = check
retrieved, err := svc.Get(context.Background(), "hc-test-1")
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if retrieved.Endpoint != "example.com:443" {
t.Errorf("Expected endpoint example.com:443, got %s", retrieved.Endpoint)
}
}
func TestHealthCheckService_Get_NotFound(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
_, err := svc.Get(context.Background(), "nonexistent")
if err == nil {
t.Fatal("Expected error for nonexistent check")
}
}
func TestHealthCheckService_List_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
check1 := &domain.EndpointHealthCheck{
ID: "hc-1",
Endpoint: "api.example.com:443",
Status: domain.HealthStatusHealthy,
}
check2 := &domain.EndpointHealthCheck{
ID: "hc-2",
Endpoint: "web.example.com:443",
Status: domain.HealthStatusDegraded,
}
repo.checks["hc-1"] = check1
repo.checks["hc-2"] = check2
checks, total, err := svc.List(context.Background(), nil)
if err != nil {
t.Fatalf("List failed: %v", err)
}
if len(checks) != 2 {
t.Errorf("Expected 2 checks, got %d", len(checks))
}
if total != 2 {
t.Errorf("Expected total 2, got %d", total)
}
}
func TestHealthCheckService_Delete_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
check := &domain.EndpointHealthCheck{
ID: "hc-test-1",
Endpoint: "example.com:443",
}
repo.checks["hc-test-1"] = check
err := svc.Delete(context.Background(), "hc-test-1")
if err != nil {
t.Fatalf("Delete failed: %v", err)
}
if _, ok := repo.checks["hc-test-1"]; ok {
t.Fatal("Expected check to be deleted")
}
}
func TestHealthCheckService_AcknowledgeIncident_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
check := &domain.EndpointHealthCheck{
ID: "hc-test-1",
Endpoint: "example.com:443",
Status: domain.HealthStatusDown,
Acknowledged: false,
}
repo.checks["hc-test-1"] = check
err := svc.AcknowledgeIncident(context.Background(), "hc-test-1", "user@example.com")
if err != nil {
t.Fatalf("AcknowledgeIncident failed: %v", err)
}
retrieved := repo.checks["hc-test-1"]
if !retrieved.Acknowledged {
t.Fatal("Expected Acknowledged to be true")
}
if retrieved.AcknowledgedBy != "user@example.com" {
t.Errorf("Expected AcknowledgedBy to be user@example.com, got %s", retrieved.AcknowledgedBy)
}
if retrieved.AcknowledgedAt == nil {
t.Fatal("Expected AcknowledgedAt to be set")
}
}
func TestHealthCheckService_GetSummary_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
repo.getSummaryResult = &domain.HealthCheckSummary{
Healthy: 5,
Degraded: 2,
Down: 1,
CertMismatch: 1,
Unknown: 0,
}
summary, err := svc.GetSummary(context.Background())
if err != nil {
t.Fatalf("GetSummary failed: %v", err)
}
if summary.Healthy != 5 {
t.Errorf("Expected 5 healthy, got %d", summary.Healthy)
}
}
func TestHealthCheckService_RunHealthChecks_NoEndpoints(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
err := svc.RunHealthChecks(context.Background())
if err != nil {
t.Fatalf("RunHealthChecks failed: %v", err)
}
}
func TestHealthCheckService_PurgeOldHistory_Success(t *testing.T) {
repo := newMockHealthCheckRepo()
logger := newTestLogger()
svc := NewHealthCheckService(repo, nil, logger, 10, 5*time.Second, 30*24*time.Hour, false)
err := svc.PurgeOldHistory(context.Background())
if err != nil {
t.Fatalf("PurgeOldHistory failed: %v", err)
}
}
+5 -25
View File
@@ -2,9 +2,6 @@ package service
import (
"context"
"crypto/ecdsa"
"crypto/rsa"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/pem"
@@ -16,6 +13,7 @@ import (
"github.com/shankar0123/certctl/internal/domain"
"github.com/shankar0123/certctl/internal/repository"
"github.com/shankar0123/certctl/internal/tlsprobe"
)
// SentinelAgentID is the agent ID used for network-discovered certificates.
@@ -469,16 +467,15 @@ func (s *NetworkScanService) probeTLS(ctx context.Context, address string, timeo
// tlsCertToEntry converts an x509.Certificate from a TLS handshake into a DiscoveredCertEntry.
func tlsCertToEntry(cert *x509.Certificate, address string) domain.DiscoveredCertEntry {
// Compute SHA-256 fingerprint
fingerprintBytes := sha256.Sum256(cert.Raw)
fingerprint := fmt.Sprintf("%x", fingerprintBytes)
// Compute SHA-256 fingerprint using shared tlsprobe package
fingerprint := tlsprobe.CertFingerprint(cert)
// Encode as PEM
pemBlock := &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}
pemData := string(pem.EncodeToMemory(pemBlock))
// Key algorithm and size
keyAlg, keySize := tlsCertKeyInfo(cert)
// Key algorithm and size using shared tlsprobe package
keyAlg, keySize := tlsprobe.CertKeyInfo(cert)
return domain.DiscoveredCertEntry{
FingerprintSHA256: fingerprint,
@@ -497,20 +494,3 @@ func tlsCertToEntry(cert *x509.Certificate, address string) domain.DiscoveredCer
SourceFormat: "network",
}
}
// tlsCertKeyInfo extracts key algorithm name and size from a certificate.
func tlsCertKeyInfo(cert *x509.Certificate) (string, int) {
switch pub := cert.PublicKey.(type) {
case *rsa.PublicKey:
return "RSA", pub.N.BitLen()
case *ecdsa.PublicKey:
return "ECDSA", pub.Curve.Params().BitSize
default:
switch cert.PublicKeyAlgorithm {
case x509.Ed25519:
return "Ed25519", 256
default:
return cert.PublicKeyAlgorithm.String(), 0
}
}
}