v2.0.47: HTTPS Everywhere — TLS-only control plane, agents/CLI/MCP

Breaking change release. Plaintext HTTP listener removed. The certctl control plane now terminates TLS 1.3 on :8443 via http.Server.ListenAndServeTLS. No CERTCTL_TLS_ENABLED=false escape hatch. No dual-listener mode. One-step cutover per docs/upgrade-to-tls.md. Server - cmd/server/tls.go: certHolder with SIGHUP hot-reload + atomic cert swap, buildServerTLSConfig (TLS 1.3 min, GetCertificate callback), preflightServerTLS validation - cmd/server/main.go: ListenAndServeTLS in place of ListenAndServe, watchSIGHUP wiring, cert/key path config threading - tls_test.go: 418-line regression coverage of reload, preflight, callback behavior, SAN validation Config - CERTCTL_TLS_CERT_PATH / CERTCTL_TLS_KEY_PATH (required) - Plaintext rejection: agents/CLI/MCP pre-flight-fail on http:// URLs with a pointer to docs/upgrade-to-tls.md Agents, CLI, MCP - All three pre-flight-reject http:// URLs with fail-loud diagnostic - CERTCTL_SERVER_CA_BUNDLE_PATH for private-CA trust - CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY for dev-only bypass (loud warning on startup) - install-agent.sh emits both vars as commented template lines docker-compose - certctl-tls-init sidecar generates SAN-valid self-signed cert into deploy/test/certs/ on first boot - All demo-stack curls pin against ca.crt with --cacert Helm chart - Three TLS provisioning modes, exactly one required: - server.tls.existingSecret (operator-supplied) - server.tls.certManager.enabled (cert-manager integration) - server.tls.selfSigned.enabled (eval only — not for production) - server-certificate.yaml template for cert-manager mode - helm install without a TLS source fails at template render with a pointer to docs/tls.md CI - .github/workflows/ci.yml Helm Chart Validation step renders the chart in both existingSecret and cert-manager modes, plus an inverse guard-regression test that asserts helm template MUST refuse to render when no TLS source is configured. Previously the single `helm template` invocation hit the certctl.tls.required fail-loud guard and exit-1'd CI. Four invocations now: lint (existingSecret), template (existingSecret), template (cert-manager), template (no args — must fail). Integration tests - deploy/test/integration_test.go stands up the Compose stack over HTTPS, extracts the CA bundle, and exercises every certctl API over https://localhost:8443 - All 34 integration subtests green (per Phase 8 local CI-parity) Documentation - New: docs/tls.md (provisioning patterns, rotation, SIGHUP reload) - New: docs/upgrade-to-tls.md (one-step cutover, no-downgrade warnings, fleet-roll sequencing) - CHANGELOG.md: v2.2.0 "HTTPS Everywhere — The Irony" entry (file heading unchanged; release tag is v2.0.47) - All curls in docs/, examples/, deploy/helm/ guides use https://localhost:8443 --cacert Verification - grep -rn "ListenAndServe[^T]" cmd/ internal/ → 0 hits - grep -rn "\"http://" cmd/ internal/ → 2 benign hits (Caddy admin API default, SSRF doc comment) — zero certctl endpoints - Tasks #197–#206 (Phases 0–8) all closed in the tracker Files: 65 changed, 3489 insertions, 372 deletions (pre-CI-fix).
2026-06-11 21:38:52 +00:00 · 2026-04-20 03:31:05 +00:00
parent 04c7eca615
commit 52248be717
66 changed files with 3518 additions and 375 deletions
@@ -1,6 +1,7 @@
 package config

 import (
+	"crypto/tls"
 	"fmt"
 	"log/slog"
 	"os"
@@ -677,9 +678,30 @@ type VerificationConfig struct {

 // ServerConfig contains HTTP server configuration.
 type ServerConfig struct {
-	Host        string // Server host (default: 127.0.0.1). Set via CERTCTL_SERVER_HOST.
-	Port        int    // Server port (default: 8080). Set via CERTCTL_SERVER_PORT.
-	MaxBodySize int64  // Maximum request body size in bytes (default: 1MB). Set via CERTCTL_MAX_BODY_SIZE.
+	Host        string          // Server host (default: 127.0.0.1). Set via CERTCTL_SERVER_HOST.
+	Port        int             // Server port (default: 8080). Set via CERTCTL_SERVER_PORT.
+	MaxBodySize int64           // Maximum request body size in bytes (default: 1MB). Set via CERTCTL_MAX_BODY_SIZE.
+	TLS         ServerTLSConfig // HTTPS-only TLS configuration. Both CertPath and KeyPath are required.
+}
+
+// ServerTLSConfig holds the server-side TLS material.
+//
+// The control plane is HTTPS-only as of the HTTPS-everywhere milestone
+// (§3 locked decisions: no `http` mode, no dual-listener, TLS 1.3 only).
+// Both CertPath and KeyPath are required; an empty value causes
+// Config.Validate() to return a fail-loud error and the server refuses
+// to start. There is no plaintext HTTP fallback, no N-release migration
+// bridge, and no auto-generated self-signed cert — operators either
+// supply a cert on disk (docker-compose init container, operator-managed
+// file, cert-manager mount) or the process exits non-zero.
+type ServerTLSConfig struct {
+	// CertPath is the filesystem path to the server's PEM-encoded X.509
+	// certificate. Set via CERTCTL_SERVER_TLS_CERT_PATH. Required.
+	CertPath string
+
+	// KeyPath is the filesystem path to the server's PEM-encoded private
+	// key that signs CertPath. Set via CERTCTL_SERVER_TLS_KEY_PATH. Required.
+	KeyPath string
 }

 // DatabaseConfig contains database connection configuration.
@@ -841,6 +863,13 @@ func Load() (*Config, error) {
 			Host:        getEnv("CERTCTL_SERVER_HOST", "127.0.0.1"),
 			Port:        getEnvInt("CERTCTL_SERVER_PORT", 8080),
 			MaxBodySize: getEnvInt64("CERTCTL_MAX_BODY_SIZE", 1024*1024), // 1MB default
+			// HTTPS-everywhere milestone §2.1: both paths REQUIRED. Empty defaults
+			// are intentional so Validate() emits a fail-loud error pointing at
+			// docs/tls.md rather than silently binding plaintext HTTP.
+			TLS: ServerTLSConfig{
+				CertPath: getEnv("CERTCTL_SERVER_TLS_CERT_PATH", ""),
+				KeyPath:  getEnv("CERTCTL_SERVER_TLS_KEY_PATH", ""),
+			},
 		},
 		Database: DatabaseConfig{
 			URL:            getEnv("CERTCTL_DATABASE_URL", "postgres://localhost/certctl"),
@@ -1059,6 +1088,37 @@ func (c *Config) Validate() error {
 		return fmt.Errorf("invalid server port: %d", c.Server.Port)
 	}

+	// HTTPS-everywhere milestone §2.1 + §3 locked decisions: the control plane
+	// is TLS-only and refuses to start without a cert. No plaintext HTTP fallback,
+	// no auto-generated self-signed cert, no N-release migration window. An empty
+	// CertPath or KeyPath is operator-visible misconfiguration, not a soft warning.
+	if c.Server.TLS.CertPath == "" {
+		return fmt.Errorf("server TLS cert path is required — refuse to start (HTTPS-only: set CERTCTL_SERVER_TLS_CERT_PATH to a PEM-encoded certificate; see docs/tls.md)")
+	}
+	if c.Server.TLS.KeyPath == "" {
+		return fmt.Errorf("server TLS key path is required — refuse to start (HTTPS-only: set CERTCTL_SERVER_TLS_KEY_PATH to the PEM-encoded private key matching CERTCTL_SERVER_TLS_CERT_PATH; see docs/tls.md)")
+	}
+
+	// Files must exist and be readable. Catches typos and missing mount paths
+	// up-front so the operator gets a structured error on startup instead of
+	// a deferred ListenAndServeTLS failure after the scheduler has already
+	// fanned out its goroutines.
+	if _, err := os.Stat(c.Server.TLS.CertPath); err != nil {
+		return fmt.Errorf("server TLS cert file unreadable at %q: %w — refuse to start (HTTPS-only; see docs/tls.md)", c.Server.TLS.CertPath, err)
+	}
+	if _, err := os.Stat(c.Server.TLS.KeyPath); err != nil {
+		return fmt.Errorf("server TLS key file unreadable at %q: %w — refuse to start (HTTPS-only; see docs/tls.md)", c.Server.TLS.KeyPath, err)
+	}
+
+	// Parse the cert+key pair up-front. tls.LoadX509KeyPair verifies that the
+	// key signs the cert (prevents the classic footgun of shipping a pair
+	// whose private key doesn't match). Discard the returned Certificate — the
+	// server constructs its own holder from fresh reads so SIGHUP reload is
+	// authoritative.
+	if _, err := tls.LoadX509KeyPair(c.Server.TLS.CertPath, c.Server.TLS.KeyPath); err != nil {
+		return fmt.Errorf("server TLS cert/key pair invalid (cert=%q key=%q): %w — refuse to start (HTTPS-only; see docs/tls.md)", c.Server.TLS.CertPath, c.Server.TLS.KeyPath, err)
+	}
+
 	// Validate database configuration
 	if c.Database.URL == "" {
 		return fmt.Errorf("database URL is required")
@@ -1,10 +1,18 @@
 package config

 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
 	"log/slog"
+	"math/big"
 	"os"
-	"testing"
+	"path/filepath"
 	"strings"
+	"testing"
 	"time"
 )

@@ -26,10 +34,76 @@ func clearCertctlEnv(t *testing.T) {
 }

 // setMinimalValidEnv sets the minimum env vars needed for Load() to succeed (Validate passes).
+//
+// HTTPS-everywhere milestone (§2.1 + §3 locked decisions): the control plane
+// is TLS-only and Validate() refuses to pass without a readable cert/key pair
+// on disk. setMinimalValidEnv therefore materializes a throwaway ECDSA P-256
+// self-signed pair in t.TempDir() and points the two TLS env vars at it so
+// every Load-based test inherits a valid HTTPS posture without each caller
+// having to spell out cert generation. The temp dir is cleaned up by
+// testing.T at end-of-test.
 func setMinimalValidEnv(t *testing.T) {
 	t.Helper()
 	// api-key auth requires a secret
 	t.Setenv("CERTCTL_AUTH_SECRET", "test-secret-key")
+	// HTTPS-only control plane requires a real cert/key pair on disk.
+	certPath, keyPath := generateTestTLSPair(t)
+	t.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", certPath)
+	t.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", keyPath)
+}
+
+// generateTestTLSPair writes an ECDSA P-256 self-signed certificate + private
+// key pair to files inside t.TempDir() and returns the paths. Same shape used
+// by cmd/server/tls_test.go — this duplicates the generator rather than
+// importing it so the config package tests stay independent of cmd/server.
+func generateTestTLSPair(t *testing.T) (certPath, keyPath string) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("ecdsa.GenerateKey: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "certctl-config-test"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("x509.CreateCertificate: %v", err)
+	}
+	dir := t.TempDir()
+	certPath = filepath.Join(dir, "cert.pem")
+	keyPath = filepath.Join(dir, "key.pem")
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	if err := os.WriteFile(certPath, certPEM, 0o600); err != nil {
+		t.Fatalf("write cert: %v", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		t.Fatalf("x509.MarshalECPrivateKey: %v", err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		t.Fatalf("write key: %v", err)
+	}
+	return certPath, keyPath
+}
+
+// validServerConfig returns a ServerConfig with Port=8080 plus a freshly
+// minted TLS cert/key pair on disk, so Validate() passes the HTTPS-only
+// preflight (cert empty → stat → tls.LoadX509KeyPair round-trip). Every
+// struct-based Validate test uses this so they fail for the reason they
+// claim to test, not for a missing TLS pair.
+func validServerConfig(t *testing.T) ServerConfig {
+	t.Helper()
+	certPath, keyPath := generateTestTLSPair(t)
+	return ServerConfig{
+		Port: 8080,
+		TLS:  ServerTLSConfig{CertPath: certPath, KeyPath: keyPath},
+	}
 }

 func TestLoad_DefaultValues(t *testing.T) {
@@ -135,6 +209,13 @@ func TestLoad_DefaultValues(t *testing.T) {
 func TestLoad_AllEnvVarsSet(t *testing.T) {
 	clearCertctlEnv(t)

+	// HTTPS-only control plane: Load() → Validate() refuses an empty cert path.
+	// Materialize a throwaway ECDSA P-256 pair and point the two TLS env vars
+	// at it before setting every other CERTCTL_* var this test cares about.
+	certPath, keyPath := generateTestTLSPair(t)
+	t.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", certPath)
+	t.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", keyPath)
+
 	t.Setenv("CERTCTL_SERVER_HOST", "0.0.0.0")
 	t.Setenv("CERTCTL_SERVER_PORT", "9090")
 	t.Setenv("CERTCTL_MAX_BODY_SIZE", "2097152")
@@ -319,7 +400,7 @@ func TestLoad_CommaSeparatedList(t *testing.T) {

 func TestValidate_ValidConfig(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "api-key", Secret: "test-secret"},
@@ -329,6 +410,7 @@ func TestValidate_ValidConfig(t *testing.T) {
 			JobProcessorInterval:        30 * time.Second,
 			AgentHealthCheckInterval:    2 * time.Minute,
 			NotificationProcessInterval: 1 * time.Minute,
+			NotificationRetryInterval:   2 * time.Minute,
 			RetryInterval:               5 * time.Minute,
 			JobTimeoutInterval:          10 * time.Minute,
 			AwaitingCSRTimeout:          24 * time.Hour,
@@ -342,7 +424,7 @@ func TestValidate_ValidConfig(t *testing.T) {

 func TestValidate_AuthTypeNone(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "none", Secret: ""},
@@ -352,6 +434,7 @@ func TestValidate_AuthTypeNone(t *testing.T) {
 			JobProcessorInterval:        30 * time.Second,
 			AgentHealthCheckInterval:    2 * time.Minute,
 			NotificationProcessInterval: 1 * time.Minute,
+			NotificationRetryInterval:   2 * time.Minute,
 			RetryInterval:               5 * time.Minute,
 			JobTimeoutInterval:          10 * time.Minute,
 			AwaitingCSRTimeout:          24 * time.Hour,
@@ -365,7 +448,7 @@ func TestValidate_AuthTypeNone(t *testing.T) {

 func TestValidate_InvalidAuthType(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "oauth", Secret: "key"},
@@ -384,7 +467,7 @@ func TestValidate_InvalidAuthType(t *testing.T) {

 func TestValidate_APIKeyAuth_MissingSecret(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "api-key", Secret: ""},
@@ -403,7 +486,7 @@ func TestValidate_APIKeyAuth_MissingSecret(t *testing.T) {

 func TestValidate_JWTAuth_MissingSecret(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "jwt", Secret: ""},
@@ -422,7 +505,7 @@ func TestValidate_JWTAuth_MissingSecret(t *testing.T) {

 func TestValidate_InvalidKeygenMode(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
@@ -470,9 +553,168 @@ func TestValidate_InvalidPort(t *testing.T) {
 	}
 }

+// TestValidate_TLSCertPathEmpty pins the first of the HTTPS-only fail-loud
+// gates in Validate(): an empty CertPath must produce the operator-facing
+// "server TLS cert path is required" error. Per §2.1 + §3 locked decisions,
+// there is no plaintext HTTP fallback — missing TLS config is a hard startup
+// refusal, not a warning.
+func TestValidate_TLSCertPathEmpty(t *testing.T) {
+	_, keyPath := generateTestTLSPair(t)
+	cfg := &Config{
+		Server: ServerConfig{
+			Port: 8080,
+			TLS:  ServerTLSConfig{CertPath: "", KeyPath: keyPath},
+		},
+		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
+		Log:      LogConfig{Level: "info", Format: "json"},
+		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
+		Keygen:   KeygenConfig{Mode: "agent"},
+		Scheduler: SchedulerConfig{
+			RenewalCheckInterval:        1 * time.Hour,
+			JobProcessorInterval:        30 * time.Second,
+			AgentHealthCheckInterval:    2 * time.Minute,
+			NotificationProcessInterval: 1 * time.Minute,
+		},
+	}
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatal("Validate() should return error for empty TLS cert path")
+	}
+	if !strings.Contains(err.Error(), "server TLS cert path is required") {
+		t.Errorf("error = %q, want substring %q", err.Error(), "server TLS cert path is required")
+	}
+}
+
+// TestValidate_TLSKeyPathEmpty pins the second HTTPS-only gate: empty KeyPath
+// must produce the "server TLS key path is required" error. Runs with a valid
+// CertPath so the cert-empty gate (which fires first) is cleanly bypassed —
+// proves the key-empty gate is actually reached.
+func TestValidate_TLSKeyPathEmpty(t *testing.T) {
+	certPath, _ := generateTestTLSPair(t)
+	cfg := &Config{
+		Server: ServerConfig{
+			Port: 8080,
+			TLS:  ServerTLSConfig{CertPath: certPath, KeyPath: ""},
+		},
+		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
+		Log:      LogConfig{Level: "info", Format: "json"},
+		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
+		Keygen:   KeygenConfig{Mode: "agent"},
+		Scheduler: SchedulerConfig{
+			RenewalCheckInterval:        1 * time.Hour,
+			JobProcessorInterval:        30 * time.Second,
+			AgentHealthCheckInterval:    2 * time.Minute,
+			NotificationProcessInterval: 1 * time.Minute,
+		},
+	}
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatal("Validate() should return error for empty TLS key path")
+	}
+	if !strings.Contains(err.Error(), "server TLS key path is required") {
+		t.Errorf("error = %q, want substring %q", err.Error(), "server TLS key path is required")
+	}
+}
+
+// TestValidate_TLSCertFileMissing pins the os.Stat gate on the cert path. A
+// non-existent path must surface "server TLS cert file unreadable" so the
+// operator sees the bad path in the error (file=%q) instead of a deferred
+// ListenAndServeTLS panic after the scheduler has already fanned out.
+func TestValidate_TLSCertFileMissing(t *testing.T) {
+	_, keyPath := generateTestTLSPair(t)
+	missingCert := filepath.Join(t.TempDir(), "does-not-exist.pem")
+	cfg := &Config{
+		Server: ServerConfig{
+			Port: 8080,
+			TLS:  ServerTLSConfig{CertPath: missingCert, KeyPath: keyPath},
+		},
+		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
+		Log:      LogConfig{Level: "info", Format: "json"},
+		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
+		Keygen:   KeygenConfig{Mode: "agent"},
+		Scheduler: SchedulerConfig{
+			RenewalCheckInterval:        1 * time.Hour,
+			JobProcessorInterval:        30 * time.Second,
+			AgentHealthCheckInterval:    2 * time.Minute,
+			NotificationProcessInterval: 1 * time.Minute,
+		},
+	}
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatal("Validate() should return error for missing TLS cert file")
+	}
+	if !strings.Contains(err.Error(), "server TLS cert file unreadable") {
+		t.Errorf("error = %q, want substring %q", err.Error(), "server TLS cert file unreadable")
+	}
+}
+
+// TestValidate_TLSKeyFileMissing pins the os.Stat gate on the key path. Uses a
+// valid CertPath so the cert-missing gate does not pre-empt; proves the key
+// gate is reached and reports the bad key path.
+func TestValidate_TLSKeyFileMissing(t *testing.T) {
+	certPath, _ := generateTestTLSPair(t)
+	missingKey := filepath.Join(t.TempDir(), "does-not-exist.key")
+	cfg := &Config{
+		Server: ServerConfig{
+			Port: 8080,
+			TLS:  ServerTLSConfig{CertPath: certPath, KeyPath: missingKey},
+		},
+		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
+		Log:      LogConfig{Level: "info", Format: "json"},
+		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
+		Keygen:   KeygenConfig{Mode: "agent"},
+		Scheduler: SchedulerConfig{
+			RenewalCheckInterval:        1 * time.Hour,
+			JobProcessorInterval:        30 * time.Second,
+			AgentHealthCheckInterval:    2 * time.Minute,
+			NotificationProcessInterval: 1 * time.Minute,
+		},
+	}
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatal("Validate() should return error for missing TLS key file")
+	}
+	if !strings.Contains(err.Error(), "server TLS key file unreadable") {
+		t.Errorf("error = %q, want substring %q", err.Error(), "server TLS key file unreadable")
+	}
+}
+
+// TestValidate_TLSMismatchedPair pins the tls.LoadX509KeyPair gate — the
+// classic "you shipped the wrong private key" footgun. Generates two
+// independent ECDSA pairs and crosses them (pair1 cert + pair2 key). Both
+// files exist and parse as PEM, so os.Stat passes; only the cryptographic
+// round-trip inside LoadX509KeyPair catches the mismatch.
+func TestValidate_TLSMismatchedPair(t *testing.T) {
+	certPath1, _ := generateTestTLSPair(t)
+	_, keyPath2 := generateTestTLSPair(t)
+	cfg := &Config{
+		Server: ServerConfig{
+			Port: 8080,
+			TLS:  ServerTLSConfig{CertPath: certPath1, KeyPath: keyPath2},
+		},
+		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
+		Log:      LogConfig{Level: "info", Format: "json"},
+		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
+		Keygen:   KeygenConfig{Mode: "agent"},
+		Scheduler: SchedulerConfig{
+			RenewalCheckInterval:        1 * time.Hour,
+			JobProcessorInterval:        30 * time.Second,
+			AgentHealthCheckInterval:    2 * time.Minute,
+			NotificationProcessInterval: 1 * time.Minute,
+		},
+	}
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatal("Validate() should return error for mismatched TLS cert/key pair")
+	}
+	if !strings.Contains(err.Error(), "server TLS cert/key pair invalid") {
+		t.Errorf("error = %q, want substring %q", err.Error(), "server TLS cert/key pair invalid")
+	}
+}
+
 func TestValidate_EmptyDatabaseURL(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
@@ -491,7 +733,7 @@ func TestValidate_EmptyDatabaseURL(t *testing.T) {

 func TestValidate_InvalidLogLevel(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "verbose", Format: "json"},
 		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
@@ -510,7 +752,7 @@ func TestValidate_InvalidLogLevel(t *testing.T) {

 func TestValidate_InvalidLogFormat(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 		Log:      LogConfig{Level: "info", Format: "yaml"},
 		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
@@ -572,7 +814,7 @@ func TestValidate_SchedulerIntervalTooSmall(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := &Config{
-				Server:    ServerConfig{Port: 8080},
+				Server:    validServerConfig(t),
 				Database:  DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 				Log:       LogConfig{Level: "info", Format: "json"},
 				Auth:      AuthConfig{Type: "api-key", Secret: "key"},
@@ -588,7 +830,7 @@ func TestValidate_SchedulerIntervalTooSmall(t *testing.T) {

 func TestValidate_DatabaseMaxConnectionsZero(t *testing.T) {
 	cfg := &Config{
-		Server:   ServerConfig{Port: 8080},
+		Server:   validServerConfig(t),
 		Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 0},
 		Log:      LogConfig{Level: "info", Format: "json"},
 		Auth:     AuthConfig{Type: "api-key", Secret: "key"},
@@ -795,7 +1037,7 @@ func TestConfig_Scheduler_JobTimeoutValidation(t *testing.T) {
 			// Start from a fully valid config so the I-003 timeout checks
 			// are the only potential failure point.
 			cfg := &Config{
-				Server:   ServerConfig{Port: 8080},
+				Server:   validServerConfig(t),
 				Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
 				Log:      LogConfig{Level: "info", Format: "json"},
 				Auth:     AuthConfig{Type: "api-key", Secret: "test-secret"},
@@ -805,6 +1047,7 @@ func TestConfig_Scheduler_JobTimeoutValidation(t *testing.T) {
 					JobProcessorInterval:        1 * time.Minute,
 					AgentHealthCheckInterval:    1 * time.Minute,
 					NotificationProcessInterval: 1 * time.Minute,
+					NotificationRetryInterval:   2 * time.Minute,
 					RetryInterval:               1 * time.Minute,
 					JobTimeoutInterval:          10 * time.Minute,
 					AwaitingCSRTimeout:          24 * time.Hour,