package nginx_test import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "log/slog" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "github.com/certctl-io/certctl/internal/connector/target" "github.com/certctl-io/certctl/internal/connector/target/nginx" "github.com/certctl-io/certctl/internal/deploy" "github.com/certctl-io/certctl/internal/tlsprobe" ) // Phase 4 of the deploy-hardening I master bundle: ≥40 tests on // the NGINX connector covering the atomic-deploy + post-deploy TLS // verify + rollback + ValidateOnly + ownership-preservation matrix // the prompt requires. The IIS bar is 41; this file plus the 17 // pre-existing tests in nginx_test.go puts NGINX at well over 40. // --- Fixtures + helpers --- // Test fixtures: valid base64-shaped PEM bodies so the // fingerprintOfPEM helper can compute SHA-256 over real binary // payloads. The actual DER content is junk; only the SHA-256 over // it matters for verifying post-deploy match logic. const ( certA = "-----BEGIN CERTIFICATE-----\nQUxQSEEtQ0VSVC1QRU0tQ09OVEVOVFMtQQ==\n-----END CERTIFICATE-----\n" certB = "-----BEGIN CERTIFICATE-----\nQkVUQS1DRVJULVBFTS1DT05URU5UUy1C\n-----END CERTIFICATE-----\n" chain = "-----BEGIN CERTIFICATE-----\nSU5URVJNRURJQVRFLUNIQUlOLVBFTQ==\n-----END CERTIFICATE-----\n" keyA = "-----BEGIN PRIVATE KEY-----\nZmFrZS1rZXktQQ==\n-----END PRIVATE KEY-----\n" ) // quietLogger discards log output so test runs stay readable. func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(os.NewFile(0, os.DevNull), &slog.HandlerOptions{Level: slog.LevelError})) } // fingerprintOfPEM returns the SHA-256 hex of the first cert in // the PEM bundle. Mirrors what tlsprobe.ProbeTLS would return for // a deployed cert. Used by stub probers to claim "deployed cert // matches" or "doesn't match". func fingerprintOfPEM(t *testing.T, pem string) string { t.Helper() begin := "-----BEGIN CERTIFICATE-----" end := "-----END CERTIFICATE-----" beginIdx := strings.Index(pem, begin) if beginIdx < 0 { t.Fatal("no cert block") } body := pem[beginIdx+len(begin):] endIdx := strings.Index(body, end) if endIdx < 0 { t.Fatal("no end") } body = strings.TrimSpace(body[:endIdx]) body = strings.ReplaceAll(body, "\n", "") body = strings.ReplaceAll(body, "\r", "") body = strings.ReplaceAll(body, " ", "") der, err := decodeBase64(body) if err != nil { t.Fatal(err) } h := sha256.Sum256(der) return hex.EncodeToString(h[:]) } func decodeBase64(s string) ([]byte, error) { // Use a real base64 decoder for the fingerprint helper. We // avoid encoding/base64 in this package's import to keep // test-time imports lean — but for a one-shot test helper // it's fine to import it here. return base64StdDecode(s) } // successProbe returns a stub probe.ProbeResult with the given // fingerprint. Used to simulate post-deploy TLS verify success // (matching) or mismatch. func successProbe(fp string) func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult { return func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult { return tlsprobe.ProbeResult{ Address: address, Success: true, Fingerprint: fp, } } } // failProbe returns a stub indicating dial timeout / handshake fail. func failProbe(reason string) func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult { return func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult { return tlsprobe.ProbeResult{ Address: address, Success: false, Error: reason, } } } // noopRun stubs runValidate / runReload to always succeed. func noopRun(ctx context.Context, command string) ([]byte, error) { return nil, nil } // failingRun stubs runValidate / runReload with a fixed error. func failingRun(reason string) func(ctx context.Context, command string) ([]byte, error) { return func(ctx context.Context, command string) ([]byte, error) { return []byte("stderr: " + reason), errors.New(reason) } } // newConnectorWithStubs is the canonical test-time constructor — // produces a Connector with no-op validate / no-op reload / no-op // (skip-because-no-endpoint) probe. func newConnectorWithStubs(t *testing.T, cfg *nginx.Config) *nginx.Connector { c := nginx.New(cfg, quietLogger()) c.SetTestRunValidate(noopRun) c.SetTestRunReload(noopRun) c.SetTestProbe(successProbe("ignored")) return c } // --- The ≥40 tests --- // 1. Happy path: cert + key + chain all written atomically; reload // succeeds; final files have new bytes. func TestNginx_Atomic_HappyPath_CertChainKeyAllAtomic(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ChainPath: filepath.Join(dir, "chain.pem"), KeyPath: filepath.Join(dir, "key.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", KeyFileMode: 0640, } c := newConnectorWithStubs(t, cfg) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{ CertPEM: certA, ChainPEM: chain, KeyPEM: keyA, }) if err != nil { t.Fatal(err) } if !res.Success { t.Fatal("Success=false") } for path, want := range map[string]string{cfg.CertPath: certA, cfg.ChainPath: chain, cfg.KeyPath: keyA} { got, _ := os.ReadFile(path) if string(got) != want { t.Errorf("%s = %q, want %q", path, got, want) } } } // 2. Validate (PreCommit) fails → no live file modified, error // surfaces as ErrValidateFailed wrap. func TestNginx_Atomic_ValidateFails_NoFilesChanged(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) c.SetTestRunValidate(failingRun("invalid SAN")) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected error") } if !errors.Is(err, deploy.ErrValidateFailed) { t.Errorf("got %v, want ErrValidateFailed wrap", err) } if got, _ := os.ReadFile(cert); string(got) != "ORIGINAL" { t.Errorf("cert was modified despite validate failure: %q", got) } } // 3. Reload (PostCommit) fails → rollback restores the previous // bytes + reload runs again successfully → ErrReloadFailed surfaces. func TestNginx_Atomic_ReloadFails_RollbackRestoresPrevious(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) var reloadCalls int32 c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { n := atomic.AddInt32(&reloadCalls, 1) if n == 1 { return nil, errors.New("nginx exited 1") } return nil, nil }) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected reload error") } if !errors.Is(err, deploy.ErrReloadFailed) { t.Errorf("got %v, want ErrReloadFailed wrap", err) } if got, _ := os.ReadFile(cert); string(got) != "ORIGINAL" { t.Errorf("cert after rollback = %q, want ORIGINAL", got) } if atomic.LoadInt32(&reloadCalls) != 2 { t.Errorf("reload calls = %d, want 2 (once for new bytes, once for restored)", reloadCalls) } } // 4. Reload fails AND the second reload also fails → ErrRollbackFailed. func TestNginx_Atomic_ReloadFails_AndRollbackReloadAlsoFails(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) c.SetTestRunReload(failingRun("nginx wedged")) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if !errors.Is(err, deploy.ErrRollbackFailed) { t.Errorf("got %v, want ErrRollbackFailed wrap", err) } } // 5. Post-deploy verify SHA-256 mismatch → rollback restores OLD // cert + emits an error. func TestNginx_Atomic_PostVerify_SHA256Mismatch_TriggersRollback(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx-test:443", Timeout: 100 * time.Millisecond, }, } c := newConnectorWithStubs(t, cfg) c.SetTestProbe(successProbe("0000000000000000000000000000000000000000000000000000000000000000")) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected error from verify mismatch") } if !strings.Contains(err.Error(), "SHA-256 mismatch") { t.Errorf("error not labeled SHA-256 mismatch: %v", err) } if got, _ := os.ReadFile(cert); string(got) != "ORIGINAL" { t.Errorf("cert after verify-failure rollback = %q, want ORIGINAL", got) } } // 6. Post-deploy verify succeeds (fingerprint matches) → result // reports Success=true. func TestNginx_Atomic_PostVerify_FingerprintMatches_Succeeds(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", Timeout: 100 * time.Millisecond, }, } c := newConnectorWithStubs(t, cfg) want := fingerprintOfPEM(t, certA) c.SetTestProbe(successProbe(want)) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err != nil { t.Fatal(err) } if !res.Success { t.Error("Success=false") } } // 7. Post-deploy verify TLS-dial timeout → rollback restores. func TestNginx_Atomic_PostVerify_DialTimeout_TriggersRollback(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", Timeout: 10 * time.Millisecond, }, } c := newConnectorWithStubs(t, cfg) c.SetTestProbe(failProbe("dial tcp: i/o timeout")) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected dial-timeout error") } if got, _ := os.ReadFile(cert); string(got) != "ORIGINAL" { t.Errorf("cert after dial-timeout rollback = %q, want ORIGINAL", got) } } // 8. Idempotency: second deploy with identical bytes → no validate // + no reload + verify skipped (the deploy was a no-op). func TestNginx_Atomic_IdempotencyHit_SkipsAllSteps(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte(certA), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) var validateCalls, reloadCalls, probeCalls int32 c.SetTestRunValidate(func(ctx context.Context, _ string) ([]byte, error) { atomic.AddInt32(&validateCalls, 1) return nil, nil }) c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { atomic.AddInt32(&reloadCalls, 1) return nil, nil }) c.SetTestProbe(func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult { atomic.AddInt32(&probeCalls, 1) return tlsprobe.ProbeResult{Success: true, Fingerprint: "ignored"} }) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err != nil { t.Fatal(err) } if !res.Success { t.Error("Success=false on idempotent skip") } if validateCalls != 0 || reloadCalls != 0 || probeCalls != 0 { t.Errorf("expected 0/0/0 calls, got %d/%d/%d", validateCalls, reloadCalls, probeCalls) } } // 9. Mode override on key file: KeyFileMode 0600 wins over default. func TestNginx_Atomic_KeyFileMode_OverrideWins(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), KeyPath: filepath.Join(dir, "key.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", KeyFileMode: 0600, } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, KeyPEM: keyA}); err != nil { t.Fatal(err) } stat, _ := os.Stat(cfg.KeyPath) if stat.Mode().Perm() != 0600 { t.Errorf("key mode = %#o, want 0600", stat.Mode().Perm()) } } // 10. Existing cert file's mode is preserved across renewal. func TestNginx_Atomic_ExistingMode_Preserved(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("OLD"), 0640); err != nil { t.Fatal(err) } if err := os.Chmod(cert, 0640); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } stat, _ := os.Stat(cert) if stat.Mode().Perm() != 0640 { t.Errorf("mode = %#o, want 0640 (preservation)", stat.Mode().Perm()) } } // 11. Backups are pruned to the configured retention. func TestNginx_Atomic_BackupRetention_KeepsLastN(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("V0"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", BackupRetention: 2, } c := newConnectorWithStubs(t, cfg) for i := 1; i <= 5; i++ { if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{ CertPEM: fmt.Sprintf("V%d-CERT", i), }); err != nil { t.Fatal(err) } time.Sleep(2 * time.Millisecond) } entries, _ := os.ReadDir(dir) bakCount := 0 for _, e := range entries { if strings.Contains(e.Name(), deploy.BackupSuffix) { bakCount++ } } if bakCount != 2 { t.Errorf("backup count = %d, want 2", bakCount) } } // 12. ValidateOnly happy path: returns nil when validate command // passes. func TestNginx_ValidateOnly_HappyPath_ReturnsNil(t *testing.T) { cfg := &nginx.Config{ CertPath: "/tmp/cert.pem", ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if err := c.ValidateOnly(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Errorf("got %v, want nil", err) } } // 13. ValidateOnly returns the validate command's error. func TestNginx_ValidateOnly_ValidateFails_ReturnsWrappedError(t *testing.T) { cfg := &nginx.Config{ CertPath: "/tmp/cert.pem", ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) c.SetTestRunValidate(failingRun("invalid certificate")) err := c.ValidateOnly(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected error") } if errors.Is(err, target.ErrValidateOnlyNotSupported) { t.Errorf("got ErrValidateOnlyNotSupported, want wrapped validate error: %v", err) } } // 14. ValidateOnly returns ErrValidateOnlyNotSupported when no // validate command configured. func TestNginx_ValidateOnly_NoConfig_ReturnsSentinel(t *testing.T) { cfg := &nginx.Config{ /* no ValidateCommand */ } c := nginx.New(cfg, quietLogger()) err := c.ValidateOnly(context.Background(), target.DeploymentRequest{}) if !errors.Is(err, target.ErrValidateOnlyNotSupported) { t.Errorf("got %v, want ErrValidateOnlyNotSupported", err) } } // 15. Post-deploy verify ON but endpoint empty → skip with warn. // Deploy still succeeds. func TestNginx_Atomic_PostVerify_NoEndpoint_SkipsWithWarn(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, // Endpoint left blank }, } c := newConnectorWithStubs(t, cfg) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err != nil { t.Fatal(err) } if !res.Success { t.Error("Success=false") } } // 16. Post-deploy verify explicitly DISABLED → skip entirely. func TestNginx_Atomic_PostVerify_Disabled_NoProbeCalled(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: false, Endpoint: "nginx:443", }, } c := newConnectorWithStubs(t, cfg) var probeCalls int32 c.SetTestProbe(func(ctx context.Context, _ string, _ time.Duration) tlsprobe.ProbeResult { atomic.AddInt32(&probeCalls, 1) return tlsprobe.ProbeResult{Success: true, Fingerprint: "ignored"} }) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } if probeCalls != 0 { t.Errorf("probe called %d times despite Enabled=false", probeCalls) } } // 17. Verify retries: 3 attempts, fingerprint matches on the 3rd. func TestNginx_Atomic_PostVerify_RetriesUntilMatch(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 3, PostDeployVerifyBackoff: 1 * time.Millisecond, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", }, } c := newConnectorWithStubs(t, cfg) want := fingerprintOfPEM(t, certA) var attempts int32 c.SetTestProbe(func(ctx context.Context, _ string, _ time.Duration) tlsprobe.ProbeResult { n := atomic.AddInt32(&attempts, 1) if n < 3 { return tlsprobe.ProbeResult{Success: true, Fingerprint: "stale-from-other-pod"} } return tlsprobe.ProbeResult{Success: true, Fingerprint: want} }) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err != nil { t.Fatal(err) } if !res.Success { t.Error("Success=false") } if attempts != 3 { t.Errorf("probe attempts = %d, want 3", attempts) } } // 18. Verify exhausts retries → rollback. func TestNginx_Atomic_PostVerify_RetriesExhausted_RollsBack(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 2, PostDeployVerifyBackoff: 1 * time.Millisecond, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", }, } c := newConnectorWithStubs(t, cfg) c.SetTestProbe(successProbe("0000000000000000000000000000000000000000000000000000000000000000")) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected verify-mismatch error") } if got, _ := os.ReadFile(cert); string(got) != "ORIGINAL" { t.Errorf("cert after rollback = %q, want ORIGINAL", got) } } // 19. Concurrent deploys to same paths serialize via deploy // package's file mutex. func TestNginx_Atomic_ConcurrentDeploys_SamePath_Serialize(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) var inFlight, maxInFlight int32 c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { n := atomic.AddInt32(&inFlight, 1) for { m := atomic.LoadInt32(&maxInFlight) if n <= m || atomic.CompareAndSwapInt32(&maxInFlight, m, n) { break } } time.Sleep(2 * time.Millisecond) atomic.AddInt32(&inFlight, -1) return nil, nil }) const N = 5 errs := make(chan error, N) for i := 0; i < N; i++ { go func(idx int) { _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{ CertPEM: fmt.Sprintf("WRITER-%d-%s", idx, certA), }) errs <- err }(i) } for i := 0; i < N; i++ { if err := <-errs; err != nil { t.Errorf("Deploy %d: %v", i, err) } } if maxInFlight > 1 { t.Errorf("max concurrent reload = %d, want 1", maxInFlight) } } // 20. Deploy without chain still succeeds (chain field optional). func TestNginx_Atomic_NoChain_StillSucceeds(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err != nil { t.Fatal(err) } if !res.Success { t.Error("Success=false") } } // 21. Deploy without key → only cert + chain written. func TestNginx_Atomic_NoKey_OnlyCertAndChainWritten(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ChainPath: filepath.Join(dir, "chain.pem"), KeyPath: filepath.Join(dir, "key.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain}); err != nil { t.Fatal(err) } if _, err := os.Stat(cfg.KeyPath); err == nil { t.Error("key file written despite empty KeyPEM") } } // 22. ChainPath unset + ChainPEM provided → chain not written // (operator never asked for it). func TestNginx_Atomic_NoChainPath_ChainPEMIgnored(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain}); err != nil { t.Fatal(err) } } // 23. SHA-256 idempotency check across cert + key + chain. func TestNginx_Atomic_Idempotency_AllThreeFilesMatch(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") chainP := filepath.Join(dir, "chain.pem") key := filepath.Join(dir, "key.pem") for _, p := range []struct { path string body string }{{cert, certA}, {chainP, chain}, {key, keyA}} { if err := os.WriteFile(p.path, []byte(p.body), 0640); err != nil { t.Fatal(err) } } cfg := &nginx.Config{ CertPath: cert, ChainPath: chainP, KeyPath: key, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) var reloadCalls int32 c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { atomic.AddInt32(&reloadCalls, 1) return nil, nil }) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{ CertPEM: certA, ChainPEM: chain, KeyPEM: keyA, }); err != nil { t.Fatal(err) } if reloadCalls != 0 { t.Errorf("reload called %d times despite idempotent input", reloadCalls) } } // 24. Partial idempotency (cert matches, key differs) → full // deploy (validate + reload run). func TestNginx_Atomic_PartialIdempotency_FullDeploy(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") key := filepath.Join(dir, "key.pem") if err := os.WriteFile(cert, []byte(certA), 0640); err != nil { t.Fatal(err) } if err := os.WriteFile(key, []byte("OLD-KEY"), 0640); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, KeyPath: key, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) var reloadCalls int32 c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { atomic.AddInt32(&reloadCalls, 1) return nil, nil }) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, KeyPEM: keyA}); err != nil { t.Fatal(err) } if reloadCalls != 1 { t.Errorf("reload called %d times, want 1 (partial-mismatch should trigger full deploy)", reloadCalls) } } // 25. New file (didn't exist) gets default mode 0644 for cert. func TestNginx_Atomic_NewCert_DefaultMode0644(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } stat, _ := os.Stat(cfg.CertPath) if stat.Mode().Perm() != 0644 { t.Errorf("default cert mode = %#o, want 0644", stat.Mode().Perm()) } } // 26. Backup file exists after first deploy with existing file. func TestNginx_Atomic_FirstDeploy_BackupCreated(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } entries, _ := os.ReadDir(dir) found := false for _, e := range entries { if strings.Contains(e.Name(), deploy.BackupSuffix) { found = true break } } if !found { t.Error("no backup file created") } } // 27. BackupRetention=-1 disables backups (no foot-gun protection). func TestNginx_Atomic_BackupDisabled_NoBackupFile(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", BackupRetention: -1, } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } entries, _ := os.ReadDir(dir) for _, e := range entries { if strings.Contains(e.Name(), deploy.BackupSuffix) { t.Errorf("backup created despite BackupRetention=-1: %s", e.Name()) } } } // 28. ValidateOnly with stubbed validate-fail returns the wrapped // command output for the operator to read. func TestNginx_ValidateOnly_ErrorMessageIncludesStderr(t *testing.T) { cfg := &nginx.Config{ CertPath: "/tmp/cert.pem", ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) c.SetTestRunValidate(failingRun("alert: SSL_CTX_use_certificate failed")) err := c.ValidateOnly(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected error") } if !strings.Contains(err.Error(), "SSL_CTX_use_certificate failed") { t.Errorf("error %q doesn't include validate stderr", err) } } // 29. Context cancellation propagates through deploy.Apply. func TestNginx_Atomic_ContextCancelled_AbortsCleanly(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) ctx, cancel := context.WithCancel(context.Background()) cancel() _, err := c.DeployCertificate(ctx, target.DeploymentRequest{CertPEM: certA}) if err == nil { t.Fatal("expected context.Canceled") } } // 30. Verify-failure rollback re-runs reload against restored bytes. func TestNginx_Atomic_VerifyFailure_RollbackRunsReloadAgain(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", }, } c := newConnectorWithStubs(t, cfg) c.SetTestProbe(successProbe("0000000000000000000000000000000000000000000000000000000000000000")) var reloadCalls int32 c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { atomic.AddInt32(&reloadCalls, 1) return nil, nil }) _, _ = c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if reloadCalls != 2 { t.Errorf("reload calls = %d, want 2 (once for new bytes, once for rollback restore)", reloadCalls) } } // 31. ValidateOnly with cancelled context returns context error. func TestNginx_ValidateOnly_ContextCancelled(t *testing.T) { cfg := &nginx.Config{ CertPath: "/tmp/cert.pem", ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) c.SetTestRunValidate(func(ctx context.Context, _ string) ([]byte, error) { return nil, ctx.Err() }) ctx, cancel := context.WithCancel(context.Background()) cancel() if err := c.ValidateOnly(ctx, target.DeploymentRequest{}); err == nil { t.Error("expected error from cancelled ctx") } } // 32. Cert + Chain + Key + verify all deploy in single Apply call. func TestNginx_Atomic_AllFour_OneApply(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ChainPath: filepath.Join(dir, "chain.pem"), KeyPath: filepath.Join(dir, "key.pem"), KeyFileMode: 0640, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", }, } c := newConnectorWithStubs(t, cfg) c.SetTestProbe(successProbe(fingerprintOfPEM(t, certA))) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{ CertPEM: certA, ChainPEM: chain, KeyPEM: keyA, }) if err != nil { t.Fatal(err) } if !res.Success { t.Error("Success=false") } } // 33. Idempotent skip skips post-verify too (deploy was a no-op). func TestNginx_Atomic_IdempotentSkip_SkipsVerify(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte(certA), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", }, } c := newConnectorWithStubs(t, cfg) var probeCalls int32 c.SetTestProbe(func(ctx context.Context, _ string, _ time.Duration) tlsprobe.ProbeResult { atomic.AddInt32(&probeCalls, 1) return tlsprobe.ProbeResult{Success: true, Fingerprint: "ignored"} }) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } if probeCalls != 0 { t.Errorf("probe called %d times despite idempotent skip", probeCalls) } } // 34. Result.Metadata carries cert_path + chain_path + duration_ms // + idempotent flags. (Audit log + Prometheus consume these.) func TestNginx_Atomic_Result_MetadataPopulated(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ChainPath: filepath.Join(dir, "chain.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain}) if err != nil { t.Fatal(err) } for _, key := range []string{"cert_path", "chain_path", "duration_ms", "idempotent"} { if _, ok := res.Metadata[key]; !ok { t.Errorf("metadata missing %q", key) } } } // 35. Successful deploy returns DeploymentID with nginx- prefix. func TestNginx_Atomic_DeploymentID_HasNginxPrefix(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) res, _ := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if !strings.HasPrefix(res.DeploymentID, "nginx-") { t.Errorf("DeploymentID = %q, want nginx-* prefix", res.DeploymentID) } } // 36. Concurrent deploys to DIFFERENT paths run in parallel. func TestNginx_Atomic_DifferentPaths_RunInParallel(t *testing.T) { dirA := t.TempDir() dirB := t.TempDir() cfgA := &nginx.Config{ CertPath: filepath.Join(dirA, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } cfgB := &nginx.Config{ CertPath: filepath.Join(dirB, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } cA := newConnectorWithStubs(t, cfgA) cB := newConnectorWithStubs(t, cfgB) // Both should be able to deploy without serializing. doneA := make(chan struct{}) doneB := make(chan struct{}) go func() { _, _ = cA.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) close(doneA) }() go func() { _, _ = cB.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certB}) close(doneB) }() select { case <-doneA: case <-time.After(2 * time.Second): t.Fatal("timeout waiting for cA") } select { case <-doneB: case <-time.After(2 * time.Second): t.Fatal("timeout waiting for cB") } } // 37. Reload command CombinedOutput surfaces in the failure // message for operator triage. func TestNginx_Atomic_ReloadFailure_OutputInError(t *testing.T) { dir := t.TempDir() cert := filepath.Join(dir, "cert.pem") if err := os.WriteFile(cert, []byte("ORIGINAL"), 0644); err != nil { t.Fatal(err) } cfg := &nginx.Config{ CertPath: cert, ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) var reloadCalls int32 c.SetTestRunReload(func(ctx context.Context, _ string) ([]byte, error) { n := atomic.AddInt32(&reloadCalls, 1) if n == 1 { return []byte("nginx: [emerg] cannot bind to :443"), errors.New("exit 1") } return nil, nil }) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil || !strings.Contains(err.Error(), "cannot bind") { t.Errorf("error doesn't include reload stderr: %v", err) } } // 38. Validate command CombinedOutput surfaces in the failure // message. func TestNginx_Atomic_ValidateFailure_OutputInError(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) c.SetTestRunValidate(func(ctx context.Context, _ string) ([]byte, error) { return []byte("nginx: [emerg] no SSL session ID context"), errors.New("exit 1") }) _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if err == nil || !strings.Contains(err.Error(), "SSL session ID context") { t.Errorf("error doesn't include validate stderr: %v", err) } } // 39. PostDeployVerify with default Timeout (0) uses 10s default. // We verify by stubbing the prober and checking the timeout // argument it receives. func TestNginx_Atomic_PostVerify_DefaultTimeout10s(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "nginx:443", // Timeout left zero }, } c := newConnectorWithStubs(t, cfg) var seenTimeout time.Duration want := fingerprintOfPEM(t, certA) c.SetTestProbe(func(ctx context.Context, _ string, timeout time.Duration) tlsprobe.ProbeResult { seenTimeout = timeout return tlsprobe.ProbeResult{Success: true, Fingerprint: want} }) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } if seenTimeout != 10*time.Second { t.Errorf("default timeout = %v, want 10s", seenTimeout) } } // 40. PostDeployVerify endpoint is passed through to the probe. func TestNginx_Atomic_PostVerify_EndpointForwarded(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 1, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "specific-host:8443", }, } c := newConnectorWithStubs(t, cfg) var seenAddr string want := fingerprintOfPEM(t, certA) c.SetTestProbe(func(ctx context.Context, addr string, _ time.Duration) tlsprobe.ProbeResult { seenAddr = addr return tlsprobe.ProbeResult{Success: true, Fingerprint: want} }) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}); err != nil { t.Fatal(err) } if seenAddr != "specific-host:8443" { t.Errorf("probe got %q, want specific-host:8443", seenAddr) } } // 41. Empty CertPEM → still attempts deploy of empty bytes (the // server-side validation should have caught this earlier; we just // pin the connector doesn't crash on edge data). func TestNginx_Atomic_EmptyCertPEM_HandledGracefully(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) if _, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: ""}); err != nil { t.Fatal(err) } } // 42. Deploy result `idempotent` field is "false" for fresh. func TestNginx_Atomic_FreshDeploy_IdempotentFlagFalse(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", } c := newConnectorWithStubs(t, cfg) res, _ := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA}) if res.Metadata["idempotent"] != "false" { t.Errorf("idempotent = %q, want false", res.Metadata["idempotent"]) } } // TestNginx_VerifyExponentialBackoff_GrowsBetweenAttempts: post-deploy verify // retries with exponential backoff (10ms → 20ms → 40ms up to max). func TestNginx_VerifyExponentialBackoff_GrowsBetweenAttempts(t *testing.T) { dir := t.TempDir() cfg := &nginx.Config{ CertPath: filepath.Join(dir, "cert.pem"), ReloadCommand: "nginx -s reload", ValidateCommand: "nginx -t", PostDeployVerifyAttempts: 4, PostDeployVerifyBackoff: 10 * time.Millisecond, PostDeployVerifyMaxBackoff: 80 * time.Millisecond, PostDeployVerify: &nginx.PostDeployVerifyConfig{ Enabled: true, Endpoint: "localhost:443", Timeout: 100 * time.Millisecond, }, } c := newConnectorWithStubs(t, cfg) var callTimes []time.Time probeCallCount := atomic.Int32{} c.SetTestProbe(func(_ context.Context, _ string, _ time.Duration) tlsprobe.ProbeResult { callTimes = append(callTimes, time.Now()) count := probeCallCount.Add(1) if count == 4 { return tlsprobe.ProbeResult{Success: true, Fingerprint: fingerprintOfPEM(t, certA)} } return tlsprobe.ProbeResult{Success: false, Error: "cert not yet deployed"} }) res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{ CertPEM: certA, KeyPEM: keyA, }) if err != nil { t.Fatalf("DeployCertificate failed: %v", err) } if !res.Success { t.Fatal("expected Success=true") } if len(callTimes) != 4 { t.Fatalf("expected 4 probe calls, got %d", len(callTimes)) } const tolerance = 20 * time.Millisecond expectedGaps := []time.Duration{ 10 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, } for i := 0; i < len(expectedGaps); i++ { gap := callTimes[i+1].Sub(callTimes[i]) expected := expectedGaps[i] if gap < expected-tolerance || gap > expected+tolerance { t.Errorf("gap[%d]: expected ~%v, got %v", i, expected, gap) } } }