certctl/deploy/test/deploy_e2e_test.go

//go:build integration

// Package test contains the deploy-hardening I Phase 11 cross-
// cutting end-to-end integration tests. These exercise the
// internal/deploy package's load-bearing invariants end-to-end:
//
//   - atomicity: kill mid-deploy → file is fully old or fully new;
//     never torn.
//   - post-verify: deploy a wrong-fingerprint cert + the connector's
//     verify hook → the rollback wire restores the previous bytes.
//   - idempotency: deploy the same bytes twice → the second attempt
//     is a no-op (no PreCommit/PostCommit calls).
//   - concurrency: N simultaneous deploys to the same destination
//     serialize via the deploy package's file-level mutex.
//
// Run via `INTEGRATION=1 go test -tags integration -race ./deploy/test/... -run Deploy`.
package integration

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/certctl-io/certctl/internal/deploy"
)

// TestDeploy_Atomicity_FileIsAlwaysOldOrNew pins the load-bearing
// POSIX-rename atomicity invariant. A reader hammering the
// destination during 30 alternating writes either sees the OLD
// bytes or the NEW bytes — never an intermediate state. Closes
// the operator-facing question "is my cert deploy interruption-
// safe?".
func TestDeploy_Atomicity_FileIsAlwaysOldOrNew(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "cert.pem")
	old := []byte(strings.Repeat("OLD-CERT-PEM-", 200))
	newer := []byte(strings.Repeat("NEW-CERT-PEM-", 200))
	if err := os.WriteFile(path, old, 0644); err != nil {
		t.Fatal(err)
	}

	stop := make(chan struct{})
	var torn atomic.Bool
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
			}
			b, err := os.ReadFile(path)
			if err != nil {
				continue
			}
			s := string(b)
			if s != string(old) && s != string(newer) {
				torn.Store(true)
				return
			}
		}
	}()

	for i := 0; i < 30; i++ {
		writeBytes := old
		if i%2 == 0 {
			writeBytes = newer
		}
		if _, err := deploy.AtomicWriteFile(context.Background(), path, writeBytes, deploy.WriteOptions{
			SkipIdempotent: true,
		}); err != nil {
			t.Fatalf("write %d: %v", i, err)
		}
	}
	close(stop)
	wg.Wait()
	if torn.Load() {
		t.Error("torn read observed (rename atomicity broken)")
	}
}

// TestDeploy_PostVerify_WrongCertTriggersRollback simulates a
// mis-deployed cert: the deploy.Apply succeeds at the file-write
// + reload level, but the connector's post-deploy verify (run
// AFTER Apply returns) detects the SHA-256 mismatch and rolls
// back manually using the BackupPaths that Apply returned. The
// final on-disk state matches the OLD bytes; the rollback wire
// works end-to-end.
func TestDeploy_PostVerify_WrongCertTriggersRollback(t *testing.T) {
	dir := t.TempDir()
	cert := filepath.Join(dir, "cert.pem")
	if err := os.WriteFile(cert, []byte("OLD-CERT"), 0644); err != nil {
		t.Fatal(err)
	}

	plan := deploy.Plan{
		Files: []deploy.File{{Path: cert, Bytes: []byte("WRONG-CERT")}},
		PostCommit: func(_ context.Context) error {
			// Reload would normally verify the cert via the post-deploy
			// TLS handshake. Here we simulate the verify failure by
			// returning an error from PostCommit (which triggers the
			// deploy package's automatic rollback).
			//
			// On the first call (the real deploy), return an error so
			// the rollback fires; on the second call (the rollback's
			// re-PostCommit against the restored bytes), succeed so
			// rollback completes cleanly.
			return errors.New("post-deploy verify: SHA-256 mismatch")
		},
	}

	// First call to PostCommit fails; the rollback's second call
	// would also fail with the same handler — so we use a stateful
	// counter.
	var postCalls int32
	plan.PostCommit = func(_ context.Context) error {
		if atomic.AddInt32(&postCalls, 1) == 1 {
			return errors.New("post-deploy verify: SHA-256 mismatch")
		}
		return nil
	}

	_, err := deploy.Apply(context.Background(), plan)
	if !errors.Is(err, deploy.ErrReloadFailed) {
		t.Fatalf("got %v, want ErrReloadFailed", err)
	}
	got, _ := os.ReadFile(cert)
	if string(got) != "OLD-CERT" {
		t.Errorf("cert after rollback = %q, want OLD-CERT", got)
	}
	if atomic.LoadInt32(&postCalls) != 2 {
		t.Errorf("PostCommit calls = %d, want 2 (1 deploy + 1 rollback re-call)", postCalls)
	}
}

// TestDeploy_Idempotency_SecondDeployIsNoOp pins the SHA-256
// short-circuit. Defends against agent-restart retry storms that
// otherwise hammer targets with no-op reloads.
func TestDeploy_Idempotency_SecondDeployIsNoOp(t *testing.T) {
	dir := t.TempDir()
	cert := filepath.Join(dir, "cert.pem")
	bytes := []byte("STABLE-CERT-PEM")
	if err := os.WriteFile(cert, bytes, 0644); err != nil {
		t.Fatal(err)
	}

	var preCalls, postCalls int32
	plan := deploy.Plan{
		Files: []deploy.File{{Path: cert, Bytes: bytes}},
		PreCommit: func(_ context.Context, _ map[string]string) error {
			atomic.AddInt32(&preCalls, 1)
			return nil
		},
		PostCommit: func(_ context.Context) error {
			atomic.AddInt32(&postCalls, 1)
			return nil
		},
	}
	res, err := deploy.Apply(context.Background(), plan)
	if err != nil {
		t.Fatal(err)
	}
	if !res.SkippedAsIdempotent {
		t.Error("expected SkippedAsIdempotent=true")
	}
	if preCalls != 0 || postCalls != 0 {
		t.Errorf("expected 0 calls, got %d/%d", preCalls, postCalls)
	}
}

// TestDeploy_Concurrent_SamePathsSerialize fires N simultaneous
// deploys to the same destination. The deploy package's file-
// level mutex must serialize them: max-in-flight = 1.
func TestDeploy_Concurrent_SamePathsSerialize(t *testing.T) {
	dir := t.TempDir()
	cert := filepath.Join(dir, "cert.pem")

	const N = 8
	var inFlight, maxInFlight int32
	var wg sync.WaitGroup
	for i := 0; i < N; i++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			plan := deploy.Plan{
				Files: []deploy.File{{
					Path:  cert,
					Bytes: []byte(fmt.Sprintf("WRITER-%d", idx)),
				}},
				SkipIdempotent: true,
				PostCommit: func(_ context.Context) error {
					n := atomic.AddInt32(&inFlight, 1)
					for {
						m := atomic.LoadInt32(&maxInFlight)
						if n <= m || atomic.CompareAndSwapInt32(&maxInFlight, m, n) {
							break
						}
					}
					time.Sleep(2 * time.Millisecond)
					atomic.AddInt32(&inFlight, -1)
					return nil
				},
			}
			if _, err := deploy.Apply(context.Background(), plan); err != nil {
				t.Errorf("Apply %d: %v", idx, err)
			}
		}(i)
	}
	wg.Wait()
	if maxInFlight > 1 {
		t.Errorf("max in-flight = %d, want 1 (mutex broken)", maxInFlight)
	}
	got, _ := os.ReadFile(cert)
	if !strings.HasPrefix(string(got), "WRITER-") {
		t.Errorf("file content not from any writer: %q", got)
	}
}