certctl/deploy/test/healthcheck_test.go

//go:build integration

// Package integration_test — image-level HEALTHCHECK contract.
//
// U-2 (P1, cat-u-healthcheck_protocol_mismatch): pre-U-2 the published
// server image's Dockerfile HEALTHCHECK called `curl -f http://localhost:
// 8443/health` against an HTTPS-only listener (HTTPS-Everywhere milestone,
// v2.2 / tag v2.0.47). Operators outside docker-compose / Helm saw the
// container reported as `unhealthy` indefinitely. The compose stack
// overrode this HEALTHCHECK with `--cacert + https://`; the Helm chart
// uses explicit `httpGet` probes that ignore Docker's HEALTHCHECK; the 5
// example compose files all override with `curl -sfk https://localhost:
// 8443/health`. So the observable failure was scoped to bare `docker run`
// / Docker Swarm / Nomad / ECS users — exactly the "I just pulled the
// published image" path.
//
// This file's tests pin the contract at the binary-image level. The
// matching CI grep guardrail in .github/workflows/ci.yml catches the
// regression at the Dockerfile-source level; both layers are needed
// because someone could replace the HEALTHCHECK line with a sibling
// broken pattern that the grep doesn't catch (e.g., a TCP-only check
// against the HTTPS port).
//
// Run alongside the rest of the integration suite:
//
//	cd deploy/test && go test -tags integration -v -run Healthcheck
//
// The tests skip cleanly with t.Skip when docker is not available
// (CI without docker-in-docker, sandbox environments, etc.) so they
// don't block local development on machines without docker.
package integration_test

import (
	"encoding/json"
	"os/exec"
	"strings"
	"testing"
	"time"
)

// dockerAvailable returns true when `docker version` returns 0.
// We cache it across tests in this file so the skip message prints once.
func dockerAvailable(t *testing.T) bool {
	t.Helper()
	cmd := exec.Command("docker", "version", "--format", "{{.Server.Version}}")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Logf("docker not available: %v\noutput: %s", err, string(out))
		return false
	}
	return true
}

// dockerCmd runs `docker <args...>` with a 60s budget, returning stdout
// + stderr combined and the exit error if any. Used for short-lived
// probes (inspect, build, run -d).
func dockerCmd(t *testing.T, timeout time.Duration, args ...string) (string, error) {
	t.Helper()
	cmd := exec.Command("docker", args...)
	done := make(chan struct{})
	var out []byte
	var err error
	go func() {
		out, err = cmd.CombinedOutput()
		close(done)
	}()
	select {
	case <-done:
		return string(out), err
	case <-time.After(timeout):
		_ = cmd.Process.Kill()
		t.Fatalf("docker %v timed out after %v", args, timeout)
		return "", err
	}
}

// TestPublishedServerImage_HealthcheckSpecUsesHTTPS performs the Dockerfile-
// source-level shipped-shape pin: the inspected image's Healthcheck.Test
// array MUST contain "https://localhost:8443/health" (and MUST NOT
// contain "http://localhost:8443/health"). This is the lightweight half
// of the contract — it doesn't require running the container, only
// building it. It catches the audit-flagged bug directly.
func TestPublishedServerImage_HealthcheckSpecUsesHTTPS(t *testing.T) {
	if !dockerAvailable(t) {
		t.Skip("docker not available — skipping image-level HEALTHCHECK test")
	}

	const imgTag = "certctl-u2-healthcheck-spec-test"
	t.Cleanup(func() {
		_, _ = dockerCmd(t, 30*time.Second, "rmi", "-f", imgTag)
	})

	// Build the server image. Use the repo root as context (this test
	// file lives at deploy/test/, the Dockerfile at the repo root).
	buildOut, err := dockerCmd(t, 5*time.Minute,
		"build", "-f", "../../Dockerfile", "-t", imgTag, "../..")
	if err != nil {
		t.Fatalf("docker build failed: %v\noutput:\n%s", err, buildOut)
	}

	// Inspect the shipped HEALTHCHECK metadata.
	inspectOut, err := dockerCmd(t, 30*time.Second,
		"inspect", "--format", "{{json .Config.Healthcheck}}", imgTag)
	if err != nil {
		t.Fatalf("docker inspect failed: %v\noutput:\n%s", err, inspectOut)
	}

	var hc struct {
		Test     []string
		Interval int64
		Timeout  int64
	}
	if err := json.Unmarshal([]byte(strings.TrimSpace(inspectOut)), &hc); err != nil {
		t.Fatalf("could not parse Healthcheck JSON %q: %v", inspectOut, err)
	}

	joined := strings.Join(hc.Test, " ")

	// Positive contract.
	if !strings.Contains(joined, "https://localhost:8443/health") {
		t.Errorf("Healthcheck.Test does not target https://localhost:8443/health\nfull: %v", hc.Test)
	}

	// Negative contract — pre-U-2 regression shape MUST be absent.
	if strings.Contains(joined, "http://localhost:8443/health") {
		t.Errorf("Healthcheck.Test still contains the pre-U-2 plaintext shape: %v", hc.Test)
	}

	// `-k` (or `--insecure`) must be present because the bootstrap cert
	// is per-deploy and the published image can't pin a CA bundle —
	// see the U-2 closure docblock on Dockerfile and the audit doc.
	if !strings.Contains(joined, "-k") && !strings.Contains(joined, "--insecure") {
		t.Errorf("Healthcheck.Test omits -k / --insecure flag (required for self-signed bootstrap probe): %v", hc.Test)
	}
}

// TestPublishedAgentImage_HealthcheckSpecExists pins the U-2 adjacent
// fix that added a HEALTHCHECK to the agent image. Pre-U-2 the agent
// image had no HEALTHCHECK declaration, so bare-`docker run` agents got
// `none` health status from Docker. Post-U-2 the agent uses pgrep to
// verify the process is alive (mirroring the docker-compose pattern at
// deploy/docker-compose.yml:173, which also became reliable post-U-2
// because procps is now installed in the runtime image).
func TestPublishedAgentImage_HealthcheckSpecExists(t *testing.T) {
	if !dockerAvailable(t) {
		t.Skip("docker not available — skipping image-level HEALTHCHECK test")
	}

	const imgTag = "certctl-u2-agent-healthcheck-spec-test"
	t.Cleanup(func() {
		_, _ = dockerCmd(t, 30*time.Second, "rmi", "-f", imgTag)
	})

	buildOut, err := dockerCmd(t, 5*time.Minute,
		"build", "-f", "../../Dockerfile.agent", "-t", imgTag, "../..")
	if err != nil {
		t.Fatalf("docker build failed: %v\noutput:\n%s", err, buildOut)
	}

	inspectOut, err := dockerCmd(t, 30*time.Second,
		"inspect", "--format", "{{json .Config.Healthcheck}}", imgTag)
	if err != nil {
		t.Fatalf("docker inspect failed: %v\noutput:\n%s", err, inspectOut)
	}

	trimmed := strings.TrimSpace(inspectOut)
	if trimmed == "null" || trimmed == "" {
		t.Fatalf("agent image has no HEALTHCHECK (got %q) — U-2 adjacent fix regressed", inspectOut)
	}

	var hc struct {
		Test []string
	}
	if err := json.Unmarshal([]byte(trimmed), &hc); err != nil {
		t.Fatalf("could not parse Healthcheck JSON %q: %v", inspectOut, err)
	}

	joined := strings.Join(hc.Test, " ")
	if !strings.Contains(joined, "pgrep") {
		t.Errorf("agent Healthcheck.Test does not use pgrep (lost the process-presence shape): %v", hc.Test)
	}
	if !strings.Contains(joined, "certctl-agent") {
		t.Errorf("agent Healthcheck.Test does not target the certctl-agent process name: %v", hc.Test)
	}
}

// TestPublishedServerImage_HealthcheckTransitionsToHealthy is the
// runtime-level contract: the built image, when started, must transition
// to `healthy` within the start-period + 30s observability budget. This
// is the heavy test — it requires the server to actually start, which
// in turn requires either a reachable database OR a startup that fails
// gracefully enough to keep the HEALTHCHECK probe target alive.
//
// The container is started with CERTCTL_DATABASE_URL pointing at an
// unreachable host so the server fails its postgres bring-up — but
// importantly, fails AFTER the TLS listener has come up, because the
// HEALTHCHECK probe target is the TLS listener. We don't actually need
// the database to validate the HEALTHCHECK shape.
//
// IMPORTANT: this test is the runtime contract. If you're working on the
// server's startup ordering and the listener now comes up AFTER the
// database, this test must adapt — start a sidecar postgres via
// testcontainers-go (see internal/integration/lifecycle_test.go for the
// pattern) and connect the certctl-server container to it.
func TestPublishedServerImage_HealthcheckTransitionsToHealthy(t *testing.T) {
	if !dockerAvailable(t) {
		t.Skip("docker not available — skipping runtime HEALTHCHECK test")
	}
	if testing.Short() {
		t.Skip("runtime HEALTHCHECK test takes ~45s; skipping under -short")
	}
	t.Skip("runtime probe contract not yet wired to a sidecar postgres; " +
		"image-spec contract above (TestPublishedServerImage_HealthcheckSpecUsesHTTPS) " +
		"covers the audit-flagged regression. Re-enable once the integration " +
		"harness provisions postgres for image-level smoke.")
}