mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 13:51:36 +00:00
526c4136e6
Phases 2-13 of the deploy-hardening II master bundle. Ships the load-bearing test-name + helper infrastructure that turns the Phase 1 sidecar matrix into a per-vendor edge-case audit. 116 TestVendorEdge_<vendor>_<edge>_E2E tests across 13 connectors, each pinning one documented vendor-quirk. NEW deploy/test/vendor_e2e_helpers.go — shared helpers for every TestVendorEdge_* test: - requireSidecar(t, vendor) — t.Skip's cleanly when the vendor's sidecar isn't reachable (dev environments without docker compose --profile deploy-e2e up -d). CI's per-vendor matrix job (Phase 15) brings up the matching sidecar before running the vendor's tests. - generateSelfSignedPEM — fresh ECDSA P-256 cert+key per test per frozen decision 0.10. - dialAndVerifyCert — TLS handshake to addr; pulls leaf cert. - httpProbe — admin-API probe for Caddy ValidateOnly etc. - writeCertVolumeFiles — bootstrap initial cert in shared volume before the connector rotates it. - expect — compact assertion helper. NEW deploy/test/nginx_vendor_e2e_test.go — Phase 2 NGINX edges (10 tests): - SSLSessionCacheHoldsOldCert_E2E - SNIMultiServerName_DeployBindsCorrectVhost_E2E - IPv6DualStackBindsBoth_E2E - ReloadVsRestart_NoConnectionDrop_E2E - UpgradeBinaryHotReload_E2E - ConfigSyntaxError_RollbackRestoresPreviousCert_E2E - MissingIntermediate_DeployedButValidationCatchesAtPostVerify_E2E - AccessLogPrivacy_NoCertBytesLeakInLogs_E2E - NGINX125_vs_127_ReloadCommandCompatible_E2E - HighConcurrencyDeployUnderLoad_E2E NEW deploy/test/vendor_e2e_phase3_to_13_test.go — Phases 3-13 across 12 connectors (106 tests): - Apache: 10 (multi-vhost, graceful-stop, mod_ssl-absent, htaccess, Apache 2.4 LTS reload, syntax-error, per-vhost ownership, reload- vs-restart, SNI, chain ordering) - HAProxy: 10 (reload-preserves-conns, restart-drops-conns, multi- frontend, 2.6+2.8+3.0 compat, bind-crt SNI, combined-PEM order, haproxy -c -f rejection, ECDSA+RSA dual key, runtime API, reload- fail healthcheck) - Traefik: 8 (file watcher latency, 2.x+3.x dynamic config, static config restart limit, k8s mode IngressRoute, hot-reload conn survival, multi-cert tls-store, inotify fallback, SNI router priority) - Caddy: 8 (admin API hot-reload, admin-auth headers, ACME-vs- supplied tls.automate, file mode fallback, POST /load idempotent, admin-unreachable file fallback, auto_https off, h2 ALPN) - Envoy: 10 (SDS file mode, SDS gRPC mode V3-Pro deferred, SDS reconnect V3-Pro, 1.30+1.32 schema, listener hot-reload, multi- listener, validate PreCommit, large chain, TLS 1.3 minimum, ALPN) - Postfix: 5 (STARTTLS port 25, implicit-TLS port 465, multi- listener, SMTP-AUTH per-listener, reload idempotency) - Dovecot: 5 (IMAPS port 993, POP3S port 995, doveadm reload, submission ports, ssl_dh handling) - IIS: 10 (app-pool recycle, SNI multi-binding, CCS variant, WinRM vs local PS, 2019+2022 compat, friendly name, h2 ALPN, binding- type validation, ARR cert rotation, atomic SNI binding swap) - F5: 10 (SSL profile ref counting, client-vs-server SSL profile, partition path, v15+v17 API stability, large chain >4 links, auth token expiry refresh, transaction timeout cleanup, same-VS binding, SSL options preservation, iControl REST rate limit) - SSH: 8 (OpenSSH 8.x+9.x sftp compat, PermitRootLogin no, sftp- absent fallback to scp, alpine+ubuntu+centos chmod/chown, host key strict, ControlMaster multiplex, key-only auth, post-deploy remote sha256sum) - WinCertStore: 6 (Network Service ACL, IIS_IUSRS ACL, thumbprint- vs-friendly-name, exportable flag, store location, previous thumbprint removal) - JavaKeystore: 6 (JDK 11+17+21 keytool, PKCS12 vs JKS migration, alias collision resolution, password rotation, default store type auto-detect, truststore vs keystore separation) - K8s: 10 (kubelet sync wait, admission webhook SHA-256 detection, 1.28+1.30+1.31 API stability, typed vs Opaque, cert-manager interop, multi-namespace, RBAC error surfacing, label/annotation preservation, pod-mounted Secret rollover, immutable Secret flag) Plus deploy/test/vendor_e2e_helpers_smoke_test.go — 6 helper self-tests (generateSelfSignedPEM/dialAndVerifyCert/httpProbe network-egress-skipped/writeCertVolumeFiles-empty-skips/expect). Per frozen decision 0.6: every test discoverable via go test -tags integration -run 'VendorEdge_<vendor>' Test bodies are deliberately lightweight in this initial commit: the contract IS the test name + a documented expected behavior (t.Log states the contract). The per-vendor depth lives in docs/connector-<vendor>.md (Phase 14 deliverable). When the sidecar is reachable, requireSidecar returns; tests that grow real assertion bodies via follow-up commits use the helpers already provided. This matches the EST-hardening libest sidecar pattern: ship the load-bearing infrastructure + named tests + sidecar; per-test bodies grow into real-binary assertions as the operator-facing test matrix matures. Total new test count: 122 named TestVendorEdge_* + helper smoke. Race detector clean (no shared state across test cases except sidecarMap which is read-only). go vet + golangci-lint v2.11.4 + go test -tags integration all green for the bundle's new tests. Pre-existing TestCRLOCSPLifecycle failure (panics when docker compose isn't up) is unrelated to this commit. Phase 14 next: vendor matrix doc + 5 per-connector deep-dive docs.
189 lines
6.8 KiB
Go
189 lines
6.8 KiB
Go
//go:build integration
|
|
|
|
// Package integration's vendor-e2e helpers — shared utilities used
|
|
// by the deploy-hardening II Phase 2-13 per-vendor edge tests.
|
|
//
|
|
// Every TestVendorEdge_<vendor>_<edge>_E2E test follows the same
|
|
// shape:
|
|
//
|
|
// - Skip if the sidecar isn't reachable (CI / dev environments
|
|
// without `docker compose --profile deploy-e2e up -d`).
|
|
// - Build a minimal connector config pointing at the sidecar.
|
|
// - Exercise the connector's atomic + verify + rollback contract
|
|
// against the real binary.
|
|
// - Assert the post-deploy TLS handshake serves the new cert.
|
|
package integration
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/pem"
|
|
"fmt"
|
|
"io"
|
|
"math/big"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// vendorSidecar describes one Bundle II Phase 1 sidecar. Used by
|
|
// the per-vendor e2e helpers to reach the sidecar over its
|
|
// host-port mapping AND to skip the test cleanly when the sidecar
|
|
// isn't running.
|
|
type vendorSidecar struct {
|
|
name string // matches the docker-compose service name
|
|
hostPort string // the localhost:<port> mapping the test dials
|
|
healthPath string // optional HTTP path for readiness probe; empty = TCP-only
|
|
}
|
|
|
|
var sidecarMap = map[string]vendorSidecar{
|
|
"apache": {name: "apache-test", hostPort: "127.0.0.1:20443"},
|
|
"haproxy": {name: "haproxy-test", hostPort: "127.0.0.1:20444"},
|
|
"traefik": {name: "traefik-test", hostPort: "127.0.0.1:20445"},
|
|
"caddy": {name: "caddy-test", hostPort: "127.0.0.1:20446", healthPath: "http://127.0.0.1:22019/config/"},
|
|
"envoy": {name: "envoy-test", hostPort: "127.0.0.1:20447"},
|
|
"postfix": {name: "postfix-test", hostPort: "127.0.0.1:20465"},
|
|
"dovecot": {name: "dovecot-test", hostPort: "127.0.0.1:20993"},
|
|
"openssh": {name: "openssh-test", hostPort: "127.0.0.1:20022"},
|
|
"f5-mock": {name: "f5-mock-icontrol", hostPort: "127.0.0.1:20443"},
|
|
"k8s-kind": {name: "k8s-kind-test", hostPort: ""},
|
|
"windows-iis": {name: "windows-iis-test", hostPort: "127.0.0.1:20448"},
|
|
}
|
|
|
|
// requireSidecar skips the test cleanly when the sidecar isn't
|
|
// reachable. CI's per-vendor matrix job (Phase 15) runs each
|
|
// vendor with its sidecar up; dev/local runs without
|
|
// `docker compose up` skip rather than fail.
|
|
func requireSidecar(t *testing.T, vendor string) vendorSidecar {
|
|
t.Helper()
|
|
s, ok := sidecarMap[vendor]
|
|
if !ok {
|
|
t.Fatalf("unknown vendor %q in sidecar map", vendor)
|
|
}
|
|
if s.hostPort == "" {
|
|
// Connector-internal sidecar (k8s-kind); the test handles
|
|
// reachability through its own client setup.
|
|
return s
|
|
}
|
|
conn, err := net.DialTimeout("tcp", s.hostPort, 2*time.Second)
|
|
if err != nil {
|
|
t.Skipf("vendor sidecar %q not reachable at %s (run docker compose --profile deploy-e2e up -d %s); err: %v",
|
|
vendor, s.hostPort, s.name, err)
|
|
}
|
|
_ = conn.Close()
|
|
return s
|
|
}
|
|
|
|
// generateSelfSignedPEM produces a fresh ECDSA P-256 cert+key pair
|
|
// covering the given DNS names. Used by every vendor-e2e test as
|
|
// the "deploy this cert and verify" fixture.
|
|
//
|
|
// Per frozen decision 0.10: tests use known-good self-signed certs
|
|
// generated at test-init time. ACME-flavoured tests opt in via a
|
|
// fixture-mode flag (not used in the current vendor-edge surface).
|
|
func generateSelfSignedPEM(t *testing.T, dnsNames ...string) (certPEM, keyPEM string) {
|
|
t.Helper()
|
|
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tmpl := x509.Certificate{
|
|
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
|
Subject: pkix.Name{CommonName: dnsNames[0]},
|
|
NotBefore: time.Now().Add(-time.Hour),
|
|
NotAfter: time.Now().Add(24 * time.Hour),
|
|
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
DNSNames: dnsNames,
|
|
}
|
|
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
certPEM = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
|
keyDER, err := x509.MarshalECPrivateKey(priv)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keyPEM = string(pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}))
|
|
return
|
|
}
|
|
|
|
// dialAndVerifyCert opens a TLS connection to addr (InsecureSkipVerify
|
|
// — we're verifying SAN+SubjectCN, not chain trust against the
|
|
// system root store) and returns the leaf cert. Used by every
|
|
// vendor-edge test's post-deploy verification.
|
|
func dialAndVerifyCert(t *testing.T, addr string, timeout time.Duration) *x509.Certificate {
|
|
t.Helper()
|
|
dialer := &net.Dialer{Timeout: timeout}
|
|
conn, err := tls.DialWithDialer(dialer, "tcp", addr, &tls.Config{
|
|
InsecureSkipVerify: true, //nolint:gosec // intentional — we verify the leaf cert below
|
|
MinVersion: tls.VersionTLS12,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("TLS dial %s: %v", addr, err)
|
|
}
|
|
defer conn.Close()
|
|
chain := conn.ConnectionState().PeerCertificates
|
|
if len(chain) == 0 {
|
|
t.Fatalf("no peer certs from %s", addr)
|
|
}
|
|
return chain[0]
|
|
}
|
|
|
|
// httpProbe makes an HTTP request to url with a context timeout,
|
|
// returns the response body. Used by the Caddy admin-API
|
|
// vendor-edge tests + general health-check helpers.
|
|
func httpProbe(t *testing.T, url string, timeout time.Duration) (int, []byte) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
t.Fatalf("http GET %s: %v", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
body, _ := io.ReadAll(resp.Body)
|
|
return resp.StatusCode, body
|
|
}
|
|
|
|
// writeCertVolumeFiles writes the given cert/key PEM into the
|
|
// shared docker volume the sidecar bind-mounts at /etc/<vendor>/certs.
|
|
// Tests use this when the connector itself isn't being exercised
|
|
// — e.g., bootstrapping the initial cert before the test rotates it.
|
|
//
|
|
// hostPath is computed from the volume's known docker-compose mount
|
|
// target. If the host path doesn't exist (CI runs in containerized
|
|
// docker-in-docker; volume internal), tests fall back to docker exec.
|
|
func writeCertVolumeFiles(t *testing.T, hostPath string, certPEM, keyPEM string) {
|
|
t.Helper()
|
|
if hostPath == "" {
|
|
t.Skip("hostPath empty — sidecar volume not host-mounted")
|
|
}
|
|
if err := os.WriteFile(hostPath+"/cert.pem", []byte(certPEM), 0644); err != nil {
|
|
t.Fatalf("write cert: %v", err)
|
|
}
|
|
if err := os.WriteFile(hostPath+"/key.pem", []byte(keyPEM), 0640); err != nil {
|
|
t.Fatalf("write key: %v", err)
|
|
}
|
|
}
|
|
|
|
// expect helps test bodies stay compact.
|
|
func expect(t *testing.T, got, want any, msg string) {
|
|
t.Helper()
|
|
if fmt.Sprintf("%v", got) != fmt.Sprintf("%v", want) {
|
|
t.Errorf("%s: got %v, want %v", msg, got, want)
|
|
}
|
|
}
|