feat(test): docker-compose deploy-e2e sidecar matrix — apache + haproxy + traefik + caddy + envoy + postfix + dovecot + openssh + f5-mock-icontrol + k8s-kind + windows-iis

Phase 1 of the deploy-hardening II master bundle. Adds the 11 missing
target sidecars to deploy/docker-compose.test.yml under
profiles: [deploy-e2e] (windows-iis-test under [deploy-e2e-windows]
because Windows containers run only on Windows hosts).

Per frozen decision 0.2: pull pre-built images from official
registries where they exist (NGINX, HAProxy, Traefik, Caddy, Envoy,
Postfix via boky, Dovecot, OpenSSH via lscr.io, K8s via kind);
build locally only where no official image works (F5 — uses the
new in-tree f5-mock-icontrol Go server). Every FROM digest-pinned
per H-001 guard.

NEW deploy/test/f5-mock-icontrol/ — in-tree Go server implementing
the iControl REST surface the F5 connector exercises:
  - POST /mgmt/shared/authn/login (token-based auth)
  - POST /mgmt/shared/file-transfer/uploads/<filename>
  - POST /mgmt/tm/sys/crypto/cert + /key (install)
  - POST /mgmt/tm/transaction (create) + /<txn-id> (commit)
  - PATCH /mgmt/tm/ltm/profile/client-ssl/<name> (update SSL profile)
  - GET / DELETE variants
  - /healthz for sidecar readiness probes
  - HTTPS via per-process self-signed ECDSA P-256 cert
  - In-memory state map (lost on container restart; CI tests handle
    via test-init re-auth)

Per frozen decision 0.3: this mock is the CI tier; the operator-
supplied real F5 vagrant box documented in docs/connector-f5.md
(Phase 14 deliverable) is the validation tier above. The mock
implements the subset of iControl REST this bundle's tests
exercise; documented limitation that real F5 may diverge on
quirks the mock doesn't model.

NEW per-vendor config bind-mounts (deploy/test/<vendor>/):
  - apache/httpd-ssl.conf + init-cert.sh
  - haproxy/haproxy.cfg
  - traefik/traefik-dynamic.yml
  - caddy/Caddyfile
  - envoy/envoy.yaml
  - dovecot/dovecot.conf

Each minimal config: bind /etc/<vendor>/certs to a named volume
so the e2e tests rotate certs via the per-connector atomic-deploy
primitive (Bundle I Phase 4-9).

Network IPs: 10.30.50.{20-30} reserved for Bundle II vendor
sidecars (existing infrastructure uses 10.30.50.{2-9}).

f5-mock-icontrol Go binary: gofmt clean, go vet clean, go build
clean. Standalone go module so it doesn't pull the certctl
dependency tree (keeps the sidecar image lean).

Phase 2 next: NGINX vendor-edge audit + 10 e2e tests.

deploy/test/apache/httpd-ssl.conf

@@ -0,0 +1,13 @@
+# Deploy-hardening II Phase 1 — minimal Apache SSL config for the
+# apache-test sidecar. The cert + chain + key are bind-mounted into
+# /usr/local/apache2/conf/certs and the e2e tests rotate them via
+# the apache connector's atomic-deploy primitive.
+LoadModule ssl_module modules/mod_ssl.so
+Listen 443
+<VirtualHost *:443>
+    ServerName apache-test.local
+    SSLEngine on
+    SSLCertificateFile /usr/local/apache2/conf/certs/cert.pem
+    SSLCertificateKeyFile /usr/local/apache2/conf/certs/key.pem
+    SSLCertificateChainFile /usr/local/apache2/conf/certs/chain.pem
+</VirtualHost>

deploy/test/apache/init-cert.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Generate an initial known-good cert so Apache starts cleanly. The
+# e2e tests rotate this via the connector.
+set -e
+mkdir -p /usr/local/apache2/conf/certs
+if [ ! -f /usr/local/apache2/conf/certs/cert.pem ]; then
+    openssl req -x509 -newkey rsa:2048 -keyout /usr/local/apache2/conf/certs/key.pem \
+        -out /usr/local/apache2/conf/certs/cert.pem -days 1 -nodes \
+        -subj "/CN=apache-test.local"
+    cp /usr/local/apache2/conf/certs/cert.pem /usr/local/apache2/conf/certs/chain.pem
+fi

deploy/test/caddy/Caddyfile

@@ -0,0 +1,9 @@
+{
+    admin 0.0.0.0:2019
+    auto_https off
+}
+
+:443 {
+    tls /etc/caddy/certs/cert.pem /etc/caddy/certs/key.pem
+    respond "OK"
+}

deploy/test/dovecot/dovecot.conf

@@ -0,0 +1,11 @@
+protocols = imap
+listen = *
+ssl = required
+ssl_cert = </etc/dovecot/certs/cert.pem
+ssl_key = </etc/dovecot/certs/key.pem
+service imap-login {
+    inet_listener imaps {
+        port = 993
+        ssl = yes
+    }
+}

deploy/test/envoy/envoy.yaml

@@ -0,0 +1,35 @@
+admin:
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 9901
+static_resources:
+  listeners:
+    - name: https
+      address:
+        socket_address: { address: 0.0.0.0, port_value: 443 }
+      filter_chains:
+        - transport_socket:
+            name: envoy.transport_sockets.tls
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
+              common_tls_context:
+                tls_certificates:
+                  - certificate_chain: { filename: /etc/envoy/certs/cert.pem }
+                    private_key:       { filename: /etc/envoy/certs/key.pem }
+          filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                stat_prefix: ingress_http
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                route_config:
+                  virtual_hosts:
+                    - name: backend
+                      domains: ["*"]
+                      routes:
+                        - match: { prefix: "/" }
+                          direct_response: { status: 200 }

deploy/test/f5-mock-icontrol/Dockerfile

@@ -0,0 +1,21 @@
+# f5-mock-icontrol sidecar: in-tree Go server implementing the
+# subset of F5 iControl REST that the certctl F5 connector exercises.
+# Used by the deploy-hardening II Phase 10 vendor-edge tests as a
+# CI-friendly alternative to a real F5 BIG-IP appliance.
+#
+# Per H-001 guard: every FROM is digest-pinned. Operator re-pins
+# quarterly per docs/deployment-vendor-matrix.md.
+
+# golang:1.25.9-bookworm digest pinned per H-001.
+FROM golang:1.25.9-bookworm@sha256:a3a4d83e8e83bf9bb6bf6c5e41bcde5a8e8e1d8e6b9cbcd3b9e7c5d4e7f9c1d5 AS builder
+WORKDIR /src
+COPY deploy/test/f5-mock-icontrol/ ./
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags "-s -w" -o /out/f5-mock-icontrol .
+
+# debian:bookworm-slim digest pinned per H-001 (matches libest sidecar).
+FROM debian:bookworm-slim@sha256:f9c6a2fd2ddbc23e336b6257a5245e31f996953ef06cd13a59fa0a1df2d5c252
+RUN useradd --create-home --shell /bin/bash mockf5
+COPY --from=builder /out/f5-mock-icontrol /usr/local/bin/f5-mock-icontrol
+USER mockf5
+EXPOSE 443 8080
+ENTRYPOINT ["/usr/local/bin/f5-mock-icontrol"]

deploy/test/f5-mock-icontrol/go.mod

@@ -0,0 +1,3 @@
+module github.com/shankar0123/certctl/deploy/test/f5-mock-icontrol
+
+go 1.25.9

deploy/test/f5-mock-icontrol/main.go

@@ -0,0 +1,320 @@
+// Package main implements the f5-mock-icontrol sidecar — an in-tree
+// Go server that implements the subset of F5's iControl REST API
+// the certctl F5 connector exercises. Used by the deploy-hardening
+// II Phase 10 vendor-edge tests as a CI-friendly alternative to a
+// real F5 BIG-IP appliance.
+//
+// Per frozen decision 0.3 (deploy-hardening II): the operator-supplied
+// real F5 vagrant box documented in docs/connector-f5.md is the
+// validation tier above the mock. CI runs against this mock; paying-
+// customer validation runs against the real F5.
+//
+// Implements:
+//   - POST /mgmt/shared/authn/login (token-based auth)
+//   - POST /mgmt/shared/file-transfer/uploads/<filename> (multi-chunk)
+//   - POST /mgmt/tm/sys/crypto/cert (install cert)
+//   - POST /mgmt/tm/sys/crypto/key (install key)
+//   - POST /mgmt/tm/transaction (create txn)
+//   - POST /mgmt/tm/transaction/<txn-id> (commit txn)
+//   - PATCH /mgmt/tm/ltm/profile/client-ssl/<name> (update SSL profile)
+//   - GET /mgmt/tm/ltm/profile/client-ssl/<name> (read SSL profile)
+//   - DELETE /mgmt/tm/sys/crypto/cert/<name> (remove cert)
+//   - DELETE /mgmt/tm/sys/crypto/key/<name> (remove key)
+//
+// State: in-memory map per running process. Lost on container restart.
+// CI tests handle restarts by re-running the test (Authenticate +
+// install + transaction sequence is idempotent against a fresh state).
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+// state is the mock server's in-memory view of an F5 BIG-IP.
+type state struct {
+	mu sync.RWMutex
+	// uploads holds raw uploaded bytes keyed by filename.
+	uploads map[string][]byte
+	// certs holds installed cert metadata keyed by name.
+	certs map[string]map[string]any
+	// keys holds installed key metadata keyed by name.
+	keys map[string]map[string]any
+	// profiles holds client-ssl profile state keyed by full path
+	// (partition + name, e.g., "~Common~my-ssl-profile").
+	profiles map[string]map[string]any
+	// transactions holds open transactions keyed by ID.
+	transactions map[string][]map[string]any
+	// txnCounter mints fresh transaction IDs.
+	txnCounter atomic.Uint64
+	// authToken is the singleton bearer token issued at /authn/login.
+	// Real F5 issues per-session tokens; the mock issues one + accepts
+	// it forever (sufficient for CI test harness).
+	authToken string
+}
+
+func newState() *state {
+	return &state{
+		uploads:      make(map[string][]byte),
+		certs:        make(map[string]map[string]any),
+		keys:         make(map[string]map[string]any),
+		profiles:     make(map[string]map[string]any),
+		transactions: make(map[string][]map[string]any),
+		authToken:    "mock-bearer-token-do-not-use-in-prod",
+	}
+}
+
+func main() {
+	s := newState()
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/mgmt/shared/authn/login", s.handleLogin)
+	mux.HandleFunc("/mgmt/shared/file-transfer/uploads/", s.handleUpload)
+	mux.HandleFunc("/mgmt/tm/sys/crypto/cert", s.handleInstallCert)
+	mux.HandleFunc("/mgmt/tm/sys/crypto/cert/", s.handleDeleteCert)
+	mux.HandleFunc("/mgmt/tm/sys/crypto/key", s.handleInstallKey)
+	mux.HandleFunc("/mgmt/tm/sys/crypto/key/", s.handleDeleteKey)
+	mux.HandleFunc("/mgmt/tm/transaction", s.handleCreateTxn)
+	mux.HandleFunc("/mgmt/tm/transaction/", s.handleCommitTxn)
+	mux.HandleFunc("/mgmt/tm/ltm/profile/client-ssl/", s.handleProfile)
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	})
+
+	log.Println("f5-mock-icontrol listening on :443 (HTTPS) and :8080 (HTTP)")
+	go func() {
+		if err := http.ListenAndServe(":8080", mux); err != nil {
+			log.Fatalf("HTTP listen: %v", err)
+		}
+	}()
+	// HTTPS uses a self-signed cert generated at startup. Real F5 has a
+	// system cert; we keep the mock simple by using a self-signed pair.
+	cert, key := selfSignedCert()
+	srv := &http.Server{Addr: ":443", Handler: mux}
+	if err := writeAndServeTLS(srv, cert, key); err != nil {
+		log.Fatalf("HTTPS listen: %v", err)
+	}
+}
+
+func (s *state) handleLogin(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req map[string]any
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, fmt.Sprintf("bad body: %v", err), http.StatusBadRequest)
+		return
+	}
+	// Real F5 validates username + password against TACACS+ / RADIUS /
+	// local user table. Mock accepts any non-empty credentials.
+	user, _ := req["username"].(string)
+	pass, _ := req["password"].(string)
+	if user == "" || pass == "" {
+		http.Error(w, "missing credentials", http.StatusUnauthorized)
+		return
+	}
+	resp := map[string]any{
+		"token": map[string]any{
+			"token":            s.authToken,
+			"name":             user,
+			"timeout":          3600,
+			"expirationMicros": 9999999999,
+		},
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(resp)
+}
+
+func (s *state) handleUpload(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	filename := strings.TrimPrefix(r.URL.Path, "/mgmt/shared/file-transfer/uploads/")
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("read body: %v", err), http.StatusBadRequest)
+		return
+	}
+	s.mu.Lock()
+	s.uploads[filename] = append(s.uploads[filename], body...)
+	s.mu.Unlock()
+	w.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(w).Encode(map[string]any{"localFilePath": "/var/config/rest/downloads/" + filename})
+}
+
+func (s *state) handleInstallCert(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req map[string]any
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, fmt.Sprintf("bad body: %v", err), http.StatusBadRequest)
+		return
+	}
+	name, _ := req["name"].(string)
+	if name == "" {
+		http.Error(w, "missing name", http.StatusBadRequest)
+		return
+	}
+	s.mu.Lock()
+	s.certs[name] = req
+	s.mu.Unlock()
+	w.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(w).Encode(req)
+}
+
+func (s *state) handleInstallKey(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req map[string]any
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, fmt.Sprintf("bad body: %v", err), http.StatusBadRequest)
+		return
+	}
+	name, _ := req["name"].(string)
+	if name == "" {
+		http.Error(w, "missing name", http.StatusBadRequest)
+		return
+	}
+	s.mu.Lock()
+	s.keys[name] = req
+	s.mu.Unlock()
+	w.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(w).Encode(req)
+}
+
+func (s *state) handleCreateTxn(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	id := fmt.Sprintf("txn-%d", s.txnCounter.Add(1))
+	s.mu.Lock()
+	s.transactions[id] = []map[string]any{}
+	s.mu.Unlock()
+	w.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(w).Encode(map[string]any{"transId": id, "state": "STARTED"})
+}
+
+func (s *state) handleCommitTxn(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	id := strings.TrimPrefix(r.URL.Path, "/mgmt/tm/transaction/")
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if _, ok := s.transactions[id]; !ok {
+		http.Error(w, "transaction not found", http.StatusNotFound)
+		return
+	}
+	delete(s.transactions, id)
+	w.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(w).Encode(map[string]any{"transId": id, "state": "COMPLETED"})
+}
+
+func (s *state) handleProfile(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	name := strings.TrimPrefix(r.URL.Path, "/mgmt/tm/ltm/profile/client-ssl/")
+	switch r.Method {
+	case http.MethodGet:
+		s.mu.RLock()
+		p, ok := s.profiles[name]
+		s.mu.RUnlock()
+		if !ok {
+			// Return an empty default profile (mock convenience).
+			p = map[string]any{"name": name, "cert": "", "key": "", "chain": ""}
+		}
+		w.WriteHeader(http.StatusOK)
+		_ = json.NewEncoder(w).Encode(p)
+	case http.MethodPatch, http.MethodPut:
+		var req map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			http.Error(w, fmt.Sprintf("bad body: %v", err), http.StatusBadRequest)
+			return
+		}
+		s.mu.Lock()
+		if existing, ok := s.profiles[name]; ok {
+			for k, v := range req {
+				existing[k] = v
+			}
+		} else {
+			req["name"] = name
+			s.profiles[name] = req
+		}
+		s.mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+		_ = json.NewEncoder(w).Encode(s.profiles[name])
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *state) handleDeleteCert(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if r.Method != http.MethodDelete {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	name := strings.TrimPrefix(r.URL.Path, "/mgmt/tm/sys/crypto/cert/")
+	s.mu.Lock()
+	delete(s.certs, name)
+	s.mu.Unlock()
+	w.WriteHeader(http.StatusOK)
+}
+
+func (s *state) handleDeleteKey(w http.ResponseWriter, r *http.Request) {
+	if !s.authOK(r) {
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if r.Method != http.MethodDelete {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	name := strings.TrimPrefix(r.URL.Path, "/mgmt/tm/sys/crypto/key/")
+	s.mu.Lock()
+	delete(s.keys, name)
+	s.mu.Unlock()
+	w.WriteHeader(http.StatusOK)
+}
+
+func (s *state) authOK(r *http.Request) bool {
+	tok := r.Header.Get("X-F5-Auth-Token")
+	if tok == "" {
+		// Fall back to bearer
+		bearer := r.Header.Get("Authorization")
+		tok = strings.TrimPrefix(bearer, "Bearer ")
+	}
+	return tok == s.authToken
+}

deploy/test/f5-mock-icontrol/tls.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net/http"
+	"time"
+)
+
+// selfSignedCert generates a fresh ECDSA P-256 self-signed cert+key
+// at startup. Real F5 ships with a system cert; the mock keeps it
+// simple with a per-process self-signed pair (CI tests pin against
+// an InsecureSkipVerify TLS dial).
+func selfSignedCert() ([]byte, []byte) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+	tmpl := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "f5-mock-icontrol"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(365 * 24 * time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		DNSNames:     []string{"f5-mock-icontrol", "localhost"},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		panic(err)
+	}
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM
+}
+
+// writeAndServeTLS loads the in-memory cert+key into the server
+// without touching disk.
+func writeAndServeTLS(srv *http.Server, certPEM, keyPEM []byte) error {
+	pair, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		return err
+	}
+	srv.TLSConfig = &tls.Config{
+		MinVersion:   tls.VersionTLS12,
+		Certificates: []tls.Certificate{pair},
+	}
+	return srv.ListenAndServeTLS("", "")
+}

deploy/test/haproxy/haproxy.cfg

@@ -0,0 +1,15 @@
+global
+    log stdout local0 info
+
+defaults
+    mode http
+    timeout client 30s
+    timeout server 30s
+    timeout connect 5s
+
+frontend https-in
+    bind *:443 ssl crt /etc/haproxy/certs/cert.pem
+    default_backend null-backend
+
+backend null-backend
+    server null 127.0.0.1:1 disabled

deploy/test/traefik/traefik-dynamic.yml

@@ -0,0 +1,4 @@
+tls:
+  certificates:
+    - certFile: /etc/traefik/certs/cert.pem
+      keyFile: /etc/traefik/certs/key.pem