certctl/deploy/test/loadtest/docker-compose.yml

# =============================================================================
# certctl Load-Test Harness — Docker Compose
# =============================================================================
#
# Spins up a minimal certctl stack and runs a k6 driver against it to capture
# p50 / p95 / p99 latency for the certificate-management API hot path AND
# (Bundle 10 of the 2026-05-02 deployment-target audit) per-target-type
# TCP+TLS handshake throughput against four target sidecars (nginx, apache,
# haproxy, f5-mock).
#
# Stack:
#   1. postgres                 — empty database (server runs migrations + seeds at boot)
#   2. certctl-tls-init         — one-shot init container; writes self-signed
#                                  server.crt/.key/ca.crt into ./certs (bind
#                                  mount, host-readable so the k6 container
#                                  can pin against it via volumes)
#   3. certctl-server           — HTTPS API on :8443, demo-seed enabled so
#                                  the k6 script has iss-local + an operator
#                                  + a team ready to reference in
#                                  CreateCertificate payloads
#   4. target-tls-init          — Bundle 10: shared starter cert+key for the
#                                  four target sidecars (nginx, apache,
#                                  haproxy, f5-mock). Each daemon boots with
#                                  this cert; the loadtest scenarios connect
#                                  at sustained rates to measure handshake
#                                  latency tagged by target_type.
#   5. nginx-target             — Bundle 10: HTTPS on internal :443.
#   6. apache-target            — Bundle 10: HTTPS on internal :443.
#   7. haproxy-target           — Bundle 10: HTTPS on internal :443.
#   8. f5-mock-target           — Bundle 10: iControl REST on internal :443
#                                  + plaintext HTTP on internal :8080. Runs
#                                  the in-tree f5-mock-icontrol image
#                                  (deploy/test/f5-mock-icontrol/).
#   9. k6                       — runs k6.js once and exits with the
#                                  threshold-driven exit code (zero on green,
#                                  non-zero on any threshold breach so
#                                  `make loadtest` surfaces regressions as a
#                                  failed shell command).
#
# Out of scope for v1 of the connector-tier harness (Bundle 10):
#   - Kubernetes target via kind-in-docker. kind requires `privileged: true`
#     and Docker-in-Docker semantics that are operationally fragile in CI;
#     the K8s connector loadtest is a follow-up that needs Bundle 2's real
#     k8s.io/client-go to land first.
#   - Full agent-driven deploy poll loop (POST cert → poll deployments →
#     verify served cert matches what was deployed). The harness measures
#     handshake throughput against the target sidecars directly — that's
#     enough to validate the sidecars are operational under load and gives
#     procurement a per-target latency number that doesn't depend on the
#     agent registration + target-binding API surface being plumbed
#     end-to-end in the loadtest stack.
#
# Usage:  make loadtest  (from the repo root)
# Manual: cd deploy/test/loadtest && docker compose up --abort-on-container-exit --exit-code-from k6
#
# Audit reference (API tier):       2026-05-01 issuer coverage audit fix #8.
# Audit reference (connector tier): 2026-05-02 deployment-target audit Bundle 10.
# =============================================================================

services:

  # ---------------------------------------------------------------------------
  # Self-signed TLS bootstrap. Mirrors the deploy/docker-compose.test.yml
  # tls-init pattern exactly: bind-mount instead of named volume so the host
  # (and the sibling k6 container) can read ca.crt without a chown dance.
  # See deploy/docker-compose.test.yml::certctl-tls-init for the full rationale.
  # ---------------------------------------------------------------------------
  certctl-tls-init:
    image: alpine/openssl:latest
    container_name: certctl-loadtest-tls-init
    restart: "no"
    entrypoint: /bin/sh
    command:
      - -c
      - |
        set -eu
        CERT=/etc/certctl/tls/server.crt
        KEY=/etc/certctl/tls/server.key
        CA=/etc/certctl/tls/ca.crt
        if [ -f "$$CERT" ] && [ -f "$$KEY" ] && [ -f "$$CA" ]; then
          echo "TLS cert already present — skipping generation"
        else
          mkdir -p /etc/certctl/tls
          openssl req -x509 -newkey ec \
            -pkeyopt ec_paramgen_curve:P-256 \
            -nodes \
            -keyout "$$KEY" \
            -out "$$CERT" \
            -days 3650 \
            -subj "/CN=certctl-server" \
            -addext "subjectAltName=DNS:certctl-server,DNS:localhost,IP:127.0.0.1"
          cp "$$CERT" "$$CA"
          echo "Generated self-signed TLS cert (ECDSA-P256, 3650d, CN=certctl-server)"
        fi
        chmod 0644 "$$CERT" "$$CA"
        chmod 0600 "$$KEY"
    volumes:
      - ./certs:/etc/certctl/tls

  # ---------------------------------------------------------------------------
  # Database. The server runs migrations + seed.sql + (because
  # CERTCTL_DEMO_SEED=true below) seed_demo.sql at boot — so the load-test
  # k6 script can reference iss-local, o-alice, t-platform, and rp-default
  # without a separate seed step.
  # ---------------------------------------------------------------------------
  postgres:
    image: postgres:16-alpine
    container_name: certctl-loadtest-postgres
    environment:
      POSTGRES_DB: certctl
      POSTGRES_USER: certctl
      POSTGRES_PASSWORD: loadtestpass
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U certctl"]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 30s

  # ---------------------------------------------------------------------------
  # certctl server. Built from the repo root Dockerfile (same as production).
  # Demo seed is enabled so referenced FK rows exist when the k6 script
  # POSTs CreateCertificate payloads. Auth is api-key with a deterministic
  # token the k6 script knows.
  # ---------------------------------------------------------------------------
  certctl-server:
    build:
      context: ../../..
      dockerfile: Dockerfile
      args:
        HTTP_PROXY: ${HTTP_PROXY:-}
        HTTPS_PROXY: ${HTTPS_PROXY:-}
        NO_PROXY: ${NO_PROXY:-}
    container_name: certctl-loadtest-server
    depends_on:
      postgres:
        condition: service_healthy
      certctl-tls-init:
        condition: service_completed_successfully
    environment:
      CERTCTL_DATABASE_URL: postgres://certctl:loadtestpass@postgres:5432/certctl?sslmode=disable
      CERTCTL_SERVER_HOST: 0.0.0.0
      CERTCTL_SERVER_PORT: 8443
      CERTCTL_SERVER_TLS_CERT_PATH: /etc/certctl/tls/server.crt
      CERTCTL_SERVER_TLS_KEY_PATH: /etc/certctl/tls/server.key
      CERTCTL_LOG_LEVEL: warn
      CERTCTL_AUTH_TYPE: api-key
      CERTCTL_AUTH_SECRET: load-test-token
      CERTCTL_KEYGEN_MODE: agent
      # CERTCTL_DEMO_SEED=true triggers seed_demo.sql which creates iss-local,
      # o-alice, t-platform, rp-standard so CreateCertificate FK validation
      # has rows to bind to.
      CERTCTL_DEMO_SEED: "true"
      # Bigger body limit so listing 100s of certs in the GET scenario
      # doesn't 413 once the harness has been running for a few minutes.
      CERTCTL_MAX_BODY_SIZE: "10485760"
      # Encryption key (≥32 bytes per H-1 floor — the test compose's
      # documented value).
      CERTCTL_CONFIG_ENCRYPTION_KEY: "loadtest-key-must-be-32-bytes-long-yes"
    volumes:
      - ./certs:/etc/certctl/tls:ro
    healthcheck:
      # /healthz is unauthenticated. -k because the cert is self-signed.
      test: ["CMD-SHELL", "wget -q --no-check-certificate -O- https://localhost:8443/healthz || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 30
      start_period: 60s

  # ---------------------------------------------------------------------------
  # Bundle 10: target-side TLS bootstrap. Mints a single ECDSA-P256 self-
  # signed cert + key into a shared ./fixtures/target-certs/ volume that the
  # four target sidecars (nginx, apache, haproxy) mount read-only. f5-mock
  # generates its own self-signed cert at startup (see
  # deploy/test/f5-mock-icontrol/tls.go) so it doesn't need this volume.
  #
  # The loadtest scenarios don't care which cert the target serves — only
  # that the daemon is up and completing TLS handshakes at the configured
  # rate. The starter cert exists so each daemon boots green; once Bundle 2
  # (real K8s client) + agent-driven deploy poll is plumbed in v2 of the
  # harness, deploys would overwrite this cert.
  # ---------------------------------------------------------------------------
  target-tls-init:
    image: alpine/openssl:latest
    container_name: certctl-loadtest-target-tls-init
    restart: "no"
    entrypoint: /bin/sh
    command:
      - -c
      - |
        set -eu
        CERT=/certs/target.crt
        KEY=/certs/target.key
        PEM=/certs/target.pem
        if [ -f "$$CERT" ] && [ -f "$$KEY" ] && [ -f "$$PEM" ]; then
          echo "Target TLS cert already present — skipping generation"
        else
          mkdir -p /certs
          openssl req -x509 -newkey ec \
            -pkeyopt ec_paramgen_curve:P-256 \
            -nodes \
            -keyout "$$KEY" \
            -out "$$CERT" \
            -days 365 \
            -subj "/CN=loadtest-target" \
            -addext "subjectAltName=DNS:nginx-target,DNS:apache-target,DNS:haproxy-target,DNS:f5-mock-target,DNS:localhost,IP:127.0.0.1"
          # HAProxy expects cert+key concatenated into a single PEM file
          # at the path supplied to `bind ... ssl crt <path>`. Build it
          # alongside the cert/key pair so the haproxy-target's mount
          # works without a per-daemon ENTRYPOINT shim.
          cat "$$CERT" "$$KEY" > "$$PEM"
          echo "Generated target starter cert (ECDSA-P256, 365d, multi-SAN)"
        fi
        # World-readable so non-root container users (haproxy uses uid 99,
        # apache uses uid 1) can read the key. This is fine for a load-test
        # starter cert; production wouldn't do this.
        chmod 0644 "$$CERT" "$$KEY" "$$PEM"
    volumes:
      - ./fixtures/target-certs:/certs

  # ---------------------------------------------------------------------------
  # nginx-target. Listens on internal :443 with the starter cert. The
  # k6 nginx_handshake scenario connects at 100 conns/min for 5 minutes.
  # ---------------------------------------------------------------------------
  nginx-target:
    image: nginx:1.27-alpine
    container_name: certctl-loadtest-nginx
    depends_on:
      target-tls-init:
        condition: service_completed_successfully
    volumes:
      - ./fixtures/target-certs:/etc/nginx/certs:ro
      - ./fixtures/nginx.conf:/etc/nginx/nginx.conf:ro
    healthcheck:
      test: ["CMD-SHELL", "wget -q --no-check-certificate -O- https://localhost:443/ || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 20
      start_period: 15s

  # ---------------------------------------------------------------------------
  # apache-target. Listens on internal :443. The bundled httpd.conf loads
  # the minimum module set + a single SSL-terminated vhost.
  # ---------------------------------------------------------------------------
  apache-target:
    image: httpd:2.4-alpine
    container_name: certctl-loadtest-apache
    depends_on:
      target-tls-init:
        condition: service_completed_successfully
    volumes:
      - ./fixtures/target-certs:/usr/local/apache2/conf/certs:ro
      - ./fixtures/httpd.conf:/usr/local/apache2/conf/httpd.conf:ro
    healthcheck:
      test: ["CMD-SHELL", "wget -q --no-check-certificate -O- https://localhost:443/ || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 20
      start_period: 15s

  # ---------------------------------------------------------------------------
  # haproxy-target. Listens on internal :443 with SSL termination. The
  # haproxy.cfg references /usr/local/etc/haproxy/certs/target.pem which
  # target-tls-init writes (cert + key concatenated).
  # ---------------------------------------------------------------------------
  haproxy-target:
    image: haproxy:2.9-alpine
    container_name: certctl-loadtest-haproxy
    depends_on:
      target-tls-init:
        condition: service_completed_successfully
    volumes:
      - ./fixtures/target-certs:/usr/local/etc/haproxy/certs:ro
      - ./fixtures/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
    healthcheck:
      # HAProxy doesn't ship with wget/curl; use the openssl-based handshake
      # check instead. The /dev/null redirect drops the response body so
      # large logs don't accumulate over the run.
      test: ["CMD-SHELL", "echo Q | openssl s_client -connect localhost:443 -servername localhost 2>/dev/null | grep -q 'BEGIN CERTIFICATE'"]
      interval: 5s
      timeout: 3s
      retries: 20
      start_period: 15s

  # ---------------------------------------------------------------------------
  # f5-mock target. Re-uses the in-tree f5-mock-icontrol image (already
  # used by the deploy-vendor-e2e CI job). Generates its own self-signed
  # cert at startup; listens on internal :443 (HTTPS, iControl REST) and
  # :8080 (plaintext HTTP). The k6 f5_handshake scenario hits the
  # /healthz endpoint.
  # ---------------------------------------------------------------------------
  f5-mock-target:
    # Long-form build to match docker-compose.test.yml: the Dockerfile
    # has `COPY deploy/test/f5-mock-icontrol/ ./` which assumes the
    # build context is the REPO ROOT. The previous shorthand form
    # `build: ../f5-mock-icontrol` set the context to the
    # f5-mock-icontrol directory itself, breaking the COPY at CI build
    # time (run #25305811340: "deploy/test/f5-mock-icontrol: not found").
    build:
      context: ../../..
      dockerfile: deploy/test/f5-mock-icontrol/Dockerfile
    container_name: certctl-loadtest-f5-mock
    healthcheck:
      test: ["CMD-SHELL", "wget -q -O- http://localhost:8080/healthz || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 20
      start_period: 15s

  # ---------------------------------------------------------------------------
  # k6 driver. Pinned to a specific version so threshold expressions stay
  # stable across runs. --insecure-skip-tls-verify because the server cert is
  # self-signed; the load test isn't a TLS conformance test. The k6 process
  # exits non-zero if any threshold is breached, which the parent
  # `docker compose up --exit-code-from k6` propagates as the compose exit
  # code, which `make loadtest` then surfaces as the make-target exit code.
  # ---------------------------------------------------------------------------
  k6:
    image: grafana/k6:0.54.0
    container_name: certctl-loadtest-k6
    depends_on:
      certctl-server:
        condition: service_healthy
      # Bundle 10: wait for the four target sidecars to be healthy before
      # firing the connector-tier scenarios. Saves the operator from
      # spurious "connection refused" errors during the first ~15s of the
      # run while target daemons are coming up.
      nginx-target:
        condition: service_healthy
      apache-target:
        condition: service_healthy
      haproxy-target:
        condition: service_healthy
      f5-mock-target:
        condition: service_healthy
    environment:
      CERTCTL_BASE: https://certctl-server:8443
      CERTCTL_TOKEN: load-test-token
      K6_INSECURE_SKIP_TLS_VERIFY: "true"
      # Bundle 10: per-target sidecar URLs the connector-tier scenarios
      # connect to. Internal docker-compose DNS — k6 resolves these via
      # the default user network's resolver.
      NGINX_TARGET_URL:   https://nginx-target:443
      APACHE_TARGET_URL:  https://apache-target:443
      HAPROXY_TARGET_URL: https://haproxy-target:443
      F5_TARGET_URL:      https://f5-mock-target:443
    volumes:
      - ./k6.js:/scripts/k6.js:ro
      - ./results:/results
    command:
      - run
      - --summary-export=/results/summary.json
      - /scripts/k6.js

  # ===========================================================================
  # Phase 8 SCALE-H2 — scale-tier scenarios (opt-in via `--profile scale`).
  #
  # The default `make loadtest` path runs the API tier + connector tier
  # scenarios above against the demo-scale seed. The Phase 8 scenarios are
  # heavier (10K cert + 5K agent fixtures) and would slow the default path
  # without serving the per-PR signal the existing run targets, so they live
  # behind a separate compose profile.
  #
  # Three components, all profile-gated:
  #   1. scale-seed    — one-shot init that runs ./seed/*.sql against the
  #                      same postgres the server uses. Idempotent.
  #   2. k6-scale-bulk / k6-scale-acme / k6-scale-agent — one driver each
  #                      for the three Phase 8 scenarios. The matrix dispatch
  #                      in .github/workflows/loadtest.yml picks one per job.
  #
  # Run a single scale scenario locally:
  #   docker compose --profile scale up \
  #       --abort-on-container-exit --exit-code-from k6-scale-bulk \
  #       scale-seed k6-scale-bulk
  # ===========================================================================

  scale-seed:
    # postgres:16-alpine bundles psql; no extra image needed.
    image: postgres:16-alpine
    container_name: certctl-loadtest-scale-seed
    restart: "no"
    profiles: ["scale"]
    depends_on:
      postgres:
        condition: service_healthy
      # Wait for certctl-server to be healthy — the server runs schema
      # migrations + seed_demo.sql at boot. The Phase 8 seeds reference
      # FKs (iss-local, o-alice, t-platform, rp-standard) that
      # seed_demo.sql creates, so the order MUST be:
      #   postgres up → server runs migrations + seed_demo.sql → scale-seed runs
      certctl-server:
        condition: service_healthy
    environment:
      PGHOST: postgres
      PGUSER: certctl
      PGPASSWORD: loadtestpass
      PGDATABASE: certctl
    volumes:
      - ./seed:/seed:ro
    entrypoint: /bin/sh
    command:
      - -c
      - |
        set -eu
        echo "==> Phase 8 scale-seed: running SQL fixtures (lexical order)"
        for f in /seed/*.sql; do
            echo "----> $$f"
            psql -v ON_ERROR_STOP=1 -f "$$f"
        done
        echo "==> Phase 8 scale-seed: complete"

  k6-scale-bulk:
    image: grafana/k6:0.54.0
    container_name: certctl-loadtest-k6-bulk
    profiles: ["scale"]
    depends_on:
      certctl-server:
        condition: service_healthy
      scale-seed:
        condition: service_completed_successfully
    environment:
      CERTCTL_BASE: https://certctl-server:8443
      CERTCTL_TOKEN: load-test-token
      K6_INSECURE_SKIP_TLS_VERIFY: "true"
    volumes:
      - ./k6/bulk_renewal.js:/scripts/bulk_renewal.js:ro
      - ./results:/results
    command:
      - run
      - --summary-export=/results/summary-bulk-renewal.json
      - /scripts/bulk_renewal.js

  k6-scale-acme:
    image: grafana/k6:0.54.0
    container_name: certctl-loadtest-k6-acme
    profiles: ["scale"]
    depends_on:
      certctl-server:
        condition: service_healthy
      # ACME scenario doesn't depend on the SQL seeds (it hits the
      # unauthenticated directory + nonce + ARI surface) but routing
      # it through the same dependency chain keeps the compose
      # ordering predictable across the three scale jobs.
      scale-seed:
        condition: service_completed_successfully
    environment:
      CERTCTL_ACME_DIRECTORY: https://certctl-server:8443/acme/profile/prof-test/directory
      K6_INSECURE_SKIP_TLS_VERIFY: "true"
    volumes:
      - ./k6/acme_burst.js:/scripts/acme_burst.js:ro
      - ./results:/results
    command:
      - run
      - --summary-export=/results/summary-acme-burst.json
      - /scripts/acme_burst.js

  k6-scale-agent:
    image: grafana/k6:0.54.0
    container_name: certctl-loadtest-k6-agent
    profiles: ["scale"]
    depends_on:
      certctl-server:
        condition: service_healthy
      scale-seed:
        condition: service_completed_successfully
    environment:
      CERTCTL_BASE: https://certctl-server:8443
      CERTCTL_TOKEN: load-test-token
      K6_INSECURE_SKIP_TLS_VERIFY: "true"
      # Match the seed's 5K-agent fleet.
      K6_AGENT_FLEET: "5000"
    volumes:
      - ./k6/agent_storm.js:/scripts/agent_storm.js:ro
      - ./results:/results
    command:
      - run
      - --summary-export=/results/summary-agent-storm.json
      - /scripts/agent_storm.js