mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 22:31:36 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 52248be717 | |||
| 04c7eca615 | |||
| 6e646e0fe8 | |||
| 675b87ba63 | |||
| 707d8de4fb | |||
| 0725713e19 | |||
| 1ee77c89f8 |
@@ -148,8 +148,34 @@ jobs:
|
||||
with:
|
||||
version: '3.13.0'
|
||||
|
||||
# HTTPS-Everywhere (v2.0.47): the chart fails render when no TLS source is
|
||||
# configured. Every lint/template invocation below must pick exactly one
|
||||
# provisioning mode — see deploy/helm/certctl/templates/_helpers.tpl
|
||||
# (certctl.tls.required) and docs/tls.md.
|
||||
- name: Lint Helm Chart
|
||||
run: helm lint deploy/helm/certctl/
|
||||
run: |
|
||||
helm lint deploy/helm/certctl/ \
|
||||
--set server.tls.existingSecret=certctl-tls-ci
|
||||
|
||||
- name: Template Helm Chart
|
||||
run: helm template certctl deploy/helm/certctl/ > /dev/null
|
||||
- name: Template Helm Chart (existingSecret mode)
|
||||
run: |
|
||||
helm template certctl deploy/helm/certctl/ \
|
||||
--set server.tls.existingSecret=certctl-tls-ci \
|
||||
> /dev/null
|
||||
|
||||
- name: Template Helm Chart (cert-manager mode)
|
||||
run: |
|
||||
helm template certctl deploy/helm/certctl/ \
|
||||
--set server.tls.certManager.enabled=true \
|
||||
--set server.tls.certManager.issuerRef.name=letsencrypt-prod \
|
||||
> /dev/null
|
||||
|
||||
- name: Template Helm Chart (guard fails without TLS)
|
||||
run: |
|
||||
# Inverse test: the chart MUST refuse to render when no TLS source is
|
||||
# configured. If this ever renders successfully, the fail-loud guard
|
||||
# in certctl.tls.required has regressed.
|
||||
if helm template certctl deploy/helm/certctl/ > /dev/null 2>&1; then
|
||||
echo "::error::Helm chart rendered without a TLS source — fail-loud guard regressed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
+13
-1
@@ -63,6 +63,7 @@ certctl-cli
|
||||
/server
|
||||
/agent
|
||||
/cli
|
||||
/mcp-server
|
||||
|
||||
# Private strategy docs
|
||||
strategy.md
|
||||
@@ -71,9 +72,20 @@ SECURITY_REMEDIATION.md
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
mcp-server
|
||||
|
||||
# Local Go build/module caches (session-scoped, never committed)
|
||||
/.gocache/
|
||||
/.gomodcache/
|
||||
/.gopath/
|
||||
/.gomodcache-gopath/
|
||||
|
||||
# Design scratch files (session-scoped)
|
||||
/.i004-design.md
|
||||
/.i005-design.md
|
||||
|
||||
# HTTPS-Everywhere (M-007) Phase 6: the docker-compose.test.yml tls-init
|
||||
# container writes ca.crt / server.crt / server.key into this directory so
|
||||
# the host-side integration_test.go binary can pin the CA via
|
||||
# CERTCTL_TEST_CA_BUNDLE=./certs/ca.crt. Material is regenerated on every
|
||||
# `docker compose up` and never belongs in git.
|
||||
/deploy/test/certs/
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to certctl are documented in this file. Dates use ISO 8601. Versions follow [Semantic Versioning](https://semver.org/).
|
||||
|
||||
## [2.2.0] — 2026-04-19
|
||||
|
||||
### HTTPS Everywhere — The Irony
|
||||
|
||||
> certctl manages other teams' certificates. Until v2.2, it didn't terminate TLS on its own control plane. We treated the server as an internal service sitting behind whatever TLS-terminating infrastructure the operator already owned — reverse proxies, Kubernetes Ingress controllers, service mesh sidecars. Working through an EST coverage-gap audit surfaced this as a credibility problem we wanted to fix head-on: a cert-lifecycle product should ship with HTTPS by default. This release flips that. Self-signed bootstrap for docker-compose demos, operator-supplied Secret for Helm (with optional cert-manager integration), and a one-step cutover with no backward-compat bridge. Out-of-date agents will fail at the TLS handshake layer on upgrade; the upgrade guide walks operators through the roll.
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- **HTTPS-only control plane. The plaintext HTTP listener is gone.** There is no `CERTCTL_TLS_ENABLED=false` escape hatch and no `:8080` fallback. Operators who were running certctl behind their own TLS terminator must either (a) continue doing so and let the downstream TLS terminator talk to certctl's HTTPS listener, or (b) bring their own cert/key and terminate on certctl directly. Either path requires config changes — see `docs/upgrade-to-tls.md` for a one-step cutover.
|
||||
- **Agents reject `CERTCTL_SERVER_URL=http://...` at startup.** This is a pre-flight config validation failure with a fail-loud diagnostic pointing at `docs/upgrade-to-tls.md`. Not a TCP-refused, not a TLS-handshake-error — the agent will not even attempt the network call. Every agent deployment must be reconfigured before upgrading the server.
|
||||
- **CLI and MCP clients require `https://` URLs.** Same pre-flight rejection of plaintext schemes.
|
||||
- **TLS 1.2 is not supported. TLS 1.3 only.** The server's `tls.Config.MinVersion` is pinned to `tls.VersionTLS13`. Any client still negotiating TLS 1.2 will fail at the handshake. Modern curl, Go stdlib, browsers, and Kubernetes tooling all default to 1.3-capable; legacy clients may need an upgrade.
|
||||
- **Helm chart requires a TLS source.** `helm install` without one of `server.tls.existingSecret`, `server.tls.certManager.enabled`, or (for eval only) `server.tls.selfSigned.enabled` fails at template time with a diagnostic pointing at `docs/tls.md`. There is no default-to-plaintext path.
|
||||
|
||||
### Added
|
||||
|
||||
- **Self-signed bootstrap for Docker Compose demos.** A `certctl-tls-init` init container runs before the server on first boot, generates a SAN-valid self-signed cert into `deploy/test/certs/`, and exits. The server mounts the resulting cert/key. Every curl in the demo stack pins against `./deploy/test/certs/ca.crt` with `--cacert`.
|
||||
- **Helm chart TLS provisioning — three modes.** Operator-supplied Secret (`server.tls.existingSecret`), cert-manager integration (`server.tls.certManager.enabled` with issuer selection), or self-signed (`server.tls.selfSigned.enabled` — eval only, not supported for production). Chart templates enforce exactly one is active.
|
||||
- **Hot-reload of TLS cert/key on `SIGHUP`.** Overwrite the cert/key on disk, send `SIGHUP` to the server PID, watch the `slog.Info("tls.reload", ...)` log line, and new TLS connections use the new cert. Failure during reload is logged and does not crash the server; the previous cert remains in use.
|
||||
- **Agent CA-bundle env vars.** `CERTCTL_SERVER_CA_BUNDLE_PATH` points at a PEM file the agent's HTTP client will trust. `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY` disables verification (development only — the agent logs a loud warning at startup). `install-agent.sh` writes both as commented template lines into the generated `agent.env`.
|
||||
- **Integration test suite runs over HTTPS.** `go test -tags=integration ./deploy/test/...` stands up the full Compose stack, extracts the self-signed CA bundle, and exercises every certctl API over `https://localhost:8443`. All 34 subtests green.
|
||||
- **`docs/tls.md`** — cert provisioning patterns: bring-your-own Secret, cert-manager, self-signed bootstrap, SAN requirements, rotation workflows, SIGHUP reload semantics, troubleshooting.
|
||||
- **`docs/upgrade-to-tls.md`** — one-step cutover guide for existing v2.1 operators. Walks through the agent fleet roll, Helm upgrade sequencing, downgrade-is-not-supported warnings, and cert-provisioning decision tree.
|
||||
|
||||
### Changed
|
||||
|
||||
- `cmd/server/main.go` now calls `http.Server.ListenAndServeTLS(certFile, keyFile)`. The plaintext `ListenAndServe` code path is deleted — `grep -rn "ListenAndServe[^T]" cmd/ internal/` returns zero hits.
|
||||
- All documentation curls (`docs/testing-guide.md`, `docs/quickstart.md`, `deploy/helm/INSTALLATION.md`, `deploy/helm/DEPLOYMENT_GUIDE.md`, `deploy/ENVIRONMENTS.md`, `docs/openapi.md`, migration guides, example READMEs) use `https://localhost:8443` and `--cacert` against the demo stack's bundle.
|
||||
- OpenAPI spec (`api/openapi.yaml`) `servers` blocks default to `https://localhost:8443`.
|
||||
|
||||
### Security
|
||||
|
||||
- TLS 1.3 pinned via `tls.Config.MinVersion = tls.VersionTLS13`.
|
||||
- Plaintext HTTP listener removed entirely — no port 8080, no `Upgrade-Insecure-Requests`, no HSTS-required redirect dance. There is only one port: 8443, TLS 1.3.
|
||||
- `grep -rn "http://" cmd/ internal/` returns zero hits outside test fixtures and the agent-side URL-scheme rejection error message.
|
||||
|
||||
### Upgrade Notes
|
||||
|
||||
Read `docs/upgrade-to-tls.md` before upgrading. The short version:
|
||||
|
||||
1. Pick a TLS source — bring-your-own cert, cert-manager, or self-signed bootstrap.
|
||||
2. Upgrade the server with TLS configured. First boot over HTTPS.
|
||||
3. Roll the agent fleet: set `CERTCTL_SERVER_URL=https://...` and, if using a private CA, `CERTCTL_SERVER_CA_BUNDLE_PATH`. Old agents will fail loud at startup — expected.
|
||||
4. Roll CLI/MCP clients the same way.
|
||||
|
||||
There is no backward-compat bridge. There is no dual-listener mode. The cutover is one step.
|
||||
@@ -197,7 +197,7 @@ cd certctl
|
||||
docker compose -f deploy/docker-compose.yml up -d --build
|
||||
```
|
||||
|
||||
Wait ~30 seconds, then open **http://localhost:8443** in your browser. The onboarding wizard walks you through connecting a CA, deploying an agent, and issuing your first certificate.
|
||||
Wait ~30 seconds, then open **https://localhost:8443** in your browser. (The shipped `docker-compose.yml` self-signs a cert via the `certctl-tls-init` init container on first boot — accept the browser warning for the demo, or feed the generated `ca.crt` to your client.) The onboarding wizard walks you through connecting a CA, deploying an agent, and issuing your first certificate.
|
||||
|
||||
**Want a pre-populated demo instead?** Add the demo override to see 32 certificates across 10 issuers, 8 agents, and 180 days of realistic history:
|
||||
|
||||
@@ -208,10 +208,12 @@ docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.demo.yml up
|
||||
The `deploy/` directory has four compose files: `docker-compose.yml` (base platform), `docker-compose.demo.yml` (demo data overlay), `docker-compose.dev.yml` (PgAdmin + debug logging), and `docker-compose.test.yml` (standalone integration tests with real CA backends). See the [Docker Compose Environments Guide](deploy/ENVIRONMENTS.md) for a service-by-service walkthrough, or the [Quick Start](docs/quickstart.md#docker-compose-environments) for a summary.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8443/health
|
||||
curl --cacert $(docker compose -f deploy/docker-compose.yml exec -T certctl-server cat /etc/certctl/tls/ca.crt) https://localhost:8443/health
|
||||
# {"status":"healthy"}
|
||||
```
|
||||
|
||||
The control plane is HTTPS-only (TLS 1.3, no plaintext listener). See [`docs/tls.md`](docs/tls.md) for cert provisioning patterns and [`docs/upgrade-to-tls.md`](docs/upgrade-to-tls.md) if you're upgrading from a pre-v2.2 release.
|
||||
|
||||
### Agent Install (One-Liner)
|
||||
|
||||
```bash
|
||||
@@ -326,8 +328,9 @@ Each directory contains a `docker-compose.yml` and a `README.md` explaining the
|
||||
go install github.com/shankar0123/certctl/cmd/cli@latest
|
||||
|
||||
# Configure
|
||||
export CERTCTL_SERVER_URL=http://localhost:8443
|
||||
export CERTCTL_SERVER_URL=https://localhost:8443
|
||||
export CERTCTL_API_KEY=your-api-key
|
||||
export CERTCTL_SERVER_CA_BUNDLE_PATH=/path/to/ca.crt # or --ca-bundle on the CLI; --insecure for dev self-signed
|
||||
|
||||
# Usage
|
||||
certctl-cli certs list # List all certificates
|
||||
@@ -347,11 +350,14 @@ certctl ships a standalone MCP (Model Context Protocol) server that exposes all
|
||||
```bash
|
||||
# Install and run
|
||||
go install github.com/shankar0123/certctl/cmd/mcp-server@latest
|
||||
export CERTCTL_SERVER_URL=http://localhost:8443
|
||||
export CERTCTL_SERVER_URL=https://localhost:8443
|
||||
export CERTCTL_API_KEY=your-api-key
|
||||
export CERTCTL_SERVER_CA_BUNDLE_PATH=/path/to/ca.crt # required for self-signed bootstrap
|
||||
mcp-server
|
||||
```
|
||||
|
||||
The MCP server is env-vars-only — there are no CLI flags for TLS. If you must bypass verification for local development against a self-signed cert, set `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true`. Never set that in production.
|
||||
|
||||
**Claude Desktop** (`claude_desktop_config.json`):
|
||||
```json
|
||||
{
|
||||
@@ -359,8 +365,9 @@ mcp-server
|
||||
"certctl": {
|
||||
"command": "mcp-server",
|
||||
"env": {
|
||||
"CERTCTL_SERVER_URL": "http://localhost:8443",
|
||||
"CERTCTL_API_KEY": "your-api-key"
|
||||
"CERTCTL_SERVER_URL": "https://localhost:8443",
|
||||
"CERTCTL_API_KEY": "your-api-key",
|
||||
"CERTCTL_SERVER_CA_BUNDLE_PATH": "/path/to/ca.crt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+292
-5
@@ -17,10 +17,8 @@ info:
|
||||
url: https://github.com/shankar0123/certctl/blob/master/LICENSE
|
||||
|
||||
servers:
|
||||
- url: http://localhost:8080
|
||||
description: Local development
|
||||
- url: http://localhost:8443
|
||||
description: Docker Compose demo
|
||||
- url: https://localhost:8443
|
||||
description: Docker Compose demo (self-signed cert; pin with ./deploy/test/certs/ca.crt)
|
||||
|
||||
security:
|
||||
- bearerAuth: []
|
||||
@@ -880,6 +878,40 @@ paths:
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
|
||||
/api/v1/agents/retired:
|
||||
get:
|
||||
tags: [Agents]
|
||||
summary: List retired agents
|
||||
description: |
|
||||
I-004: opt-in listing of soft-retired agents. The default
|
||||
`GET /api/v1/agents` endpoint filters retired rows out; this is the
|
||||
dedicated surface for reading them back (e.g., the operator UI's
|
||||
"Retired" tab, audit and forensics workflows). Pagination defaults
|
||||
match the default agent listing (page=1, per_page=50, max 500). Go
|
||||
1.22's enhanced ServeMux routes `/agents/retired` to this handler
|
||||
via the literal-beats-pattern-var precedence rule, so the sibling
|
||||
`/agents/{id}` route does not shadow it.
|
||||
operationId: listRetiredAgents
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/page"
|
||||
- $ref: "#/components/parameters/per_page"
|
||||
responses:
|
||||
"200":
|
||||
description: Paginated list of retired agents
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/PaginationEnvelope"
|
||||
- type: object
|
||||
properties:
|
||||
data:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/Agent"
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
|
||||
/api/v1/agents/{id}:
|
||||
get:
|
||||
tags: [Agents]
|
||||
@@ -900,12 +932,116 @@ paths:
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
delete:
|
||||
tags: [Agents]
|
||||
summary: Soft-retire agent
|
||||
description: |
|
||||
I-004: soft-retirement. The agent row is preserved (so its audit
|
||||
trail and historical job links remain intact) and `retired_at` is
|
||||
stamped. A retired agent receives `410 Gone` on subsequent
|
||||
heartbeats so it can shut down cleanly.
|
||||
|
||||
Behavior matrix:
|
||||
|
||||
| Scenario | Query | Status | Body |
|
||||
| --- | --- | --- | --- |
|
||||
| Clean retire (no active dependencies) | none | `200` | `RetireAgentResponse` with `cascade=false`, zero counts |
|
||||
| Blocked by active targets/certs/jobs | none | `409` | `BlockedByDependenciesResponse` with per-bucket counts |
|
||||
| Force-cascade retire | `force=true&reason=...` | `200` | `RetireAgentResponse` with `cascade=true`, pre-cascade counts |
|
||||
| Idempotent re-retire | either | `204` | (empty — downstream consumers break on stray bodies) |
|
||||
| `force=true` without reason | `force=true` | `400` | ErrorResponse (ErrForceReasonRequired) |
|
||||
| Reserved sentinel agent | any | `403` | ErrorResponse (ErrAgentIsSentinel) |
|
||||
| Unknown agent id | any | `404` | ErrorResponse |
|
||||
|
||||
Sentinel agents are the four reserved identities backing non-agent
|
||||
discovery subsystems (`server-scanner`, `cloud-aws-sm`,
|
||||
`cloud-azure-kv`, `cloud-gcp-sm`). Retiring them would orphan the
|
||||
scanner or a cloud secret-manager source, so the handler refuses
|
||||
unconditionally — even with `force=true`.
|
||||
operationId: retireAgent
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/resourceId"
|
||||
- name: force
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: boolean
|
||||
default: false
|
||||
description: |
|
||||
Cascade-retire active downstream targets, certificates, and
|
||||
jobs. When `true`, a non-empty `reason` is required. A
|
||||
malformed value (anything strconv.ParseBool rejects) is
|
||||
silently treated as `false` so a typoed query can never
|
||||
accidentally enable the cascade.
|
||||
- name: reason
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
description: |
|
||||
Human-readable reason recorded on the retired row and in the
|
||||
immutable audit trail. Required (non-empty after trimming)
|
||||
when `force=true`.
|
||||
responses:
|
||||
"200":
|
||||
description: |
|
||||
Agent retired (clean retire or successful force-cascade). Body
|
||||
is `RetireAgentResponse`.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/RetireAgentResponse"
|
||||
"204":
|
||||
description: |
|
||||
Idempotent retire — the agent was already retired. Response
|
||||
body is empty (the 200-path shape does not apply, and
|
||||
downstream clients that tee responses into dashboards would
|
||||
break on spurious bodies).
|
||||
"400":
|
||||
description: |
|
||||
`force=true` was sent without a non-empty `reason`
|
||||
(ErrForceReasonRequired).
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorResponse"
|
||||
"403":
|
||||
description: |
|
||||
Agent is a reserved sentinel and cannot be retired even with
|
||||
`?force=true` (ErrAgentIsSentinel).
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorResponse"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"409":
|
||||
description: |
|
||||
Blocked by active downstream dependencies. Body carries
|
||||
per-bucket counts so the operator UI can show the user which
|
||||
dependency is holding up the retire. Re-run with
|
||||
`?force=true&reason=...` to cascade.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlockedByDependenciesResponse"
|
||||
"405":
|
||||
description: Method not allowed (only DELETE, GET are routed to this path)
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
|
||||
/api/v1/agents/{id}/heartbeat:
|
||||
post:
|
||||
tags: [Agents]
|
||||
summary: Agent heartbeat
|
||||
description: Reports agent liveness and metadata (OS, architecture, IP, version).
|
||||
description: |
|
||||
Reports agent liveness and metadata (OS, architecture, IP, version).
|
||||
|
||||
I-004: a retired agent still polling the heartbeat endpoint receives
|
||||
`410 Gone` so `cmd/agent` detects the terminal signal and shuts down
|
||||
cleanly instead of looping forever against a decommissioned identity.
|
||||
The retired-agent check runs before any "not found" string match so
|
||||
it can never be masked by a sibling error branch.
|
||||
operationId: agentHeartbeat
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/resourceId"
|
||||
@@ -936,6 +1072,14 @@ paths:
|
||||
$ref: "#/components/responses/BadRequest"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"410":
|
||||
description: |
|
||||
I-004: the agent has been soft-retired. The agent process should
|
||||
treat this as a terminal signal and shut down cleanly.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorResponse"
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
|
||||
@@ -1891,6 +2035,16 @@ paths:
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/page"
|
||||
- $ref: "#/components/parameters/per_page"
|
||||
- name: status
|
||||
in: query
|
||||
required: false
|
||||
description: |
|
||||
Filter by lifecycle status. I-005: `dead` powers the Dead letter
|
||||
tab on the GUI; empty/omitted returns the default all-statuses
|
||||
listing to preserve pre-I-005 behavior.
|
||||
schema:
|
||||
type: string
|
||||
enum: [pending, sent, failed, dead, read]
|
||||
responses:
|
||||
"200":
|
||||
description: Paginated list of notifications
|
||||
@@ -1948,6 +2102,36 @@ paths:
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
|
||||
/api/v1/notifications/{id}/requeue:
|
||||
post:
|
||||
tags: [Notifications]
|
||||
summary: Requeue a dead notification
|
||||
description: |
|
||||
I-005: flip a notification from the `dead` dead-letter queue back to
|
||||
`pending` so the retry sweep (default 2 minutes) picks it up on its
|
||||
next tick. Used by operators after fixing the underlying delivery
|
||||
failure (SMTP config, webhook endpoint, etc.). Clears `next_retry_at`
|
||||
and resets the `retry_count` budget; `last_error` is preserved for
|
||||
audit continuity.
|
||||
operationId: requeueNotification
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/resourceId"
|
||||
responses:
|
||||
"200":
|
||||
description: Requeued
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/StatusResponse"
|
||||
"400":
|
||||
$ref: "#/components/responses/BadRequest"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"405":
|
||||
description: Method not allowed (POST only)
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalError"
|
||||
|
||||
# ─── Stats ───────────────────────────────────────────────────────────
|
||||
/api/v1/stats/summary:
|
||||
get:
|
||||
@@ -3373,6 +3557,85 @@ components:
|
||||
type: string
|
||||
version:
|
||||
type: string
|
||||
retired_at:
|
||||
type: string
|
||||
format: date-time
|
||||
nullable: true
|
||||
description: |
|
||||
I-004: soft-retirement timestamp. `null` (or field absent) means the
|
||||
agent is active. A non-null value is the canonical "retired" state —
|
||||
the operational `status` column is preserved at retirement time as
|
||||
the last-seen value, but `retired_at` is the source of truth for
|
||||
filtering agents out of active listings.
|
||||
retired_reason:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
I-004: human-readable reason captured at retirement time. Only set
|
||||
when the agent was retired via `?force=true&reason=...` cascade; a
|
||||
default soft-retire leaves this field null.
|
||||
|
||||
AgentDependencyCounts:
|
||||
type: object
|
||||
description: |
|
||||
I-004: preflight counts of active downstream rows that would be
|
||||
orphaned by retiring an agent. Returned in the 409
|
||||
`blocked_by_dependencies` body so the operator UI can tell the user
|
||||
which bucket is blocking the retire, and also in the 200 response
|
||||
body on a successful `?force=true` cascade as a snapshot of what
|
||||
was cascaded.
|
||||
properties:
|
||||
active_targets:
|
||||
type: integer
|
||||
description: Deployment targets with this agent assigned and retired_at IS NULL
|
||||
active_certificates:
|
||||
type: integer
|
||||
description: Certificates currently deployed via one of this agent's active targets
|
||||
pending_jobs:
|
||||
type: integer
|
||||
description: Jobs with agent_id=this in status Pending, AwaitingCSR, AwaitingApproval, or Running
|
||||
|
||||
RetireAgentResponse:
|
||||
type: object
|
||||
description: |
|
||||
I-004: response body for a successful retire on DELETE /api/v1/agents/{id}.
|
||||
Returned on both clean retires (cascade=false, zero counts) and
|
||||
force-cascade retires (cascade=true, counts snapshot of the
|
||||
pre-cascade dependency state). The 204 idempotent-retire path does
|
||||
NOT emit this body — re-retiring an already-retired agent returns
|
||||
an empty response.
|
||||
properties:
|
||||
retired_at:
|
||||
type: string
|
||||
format: date-time
|
||||
already_retired:
|
||||
type: boolean
|
||||
description: |
|
||||
Always false on the 200 response — the already-retired path
|
||||
returns 204 No Content with no body. Surfaced in the schema
|
||||
only so downstream consumers have a complete field map.
|
||||
cascade:
|
||||
type: boolean
|
||||
description: True when the retire was invoked with ?force=true
|
||||
counts:
|
||||
$ref: "#/components/schemas/AgentDependencyCounts"
|
||||
|
||||
BlockedByDependenciesResponse:
|
||||
type: object
|
||||
description: |
|
||||
I-004: 409 response body for a retire request blocked by active
|
||||
downstream dependencies. Returned when `force=true` is not set and
|
||||
any of the three counts is non-zero. The operator UI renders these
|
||||
counts so the human can retire or reassign the blocking rows
|
||||
before re-running the retire, or tick the force checkbox to cascade.
|
||||
properties:
|
||||
error:
|
||||
type: string
|
||||
example: blocked_by_dependencies
|
||||
message:
|
||||
type: string
|
||||
counts:
|
||||
$ref: "#/components/schemas/AgentDependencyCounts"
|
||||
|
||||
WorkItem:
|
||||
type: object
|
||||
@@ -3680,8 +3943,32 @@ components:
|
||||
format: date-time
|
||||
status:
|
||||
type: string
|
||||
enum: [pending, sent, failed, dead, read]
|
||||
description: |
|
||||
Notification lifecycle status. I-005 adds `dead` for notifications
|
||||
that exhausted their 5-attempt retry budget and were moved to the
|
||||
dead-letter queue; operators triage these in the GUI's Dead letter
|
||||
tab and use POST /notifications/{id}/requeue to resurrect them.
|
||||
error:
|
||||
type: string
|
||||
retry_count:
|
||||
type: integer
|
||||
description: |
|
||||
Number of delivery attempts made. I-005 retry-sweep field; caps
|
||||
at max_attempts=5 before the notification transitions to `dead`.
|
||||
next_retry_at:
|
||||
type: string
|
||||
format: date-time
|
||||
description: |
|
||||
When the next retry attempt is scheduled. I-005 retry-sweep field;
|
||||
null for `sent`, `dead`, and `read` statuses. Backoff follows
|
||||
`min(2^retry_count * 1m, 1h)`.
|
||||
last_error:
|
||||
type: string
|
||||
description: |
|
||||
Most recent transient delivery error (SMTP failure, webhook 5xx,
|
||||
etc.). I-005 retry-sweep field; surfaced on the Dead letter tab
|
||||
so operators can triage without chasing server logs.
|
||||
created_at:
|
||||
type: string
|
||||
format: date-time
|
||||
|
||||
+273
-31
@@ -7,6 +7,7 @@ import (
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
@@ -72,7 +73,7 @@ func TestAgent_Heartbeat_Success(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Should not panic
|
||||
agent.sendHeartbeat(context.Background())
|
||||
@@ -93,7 +94,7 @@ func TestAgent_Heartbeat_ServerError(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Should increment consecutive failures
|
||||
failureBefore := agent.consecutiveFailures
|
||||
@@ -115,7 +116,7 @@ func TestAgent_Heartbeat_ConnectionError(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Should fail due to connection error
|
||||
agent.sendHeartbeat(context.Background())
|
||||
@@ -150,7 +151,7 @@ func TestAgent_PollWork_NoWork(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Should not panic
|
||||
agent.pollForWork(context.Background())
|
||||
@@ -195,7 +196,7 @@ func TestAgent_PollWork_Success(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Should not panic; work items are processed in separate gorines in real usage
|
||||
agent.pollForWork(context.Background())
|
||||
@@ -285,7 +286,7 @@ func TestParsePEMFile(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Parse the file
|
||||
entries := agent.parsePEMFile(certPath)
|
||||
@@ -336,7 +337,7 @@ func TestParsePEMFile_MultipleCerts(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
entries := agent.parsePEMFile(certPath)
|
||||
|
||||
@@ -362,7 +363,7 @@ func TestParseDERFile(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
entry, err := agent.parseDERFile(derPath)
|
||||
if err != nil {
|
||||
@@ -397,7 +398,7 @@ func TestParseDERFile_Invalid(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
_, err := agent.parseDERFile(derPath)
|
||||
if err == nil {
|
||||
@@ -439,7 +440,7 @@ func TestScanDirectory(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpdir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Simulate directory walk manually (as runDiscoveryScan does)
|
||||
var certs []discoveredCertEntry
|
||||
@@ -474,7 +475,7 @@ func TestCreateTargetConnector_NGINX(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
configJSON := json.RawMessage(`{"cert_path":"/etc/nginx/cert.pem"}`)
|
||||
connector, err := agent.createTargetConnector("NGINX", configJSON)
|
||||
@@ -496,7 +497,7 @@ func TestCreateTargetConnector_Unsupported(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
_, err := agent.createTargetConnector("UnsupportedType", nil)
|
||||
|
||||
@@ -530,7 +531,7 @@ func TestFetchCertificate_Success(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
certPEM, err := agent.fetchCertificate(context.Background(), "mc-001")
|
||||
if err != nil {
|
||||
@@ -556,7 +557,7 @@ func TestFetchCertificate_NotFound(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
_, err := agent.fetchCertificate(context.Background(), "mc-nonexistent")
|
||||
if err == nil {
|
||||
@@ -592,7 +593,7 @@ func TestReportJobStatus_Success(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
err := agent.reportJobStatus(context.Background(), "j-001", "Completed", "")
|
||||
if err != nil {
|
||||
@@ -624,7 +625,7 @@ func TestReportJobStatus_WithError(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
err := agent.reportJobStatus(context.Background(), "j-001", "Failed", "deployment failed")
|
||||
if err != nil {
|
||||
@@ -658,7 +659,7 @@ func TestMakeRequest_Success(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
resp, err := agent.makeRequest(context.Background(), http.MethodPost, "/test", map[string]string{"key": "value"})
|
||||
if err != nil {
|
||||
@@ -680,7 +681,7 @@ func TestMakeRequest_InvalidURL(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
_, err := agent.makeRequest(context.Background(), http.MethodGet, "/test", nil)
|
||||
if err == nil {
|
||||
@@ -765,7 +766,7 @@ func TestNewAgent(t *testing.T) {
|
||||
}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
if agent.config != cfg {
|
||||
t.Error("config not set correctly")
|
||||
@@ -791,7 +792,7 @@ func TestNewAgent_WithLogger(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
if agent.logger != logger {
|
||||
t.Error("logger not set correctly")
|
||||
@@ -954,7 +955,7 @@ func TestCreateTargetConnector_AllSupportedTypes(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -1007,7 +1008,7 @@ func TestCreateTargetConnector_InvalidJSON(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
invalidJSON := json.RawMessage("{invalid json}")
|
||||
|
||||
@@ -1031,7 +1032,7 @@ func TestCreateTargetConnector_UnknownType(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
_, err := agent.createTargetConnector("MagicBox", nil)
|
||||
|
||||
@@ -1061,7 +1062,7 @@ func TestCreateTargetConnector_EmptyConfig(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
for _, typeName := range tests {
|
||||
t.Run(typeName, func(t *testing.T) {
|
||||
@@ -1137,7 +1138,7 @@ func TestRunDiscoveryScan_ValidCerts(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpDir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Run discovery scan
|
||||
agent.runDiscoveryScan(context.Background())
|
||||
@@ -1165,7 +1166,7 @@ func TestRunDiscoveryScan_NoCertificates(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpDir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Run discovery scan - should complete without error even with empty directory
|
||||
agent.runDiscoveryScan(context.Background())
|
||||
@@ -1222,7 +1223,7 @@ func TestRunDiscoveryScan_MultipleCerts(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpDir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Run discovery scan
|
||||
agent.runDiscoveryScan(context.Background())
|
||||
@@ -1273,7 +1274,7 @@ func TestRunDiscoveryScan_DERCertificate(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpDir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Run discovery scan
|
||||
agent.runDiscoveryScan(context.Background())
|
||||
@@ -1331,7 +1332,7 @@ func TestRunDiscoveryScan_Subdirectories(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpDir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Run discovery scan - should recursively find certs in subdirs
|
||||
agent.runDiscoveryScan(context.Background())
|
||||
@@ -1369,7 +1370,7 @@ func TestRunDiscoveryScan_ServerError(t *testing.T) {
|
||||
DiscoveryDirs: []string{tmpDir},
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
// Should handle server error gracefully without panicking
|
||||
agent.runDiscoveryScan(context.Background())
|
||||
@@ -1396,7 +1397,7 @@ func TestDiscoveredCertEntry_ValidFields(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent := NewAgent(cfg, logger)
|
||||
agent, _ := NewAgent(cfg, logger)
|
||||
|
||||
entries := agent.parsePEMFile(certPath)
|
||||
|
||||
@@ -1447,3 +1448,244 @@ func TestDiscoveredCertEntry_ValidFields(t *testing.T) {
|
||||
t.Error("PEMData should not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTPS-Everywhere milestone (v2.2, §3.2 / §7) — Phase 5 client-side tests.
|
||||
//
|
||||
// These tests pin the agent's pre-flight HTTPS-scheme guard and the TLS
|
||||
// configuration surface (CA bundle loading + TLS 1.3 round-trip) so that
|
||||
// regressions surface at unit-test time, not at the first heartbeat of a
|
||||
// production rollout. Matches the same contract asserted by the sibling
|
||||
// binaries cmd/cli/main_test.go and cmd/mcp-server/main_test.go — the three
|
||||
// must stay in lock-step because all three are HTTPS-only clients of the
|
||||
// same control plane.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestValidateHTTPSScheme pins the pre-flight URL-scheme guard that the
|
||||
// HTTPS-Everywhere milestone requires on the agent binary startup path. The
|
||||
// agent's diagnostic is distinct from the CLI/MCP variants because it names
|
||||
// CERTCTL_SERVER_URL (the only input channel — no --server flag on the
|
||||
// agent). Every case here mirrors the dispatch arms in cmd/agent/main.go:
|
||||
// validateHTTPSScheme; drifting the error-message substrings is what this
|
||||
// test is here to catch.
|
||||
func TestValidateHTTPSScheme(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
serverURL string
|
||||
wantErr bool
|
||||
wantErrSub string
|
||||
}{
|
||||
{
|
||||
name: "https URL passes",
|
||||
serverURL: "https://certctl-server:8443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "https URL with path passes",
|
||||
serverURL: "https://certctl.example.com/api/v1",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "uppercase HTTPS scheme passes (url.Parse lowercases)",
|
||||
serverURL: "HTTPS://certctl-server:8443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty URL rejected names CERTCTL_SERVER_URL",
|
||||
serverURL: "",
|
||||
wantErr: true,
|
||||
wantErrSub: "CERTCTL_SERVER_URL is empty",
|
||||
},
|
||||
{
|
||||
name: "plaintext http rejected",
|
||||
serverURL: "http://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "plaintext http://",
|
||||
},
|
||||
{
|
||||
name: "bare host missing scheme falls through to unsupported",
|
||||
serverURL: "localhost:8443",
|
||||
wantErr: true,
|
||||
// url.Parse treats "localhost:8443" as scheme=localhost,
|
||||
// opaque=8443 — exercises the default arm (unsupported scheme)
|
||||
// rather than the empty-scheme arm. Both are fail-closed, which
|
||||
// is what we care about.
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
{
|
||||
name: "path-only URL rejected",
|
||||
serverURL: "//certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "missing a scheme",
|
||||
},
|
||||
{
|
||||
name: "unsupported scheme rejected",
|
||||
serverURL: "ftp://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
{
|
||||
name: "ws scheme rejected",
|
||||
serverURL: "ws://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateHTTPSScheme(tt.serverURL)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("validateHTTPSScheme(%q) err=%v wantErr=%v", tt.serverURL, err, tt.wantErr)
|
||||
}
|
||||
if tt.wantErr && tt.wantErrSub != "" && !strings.Contains(err.Error(), tt.wantErrSub) {
|
||||
t.Errorf("validateHTTPSScheme(%q) err=%q must contain %q so operators see the right diagnostic",
|
||||
tt.serverURL, err.Error(), tt.wantErrSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeTestCABundle PEM-encodes a cert's DER bytes and writes the result to a
|
||||
// tmp file inside dir. Used by CA-bundle tests so each case owns a distinct
|
||||
// file path (matters for the "missing file" case which must point at a path
|
||||
// that provably does not exist). Returns the path.
|
||||
func writeTestCABundle(t *testing.T, dir string, certDER []byte, filename string) string {
|
||||
t.Helper()
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
||||
path := filepath.Join(dir, filename)
|
||||
if err := os.WriteFile(path, pemBytes, 0644); err != nil {
|
||||
t.Fatalf("writing CA bundle %q: %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// TestNewAgent_CABundle_Success confirms that a well-formed PEM bundle gets
|
||||
// parsed into an x509.CertPool and wired onto the agent's HTTP client
|
||||
// transport. This is the happy path the docs/tls.md "Private CA signed
|
||||
// server cert" section depends on.
|
||||
func TestNewAgent_CABundle_Success(t *testing.T) {
|
||||
cert, err := generateTestCertWithCN("test.certctl.local")
|
||||
if err != nil {
|
||||
t.Fatalf("generateTestCertWithCN: %v", err)
|
||||
}
|
||||
bundlePath := writeTestCABundle(t, t.TempDir(), cert.Raw, "ca-bundle.pem")
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent, err := NewAgent(&AgentConfig{
|
||||
ServerURL: "https://certctl-server:8443",
|
||||
APIKey: "test-key",
|
||||
AgentID: "a-test",
|
||||
Hostname: "test-host",
|
||||
CABundlePath: bundlePath,
|
||||
}, logger)
|
||||
if err != nil {
|
||||
t.Fatalf("NewAgent with valid CA bundle err=%v want nil", err)
|
||||
}
|
||||
|
||||
transport, ok := agent.client.Transport.(*http.Transport)
|
||||
if !ok {
|
||||
t.Fatalf("agent.client.Transport is %T; want *http.Transport", agent.client.Transport)
|
||||
}
|
||||
if transport.TLSClientConfig == nil {
|
||||
t.Fatal("TLSClientConfig is nil; HTTPS-everywhere milestone requires a non-nil TLS config")
|
||||
}
|
||||
if transport.TLSClientConfig.MinVersion != tls.VersionTLS13 {
|
||||
t.Errorf("MinVersion=%x want TLS 1.3 (%x) per §2.3 of the milestone spec",
|
||||
transport.TLSClientConfig.MinVersion, tls.VersionTLS13)
|
||||
}
|
||||
if transport.TLSClientConfig.RootCAs == nil {
|
||||
t.Error("RootCAs is nil; the configured CA bundle was silently dropped")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAgent_CABundle_MissingFile pins the fail-loud behavior when the
|
||||
// operator points CERTCTL_SERVER_CA_BUNDLE_PATH at a path that does not
|
||||
// exist. Falling back to system roots here would mask a misconfiguration as
|
||||
// a much harder-to-debug TLS handshake failure downstream.
|
||||
func TestNewAgent_CABundle_MissingFile(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
missingPath := filepath.Join(t.TempDir(), "does-not-exist.pem")
|
||||
_, err := NewAgent(&AgentConfig{
|
||||
ServerURL: "https://certctl-server:8443",
|
||||
APIKey: "test-key",
|
||||
AgentID: "a-test",
|
||||
Hostname: "test-host",
|
||||
CABundlePath: missingPath,
|
||||
}, logger)
|
||||
if err == nil {
|
||||
t.Fatal("NewAgent err=nil for missing CA bundle path; must fail loud at startup")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "reading CA bundle") {
|
||||
t.Errorf("err=%q must contain \"reading CA bundle\" so operators can trace the cause", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAgent_CABundle_EmptyPEM covers the "file exists but contains no
|
||||
// valid certs" case (garbage, wrong-format, stripped PEM). AppendCertsFromPEM
|
||||
// returns false in this case; NewAgent must translate that into a fail-loud
|
||||
// startup error rather than quietly carry on with an empty pool.
|
||||
func TestNewAgent_CABundle_EmptyPEM(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
bundlePath := filepath.Join(t.TempDir(), "empty.pem")
|
||||
if err := os.WriteFile(bundlePath, []byte("not a pem-encoded certificate, just garbage\n"), 0644); err != nil {
|
||||
t.Fatalf("writing garbage bundle: %v", err)
|
||||
}
|
||||
_, err := NewAgent(&AgentConfig{
|
||||
ServerURL: "https://certctl-server:8443",
|
||||
APIKey: "test-key",
|
||||
AgentID: "a-test",
|
||||
Hostname: "test-host",
|
||||
CABundlePath: bundlePath,
|
||||
}, logger)
|
||||
if err == nil {
|
||||
t.Fatal("NewAgent err=nil for empty-PEM CA bundle; must fail loud at startup")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no valid PEM-encoded certificates") {
|
||||
t.Errorf("err=%q must contain \"no valid PEM-encoded certificates\" so operators see why the bundle was rejected", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAgent_TLSRoundTrip is the end-to-end integration-style check: spin
|
||||
// up an httptest.NewTLSServer (which presents a self-signed cert over TLS
|
||||
// 1.3), feed that cert into the agent as a CA bundle, and confirm the agent
|
||||
// successfully completes a heartbeat round-trip over HTTPS. This proves that
|
||||
// (a) the CA pool is actually being consulted during verification and (b)
|
||||
// the TLS 1.3 MinVersion doesn't break against httptest's default
|
||||
// negotiation. Equivalent to the "TLS handshake succeeds against a
|
||||
// self-signed control plane" integration gate, but runs in-process with no
|
||||
// Docker dependency.
|
||||
func TestNewAgent_TLSRoundTrip(t *testing.T) {
|
||||
var heartbeatHit int
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v1/agents/a-tls-test/heartbeat" && r.Method == http.MethodPost {
|
||||
heartbeatHit++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// server.Certificate() returns the *x509.Certificate httptest presents;
|
||||
// PEM-encode its DER bytes so NewAgent's AppendCertsFromPEM can ingest it.
|
||||
bundlePath := writeTestCABundle(t, t.TempDir(), server.Certificate().Raw, "httptest-ca.pem")
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
agent, err := NewAgent(&AgentConfig{
|
||||
ServerURL: server.URL,
|
||||
APIKey: "test-key",
|
||||
AgentID: "a-tls-test",
|
||||
Hostname: "tls-test-host",
|
||||
CABundlePath: bundlePath,
|
||||
}, logger)
|
||||
if err != nil {
|
||||
t.Fatalf("NewAgent with httptest CA bundle err=%v want nil", err)
|
||||
}
|
||||
|
||||
agent.sendHeartbeat(context.Background())
|
||||
|
||||
if heartbeatHit != 1 {
|
||||
t.Fatalf("heartbeat handler hit %d times; want 1 — the TLS round-trip must actually complete", heartbeatHit)
|
||||
}
|
||||
}
|
||||
|
||||
+234
-19
@@ -8,21 +8,25 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -44,15 +48,27 @@ import (
|
||||
|
||||
// AgentConfig represents the agent-side configuration.
|
||||
type AgentConfig struct {
|
||||
ServerURL string // Control plane server URL (e.g., http://localhost:8443)
|
||||
APIKey string // Agent API key for authentication
|
||||
AgentName string // Agent name for identification
|
||||
AgentID string // Agent ID for API calls (set after registration or from env)
|
||||
Hostname string // Server hostname
|
||||
KeyDir string // Directory for storing private keys (default: /var/lib/certctl/keys)
|
||||
DiscoveryDirs []string // Directories to scan for certificates (comma-separated via env)
|
||||
ServerURL string // Control plane server URL (e.g., https://localhost:8443) — must be https:// scheme
|
||||
APIKey string // Agent API key for authentication
|
||||
AgentName string // Agent name for identification
|
||||
AgentID string // Agent ID for API calls (set after registration or from env)
|
||||
Hostname string // Server hostname
|
||||
KeyDir string // Directory for storing private keys (default: /var/lib/certctl/keys)
|
||||
DiscoveryDirs []string // Directories to scan for certificates (comma-separated via env)
|
||||
CABundlePath string // Optional path to a PEM-encoded CA bundle that signed the server's cert (empty = system roots)
|
||||
InsecureSkipVerify bool // Dev-only: skip TLS certificate verification. Never enable in production. See docs/tls.md.
|
||||
}
|
||||
|
||||
// ErrAgentRetired is the sentinel returned by [Agent.Run] when the control
|
||||
// plane responds with HTTP 410 Gone to a heartbeat or work-poll request — the
|
||||
// canonical signal that this agent's row has been soft-retired server-side
|
||||
// (see I-004 in cowork/certctl-coverage-gap-audit.md). The binary must
|
||||
// terminate cleanly: an init-system restart would only produce another 410
|
||||
// and wedge the host in a restart loop. main() translates this sentinel into
|
||||
// a zero exit code so systemd (Restart=on-failure) and launchd do not respawn
|
||||
// the process. Do not wrap this error — main() matches it with errors.Is.
|
||||
var ErrAgentRetired = fmt.Errorf("agent retired by control plane")
|
||||
|
||||
// Agent represents the local agent that runs on target servers.
|
||||
// It periodically sends heartbeats, polls for work, executes deployment and CSR jobs,
|
||||
// and scans configured directories for existing certificates.
|
||||
@@ -68,6 +84,17 @@ type Agent struct {
|
||||
pollInterval time.Duration
|
||||
discoveryInterval time.Duration
|
||||
consecutiveFailures int
|
||||
|
||||
// I-004: terminal retirement signal. retiredSignal is closed exactly once
|
||||
// (guarded by retiredOnce) when either sendHeartbeat or pollForWork
|
||||
// observes HTTP 410 Gone. The Run() select loop picks up the close and
|
||||
// returns ErrAgentRetired, unwinding the goroutine cleanly so main() can
|
||||
// log + exit(0). Using a channel + sync.Once (rather than an atomic bool
|
||||
// + polling) lets us fall through the select statement immediately instead
|
||||
// of waiting for the next ticker; the zero-allocation close is safe to
|
||||
// race with ctx.Done() and other cases.
|
||||
retiredOnce sync.Once
|
||||
retiredSignal chan struct{}
|
||||
}
|
||||
|
||||
// WorkResponse represents the response from the work polling endpoint.
|
||||
@@ -90,15 +117,78 @@ type JobItem struct {
|
||||
}
|
||||
|
||||
// NewAgent creates a new agent instance.
|
||||
func NewAgent(cfg *AgentConfig, logger *slog.Logger) *Agent {
|
||||
//
|
||||
// The returned HTTP client enforces HTTPS-only control-plane access per the
|
||||
// HTTPS-Everywhere milestone (see docs/tls.md). TLS 1.3 is required; the
|
||||
// optional CABundlePath loads a PEM bundle into RootCAs so the agent can
|
||||
// trust internal / self-signed server certs without touching system trust
|
||||
// stores. InsecureSkipVerify is a dev-only escape hatch — callers must log a
|
||||
// loud warning when it's set; never enable in production (see §2.4 of the
|
||||
// milestone spec and docs/upgrade-to-tls.md).
|
||||
//
|
||||
// Returns an error if CABundlePath is set but unreadable or malformed — fail
|
||||
// loud at startup rather than silently fall back to system roots, which would
|
||||
// turn a misconfigured bundle path into a cryptic "x509: certificate signed
|
||||
// by unknown authority" on the first heartbeat.
|
||||
func NewAgent(cfg *AgentConfig, logger *slog.Logger) (*Agent, error) {
|
||||
tlsConfig := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: cfg.InsecureSkipVerify, //nolint:gosec // opt-in dev escape hatch, documented in docs/tls.md
|
||||
}
|
||||
if cfg.CABundlePath != "" {
|
||||
pemBytes, err := os.ReadFile(cfg.CABundlePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading CA bundle at %q: %w", cfg.CABundlePath, err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pemBytes) {
|
||||
return nil, fmt.Errorf("CA bundle at %q contains no valid PEM-encoded certificates", cfg.CABundlePath)
|
||||
}
|
||||
tlsConfig.RootCAs = pool
|
||||
}
|
||||
|
||||
httpClient := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
return &Agent{
|
||||
config: cfg,
|
||||
logger: logger,
|
||||
client: &http.Client{Timeout: 30 * time.Second},
|
||||
client: httpClient,
|
||||
heartbeatInterval: 60 * time.Second,
|
||||
pollInterval: 30 * time.Second,
|
||||
discoveryInterval: 6 * time.Hour, // scan for certs every 6 hours
|
||||
}
|
||||
retiredSignal: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// markRetired records that the control plane has declared this agent retired
|
||||
// (HTTP 410 Gone on heartbeat or work poll). Idempotent via sync.Once — if
|
||||
// both the heartbeat and work-poll paths observe 410 in the same tick, only
|
||||
// the first close() runs and we avoid a runtime panic. Emits an ERROR-level
|
||||
// log line so init-system journaling captures it prominently, and includes
|
||||
// the source (heartbeat/work_poll), response body, and status code so the
|
||||
// operator can verify it's a genuine retirement signal rather than a
|
||||
// misrouted request. After this returns, the select-loop case in Run()
|
||||
// observes the closed channel on its next iteration and returns
|
||||
// ErrAgentRetired.
|
||||
func (a *Agent) markRetired(source string, statusCode int, body string) {
|
||||
a.retiredOnce.Do(func() {
|
||||
a.logger.Error("agent has been retired by control plane — shutting down",
|
||||
"source", source,
|
||||
"status", statusCode,
|
||||
"body", body,
|
||||
"agent_id", a.config.AgentID)
|
||||
close(a.retiredSignal)
|
||||
})
|
||||
}
|
||||
|
||||
// Run starts the agent's main loop.
|
||||
@@ -154,6 +244,19 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
a.logger.Info("agent shutting down", "reason", ctx.Err())
|
||||
return ctx.Err()
|
||||
|
||||
// I-004: retiredSignal is closed exactly once (via markRetired's
|
||||
// sync.Once) when either sendHeartbeat or pollForWork observes HTTP 410
|
||||
// Gone from the control plane. Falling through this case immediately
|
||||
// (rather than waiting for the next ticker) lets the agent shut down
|
||||
// quickly once retirement is confirmed — every extra heartbeat against a
|
||||
// retired row is wasted work and noise in the audit trail. Returning
|
||||
// ErrAgentRetired propagates up to main(), which matches it with
|
||||
// errors.Is and exits(0) so systemd/launchd do not respawn the process.
|
||||
case <-a.retiredSignal:
|
||||
a.logger.Info("agent retired signal received — exiting event loop",
|
||||
"agent_id", a.config.AgentID)
|
||||
return ErrAgentRetired
|
||||
|
||||
case <-heartbeatTicker.C:
|
||||
a.sendHeartbeat(ctx)
|
||||
|
||||
@@ -209,6 +312,22 @@ func (a *Agent) sendHeartbeat(ctx context.Context) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// I-004: HTTP 410 Gone is the terminal signal from the control plane that
|
||||
// this agent's row has been soft-retired (see internal/api/handler/agent.go
|
||||
// heartbeat path + AgentRetirementService). Treat it separately from the
|
||||
// generic non-200 error branch: record the event to markRetired (which closes
|
||||
// retiredSignal exactly once via sync.Once) and return without bumping
|
||||
// consecutiveFailures — this is not a transient failure, it's a clean
|
||||
// shutdown. The Run() select loop picks up the closed channel on its next
|
||||
// iteration and returns ErrAgentRetired, which main() translates into an
|
||||
// exit(0) so systemd/launchd don't respawn the process into another 410
|
||||
// loop.
|
||||
if resp.StatusCode == http.StatusGone {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
a.markRetired("heartbeat", resp.StatusCode, string(body))
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
a.logger.Error("heartbeat rejected",
|
||||
@@ -237,6 +356,19 @@ func (a *Agent) pollForWork(ctx context.Context) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// I-004: same terminal-retirement handling as sendHeartbeat. Work-poll is the
|
||||
// other hot path that can observe an agent's soft-retirement; if the
|
||||
// heartbeat tick happens to fire after a work-poll tick within the same
|
||||
// retirement window, this branch catches it first. markRetired's sync.Once
|
||||
// guards idempotency so racing both paths in the same tick only closes the
|
||||
// signal channel once. No consecutiveFailures increment — retirement is
|
||||
// not a transient failure.
|
||||
if resp.StatusCode == http.StatusGone {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
a.markRetired("work_poll", resp.StatusCode, string(body))
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
a.logger.Error("work poll rejected",
|
||||
@@ -1031,12 +1163,14 @@ func certKeyInfo(cert *x509.Certificate) (string, int) {
|
||||
|
||||
func main() {
|
||||
// Parse command-line flags (with env var fallbacks for Docker deployment)
|
||||
serverURL := flag.String("server", getEnvDefault("CERTCTL_SERVER_URL", "http://localhost:8443"), "Control plane server URL")
|
||||
serverURL := flag.String("server", getEnvDefault("CERTCTL_SERVER_URL", "https://localhost:8443"), "Control plane server URL (must be https://)")
|
||||
apiKey := flag.String("api-key", getEnvDefault("CERTCTL_API_KEY", ""), "Agent API key")
|
||||
agentName := flag.String("name", getEnvDefault("CERTCTL_AGENT_NAME", "certctl-agent"), "Agent name")
|
||||
agentID := flag.String("agent-id", getEnvDefault("CERTCTL_AGENT_ID", ""), "Agent ID (from registration)")
|
||||
keyDir := flag.String("key-dir", getEnvDefault("CERTCTL_KEY_DIR", "/var/lib/certctl/keys"), "Directory for storing private keys")
|
||||
discoveryDirsStr := flag.String("discovery-dirs", getEnvDefault("CERTCTL_DISCOVERY_DIRS", ""), "Comma-separated directories to scan for certificates")
|
||||
caBundlePath := flag.String("ca-bundle", getEnvDefault("CERTCTL_SERVER_CA_BUNDLE_PATH", ""), "Path to a PEM-encoded CA bundle that signed the server's TLS cert (optional; falls back to system roots)")
|
||||
insecureSkipVerify := flag.Bool("insecure-skip-verify", getEnvBoolDefault("CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY", false), "Dev-only: skip TLS certificate verification. Never enable in production. See docs/tls.md.")
|
||||
flag.Parse()
|
||||
|
||||
if *apiKey == "" {
|
||||
@@ -1050,6 +1184,18 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Pre-flight URL-scheme validation — reject plaintext http:// before any
|
||||
// network call. The HTTPS-Everywhere milestone (§2.4, §7) mandates that
|
||||
// mis-configured agents fail loudly at startup with a diagnostic pointing
|
||||
// at the upgrade guide, rather than producing a TCP-refused or
|
||||
// TLS-handshake-error that obscures the actual cause.
|
||||
if err := validateHTTPSScheme(*serverURL); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "\nThe certctl control plane is HTTPS-only as of v2.2.\n")
|
||||
fmt.Fprintf(os.Stderr, "See docs/upgrade-to-tls.md for the cutover walkthrough.\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Set up structured logging
|
||||
logLevel := slog.LevelInfo
|
||||
if getEnvDefault("CERTCTL_LOG_LEVEL", "info") == "debug" {
|
||||
@@ -1078,17 +1224,27 @@ func main() {
|
||||
|
||||
// Create agent configuration
|
||||
agentCfg := &AgentConfig{
|
||||
ServerURL: *serverURL,
|
||||
APIKey: *apiKey,
|
||||
AgentName: *agentName,
|
||||
AgentID: *agentID,
|
||||
Hostname: hostname,
|
||||
KeyDir: *keyDir,
|
||||
DiscoveryDirs: discoveryDirs,
|
||||
ServerURL: *serverURL,
|
||||
APIKey: *apiKey,
|
||||
AgentName: *agentName,
|
||||
AgentID: *agentID,
|
||||
Hostname: hostname,
|
||||
KeyDir: *keyDir,
|
||||
DiscoveryDirs: discoveryDirs,
|
||||
CABundlePath: *caBundlePath,
|
||||
InsecureSkipVerify: *insecureSkipVerify,
|
||||
}
|
||||
|
||||
if agentCfg.InsecureSkipVerify {
|
||||
logger.Warn("TLS certificate verification is disabled (CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true) — never enable this in production")
|
||||
}
|
||||
|
||||
// Create and start agent
|
||||
agent := NewAgent(agentCfg, logger)
|
||||
agent, err := NewAgent(agentCfg, logger)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed to initialize agent: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create context with cancellation for graceful shutdown
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -1117,6 +1273,19 @@ func main() {
|
||||
cancel()
|
||||
<-errChan
|
||||
case err := <-errChan:
|
||||
// I-004: ErrAgentRetired is a terminal, *clean* shutdown — the control
|
||||
// plane responded HTTP 410 Gone on heartbeat/work-poll, meaning this
|
||||
// agent's row has been soft-retired and will never be reachable again.
|
||||
// Exit 0 so systemd's Restart=on-failure and launchd's KeepAlive do NOT
|
||||
// respawn the process into another 410 loop (which would wedge the host
|
||||
// and spam the control plane). Operators can observe the retirement via
|
||||
// audit_events or the AgentsPage retired tab; the terminal log line on
|
||||
// the way out is enough for post-mortem forensics.
|
||||
if errors.Is(err, ErrAgentRetired) {
|
||||
logger.Info("agent retired by control plane — exiting without restart",
|
||||
"agent_id", agentCfg.AgentID)
|
||||
return
|
||||
}
|
||||
if err != context.Canceled {
|
||||
logger.Error("agent error", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -1133,3 +1302,49 @@ func getEnvDefault(key, defaultValue string) string {
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
// getEnvBoolDefault parses an environment variable as a boolean. Accepts "1",
|
||||
// "t", "true", "T", "TRUE", "True" as true; anything else (including empty)
|
||||
// returns the provided default. Kept permissive on purpose so operators can
|
||||
// flip the dev-only TLS skip-verify toggle with any common truthy spelling
|
||||
// without having to remember exactly what we parse.
|
||||
func getEnvBoolDefault(key string, defaultValue bool) bool {
|
||||
raw := os.Getenv(key)
|
||||
if raw == "" {
|
||||
return defaultValue
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "1", "t", "true", "yes", "on":
|
||||
return true
|
||||
case "0", "f", "false", "no", "off":
|
||||
return false
|
||||
default:
|
||||
return defaultValue
|
||||
}
|
||||
}
|
||||
|
||||
// validateHTTPSScheme enforces the HTTPS-Everywhere milestone's §7 acceptance
|
||||
// criterion: "Agent with CERTCTL_SERVER_URL=http://... fails at startup with
|
||||
// a fail-loud diagnostic pointing at docs/upgrade-to-tls.md. Not TCP-refused,
|
||||
// not TLS-handshake-error — a pre-flight config validation failure before any
|
||||
// network call." Returns a descriptive error; the caller prints the upgrade
|
||||
// guide pointer and exits non-zero.
|
||||
func validateHTTPSScheme(serverURL string) error {
|
||||
if serverURL == "" {
|
||||
return fmt.Errorf("CERTCTL_SERVER_URL is empty — set it to an https:// URL (e.g., https://certctl-server:8443)")
|
||||
}
|
||||
u, err := url.Parse(serverURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("CERTCTL_SERVER_URL %q is not a valid URL: %w", serverURL, err)
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "https":
|
||||
return nil
|
||||
case "http":
|
||||
return fmt.Errorf("CERTCTL_SERVER_URL %q uses plaintext http:// — the certctl control plane is HTTPS-only", serverURL)
|
||||
case "":
|
||||
return fmt.Errorf("CERTCTL_SERVER_URL %q is missing a scheme — expected https://", serverURL)
|
||||
default:
|
||||
return fmt.Errorf("CERTCTL_SERVER_URL %q uses unsupported scheme %q — expected https://", serverURL, u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,7 +228,7 @@ func TestReportVerificationResult_Success(t *testing.T) {
|
||||
ServerURL: server.URL,
|
||||
APIKey: "test-api-key",
|
||||
}
|
||||
agent := NewAgent(cfg, nil)
|
||||
agent, _ := NewAgent(cfg, nil)
|
||||
|
||||
result := &VerificationResult{
|
||||
ExpectedFingerprint: "abc123",
|
||||
@@ -244,7 +244,7 @@ func TestReportVerificationResult_Success(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReportVerificationResult_MissingFields(t *testing.T) {
|
||||
agent := NewAgent(&AgentConfig{}, nil)
|
||||
agent, _ := NewAgent(&AgentConfig{}, nil)
|
||||
|
||||
result := &VerificationResult{
|
||||
Verified: true,
|
||||
@@ -343,7 +343,7 @@ func TestReportVerificationResult_ServerError(t *testing.T) {
|
||||
ServerURL: server.URL,
|
||||
APIKey: "test-api-key",
|
||||
}
|
||||
agent := NewAgent(cfg, nil)
|
||||
agent, _ := NewAgent(cfg, nil)
|
||||
|
||||
result := &VerificationResult{
|
||||
ExpectedFingerprint: "abc123",
|
||||
|
||||
+82
-10
@@ -3,7 +3,9 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/cli"
|
||||
)
|
||||
@@ -27,8 +29,9 @@ Commands:
|
||||
certs renew ID Trigger certificate renewal
|
||||
certs revoke ID Revoke a certificate
|
||||
|
||||
agents list List agents
|
||||
agents get ID Get agent details
|
||||
agents list List agents (add --retired to list soft-retired agents)
|
||||
agents get ID Get agent details
|
||||
agents retire ID Soft-retire an agent (add --force --reason "…" to cascade)
|
||||
|
||||
jobs list List jobs
|
||||
jobs get ID Get job details
|
||||
@@ -42,22 +45,34 @@ Commands:
|
||||
version Show CLI version
|
||||
|
||||
Examples:
|
||||
certctl-cli --server http://localhost:8443 --api-key mykey certs list
|
||||
certctl-cli --server https://localhost:8443 --api-key mykey certs list
|
||||
certctl-cli certs renew mc-prod --format json
|
||||
certctl-cli import certs.pem
|
||||
`)
|
||||
}
|
||||
|
||||
serverURL := fs.String("server", os.Getenv("CERTCTL_SERVER_URL"), "certctl server URL (env: CERTCTL_SERVER_URL)")
|
||||
if *serverURL == "" {
|
||||
*serverURL = "http://localhost:8443"
|
||||
// HTTPS-Everywhere (v2.2): the server is HTTPS-only. The default URL uses
|
||||
// https://; plaintext http:// is rejected by validateHTTPSScheme below.
|
||||
defaultServer := os.Getenv("CERTCTL_SERVER_URL")
|
||||
if defaultServer == "" {
|
||||
defaultServer = "https://localhost:8443"
|
||||
}
|
||||
serverURL := fs.String("server", defaultServer, "certctl server URL — must be https:// (env: CERTCTL_SERVER_URL)")
|
||||
|
||||
apiKey := fs.String("api-key", os.Getenv("CERTCTL_API_KEY"), "API key for authentication (env: CERTCTL_API_KEY)")
|
||||
format := fs.String("format", "table", "Output format: table, json")
|
||||
caBundlePath := fs.String("ca-bundle", os.Getenv("CERTCTL_SERVER_CA_BUNDLE_PATH"), "Path to a PEM-encoded CA bundle that signed the server cert (env: CERTCTL_SERVER_CA_BUNDLE_PATH)")
|
||||
insecure := fs.Bool("insecure", strings.EqualFold(os.Getenv("CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY"), "true"), "Skip TLS certificate verification — dev only, never set in production (env: CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY)")
|
||||
|
||||
fs.Parse(os.Args[1:])
|
||||
|
||||
if err := validateHTTPSScheme(*serverURL); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "\nThe certctl control plane is HTTPS-only as of v2.2.\n")
|
||||
fmt.Fprintf(os.Stderr, "See docs/upgrade-to-tls.md for the cutover walkthrough.\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
args := fs.Args()
|
||||
if len(args) == 0 {
|
||||
fs.Usage()
|
||||
@@ -65,13 +80,16 @@ Examples:
|
||||
}
|
||||
|
||||
// Create client
|
||||
client := cli.NewClient(*serverURL, *apiKey, *format)
|
||||
client, err := cli.NewClient(*serverURL, *apiKey, *format, *caBundlePath, *insecure)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Dispatch to appropriate command
|
||||
command := args[0]
|
||||
cmdArgs := args[1:]
|
||||
|
||||
var err error
|
||||
switch command {
|
||||
case "certs":
|
||||
err = handleCerts(client, cmdArgs)
|
||||
@@ -140,9 +158,19 @@ func handleCerts(client *cli.Client, args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// handleAgents dispatches the `agents` subcommands.
|
||||
//
|
||||
// I-004 additions:
|
||||
//
|
||||
// agents list --retired — hit the opt-in /agents/retired endpoint
|
||||
// instead of the default listing (which
|
||||
// filters retired rows out).
|
||||
// agents retire <id> — soft-retire an agent (DELETE /agents/{id}).
|
||||
// --force cascades; --reason is required with
|
||||
// --force (mirrors ErrForceReasonRequired).
|
||||
func handleAgents(client *cli.Client, args []string) error {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "usage: agents <list|get> [options]\n")
|
||||
fmt.Fprintf(os.Stderr, "usage: agents <list|get|retire> [options]\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -151,13 +179,34 @@ func handleAgents(client *cli.Client, args []string) error {
|
||||
|
||||
switch subcommand {
|
||||
case "list":
|
||||
return client.ListAgents(subArgs)
|
||||
// --retired flag splits to a separate endpoint. We intercept it
|
||||
// client-side and strip it before delegating, so both code paths
|
||||
// share the --page/--per-page flag parsing inside the client.
|
||||
retired := false
|
||||
rest := make([]string, 0, len(subArgs))
|
||||
for _, a := range subArgs {
|
||||
if a == "--retired" {
|
||||
retired = true
|
||||
continue
|
||||
}
|
||||
rest = append(rest, a)
|
||||
}
|
||||
if retired {
|
||||
return client.ListRetiredAgents(rest)
|
||||
}
|
||||
return client.ListAgents(rest)
|
||||
case "get":
|
||||
if len(subArgs) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "usage: agents get <id>\n")
|
||||
return nil
|
||||
}
|
||||
return client.GetAgent(subArgs[0])
|
||||
case "retire":
|
||||
if len(subArgs) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "usage: agents retire <id> [--force] [--reason <reason>]\n")
|
||||
return nil
|
||||
}
|
||||
return client.RetireAgent(subArgs)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown subcommand: agents %s\n", subcommand)
|
||||
return nil
|
||||
@@ -205,3 +254,26 @@ func handleImport(client *cli.Client, args []string) error {
|
||||
func handleStatus(client *cli.Client) error {
|
||||
return client.GetStatus()
|
||||
}
|
||||
|
||||
// validateHTTPSScheme rejects plaintext and empty-scheme server URLs at
|
||||
// startup so operators get a fail-loud diagnostic before any network call,
|
||||
// not a TCP-refused or TLS-handshake-error downstream. See docs/upgrade-to-tls.md.
|
||||
func validateHTTPSScheme(serverURL string) error {
|
||||
if serverURL == "" {
|
||||
return fmt.Errorf("server URL is empty — set --server (or CERTCTL_SERVER_URL) to an https:// URL (e.g., https://certctl-server:8443)")
|
||||
}
|
||||
u, err := url.Parse(serverURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("server URL %q is not a valid URL: %w", serverURL, err)
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "https":
|
||||
return nil
|
||||
case "http":
|
||||
return fmt.Errorf("server URL %q uses plaintext http:// — the certctl control plane is HTTPS-only", serverURL)
|
||||
case "":
|
||||
return fmt.Errorf("server URL %q is missing a scheme — expected https://", serverURL)
|
||||
default:
|
||||
return fmt.Errorf("server URL %q uses unsupported scheme %q — expected https://", serverURL, u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestValidateHTTPSScheme pins the pre-flight URL-scheme guard that the
|
||||
// HTTPS-Everywhere milestone (v2.2, §3.2) requires on the certctl-cli binary
|
||||
// startup path. The CLI's diagnostic is distinct from the agent and MCP server
|
||||
// because it surfaces the --server flag alongside CERTCTL_SERVER_URL — so the
|
||||
// empty-URL case pins that flag-name substring separately. Every other case
|
||||
// mirrors the dispatch arms in cmd/cli/main.go:validateHTTPSScheme; drifting
|
||||
// the substrings is what this test is here to catch.
|
||||
func TestValidateHTTPSScheme(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
serverURL string
|
||||
wantErr bool
|
||||
wantErrSub string // substring that MUST appear in the error message
|
||||
}{
|
||||
{
|
||||
name: "https URL passes",
|
||||
serverURL: "https://certctl-server:8443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "https URL with path passes",
|
||||
serverURL: "https://certctl.example.com/api/v1",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "uppercase HTTPS scheme passes (url.Parse lowercases)",
|
||||
serverURL: "HTTPS://certctl-server:8443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty URL rejected mentions --server flag",
|
||||
serverURL: "",
|
||||
wantErr: true,
|
||||
wantErrSub: "--server",
|
||||
},
|
||||
{
|
||||
name: "empty URL rejected also mentions CERTCTL_SERVER_URL",
|
||||
serverURL: "",
|
||||
wantErr: true,
|
||||
wantErrSub: "CERTCTL_SERVER_URL",
|
||||
},
|
||||
{
|
||||
name: "plaintext http rejected",
|
||||
serverURL: "http://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "plaintext http://",
|
||||
},
|
||||
{
|
||||
name: "bare host missing scheme rejected",
|
||||
serverURL: "localhost:8443",
|
||||
wantErr: true,
|
||||
// url.Parse treats "localhost:8443" as scheme=localhost, opaque=8443
|
||||
// — exercises the default arm (unsupported scheme) rather than the
|
||||
// empty-scheme arm. Both are fail-closed, which is what we care about.
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
{
|
||||
name: "path-only URL rejected",
|
||||
serverURL: "//certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "missing a scheme",
|
||||
},
|
||||
{
|
||||
name: "unsupported scheme rejected",
|
||||
serverURL: "ftp://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
{
|
||||
name: "ws scheme rejected",
|
||||
serverURL: "ws://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateHTTPSScheme(tt.serverURL)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("validateHTTPSScheme(%q) err=%v wantErr=%v", tt.serverURL, err, tt.wantErr)
|
||||
}
|
||||
if tt.wantErr && tt.wantErrSub != "" && !strings.Contains(err.Error(), tt.wantErrSub) {
|
||||
t.Errorf("validateHTTPSScheme(%q) err=%q must contain %q so operators see the right diagnostic",
|
||||
tt.serverURL, err.Error(), tt.wantErrSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+46
-2
@@ -4,8 +4,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
|
||||
gomcp "github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
|
||||
@@ -16,14 +18,33 @@ import (
|
||||
var Version = "dev"
|
||||
|
||||
func main() {
|
||||
// HTTPS-Everywhere (v2.2): the server is HTTPS-only. The default URL
|
||||
// uses https://; plaintext http:// is rejected by validateHTTPSScheme
|
||||
// below with a fail-loud pre-flight diagnostic pointing at
|
||||
// docs/upgrade-to-tls.md, so operators never get a TCP-refused or
|
||||
// TLS-handshake-error downstream. See docs/tls.md for CA bundle and
|
||||
// insecure-skip-verify guidance.
|
||||
serverURL := os.Getenv("CERTCTL_SERVER_URL")
|
||||
if serverURL == "" {
|
||||
serverURL = "http://localhost:8443"
|
||||
serverURL = "https://localhost:8443"
|
||||
}
|
||||
|
||||
if err := validateHTTPSScheme(serverURL); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "\nThe certctl control plane is HTTPS-only as of v2.2.\n")
|
||||
fmt.Fprintf(os.Stderr, "See docs/upgrade-to-tls.md for the cutover walkthrough.\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
apiKey := os.Getenv("CERTCTL_API_KEY")
|
||||
caBundlePath := os.Getenv("CERTCTL_SERVER_CA_BUNDLE_PATH")
|
||||
insecure := strings.EqualFold(os.Getenv("CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY"), "true")
|
||||
|
||||
client := mcp.NewClient(serverURL, apiKey)
|
||||
client, err := mcp.NewClient(serverURL, apiKey, caBundlePath, insecure)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
server := gomcp.NewServer(&gomcp.Implementation{
|
||||
Name: "certctl",
|
||||
@@ -41,3 +62,26 @@ func main() {
|
||||
log.Fatalf("MCP server error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// validateHTTPSScheme rejects plaintext and empty-scheme server URLs at
|
||||
// startup so operators get a fail-loud diagnostic before any network call,
|
||||
// not a TCP-refused or TLS-handshake-error downstream. See docs/upgrade-to-tls.md.
|
||||
func validateHTTPSScheme(serverURL string) error {
|
||||
if serverURL == "" {
|
||||
return fmt.Errorf("server URL is empty — set CERTCTL_SERVER_URL to an https:// URL (e.g., https://certctl-server:8443)")
|
||||
}
|
||||
u, err := url.Parse(serverURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("server URL %q is not a valid URL: %w", serverURL, err)
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "https":
|
||||
return nil
|
||||
case "http":
|
||||
return fmt.Errorf("server URL %q uses plaintext http:// — the certctl control plane is HTTPS-only", serverURL)
|
||||
case "":
|
||||
return fmt.Errorf("server URL %q is missing a scheme — expected https://", serverURL)
|
||||
default:
|
||||
return fmt.Errorf("server URL %q uses unsupported scheme %q — expected https://", serverURL, u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestValidateHTTPSScheme pins the pre-flight URL-scheme guard that the
|
||||
// HTTPS-Everywhere milestone (v2.2, §3.2) requires on the MCP server binary
|
||||
// startup path. The whole point is to fail loud with a diagnostic that points
|
||||
// at docs/upgrade-to-tls.md *before* any network call — not a cryptic
|
||||
// TCP-refused or TLS-handshake-error two ticks later. Every case here mirrors
|
||||
// the dispatch arms in cmd/mcp-server/main.go:validateHTTPSScheme; drifting
|
||||
// the error-message substrings is what this test is here to catch.
|
||||
func TestValidateHTTPSScheme(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
serverURL string
|
||||
wantErr bool
|
||||
wantErrSub string // substring that MUST appear in the error message
|
||||
}{
|
||||
{
|
||||
name: "https URL passes",
|
||||
serverURL: "https://certctl-server:8443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "https URL with path passes",
|
||||
serverURL: "https://certctl.example.com/api/v1",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "uppercase HTTPS scheme passes (url.Parse lowercases)",
|
||||
serverURL: "HTTPS://certctl-server:8443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty URL rejected",
|
||||
serverURL: "",
|
||||
wantErr: true,
|
||||
wantErrSub: "server URL is empty",
|
||||
},
|
||||
{
|
||||
name: "plaintext http rejected",
|
||||
serverURL: "http://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "plaintext http://",
|
||||
},
|
||||
{
|
||||
name: "bare host missing scheme rejected",
|
||||
serverURL: "localhost:8443",
|
||||
wantErr: true,
|
||||
// url.Parse treats "localhost:8443" as scheme=localhost, opaque=8443
|
||||
// — exercises the default arm (unsupported scheme) rather than the
|
||||
// empty-scheme arm. Both are fail-closed, which is what we care about.
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
{
|
||||
name: "path-only URL rejected",
|
||||
serverURL: "//certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "missing a scheme",
|
||||
},
|
||||
{
|
||||
name: "unsupported scheme rejected",
|
||||
serverURL: "ftp://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
{
|
||||
name: "ws scheme rejected",
|
||||
serverURL: "ws://certctl-server:8443",
|
||||
wantErr: true,
|
||||
wantErrSub: "unsupported scheme",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateHTTPSScheme(tt.serverURL)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("validateHTTPSScheme(%q) err=%v wantErr=%v", tt.serverURL, err, tt.wantErr)
|
||||
}
|
||||
if tt.wantErr && tt.wantErrSub != "" && !strings.Contains(err.Error(), tt.wantErrSub) {
|
||||
t.Errorf("validateHTTPSScheme(%q) err=%q must contain %q so operators see the right diagnostic",
|
||||
tt.serverURL, err.Error(), tt.wantErrSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestBuildFinalHandler_Dispatch is the M-001 regression harness for the outer
|
||||
// HTTP dispatch layer. It pins which path prefixes ride the no-auth middleware
|
||||
// chain (EST, SCEP, /.well-known/pki, health/ready, /api/v1/auth/info) versus
|
||||
// the authenticated chain (/api/v1/*).
|
||||
//
|
||||
// The concern under test is ONLY the dispatch in buildFinalHandler — the
|
||||
// handlers themselves are mocked as marker handlers that stamp "AUTH" or
|
||||
// "NOAUTH" into the response body. Service-layer concerns (SCEP password
|
||||
// validation, EST CSR validation, API auth enforcement) are covered by their
|
||||
// respective test suites.
|
||||
//
|
||||
// Case (i) is the central guard: EST with NO client cert / NO Bearer token
|
||||
// MUST reach the no-auth handler (pre-M-001 it was 401'd by the Auth
|
||||
// middleware, blocking enrollment for every real-world EST client).
|
||||
func TestBuildFinalHandler_Dispatch(t *testing.T) {
|
||||
// Marker handlers — each stamps a unique body so tests can verify which
|
||||
// chain the request traversed.
|
||||
authHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("X-Chain", "auth")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("AUTH"))
|
||||
})
|
||||
noAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("X-Chain", "noauth")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("NOAUTH"))
|
||||
})
|
||||
|
||||
// Dashboard directory with index.html + assets/ for SPA fallback and
|
||||
// static-asset tests. Cleaned up by t.TempDir.
|
||||
webDir := t.TempDir()
|
||||
indexHTML := []byte("<!doctype html><html><body>certctl dashboard</body></html>")
|
||||
if err := os.WriteFile(filepath.Join(webDir, "index.html"), indexHTML, 0o644); err != nil {
|
||||
t.Fatalf("write index.html: %v", err)
|
||||
}
|
||||
assetsDir := filepath.Join(webDir, "assets")
|
||||
if err := os.MkdirAll(assetsDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir assets: %v", err)
|
||||
}
|
||||
assetJS := []byte("console.log('certctl');")
|
||||
if err := os.WriteFile(filepath.Join(assetsDir, "app.js"), assetJS, 0o644); err != nil {
|
||||
t.Fatalf("write app.js: %v", err)
|
||||
}
|
||||
|
||||
handler := buildFinalHandler(authHandler, noAuthHandler, webDir, true /* dashboardEnabled */)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
path string
|
||||
wantBody string // "AUTH" | "NOAUTH" | "" (== substring match against response body)
|
||||
wantBodyPrefix string
|
||||
wantStatus int
|
||||
description string
|
||||
}{
|
||||
// ---- Case (i): M-001 central regression guard ----
|
||||
{
|
||||
name: "est_cacerts_no_auth_reaches_noauth_handler",
|
||||
method: http.MethodGet,
|
||||
path: "/.well-known/est/cacerts",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "EST clients cannot present Bearer tokens — must NOT be 401'd before reaching the handler (RFC 7030 §4.1.1)",
|
||||
},
|
||||
{
|
||||
name: "est_simpleenroll_no_auth_reaches_noauth_handler",
|
||||
method: http.MethodPost,
|
||||
path: "/.well-known/est/simpleenroll",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "RFC 7030 §4.2 simpleenroll served from no-auth chain (option D)",
|
||||
},
|
||||
{
|
||||
name: "est_simplereenroll_no_auth_reaches_noauth_handler",
|
||||
method: http.MethodPost,
|
||||
path: "/.well-known/est/simplereenroll",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "RFC 7030 §4.2.2 simplereenroll also on no-auth chain",
|
||||
},
|
||||
{
|
||||
name: "est_csrattrs_no_auth_reaches_noauth_handler",
|
||||
method: http.MethodGet,
|
||||
path: "/.well-known/est/csrattrs",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "RFC 7030 §4.5 csrattrs also on no-auth chain",
|
||||
},
|
||||
|
||||
// ---- Cases (ii) + (iii): SCEP dispatch ----
|
||||
// The actual challengePassword validation lives in the service layer
|
||||
// (internal/service/scep.go). This test pins that ALL /scep* requests
|
||||
// reach the no-auth chain — the service layer is then responsible for
|
||||
// rejecting or accepting based on password contents.
|
||||
{
|
||||
name: "scep_exact_path_reaches_noauth_handler",
|
||||
method: http.MethodGet,
|
||||
path: "/scep",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "SCEP clients authenticate via CSR challengePassword, not Bearer (RFC 8894 §3.2)",
|
||||
},
|
||||
{
|
||||
name: "scep_subpath_reaches_noauth_handler",
|
||||
method: http.MethodPost,
|
||||
path: "/scep/",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "Trailing-slash variant must also ride no-auth chain",
|
||||
},
|
||||
{
|
||||
name: "scep_query_string_reaches_noauth_handler",
|
||||
method: http.MethodGet,
|
||||
path: "/scep?operation=GetCACaps",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "Query string does not affect dispatch — operation dispatch is handler-internal",
|
||||
},
|
||||
// Defensive: /scepxyz MUST NOT match the SCEP prefix (guards against
|
||||
// over-broad matching that would leak non-SCEP paths into no-auth).
|
||||
{
|
||||
name: "scepxyz_does_not_match_scep_prefix",
|
||||
method: http.MethodGet,
|
||||
path: "/scepxyz",
|
||||
wantStatus: http.StatusOK,
|
||||
wantBody: "certctl dashboard",
|
||||
description: "SPA fallback — /scepxyz must not be confused with /scep or /scep/",
|
||||
},
|
||||
|
||||
// ---- Case (iv): RFC 5280 CRL + RFC 6960 OCSP ----
|
||||
{
|
||||
name: "pki_crl_no_auth_reaches_noauth_handler",
|
||||
method: http.MethodGet,
|
||||
path: "/.well-known/pki/crl/abc123",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "RFC 5280 CRL distribution point must be served without auth",
|
||||
},
|
||||
{
|
||||
name: "pki_ocsp_no_auth_reaches_noauth_handler",
|
||||
method: http.MethodGet,
|
||||
path: "/.well-known/pki/ocsp/abc123/serial",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "RFC 6960 OCSP responder must be served without auth",
|
||||
},
|
||||
|
||||
// ---- Case (v): Authenticated API routes ----
|
||||
{
|
||||
name: "api_v1_certificates_goes_through_auth",
|
||||
method: http.MethodGet,
|
||||
path: "/api/v1/certificates",
|
||||
wantBody: "AUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "Primary API surface must still require Bearer token",
|
||||
},
|
||||
{
|
||||
name: "api_v1_auth_check_goes_through_auth",
|
||||
method: http.MethodGet,
|
||||
path: "/api/v1/auth/check",
|
||||
wantBody: "AUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "auth/check validates the caller's Bearer — auth chain required",
|
||||
},
|
||||
{
|
||||
name: "api_v1_jobs_goes_through_auth",
|
||||
method: http.MethodGet,
|
||||
path: "/api/v1/jobs",
|
||||
wantBody: "AUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "Jobs API is part of the privileged surface",
|
||||
},
|
||||
|
||||
// ---- Health probes bypass auth ----
|
||||
{
|
||||
name: "health_bypasses_auth",
|
||||
method: http.MethodGet,
|
||||
path: "/health",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "Docker/K8s health probes cannot carry Bearer tokens",
|
||||
},
|
||||
{
|
||||
name: "ready_bypasses_auth",
|
||||
method: http.MethodGet,
|
||||
path: "/ready",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "Readiness probe also unauthenticated",
|
||||
},
|
||||
{
|
||||
name: "auth_info_bypasses_auth",
|
||||
method: http.MethodGet,
|
||||
path: "/api/v1/auth/info",
|
||||
wantBody: "NOAUTH",
|
||||
wantStatus: http.StatusOK,
|
||||
description: "React app calls auth/info BEFORE login to discover auth mode",
|
||||
},
|
||||
|
||||
// ---- Static assets served by file server ----
|
||||
{
|
||||
name: "static_asset_served_by_file_server",
|
||||
method: http.MethodGet,
|
||||
path: "/assets/app.js",
|
||||
wantStatus: http.StatusOK,
|
||||
wantBody: "console.log('certctl');",
|
||||
description: "Built Vite assets served directly without auth",
|
||||
},
|
||||
|
||||
// ---- SPA fallback ----
|
||||
{
|
||||
name: "spa_fallback_serves_index_html",
|
||||
method: http.MethodGet,
|
||||
path: "/",
|
||||
wantStatus: http.StatusOK,
|
||||
wantBody: "certctl dashboard",
|
||||
description: "Root path serves SPA entry point",
|
||||
},
|
||||
{
|
||||
name: "spa_fallback_for_unknown_route",
|
||||
method: http.MethodGet,
|
||||
path: "/certificates",
|
||||
wantStatus: http.StatusOK,
|
||||
wantBody: "certctl dashboard",
|
||||
description: "React Router routes fall through to index.html",
|
||||
},
|
||||
{
|
||||
name: "spa_fallback_deep_route",
|
||||
method: http.MethodGet,
|
||||
path: "/certificates/mc-api-prod/detail",
|
||||
wantStatus: http.StatusOK,
|
||||
wantBody: "certctl dashboard",
|
||||
description: "Deep React Router routes also fall through to SPA",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(tc.method, tc.path, nil)
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != tc.wantStatus {
|
||||
t.Errorf("status = %d, want %d (%s)", w.Code, tc.wantStatus, tc.description)
|
||||
}
|
||||
body := w.Body.String()
|
||||
if tc.wantBody != "" && !strings.Contains(body, tc.wantBody) {
|
||||
t.Errorf("body %q does not contain %q (%s)", body, tc.wantBody, tc.description)
|
||||
}
|
||||
if tc.wantBodyPrefix != "" && !strings.HasPrefix(body, tc.wantBodyPrefix) {
|
||||
t.Errorf("body %q does not start with %q (%s)", body, tc.wantBodyPrefix, tc.description)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildFinalHandler_NoDashboard pins the API-only (dashboard-absent)
|
||||
// dispatch behavior. When web/dist/index.html is missing, everything that's
|
||||
// not a no-auth bypass route falls through to the authenticated apiHandler
|
||||
// (pre-M-001 behavior for headless deployments). EST/SCEP/PKI still ride the
|
||||
// no-auth chain.
|
||||
func TestBuildFinalHandler_NoDashboard(t *testing.T) {
|
||||
authHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("AUTH"))
|
||||
})
|
||||
noAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("NOAUTH"))
|
||||
})
|
||||
|
||||
handler := buildFinalHandler(authHandler, noAuthHandler, "/nonexistent", false /* dashboardEnabled */)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantBody string
|
||||
}{
|
||||
{"est_still_no_auth", "/.well-known/est/cacerts", "NOAUTH"},
|
||||
{"scep_still_no_auth", "/scep", "NOAUTH"},
|
||||
{"pki_still_no_auth", "/.well-known/pki/crl/x", "NOAUTH"},
|
||||
{"health_still_no_auth", "/health", "NOAUTH"},
|
||||
{"api_still_auth", "/api/v1/certificates", "AUTH"},
|
||||
// The difference: non-API, non-special paths go through auth chain when
|
||||
// there's no dashboard to serve (preserves legacy headless behavior).
|
||||
{"unknown_path_falls_through_to_auth", "/", "AUTH"},
|
||||
{"unknown_deep_path_falls_through_to_auth", "/random/path", "AUTH"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, tc.path, nil)
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", w.Code)
|
||||
}
|
||||
if got := w.Body.String(); !strings.Contains(got, tc.wantBody) {
|
||||
t.Errorf("body = %q, want to contain %q", got, tc.wantBody)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+181
-75
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/shankar0123/certctl/internal/api/middleware"
|
||||
"github.com/shankar0123/certctl/internal/api/router"
|
||||
"github.com/shankar0123/certctl/internal/config"
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
discoveryawssm "github.com/shankar0123/certctl/internal/connector/discovery/awssm"
|
||||
discoveryazurekv "github.com/shankar0123/certctl/internal/connector/discovery/azurekv"
|
||||
discoverygcpsm "github.com/shankar0123/certctl/internal/connector/discovery/gcpsm"
|
||||
@@ -26,6 +25,7 @@ import (
|
||||
notifypagerduty "github.com/shankar0123/certctl/internal/connector/notifier/pagerduty"
|
||||
notifyslack "github.com/shankar0123/certctl/internal/connector/notifier/slack"
|
||||
notifyteams "github.com/shankar0123/certctl/internal/connector/notifier/teams"
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
"github.com/shankar0123/certctl/internal/repository/postgres"
|
||||
"github.com/shankar0123/certctl/internal/scheduler"
|
||||
"github.com/shankar0123/certctl/internal/service"
|
||||
@@ -353,6 +353,12 @@ func main() {
|
||||
|
||||
// Initialize stats and metrics services
|
||||
statsService := service.NewStatsService(certificateRepo, jobRepo, agentRepo)
|
||||
// I-005: wire the notification repository so DashboardSummary.NotificationsDead
|
||||
// is populated, which in turn drives the Prometheus counter
|
||||
// certctl_notification_dead_total in GetPrometheusMetrics. Setter
|
||||
// pattern keeps NewStatsService's nine call sites (main.go + stats_test.go
|
||||
// + 8 digest_test.go sites) untouched.
|
||||
statsService.SetNotifRepo(notificationRepo)
|
||||
logger.Info("initialized stats service")
|
||||
|
||||
// Initialize API handlers
|
||||
@@ -447,6 +453,14 @@ func main() {
|
||||
sched.SetJobRetryInterval(cfg.Scheduler.RetryInterval)
|
||||
sched.SetAgentHealthCheckInterval(cfg.Scheduler.AgentHealthCheckInterval)
|
||||
sched.SetNotificationProcessInterval(cfg.Scheduler.NotificationProcessInterval)
|
||||
// I-005: drive the failed-notification retry sweep. Runs every
|
||||
// NotificationRetryInterval (default 2m, CERTCTL_NOTIFICATION_RETRY_INTERVAL)
|
||||
// and transitions eligible Failed notifications whose next_retry_at has
|
||||
// arrived back to Pending so the notification processor picks them up on
|
||||
// its next tick. Kept adjacent to the notification processor setter
|
||||
// because they share the NotificationServicer dependency (same placement
|
||||
// pattern as I-001's SetJobRetryInterval above).
|
||||
sched.SetNotificationRetryInterval(cfg.Scheduler.NotificationRetryInterval)
|
||||
if cfg.NetworkScan.Enabled {
|
||||
sched.SetNetworkScanInterval(cfg.NetworkScan.ScanInterval)
|
||||
logger.Info("network scanning enabled", "interval", cfg.NetworkScan.ScanInterval.String())
|
||||
@@ -469,6 +483,16 @@ func main() {
|
||||
"sources", cloudDiscoveryService.SourceCount())
|
||||
}
|
||||
|
||||
// Wire job timeout reaper (I-003)
|
||||
sched.SetJobReaperService(jobService)
|
||||
sched.SetJobTimeoutInterval(cfg.Scheduler.JobTimeoutInterval)
|
||||
sched.SetAwaitingCSRTimeout(cfg.Scheduler.AwaitingCSRTimeout)
|
||||
sched.SetAwaitingApprovalTimeout(cfg.Scheduler.AwaitingApprovalTimeout)
|
||||
logger.Info("job timeout reaper enabled",
|
||||
"interval", cfg.Scheduler.JobTimeoutInterval.String(),
|
||||
"csr_timeout", cfg.Scheduler.AwaitingCSRTimeout.String(),
|
||||
"approval_timeout", cfg.Scheduler.AwaitingApprovalTimeout.String())
|
||||
|
||||
// Start scheduler
|
||||
logger.Info("starting scheduler")
|
||||
startedChan := sched.Start(ctx)
|
||||
@@ -478,28 +502,28 @@ func main() {
|
||||
// Build the API router with all handlers
|
||||
apiRouter := router.New()
|
||||
apiRouter.RegisterHandlers(router.HandlerRegistry{
|
||||
Certificates: certificateHandler,
|
||||
Issuers: issuerHandler,
|
||||
Targets: targetHandler,
|
||||
Agents: agentHandler,
|
||||
Jobs: jobHandler,
|
||||
Policies: policyHandler,
|
||||
Profiles: profileHandler,
|
||||
Teams: teamHandler,
|
||||
Owners: ownerHandler,
|
||||
AgentGroups: agentGroupHandler,
|
||||
Audit: auditHandler,
|
||||
Notifications: notificationHandler,
|
||||
Stats: statsHandler,
|
||||
Metrics: metricsHandler,
|
||||
Health: healthHandler,
|
||||
Discovery: discoveryHandler,
|
||||
NetworkScan: networkScanHandler,
|
||||
Verification: verificationHandler,
|
||||
Export: exportHandler,
|
||||
Digest: *digestHandler,
|
||||
HealthChecks: healthCheckHandler,
|
||||
BulkRevocation: bulkRevocationHandler,
|
||||
Certificates: certificateHandler,
|
||||
Issuers: issuerHandler,
|
||||
Targets: targetHandler,
|
||||
Agents: agentHandler,
|
||||
Jobs: jobHandler,
|
||||
Policies: policyHandler,
|
||||
Profiles: profileHandler,
|
||||
Teams: teamHandler,
|
||||
Owners: ownerHandler,
|
||||
AgentGroups: agentGroupHandler,
|
||||
Audit: auditHandler,
|
||||
Notifications: notificationHandler,
|
||||
Stats: statsHandler,
|
||||
Metrics: metricsHandler,
|
||||
Health: healthHandler,
|
||||
Discovery: discoveryHandler,
|
||||
NetworkScan: networkScanHandler,
|
||||
Verification: verificationHandler,
|
||||
Export: exportHandler,
|
||||
Digest: *digestHandler,
|
||||
HealthChecks: healthCheckHandler,
|
||||
BulkRevocation: bulkRevocationHandler,
|
||||
})
|
||||
// Register EST (RFC 7030) handlers if enabled
|
||||
if cfg.EST.Enabled {
|
||||
@@ -701,74 +725,65 @@ func main() {
|
||||
middleware.Recovery,
|
||||
)
|
||||
|
||||
dashboardEnabled := false
|
||||
if _, err := os.Stat(webDir + "/index.html"); err == nil {
|
||||
fileServer := http.FileServer(http.Dir(webDir))
|
||||
finalHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
// Health/ready and auth/info bypass auth middleware.
|
||||
// Health/ready: Docker/K8s health probes don't carry Bearer tokens.
|
||||
// auth/info: React app calls this before login to detect auth mode.
|
||||
if path == "/health" || path == "/ready" || path == "/api/v1/auth/info" {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
// RFC 5280 CRL and RFC 6960 OCSP live under /.well-known/pki/ and
|
||||
// MUST be served unauthenticated — relying parties (browsers,
|
||||
// OpenSSL, OCSP stapling sidecars, mTLS clients) cannot present
|
||||
// certctl Bearer tokens. See router.RegisterPKIHandlers.
|
||||
if len(path) >= 16 && path[:16] == "/.well-known/pki" {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
// All other API and EST routes go through the full middleware stack (with auth)
|
||||
if (len(path) >= 8 && path[:8] == "/api/v1/") ||
|
||||
(len(path) >= 16 && path[:16] == "/.well-known/est") {
|
||||
apiHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
// Try to serve static files (JS, CSS, assets)
|
||||
if len(path) > 8 && path[:8] == "/assets/" {
|
||||
fileServer.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
// SPA fallback: serve index.html for all other routes
|
||||
http.ServeFile(w, r, webDir+"/index.html")
|
||||
})
|
||||
dashboardEnabled = true
|
||||
}
|
||||
finalHandler = buildFinalHandler(apiHandler, noAuthHandler, webDir, dashboardEnabled)
|
||||
if dashboardEnabled {
|
||||
logger.Info("dashboard available at /", "web_dir", webDir)
|
||||
} else {
|
||||
// No dashboard: route health/auth-info and /.well-known/pki without
|
||||
// auth, everything else through full stack.
|
||||
finalHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
if path == "/health" || path == "/ready" || path == "/api/v1/auth/info" {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if len(path) >= 16 && path[:16] == "/.well-known/pki" {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
apiHandler.ServeHTTP(w, r)
|
||||
})
|
||||
logger.Info("dashboard directory not found, serving API only")
|
||||
}
|
||||
|
||||
// HTTPS-everywhere milestone §2.1: fail-loud if the TLS configuration is
|
||||
// missing or malformed. Duplicates config.Validate() for defense in depth
|
||||
// (same pattern as preflightSCEPChallengePassword).
|
||||
if err := preflightServerTLS(cfg.Server.TLS.CertPath, cfg.Server.TLS.KeyPath); err != nil {
|
||||
logger.Error("startup refused: HTTPS cert unusable; control plane is HTTPS-only",
|
||||
"error", err,
|
||||
"cert_path", cfg.Server.TLS.CertPath,
|
||||
"key_path", cfg.Server.TLS.KeyPath)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Load the cert+key into a SIGHUP-reloadable holder. Any subsequent
|
||||
// SIGHUP triggers a fresh read and atomic swap so rotations do not need
|
||||
// a restart. Reload failures keep the previous cert and log a warning.
|
||||
tlsCertHolder, err := newCertHolder(cfg.Server.TLS.CertPath, cfg.Server.TLS.KeyPath)
|
||||
if err != nil {
|
||||
logger.Error("startup refused: failed to load TLS cert holder",
|
||||
"error", err,
|
||||
"cert_path", cfg.Server.TLS.CertPath,
|
||||
"key_path", cfg.Server.TLS.KeyPath)
|
||||
os.Exit(1)
|
||||
}
|
||||
stopTLSWatcher := tlsCertHolder.watchSIGHUP(logger)
|
||||
defer stopTLSWatcher()
|
||||
|
||||
// Server configuration
|
||||
addr := net.JoinHostPort(cfg.Server.Host, strconv.Itoa(cfg.Server.Port))
|
||||
httpServer := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: finalHandler,
|
||||
TLSConfig: buildServerTLSConfig(tlsCertHolder),
|
||||
ReadTimeout: 30 * time.Second,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
WriteTimeout: 120 * time.Second, // Must accommodate ACME issuance (order + challenge + finalize)
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
// Start HTTP server in background
|
||||
logger.Info("starting HTTP server", "address", addr)
|
||||
// Start HTTPS server in background. ListenAndServeTLS is called with
|
||||
// empty cert+key arguments because the cert is sourced through
|
||||
// TLSConfig.GetCertificate (the SIGHUP-reloadable holder). Passing file
|
||||
// paths here would pin the first-loaded cert and defeat hot reload.
|
||||
logger.Info("HTTPS server listening",
|
||||
"address", addr,
|
||||
"cert_path", cfg.Server.TLS.CertPath,
|
||||
"min_version", "TLS1.3")
|
||||
go func() {
|
||||
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
logger.Error("HTTP server error", "error", err)
|
||||
if err := httpServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
|
||||
logger.Error("HTTPS server error", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -791,9 +806,9 @@ func main() {
|
||||
logger.Warn("scheduler work did not complete in time", "error", err)
|
||||
}
|
||||
|
||||
logger.Info("shutting down HTTP server")
|
||||
logger.Info("shutting down HTTPS server")
|
||||
if err := httpServer.Shutdown(shutdownCtx); err != nil {
|
||||
logger.Error("HTTP server shutdown error", "error", err)
|
||||
logger.Error("HTTPS server shutdown error", "error", err)
|
||||
}
|
||||
|
||||
// Drain in-flight audit-recording goroutines before closing the DB pool.
|
||||
@@ -835,3 +850,94 @@ func preflightSCEPChallengePassword(enabled bool, challengePassword string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildFinalHandler builds the outer HTTP dispatch handler that routes incoming
|
||||
// requests to either the authenticated apiHandler chain or the unauthenticated
|
||||
// noAuthHandler chain based on URL path prefix. Extracted from main() so the
|
||||
// dispatch logic can be unit tested without booting the full server stack
|
||||
// (see cmd/server/finalhandler_test.go).
|
||||
//
|
||||
// Dispatch rules (M-001, audit 2026-04-19, option D):
|
||||
//
|
||||
// - /health, /ready, /api/v1/auth/info → no-auth (probes + login detection)
|
||||
// - /.well-known/pki/* → no-auth (RFC 5280 CRL, RFC 6960 OCSP)
|
||||
// - /.well-known/est/* → no-auth (RFC 7030 §3.2.3)
|
||||
// - /scep, /scep/* → no-auth (RFC 8894 §3.2, CSR challengePassword)
|
||||
// - /api/v1/* → auth (Bearer token required)
|
||||
// - /assets/* → static file server (dashboard only)
|
||||
// - anything else → SPA index.html fallback (dashboard only)
|
||||
// OR apiHandler (no dashboard)
|
||||
//
|
||||
// EST/SCEP clients (IoT devices, 802.1X supplicants, MDM endpoints, network
|
||||
// appliances) cannot present certctl Bearer tokens, so those endpoints must be
|
||||
// reachable without the Auth middleware. Authentication is instead enforced by
|
||||
// CSR signature verification, profile policy gates, and for SCEP the
|
||||
// challengePassword shared secret (fail-loud gated by preflightSCEPChallengePassword
|
||||
// above).
|
||||
//
|
||||
// webDir must point to a directory containing index.html + assets/ when
|
||||
// dashboardEnabled is true; it is ignored otherwise.
|
||||
func buildFinalHandler(apiHandler, noAuthHandler http.Handler, webDir string, dashboardEnabled bool) http.Handler {
|
||||
var fileServer http.Handler
|
||||
if dashboardEnabled {
|
||||
fileServer = http.FileServer(http.Dir(webDir))
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
|
||||
// Health/ready and auth/info bypass auth middleware.
|
||||
// Health/ready: Docker/K8s health probes don't carry Bearer tokens.
|
||||
// auth/info: React app calls this before login to detect auth mode.
|
||||
if path == "/health" || path == "/ready" || path == "/api/v1/auth/info" {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// RFC 5280 CRL and RFC 6960 OCSP live under /.well-known/pki/ and MUST
|
||||
// be served unauthenticated — relying parties (browsers, OpenSSL, OCSP
|
||||
// stapling sidecars, mTLS clients) cannot present certctl Bearer tokens.
|
||||
if strings.HasPrefix(path, "/.well-known/pki") {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// RFC 7030 EST endpoints ride the no-auth middleware chain (M-001,
|
||||
// option D, audit 2026-04-19). Trust boundary is CSR signature + profile
|
||||
// policy, not HTTP Bearer. /.well-known/est/cacerts is explicitly
|
||||
// anonymous per RFC 7030 §4.1.1.
|
||||
if strings.HasPrefix(path, "/.well-known/est") {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// RFC 8894 SCEP rides the no-auth chain (M-001, option D). SCEP clients
|
||||
// authenticate via the challengePassword attribute in the PKCS#10 CSR,
|
||||
// not via HTTP Bearer tokens. preflightSCEPChallengePassword refuses to
|
||||
// start the server if SCEP is enabled without a non-empty shared secret.
|
||||
if path == "/scep" || strings.HasPrefix(path, "/scep/") {
|
||||
noAuthHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Authenticated API routes — full middleware stack including Auth.
|
||||
if strings.HasPrefix(path, "/api/v1/") {
|
||||
apiHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if !dashboardEnabled {
|
||||
// No dashboard: everything non-special falls through to the
|
||||
// authenticated handler (preserves pre-M-001 behavior for API-only
|
||||
// deployments).
|
||||
apiHandler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Dashboard-present: serve static assets directly, SPA fallback for
|
||||
// everything else.
|
||||
if strings.HasPrefix(path, "/assets/") {
|
||||
fileServer.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
http.ServeFile(w, r, webDir+"/index.html")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -214,6 +214,8 @@ func TestMain_ServerConfigFromEnvironment(t *testing.T) {
|
||||
oldAuthType := os.Getenv("CERTCTL_AUTH_TYPE")
|
||||
oldServerHost := os.Getenv("CERTCTL_SERVER_HOST")
|
||||
oldServerPort := os.Getenv("CERTCTL_SERVER_PORT")
|
||||
oldTLSCert := os.Getenv("CERTCTL_SERVER_TLS_CERT_PATH")
|
||||
oldTLSKey := os.Getenv("CERTCTL_SERVER_TLS_KEY_PATH")
|
||||
defer func() {
|
||||
if oldAuthType != "" {
|
||||
os.Setenv("CERTCTL_AUTH_TYPE", oldAuthType)
|
||||
@@ -230,12 +232,32 @@ func TestMain_ServerConfigFromEnvironment(t *testing.T) {
|
||||
} else {
|
||||
os.Unsetenv("CERTCTL_SERVER_PORT")
|
||||
}
|
||||
if oldTLSCert != "" {
|
||||
os.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", oldTLSCert)
|
||||
} else {
|
||||
os.Unsetenv("CERTCTL_SERVER_TLS_CERT_PATH")
|
||||
}
|
||||
if oldTLSKey != "" {
|
||||
os.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", oldTLSKey)
|
||||
} else {
|
||||
os.Unsetenv("CERTCTL_SERVER_TLS_KEY_PATH")
|
||||
}
|
||||
}()
|
||||
|
||||
// HTTPS-only control plane: Validate() refuses to pass without a readable
|
||||
// cert/key pair on disk. Materialize a throwaway ECDSA P-256 pair using the
|
||||
// same generator cmd/server/tls_test.go uses for the certHolder tests.
|
||||
dir := t.TempDir()
|
||||
certPath := dir + "/server.crt"
|
||||
keyPath := dir + "/server.key"
|
||||
generateTestCert(t, certPath, keyPath, "main-test-cn")
|
||||
|
||||
// Set test env vars
|
||||
os.Setenv("CERTCTL_AUTH_TYPE", "none")
|
||||
os.Setenv("CERTCTL_SERVER_HOST", "127.0.0.1")
|
||||
os.Setenv("CERTCTL_SERVER_PORT", "8080")
|
||||
os.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", certPath)
|
||||
os.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", keyPath)
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
@@ -260,6 +282,8 @@ func TestMain_AuthTypeConfiguration(t *testing.T) {
|
||||
// Save original env vars
|
||||
oldAuthType := os.Getenv("CERTCTL_AUTH_TYPE")
|
||||
oldAuthSecret := os.Getenv("CERTCTL_AUTH_SECRET")
|
||||
oldTLSCert := os.Getenv("CERTCTL_SERVER_TLS_CERT_PATH")
|
||||
oldTLSKey := os.Getenv("CERTCTL_SERVER_TLS_KEY_PATH")
|
||||
defer func() {
|
||||
if oldAuthType != "" {
|
||||
os.Setenv("CERTCTL_AUTH_TYPE", oldAuthType)
|
||||
@@ -271,8 +295,28 @@ func TestMain_AuthTypeConfiguration(t *testing.T) {
|
||||
} else {
|
||||
os.Unsetenv("CERTCTL_AUTH_SECRET")
|
||||
}
|
||||
if oldTLSCert != "" {
|
||||
os.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", oldTLSCert)
|
||||
} else {
|
||||
os.Unsetenv("CERTCTL_SERVER_TLS_CERT_PATH")
|
||||
}
|
||||
if oldTLSKey != "" {
|
||||
os.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", oldTLSKey)
|
||||
} else {
|
||||
os.Unsetenv("CERTCTL_SERVER_TLS_KEY_PATH")
|
||||
}
|
||||
}()
|
||||
|
||||
// HTTPS-only control plane: config.Load()→Validate() refuses to pass
|
||||
// without a readable cert/key pair. Mint one throwaway pair for the whole
|
||||
// sub-test cohort — auth type toggles don't care about the TLS surface.
|
||||
dir := t.TempDir()
|
||||
certPath := dir + "/server.crt"
|
||||
keyPath := dir + "/server.key"
|
||||
generateTestCert(t, certPath, keyPath, "main-test-cn")
|
||||
os.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", certPath)
|
||||
os.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", keyPath)
|
||||
|
||||
// Set auth secret for api-key mode
|
||||
os.Setenv("CERTCTL_AUTH_SECRET", "test-secret")
|
||||
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// certHolder stores the server's TLS certificate under a mutex so it can be
|
||||
// swapped atomically by a SIGHUP handler without restarting the server. A
|
||||
// *tls.Config that wires GetCertificate → (*certHolder).GetCertificate reads
|
||||
// through the holder on every ClientHello, so a successful reload takes
|
||||
// effect on the next new connection immediately and without dropping
|
||||
// in-flight requests.
|
||||
//
|
||||
// Concurrency: GetCertificate is invoked from crypto/tls handshake goroutines
|
||||
// on every new inbound connection; Reload is invoked from the SIGHUP watcher
|
||||
// goroutine. sync.Mutex is sufficient — TLS handshakes are not an inner-loop
|
||||
// hot path and the critical section is a single pointer read.
|
||||
type certHolder struct {
|
||||
mu sync.Mutex
|
||||
cert *tls.Certificate
|
||||
certPath string
|
||||
keyPath string
|
||||
}
|
||||
|
||||
// newCertHolder loads the initial cert+key pair from disk and returns a
|
||||
// holder ready to serve handshakes. Returns a non-nil error if either file
|
||||
// is missing, unreadable, or the pair does not round-trip through
|
||||
// tls.LoadX509KeyPair (for example the key does not sign the cert). The
|
||||
// caller is expected to treat a non-nil error as a fail-loud startup gate
|
||||
// and os.Exit(1) — the HTTPS-everywhere milestone (§3 locked decisions)
|
||||
// prohibits plaintext HTTP fallback.
|
||||
func newCertHolder(certPath, keyPath string) (*certHolder, error) {
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load TLS cert/key (cert=%q key=%q): %w", certPath, keyPath, err)
|
||||
}
|
||||
return &certHolder{
|
||||
cert: &cert,
|
||||
certPath: certPath,
|
||||
keyPath: keyPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetCertificate is the tls.Config.GetCertificate hook. Returns the current
|
||||
// cert under the holder's mutex. ClientHelloInfo is ignored — the control
|
||||
// plane does not multiplex by SNI.
|
||||
func (h *certHolder) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.cert, nil
|
||||
}
|
||||
|
||||
// Reload re-reads the cert+key pair from disk and swaps the holder
|
||||
// atomically on success. On failure the holder retains its previous cert
|
||||
// and the error is propagated to the caller — the SIGHUP watcher logs and
|
||||
// keeps serving the previous cert rather than crashing on a bad reload.
|
||||
// This is deliberately "fail-safe on reload, fail-loud on startup": an
|
||||
// operator rotating certs wants a recoverable error, not a restart loop.
|
||||
func (h *certHolder) Reload() error {
|
||||
cert, err := tls.LoadX509KeyPair(h.certPath, h.keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reload TLS cert/key (cert=%q key=%q): %w", h.certPath, h.keyPath, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.cert = &cert
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchSIGHUP installs a signal handler that calls Reload() on each SIGHUP.
|
||||
// The returned stop function closes the internal done channel and stops
|
||||
// signal delivery so the goroutine can exit cleanly during shutdown. Errors
|
||||
// from Reload are logged but do not terminate the watcher — the operator
|
||||
// can fix the files and send another SIGHUP.
|
||||
//
|
||||
// Defensive design note: this deliberately does NOT panic on Reload error
|
||||
// even though HTTPS is mission-critical. A rotation that writes half-files
|
||||
// (operator overwrites cert.pem then key.pem as two separate copies) would
|
||||
// otherwise crash the server mid-rotation. Logging + retaining the old
|
||||
// cert gives the operator a bounded window to fix and re-SIGHUP.
|
||||
func (h *certHolder) watchSIGHUP(logger *slog.Logger) (stop func()) {
|
||||
ch := make(chan os.Signal, 1)
|
||||
signal.Notify(ch, syscall.SIGHUP)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ch:
|
||||
if err := h.Reload(); err != nil {
|
||||
logger.Error("TLS cert reload failed; continuing with previous cert",
|
||||
"error", err,
|
||||
"cert_path", h.certPath,
|
||||
"key_path", h.keyPath)
|
||||
continue
|
||||
}
|
||||
logger.Info("TLS cert reloaded via SIGHUP",
|
||||
"cert_path", h.certPath,
|
||||
"key_path", h.keyPath)
|
||||
case <-done:
|
||||
signal.Stop(ch)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { close(done) }
|
||||
}
|
||||
|
||||
// buildServerTLSConfig returns the TLS 1.3-only *tls.Config for the HTTPS
|
||||
// server. Pinned per HTTPS-everywhere milestone §2.1 + §3 locked decisions:
|
||||
//
|
||||
// - MinVersion: TLS 1.3 (no TLS 1.2 escape hatch). Go 1.25's crypto/tls
|
||||
// automatically rejects older versions.
|
||||
// - CurvePreferences: explicit [X25519, P-256]. Explicit ordering keeps
|
||||
// the handshake deterministic and documents the accepted curves.
|
||||
// - No CipherSuites field: TLS 1.3 cipher suites are not negotiable in
|
||||
// the handshake (all three mandatory suites — AES-128-GCM-SHA256,
|
||||
// AES-256-GCM-SHA384, CHACHA20-POLY1305-SHA256 — are always offered).
|
||||
// Go's crypto/tls ignores CipherSuites for TLS 1.3.
|
||||
// - GetCertificate: reads through the holder so SIGHUP rotations take
|
||||
// effect on the next new connection without a restart. Setting
|
||||
// tls.Config.Certificates directly would pin the first-loaded cert
|
||||
// and defeat SIGHUP reload.
|
||||
func buildServerTLSConfig(holder *certHolder) *tls.Config {
|
||||
return &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256},
|
||||
GetCertificate: holder.GetCertificate,
|
||||
}
|
||||
}
|
||||
|
||||
// preflightServerTLS is the fail-loud startup gate for HTTPS. Returns a
|
||||
// non-nil error when the TLS configuration is missing or the cert+key pair
|
||||
// cannot be parsed, so the caller refuses to start the control plane
|
||||
// (HTTPS-everywhere §3 locked decisions: no plaintext HTTP fallback).
|
||||
//
|
||||
// Duplicates the emptiness + stat + parse checks in config.Validate() for
|
||||
// defense in depth, mirroring the pattern established by
|
||||
// preflightSCEPChallengePassword (which itself duplicates
|
||||
// config.Validate()'s SCEP check for CWE-306). Extracted into a separate
|
||||
// function so the gate is unit-testable without booting the full server.
|
||||
func preflightServerTLS(certPath, keyPath string) error {
|
||||
if certPath == "" {
|
||||
return fmt.Errorf("CERTCTL_SERVER_TLS_CERT_PATH is empty: HTTPS-only control plane refuses to start (see docs/tls.md)")
|
||||
}
|
||||
if keyPath == "" {
|
||||
return fmt.Errorf("CERTCTL_SERVER_TLS_KEY_PATH is empty: HTTPS-only control plane refuses to start (see docs/tls.md)")
|
||||
}
|
||||
if _, err := os.Stat(certPath); err != nil {
|
||||
return fmt.Errorf("TLS cert file %q unreadable: %w (see docs/tls.md)", certPath, err)
|
||||
}
|
||||
if _, err := os.Stat(keyPath); err != nil {
|
||||
return fmt.Errorf("TLS key file %q unreadable: %w (see docs/tls.md)", keyPath, err)
|
||||
}
|
||||
if _, err := tls.LoadX509KeyPair(certPath, keyPath); err != nil {
|
||||
return fmt.Errorf("TLS cert/key pair invalid (cert=%q key=%q): %w (see docs/tls.md)", certPath, keyPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// generateTestCert writes a PEM-encoded self-signed leaf cert + ECDSA P-256
|
||||
// key pair to certPath/keyPath. The subject is derived from cn so tests can
|
||||
// tell reloaded certs apart from original certs by re-parsing the served
|
||||
// Certificate and comparing the CN.
|
||||
func generateTestCert(t *testing.T, certPath, keyPath, cn string) {
|
||||
t.Helper()
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ecdsa.GenerateKey: %v", err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
||||
Subject: pkix.Name{CommonName: cn},
|
||||
NotBefore: time.Now().Add(-1 * time.Hour),
|
||||
NotAfter: time.Now().Add(24 * time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
DNSNames: []string{"localhost"},
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
t.Fatalf("x509.CreateCertificate: %v", err)
|
||||
}
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalECPrivateKey: %v", err)
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
if err := os.WriteFile(certPath, certPEM, 0o600); err != nil {
|
||||
t.Fatalf("write cert: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
t.Fatalf("write key: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// readCertCN returns the CommonName from the leaf cert currently held by the
|
||||
// holder, by exercising the same GetCertificate path the tls handshake would
|
||||
// take. Lets tests assert which generation of the cert is being served.
|
||||
func readCertCN(t *testing.T, h *certHolder) string {
|
||||
t.Helper()
|
||||
c, err := h.GetCertificate(&tls.ClientHelloInfo{})
|
||||
if err != nil {
|
||||
t.Fatalf("GetCertificate: %v", err)
|
||||
}
|
||||
leaf, err := x509.ParseCertificate(c.Certificate[0])
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCertificate: %v", err)
|
||||
}
|
||||
return leaf.Subject.CommonName
|
||||
}
|
||||
|
||||
func silentLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError}))
|
||||
}
|
||||
|
||||
func TestNewCertHolder_ValidPair_LoadsCert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-initial")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
if got := readCertCN(t, h); got != "cn-initial" {
|
||||
t.Fatalf("CN mismatch: got %q want %q", got, "cn-initial")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCertHolder_MissingFile_Fails(t *testing.T) {
|
||||
_, err := newCertHolder("/nonexistent/cert.pem", "/nonexistent/key.pem")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing files, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCertHolder_MalformedCert_Fails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "bad.crt")
|
||||
keyPath := filepath.Join(dir, "bad.key")
|
||||
if err := os.WriteFile(certPath, []byte("not a pem cert"), 0o600); err != nil {
|
||||
t.Fatalf("write cert: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, []byte("not a pem key"), 0o600); err != nil {
|
||||
t.Fatalf("write key: %v", err)
|
||||
}
|
||||
_, err := newCertHolder(certPath, keyPath)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for malformed PEM, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertHolder_Reload_SwapsCert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-v1")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
if got := readCertCN(t, h); got != "cn-v1" {
|
||||
t.Fatalf("initial CN: got %q want cn-v1", got)
|
||||
}
|
||||
|
||||
// Rotate on disk and reload.
|
||||
generateTestCert(t, certPath, keyPath, "cn-v2")
|
||||
if err := h.Reload(); err != nil {
|
||||
t.Fatalf("Reload: %v", err)
|
||||
}
|
||||
if got := readCertCN(t, h); got != "cn-v2" {
|
||||
t.Fatalf("post-reload CN: got %q want cn-v2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertHolder_Reload_FailureRetainsPreviousCert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-v1")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
|
||||
// Corrupt the cert file and attempt reload.
|
||||
if err := os.WriteFile(certPath, []byte("garbage"), 0o600); err != nil {
|
||||
t.Fatalf("corrupt cert: %v", err)
|
||||
}
|
||||
if err := h.Reload(); err == nil {
|
||||
t.Fatal("expected Reload error for corrupt file, got nil")
|
||||
}
|
||||
// Holder should still serve the v1 cert.
|
||||
if got := readCertCN(t, h); got != "cn-v1" {
|
||||
t.Fatalf("post-failed-reload CN: got %q want cn-v1 (reload must not clobber on failure)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertHolder_GetCertificate_Concurrent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-concurrent")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
|
||||
// 64 readers + 1 rotator for 500ms. Race detector catches any unsynchronized
|
||||
// swap of h.cert. Rotator writes fresh files + Reload, readers call
|
||||
// GetCertificate in a tight loop.
|
||||
var wg sync.WaitGroup
|
||||
done := make(chan struct{})
|
||||
const readers = 64
|
||||
for i := 0; i < readers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
default:
|
||||
if _, err := h.GetCertificate(&tls.ClientHelloInfo{}); err != nil {
|
||||
t.Errorf("GetCertificate: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 20; i++ {
|
||||
generateTestCert(t, certPath, keyPath, "cn-concurrent")
|
||||
_ = h.Reload()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
close(done)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestCertHolder_WatchSIGHUP_ReloadsOnSignal(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-before-sighup")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
stop := h.watchSIGHUP(silentLogger())
|
||||
defer stop()
|
||||
|
||||
// Rotate on disk, then fire SIGHUP to our own process and poll for the swap.
|
||||
generateTestCert(t, certPath, keyPath, "cn-after-sighup")
|
||||
if err := syscall.Kill(syscall.Getpid(), syscall.SIGHUP); err != nil {
|
||||
t.Fatalf("SIGHUP: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if readCertCN(t, h) == "cn-after-sighup" {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("watcher did not reload cert within 2s (CN still %q)", readCertCN(t, h))
|
||||
}
|
||||
|
||||
func TestCertHolder_WatchSIGHUP_StopExits(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-stop")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
stop := h.watchSIGHUP(silentLogger())
|
||||
|
||||
// Closing should be synchronous and safe; a subsequent SIGHUP must not
|
||||
// cause a reload (the watcher goroutine is gone).
|
||||
stop()
|
||||
time.Sleep(50 * time.Millisecond) // let goroutine exit
|
||||
|
||||
// After stop, the signal may still be delivered to the process but the
|
||||
// watcher has called signal.Stop so this channel is no longer receiving.
|
||||
// Simply assert that calling stop() twice does not panic — the goroutine
|
||||
// has already exited, so a second close would panic on the `done`
|
||||
// channel; we do NOT call stop twice. Instead verify no regression in
|
||||
// the held cert.
|
||||
if got := readCertCN(t, h); got != "cn-stop" {
|
||||
t.Fatalf("unexpected cert rotation after stop: got %q want cn-stop", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildServerTLSConfig_IsTLS13Only(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-cfg")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
cfg := buildServerTLSConfig(h)
|
||||
if cfg.MinVersion != tls.VersionTLS13 {
|
||||
t.Fatalf("MinVersion: got %#x want %#x (TLS 1.3)", cfg.MinVersion, tls.VersionTLS13)
|
||||
}
|
||||
wantCurves := []tls.CurveID{tls.X25519, tls.CurveP256}
|
||||
if len(cfg.CurvePreferences) != len(wantCurves) {
|
||||
t.Fatalf("CurvePreferences length: got %d want %d", len(cfg.CurvePreferences), len(wantCurves))
|
||||
}
|
||||
for i, c := range cfg.CurvePreferences {
|
||||
if c != wantCurves[i] {
|
||||
t.Fatalf("CurvePreferences[%d]: got %v want %v", i, c, wantCurves[i])
|
||||
}
|
||||
}
|
||||
if cfg.GetCertificate == nil {
|
||||
t.Fatal("GetCertificate: nil (holder not wired; SIGHUP reload would be broken)")
|
||||
}
|
||||
if len(cfg.Certificates) != 0 {
|
||||
t.Fatalf("Certificates: got %d want 0 (static cert would pin the first load and defeat reload)", len(cfg.Certificates))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildServerTLSConfig_Handshake_TLS12Rejected(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-handshake")
|
||||
|
||||
h, err := newCertHolder(certPath, keyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("newCertHolder: %v", err)
|
||||
}
|
||||
serverCfg := buildServerTLSConfig(h)
|
||||
|
||||
ln, err := tls.Listen("tcp", "127.0.0.1:0", serverCfg)
|
||||
if err != nil {
|
||||
t.Fatalf("tls.Listen: %v", err)
|
||||
}
|
||||
defer ln.Close()
|
||||
|
||||
// Server loop: accept and immediately close (we only care about the
|
||||
// handshake outcome).
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Force handshake so the server-side error surfaces.
|
||||
_ = conn.(*tls.Conn).Handshake()
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// TLS 1.3 client — should succeed.
|
||||
clientOK := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
MaxVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
c, err := tls.Dial("tcp", ln.Addr().String(), clientOK)
|
||||
if err != nil {
|
||||
t.Fatalf("TLS 1.3 dial failed (expected success): %v", err)
|
||||
}
|
||||
if c.ConnectionState().Version != tls.VersionTLS13 {
|
||||
t.Fatalf("negotiated version: got %#x want TLS 1.3 (%#x)", c.ConnectionState().Version, tls.VersionTLS13)
|
||||
}
|
||||
c.Close()
|
||||
|
||||
// TLS 1.2 client — must be rejected at handshake.
|
||||
clientOld := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
MaxVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
if _, err := tls.Dial("tcp", ln.Addr().String(), clientOld); err == nil {
|
||||
t.Fatal("TLS 1.2 dial succeeded; HTTPS-everywhere requires server to refuse TLS 1.2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightServerTLS_MissingCertPath(t *testing.T) {
|
||||
err := preflightServerTLS("", "/any/key.pem")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty cert path, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightServerTLS_MissingKeyPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-preflight")
|
||||
err := preflightServerTLS(certPath, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty key path, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightServerTLS_CertFileNotReadable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
if err := os.WriteFile(keyPath, []byte("k"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := preflightServerTLS(filepath.Join(dir, "nope.crt"), keyPath)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unreadable cert path, got nil")
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("expected os.ErrNotExist wrapped in error chain, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightServerTLS_InvalidKeyPair(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
// Pair of valid cert + garbage key — files are readable but the pair
|
||||
// doesn't round-trip tls.LoadX509KeyPair.
|
||||
generateTestCert(t, certPath, keyPath, "cn-bad-pair")
|
||||
if err := os.WriteFile(keyPath, []byte("-----BEGIN EC PRIVATE KEY-----\nBAD\n-----END EC PRIVATE KEY-----\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := preflightServerTLS(certPath, keyPath)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid key pair, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreflightServerTLS_ValidPair_NoError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
generateTestCert(t, certPath, keyPath, "cn-ok")
|
||||
if err := preflightServerTLS(certPath, keyPath); err != nil {
|
||||
t.Fatalf("unexpected error for valid pair: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ A compose file defines **services** (containers), **networks** (how they talk to
|
||||
|
||||
**Overlay files** let you layer changes. Running `docker compose -f base.yml -f overlay.yml up` merges both files. The overlay can add services, change environment variables, or mount extra volumes without editing the base.
|
||||
|
||||
**Port mapping** (`"8443:8443"`) maps host port (left) to container port (right). After startup, `http://localhost:8443` on your machine reaches the certctl server inside its container.
|
||||
**Port mapping** (`"8443:8443"`) maps host port (left) to container port (right). After startup, `https://localhost:8443` on your machine reaches the certctl server inside its container (HTTPS-only as of v2.2; the `certctl-tls-init` init container bootstraps a self-signed cert into `deploy/test/certs/`).
|
||||
|
||||
---
|
||||
|
||||
@@ -91,11 +91,13 @@ Wait about 30 seconds, then verify:
|
||||
docker compose -f deploy/docker-compose.yml ps
|
||||
# All three services should show "Up (healthy)"
|
||||
|
||||
curl http://localhost:8443/health
|
||||
curl --cacert ./deploy/test/certs/ca.crt https://localhost:8443/health
|
||||
# {"status":"healthy"}
|
||||
```
|
||||
|
||||
Open **http://localhost:8443** in your browser. You'll see the onboarding wizard guiding you through: connecting a CA, deploying an agent, and adding your first certificate.
|
||||
The control plane is HTTPS-only as of v2.2. The `certctl-tls-init` init container bootstraps a self-signed cert into `deploy/test/certs/` on first boot; pin it with `--cacert` (as above) or pass `-k` for one-off smoke tests (never in production).
|
||||
|
||||
Open **https://localhost:8443** in your browser. You'll see the onboarding wizard guiding you through: connecting a CA, deploying an agent, and adding your first certificate. Your browser will flag the self-signed cert as untrusted — accept the warning for local evaluation, or import `deploy/test/certs/ca.crt` into your OS trust store to make the warning go away.
|
||||
|
||||
### Service-by-service walkthrough
|
||||
|
||||
@@ -307,8 +309,9 @@ docker compose -f deploy/docker-compose.test.yml up --build
|
||||
Wait for all health checks to pass (about 60 seconds for step-ca's first-run bootstrap). Then:
|
||||
|
||||
```bash
|
||||
# Dashboard with auth enabled
|
||||
open http://localhost:8443
|
||||
# Dashboard with auth enabled (HTTPS-only as of v2.2; browser will warn on the self-signed cert —
|
||||
# accept the warning or trust `deploy/test/certs/ca.crt` in your OS keychain)
|
||||
open https://localhost:8443
|
||||
# API key: test-key-2026
|
||||
|
||||
# NGINX serving a self-signed placeholder
|
||||
|
||||
@@ -4,8 +4,12 @@
|
||||
#
|
||||
# Spins up the full certctl platform with real CA backends for manual QA:
|
||||
#
|
||||
# 0. certctl-tls-init — one-shot init container; writes self-signed
|
||||
# server.crt/.key/ca.crt into ./test/certs (bind
|
||||
# mount, not a named volume — host-readable for
|
||||
# the Go integration test binary)
|
||||
# 1. PostgreSQL 16 — database (clean, no demo data)
|
||||
# 2. certctl-server — control plane API + web dashboard on :8443
|
||||
# 2. certctl-server — control plane API + web dashboard on :8443 (HTTPS)
|
||||
# 3. certctl-agent — polls for work, deploys certs to NGINX
|
||||
# 4. step-ca — private CA (JWK provisioner, auto-bootstraps)
|
||||
# 5. Pebble — ACME test server (simulates Let's Encrypt)
|
||||
@@ -16,15 +20,74 @@
|
||||
# cd deploy
|
||||
# docker compose -f docker-compose.test.yml up --build
|
||||
#
|
||||
# Dashboard: http://localhost:8443
|
||||
# Dashboard: https://localhost:8443 (self-signed — use --cacert test/certs/ca.crt)
|
||||
# API key: test-key-2026
|
||||
# NGINX: https://localhost:8444 (self-signed placeholder until cert deployed)
|
||||
#
|
||||
# Integration tests: `go test -tags integration ./deploy/test/...` picks up
|
||||
# the CA bundle at ./test/certs/ca.crt automatically via CERTCTL_TEST_CA_BUNDLE.
|
||||
#
|
||||
# See docs/test-env.md for the full walkthrough.
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTPS-Everywhere Phase 6 — self-signed TLS bootstrap for the test harness.
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mirrors the production `certctl-tls-init` (see docker-compose.yml §10-43)
|
||||
# but writes into a *host bind mount* (./test/certs) instead of a named
|
||||
# volume. The named-volume approach works fine inside Docker but hides the
|
||||
# CA bundle from the Go integration test binary that runs on the host; the
|
||||
# bind mount exposes /etc/certctl/tls/ca.crt at deploy/test/certs/ca.crt
|
||||
# so `newTestClient()` can load it into an x509.CertPool and validate the
|
||||
# self-signed server cert. Test-only divergence, explicitly documented.
|
||||
#
|
||||
# The generated cert has SAN=DNS:certctl-server,DNS:localhost,IP:127.0.0.1
|
||||
# so both in-cluster traffic (agent → certctl-server:8443) and host traffic
|
||||
# (go test → localhost:8443) validate cleanly. Destroy via
|
||||
# `docker compose -f docker-compose.test.yml down -v` + `rm -rf test/certs`
|
||||
# to force regeneration. Keys written 0600, certs 0644, owned 1000:1000
|
||||
# (the UID the server binary runs as inside its container per Dockerfile:64).
|
||||
certctl-tls-init:
|
||||
image: alpine/openssl:latest
|
||||
container_name: certctl-test-tls-init
|
||||
restart: "no"
|
||||
entrypoint: /bin/sh
|
||||
command:
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
CERT=/etc/certctl/tls/server.crt
|
||||
KEY=/etc/certctl/tls/server.key
|
||||
CA=/etc/certctl/tls/ca.crt
|
||||
if [ -f "$$CERT" ] && [ -f "$$KEY" ] && [ -f "$$CA" ]; then
|
||||
echo "TLS cert already present at $$CERT — skipping generation"
|
||||
else
|
||||
mkdir -p /etc/certctl/tls
|
||||
openssl req -x509 -newkey ed25519 -nodes \
|
||||
-keyout "$$KEY" \
|
||||
-out "$$CERT" \
|
||||
-days 3650 \
|
||||
-subj "/CN=certctl-server" \
|
||||
-addext "subjectAltName=DNS:certctl-server,DNS:localhost,IP:127.0.0.1,IP:::1"
|
||||
cp "$$CERT" "$$CA"
|
||||
echo "Generated self-signed TLS cert for certctl-test-server (ed25519, 3650d, CN=certctl-server)"
|
||||
fi
|
||||
# The test server container runs as root (see `user: "0:0"` below)
|
||||
# because setup-trust.sh needs to update the system trust store, so
|
||||
# the perms here are really about host-side readability — 0644 on
|
||||
# the CA/cert lets `go test` on the host read the bundle without a
|
||||
# chown dance.
|
||||
chown 1000:1000 "$$CERT" "$$KEY" "$$CA" || true
|
||||
chmod 0644 "$$CERT" "$$CA"
|
||||
chmod 0600 "$$KEY"
|
||||
volumes:
|
||||
- ./test/certs:/etc/certctl/tls
|
||||
networks:
|
||||
certctl-test:
|
||||
ipv4_address: 10.30.50.9
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -168,6 +231,12 @@ services:
|
||||
condition: service_started
|
||||
step-ca:
|
||||
condition: service_healthy
|
||||
# HTTPS-Everywhere Phase 6: block server boot until the init container
|
||||
# has written server.crt / server.key / ca.crt into ./test/certs. The
|
||||
# init container runs once and exits 0; service_completed_successfully
|
||||
# makes that a gating dependency rather than a liveness one.
|
||||
certctl-tls-init:
|
||||
condition: service_completed_successfully
|
||||
# Run as root so update-ca-certificates can write to /etc/ssl/certs.
|
||||
# Container isolation provides the security boundary.
|
||||
user: "0:0"
|
||||
@@ -179,6 +248,12 @@ services:
|
||||
# Server
|
||||
CERTCTL_SERVER_HOST: 0.0.0.0
|
||||
CERTCTL_SERVER_PORT: 8443
|
||||
# HTTPS-Everywhere Phase 6: point the server at the init-container-generated
|
||||
# cert/key pair (bind-mounted from ./test/certs). Same paths as production
|
||||
# compose so the server binary code path is identical; only the host-side
|
||||
# storage differs (bind mount vs named volume — see §certctl-tls-init block).
|
||||
CERTCTL_SERVER_TLS_CERT_PATH: /etc/certctl/tls/server.crt
|
||||
CERTCTL_SERVER_TLS_KEY_PATH: /etc/certctl/tls/server.key
|
||||
CERTCTL_LOG_LEVEL: debug
|
||||
|
||||
# Auth — API key required (production-like)
|
||||
@@ -224,12 +299,22 @@ services:
|
||||
- ./test/setup-trust.sh:/app/setup-trust.sh:ro
|
||||
# step-ca data volume (root cert at /certs/root_ca.crt, key at /secrets/provisioner_key)
|
||||
- stepca_data:/stepca-data:ro
|
||||
# HTTPS-Everywhere Phase 6: read-only bind mount of the init-generated
|
||||
# TLS material. The init container writes here; server reads here; the
|
||||
# agent mounts the same host path at the same container path (see below)
|
||||
# so /etc/certctl/tls/ca.crt resolves to the *same* bytes on both sides.
|
||||
- ./test/certs:/etc/certctl/tls:ro
|
||||
networks:
|
||||
certctl-test:
|
||||
ipv4_address: 10.30.50.6
|
||||
healthcheck:
|
||||
# /health requires auth when CERTCTL_AUTH_TYPE=api-key, so include the Bearer token
|
||||
test: ["CMD", "curl", "-f", "-H", "Authorization: Bearer test-key-2026", "http://localhost:8443/health"]
|
||||
# HTTPS-Everywhere Phase 6: healthcheck now speaks TLS with --cacert to
|
||||
# verify the self-signed server cert against the init-generated bundle.
|
||||
# /health requires auth when CERTCTL_AUTH_TYPE=api-key, so include the
|
||||
# Bearer token. curl exits non-zero on both TLS handshake failure and
|
||||
# non-2xx status — either failure keeps depends_on: {condition:
|
||||
# service_healthy} from unblocking the agent, which is what we want.
|
||||
test: ["CMD", "curl", "--cacert", "/etc/certctl/tls/ca.crt", "-f", "-H", "Authorization: Bearer test-key-2026", "https://localhost:8443/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
start_period: 30s
|
||||
@@ -290,7 +375,13 @@ services:
|
||||
certctl-server:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
CERTCTL_SERVER_URL: http://certctl-server:8443
|
||||
# HTTPS-Everywhere Phase 6: agent dials the server over TLS and validates
|
||||
# the self-signed cert against the CA bundle pinned by
|
||||
# CERTCTL_SERVER_CA_BUNDLE_PATH. Same env vars + container paths as
|
||||
# production compose so the agent binary code path (loadCABundle →
|
||||
# x509.CertPool → *tls.Config{RootCAs, MinVersion: TLS13}) is identical.
|
||||
CERTCTL_SERVER_URL: https://certctl-server:8443
|
||||
CERTCTL_SERVER_CA_BUNDLE_PATH: /etc/certctl/tls/ca.crt
|
||||
CERTCTL_API_KEY: test-key-2026
|
||||
CERTCTL_AGENT_NAME: test-agent-01
|
||||
CERTCTL_AGENT_ID: agent-test-01
|
||||
@@ -300,6 +391,10 @@ services:
|
||||
volumes:
|
||||
- agent_keys:/var/lib/certctl/keys
|
||||
- nginx_certs:/nginx-certs
|
||||
# HTTPS-Everywhere Phase 6: same bind mount as the server, same path,
|
||||
# so /etc/certctl/tls/ca.crt resolves to the identical bytes. This is
|
||||
# the only way the CN=certctl-server cert validates on the agent side.
|
||||
- ./test/certs:/etc/certctl/tls:ro
|
||||
networks:
|
||||
certctl-test:
|
||||
ipv4_address: 10.30.50.8
|
||||
|
||||
@@ -1,4 +1,47 @@
|
||||
services:
|
||||
# HTTPS-Everywhere Phase 3 — self-signed TLS bootstrap (init container).
|
||||
# Generates a CN=certctl-server ed25519 cert with the SAN list locked by
|
||||
# milestone §3.6 on first boot; subsequent boots see the cert already
|
||||
# present in the `certs` named volume and no-op out. Server + agent mount
|
||||
# the volume read-only. Destroy via `docker compose down -v` to force
|
||||
# regeneration. This bootstrap is for docker-compose demos and local dev
|
||||
# only; Helm operators supply a Secret / cert-manager Certificate per
|
||||
# docs/tls.md.
|
||||
certctl-tls-init:
|
||||
image: alpine/openssl:latest
|
||||
container_name: certctl-tls-init
|
||||
restart: "no"
|
||||
entrypoint: /bin/sh
|
||||
command:
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
CERT=/etc/certctl/tls/server.crt
|
||||
KEY=/etc/certctl/tls/server.key
|
||||
CA=/etc/certctl/tls/ca.crt
|
||||
if [ -f "$$CERT" ] && [ -f "$$KEY" ] && [ -f "$$CA" ]; then
|
||||
echo "TLS cert already present at $$CERT — skipping generation"
|
||||
else
|
||||
mkdir -p /etc/certctl/tls
|
||||
openssl req -x509 -newkey ed25519 -nodes \
|
||||
-keyout "$$KEY" \
|
||||
-out "$$CERT" \
|
||||
-days 3650 \
|
||||
-subj "/CN=certctl-server" \
|
||||
-addext "subjectAltName=DNS:certctl-server,DNS:localhost,IP:127.0.0.1,IP:::1"
|
||||
cp "$$CERT" "$$CA"
|
||||
echo "Generated self-signed TLS cert for certctl-server (ed25519, 3650d, CN=certctl-server)"
|
||||
fi
|
||||
# certctl binary runs as UID 1000 inside the server container per
|
||||
# Dockerfile:64-65; the cert + key must be readable by that UID.
|
||||
chown 1000:1000 "$$CERT" "$$KEY" "$$CA"
|
||||
chmod 0644 "$$CERT" "$$CA"
|
||||
chmod 0600 "$$KEY"
|
||||
volumes:
|
||||
- certs:/etc/certctl/tls
|
||||
networks:
|
||||
- certctl-network
|
||||
|
||||
# PostgreSQL database
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
@@ -50,10 +93,14 @@ services:
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
certctl-tls-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
CERTCTL_DATABASE_URL: postgres://certctl:${POSTGRES_PASSWORD:-certctl}@postgres:5432/certctl?sslmode=disable
|
||||
CERTCTL_SERVER_HOST: 0.0.0.0
|
||||
CERTCTL_SERVER_PORT: 8443
|
||||
CERTCTL_SERVER_TLS_CERT_PATH: /etc/certctl/tls/server.crt
|
||||
CERTCTL_SERVER_TLS_KEY_PATH: /etc/certctl/tls/server.key
|
||||
CERTCTL_LOG_LEVEL: info
|
||||
CERTCTL_AUTH_TYPE: none
|
||||
CERTCTL_KEYGEN_MODE: server # Demo uses server-side keygen; production should use "agent"
|
||||
@@ -61,10 +108,12 @@ services:
|
||||
CERTCTL_CONFIG_ENCRYPTION_KEY: ${CERTCTL_CONFIG_ENCRYPTION_KEY:-change-me-32-char-encryption-key} # AES-256-GCM for dynamic issuer/target config
|
||||
ports:
|
||||
- "8443:8443"
|
||||
volumes:
|
||||
- certs:/etc/certctl/tls:ro
|
||||
networks:
|
||||
- certctl-network
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8443/health"]
|
||||
test: ["CMD", "curl", "--cacert", "/etc/certctl/tls/ca.crt", "-f", "https://localhost:8443/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -99,13 +148,15 @@ services:
|
||||
certctl-server:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
CERTCTL_SERVER_URL: http://certctl-server:8443
|
||||
CERTCTL_SERVER_URL: https://certctl-server:8443
|
||||
CERTCTL_SERVER_CA_BUNDLE_PATH: /etc/certctl/tls/ca.crt
|
||||
CERTCTL_API_KEY: ${CERTCTL_API_KEY:-change-me-in-production}
|
||||
CERTCTL_AGENT_NAME: docker-agent
|
||||
CERTCTL_LOG_LEVEL: info
|
||||
CERTCTL_DISCOVERY_DIRS: /var/lib/certctl/keys # Agent scans this directory for existing certificates
|
||||
volumes:
|
||||
- agent_keys:/var/lib/certctl/keys
|
||||
- certs:/etc/certctl/tls:ro
|
||||
networks:
|
||||
- certctl-network
|
||||
healthcheck:
|
||||
@@ -134,3 +185,5 @@ volumes:
|
||||
driver: local
|
||||
agent_keys:
|
||||
driver: local
|
||||
certs:
|
||||
driver: local
|
||||
|
||||
@@ -236,10 +236,12 @@ kubectl get svc -l app.kubernetes.io/instance=certctl
|
||||
kubectl get ingress
|
||||
kubectl describe ingress certctl
|
||||
|
||||
# Test API connectivity
|
||||
# Test API connectivity (HTTPS-only as of v2.2)
|
||||
POD=$(kubectl get pods -l app.kubernetes.io/component=server -o jsonpath='{.items[0].metadata.name}')
|
||||
kubectl port-forward $POD 8443:8443 &
|
||||
curl -H "Authorization: Bearer $API_KEY" http://localhost:8443/health
|
||||
# If the chart provisioned a self-signed cert, fetch the CA bundle from the TLS secret first:
|
||||
# kubectl get secret certctl-server-tls -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/certctl-ca.crt
|
||||
curl --cacert /tmp/certctl-ca.crt -H "Authorization: Bearer $API_KEY" https://localhost:8443/health
|
||||
```
|
||||
|
||||
### Step 6: Access the Dashboard
|
||||
@@ -333,9 +335,10 @@ kubectl logs $POD | tail -20
|
||||
# Port forward to API
|
||||
kubectl port-forward svc/certctl-server 8443:8443 &
|
||||
|
||||
# Create a test certificate
|
||||
# Create a test certificate (HTTPS-only as of v2.2 — pin the chart-provisioned CA bundle)
|
||||
# kubectl get secret certctl-server-tls -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/certctl-ca.crt
|
||||
API_KEY="your-api-key"
|
||||
curl -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl --cacert /tmp/certctl-ca.crt -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H "Authorization: Bearer $API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
|
||||
@@ -33,9 +33,11 @@ kubectl get pods -l app.kubernetes.io/instance=certctl
|
||||
# View server logs
|
||||
kubectl logs -l app.kubernetes.io/component=server -f
|
||||
|
||||
# Access the API
|
||||
# Access the API (HTTPS-only as of v2.2; use --cacert or -k depending on your cert provisioning)
|
||||
kubectl port-forward svc/certctl-server 8443:8443 &
|
||||
curl http://localhost:8443/health
|
||||
# If the chart provisioned a self-signed cert, fetch the CA bundle from the secret first:
|
||||
# kubectl get secret certctl-server-tls -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/certctl-ca.crt
|
||||
curl --cacert /tmp/certctl-ca.crt https://localhost:8443/health
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
@@ -4,36 +4,46 @@
|
||||
{{- else if contains "NodePort" .Values.server.service.type }}
|
||||
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
|
||||
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "certctl.fullname" . }}-server)
|
||||
echo http://$NODE_IP:$NODE_PORT
|
||||
echo https://$NODE_IP:$NODE_PORT
|
||||
{{- else if contains "LoadBalancer" .Values.server.service.type }}
|
||||
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "certctl.fullname" . }}-server --template "{.status.loadBalancer.ingress[0].ip}")
|
||||
echo http://$SERVICE_IP:{{ .Values.server.service.port }}
|
||||
echo https://$SERVICE_IP:{{ .Values.server.service.port }}
|
||||
{{- else }}
|
||||
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "certctl.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=server" -o jsonpath="{.items[0].metadata.name}")
|
||||
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
|
||||
echo "Visit http://127.0.0.1:8080 to use your application"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
|
||||
echo "Visit https://127.0.0.1:8443 to use your application"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8443:$CONTAINER_PORT
|
||||
{{- end }}
|
||||
|
||||
2. Get the default API key:
|
||||
2. Talk to the HTTPS-only server from your workstation:
|
||||
# Export the CA bundle that signed the server cert (self-signed or cert-manager-issued)
|
||||
kubectl get secret --namespace {{ .Release.Namespace }} {{ include "certctl.tls.secretName" . }} \
|
||||
-o jsonpath='{.data.ca\.crt}' | base64 --decode > /tmp/certctl-ca.crt
|
||||
# (If ca.crt is empty, fall back to tls.crt — typical when the Secret
|
||||
# was created from a self-signed bootstrap cert without a separate CA.)
|
||||
|
||||
# Adapt the URL below to match the Server URL printed in step 1.
|
||||
curl --cacert /tmp/certctl-ca.crt https://127.0.0.1:8443/health
|
||||
|
||||
3. Get the default API key:
|
||||
kubectl get secret --namespace {{ .Release.Namespace }} {{ include "certctl.fullname" . }}-server -o jsonpath="{.data.api-key}" | base64 --decode; echo
|
||||
|
||||
3. Get PostgreSQL connection details:
|
||||
4. Get PostgreSQL connection details:
|
||||
Host: {{ include "certctl.fullname" . }}-postgres.{{ .Release.Namespace }}.svc.cluster.local
|
||||
Port: 5432
|
||||
Database: {{ .Values.postgresql.auth.database }}
|
||||
Username: {{ .Values.postgresql.auth.username }}
|
||||
Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "certctl.fullname" . }}-postgres -o jsonpath="{.data.password}" | base64 --decode)
|
||||
|
||||
4. Check deployment status:
|
||||
5. Check deployment status:
|
||||
kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }}
|
||||
|
||||
5. View server logs:
|
||||
6. View server logs:
|
||||
kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/name={{ include "certctl.name" . }},app.kubernetes.io/component=server -f
|
||||
|
||||
{{- if .Values.agent.enabled }}
|
||||
|
||||
6. View agent logs:
|
||||
7. View agent logs:
|
||||
kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/name={{ include "certctl.name" . }},app.kubernetes.io/component=agent -f
|
||||
|
||||
{{- end }}
|
||||
@@ -58,11 +68,7 @@ IMPORTANT NOTES FOR PRODUCTION:
|
||||
- Use an external PostgreSQL managed service (AWS RDS, Cloud SQL, etc.)
|
||||
- Set postgresql.enabled=false and configure CERTCTL_DATABASE_URL in values
|
||||
|
||||
5. Enable HTTPS/TLS using an Ingress with certificate management:
|
||||
- Configure cert-manager for automatic TLS certificate renewal
|
||||
- Update ingress values with your domain and certificate issuer
|
||||
|
||||
6. Review security contexts and network policies:
|
||||
5. Review security contexts and network policies:
|
||||
- All containers run as non-root
|
||||
- Implement network policies to restrict traffic between components
|
||||
- Consider pod security policies or security standards for your cluster
|
||||
|
||||
@@ -118,8 +118,54 @@ postgres://{{ .Values.postgresql.auth.username }}:$(POSTGRES_PASSWORD)@{{ includ
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Server URL (for agents)
|
||||
Server URL (for agents). HTTPS-only as of v2.2 — see docs/tls.md.
|
||||
*/}}
|
||||
{{- define "certctl.serverURL" -}}
|
||||
http://{{ include "certctl.fullname" . }}-server:{{ .Values.server.service.port }}
|
||||
https://{{ include "certctl.fullname" . }}-server:{{ .Values.server.service.port }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
TLS Secret name resolver.
|
||||
|
||||
Operator-facing precedence:
|
||||
1. server.tls.existingSecret — operator points at a pre-existing kubernetes.io/tls Secret
|
||||
2. server.tls.certManager.secretName — explicit secret name for the cert-manager Certificate CR
|
||||
3. "<fullname>-tls" — default when cert-manager is enabled but secretName is blank
|
||||
|
||||
Never emits an empty string — that case is already excluded by certctl.tls.required below,
|
||||
which must be invoked by any template that depends on the resolved secret name.
|
||||
*/}}
|
||||
{{- define "certctl.tls.secretName" -}}
|
||||
{{- if .Values.server.tls.existingSecret -}}
|
||||
{{- .Values.server.tls.existingSecret -}}
|
||||
{{- else if .Values.server.tls.certManager.secretName -}}
|
||||
{{- .Values.server.tls.certManager.secretName -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-tls" (include "certctl.fullname" .) -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
TLS configuration gate.
|
||||
|
||||
HTTPS is the only supported listener mode (v2.2+). The server refuses to start
|
||||
without a cert/key pair mounted at server.tls.mountPath, so `helm template` /
|
||||
`helm install` must fail loudly at render-time rather than shipping a broken
|
||||
Deployment that crash-loops with "tls config required".
|
||||
|
||||
Operators MUST configure EXACTLY ONE of:
|
||||
(a) server.tls.existingSecret: <name-of-kubernetes.io/tls-secret>
|
||||
(b) server.tls.certManager.enabled: true (+ issuerRef.name populated)
|
||||
|
||||
Any template that mounts the TLS Secret must call
|
||||
`{{ include "certctl.tls.required" . }}` at the top so this guard runs once
|
||||
per affected resource. No-op when configured correctly.
|
||||
*/}}
|
||||
{{- define "certctl.tls.required" -}}
|
||||
{{- if and (not .Values.server.tls.existingSecret) (not .Values.server.tls.certManager.enabled) -}}
|
||||
{{- fail "\n\ncertctl refuses to start without TLS.\n\nSet EXACTLY ONE of:\n --set server.tls.existingSecret=<your-kubernetes.io/tls-secret-name>\nOR\n --set server.tls.certManager.enabled=true \\\n --set server.tls.certManager.issuerRef.name=<your-issuer-or-clusterissuer>\n\nSee docs/tls.md for the full setup walkthrough, including bootstrap\nguidance for air-gapped clusters without cert-manager.\n" -}}
|
||||
{{- end -}}
|
||||
{{- if and .Values.server.tls.certManager.enabled (not .Values.server.tls.certManager.issuerRef.name) -}}
|
||||
{{- fail "\n\nserver.tls.certManager.enabled=true but server.tls.certManager.issuerRef.name is empty.\n\nSet:\n --set server.tls.certManager.issuerRef.name=<your-issuer-or-clusterissuer>\n\nSee docs/tls.md.\n" -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{{- if .Values.agent.enabled }}
|
||||
{{- include "certctl.tls.required" . }}
|
||||
{{- if eq .Values.agent.kind "DaemonSet" }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
@@ -53,6 +54,8 @@ spec:
|
||||
fieldPath: metadata.name
|
||||
- name: CERTCTL_KEY_DIR
|
||||
value: {{ .Values.agent.keyDir }}
|
||||
- name: CERTCTL_SERVER_CA_BUNDLE_PATH
|
||||
value: "{{ .Values.server.tls.mountPath }}/ca.crt"
|
||||
{{- if .Values.agent.discoveryDirs }}
|
||||
- name: CERTCTL_DISCOVERY_DIRS
|
||||
valueFrom:
|
||||
@@ -70,12 +73,19 @@ spec:
|
||||
mountPath: {{ .Values.agent.keyDir }}
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: server-tls
|
||||
mountPath: {{ .Values.server.tls.mountPath }}
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: agent-keys
|
||||
emptyDir:
|
||||
sizeLimit: 1Gi
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: server-tls
|
||||
secret:
|
||||
secretName: {{ include "certctl.tls.secretName" . }}
|
||||
defaultMode: 0400
|
||||
{{- else if eq .Values.agent.kind "Deployment" }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -135,6 +145,8 @@ spec:
|
||||
{{- end }}
|
||||
- name: CERTCTL_KEY_DIR
|
||||
value: {{ .Values.agent.keyDir }}
|
||||
- name: CERTCTL_SERVER_CA_BUNDLE_PATH
|
||||
value: "{{ .Values.server.tls.mountPath }}/ca.crt"
|
||||
{{- if .Values.agent.discoveryDirs }}
|
||||
- name: CERTCTL_DISCOVERY_DIRS
|
||||
valueFrom:
|
||||
@@ -152,11 +164,18 @@ spec:
|
||||
mountPath: {{ .Values.agent.keyDir }}
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: server-tls
|
||||
mountPath: {{ .Values.server.tls.mountPath }}
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: agent-keys
|
||||
emptyDir:
|
||||
sizeLimit: 1Gi
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: server-tls
|
||||
secret:
|
||||
secretName: {{ include "certctl.tls.secretName" . }}
|
||||
defaultMode: 0400
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,14 +1,24 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
{{- if and .Values.ingress.certManager.enabled (not .Values.ingress.certManager.issuerRef.name) -}}
|
||||
{{- fail "\n\ningress.certManager.enabled=true but ingress.certManager.issuerRef.name is empty.\n\nSet:\n --set ingress.certManager.issuerRef.name=<your-issuer-or-clusterissuer>\n\nThis is separate from server.tls.certManager — it issues the external-facing\nIngress cert, not the in-cluster server TLS cert. See docs/tls.md.\n" -}}
|
||||
{{- end -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "certctl.fullname" . }}
|
||||
labels:
|
||||
{{- include "certctl.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- if .Values.ingress.certManager.enabled }}
|
||||
{{- if eq .Values.ingress.certManager.issuerRef.kind "ClusterIssuer" }}
|
||||
cert-manager.io/cluster-issuer: {{ .Values.ingress.certManager.issuerRef.name | quote }}
|
||||
{{- else }}
|
||||
cert-manager.io/issuer: {{ .Values.ingress.certManager.issuerRef.name | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
@@ -33,7 +43,7 @@ spec:
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "certctl.fullname" . }}-server
|
||||
name: {{ include "certctl.fullname" $ }}-server
|
||||
port:
|
||||
number: {{ $.Values.server.service.port }}
|
||||
{{- end }}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
{{- if .Values.server.tls.certManager.enabled }}
|
||||
{{- include "certctl.tls.required" . }}
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ include "certctl.fullname" . }}-server-tls
|
||||
labels:
|
||||
{{- include "certctl.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
spec:
|
||||
secretName: {{ include "certctl.tls.secretName" . }}
|
||||
commonName: {{ .Values.server.tls.certManager.commonName | quote }}
|
||||
dnsNames:
|
||||
{{- range .Values.server.tls.certManager.dnsNames }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
duration: {{ .Values.server.tls.certManager.duration }}
|
||||
renewBefore: {{ .Values.server.tls.certManager.renewBefore }}
|
||||
usages:
|
||||
- server auth
|
||||
- digital signature
|
||||
- key encipherment
|
||||
privateKey:
|
||||
algorithm: ECDSA
|
||||
size: 256
|
||||
rotationPolicy: Always
|
||||
issuerRef:
|
||||
name: {{ .Values.server.tls.certManager.issuerRef.name | quote }}
|
||||
kind: {{ .Values.server.tls.certManager.issuerRef.kind }}
|
||||
group: {{ .Values.server.tls.certManager.issuerRef.group }}
|
||||
{{- end }}
|
||||
@@ -1,3 +1,4 @@
|
||||
{{- include "certctl.tls.required" . }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -32,7 +33,7 @@ spec:
|
||||
image: {{ include "certctl.serverImage" . }}
|
||||
imagePullPolicy: {{ .Values.server.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
- name: https
|
||||
containerPort: {{ .Values.server.port }}
|
||||
protocol: TCP
|
||||
env:
|
||||
@@ -40,6 +41,10 @@ spec:
|
||||
value: "0.0.0.0"
|
||||
- name: CERTCTL_SERVER_PORT
|
||||
value: "{{ .Values.server.port }}"
|
||||
- name: CERTCTL_SERVER_TLS_CERT_PATH
|
||||
value: "{{ .Values.server.tls.mountPath }}/tls.crt"
|
||||
- name: CERTCTL_SERVER_TLS_KEY_PATH
|
||||
value: "{{ .Values.server.tls.mountPath }}/tls.key"
|
||||
- name: CERTCTL_DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -172,12 +177,19 @@ spec:
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: tls
|
||||
mountPath: {{ .Values.server.tls.mountPath }}
|
||||
readOnly: true
|
||||
{{- if .Values.server.volumeMounts }}
|
||||
{{- toYaml .Values.server.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: tls
|
||||
secret:
|
||||
secretName: {{ include "certctl.tls.secretName" . }}
|
||||
defaultMode: 0400
|
||||
{{- if .Values.server.volumes }}
|
||||
{{- toYaml .Values.server.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -13,8 +13,8 @@ spec:
|
||||
type: {{ .Values.server.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.server.service.port }}
|
||||
targetPort: http
|
||||
targetPort: https
|
||||
protocol: TCP
|
||||
name: http
|
||||
name: https
|
||||
selector:
|
||||
{{- include "certctl.serverSelectorLabels" . | nindent 4 }}
|
||||
|
||||
@@ -48,11 +48,12 @@ server:
|
||||
drop:
|
||||
- ALL
|
||||
|
||||
# Liveness and readiness probes
|
||||
# Liveness and readiness probes (HTTPS-only as of v2.2)
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -61,12 +62,50 @@ server:
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: http
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
|
||||
# TLS configuration — REQUIRED. HTTPS is the only supported mode (v2.2+).
|
||||
# Operator must configure EXACTLY ONE of:
|
||||
# (a) server.tls.existingSecret: <name> # pre-existing kubernetes.io/tls Secret
|
||||
# (b) server.tls.certManager.enabled: true # provision a cert-manager Certificate CR
|
||||
# Refusing to set either makes `helm template` fail with a diagnostic pointing at docs/tls.md.
|
||||
tls:
|
||||
# Name of a pre-existing Secret (type kubernetes.io/tls) holding tls.crt + tls.key (+ optional ca.crt).
|
||||
# Leave empty to fall through to the cert-manager path.
|
||||
existingSecret: ""
|
||||
|
||||
# Mount path for the TLS Secret inside the server + agent containers.
|
||||
mountPath: /etc/certctl/tls
|
||||
|
||||
# cert-manager auto-provisioning. Opt-in (off by default per milestone §3.4).
|
||||
certManager:
|
||||
enabled: false
|
||||
|
||||
# Secret name the cert-manager Certificate CR writes into. Agents and the server
|
||||
# both read from this Secret. If empty, defaults to "<fullname>-tls".
|
||||
secretName: ""
|
||||
|
||||
# Cert-manager issuer reference.
|
||||
issuerRef:
|
||||
name: "" # e.g. "letsencrypt-prod" or "internal-ca"
|
||||
kind: ClusterIssuer # ClusterIssuer or Issuer
|
||||
group: cert-manager.io
|
||||
|
||||
# Subject fields on the issued cert.
|
||||
commonName: "certctl-server"
|
||||
dnsNames:
|
||||
- certctl-server
|
||||
- localhost
|
||||
|
||||
# Certificate lifetime + renewal window.
|
||||
duration: 2160h # 90 days
|
||||
renewBefore: 360h # 15 days
|
||||
|
||||
# Service type (ClusterIP, LoadBalancer, NodePort)
|
||||
service:
|
||||
type: ClusterIP
|
||||
@@ -356,7 +395,16 @@ ingress:
|
||||
className: ""
|
||||
annotations: {}
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
# cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
|
||||
# Optional cert-manager integration for the public-facing Ingress cert.
|
||||
# This is completely independent of server.tls.* — the Ingress terminates
|
||||
# an *additional* TLS hop between the internet and the in-cluster Service.
|
||||
# Leave disabled unless an Ingress is exposing certctl to the outside world.
|
||||
certManager:
|
||||
enabled: false
|
||||
issuerRef:
|
||||
name: "" # e.g. "letsencrypt-prod"
|
||||
kind: ClusterIssuer # ClusterIssuer or Issuer
|
||||
hosts:
|
||||
- host: certctl.local
|
||||
paths:
|
||||
|
||||
@@ -47,11 +47,30 @@ func envOr(key, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
// HTTPS-Everywhere Phase 6: the test harness now dials the server over TLS and
|
||||
// validates the self-signed cert against the init-container-generated CA bundle
|
||||
// bind-mounted at ./test/certs/ca.crt. The defaults assume the compose setup in
|
||||
// deploy/docker-compose.test.yml; override via the usual env vars when pointing
|
||||
// the suite at a different deployment.
|
||||
//
|
||||
// - CERTCTL_TEST_SERVER_URL — must be https:// for the Phase 6 wiring
|
||||
// - CERTCTL_TEST_CA_BUNDLE — PEM bundle; must contain the server's issuing
|
||||
// CA (self-signed in the compose setup, so server.crt doubles as ca.crt)
|
||||
// - CERTCTL_TEST_INSECURE — set to "true" to fall back to
|
||||
// InsecureSkipVerify when the CA bundle path is unavailable (CI smoke or
|
||||
// exploratory runs only — CI-parity runs MUST use the pinned bundle).
|
||||
//
|
||||
// Under no circumstance does the suite silently downgrade to plaintext HTTP:
|
||||
// Phase 5 (#203) pre-flight guards in cmd/server will refuse to start with an
|
||||
// http:// URL anyway, so a misconfiguration fails loud at test-harness startup
|
||||
// rather than flaking mid-suite.
|
||||
var (
|
||||
serverURL = envOr("CERTCTL_TEST_SERVER_URL", "http://localhost:8443")
|
||||
apiKey = envOr("CERTCTL_TEST_API_KEY", "test-key-2026")
|
||||
dbURL = envOr("CERTCTL_TEST_DB_URL", "postgres://certctl:testpass@localhost:5432/certctl?sslmode=disable")
|
||||
nginxTLS = envOr("CERTCTL_TEST_NGINX_TLS", "localhost:8444")
|
||||
serverURL = envOr("CERTCTL_TEST_SERVER_URL", "https://localhost:8443")
|
||||
apiKey = envOr("CERTCTL_TEST_API_KEY", "test-key-2026")
|
||||
dbURL = envOr("CERTCTL_TEST_DB_URL", "postgres://certctl:testpass@localhost:5432/certctl?sslmode=disable")
|
||||
nginxTLS = envOr("CERTCTL_TEST_NGINX_TLS", "localhost:8444")
|
||||
caBundlePath = envOr("CERTCTL_TEST_CA_BUNDLE", "./certs/ca.crt")
|
||||
insecureTLS = strings.EqualFold(os.Getenv("CERTCTL_TEST_INSECURE"), "true")
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -75,16 +94,74 @@ type testClient struct {
|
||||
apiKey string
|
||||
}
|
||||
|
||||
// buildTLSConfig wires up the x509.CertPool with the self-signed CA bundle
|
||||
// emitted by the certctl-tls-init container. Panics via t.Fatal on the happy
|
||||
// path if both CERTCTL_TEST_CA_BUNDLE is unreadable *and* CERTCTL_TEST_INSECURE
|
||||
// is not set — that combination is almost always a misconfigured test harness
|
||||
// and silently downgrading to InsecureSkipVerify would hide real failures.
|
||||
//
|
||||
// MinVersion is pinned to TLS 1.3 so this matches what cmd/server negotiates
|
||||
// by default; a drift there would surface here first.
|
||||
func buildTLSConfig() *tls.Config {
|
||||
cfg := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
}
|
||||
if insecureTLS {
|
||||
// Opt-in smoke-run mode; log but don't fail so operators running
|
||||
// `CERTCTL_TEST_INSECURE=true go test -tags integration ./deploy/test/...`
|
||||
// against an ad-hoc environment still get a green suite when the server
|
||||
// is reachable. CI must not set this.
|
||||
cfg.InsecureSkipVerify = true
|
||||
return cfg
|
||||
}
|
||||
pem, err := os.ReadFile(caBundlePath)
|
||||
if err != nil {
|
||||
// Can't use t.Fatal here (called from package-level helpers); fall
|
||||
// back to a panic so the harness dies loud at the first HTTP call.
|
||||
// Operators see a clear "CA bundle missing" message and fix their
|
||||
// setup instead of chasing a confusing TLS handshake error.
|
||||
panic(fmt.Sprintf("integration test: read CA bundle %q: %v — "+
|
||||
"run `docker compose -f deploy/docker-compose.test.yml up` first, or "+
|
||||
"set CERTCTL_TEST_CA_BUNDLE to a valid PEM path, or "+
|
||||
"set CERTCTL_TEST_INSECURE=true for a smoke run", caBundlePath, err))
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
panic(fmt.Sprintf("integration test: no PEM certificates parsed from %q", caBundlePath))
|
||||
}
|
||||
cfg.RootCAs = pool
|
||||
return cfg
|
||||
}
|
||||
|
||||
// newTestClient builds a Bearer-authenticated HTTPS client pinned to the
|
||||
// init-container CA. Every phase uses this for REST calls.
|
||||
func newTestClient() *testClient {
|
||||
return &testClient{
|
||||
http: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: buildTLSConfig(),
|
||||
},
|
||||
},
|
||||
baseURL: serverURL,
|
||||
apiKey: apiKey,
|
||||
}
|
||||
}
|
||||
|
||||
// newUnauthHTTPClient returns an *http.Client with the same TLS configuration
|
||||
// but no Bearer token. Used for the Phase 7 RFC 5280 CRL / RFC 8615
|
||||
// `/.well-known/pki/*` probes — those endpoints must be reachable by
|
||||
// *unauthenticated* relying parties per M-006, so we explicitly omit the
|
||||
// Authorization header to prove it.
|
||||
func newUnauthHTTPClient() *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: buildTLSConfig(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *testClient) do(method, path string, body io.Reader) (*http.Response, error) {
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequest(method, url, body)
|
||||
@@ -724,11 +801,18 @@ func TestIntegrationSuite(t *testing.T) {
|
||||
}
|
||||
|
||||
// Check DER CRL served unauthenticated under /.well-known/pki/ per
|
||||
// RFC 5280 §5 + RFC 8615 (M-006). Use a plain http.Get — no Bearer
|
||||
// token — to prove the endpoint is reachable by relying parties that
|
||||
// have no certctl API credentials.
|
||||
// RFC 5280 §5 + RFC 8615 (M-006). Use newUnauthHTTPClient() — no
|
||||
// Bearer token — to prove the endpoint is reachable by relying
|
||||
// parties that have no certctl API credentials. Post HTTPS-Everywhere
|
||||
// (M-007, Phase 6) the client still speaks TLS 1.3 against the pinned
|
||||
// CA bundle from ./certs/ca.crt; we just skip the Authorization header
|
||||
// to exercise the unauthenticated RFC 5280 / RFC 8615 relying-party
|
||||
// path. Switching from the stdlib http.DefaultClient (plaintext OK,
|
||||
// system trust store only) to the helper keeps the no-auth semantic
|
||||
// while preventing silent plaintext downgrade — the whole point of
|
||||
// this milestone.
|
||||
t.Run("CRL_DER_Unauthenticated", func(t *testing.T) {
|
||||
resp, err := http.Get(serverURL + "/.well-known/pki/crl/iss-local")
|
||||
resp, err := newUnauthHTTPClient().Get(serverURL + "/.well-known/pki/crl/iss-local")
|
||||
if err != nil {
|
||||
t.Fatalf("GET DER CRL: %v", err)
|
||||
}
|
||||
@@ -1141,4 +1225,243 @@ func TestIntegrationSuite(t *testing.T) {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Phase 13: I-005 Phase 1 Red — Notification Retry + Dead Letter Queue (E2E)
|
||||
//
|
||||
// Pins the full retry-loop contract end-to-end. Phase 2 Green must turn
|
||||
// every subtest Green with a single coherent change set (migration 000016
|
||||
// live, scheduler notificationRetryLoop wired as the 11th loop bumping
|
||||
// the total from 10 → 11, service RetryFailedNotifications + MarkAsDead +
|
||||
// RequeueNotification implemented, handler POST
|
||||
// /api/v1/notifications/{id}/requeue routed, list handler parsing the
|
||||
// status query param).
|
||||
//
|
||||
// Subtests:
|
||||
//
|
||||
// 1. MarkAsDead_OnMaxAttempts — a notification seeded at retry_count=4
|
||||
// (one failure shy of the max_attempts=5 gate) with next_retry_at in
|
||||
// the past is promoted to status='dead' on the first retry-loop
|
||||
// tick. The pre-increment arithmetic `retry_count + 1 = 5 =
|
||||
// max_attempts` triggers MarkAsDead instead of scheduling another
|
||||
// retry.
|
||||
//
|
||||
// 2. Requeue_FlipsDeadToPending — POST
|
||||
// /api/v1/notifications/{id}/requeue on a dead row flips status back
|
||||
// to 'pending', resets retry_count to 0, and clears next_retry_at
|
||||
// so the existing ProcessPendingNotifications loop (not the retry
|
||||
// sweep) picks it up on its next tick.
|
||||
//
|
||||
// 3. ListFilter_StatusDead — GET /api/v1/notifications?status=dead
|
||||
// returns only rows in status='dead' so the UI's Dead Letter tab
|
||||
// (web/src/pages/NotificationsPage.test.tsx subtest #1) can isolate
|
||||
// them without client-side filtering.
|
||||
//
|
||||
// Red behavior at HEAD (what Phase 2 Green must flip):
|
||||
//
|
||||
// * Schema: the INSERTs reference retry_count, next_retry_at,
|
||||
// last_error. Migration 000016 is already written (file (a) of
|
||||
// Phase 1 Red) but until it is applied the INSERTs fail with
|
||||
// "column does not exist" — schema-level Red halt.
|
||||
//
|
||||
// * Subtest 1: no retry loop exists at HEAD. The seeded row stays at
|
||||
// status='failed' retry_count=4 forever. The 4-minute waitFor
|
||||
// therefore times out.
|
||||
//
|
||||
// * Subtest 2: /notifications/{id}/requeue is not routed at HEAD
|
||||
// (internal/api/handler/notifications.go registers only list / get /
|
||||
// mark-read). The POST returns 404.
|
||||
//
|
||||
// * Subtest 3: the list handler does not parse the status query param
|
||||
// at HEAD. The response includes rows of every status, so the
|
||||
// "leaked non-dead row" assertion fires.
|
||||
// -----------------------------------------------------------------------
|
||||
t.Run("Phase13_NotificationRetryDLQ", func(t *testing.T) {
|
||||
// Unreachable endpoint so every webhook delivery attempt fails
|
||||
// deterministically — port 1 is never bound. Pinning retry_count=4
|
||||
// + a guaranteed-failing channel is what turns the seeded row into
|
||||
// 'dead' on the very next scheduler tick (one delivery attempt,
|
||||
// retry_count 4→5, crosses max_attempts=5 → MarkAsDead).
|
||||
const blackHole = "http://127.0.0.1:1/i005-red-black-hole"
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Subtest 1: failed → dead transition after one retry-loop tick
|
||||
// ---------------------------------------------------------------
|
||||
t.Run("MarkAsDead_OnMaxAttempts", func(t *testing.T) {
|
||||
id := fmt.Sprintf("notif-i005-dead-%d", time.Now().UnixNano())
|
||||
|
||||
// retry_count=4 + next attempt = 5 = max_attempts → MarkAsDead.
|
||||
// next_retry_at is backdated so the row is immediately eligible
|
||||
// for the retry sweep rather than having to wait for its own
|
||||
// backoff to elapse.
|
||||
past := time.Now().Add(-30 * time.Second).UTC()
|
||||
db.Exec(t, `
|
||||
INSERT INTO notification_events
|
||||
(id, type, channel, recipient, message, status,
|
||||
retry_count, next_retry_at, last_error)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
`,
|
||||
id, "ExpirationWarning", "Webhook", blackHole,
|
||||
"I-005 integration: DLQ promotion on max_attempts",
|
||||
"failed", 4, past, "transient webhook 500",
|
||||
)
|
||||
|
||||
// Give the retry sweep up to 4m to tick at least once (default
|
||||
// 2m interval + seed/sweep/notifier slop). On success the row
|
||||
// carries status='dead' and retry_count has advanced to 5.
|
||||
waitFor(t, "notification transitions to dead", 4*time.Minute, 5*time.Second,
|
||||
func() (bool, error) {
|
||||
var status string
|
||||
var retry int
|
||||
err := db.db.QueryRow(
|
||||
"SELECT status, retry_count FROM notification_events WHERE id = $1",
|
||||
id,
|
||||
).Scan(&status, &retry)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return strings.EqualFold(status, "dead") && retry >= 5, nil
|
||||
})
|
||||
|
||||
// The dead-letter tab is only useful if operators can see why
|
||||
// the row died. MarkAsDead must preserve the most recent
|
||||
// failure string in last_error rather than nil'ing it.
|
||||
var lastErr sql.NullString
|
||||
if err := db.db.QueryRow(
|
||||
"SELECT last_error FROM notification_events WHERE id = $1", id,
|
||||
).Scan(&lastErr); err != nil {
|
||||
t.Fatalf("read last_error: %v", err)
|
||||
}
|
||||
if !lastErr.Valid || lastErr.String == "" {
|
||||
t.Errorf("dead notification %s has empty last_error — "+
|
||||
"retry loop must preserve the most recent failure", id)
|
||||
}
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Subtest 2: dead → pending via manual Requeue endpoint
|
||||
// ---------------------------------------------------------------
|
||||
t.Run("Requeue_FlipsDeadToPending", func(t *testing.T) {
|
||||
id := fmt.Sprintf("notif-i005-requeue-%d", time.Now().UnixNano())
|
||||
|
||||
// Seed directly at status='dead' rather than waiting for a
|
||||
// scheduler tick — this subtest isolates the requeue handler,
|
||||
// not the retry loop (subtest 1 already pins that).
|
||||
past := time.Now().Add(-10 * time.Minute).UTC()
|
||||
db.Exec(t, `
|
||||
INSERT INTO notification_events
|
||||
(id, type, channel, recipient, message, status,
|
||||
retry_count, next_retry_at, last_error)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
`,
|
||||
id, "ExpirationWarning", "Webhook", blackHole,
|
||||
"I-005 integration: manual requeue",
|
||||
"dead", 5, past, "max attempts reached",
|
||||
)
|
||||
|
||||
resp, err := c.Post("/api/v1/notifications/"+id+"/requeue", "")
|
||||
if err != nil {
|
||||
t.Fatalf("POST requeue: %v", err)
|
||||
}
|
||||
body := readBody(resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("requeue status %d, want 200 (body: %s)",
|
||||
resp.StatusCode, body)
|
||||
}
|
||||
// Phase 2 Green handler responds with {"status":"requeued"}
|
||||
// to mirror MarkAsRead's {"status":"marked_as_read"} envelope.
|
||||
if !strings.Contains(body, "requeued") {
|
||||
t.Errorf("requeue body missing 'requeued' marker: %s", body)
|
||||
}
|
||||
|
||||
// DB must reflect the full flip: pending status, reset counter,
|
||||
// cleared next_retry_at. Clearing next_retry_at is what moves
|
||||
// the row out of the retry-sweep partial index and back under
|
||||
// ProcessPendingNotifications.
|
||||
var status string
|
||||
var retry int
|
||||
var nextRetry sql.NullTime
|
||||
if err := db.db.QueryRow(`
|
||||
SELECT status, retry_count, next_retry_at
|
||||
FROM notification_events WHERE id = $1
|
||||
`, id).Scan(&status, &retry, &nextRetry); err != nil {
|
||||
t.Fatalf("read requeued row: %v", err)
|
||||
}
|
||||
if !strings.EqualFold(status, "pending") {
|
||||
t.Errorf("after requeue: status=%q, want 'pending'", status)
|
||||
}
|
||||
if retry != 0 {
|
||||
t.Errorf("after requeue: retry_count=%d, want 0", retry)
|
||||
}
|
||||
if nextRetry.Valid {
|
||||
t.Errorf("after requeue: next_retry_at=%v, want NULL",
|
||||
nextRetry.Time)
|
||||
}
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Subtest 3: GET /notifications?status=dead isolates DLQ rows
|
||||
// ---------------------------------------------------------------
|
||||
t.Run("ListFilter_StatusDead", func(t *testing.T) {
|
||||
suffix := fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
deadID := "notif-i005-filter-dead-" + suffix
|
||||
pendingID := "notif-i005-filter-pending-" + suffix
|
||||
|
||||
// One row at each end of the lifecycle so we can prove the
|
||||
// filter both matches and excludes.
|
||||
db.Exec(t, `
|
||||
INSERT INTO notification_events
|
||||
(id, type, channel, recipient, message, status, retry_count)
|
||||
VALUES ($1, 'ExpirationWarning', 'Webhook', $2,
|
||||
'I-005 filter test: dead row', 'dead', 5)
|
||||
`, deadID, blackHole)
|
||||
db.Exec(t, `
|
||||
INSERT INTO notification_events
|
||||
(id, type, channel, recipient, message, status, retry_count)
|
||||
VALUES ($1, 'ExpirationWarning', 'Webhook', $2,
|
||||
'I-005 filter test: pending row', 'pending', 0)
|
||||
`, pendingID, blackHole)
|
||||
|
||||
// per_page large enough to rule out pagination artifacts as
|
||||
// the reason a seeded row might be missing from the response.
|
||||
resp, err := c.Get("/api/v1/notifications?status=dead&per_page=500")
|
||||
if err != nil {
|
||||
t.Fatalf("GET notifications?status=dead: %v", err)
|
||||
}
|
||||
var pr pagedResponse
|
||||
if err := decodeJSON(resp, &pr); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
|
||||
type row struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
var rows []row
|
||||
if err := json.Unmarshal(pr.Data, &rows); err != nil {
|
||||
t.Fatalf("unmarshal rows: %v", err)
|
||||
}
|
||||
|
||||
var sawDead, sawPending bool
|
||||
for _, r := range rows {
|
||||
if r.ID == deadID {
|
||||
sawDead = true
|
||||
}
|
||||
if r.ID == pendingID {
|
||||
sawPending = true
|
||||
}
|
||||
if !strings.EqualFold(r.Status, "dead") {
|
||||
t.Errorf("status=dead filter leaked non-dead row: "+
|
||||
"id=%s status=%s", r.ID, r.Status)
|
||||
}
|
||||
}
|
||||
if !sawDead {
|
||||
t.Errorf("status=dead filter missed seeded dead row %s", deadID)
|
||||
}
|
||||
if sawPending {
|
||||
t.Errorf("status=dead filter leaked seeded pending row %s",
|
||||
pendingID)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
+53
-9
@@ -19,16 +19,29 @@
|
||||
//
|
||||
// Environment overrides:
|
||||
//
|
||||
// CERTCTL_QA_SERVER_URL (default: http://localhost:8443)
|
||||
// CERTCTL_QA_API_KEY (default: change-me-in-production)
|
||||
// CERTCTL_QA_DB_URL (default: postgres://certctl:certctl@localhost:5432/certctl?sslmode=disable)
|
||||
// CERTCTL_QA_REPO_DIR (default: ../.. — the certctl repo root)
|
||||
// CERTCTL_QA_SERVER_URL (default: https://localhost:8443)
|
||||
// CERTCTL_QA_API_KEY (default: change-me-in-production)
|
||||
// CERTCTL_QA_DB_URL (default: postgres://certctl:certctl@localhost:5432/certctl?sslmode=disable)
|
||||
// CERTCTL_QA_REPO_DIR (default: ../.. — the certctl repo root)
|
||||
// CERTCTL_QA_CA_BUNDLE (default: ./certs/ca.crt — the demo stack's init container writes here)
|
||||
// CERTCTL_QA_INSECURE (default: false — set to "true" to skip TLS verify, e.g. before the init container finishes)
|
||||
//
|
||||
// TLS note (HTTPS-Everywhere M-007, Phase 6): the demo compose stack now
|
||||
// listens on https://localhost:8443 with a self-signed cert written by the
|
||||
// tls-init container. This suite pins the issuing CA via
|
||||
// CERTCTL_QA_CA_BUNDLE so cert rotation or a tampered proxy fails the
|
||||
// handshake instead of being silently trusted. CERTCTL_QA_INSECURE="true"
|
||||
// is an explicit opt-out for bootstrap scenarios — there is no silent
|
||||
// plaintext downgrade, matching the server-side pre-flight guard added in
|
||||
// Phase 5 (task #203).
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -50,10 +63,12 @@ func qaEnv(key, fallback string) string {
|
||||
}
|
||||
|
||||
var (
|
||||
qaServerURL = qaEnv("CERTCTL_QA_SERVER_URL", "http://localhost:8443")
|
||||
qaAPIKey = qaEnv("CERTCTL_QA_API_KEY", "change-me-in-production")
|
||||
qaDBURL = qaEnv("CERTCTL_QA_DB_URL", "postgres://certctl:certctl@localhost:5432/certctl?sslmode=disable")
|
||||
qaRepoDir = qaEnv("CERTCTL_QA_REPO_DIR", filepath.Join("..", ".."))
|
||||
qaServerURL = qaEnv("CERTCTL_QA_SERVER_URL", "https://localhost:8443")
|
||||
qaAPIKey = qaEnv("CERTCTL_QA_API_KEY", "change-me-in-production")
|
||||
qaDBURL = qaEnv("CERTCTL_QA_DB_URL", "postgres://certctl:certctl@localhost:5432/certctl?sslmode=disable")
|
||||
qaRepoDir = qaEnv("CERTCTL_QA_REPO_DIR", filepath.Join("..", ".."))
|
||||
qaCABundlePath = qaEnv("CERTCTL_QA_CA_BUNDLE", "./certs/ca.crt")
|
||||
qaInsecure = strings.EqualFold(os.Getenv("CERTCTL_QA_INSECURE"), "true")
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -66,9 +81,38 @@ type qaClient struct {
|
||||
apiKey string
|
||||
}
|
||||
|
||||
// buildQATLSConfig returns the *tls.Config used by every qaClient. TLS 1.3
|
||||
// minimum matches the server-side config pinned in Phase 2 (cmd/server).
|
||||
// When CERTCTL_QA_INSECURE=true we skip verification entirely — useful
|
||||
// when running against a compose stack where the tls-init container hasn't
|
||||
// written ca.crt yet, or when pointing at a dev server with a rotated cert.
|
||||
// Otherwise we pin CERTCTL_QA_CA_BUNDLE and panic on read/parse failure
|
||||
// rather than silently downgrading to the system trust store (which would
|
||||
// mask a missing init container).
|
||||
func buildQATLSConfig() *tls.Config {
|
||||
cfg := &tls.Config{MinVersion: tls.VersionTLS13}
|
||||
if qaInsecure {
|
||||
cfg.InsecureSkipVerify = true
|
||||
return cfg
|
||||
}
|
||||
pem, err := os.ReadFile(qaCABundlePath)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("qa test: read CA bundle %q: %v — set CERTCTL_QA_CA_BUNDLE or CERTCTL_QA_INSECURE=true", qaCABundlePath, err))
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
panic(fmt.Sprintf("qa test: no PEM certificates parsed from %q", qaCABundlePath))
|
||||
}
|
||||
cfg.RootCAs = pool
|
||||
return cfg
|
||||
}
|
||||
|
||||
func newQAClient() *qaClient {
|
||||
return &qaClient{
|
||||
http: &http.Client{Timeout: 30 * time.Second},
|
||||
http: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{TLSClientConfig: buildQATLSConfig()},
|
||||
},
|
||||
baseURL: qaServerURL,
|
||||
apiKey: qaAPIKey,
|
||||
}
|
||||
|
||||
+30
-4
@@ -1,5 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# DEPRECATED — prefer `go test -tags integration ./deploy/test/...`
|
||||
# =============================================================================
|
||||
#
|
||||
# This bash harness predates the Go integration test suite in
|
||||
# deploy/test/integration_test.go (build tag `integration`, 34 subtests across
|
||||
# 13 phases — health, agent heartbeat, Local CA issuance, ACME, step-ca, EST,
|
||||
# S/MIME, discovery, network scan, revocation + CRL, deployment verification).
|
||||
# The Go suite uses crypto/x509, crypto/tls, and database/sql to parse certs,
|
||||
# probe TLS, and talk to PostgreSQL directly — no openssl text-scraping or
|
||||
# brittle curl pipelines. It is the authoritative integration test surface as
|
||||
# of milestone M-007 (HTTPS Everywhere, Phase 6), where the test compose
|
||||
# stack wires the server on https://localhost:8443 behind a pinned CA bundle
|
||||
# at ./certs/ca.crt.
|
||||
#
|
||||
# Run the Go suite:
|
||||
# (cd deploy && docker compose -f docker-compose.test.yml up -d --build)
|
||||
# go test -tags integration -v -count=1 ./deploy/test/...
|
||||
#
|
||||
# Keep this bash script around because:
|
||||
# * It is cited in docs/test-env.md and muscle-memory for contributors.
|
||||
# * It exercises the CLI / curl path end-to-end (a different failure mode
|
||||
# than the Go HTTP client path).
|
||||
# But any NEW integration coverage goes in integration_test.go — not here.
|
||||
#
|
||||
# =============================================================================
|
||||
# certctl End-to-End Test Script
|
||||
# =============================================================================
|
||||
#
|
||||
@@ -32,10 +57,11 @@ set -euo pipefail
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
COMPOSE_FILE="docker-compose.test.yml"
|
||||
API_URL="http://localhost:8443"
|
||||
API_URL="https://localhost:8443"
|
||||
API_KEY="test-key-2026"
|
||||
NGINX_TLS="localhost:8444"
|
||||
AUTH_HEADER="Authorization: Bearer ${API_KEY}"
|
||||
CACERT="./certs/ca.crt"
|
||||
|
||||
# Flags
|
||||
BUILD=true
|
||||
@@ -91,7 +117,7 @@ header() {
|
||||
# API helper: GET endpoint, return JSON body. Exits 1 on HTTP error.
|
||||
api_get() {
|
||||
local path="$1"
|
||||
curl -sf -H "${AUTH_HEADER}" "${API_URL}${path}" 2>/dev/null
|
||||
curl -sf --cacert "${CACERT}" -H "${AUTH_HEADER}" "${API_URL}${path}" 2>/dev/null
|
||||
}
|
||||
|
||||
# API helper: POST with optional JSON body
|
||||
@@ -99,10 +125,10 @@ api_post() {
|
||||
local path="$1"
|
||||
local body="${2:-}"
|
||||
if [ -n "$body" ]; then
|
||||
curl -sf -X POST -H "${AUTH_HEADER}" -H "Content-Type: application/json" \
|
||||
curl -sf --cacert "${CACERT}" -X POST -H "${AUTH_HEADER}" -H "Content-Type: application/json" \
|
||||
-d "$body" "${API_URL}${path}" 2>/dev/null
|
||||
else
|
||||
curl -sf -X POST -H "${AUTH_HEADER}" "${API_URL}${path}" 2>/dev/null
|
||||
curl -sf --cacert "${CACERT}" -X POST -H "${AUTH_HEADER}" "${API_URL}${path}" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
+58
-17
@@ -61,7 +61,7 @@ flowchart TB
|
||||
API["REST API\n(Go net/http, :8443)"]
|
||||
SVC["Service Layer"]
|
||||
REPO["Repository Layer\n(database/sql + lib/pq)"]
|
||||
SCHED["Background Scheduler\n7 loops"]
|
||||
SCHED["Background Scheduler\n8 always-on + 4 optional loops"]
|
||||
DASH["Web Dashboard\n(React SPA)"]
|
||||
end
|
||||
|
||||
@@ -139,6 +139,16 @@ The agent runs two background loops: a heartbeat (every 60 seconds) to signal it
|
||||
|
||||
**Agent groups (M11b):** Dynamic device grouping allows organizing agents by metadata criteria. Agent groups can match by OS, architecture, IP CIDR, and version. Groups support both dynamic matching (agents automatically join when criteria match) and manual membership (explicit include/exclude). Renewal policies can be scoped to agent groups via the `agent_group_id` foreign key. The GUI provides full CRUD management for agent groups with visual match criteria badges.
|
||||
|
||||
**Agent soft-retirement (I-004):** `DELETE /api/v1/agents/{id}` is a soft-delete surface — the row is never removed. Retirement stamps `agents.retired_at` (TIMESTAMPTZ) and `agents.retired_reason` (TEXT) and flips the operational status to `Offline`. Default listings (`GET /api/v1/agents`, the dashboard stats counter, and the stale-offline sweeper) filter retired rows out via `AgentRepository.ListActive`; retired rows are surfaced only through the opt-in `GET /api/v1/agents/retired` view. The endpoint follows a preflight → block → escape-hatch contract:
|
||||
|
||||
- **Clean retire** (no active dependencies) — `200 OK` with `RetireAgentResponse` (`cascade=false`, zero counts).
|
||||
- **Blocked by active dependencies** — `409 Conflict` with `BlockedByDependenciesResponse`. The three counts (`active_targets`, `active_certificates`, `pending_jobs`) tell the operator exactly which rows would be orphaned. The schema diverges from `ErrorResponse` because downstream dashboards parse the stable three-key shape.
|
||||
- **Force cascade** — `DELETE /api/v1/agents/{id}?force=true&reason=...`. `reason` is required (400 otherwise). Transactionally soft-retires downstream `deployment_targets`, cancels pending jobs, and soft-retires the agent, emitting an `agent_retirement_cascaded` audit event with actor + reason + per-bucket counts.
|
||||
- **Idempotent re-retire** — a retire attempt against an already-retired agent returns `204 No Content` with an empty body (no second audit event, no response shape — callers that POST again on a retry get a clean no-op).
|
||||
- **Sentinel refusal** — the four sentinel agent IDs (`server-scanner`, `cloud-aws-sm`, `cloud-azure-kv`, `cloud-gcp-sm`) back non-agent discovery subsystems (the network scanner and the three cloud secret-manager sources). They are refused unconditionally — even with `force=true` — via `ErrAgentIsSentinel` → `403 Forbidden`. The ID list lives in `internal/domain/connector.go` (`SentinelAgentIDs`) so handler, repository, and scheduler code can filter them without importing `service`.
|
||||
|
||||
Retired agents receive `410 Gone` on subsequent heartbeats (`service.ErrAgentRetired`). `cmd/agent` treats 410 as a terminal signal and exits cleanly so retired agents stop phoning home. Migration `000015` flipped `deployment_targets.agent_id` from `ON DELETE CASCADE` to `ON DELETE RESTRICT`, making the old hard-delete path a schema error and forcing all retirement through this contract.
|
||||
|
||||
### Web Dashboard
|
||||
|
||||
The web dashboard is the primary operational interface for certctl. It is built with Vite + React + TypeScript and uses TanStack Query for server state management (caching, background refetching, optimistic updates).
|
||||
@@ -275,6 +285,9 @@ erDiagram
|
||||
text channel
|
||||
text recipient
|
||||
text status
|
||||
int retry_count
|
||||
timestamptz next_retry_at
|
||||
text last_error
|
||||
}
|
||||
certificate_profiles {
|
||||
text id PK
|
||||
@@ -473,40 +486,55 @@ For compliance events requiring fleet-wide revocation (key compromise, CA distru
|
||||
|
||||
### 4. Automatic Renewal
|
||||
|
||||
The control plane runs a scheduler with seven background loops:
|
||||
The control plane runs a scheduler with 8 always-on loops plus up to 4 optional loops (enabled by configuration). `internal/scheduler/scheduler.go:262-265` is the authoritative count.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph "Scheduler (Background Goroutines)"
|
||||
R["Renewal Checker\n⏱ every 1h"]
|
||||
J["Job Processor\n⏱ every 30s"]
|
||||
JR["Job Retry\n⏱ every 5m"]
|
||||
JT["Job Timeout\n⏱ every 10m"]
|
||||
H["Agent Health\n⏱ every 2m"]
|
||||
N["Notification Processor\n⏱ every 1m"]
|
||||
NR["Notification Retry\n⏱ every 2m"]
|
||||
SL["Short-Lived Expiry\n⏱ every 30s"]
|
||||
NS["Network Scanner\n⏱ every 6h"]
|
||||
DG["Certificate Digest\n⏱ every 24h"]
|
||||
HC["Endpoint Health\n⏱ every 60s"]
|
||||
CD["Cloud Discovery\n⏱ every 6h"]
|
||||
end
|
||||
|
||||
R -->|"Find expiring certs\nCreate renewal jobs"| DB[("PostgreSQL")]
|
||||
J -->|"Process pending jobs\nCoordinate issuance"| DB
|
||||
JR -->|"Retry Failed jobs\nFailed→Pending"| DB
|
||||
JT -->|"Reap stalled AwaitingCSR / AwaitingApproval jobs"| DB
|
||||
H -->|"Check heartbeat staleness\nMark agents offline"| DB
|
||||
N -->|"Send pending notifications\nEmail / Webhook / Slack"| DB
|
||||
NR -->|"Retry failed notifications\n2^n-min backoff, DLQ after 5 attempts"| DB
|
||||
SL -->|"Expire short-lived certs\nMark as Expired"| DB
|
||||
NS -->|"Probe TLS endpoints\nStore discovered certs"| DB
|
||||
DG -->|"Generate & send HTML digest\nEmail to recipients"| DB
|
||||
HC -->|"Probe deployed TLS endpoints\nState machine + mismatch"| DB
|
||||
CD -->|"AWS SM / Azure KV / GCP SM\nFeed discovery pipeline"| DB
|
||||
```
|
||||
|
||||
| Loop | Interval | Timeout | Purpose |
|
||||
|------|----------|---------|---------|
|
||||
| Renewal checker | 1 hour | 5 minutes | Finds certificates approaching expiry, creates renewal jobs |
|
||||
| Job processor | 30 seconds | 2 minutes | Processes pending jobs (issuance, renewal, deployment) |
|
||||
| Agent health check | 2 minutes | 1 minute | Marks agents as offline if heartbeat is stale |
|
||||
| Notification processor | 1 minute | 1 minute | Sends pending notifications via configured channels |
|
||||
| Short-lived expiry | 30 seconds | 30 seconds | Marks expired short-lived certificates (profile TTL < 1 hour) |
|
||||
| Network scanner | 6 hours | 30 minutes | Probes TLS endpoints on configured CIDR ranges, stores discovered certs (M21, opt-in via `CERTCTL_NETWORK_SCAN_ENABLED`). CIDR size validated at API level — max /20 (4096 IPs) per range. |
|
||||
| Certificate digest | 24 hours | 5 minutes | Generates HTML email with certificate stats, expiration timeline, job health, agent count. Does NOT run on startup — waits for first scheduled tick. Configurable interval and recipients via `CERTCTL_DIGEST_INTERVAL` and `CERTCTL_DIGEST_RECIPIENTS`. Falls back to certificate owner emails if no explicit recipients configured. |
|
||||
| Loop | Interval | Always-on? | Purpose |
|
||||
|------|----------|------------|---------|
|
||||
| Renewal checker | 1 hour | Yes | Finds certificates approaching expiry (threshold-based or ARI-directed), creates renewal jobs |
|
||||
| Job processor | 30 seconds | Yes | Processes pending jobs (issuance, renewal, deployment) |
|
||||
| Job retry | 5 minutes (`CERTCTL_SCHEDULER_RETRY_INTERVAL`) | Yes | Transitions `Failed` jobs back to `Pending` for re-dispatch (I-001) |
|
||||
| Job timeout | 10 minutes (`CERTCTL_JOB_TIMEOUT_INTERVAL`) | Yes | Reaps `AwaitingCSR` jobs older than 24h and `AwaitingApproval` jobs older than 7d to `Failed`, feeding the retry loop (I-003) |
|
||||
| Agent health check | 2 minutes | Yes | Marks agents as offline if heartbeat is stale |
|
||||
| Notification processor | 1 minute | Yes | Sends pending notifications via configured channels |
|
||||
| Notification retry | 2 minutes (`CERTCTL_NOTIFICATION_RETRY_INTERVAL`) | Yes | Re-dispatches `Failed` notifications whose `next_retry_at` has elapsed; exponential backoff (2^n minutes, capped at 1h), 5-attempt budget, terminal `dead` status after exhaustion (I-005) |
|
||||
| Short-lived expiry | 30 seconds | Yes | Marks expired short-lived certificates (profile TTL < 1 hour) |
|
||||
| Network scanner | 6 hours | Opt-in (`CERTCTL_NETWORK_SCAN_ENABLED`) | Probes TLS endpoints on configured CIDR ranges, stores discovered certs (M21). CIDR size validated at API level — max /20 (4096 IPs) per range. |
|
||||
| Certificate digest | 24 hours (`CERTCTL_DIGEST_INTERVAL`) | Opt-in (digest service) | Generates HTML email with certificate stats, expiration timeline, job health, agent count. Does NOT run on startup — waits for first scheduled tick. Falls back to certificate owner emails if no explicit recipients configured. |
|
||||
| Endpoint health | 60 seconds (`CERTCTL_HEALTH_CHECK_INTERVAL`) | Opt-in (health check service) | Probes deployed TLS endpoints, drives the healthy/degraded/down/cert_mismatch state machine (M48) |
|
||||
| Cloud discovery | 6 hours | Opt-in (at least one cloud source configured) | Walks AWS Secrets Manager / Azure Key Vault / GCP Secret Manager, feeds discovery pipeline (M50) |
|
||||
|
||||
Each loop uses `sync/atomic.Bool` idempotency guards to prevent concurrent tick execution — if a loop iteration is still running when the next tick fires, the tick is skipped with a warning log. All loops (including short-lived expiry check) run immediately on startup before entering their ticker interval, ensuring no gap between scheduler start and first execution. The certificate digest loop is the exception — it does NOT run on startup, only on scheduled ticks. Graceful shutdown uses `sync.WaitGroup` with `WaitForCompletion()` to drain all in-flight work before process exit.
|
||||
Each loop uses `sync/atomic.Bool` idempotency guards to prevent concurrent tick execution — if a loop iteration is still running when the next tick fires, the tick is skipped with a warning log. Most loops (including short-lived expiry, job retry, job timeout, and notification retry) run immediately on startup before entering their ticker interval, ensuring no gap between scheduler start and first execution. The certificate digest loop is the exception — it does NOT run on startup, only on scheduled ticks. Graceful shutdown uses `sync.WaitGroup` with `WaitForCompletion()` to drain all in-flight work before process exit.
|
||||
|
||||
Each operation has a context timeout to prevent indefinite hangs if external services become unresponsive.
|
||||
|
||||
@@ -648,6 +676,16 @@ Built-in notifiers: **Email** (SMTP), **Webhook** (HTTP POST), **Slack** (incomi
|
||||
|
||||
See the [Connector Development Guide](connectors.md) for details on building custom connectors.
|
||||
|
||||
### Notification Retry & Dead-Letter Queue
|
||||
|
||||
A transient notifier failure (SMTP timeout, 5xx webhook response, Slack rate-limit) must not silently drop a critical alert. Migration `000016_notification_retry` adds three columns to `notification_events` — `retry_count INTEGER NOT NULL DEFAULT 0`, `next_retry_at TIMESTAMPTZ` (nullable — only meaningful while a row is in `failed` state), and `last_error TEXT` (the most recent transient error, preserved for operator triage) — together with a partial index `idx_notification_events_retry_sweep ON notification_events(next_retry_at) WHERE status = 'failed' AND next_retry_at IS NOT NULL` so the retry hot path scales with the retry-eligible slice rather than the full notification history.
|
||||
|
||||
The scheduler's notification-retry loop (see the scheduler section above) calls `NotificationService.RetryFailedNotifications(ctx)` every `CERTCTL_NOTIFICATION_RETRY_INTERVAL` (default `2m`). Each tick pulls up to 1000 rows via `notifRepo.ListRetryEligible(ctx, now, maxAttempts, sweepLimit)` — a partial-index-driven query that filters on `status='failed' AND next_retry_at <= now() AND retry_count < 5` — and redispatches them through the same notifier registry used by `ProcessPendingNotifications`. A successful redispatch transitions the row directly to `sent` without incrementing `retry_count`, so the audit trail preserves "delivered on attempt N". A failed redispatch re-arms `next_retry_at` using exponential backoff — `wait = min(2^retry_count minutes, 1h)` — bumps `retry_count`, and stamps `last_error`. When `retry_count >= 4` (the fifth attempt has just failed) the row is promoted to the terminal `dead` status via `notifRepo.MarkAsDead`, which clears `next_retry_at` so the partial retry-sweep index stops matching and the row cannot be re-entered into the retry rotation without operator action.
|
||||
|
||||
`NotificationService.RequeueNotification(ctx, id)` is the operator-driven escape hatch from `dead`. It atomically resets `retry_count → 0`, `next_retry_at → NULL`, `last_error → NULL`, and `status → pending`, handing the row back to `ProcessPendingNotifications` on the next 1m tick. This is the correct response to "the notifier outage is resolved, redeliver the queue"; it is not a retry, which is why the retry counter is reset rather than incremented.
|
||||
|
||||
The dead-letter depth is surfaced in two places. First, `DashboardSummary.NotificationsDead` is populated by `StatsService.GetDashboardSummary` via `notifRepo.CountByStatus(ctx, "dead")`. The injection uses a `SetNotifRepo` setter pattern (mirroring `CertificateService.SetTargetRepo`) rather than a new positional argument to `NewStatsService`, which keeps all nine existing `NewStatsService` call sites (main.go plus eight digest tests and stats_test.go) signature-stable — when the notification repository has not been wired in, `NotificationsDead` falls through to zero. Second, the `/api/v1/metrics/prometheus` endpoint emits `certctl_notification_dead_total` as a counter (operator alert thresholds per the I-005 spec: `> 0` warning, `> 10` critical) using the same `DashboardSummary` snapshot so the dashboard card and the Prometheus counter cannot skew. The web dashboard exposes a two-tab toolbar on `/notifications` — "All" (the pre-I-005 inbox) and "Dead letter" (threads `?status=dead` into the list query, surfaces `Retry N/5` and the truncated `last_error` with a full-text tooltip per row, and binds a Requeue button to `POST /api/v1/notifications/{id}/requeue`).
|
||||
|
||||
### EST Server (RFC 7030)
|
||||
|
||||
The EST (Enrollment over Secure Transport) server provides an industry-standard enrollment interface for devices that need certificates without using the REST API. It runs under `/.well-known/est/` per RFC 7030 and supports four operations: CA certificate distribution (`/cacerts`), initial enrollment (`/simpleenroll`), re-enrollment (`/simplereenroll`), and CSR attributes (`/csrattrs`).
|
||||
@@ -685,6 +723,8 @@ type ESTService interface {
|
||||
|
||||
**Issuer connector extension:** EST required adding `GetCACertPEM(ctx) (string, error)` to the issuer connector interface so the `/cacerts` endpoint can serve the CA chain. The Local CA returns its CA certificate PEM; Vault PKI fetches via `GET /v1/{mount}/ca/pem`; Google CAS fetches via API; AWS ACM PCA retrieves via `GetCertificateAuthorityCertificate`. ACME, step-ca, OpenSSL, DigiCert, and Sectigo connectors return errors (they don't expose a static CA chain — their chains are per-issuance).
|
||||
|
||||
**Authentication:** EST endpoints are served unauthenticated at the HTTP layer under `/.well-known/est/*` — no Bearer token required. Per RFC 7030 §3.2.3 EST authentication is deployment-specific, and per §4.1.1 `/cacerts` is explicitly anonymous. certctl enforces authentication via CSR signature verification inside `ESTService.SimpleEnroll`/`SimpleReEnroll` plus profile policy gates (allowed key algorithms, minimum key size, permitted SANs, permitted EKUs, MaxTTL). The HTTP dispatch is implemented in `cmd/server/main.go:buildFinalHandler`, which routes `/.well-known/est/*` through `noAuthHandler` (RequestID + structuredLogger + Recovery only). Operators who need stronger client identification should terminate mTLS at an upstream reverse proxy and pin the CSR's SAN to the client cert subject at the profile level.
|
||||
|
||||
**Audit:** Every EST enrollment is recorded in the audit trail with `protocol: "EST"`, the CN, SANs, issuer ID, serial number, and optional profile ID.
|
||||
|
||||
### SCEP Server (RFC 8894)
|
||||
@@ -711,7 +751,7 @@ Signed certificate returned as PKCS#7 certs-only
|
||||
|
||||
**Wire format:** SCEP clients wrap CSRs in PKCS#7 SignedData envelopes. The handler parses the outer ASN.1 ContentInfo → SignedData → EncapsulatedContentInfo to extract the CSR bytes. Fallback paths handle base64-encoded PKCS#7 and raw CSR submissions (for simpler clients). Responses use PKCS#7 certs-only via the shared `internal/pkcs7` package (same as EST). Single certs are returned as raw DER for `GetCACert`, chains as PKCS#7.
|
||||
|
||||
**Authentication:** SCEP uses challenge passwords embedded in CSR attributes (OID 1.2.840.113549.1.9.7) rather than TLS client certificates. The server validates the challenge password against `CERTCTL_SCEP_CHALLENGE_PASSWORD`. When no challenge password is configured, any value is accepted.
|
||||
**Authentication:** SCEP endpoints at `/scep` and `/scep/*` are served unauthenticated at the HTTP layer — no Bearer token required — per RFC 8894 §3.2, which defines authentication via the `challengePassword` attribute (OID 1.2.840.113549.1.9.7) embedded in the PKCS#10 CSR rather than an HTTP credential. The HTTP dispatch is implemented in `cmd/server/main.go:buildFinalHandler`, which routes `/scep` and `/scep/*` through `noAuthHandler` (RequestID + structuredLogger + Recovery only). The `challengePassword` is mandatory: `preflightSCEPChallengePassword` at startup refuses to boot the control plane when `CERTCTL_SCEP_ENABLED=true` is set without `CERTCTL_SCEP_CHALLENGE_PASSWORD`, closing CWE-306 (missing authentication for a critical function). `SCEPService.PKCSReq` enforces the same invariant defense-in-depth — an empty `s.challengePassword` rejects every enrollment — and the password comparison uses `crypto/subtle.ConstantTimeCompare` to prevent response-time side-channel leakage. The startup log line `SCEP server enabled` emits a `challenge_password_set` boolean for operator visibility.
|
||||
|
||||
**Interface:** The `SCEPHandler` defines an `SCEPService` interface (dependency inversion):
|
||||
|
||||
@@ -768,10 +808,11 @@ The control plane only handles public material: certificates, chains, and CSRs.
|
||||
|
||||
### Authentication
|
||||
|
||||
- **API clients → Server**: API key in `Authorization: Bearer` header, or `none` for demo mode
|
||||
- **API clients → Server**: API key in `Authorization: Bearer` header, or `none` for demo mode. Applies to every path under `/api/v1/*`.
|
||||
- **Agent → Server**: API key registered at agent creation, included in all requests
|
||||
- **Server → Issuers**: ACME account key, or connector-specific credentials
|
||||
- **Agent → Targets**: API tokens, WinRM credentials (stored locally on agent or proxy agent — never on server). Credential scope is limited to the agent's network zone.
|
||||
- **Standards-based enrollment and PKI distribution endpoints**: `/.well-known/est/*` (RFC 7030), `/scep` and `/scep/*` (RFC 8894), and `/.well-known/pki/crl/{issuer_id}` + `/.well-known/pki/ocsp/{issuer_id}/{serial}` (RFC 5280 §5 / RFC 6960 / RFC 8615) are served unauthenticated at the HTTP layer. These protocols carry their own authentication semantics — CSR signature + profile policy for EST (§3.2.3 says EST auth is deployment-specific; §4.1.1 makes `/cacerts` explicitly anonymous), `challengePassword` in CSR attributes for SCEP (§3.2), and relying-party accessibility for CRL/OCSP — and cannot present certctl Bearer tokens. The dispatch is implemented in `cmd/server/main.go:buildFinalHandler`, which routes these prefixes through `noAuthHandler` (RequestID + structuredLogger + Recovery only, no auth or rate-limit middleware). CWE-306 is closed for SCEP by `preflightSCEPChallengePassword`, which refuses to start the server when SCEP is enabled without `CERTCTL_SCEP_CHALLENGE_PASSWORD`. The 27-subtest regression harness `cmd/server/finalhandler_test.go` pins this dispatch surface (EST 4-endpoint, SCEP exact + trailing-slash + query-string, PKI CRL+OCSP, health probes, `/api/v1/*` authenticated, `/assets/*` file server, SPA fallback).
|
||||
|
||||
### Audit Trail
|
||||
|
||||
@@ -855,7 +896,7 @@ The HTTP middleware stack processes requests in the following order (see `cmd/se
|
||||
|
||||
### Concurrency Safety
|
||||
|
||||
The background scheduler uses `sync/atomic.Bool` idempotency guards on all 7 loops — if a tick fires while the previous iteration is still running, it skips. A `sync.WaitGroup` tracks all in-flight goroutines. `WaitForCompletion(timeout)` blocks during shutdown until all work finishes or the timeout expires, preventing state corruption from mid-flight database operations during process exit.
|
||||
The background scheduler uses `sync/atomic.Bool` idempotency guards on every loop (8 always-on plus up to 4 optional) — if a tick fires while the previous iteration is still running, it skips. A `sync.WaitGroup` tracks all in-flight goroutines. `WaitForCompletion(timeout)` blocks during shutdown until all work finishes or the timeout expires, preventing state corruption from mid-flight database operations during process exit.
|
||||
|
||||
### Logging
|
||||
|
||||
@@ -1051,7 +1092,7 @@ flowchart TB
|
||||
|
||||
1. **Pluggable sources** — Each cloud provider implements the `DiscoverySource` interface (Name, Type, Discover, ValidateConfig). Three built-in sources: AWS Secrets Manager, Azure Key Vault, GCP Secret Manager
|
||||
2. **CloudDiscoveryService orchestrator** — Iterates registered sources, calls `Discover()` on each, feeds reports into `ProcessDiscoveryReport()`. Errors from one source don't prevent other sources from running
|
||||
3. **Scheduler integration** — 9th scheduler loop (6h default), runs immediately on startup, `atomic.Bool` idempotency guard
|
||||
3. **Scheduler integration** — opt-in cloud discovery scheduler loop (6h default; see `docs/architecture.md` 12-loop topology), runs immediately on startup, `atomic.Bool` idempotency guard
|
||||
4. **Sentinel agents** — Each source uses its own sentinel agent ID (`cloud-aws-sm`, `cloud-azure-kv`, `cloud-gcp-sm`) for dedup and triage filtering
|
||||
5. **Source path format** — `aws-sm://{region}/{secret}`, `azure-kv://{cert-name}/{version}`, `gcp-sm://{project}/{secret}`
|
||||
6. **No new schema** — Reuses existing `discovered_certificates` and `discovery_scans` tables. Sentinel agent IDs leverage existing `(fingerprint_sha256, agent_id, source_path)` dedup constraint
|
||||
@@ -1073,7 +1114,7 @@ This data flow is pull-based and non-blocking. Agents discover at their own pace
|
||||
|
||||
Beyond one-time discovery, certctl continuously monitors TLS endpoints for certificate health using a shared TLS probing package and a state-machine-driven health check service. Endpoints transition between states (Healthy → Degraded → Down) based on consecutive failures, and `cert_mismatch` status alerts when a deployed certificate is unexpectedly replaced.
|
||||
|
||||
**Architecture:** Probing is extracted into a shared `internal/tlsprobe/` package used by both the network scanner (M21) and the health monitor. The `HealthCheckService` manages 8 API endpoints for CRUD operations and state transitions. A dedicated 8th scheduler loop runs every 60 seconds (configurable via `CERTCTL_HEALTH_CHECK_INTERVAL`). Individual health check targets have their own check intervals (default 300 seconds) — the scheduler queries only endpoints due for check via `ListDueForCheck()`. Results are stored with historical tracking for 30 days (configurable via `CERTCTL_HEALTH_CHECK_HISTORY_RETENTION`). State transitions trigger notifications (critical for down endpoints, warning for degraded, high for cert_mismatch).
|
||||
**Architecture:** Probing is extracted into a shared `internal/tlsprobe/` package used by both the network scanner (M21) and the health monitor. The `HealthCheckService` manages 8 API endpoints for CRUD operations and state transitions. A dedicated opt-in endpoint health scheduler loop runs every 60 seconds (configurable via `CERTCTL_HEALTH_CHECK_INTERVAL`). Individual health check targets have their own check intervals (default 300 seconds) — the scheduler queries only endpoints due for check via `ListDueForCheck()`. Results are stored with historical tracking for 30 days (configurable via `CERTCTL_HEALTH_CHECK_HISTORY_RETENTION`). State transitions trigger notifications (critical for down endpoints, warning for degraded, high for cert_mismatch).
|
||||
|
||||
**State Machine:** Healthy → Degraded (configurable threshold, default 2 consecutive failures) → Down (default 5 failures). The `cert_mismatch` status is special — it fires whenever the observed certificate fingerprint differs from the expected (deployed) fingerprint, catching silent rollbacks and unauthorized cert replacements. Recovery from degraded/down transitions back to healthy and resets the failure counter.
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ Deploy certctl control plane once (Docker Compose, Kubernetes Helm chart, or sel
|
||||
```bash
|
||||
cd /opt/certctl
|
||||
docker compose up -d
|
||||
# Dashboard & API: http://localhost:8443
|
||||
# Dashboard & API: https://localhost:8443 (self-signed cert — pin with --cacert ./deploy/test/certs/ca.crt)
|
||||
```
|
||||
|
||||
**Option B: Kubernetes** (recommended for prod)
|
||||
@@ -59,7 +59,8 @@ chmod +x /usr/local/bin/certctl-agent
|
||||
|
||||
# Config
|
||||
sudo tee /etc/certctl/agent.env > /dev/null <<EOF
|
||||
CERTCTL_SERVER_URL=http://certctl-control-plane:8443
|
||||
CERTCTL_SERVER_URL=https://certctl-control-plane:8443
|
||||
CERTCTL_SERVER_CA_BUNDLE_PATH=/etc/certctl/tls/ca.crt
|
||||
CERTCTL_API_KEY=your-api-key
|
||||
CERTCTL_DISCOVERY_DIRS=/etc/nginx/certs,/etc/ssl,/etc/letsencrypt/live
|
||||
CERTCTL_KEY_DIR=/var/lib/certctl/keys
|
||||
|
||||
@@ -387,12 +387,12 @@ This requirement covers key generation, storage, rotation, and destruction. Cert
|
||||
- API key transmitted in Authorization header (not URL parameter, not cookie).
|
||||
- Browser to server: TLS.
|
||||
- Agent to server: TLS.
|
||||
- No credential logging (API key hash only, never plaintext).
|
||||
- No credential logging (audit records the per-key actor `Name`, never the Bearer token; logs redact the `Authorization` header).
|
||||
|
||||
**Evidence You Can Provide**:
|
||||
- API configuration: `CERTCTL_AUTH_TYPE=api-key` in deployment manifest.
|
||||
- Database schema: `api_keys` table showing SHA-256 hash column, not plaintext.
|
||||
- API audit log: `GET /api/v1/audit?action=api_call` showing Bearer token validation (no plaintext keys logged).
|
||||
- Key inventory: `CERTCTL_API_KEYS_NAMED` env var (format `name:key:admin,...`) — seeds the in-memory `NamedAPIKey{Name, Key, Admin}` struct at `internal/api/middleware/middleware.go:29`. Keys are constant-time-compared (`subtle.ConstantTimeCompare`) against the Bearer token. No database table stores them; protect the env var contents at rest via a secrets manager (Vault / AWS Secrets Manager / Kubernetes Secrets / Docker Secrets).
|
||||
- API audit log: `GET /api/v1/audit?action=api_call` showing per-key actor names (`Name` field of matched `NamedAPIKey`) on every call, with zero plaintext or hashed key material recorded.
|
||||
- TLS certificate on control plane: `openssl s_client -connect {server}:8443` showing valid certificate, TLS 1.2+, strong cipher.
|
||||
- GUI login flow: browser network tab showing Authorization header (token value redacted in compliance report).
|
||||
|
||||
@@ -562,6 +562,7 @@ This requirement covers key generation, storage, rotation, and destruction. Cert
|
||||
- **Alert Notifications** (M3, M16a) — Configurable escalation:
|
||||
- Email alerts: certificate approaching expiration, renewal failure, revocation notification.
|
||||
- Webhook: custom HTTP POST to your monitoring system (Slack, Teams, PagerDuty, OpsGenie, custom webhook).
|
||||
- **Retry & Dead-Letter Queue** (I-005) — Transient notifier failures (SMTP timeout, webhook 5xx) are retried with exponential backoff (`2^n` minutes capped at 1h, 5-attempt budget) before landing in the terminal `dead` status. Operators monitor DLQ depth via the `certctl_notification_dead_total` Prometheus counter and requeue via the Notifications page Dead letter tab once the underlying outage is resolved. Closes the pre-I-005 silent-drop gap where a single 5xx could lose a compliance-relevant alert without evidence.
|
||||
- Deduplication: one alert per threshold/certificate per day (avoid alert fatigue).
|
||||
|
||||
- **Audit Trail Filtering and Export** (M13) — Compliance reporting:
|
||||
|
||||
+23
-12
@@ -44,7 +44,8 @@ Each section includes:
|
||||
|
||||
**certctl Implementation** (V2 — Community Edition):
|
||||
|
||||
- **API Key Authentication** — All API calls require a Bearer token (hashed with SHA-256, stored securely, validated with constant-time comparison) or are rejected with 401 Unauthorized. Environment: `CERTCTL_AUTH_TYPE` (default `api-key`; `none` requires explicit opt-in with log warning)
|
||||
- **API Key Authentication** — All `/api/v1/*` calls require a Bearer token (hashed with SHA-256, stored securely, validated with constant-time comparison) or are rejected with 401 Unauthorized. Environment: `CERTCTL_AUTH_TYPE` (default `api-key`; `none` requires explicit opt-in with log warning)
|
||||
- **Standards-based enrollment and PKI distribution endpoints** — EST (`/.well-known/est/*`, RFC 7030), SCEP (`/scep`, `/scep/*`, RFC 8894), and CRL/OCSP (`/.well-known/pki/crl/{issuer_id}`, `/.well-known/pki/ocsp/{issuer_id}/{serial}`, RFC 5280 §5 / RFC 6960 / RFC 8615) are served unauthenticated at the HTTP layer because these protocols cannot present certctl Bearer tokens. Authentication is enforced in-protocol: EST relies on CSR signature verification plus profile policy (RFC 7030 §3.2.3 says EST auth is deployment-specific; §4.1.1 makes `/cacerts` explicitly anonymous); SCEP requires a shared `challengePassword` in the PKCS#10 CSR attributes (OID 1.2.840.113549.1.9.7, RFC 8894 §3.2), validated with `crypto/subtle.ConstantTimeCompare`; CRL and OCSP are intentionally anonymous for relying-party accessibility. CWE-306 (missing authentication for a critical function) is closed for SCEP by `preflightSCEPChallengePassword` in `cmd/server/main.go`, which refuses to start the control plane when `CERTCTL_SCEP_ENABLED=true` is set without `CERTCTL_SCEP_CHALLENGE_PASSWORD`. The HTTP dispatch is implemented in `cmd/server/main.go:buildFinalHandler`, which routes these prefixes through `noAuthHandler` (RequestID + structuredLogger + Recovery only, no auth or rate-limit middleware) and is pinned by the 27-subtest regression harness at `cmd/server/finalhandler_test.go`.
|
||||
- **GUI Authentication** — Web dashboard includes login screen requiring API key entry. Failed auth redirects to login on 401. Auth context persists across page navigation. Logout clears session.
|
||||
- **Configurable CORS** — API restricts cross-origin requests via `CERTCTL_CORS_ORIGINS` allowlist or wildcard. Preflight caching prevents chatty browser auth flows.
|
||||
- **Token Bucket Rate Limiting** — Per-IP rate limiting (configurable via `CERTCTL_RATE_LIMIT_RPS` / `CERTCTL_RATE_LIMIT_BURST`) returns 429 Too Many Requests with Retry-After header. Prevents credential stuffing and brute-force attacks.
|
||||
@@ -58,6 +59,11 @@ Each section includes:
|
||||
- Auth info endpoint: `GET /api/v1/auth/info` (returns current auth mode, served without auth so GUI detects mode)
|
||||
- Rate limiting middleware: `internal/api/middleware/rate_limit.go`
|
||||
- CORS configuration: `cmd/server/main.go`, search for `CERTCTL_CORS_ORIGINS`
|
||||
- Final handler dispatch (authenticated vs. unauthenticated routing): `cmd/server/main.go:buildFinalHandler`
|
||||
- SCEP preflight gate (CWE-306 closure): `cmd/server/main.go:preflightSCEPChallengePassword`
|
||||
- SCEP service-layer defense-in-depth (rejects enrollment on empty challenge password, `crypto/subtle.ConstantTimeCompare`): `internal/service/scep.go`
|
||||
- Final handler dispatch regression harness (27 subtests): `cmd/server/finalhandler_test.go`
|
||||
- OpenAPI spec `security: []` overrides on unauthenticated paths: `api/openapi.yaml` (EST `/cacerts`, `/simpleenroll`, `/simplereenroll`, `/csrattrs`; SCEP `/scep` GET+POST; PKI `/crl/{issuer_id}`, `/ocsp/{issuer_id}/{serial}`)
|
||||
|
||||
**V3 Enhancement**:
|
||||
|
||||
@@ -110,7 +116,7 @@ Each section includes:
|
||||
|
||||
**certctl Implementation** (V2):
|
||||
|
||||
- **API Key Policy** — All API access requires an API key or explicit opt-out. Opt-out (`CERTCTL_AUTH_TYPE=none`) logs a warning: "WARNING: Auth disabled (CERTCTL_AUTH_TYPE=none) — this is insecure and only for development". Configuration choice is logged at startup.
|
||||
- **API Key Policy** — All `/api/v1/*` access requires an API key or explicit opt-out. Opt-out (`CERTCTL_AUTH_TYPE=none`) logs a warning: "WARNING: Auth disabled (CERTCTL_AUTH_TYPE=none) — this is insecure and only for development". Configuration choice is logged at startup. The standards-based enrollment and PKI distribution endpoints (EST, SCEP, CRL, OCSP) are served unauthenticated at the HTTP layer per their respective RFCs; see CC6.1 for the full authentication contract and CWE-306 closure via `preflightSCEPChallengePassword`.
|
||||
- **Agent Authentication** — Agents authenticate to the server via API keys (same mechanism as users). Agent credentials are separate from user API keys.
|
||||
- **Private Key Policy** — Agent-side key generation is the default (`CERTCTL_KEYGEN_MODE=agent`). Server-side keygen (`CERTCTL_KEYGEN_MODE=server`) requires explicit configuration and logs a warning: "server-side key generation enabled (CERTCTL_KEYGEN_MODE=server) — private keys touch control plane, demo only".
|
||||
- **Password Policy** — Not applicable; certctl uses API keys exclusively. Password management is delegated to your organization's IAM system if you integrate OIDC/SSO (V3).
|
||||
@@ -183,15 +189,20 @@ Each section includes:
|
||||
|
||||
- **Health Endpoint** — `GET /health` returns 200 OK with service status. Consumed by Docker health checks and Kubernetes probes.
|
||||
- **Readiness Endpoint** — `GET /ready` returns 200 OK when the database is connected and migrations are applied.
|
||||
- **Background Scheduler Monitoring** — 7 background loops run on a fixed schedule:
|
||||
- Renewal loop: every 1 hour, scans for certificates approaching renewal threshold
|
||||
- Job processor loop: every 30 seconds, picks up pending/waiting jobs and advances their state
|
||||
- Health check loop: every 2 minutes, pings agents to detect downtime
|
||||
- Notification dispatcher loop: every 1 minute, sends queued alerts
|
||||
- Short-lived cert expiry loop: every 30 seconds, marks expired short-lived credentials
|
||||
- Network scanner loop: every 6 hours, scans enabled TLS endpoints for certificate discovery
|
||||
- Digest emailer loop: every 24 hours, sends scheduled certificate digest email to configured recipients
|
||||
Each loop includes error handling and logs failures via structured slog.
|
||||
- **Background Scheduler Monitoring** — 12 background loops (8 always-on + 4 opt-in) run on a fixed schedule. Authoritative topology in `docs/architecture.md`:
|
||||
- Renewal loop (always-on, 1 hour): scans for certificates approaching renewal threshold
|
||||
- Job processor loop (always-on, 30 seconds): picks up pending/waiting jobs and advances their state
|
||||
- Job retry loop (always-on, 5 minutes, `CERTCTL_SCHEDULER_RETRY_INTERVAL`): retries Failed jobs (I-001)
|
||||
- Job timeout reaper loop (always-on, 10 minutes, `CERTCTL_JOB_TIMEOUT_INTERVAL`): fails AwaitingCSR/AwaitingApproval jobs past timeout (I-003)
|
||||
- Agent health check loop (always-on, 2 minutes): pings agents to detect downtime
|
||||
- Notification dispatcher loop (always-on, 1 minute): sends queued alerts
|
||||
- Notification retry loop (always-on, 2 minutes, `CERTCTL_NOTIFICATION_RETRY_INTERVAL`): exponential backoff retry for failed notifications; promote to dead-letter after 5 attempts (I-005)
|
||||
- Short-lived cert expiry loop (always-on, 30 seconds): marks expired short-lived credentials
|
||||
- Network scanner loop (opt-in, 6 hours, `CERTCTL_NETWORK_SCAN_ENABLED`): scans enabled TLS endpoints for certificate discovery
|
||||
- Digest emailer loop (opt-in, 24 hours, `CERTCTL_DIGEST_INTERVAL`): sends scheduled certificate digest email to configured recipients
|
||||
- Endpoint health loop (opt-in, 60 seconds, `CERTCTL_HEALTH_CHECK_INTERVAL`): continuous TLS health probes (M48)
|
||||
- Cloud discovery loop (opt-in, 6 hours, `CERTCTL_CLOUD_DISCOVERY_INTERVAL`): cloud secret manager certificate discovery (M50)
|
||||
Each loop includes `atomic.Bool` idempotency guards, error handling, and structured slog failure logs.
|
||||
- **Metrics Endpoints** — Two formats for monitoring integration:
|
||||
- `GET /api/v1/metrics` — JSON object with gauges, counters, and uptime for custom dashboards
|
||||
- `GET /api/v1/metrics/prometheus` — Prometheus exposition format (`text/plain; version=0.0.4`) for native scraping by Prometheus, Grafana Agent, Datadog, and other OpenMetrics-compatible collectors
|
||||
@@ -453,7 +464,7 @@ Each section includes:
|
||||
| | Metrics JSON Endpoint | `GET /api/v1/metrics` (gauges, counters, uptime) | ✅ | ✅ | Set thresholds, configure alerting |
|
||||
| | Stats API (time-series) | `GET /api/v1/stats/*` (summary, status, expiration, jobs, issuance) | ✅ | ✅ | Integrate into dashboards, SLO tracking |
|
||||
| | Structured Logging | `slog` middleware with request IDs | ✅ | ✅ | Aggregate logs to SIEM, define retention policy |
|
||||
| | Background Scheduler | 7 loops (renewal 1h, jobs 30s, health 2m, notifications 1m, short-lived 30s, network scan 6h, digest 24h) | ✅ | ✅ | Alert on scheduler loop failures |
|
||||
| | Background Scheduler | 12 loops (8 always-on: renewal 1h, jobs 30s, job retry 5m I-001, job timeout 10m I-003, health 2m, notifications 1m, notif retry 2m I-005, short-lived 30s; 4 opt-in: network scan 6h, digest 24h, endpoint health 60s M48, cloud discovery 6h M50) | ✅ | ✅ | Alert on scheduler loop failures |
|
||||
| **CC7.2** Anomaly Detection | Immutable API Audit Trail | `internal/api/middleware/audit.go`, `GET /api/v1/audit` | ✅ | Enhanced (SIEM export) | Integrate into SIEM, search for anomalies, archive long-term |
|
||||
| | Expiration Threshold Alerting | Configurable per-policy (default 30/14/7/0 days) | ✅ | ✅ | Configure thresholds, integrate notifications |
|
||||
| | Status Auto-Transitions | Active → Expiring (30d) → Expired (0d) | ✅ | ✅ | Monitor status changes in audit trail |
|
||||
|
||||
@@ -123,6 +123,8 @@ At no point does the private key leave the agent. This is a fundamental security
|
||||
|
||||
Agents also report **metadata** about themselves — their operating system, CPU architecture, IP address, hostname, and version — with every heartbeat. This gives ops teams fleet-wide visibility (e.g., "how many agents are running on ARM?", "which agents are still on v1.0.0?") and powers **agent groups** — dynamic device grouping where policies can be scoped to specific agent criteria like OS type, architecture, or network subnet.
|
||||
|
||||
**Retiring an agent.** When you decommission a server, the certctl record for its agent needs to be retired, not deleted. certctl uses a **soft-delete** model: `DELETE /api/v1/agents/{id}` stamps the row with a retired-at timestamp and a reason, instead of removing it. This is deliberate — an audit trail of "who owned this certificate, on which host, for which team" stays intact forever, and the downstream deployment_targets, certificates, and jobs keep valid foreign keys. Retired agents are filtered out of default list views and the dashboard's agent counter, but remain visible through a separate retired-agents view for compliance reconciliation. If the agent still has active deployment targets, deployed certificates, or pending jobs, retirement is blocked by default so you don't silently orphan those rows; the API responds with the exact counts so you can retire or reassign each dependency explicitly. A force-retire escape hatch (`?force=true&reason=...`) is available for true decommission scenarios — it transactionally retires the downstream targets, cancels pending jobs, and records the cascade in the audit trail with the reason you provided. Four internal sentinel agents that back the network scanner and the cloud secret-manager discovery sources cannot be retired at all, even with force, because retiring them would orphan their subsystems. Once retired, an agent that still attempts to heartbeat receives `410 Gone` — the agent process reads that as "you've been retired, shut down" and exits cleanly.
|
||||
|
||||
### Deployment Targets
|
||||
|
||||
Targets are the systems where certificates actually get installed — NGINX web servers, Apache httpd servers, HAProxy load balancers, Traefik reverse proxies, Caddy servers, Envoy gateways, Postfix/Dovecot mail servers, Microsoft IIS servers, and network appliances. Each target type has a **connector** that knows how to deploy certificates to that specific system (e.g., writing files and reloading NGINX or Apache config, building a combined PEM for HAProxy).
|
||||
|
||||
+3
-3
@@ -1126,7 +1126,7 @@ The digest HTML template includes:
|
||||
- Expiring certificates table (color-coded by urgency: 7d, 14d, 30d)
|
||||
- Auto-refresh and responsive email layout
|
||||
|
||||
**Scheduler Integration:** The 7th scheduler loop runs on configurable interval (default 24 hours). It does NOT run on startup — waits for first scheduled tick. Operation timeout is 5 minutes. Each loop execution is guarded by `sync/atomic.Bool` idempotency.
|
||||
**Scheduler Integration:** The opt-in digest scheduler loop runs on configurable interval (default 24 hours). It does NOT run on startup — waits for first scheduled tick. Operation timeout is 5 minutes. Each loop execution is guarded by `sync/atomic.Bool` idempotency. See `docs/architecture.md` for the full scheduler topology (12 loops, 8 always-on + 4 opt-in).
|
||||
|
||||
Configuration:
|
||||
|
||||
@@ -1389,7 +1389,7 @@ curl -s -X DELETE http://localhost:8443/api/v1/network-scan-targets/nst-dmz
|
||||
|
||||
### Scheduler Integration
|
||||
|
||||
When `CERTCTL_NETWORK_SCAN_ENABLED=true`, the server runs a 6th scheduler loop (alongside renewal, jobs, health, notifications, and short-lived expiry). It scans all enabled targets at the configured interval (default 6h). Each target tracks `last_scan_at`, `last_scan_duration_ms`, and `last_scan_certs_found` for monitoring scan health.
|
||||
When `CERTCTL_NETWORK_SCAN_ENABLED=true`, the server runs the opt-in network scanner scheduler loop alongside the always-on loops (renewal, jobs, job retry, job timeout, agent health, notifications, notification retry, short-lived expiry). It scans all enabled targets at the configured interval (default 6h). Each target tracks `last_scan_at`, `last_scan_duration_ms`, and `last_scan_certs_found` for monitoring scan health. See `docs/architecture.md` for the full 12-loop scheduler topology.
|
||||
|
||||
### Use Cases
|
||||
|
||||
@@ -1447,7 +1447,7 @@ Source path format: `gcp-sm://{project}/{secret-name}`. Sentinel agent: `cloud-g
|
||||
|
||||
### Cloud Discovery Scheduler
|
||||
|
||||
All enabled cloud sources run on a shared scheduler loop (9th loop). The interval is configurable:
|
||||
All enabled cloud sources run on a shared opt-in cloud discovery scheduler loop (see `docs/architecture.md` for the full 12-loop scheduler topology). The interval is configurable:
|
||||
|
||||
| Variable | Description | Default |
|
||||
|---|---|---|
|
||||
|
||||
+17
-11
@@ -50,14 +50,17 @@ docker compose -f deploy/docker-compose.yml up -d --build
|
||||
docker compose -f deploy/docker-compose.yml ps
|
||||
```
|
||||
|
||||
Open **http://localhost:8443** in your browser alongside your terminal. You'll watch changes appear in the dashboard as you make API calls.
|
||||
Open **https://localhost:8443** in your browser alongside your terminal. The default compose stack ships a self-signed cert; your browser will show a warning the first time — click through (or trust `deploy/test/certs/ca.crt` in your OS keychain). You'll watch changes appear in the dashboard as you make API calls.
|
||||
|
||||
Set up a base variable for convenience:
|
||||
Set up base variables for convenience:
|
||||
|
||||
```bash
|
||||
API="http://localhost:8443"
|
||||
API="https://localhost:8443"
|
||||
CA="$PWD/deploy/test/certs/ca.crt" # pin the self-signed CA for curl
|
||||
```
|
||||
|
||||
Every `curl` in this guide uses `--cacert "$CA"` so the TLS handshake verifies against the compose-stack CA instead of the system trust store.
|
||||
|
||||
## How the pieces fit together
|
||||
|
||||
Before we start, here's the high-level flow of what we're about to do:
|
||||
@@ -730,7 +733,7 @@ Check the CRL (Certificate Revocation List) — served unauthenticated under the
|
||||
# DER-encoded X.509 CRL for the local CA (binary — pipe to openssl for inspection).
|
||||
# Note: no -H "Authorization: Bearer ..." — the endpoint is deliberately
|
||||
# unauthenticated. Content-Type is application/pkix-crl.
|
||||
curl -s http://localhost:8443/.well-known/pki/crl/iss-local -o /tmp/crl.der
|
||||
curl --cacert "$CA" -s https://localhost:8443/.well-known/pki/crl/iss-local -o /tmp/crl.der
|
||||
openssl crl -inform DER -in /tmp/crl.der -text -noout
|
||||
```
|
||||
|
||||
@@ -740,7 +743,7 @@ Check OCSP status (RFC 6960, also unauthenticated, `application/ocsp-response`):
|
||||
# Replace SERIAL with the actual serial number from the certificate version.
|
||||
# The embedded OCSP responder returns a signed DER response — parse it with
|
||||
# `openssl ocsp -respin` or similar tooling.
|
||||
curl -s http://localhost:8443/.well-known/pki/ocsp/iss-local/SERIAL -o /tmp/ocsp.der
|
||||
curl --cacert "$CA" -s https://localhost:8443/.well-known/pki/ocsp/iss-local/SERIAL -o /tmp/ocsp.der
|
||||
openssl ocsp -respin /tmp/ocsp.der -noverify -resp_text | head -40
|
||||
```
|
||||
|
||||
@@ -946,7 +949,8 @@ certctl includes a standalone CLI tool for command-line users:
|
||||
cd cmd/cli && go build -o certctl-cli .
|
||||
|
||||
# Export credentials
|
||||
export CERTCTL_SERVER_URL="http://localhost:8443"
|
||||
export CERTCTL_SERVER_URL="https://localhost:8443"
|
||||
export CERTCTL_SERVER_CA_BUNDLE_PATH="$PWD/deploy/test/certs/ca.crt"
|
||||
export CERTCTL_API_KEY="test-key-123"
|
||||
|
||||
# List certificates (JSON or table format)
|
||||
@@ -990,7 +994,8 @@ certctl exposes the full REST API via the Model Context Protocol (MCP), enabling
|
||||
cd cmd/mcp-server && go build -o mcp-server .
|
||||
|
||||
# Export credentials
|
||||
export CERTCTL_SERVER_URL="http://localhost:8443"
|
||||
export CERTCTL_SERVER_URL="https://localhost:8443"
|
||||
export CERTCTL_SERVER_CA_BUNDLE_PATH="$PWD/deploy/test/certs/ca.crt"
|
||||
export CERTCTL_API_KEY="test-key-123"
|
||||
|
||||
# Start the MCP server (listens on stdin/stdout)
|
||||
@@ -1048,7 +1053,7 @@ docker compose -f deploy/docker-compose.yml run -e CERTCTL_DISCOVERY_DIRS=/tmp/c
|
||||
Or with the CLI flag:
|
||||
|
||||
```bash
|
||||
certctl-agent --agent-id a-demo-1 --key-dir /tmp/keys --discovery-dirs /tmp/certs --server http://localhost:8443 --api-key test-key-123
|
||||
certctl-agent --agent-id a-demo-1 --key-dir /tmp/keys --discovery-dirs /tmp/certs --server https://localhost:8443 --ca-bundle "$CA" --api-key test-key-123
|
||||
```
|
||||
|
||||
### Network Discovery (Server-Side)
|
||||
@@ -1155,7 +1160,7 @@ flowchart TB
|
||||
API["REST API\nGo net/http"]
|
||||
SVC["Service Layer\nBusiness Logic"]
|
||||
REPO["Repository Layer\ndatabase/sql + lib/pq"]
|
||||
SCHED["Scheduler\n7 background loops"]
|
||||
SCHED["Scheduler\n12 background loops\n(8 always-on + 4 opt-in)"]
|
||||
CONN["Connector Registry\nIssuer + Target + Notifier"]
|
||||
end
|
||||
|
||||
@@ -1191,7 +1196,8 @@ Here's a single script that runs the entire demo end-to-end. Save it as `demo.sh
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
API="http://localhost:8443"
|
||||
API="https://localhost:8443"
|
||||
CA="$PWD/deploy/test/certs/ca.crt" # pin the self-signed CA for curl
|
||||
BLUE='\033[0;34m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
@@ -1299,7 +1305,7 @@ echo " 5. Revoked the certificate with RFC 5280 reason codes"
|
||||
echo " 6. Checked dashboard stats and metrics"
|
||||
echo " 7. All actions recorded in the audit trail"
|
||||
echo ""
|
||||
echo -e "Open ${GREEN}http://localhost:8443${NC} to see everything in the dashboard."
|
||||
echo -e "Open ${GREEN}https://localhost:8443${NC} to see everything in the dashboard."
|
||||
echo "Look for certificate: $CERT_ID"
|
||||
```
|
||||
|
||||
|
||||
+17
-12
@@ -16,7 +16,7 @@ Complete reference of every feature shipped in certctl through v2.1.0 (April 202
|
||||
| Target connectors | 14 |
|
||||
| Notifier connectors | 6 channels |
|
||||
| Database tables | 21 (across 10 migrations) |
|
||||
| Background scheduler loops | 7 |
|
||||
| Background scheduler loops | 12 (8 always-on + 4 opt-in) |
|
||||
| Web dashboard pages | 24 |
|
||||
| Test functions | 1850+ |
|
||||
| Supported platforms | linux/amd64, linux/arm64, darwin/amd64, darwin/arm64 |
|
||||
@@ -903,7 +903,7 @@ Server-side active TLS scanning of CIDR ranges. Concurrent probing with semaphor
|
||||
|
||||
<!-- Source: internal/connector/discovery/awssm/, azurekv/, gcpsm/, internal/service/cloud_discovery.go -->
|
||||
|
||||
Discovers certificates stored in cloud secret managers and brings them into the certctl inventory. Extends the existing discovery pipeline with pluggable `DiscoverySource` implementations. Each source runs as part of the 9th scheduler loop (6h default).
|
||||
Discovers certificates stored in cloud secret managers and brings them into the certctl inventory. Extends the existing discovery pipeline with pluggable `DiscoverySource` implementations. Each source runs as part of the opt-in cloud discovery scheduler loop (6h default; see `docs/architecture.md` for the full 12-loop scheduler topology).
|
||||
|
||||
**Supported sources:**
|
||||
|
||||
@@ -1097,17 +1097,22 @@ Single SQL `UNION` query replaces the previous "fetch all, filter in Go" approac
|
||||
|
||||
<!-- Source: internal/scheduler/scheduler.go -->
|
||||
|
||||
7 background loops, each with an `atomic.Bool` idempotency guard preventing concurrent tick execution. `sync.WaitGroup` + `WaitForCompletion()` for graceful shutdown.
|
||||
12 background loops (8 always-on + 4 opt-in), each with an `atomic.Bool` idempotency guard preventing concurrent tick execution. `sync.WaitGroup` + `WaitForCompletion()` for graceful shutdown. Authoritative topology table lives in `docs/architecture.md`.
|
||||
|
||||
| Loop | Default Interval | Description |
|
||||
|---|---|---|
|
||||
| Renewal check | 1 hour | Check expiring certs, query ARI, create renewal jobs |
|
||||
| Job processor | 30 seconds | Process pending jobs |
|
||||
| Agent health check | 2 minutes | Check agent heartbeat staleness |
|
||||
| Notification processor | 1 minute | Send queued notifications |
|
||||
| Short-lived expiry check | 30 seconds | Mark short-lived certs expired |
|
||||
| Network scan | 6 hours | Run network discovery scans |
|
||||
| Digest | 24 hours | Send certificate digest email (does not run on startup) |
|
||||
| Loop | Default Interval | Always-on | Env Var | Description |
|
||||
|---|---|---|---|---|
|
||||
| Renewal check | 1 hour | Yes | — | Check expiring certs, query ARI, create renewal jobs |
|
||||
| Job processor | 30 seconds | Yes | — | Process pending jobs |
|
||||
| Job retry | 5 minutes | Yes | `CERTCTL_SCHEDULER_RETRY_INTERVAL` | Retry Failed jobs (I-001) |
|
||||
| Job timeout reaper | 10 minutes | Yes | `CERTCTL_JOB_TIMEOUT_INTERVAL` | Fail AwaitingCSR/AwaitingApproval jobs past timeout (I-003) |
|
||||
| Agent health check | 2 minutes | Yes | — | Check agent heartbeat staleness |
|
||||
| Notification processor | 1 minute | Yes | — | Send queued notifications |
|
||||
| Notification retry | 2 minutes | Yes | `CERTCTL_NOTIFICATION_RETRY_INTERVAL` | Exponential backoff retry for failed notifications; promote to dead-letter after 5 attempts (I-005) |
|
||||
| Short-lived expiry check | 30 seconds | Yes | — | Mark short-lived certs expired |
|
||||
| Network scan | 6 hours | Opt-in | `CERTCTL_NETWORK_SCAN_ENABLED` | Run network discovery scans |
|
||||
| Digest | 24 hours | Opt-in | `CERTCTL_DIGEST_INTERVAL` | Send certificate digest email (does not run on startup) |
|
||||
| Endpoint health | 60 seconds | Opt-in | `CERTCTL_HEALTH_CHECK_INTERVAL` | Continuous TLS health probes (M48) |
|
||||
| Cloud discovery | 6 hours | Opt-in | `CERTCTL_CLOUD_DISCOVERY_INTERVAL` | Cloud secret manager certificate discovery (M50) |
|
||||
|
||||
---
|
||||
|
||||
|
||||
+11
-5
@@ -29,15 +29,18 @@ The binary has zero runtime dependencies beyond the certctl server it connects t
|
||||
|
||||
## Configuration
|
||||
|
||||
The MCP server reads two environment variables:
|
||||
The MCP server reads three environment variables:
|
||||
|
||||
| Variable | Required | Default | Description |
|
||||
|----------|----------|---------|-------------|
|
||||
| `CERTCTL_SERVER_URL` | No | `http://localhost:8443` | URL of the certctl REST API |
|
||||
| `CERTCTL_SERVER_URL` | No | `https://localhost:8443` | URL of the certctl REST API (HTTPS-only as of v2.2) |
|
||||
| `CERTCTL_API_KEY` | No | (empty) | API key for authentication (passed as `Bearer` token) |
|
||||
| `CERTCTL_SERVER_CA_BUNDLE_PATH` | Yes (for self-signed / internal CA) | (empty) | Path to PEM CA bundle that signed the server cert. Required when the server cert isn't rooted in the system trust store (the default compose stack ships a self-signed cert at `deploy/test/certs/ca.crt`). |
|
||||
|
||||
If your certctl server has auth enabled (the default), you must provide the API key. The MCP server passes it through to every HTTP request.
|
||||
|
||||
Since v2.2 the certctl control plane is HTTPS-only. If the server cert is self-signed or chained to an internal CA, set `CERTCTL_SERVER_CA_BUNDLE_PATH` so the MCP server can verify the TLS handshake. Never set `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true` outside local development — it disables all certificate validation.
|
||||
|
||||
## Setting Up with Claude Desktop
|
||||
|
||||
Add this to your Claude Desktop MCP configuration file (`~/Library/Application Support/Claude/claude_desktop_config.json` on macOS, `%APPDATA%\Claude\claude_desktop_config.json` on Windows):
|
||||
@@ -48,7 +51,8 @@ Add this to your Claude Desktop MCP configuration file (`~/Library/Application S
|
||||
"certctl": {
|
||||
"command": "/path/to/certctl-mcp",
|
||||
"env": {
|
||||
"CERTCTL_SERVER_URL": "http://localhost:8443",
|
||||
"CERTCTL_SERVER_URL": "https://localhost:8443",
|
||||
"CERTCTL_SERVER_CA_BUNDLE_PATH": "/path/to/certctl/deploy/test/certs/ca.crt",
|
||||
"CERTCTL_API_KEY": "your-api-key-here"
|
||||
}
|
||||
}
|
||||
@@ -67,7 +71,8 @@ In Cursor, go to Settings → MCP Servers and add:
|
||||
"certctl": {
|
||||
"command": "/path/to/certctl-mcp",
|
||||
"env": {
|
||||
"CERTCTL_SERVER_URL": "http://localhost:8443",
|
||||
"CERTCTL_SERVER_URL": "https://localhost:8443",
|
||||
"CERTCTL_SERVER_CA_BUNDLE_PATH": "/path/to/certctl/deploy/test/certs/ca.crt",
|
||||
"CERTCTL_API_KEY": "your-api-key-here"
|
||||
}
|
||||
}
|
||||
@@ -84,7 +89,8 @@ Add certctl as an MCP server in your project's `.mcp.json`:
|
||||
"certctl": {
|
||||
"command": "/path/to/certctl-mcp",
|
||||
"env": {
|
||||
"CERTCTL_SERVER_URL": "http://localhost:8443",
|
||||
"CERTCTL_SERVER_URL": "https://localhost:8443",
|
||||
"CERTCTL_SERVER_CA_BUNDLE_PATH": "/path/to/certctl/deploy/test/certs/ca.crt",
|
||||
"CERTCTL_API_KEY": "your-api-key-here"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ cd certctl/deploy
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Access the dashboard at `http://localhost:8443` with API key from `.env` file.
|
||||
Access the dashboard at `https://localhost:8443` with the API key from `.env`. The default compose stack ships a self-signed cert; pin with `--cacert ./deploy/test/certs/ca.crt` when calling the API from the host.
|
||||
|
||||
### 2. Deploy Agents
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ Option A: Docker Compose (quickest for evaluation)
|
||||
```bash
|
||||
cd /opt/certctl
|
||||
docker compose up -d
|
||||
# Dashboard & API: http://localhost:8443
|
||||
# Dashboard & API: https://localhost:8443 (self-signed cert — use --cacert ./deploy/test/certs/ca.crt for the default compose stack)
|
||||
# Default API key in logs (grep CERTCTL_API_KEY docker logs certctl-server)
|
||||
```
|
||||
|
||||
@@ -45,7 +45,8 @@ chmod +x /usr/local/bin/certctl-agent
|
||||
# Create config
|
||||
sudo mkdir -p /etc/certctl /var/lib/certctl/keys
|
||||
sudo tee /etc/certctl/agent.env > /dev/null <<EOF
|
||||
CERTCTL_SERVER_URL=http://certctl-control-plane.example.com:8443
|
||||
CERTCTL_SERVER_URL=https://certctl-control-plane.example.com:8443
|
||||
CERTCTL_SERVER_CA_BUNDLE_PATH=/etc/certctl/tls/ca.crt
|
||||
CERTCTL_API_KEY=your-api-key-here
|
||||
CERTCTL_DISCOVERY_DIRS=/etc/letsencrypt/live
|
||||
CERTCTL_KEY_DIR=/var/lib/certctl/keys
|
||||
|
||||
+9
-4
@@ -68,8 +68,10 @@ The spec organizes endpoints into 16 tags:
|
||||
The spec declares a `bearerAuth` security scheme applied globally. All endpoints under `/api/v1/` require a Bearer token by default:
|
||||
|
||||
```bash
|
||||
curl -H "Authorization: Bearer your-api-key" \
|
||||
http://localhost:8443/api/v1/certificates
|
||||
# The default compose stack uses a self-signed cert; pin with --cacert
|
||||
curl --cacert ./deploy/test/certs/ca.crt \
|
||||
-H "Authorization: Bearer your-api-key" \
|
||||
https://localhost:8443/api/v1/certificates
|
||||
```
|
||||
|
||||
Three endpoints are exempt from auth (declared with `security: []` in the spec): `/health`, `/ready`, and `/api/v1/auth/info`. The auth info endpoint tells clients whether authentication is enabled and what type is required — useful for GUIs that need to show/hide a login screen.
|
||||
@@ -150,8 +152,9 @@ Import the spec directly into Postman:
|
||||
|
||||
1. Open Postman → Import → File → select `api/openapi.yaml`
|
||||
2. Postman creates a collection with all 78 documented operations organized by tag
|
||||
3. Set the `baseUrl` variable to `http://localhost:8443`
|
||||
3. Set the `baseUrl` variable to `https://localhost:8443` (HTTPS-only as of v2.2)
|
||||
4. Add an `Authorization: Bearer your-api-key` header to the collection
|
||||
5. Import the demo stack CA bundle (`deploy/test/certs/ca.crt`) into Postman's Settings → Certificates → CA Certificates, or disable certificate verification for the `localhost` host (Settings → General → SSL certificate verification)
|
||||
|
||||
## Key Schemas
|
||||
|
||||
@@ -176,8 +179,10 @@ Use the spec to generate contract tests that verify the API matches the spec:
|
||||
```bash
|
||||
# Using schemathesis for fuzz testing against the spec
|
||||
pip install schemathesis
|
||||
# The default compose stack uses a self-signed cert — export a CA bundle or set REQUESTS_CA_BUNDLE
|
||||
export REQUESTS_CA_BUNDLE=$(pwd)/deploy/test/certs/ca.crt
|
||||
schemathesis run api/openapi.yaml \
|
||||
--base-url http://localhost:8443 \
|
||||
--base-url https://localhost:8443 \
|
||||
--header "Authorization: Bearer your-api-key"
|
||||
```
|
||||
|
||||
|
||||
@@ -85,10 +85,12 @@ go test -tags qa -v -timeout 10m ./...
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `CERTCTL_QA_SERVER_URL` | `http://localhost:8443` | certctl server URL |
|
||||
| `CERTCTL_QA_SERVER_URL` | `https://localhost:8443` | certctl server URL (HTTPS-only as of v2.2) |
|
||||
| `CERTCTL_QA_API_KEY` | `change-me-in-production` | API key for Bearer auth |
|
||||
| `CERTCTL_QA_DB_URL` | `postgres://certctl:certctl@localhost:5432/certctl?sslmode=disable` | PostgreSQL connection string |
|
||||
| `CERTCTL_QA_REPO_DIR` | `../..` | Path to certctl repo root (for source file checks) |
|
||||
| `CERTCTL_QA_CA_BUNDLE` | `./certs/ca.crt` | PEM CA bundle pinned for TLS verification. The demo stack's `certctl-tls-init` container writes here. |
|
||||
| `CERTCTL_QA_INSECURE` | `false` | Set to `"true"` to skip TLS verification (e.g. before the init container finishes). Never use outside the demo harness. |
|
||||
|
||||
## Part-by-Part Coverage Map
|
||||
|
||||
@@ -256,8 +258,8 @@ docker compose -f docker-compose.yml -f docker-compose.demo.yml ps
|
||||
# Check server logs
|
||||
docker compose -f docker-compose.yml -f docker-compose.demo.yml logs certctl-server
|
||||
|
||||
# Check if the port is exposed
|
||||
curl -s http://localhost:8443/health
|
||||
# Check if the port is exposed (self-signed cert — pin CA bundle)
|
||||
curl --cacert ./deploy/test/certs/ca.crt -s https://localhost:8443/health
|
||||
```
|
||||
|
||||
### "connect to QA DB" failure
|
||||
|
||||
+59
-46
@@ -105,16 +105,24 @@ certctl-server Up (healthy)
|
||||
certctl-agent Up
|
||||
```
|
||||
|
||||
The control plane is HTTPS-only as of v2.2. The `certctl-tls-init` init container in the shipped `deploy/docker-compose.yml` self-signs a cert on first boot and drops it into a named volume. Extract the CA bundle once and reuse it for every API call in this guide:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8443/health
|
||||
export CA=/tmp/certctl-ca.crt
|
||||
docker compose -f deploy/docker-compose.yml exec -T certctl-server \
|
||||
cat /etc/certctl/tls/ca.crt > "$CA"
|
||||
|
||||
curl --cacert "$CA" https://localhost:8443/health
|
||||
```
|
||||
```json
|
||||
{"status":"healthy"}
|
||||
```
|
||||
|
||||
If you're bringing your own cert (internal CA, cert-manager, operator-supplied Secret), see [`docs/tls.md`](tls.md) for the full provisioning matrix. If you're cutting over an existing install, see [`docs/upgrade-to-tls.md`](upgrade-to-tls.md) for the failure modes (out-of-date `http://…` agents fail at the TLS handshake) and the one-step procedure.
|
||||
|
||||
## Open the Dashboard
|
||||
|
||||
Open **http://localhost:8443** in your browser.
|
||||
Open **https://localhost:8443** in your browser. Your browser will warn about the self-signed cert — that's expected for the demo bootstrap. Trust the CA bundle you just exported, or click through the warning.
|
||||
|
||||
> **Note:** The Docker Compose demo runs with authentication disabled (`CERTCTL_AUTH_TYPE=none`) so you can explore immediately. For production, set `CERTCTL_AUTH_TYPE=api-key` and `CERTCTL_AUTH_SECRET=<your-secret>` in your environment, then pass `Authorization: Bearer <your-secret>` on all API requests. The dashboard will prompt for your API key on first load.
|
||||
>
|
||||
@@ -154,62 +162,64 @@ Everything you see in the dashboard is backed by the REST API. All endpoints liv
|
||||
|
||||
### Core operations
|
||||
|
||||
Every request below uses `--cacert "$CA"` to pin the self-signed CA bundle extracted above. In production, point `$CA` at your internal CA root or the bundle you distributed to the fleet.
|
||||
|
||||
```bash
|
||||
# List all certificates
|
||||
curl -s http://localhost:8443/api/v1/certificates | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/certificates | jq .
|
||||
|
||||
# Filter by status
|
||||
curl -s "http://localhost:8443/api/v1/certificates?status=Expiring" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?status=Expiring" | jq .
|
||||
|
||||
# Filter by environment
|
||||
curl -s "http://localhost:8443/api/v1/certificates?environment=production" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?environment=production" | jq .
|
||||
|
||||
# Get a specific certificate
|
||||
curl -s http://localhost:8443/api/v1/certificates/mc-api-prod | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/certificates/mc-api-prod | jq .
|
||||
|
||||
# Get deployment targets for a certificate
|
||||
curl -s http://localhost:8443/api/v1/certificates/mc-api-prod/deployments | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/certificates/mc-api-prod/deployments | jq .
|
||||
|
||||
# List agents
|
||||
curl -s http://localhost:8443/api/v1/agents | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/agents | jq .
|
||||
|
||||
# Check agent pending work
|
||||
curl -s http://localhost:8443/api/v1/agents/ag-web-prod/work | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/agents/ag-web-prod/work | jq .
|
||||
|
||||
# View audit trail
|
||||
curl -s http://localhost:8443/api/v1/audit | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/audit | jq .
|
||||
|
||||
# View policies and violations
|
||||
curl -s http://localhost:8443/api/v1/policies | jq .
|
||||
curl -s http://localhost:8443/api/v1/policies/pr-require-owner/violations | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/policies | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/policies/pr-require-owner/violations | jq .
|
||||
|
||||
# Notifications
|
||||
curl -s http://localhost:8443/api/v1/notifications | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/notifications | jq .
|
||||
|
||||
# Profiles and agent groups
|
||||
curl -s http://localhost:8443/api/v1/profiles | jq .
|
||||
curl -s http://localhost:8443/api/v1/agent-groups | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/profiles | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/agent-groups | jq .
|
||||
```
|
||||
|
||||
### Sorting, filtering, and pagination
|
||||
|
||||
```bash
|
||||
# Sort by expiration date (ascending)
|
||||
curl -s "http://localhost:8443/api/v1/certificates?sort=notAfter" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?sort=notAfter" | jq .
|
||||
|
||||
# Sort descending (prefix with -)
|
||||
curl -s "http://localhost:8443/api/v1/certificates?sort=-createdAt" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?sort=-createdAt" | jq .
|
||||
|
||||
# Time-range filters (RFC3339)
|
||||
curl -s "http://localhost:8443/api/v1/certificates?expires_before=2026-05-01T00:00:00Z" | jq .
|
||||
curl -s "http://localhost:8443/api/v1/certificates?created_after=2026-03-01T00:00:00Z" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?expires_before=2026-05-01T00:00:00Z" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?created_after=2026-03-01T00:00:00Z" | jq .
|
||||
|
||||
# Sparse fields — request only what you need
|
||||
curl -s "http://localhost:8443/api/v1/certificates?fields=id,common_name,status,expires_at" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?fields=id,common_name,status,expires_at" | jq .
|
||||
|
||||
# Cursor pagination — efficient for large inventories
|
||||
curl -s "http://localhost:8443/api/v1/certificates?page_size=5" | jq '{next_cursor: .next_cursor, count: (.data | length)}'
|
||||
curl -s "http://localhost:8443/api/v1/certificates?cursor=<next_cursor_value>&page_size=5" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?page_size=5" | jq '{next_cursor: .next_cursor, count: (.data | length)}'
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/certificates?cursor=<next_cursor_value>&page_size=5" | jq .
|
||||
```
|
||||
|
||||
Supported sort fields: `notAfter`, `expiresAt`, `createdAt`, `updatedAt`, `commonName`, `name`, `status`, `environment`.
|
||||
@@ -218,22 +228,22 @@ Supported sort fields: `notAfter`, `expiresAt`, `createdAt`, `updatedAt`, `commo
|
||||
|
||||
```bash
|
||||
# Dashboard summary
|
||||
curl -s http://localhost:8443/api/v1/stats/summary | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/stats/summary | jq .
|
||||
|
||||
# Certificates by status
|
||||
curl -s http://localhost:8443/api/v1/stats/certificates-by-status | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/stats/certificates-by-status | jq .
|
||||
|
||||
# Expiration timeline (next 90 days)
|
||||
curl -s "http://localhost:8443/api/v1/stats/expiration-timeline?days=90" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/stats/expiration-timeline?days=90" | jq .
|
||||
|
||||
# Job trends (last 30 days)
|
||||
curl -s "http://localhost:8443/api/v1/stats/job-trends?days=30" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/stats/job-trends?days=30" | jq .
|
||||
|
||||
# JSON metrics
|
||||
curl -s http://localhost:8443/api/v1/metrics | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/metrics | jq .
|
||||
|
||||
# Prometheus format (for Prometheus, Grafana Agent, Datadog)
|
||||
curl -s http://localhost:8443/api/v1/metrics/prometheus
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/metrics/prometheus
|
||||
```
|
||||
|
||||
## Create Your First Certificate
|
||||
@@ -241,7 +251,7 @@ curl -s http://localhost:8443/api/v1/metrics/prometheus
|
||||
Create a certificate record that certctl will track, renew, and deploy automatically.
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "My First Certificate",
|
||||
@@ -264,22 +274,22 @@ CERT_ID="<paste the id from the response>"
|
||||
|
||||
Trigger renewal:
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/$CERT_ID/renew | jq .
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/$CERT_ID/renew | jq .
|
||||
```
|
||||
|
||||
Check the result:
|
||||
```bash
|
||||
curl -s http://localhost:8443/api/v1/certificates/$CERT_ID | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/certificates/$CERT_ID | jq .
|
||||
```
|
||||
|
||||
Refresh the dashboard at http://localhost:8443 — your new certificate appears in the inventory.
|
||||
Refresh the dashboard at https://localhost:8443 — your new certificate appears in the inventory.
|
||||
|
||||
### Revoke a certificate
|
||||
|
||||
When a private key is compromised or a service is decommissioned:
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/$CERT_ID/revoke \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/$CERT_ID/revoke \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"reason": "superseded"}' | jq .
|
||||
```
|
||||
@@ -289,7 +299,8 @@ Supported RFC 5280 reason codes: `unspecified`, `keyCompromise`, `caCompromise`,
|
||||
Confirm via the unauthenticated DER CRL (RFC 5280 §5, RFC 8615):
|
||||
```bash
|
||||
# Fetch the CRL without any API key — relying parties shouldn't need one.
|
||||
curl -s http://localhost:8443/.well-known/pki/crl/iss-local -o /tmp/crl.der
|
||||
# The CRL path is unauthenticated, but it's still served over TLS.
|
||||
curl --cacert "$CA" -s https://localhost:8443/.well-known/pki/crl/iss-local -o /tmp/crl.der
|
||||
openssl crl -inform der -in /tmp/crl.der -noout -text | head -40
|
||||
```
|
||||
|
||||
@@ -299,15 +310,15 @@ For high-value certificates where you want human oversight. The demo includes 2
|
||||
|
||||
```bash
|
||||
# List jobs awaiting approval (demo includes 2)
|
||||
curl -s "http://localhost:8443/api/v1/jobs?status=AwaitingApproval" | jq '.data[] | {id, certificate_id, status}'
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/jobs?status=AwaitingApproval" | jq '.data[] | {id, certificate_id, status}'
|
||||
|
||||
# Approve a pending job
|
||||
curl -s -X POST http://localhost:8443/api/v1/jobs/JOB_ID/approve \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/jobs/JOB_ID/approve \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"reason": "Approved for production deployment"}' | jq .
|
||||
|
||||
# Reject a pending job
|
||||
curl -s -X POST http://localhost:8443/api/v1/jobs/JOB_ID/reject \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/jobs/JOB_ID/reject \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"reason": "Key type does not meet compliance requirements"}' | jq .
|
||||
```
|
||||
@@ -333,7 +344,7 @@ export CERTCTL_DISCOVERY_DIRS="/etc/nginx/certs,/etc/ssl/certs,/var/lib/certs"
|
||||
export CERTCTL_NETWORK_SCAN_ENABLED=true
|
||||
|
||||
# Create a scan target
|
||||
curl -s -X POST http://localhost:8443/api/v1/network-scan-targets \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/network-scan-targets \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "Internal Network",
|
||||
@@ -345,20 +356,20 @@ curl -s -X POST http://localhost:8443/api/v1/network-scan-targets \
|
||||
}' | jq .
|
||||
|
||||
# Trigger an immediate scan
|
||||
curl -s -X POST http://localhost:8443/api/v1/network-scan-targets/nst-internal-network/scan | jq .
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/network-scan-targets/nst-internal-network/scan | jq .
|
||||
```
|
||||
|
||||
### Triage discovered certificates
|
||||
|
||||
```bash
|
||||
# List discovered certs
|
||||
curl -s "http://localhost:8443/api/v1/discovered-certificates?agent_id=agent-nginx-prod" | jq .
|
||||
curl --cacert "$CA" -s "https://localhost:8443/api/v1/discovered-certificates?agent_id=agent-nginx-prod" | jq .
|
||||
|
||||
# Summary counts
|
||||
curl -s http://localhost:8443/api/v1/discovery-summary | jq .
|
||||
curl --cacert "$CA" -s https://localhost:8443/api/v1/discovery-summary | jq .
|
||||
|
||||
# Claim a discovered cert (bring under management)
|
||||
curl -s -X POST "http://localhost:8443/api/v1/discovered-certificates/DISCOVERY_ID/claim" \
|
||||
curl --cacert "$CA" -s -X POST "https://localhost:8443/api/v1/discovered-certificates/DISCOVERY_ID/claim" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"managed_certificate_id": "mc-api-prod"}' | jq .
|
||||
```
|
||||
@@ -368,8 +379,9 @@ curl -s -X POST "http://localhost:8443/api/v1/discovered-certificates/DISCOVERY_
|
||||
```bash
|
||||
cd cmd/cli && go build -o certctl-cli .
|
||||
|
||||
export CERTCTL_SERVER_URL="http://localhost:8443"
|
||||
export CERTCTL_SERVER_URL="https://localhost:8443"
|
||||
export CERTCTL_API_KEY="test-key-123"
|
||||
export CERTCTL_SERVER_CA_BUNDLE_PATH="$CA" # or pass --ca-bundle; --insecure for dev self-signed
|
||||
|
||||
./certctl-cli certs list # List certificates
|
||||
./certctl-cli certs get mc-api-prod # Certificate details
|
||||
@@ -402,10 +414,10 @@ export CERTCTL_DIGEST_RECIPIENTS=ops@example.com,security@example.com
|
||||
|
||||
Preview the digest HTML before enabling scheduled delivery:
|
||||
```bash
|
||||
curl http://localhost:8443/api/v1/digest/preview | jq '.html' | grep -o '<html>' # Shows HTML is ready
|
||||
curl --cacert "$CA" https://localhost:8443/api/v1/digest/preview | jq '.html' | grep -o '<html>' # Shows HTML is ready
|
||||
|
||||
# Trigger a digest send immediately (outside of schedule)
|
||||
curl -X POST http://localhost:8443/api/v1/digest/send
|
||||
curl --cacert "$CA" -X POST https://localhost:8443/api/v1/digest/send
|
||||
```
|
||||
|
||||
If no recipients are configured (`CERTCTL_DIGEST_RECIPIENTS` empty), the digest falls back to certificate owner emails. Digests include total certificates, expiring soon, expired, active agents, completed/failed jobs (30-day summary), and a table of expiring certs color-coded by urgency (7/14/30 days).
|
||||
@@ -415,8 +427,9 @@ If no recipients are configured (`CERTCTL_DIGEST_RECIPIENTS` empty), the digest
|
||||
```bash
|
||||
cd cmd/mcp-server && go build -o mcp-server .
|
||||
|
||||
export CERTCTL_SERVER_URL="http://localhost:8443"
|
||||
export CERTCTL_SERVER_URL="https://localhost:8443"
|
||||
export CERTCTL_API_KEY="test-key-123"
|
||||
export CERTCTL_SERVER_CA_BUNDLE_PATH="$CA" # MCP is env-vars-only; no CLI flags
|
||||
|
||||
./mcp-server
|
||||
```
|
||||
|
||||
+64
-47
@@ -16,7 +16,7 @@ You'll start 7 Docker containers that talk to each other:
|
||||
| **pebble-challtestsrv** | DNS/HTTP challenge test server for Pebble | 10.30.50.3 | Not directly — Pebble talks to it |
|
||||
| **Pebble** | A fake Let's Encrypt (tests the ACME protocol without touching the real internet) | 10.30.50.4 | Not directly — the server talks to it |
|
||||
| **step-ca** | A private Certificate Authority (think: your company's internal CA) | 10.30.50.5 | Not directly — the server talks to it |
|
||||
| **certctl-server** | The brain. API + web dashboard + scheduler + ACME challenge server | 10.30.50.6 | **http://localhost:8443** |
|
||||
| **certctl-server** | The brain. API + web dashboard + scheduler + ACME challenge server | 10.30.50.6 | **https://localhost:8443** (self-signed — see CA-bundle note below) |
|
||||
| **NGINX** | A web server. The agent deploys certificates here. | 10.30.50.7 | **https://localhost:8444** |
|
||||
| **certctl-agent** | The hands. Generates keys, deploys certs to NGINX | 10.30.50.8 | Not directly — it talks to the server |
|
||||
|
||||
@@ -123,7 +123,7 @@ docker compose -f docker-compose.test.yml up --build
|
||||
|
||||
```
|
||||
certctl-test-server | {"level":"INFO","msg":"server started","address":"0.0.0.0:8443"}
|
||||
certctl-test-agent | {"level":"INFO","msg":"agent starting","server_url":"http://certctl-server:8443"}
|
||||
certctl-test-agent | {"level":"INFO","msg":"agent starting","server_url":"https://certctl-server:8443"}
|
||||
certctl-test-stepca | Serving HTTPS on :9000 ...
|
||||
certctl-test-pebble | Listening on: 0.0.0.0:14000
|
||||
```
|
||||
@@ -159,13 +159,29 @@ certctl-test-stepca Up (healthy)
|
||||
|
||||
**If certctl-test-server says "Restarting"**: It probably started before step-ca or Pebble were ready. Wait 30 seconds and check again. If it keeps restarting, see [Troubleshooting](#troubleshooting).
|
||||
|
||||
### Get the CA bundle for curl
|
||||
|
||||
The test harness runs HTTPS-only (the `certctl-tls-init` init container self-signs an ed25519 server cert into a bind-mounted directory before the server starts — see `docker-compose.test.yml` §`certctl-tls-init` for details). The CA cert that signed it is materialized on the host at `./test/certs/ca.crt` (relative to the `deploy/` directory). Every `curl` in the rest of this doc expects it in `$CA`:
|
||||
|
||||
```bash
|
||||
export CA=$PWD/test/certs/ca.crt
|
||||
ls -la "$CA" # sanity check: file should exist and be non-empty
|
||||
curl --cacert "$CA" -f https://localhost:8443/health
|
||||
```
|
||||
|
||||
Expect `{"status":"ok"}`. If `curl` errors with `SSL certificate problem: unable to get local issuer certificate`, the init container hasn't finished yet — wait a few seconds and retry. If the file doesn't exist at all, the bind mount didn't populate; `docker compose -f docker-compose.test.yml logs certctl-tls-init` should show the self-sign ran.
|
||||
|
||||
For a full explanation of the cert provisioning patterns (self-signed bootstrap, operator-supplied, cert-manager), see [`tls.md`](tls.md). For the one-step cutover from the old plaintext test harness to HTTPS, see [`upgrade-to-tls.md`](upgrade-to-tls.md).
|
||||
|
||||
---
|
||||
|
||||
## Step 2: Open the Dashboard
|
||||
|
||||
Open your web browser and go to:
|
||||
|
||||
**http://localhost:8443**
|
||||
**https://localhost:8443**
|
||||
|
||||
Your browser will warn you that the cert is self-signed ("Your connection is not private" / "NET::ERR_CERT_AUTHORITY_INVALID"). That's expected for the test harness — the CA that signed the cert lives at `deploy/test/certs/ca.crt` and isn't in your system trust store. Click through the warning (Chrome: "Advanced" → "Proceed"; Firefox: "Accept the Risk"; Safari: "Show Details" → "visit this website").
|
||||
|
||||
You'll see a login screen asking for an API key. Enter:
|
||||
|
||||
@@ -198,12 +214,13 @@ Go back to your second terminal. Let's verify the data loaded correctly.
|
||||
### Check the agent
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/agents | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/agents | python3 -m json.tool
|
||||
```
|
||||
|
||||
**What this command does**:
|
||||
- `curl` makes an HTTP request (like a browser but from the terminal)
|
||||
- `curl` makes an HTTPS request (like a browser but from the terminal)
|
||||
- `--cacert "$CA"` pins the test harness's self-signed root as the only trust anchor for this call — matches what you exported in Step 1
|
||||
- `-s` means "silent" (don't show progress bars)
|
||||
- `-H "Authorization: Bearer test-key-2026"` sends the API key (same one you used to log in)
|
||||
- `python3 -m json.tool` formats the JSON response so it's readable
|
||||
@@ -233,8 +250,8 @@ The important parts: `"id": "agent-test-01"` and `"status": "online"`. If the st
|
||||
### Check the issuers
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/issuers | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/issuers | python3 -m json.tool
|
||||
```
|
||||
|
||||
You should see three issuers:
|
||||
@@ -245,8 +262,8 @@ You should see three issuers:
|
||||
### Check the target
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/targets | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/targets | python3 -m json.tool
|
||||
```
|
||||
|
||||
You should see `target-test-nginx` — the NGINX deployment target, assigned to `agent-test-01`.
|
||||
@@ -255,7 +272,7 @@ The target config uses no-op commands for `reload_command` and `validate_command
|
||||
|
||||
### See it all in the dashboard
|
||||
|
||||
Open the dashboard at http://localhost:8443 and click through the sidebar:
|
||||
Open the dashboard at https://localhost:8443 and click through the sidebar:
|
||||
- **Agents** — you should see `test-agent-01`
|
||||
- **Issuers** — you should see all three CAs
|
||||
- **Targets** — you should see `Test NGINX`
|
||||
@@ -287,7 +304,7 @@ The private key **never leaves the agent**. The server only ever sees the CSR (p
|
||||
### Step 4a: Create the certificate record
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H "Authorization: Bearer test-key-2026" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
@@ -338,7 +355,7 @@ docker exec certctl-test-postgres psql -U certctl -d certctl -c \
|
||||
### Step 4c: Trigger issuance
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/mc-local-test/renew \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/mc-local-test/renew \
|
||||
-H "Authorization: Bearer test-key-2026" | python3 -m json.tool
|
||||
```
|
||||
|
||||
@@ -395,7 +412,7 @@ The `subject` should match the domain name you chose. The `issuer` should say "c
|
||||
|
||||
### Step 4f: Check the dashboard
|
||||
|
||||
Open the dashboard at http://localhost:8443 and:
|
||||
Open the dashboard at https://localhost:8443 and:
|
||||
|
||||
1. Click **Certificates** in the sidebar — you should see `mc-local-test` with status "Active"
|
||||
2. Click on it to see the detail page — you should see version history, the signed certificate details, and the deployment timeline
|
||||
@@ -414,7 +431,7 @@ This is the real deal. ACME is the protocol that Let's Encrypt uses to issue cer
|
||||
### Step 5a: Create the certificate record
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H "Authorization: Bearer test-key-2026" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
@@ -441,7 +458,7 @@ docker exec certctl-test-postgres psql -U certctl -d certctl -c \
|
||||
"INSERT INTO certificate_target_mappings (certificate_id, target_id) VALUES ('mc-acme-test', 'target-test-nginx') ON CONFLICT DO NOTHING;"
|
||||
|
||||
# Trigger issuance
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/mc-acme-test/renew \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/mc-acme-test/renew \
|
||||
-H "Authorization: Bearer test-key-2026" | python3 -m json.tool
|
||||
```
|
||||
|
||||
@@ -502,7 +519,7 @@ Revocation means "this certificate is no longer trusted, even though it hasn't e
|
||||
### Step 7a: Revoke the Local CA cert
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/mc-local-test/revoke \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/mc-local-test/revoke \
|
||||
-H "Authorization: Bearer test-key-2026" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"reason": "superseded"}' | python3 -m json.tool
|
||||
@@ -516,7 +533,7 @@ The CRL is a DER-encoded X.509 v2 CRL (RFC 5280 §5) served under the RFC 8615 w
|
||||
|
||||
```bash
|
||||
# No Authorization header — the endpoint is public by design.
|
||||
curl -s http://localhost:8443/.well-known/pki/crl/iss-local -o /tmp/crl.der
|
||||
curl --cacert "$CA" -s https://localhost:8443/.well-known/pki/crl/iss-local -o /tmp/crl.der
|
||||
openssl crl -inform der -in /tmp/crl.der -noout -text | head -40
|
||||
```
|
||||
|
||||
@@ -533,8 +550,8 @@ Go to **Certificates** in the sidebar. The `mc-local-test` cert should now show
|
||||
The agent is configured to scan `/nginx-certs` every 6 hours for existing certificates. It already ran a scan when it started up. Let's see what it found.
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/discovered-certificates | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/discovered-certificates | python3 -m json.tool
|
||||
```
|
||||
|
||||
**What you should see**: Any certificates that exist in the NGINX cert directory, including the ones you deployed in Steps 4-5. The discovery system extracts metadata (CN, SANs, issuer, expiry, fingerprint) from the PEM files.
|
||||
@@ -542,8 +559,8 @@ curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
Check the summary:
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/discovery-summary | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/discovery-summary | python3 -m json.tool
|
||||
```
|
||||
|
||||
This shows counts: how many are Unmanaged, Managed, and Dismissed.
|
||||
@@ -557,7 +574,7 @@ In the dashboard: click **Discovery** in the sidebar to see the triage view.
|
||||
Force a renewal on the ACME certificate to see the full cycle happen again:
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/mc-acme-test/renew \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/mc-acme-test/renew \
|
||||
-H "Authorization: Bearer test-key-2026" | python3 -m json.tool
|
||||
```
|
||||
|
||||
@@ -584,7 +601,7 @@ The test environment enables EST with `CERTCTL_EST_ENABLED=true` and `CERTCTL_ES
|
||||
### Step 10a: Check available CA certificates
|
||||
|
||||
```bash
|
||||
curl -sk http://localhost:8443/.well-known/est/cacerts \
|
||||
curl --cacert "$CA" -s https://localhost:8443/.well-known/est/cacerts \
|
||||
-H "Authorization: Bearer test-key-2026"
|
||||
```
|
||||
|
||||
@@ -595,7 +612,7 @@ curl -sk http://localhost:8443/.well-known/est/cacerts \
|
||||
### Step 10b: Check CSR attributes
|
||||
|
||||
```bash
|
||||
curl -sk http://localhost:8443/.well-known/est/csrattrs \
|
||||
curl --cacert "$CA" -s https://localhost:8443/.well-known/est/csrattrs \
|
||||
-H "Authorization: Bearer test-key-2026"
|
||||
```
|
||||
|
||||
@@ -615,7 +632,7 @@ openssl req -new -newkey ec -pkeyopt ec_paramgen_curve:P-256 \
|
||||
EST_CSR=$(openssl req -in /tmp/est-test.csr -outform DER | base64 -w 0)
|
||||
|
||||
# Submit to EST simpleenroll endpoint
|
||||
curl -sk -X POST http://localhost:8443/.well-known/est/simpleenroll \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/.well-known/est/simpleenroll \
|
||||
-H "Authorization: Bearer test-key-2026" \
|
||||
-H "Content-Type: application/pkcs10" \
|
||||
-d "$EST_CSR"
|
||||
@@ -628,8 +645,8 @@ curl -sk -X POST http://localhost:8443/.well-known/est/simpleenroll \
|
||||
Decode and inspect the response (if you saved it to a variable):
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/audit-events | python3 -m json.tool | head -30
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/audit-events | python3 -m json.tool | head -30
|
||||
```
|
||||
|
||||
Check the audit trail — you should see an `est_enrollment` event with the CN `est-device.certctl.test`.
|
||||
@@ -639,7 +656,7 @@ Check the audit trail — you should see an `est_enrollment` event with the CN `
|
||||
EST also supports re-enrollment (certificate renewal). The same CSR format works:
|
||||
|
||||
```bash
|
||||
curl -sk -X POST http://localhost:8443/.well-known/est/simplereenroll \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/.well-known/est/simplereenroll \
|
||||
-H "Authorization: Bearer test-key-2026" \
|
||||
-H "Content-Type: application/pkcs10" \
|
||||
-d "$EST_CSR"
|
||||
@@ -658,7 +675,7 @@ S/MIME certificates are used for email signing and encryption — a different us
|
||||
### Step 11a: Create an S/MIME certificate record
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H "Authorization: Bearer test-key-2026" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
@@ -686,7 +703,7 @@ Notice:
|
||||
docker exec certctl-test-postgres psql -U certctl -d certctl -c \
|
||||
"INSERT INTO certificate_target_mappings (certificate_id, target_id) VALUES ('mc-smime-test', 'target-test-nginx') ON CONFLICT DO NOTHING;"
|
||||
|
||||
curl -s -X POST http://localhost:8443/api/v1/certificates/mc-smime-test/renew \
|
||||
curl --cacert "$CA" -s -X POST https://localhost:8443/api/v1/certificates/mc-smime-test/renew \
|
||||
-H "Authorization: Bearer test-key-2026" | python3 -m json.tool
|
||||
```
|
||||
|
||||
@@ -695,15 +712,15 @@ curl -s -X POST http://localhost:8443/api/v1/certificates/mc-smime-test/renew \
|
||||
After the agent processes the job (30-60 seconds), check the certificate details:
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/certificates/mc-smime-test | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/certificates/mc-smime-test | python3 -m json.tool
|
||||
```
|
||||
|
||||
The certificate should show `"status": "active"`. To verify the EKU on the actual cert, you can export it:
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/certificates/mc-smime-test/export/pem | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/certificates/mc-smime-test/export/pem | python3 -m json.tool
|
||||
```
|
||||
|
||||
If you decode the certificate PEM, you should see:
|
||||
@@ -768,16 +785,16 @@ If you have Go installed, you can build and test the CLI tool:
|
||||
go build -o certctl-cli ./cmd/cli
|
||||
|
||||
# List certificates
|
||||
./certctl-cli --server http://localhost:8443 --api-key test-key-2026 list-certs
|
||||
./certctl-cli --server https://localhost:8443 --ca-bundle "$CA" --api-key test-key-2026 list-certs
|
||||
|
||||
# Get a specific certificate
|
||||
./certctl-cli --server http://localhost:8443 --api-key test-key-2026 get-cert mc-acme-test
|
||||
./certctl-cli --server https://localhost:8443 --ca-bundle "$CA" --api-key test-key-2026 get-cert mc-acme-test
|
||||
|
||||
# Check health
|
||||
./certctl-cli --server http://localhost:8443 --api-key test-key-2026 health
|
||||
./certctl-cli --server https://localhost:8443 --ca-bundle "$CA" --api-key test-key-2026 health
|
||||
|
||||
# Get metrics (JSON format)
|
||||
./certctl-cli --server http://localhost:8443 --api-key test-key-2026 --format json metrics
|
||||
./certctl-cli --server https://localhost:8443 --ca-bundle "$CA" --api-key test-key-2026 --format json metrics
|
||||
```
|
||||
|
||||
---
|
||||
@@ -924,15 +941,15 @@ Look for error messages. Common ones:
|
||||
**Step 2**: Verify the agent is registered:
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
http://localhost:8443/api/v1/agents/agent-test-01 | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
https://localhost:8443/api/v1/agents/agent-test-01 | python3 -m json.tool
|
||||
```
|
||||
|
||||
**Step 3**: Check for pending jobs:
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
"http://localhost:8443/api/v1/jobs?status=Pending&status=AwaitingCSR" | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
"https://localhost:8443/api/v1/jobs?status=Pending&status=AwaitingCSR" | python3 -m json.tool
|
||||
```
|
||||
|
||||
If there are pending jobs but the agent isn't picking them up, check that the job's `agent_id` matches `agent-test-01`.
|
||||
@@ -962,8 +979,8 @@ docker exec certctl-test-nginx nginx -s reload
|
||||
**Step 3**: If the files aren't there, the deployment job hasn't completed. Check the jobs:
|
||||
|
||||
```bash
|
||||
curl -s -H "Authorization: Bearer test-key-2026" \
|
||||
"http://localhost:8443/api/v1/jobs?type=Deployment" | python3 -m json.tool
|
||||
curl --cacert "$CA" -s -H "Authorization: Bearer test-key-2026" \
|
||||
"https://localhost:8443/api/v1/jobs?type=Deployment" | python3 -m json.tool
|
||||
```
|
||||
|
||||
Look at the job status. If it's "Running" and stuck, the server's job processor may have picked it up instead of the agent (this was a known bug — the fix skips deployment jobs with `agent_id` in the server's `ProcessPendingJobs`).
|
||||
@@ -1008,7 +1025,7 @@ Change it to a different port, like:
|
||||
- "9443:8443"
|
||||
```
|
||||
|
||||
Then access the dashboard at http://localhost:9443 instead.
|
||||
Then access the dashboard at https://localhost:9443 instead.
|
||||
|
||||
### Starting completely fresh
|
||||
|
||||
@@ -1054,7 +1071,7 @@ docker compose -f docker-compose.test.yml up --build
|
||||
|
||||
| What | Value |
|
||||
|---|---|
|
||||
| Dashboard URL | http://localhost:8443 |
|
||||
| Dashboard URL | https://localhost:8443 (use `--cacert ./test/certs/ca.crt`) |
|
||||
| API key | `test-key-2026` |
|
||||
| NGINX HTTP | http://localhost:8080 |
|
||||
| NGINX HTTPS | https://localhost:8444 |
|
||||
|
||||
+419
-6
@@ -5002,10 +5002,10 @@ curl -s -w "HTTP %{http_code}\n" -X DELETE -H "$AUTH" "$SERVER/api/v1/audit/$EVE
|
||||
|
||||
> **Tip:** Open a second terminal with `docker compose logs -f certctl-server` to watch scheduler log output in real time.
|
||||
|
||||
**Test 20.1.1 — Scheduler startup: all 7 loops registered**
|
||||
**Test 20.1.1 — Scheduler startup: all 12 loops registered**
|
||||
|
||||
```bash
|
||||
docker compose logs certctl-server 2>&1 | grep -i "scheduler\|renewal check\|job processor\|health check\|notification\|short-lived\|network scan" | head -20
|
||||
docker compose logs certctl-server 2>&1 | grep -i "scheduler\|renewal check\|job processor\|job retry\|job timeout\|health check\|notification\|notification retry\|short-lived\|network scan\|digest\|endpoint health\|cloud discovery" | head -30
|
||||
```
|
||||
|
||||
**What:** Checks server startup logs for scheduler loop registration.
|
||||
@@ -6587,6 +6587,419 @@ helm template certctl deploy/helm/certctl/ --set server.replicaCount=3 | grep 'r
|
||||
|
||||
---
|
||||
|
||||
## Part 55: Agent Soft-Retirement (I-004)
|
||||
|
||||
**What this validates:** The full `DELETE /api/v1/agents/{id}` soft-retirement contract — seven HTTP status codes (200/204/400/403/404/405/409/500), opt-in retired-agent listing, sentinel refusal, `410 Gone` heartbeat response, and the force-cascade escape hatch.
|
||||
|
||||
**Why it matters:** Before I-004, there was no retirement surface at all — `DELETE` did not exist and agents could only be removed via raw SQL against the `agents` table. Worse, the schema declared `deployment_targets.agent_id ON DELETE CASCADE`, so any such manual delete silently cascaded through four tables with zero audit trail. This part pins the replacement contract (soft-delete + preflight + force-cascade + sentinel guard + heartbeat 410) so regressions show up here first rather than as orphaned targets in production.
|
||||
|
||||
### 55.1 Migration 000015 Applied
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='agents' AND column_name IN ('retired_at','retired_reason') ORDER BY column_name;"
|
||||
```
|
||||
|
||||
**What:** Confirms migration 000015 added the archival columns to the `agents` table.
|
||||
**PASS if** both `retired_at` and `retired_reason` rows are returned. **FAIL** if either is missing (migration did not apply).
|
||||
|
||||
---
|
||||
|
||||
### 55.2 FK Constraint Flipped to RESTRICT
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT confdeltype FROM pg_constraint WHERE conname='deployment_targets_agent_id_fkey';"
|
||||
```
|
||||
|
||||
**What:** `confdeltype` is PostgreSQL's one-character code for the FK delete action: `r` = RESTRICT, `c` = CASCADE.
|
||||
**PASS if** the value is `r`. **FAIL** if it is still `c` — that means migration 000015's FK flip did not run, and a hard `DELETE` against an agent row would silently cascade.
|
||||
|
||||
---
|
||||
|
||||
### 55.3 Clean Retire — 200
|
||||
|
||||
```bash
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/ag-test-clean" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Retires an agent that has no active deployment targets, no deployed certificates, and no pending jobs.
|
||||
**PASS if** status code is `200` and response body includes `"retired_at":"<ISO8601>"`, `"cascade":false`, and zero-valued counts.
|
||||
|
||||
---
|
||||
|
||||
### 55.4 Idempotent Re-Retire — 204
|
||||
|
||||
```bash
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/ag-test-clean" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Retires an agent that is already retired.
|
||||
**PASS if** status code is `204` and response body is completely empty (not even a trailing newline from the handler). The 200-shape must NOT be emitted — this is the terminal no-op.
|
||||
|
||||
---
|
||||
|
||||
### 55.5 Blocked by Dependencies — 409
|
||||
|
||||
```bash
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/ag-with-deps" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Attempts to retire an agent that still has active targets/certificates/jobs.
|
||||
**PASS if** status code is `409` and response body is the three-key `BlockedByDependenciesResponse` shape: `{"error":"blocked_by_dependencies", "message": "...", "counts": {"active_targets": N, "active_certificates": N, "pending_jobs": N}}`. Must NOT be the generic `ErrorResponse` shape — downstream dashboards parse the `counts` key.
|
||||
|
||||
---
|
||||
|
||||
### 55.6 Force Cascade — 200
|
||||
|
||||
```bash
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/ag-with-deps?force=true&reason=decommissioning+rack-7" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Uses the force escape hatch to cascade-retire the dependencies.
|
||||
**PASS if** status code is `200`, response includes `"cascade":true` with the pre-cascade counts, and the subsequent `GET /api/v1/audit-events?action=agent_retirement_cascaded` shows the event with the supplied `reason` and actor.
|
||||
|
||||
---
|
||||
|
||||
### 55.7 Force Without Reason — 400
|
||||
|
||||
```bash
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/ag-other?force=true" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Verifies the `ErrForceReasonRequired` guard — `force=true` without `reason` must be rejected before any state mutation.
|
||||
**PASS if** status code is `400` and no agent/target/job rows were modified.
|
||||
|
||||
---
|
||||
|
||||
### 55.8 Sentinel Refusal — 403
|
||||
|
||||
```bash
|
||||
for id in server-scanner cloud-aws-sm cloud-azure-kv cloud-gcp-sm; do
|
||||
echo "=== $id ==="
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/${id}?force=true&reason=attempt" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
done
|
||||
```
|
||||
|
||||
**What:** Verifies all four sentinel agents refuse retirement even with `force=true`.
|
||||
**PASS if** every request returns `403` and the response body's `error` value is `sentinel_agent` (or the equivalent `ErrAgentIsSentinel` mapping). **FAIL** if any sentinel accepts the request — retiring one silently orphans the network scanner or one of the three cloud secret-manager discovery sources.
|
||||
|
||||
---
|
||||
|
||||
### 55.9 Unknown ID — 404
|
||||
|
||||
```bash
|
||||
curl -sS -X DELETE "http://localhost:8443/api/v1/agents/ag-does-not-exist" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Verifies `ErrAgentNotFound` maps to 404 (not 500). Ordering matters — the not-found check must come after the sentinel check so a typo'd sentinel ID still returns 403, not 404.
|
||||
**PASS if** status code is `404`.
|
||||
|
||||
---
|
||||
|
||||
### 55.10 Heartbeat on Retired Agent — 410
|
||||
|
||||
```bash
|
||||
curl -sS -X POST "http://localhost:8443/api/v1/agents/ag-test-clean/heartbeat" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"os":"linux","architecture":"amd64","hostname":"test","ip_address":"10.0.0.1","version":"2.1.0"}' \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
```
|
||||
|
||||
**What:** Retired agents get `410 Gone` — the canonical "resource is permanently gone, stop retrying" signal — so `cmd/agent` detects it and exits cleanly.
|
||||
**PASS if** status code is `410`. **FAIL** if it is `404` (wrong ordering — retired-check must run before not-found) or `200` (retired filter missing entirely — agent would keep phoning home forever).
|
||||
|
||||
---
|
||||
|
||||
### 55.11 Default List Excludes Retired
|
||||
|
||||
```bash
|
||||
curl -sS "http://localhost:8443/api/v1/agents" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
| jq -r '.data[] | select(.id=="ag-test-clean") | .id'
|
||||
```
|
||||
|
||||
**What:** Verifies the default `/agents` listing filters retired rows via `AgentRepository.ListActive`.
|
||||
**PASS if** output is empty (the retired agent does NOT appear). **FAIL** if `ag-test-clean` shows up — default listings must not expose retired rows.
|
||||
|
||||
---
|
||||
|
||||
### 55.12 Retired Agents Opt-In View
|
||||
|
||||
```bash
|
||||
curl -sS "http://localhost:8443/api/v1/agents/retired" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
| jq -r '.data[] | select(.id=="ag-test-clean") | {id, retired_at, retired_reason}'
|
||||
```
|
||||
|
||||
**What:** Verifies the opt-in retired-agents view returns the row with `retired_at` and `retired_reason` populated. Go 1.22 ServeMux literal-beats-pattern-var precedence routes `/agents/retired` to this handler rather than `/agents/{id}`.
|
||||
**PASS if** the row appears with non-null `retired_at`. **FAIL** if the row is missing (listing broken) or `retired_at` is null (serialization broken).
|
||||
|
||||
---
|
||||
|
||||
### 55.13 Dashboard Stats Counter Excludes Retired
|
||||
|
||||
```bash
|
||||
curl -sS "http://localhost:8443/api/v1/stats/summary" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
| jq -r '.total_agents'
|
||||
```
|
||||
|
||||
**What:** Stats dashboard uses `ListActive`, not `List` — retired agents must not inflate the count.
|
||||
**PASS if** the counter reflects only non-retired rows (verify against `SELECT count(*) FROM agents WHERE retired_at IS NULL`).
|
||||
|
||||
---
|
||||
|
||||
### 55.14 CLI Retire Subcommand
|
||||
|
||||
```bash
|
||||
certctl-cli agents retire ag-cli-test --force --reason "smoke test"
|
||||
certctl-cli agents list --retired | grep ag-cli-test
|
||||
```
|
||||
|
||||
**What:** Verifies the CLI `agents retire` subcommand forwards `--force` and `--reason` via `DeleteWithQuery` and the `agents list --retired` flag hits `/agents/retired` rather than the default listing.
|
||||
**PASS if** the first command succeeds and the second shows the agent in the retired view.
|
||||
|
||||
---
|
||||
|
||||
### 55.15 MCP Retire Tool Schema
|
||||
|
||||
```bash
|
||||
go test ./internal/mcp/ -run TestRetireAgent -v -count=1
|
||||
```
|
||||
|
||||
**What:** Verifies the `certctl_retire_agent` MCP tool's input schema accepts `id`, `force`, and `reason`, and that the tool actually propagates `force`/`reason` into the outbound DELETE query string (not the body).
|
||||
**PASS if** exit code 0.
|
||||
|
||||
---
|
||||
|
||||
### 55.16 HEAD-State OpenAPI Contract
|
||||
|
||||
```bash
|
||||
npx --yes @redocly/cli lint api/openapi.yaml \
|
||||
--config '{"rules":{"operation-4xx-response":"error","no-invalid-media-type-examples":"error"}}'
|
||||
python3 -c "
|
||||
import yaml
|
||||
spec = yaml.safe_load(open('api/openapi.yaml'))
|
||||
del_op = spec['paths']['/api/v1/agents/{id}']['delete']
|
||||
assert set(del_op['responses'].keys()) == {'200','204','400','403','404','405','409','500'}, del_op['responses'].keys()
|
||||
hb = spec['paths']['/api/v1/agents/{id}/heartbeat']['post']
|
||||
assert '410' in hb['responses'], hb['responses'].keys()
|
||||
assert spec['paths']['/api/v1/agents/retired']['get']['operationId'] == 'listRetiredAgents'
|
||||
print('OpenAPI I-004 contract: OK')
|
||||
"
|
||||
```
|
||||
|
||||
**What:** Two-part check. Redocly lint confirms the spec is structurally valid; the Python assertions pin the seven DELETE status codes, the 410 heartbeat response, and the retired-agents operationId.
|
||||
**PASS if** redocly prints no errors and the Python script prints `OpenAPI I-004 contract: OK`.
|
||||
|
||||
---
|
||||
|
||||
## Part 56: Notification Retry & Dead-Letter Queue (I-005)
|
||||
|
||||
**What this validates:** The full retry lifecycle for `notification_events` rows — transient notifier failures are re-armed with exponential backoff (`2^retry_count` minutes capped at 1h, 5-attempt budget), rows that exhaust the budget land in the terminal `dead` status, the dead-letter depth is surfaced both on the dashboard and via a Prometheus counter, and operators can requeue dead rows once the underlying outage is resolved.
|
||||
|
||||
**Why it matters:** Before I-005, a failed notification was a silent drop. `internal/service/notification.go` flipped `status` to `failed` and never came back to it, because `ProcessPendingNotifications` only lists rows whose `status='pending'`. A 5xx from Slack, a 30-second SMTP stall, or a misrouted webhook URL could each lose a critical alert (cert expiry, CA compromise, approval-rejected) with no trace beyond a single log line. Part 56 pins the replacement contract (retry loop + DLQ + dashboard surface + Prometheus metric + operator requeue) so regressions show up here rather than as a post-incident "why didn't we get paged?" review.
|
||||
|
||||
### 56.1 Migration 000016 Columns Applied
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='notification_events' AND column_name IN ('retry_count','next_retry_at','last_error') ORDER BY column_name;"
|
||||
```
|
||||
|
||||
**What:** Confirms migration 000016 added the retry bookkeeping columns to `notification_events`.
|
||||
**PASS if** all three rows (`last_error`, `next_retry_at`, `retry_count`) are returned. **FAIL** if any is missing — the migration did not apply and the retry loop will error on every tick.
|
||||
|
||||
---
|
||||
|
||||
### 56.2 Partial Retry-Sweep Index Present
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT indexdef FROM pg_indexes WHERE tablename='notification_events' AND indexname='idx_notification_events_retry_sweep';"
|
||||
```
|
||||
|
||||
**What:** Confirms the partial index `idx_notification_events_retry_sweep ON notification_events(next_retry_at) WHERE status = 'failed' AND next_retry_at IS NOT NULL` exists and has the expected predicate.
|
||||
**PASS if** the returned `indexdef` includes `WHERE ((status = 'failed'::text) AND (next_retry_at IS NOT NULL))`. **FAIL** if the index is missing or unpartialed — the retry sweep will scan the full notification history instead of the small retry-eligible slice.
|
||||
|
||||
---
|
||||
|
||||
### 56.3 Failed Notification Retries On Next Tick
|
||||
|
||||
```bash
|
||||
# Seed a failed notification with next_retry_at in the past
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"UPDATE notification_events SET status='failed', retry_count=0, next_retry_at=NOW() - INTERVAL '1 minute', last_error='transient SMTP timeout' WHERE id='notif-demo-1';"
|
||||
|
||||
# Wait for the retry loop to sweep (default CERTCTL_NOTIFICATION_RETRY_INTERVAL=2m)
|
||||
sleep 130
|
||||
|
||||
# Observe the post-sweep state
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT id, status, retry_count, next_retry_at IS NOT NULL AS has_next_retry FROM notification_events WHERE id='notif-demo-1';"
|
||||
```
|
||||
|
||||
**What:** Exercises the retry loop's failure path. The seeded row is re-dispatched through the notifier registry; in the demo environment the notifier does not exist for `email` so the sweep either delivers (`status='sent'`) or records a failed attempt (`retry_count=1`, `next_retry_at` re-armed).
|
||||
**PASS if** either `status='sent'` (delivered on retry) or the row is still `failed` with `retry_count >= 1` and `has_next_retry=t`. **FAIL** if the row is still `failed` with `retry_count=0` and `next_retry_at` in the past — the retry loop is not actually running.
|
||||
|
||||
---
|
||||
|
||||
### 56.4 Exhausted Notification Transitions To Dead
|
||||
|
||||
```bash
|
||||
# Seed a row one failure shy of exhaustion — retry_count=4 means the next
|
||||
# tick's failure is the 5th attempt (notifRetryMaxAttempts-1 check at
|
||||
# internal/service/notification.go:531).
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"UPDATE notification_events SET status='failed', retry_count=4, next_retry_at=NOW() - INTERVAL '1 minute', last_error='persistent outage', channel='channel-that-does-not-exist' WHERE id='notif-demo-2';"
|
||||
|
||||
sleep 130
|
||||
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT id, status, retry_count, last_error FROM notification_events WHERE id='notif-demo-2';"
|
||||
```
|
||||
|
||||
**What:** The row at `retry_count=4` enters the sweep, the notifier lookup fails (channel unknown), the exhaustion branch fires, and `MarkAsDead` flips the row. Note: the "notifier unknown" branch at notification.go:494-503 promotes to `sent` for demo parity, so for a strict DLQ assertion seed a row whose channel is a known registered notifier that will reject delivery — alternatively run against the integration test fixture where the retry-exhaustion path is deterministic.
|
||||
**PASS if** `status='dead'` and `last_error` reflects the send failure. **FAIL** if the row is still `failed` with `retry_count >= 5` — the exhaustion branch did not fire and the row will retry forever.
|
||||
|
||||
---
|
||||
|
||||
### 56.5 Dead Row Has Null next_retry_at
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT COUNT(*) FROM notification_events WHERE status='dead' AND next_retry_at IS NOT NULL;"
|
||||
```
|
||||
|
||||
**What:** `MarkAsDead` must clear `next_retry_at` so the partial retry-sweep index stops matching the row. If this invariant breaks, a dead row keeps appearing in `ListRetryEligible` and the exhaustion branch fires on every sweep.
|
||||
**PASS if** the count is `0`. **FAIL** if any dead rows still carry a non-null `next_retry_at` — the DLQ is leaky and the row will re-enter the retry rotation on the next tick.
|
||||
|
||||
---
|
||||
|
||||
### 56.6 DashboardSummary Populates NotificationsDead
|
||||
|
||||
```bash
|
||||
# Seed a dead row so the count is observable
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"UPDATE notification_events SET status='dead', next_retry_at=NULL, last_error='demo DLQ fixture' WHERE id='notif-demo-3';"
|
||||
|
||||
curl -sS "http://localhost:8443/api/v1/stats/summary" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
| python3 -c "import sys,json; s=json.load(sys.stdin); assert 'notifications_dead' in s, 'missing notifications_dead field'; assert s['notifications_dead'] >= 1, s['notifications_dead']; print('notifications_dead:', s['notifications_dead'])"
|
||||
```
|
||||
|
||||
**What:** Confirms `DashboardSummary.NotificationsDead` (`internal/service/stats.go:66`) is populated by `notifRepo.CountByStatus(ctx, "dead")` (stats.go:137-142) and surfaced in the dashboard summary JSON.
|
||||
**PASS if** the field is present and reflects at least the seeded dead row. **FAIL** if the field is missing (`SetNotifRepo` was not called on StatsService) or stuck at zero despite seeded dead rows (repository `CountByStatus` is broken).
|
||||
|
||||
---
|
||||
|
||||
### 56.7 Prometheus Counter Emits certctl_notification_dead_total
|
||||
|
||||
```bash
|
||||
curl -sS "http://localhost:8443/api/v1/metrics/prometheus" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
| grep -E '^# (HELP|TYPE) certctl_notification_dead_total|^certctl_notification_dead_total '
|
||||
```
|
||||
|
||||
**What:** The Prometheus endpoint (`internal/api/handler/metrics.go:217-219`) emits three lines: `# HELP certctl_notification_dead_total Number of notifications in the dead-letter queue.`, `# TYPE certctl_notification_dead_total counter`, and a bare `certctl_notification_dead_total <value>` value line. Operator alert thresholds per the I-005 spec: `> 0` warning, `> 10` critical.
|
||||
**PASS if** all three lines are present and the value is `>= 1` when dead rows exist. **FAIL** if any of the three lines is missing — the metric name is misspelled, the `# TYPE` is wrong, or `DashboardSummary.NotificationsDead` is not wired into the metrics handler.
|
||||
|
||||
---
|
||||
|
||||
### 56.8 Requeue Resets Retry Bookkeeping
|
||||
|
||||
```bash
|
||||
# Confirm the row is in 'dead' with the full retry history
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT id, status, retry_count, next_retry_at, last_error FROM notification_events WHERE id='notif-demo-3';"
|
||||
|
||||
# Requeue via the operator endpoint
|
||||
curl -sS -X POST "http://localhost:8443/api/v1/notifications/notif-demo-3/requeue" \
|
||||
-H "Authorization: Bearer ${CERTCTL_API_KEY}" \
|
||||
-w "\nHTTP %{http_code}\n"
|
||||
|
||||
# Confirm the atomic reset
|
||||
docker compose -f deploy/docker-compose.yml exec postgres \
|
||||
psql -U certctl -d certctl -c \
|
||||
"SELECT id, status, retry_count, next_retry_at, last_error FROM notification_events WHERE id='notif-demo-3';"
|
||||
```
|
||||
|
||||
**What:** Exercises the operator-driven escape hatch (`POST /api/v1/notifications/{id}/requeue`). The repository's `Requeue` must atomically flip `status → pending`, reset `retry_count → 0`, clear `next_retry_at → NULL`, and clear `last_error → NULL` — see `internal/service/notification.go:571-576` and the pinning test at `notification_handler_test.go:307-347`.
|
||||
**PASS if** HTTP `200` with JSON body `{"status":"requeued"}` AND the post-requeue row has `status='pending'`, `retry_count=0`, `next_retry_at IS NULL`, `last_error IS NULL`. **FAIL** if any of the four fields is not reset — `ProcessPendingNotifications` will not treat this as a fresh attempt and the audit trail will be ambiguous.
|
||||
|
||||
---
|
||||
|
||||
### 56.9 GUI Dead Letter Tab Threads ?status=dead
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npx vitest run src/pages/NotificationsPage.test.tsx -t 'Dead letter tab fetches notifications with status=dead'
|
||||
```
|
||||
|
||||
**What:** The two-tab toolbar on `/notifications` routes the "Dead letter" tab's query through `getNotifications({ status: 'dead', per_page: '100' })`. This test verifies the React Query's `queryKey: ['notifications', activeTab]` (`NotificationsPage.tsx:31`) actually translates the tab click into the server-side filter — not client-side filtering of the full inbox.
|
||||
**PASS if** the Vitest assertion at `NotificationsPage.test.tsx:104-128` passes. **FAIL** if the Dead letter tab is merely a client-side filter on the `all` response — the DLQ-only code path (`NotificationRepository.ListByStatus`) is not exercised, which matters for pagination correctness once the inbox grows beyond 100 rows.
|
||||
|
||||
---
|
||||
|
||||
### 56.10 Requeue Button MutationFn Wrapper
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npx vitest run src/pages/NotificationsPage.test.tsx -t 'clicking Requeue invokes requeueNotification'
|
||||
```
|
||||
|
||||
**What:** `react-query` v5's `mutate(id)` passes a second positional argument (the mutation context object) to the `mutationFn`. If `mutationFn: requeueNotification` is used directly, the API client receives `(id, { client })` — an extra argument that the strict-match `toHaveBeenCalledWith('notif-dead-001')` assertion at `NotificationsPage.test.tsx:181` rejects. The fix is an explicit single-arg arrow: `mutationFn: (id: string) => requeueNotification(id)` at `NotificationsPage.tsx:64`.
|
||||
**PASS if** the Vitest assertion passes (the API client was called with exactly one argument). **FAIL** if the wrapper is inadvertently removed — silent success in runtime, loud failure in this contract.
|
||||
|
||||
---
|
||||
|
||||
### 56.11 HEAD-State OpenAPI Contract
|
||||
|
||||
```bash
|
||||
npx --yes @redocly/cli lint api/openapi.yaml \
|
||||
--config '{"rules":{"operation-4xx-response":"error","no-invalid-media-type-examples":"error"}}'
|
||||
python3 -c "
|
||||
import yaml
|
||||
spec = yaml.safe_load(open('api/openapi.yaml'))
|
||||
post = spec['paths']['/api/v1/notifications/{id}/requeue']['post']
|
||||
assert post['operationId'] == 'requeueNotification', post['operationId']
|
||||
assert set(post['responses'].keys()) >= {'200','400','404','405','500'}, post['responses'].keys()
|
||||
print('OpenAPI I-005 contract: OK')
|
||||
"
|
||||
```
|
||||
|
||||
**What:** Two-part check. Redocly lint confirms the spec is structurally valid; the Python assertions pin the requeue endpoint's `operationId` and the five minimum response codes (200/400/404/405/500).
|
||||
**PASS if** redocly prints no errors and the Python script prints `OpenAPI I-005 contract: OK`. **FAIL** if the `operationId` changed or any of the five responses is missing — downstream MCP/CLI clients rely on the contract.
|
||||
|
||||
---
|
||||
|
||||
## Release Sign-Off
|
||||
|
||||
All tests below must pass before tagging v2.1.0. Each row is one individual test from the guide above. The **Method** column indicates whether `qa-smoke-test.sh` covers the test automatically (**Auto**) or requires hands-on verification (**Manual**).
|
||||
@@ -6927,7 +7340,7 @@ These must be green before starting manual QA:
|
||||
|
||||
| Test | Description | Method | Pass? | Date | Notes |
|
||||
|------|-------------|--------|-------|------|-------|
|
||||
| 20.1.1 | Scheduler startup: all 7 loops registered | Manual | ☐ | | |
|
||||
| 20.1.1 | Scheduler startup: all 12 loops registered | Manual | ☐ | | |
|
||||
| 20.1.2 | Job processor loop fires (30s interval) | Manual | ☐ | | |
|
||||
| 20.1.3 | Agent health check marks offline (2m interval) | Manual | ☐ | | |
|
||||
| 20.1.4 | Notification processor fires (1m interval) | Manual | ☐ | | |
|
||||
@@ -7588,10 +8001,10 @@ These must be green before starting manual QA:
|
||||
| Category | Count |
|
||||
|----------|-------|
|
||||
| ☑ Auto (passed in `qa-smoke-test.sh`) | 144 |
|
||||
| ☐ Auto (not yet run) | 129 |
|
||||
| ☐ Auto (not yet run) | 136 |
|
||||
| — Skipped (preconditions not met in demo) | 5 |
|
||||
| ☐ Manual (requires hands-on verification) | 282 |
|
||||
| **Total** | **560** |
|
||||
| ☐ Manual (requires hands-on verification) | 286 |
|
||||
| **Total** | **571** |
|
||||
|
||||
**Automated tests must also be green.** CI passing is necessary but not sufficient — this manual QA catches integration issues that isolated unit tests miss.
|
||||
|
||||
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
# TLS on the Control Plane
|
||||
|
||||
certctl's control plane is HTTPS-only as of v2.2. There is no plaintext `http://` listener, no `auto` mode, no dual-listener bridge, no TLS 1.2 escape hatch. The server refuses to start without a cert+key pair, the agent/CLI/MCP clients reject `http://` URLs at startup, and the Helm chart refuses to render without either an operator-supplied Secret or a cert-manager Certificate CR.
|
||||
|
||||
This doc covers four cert provisioning patterns, SIGHUP-based cert rotation, and the client-side CA-trust configuration agents and the CLI need to talk to the server. If you are upgrading from a pre-HTTPS release and want the step-by-step cutover procedure, read [`upgrade-to-tls.md`](upgrade-to-tls.md) first and come back here for reference.
|
||||
|
||||
## What you get
|
||||
|
||||
The server binds TLS 1.3 only with an explicit curve preference of `[X25519, P-256]`. TLS 1.3 cipher suites are non-negotiable (all three mandatory suites — AES-128-GCM-SHA256, AES-256-GCM-SHA384, CHACHA20-POLY1305-SHA256 — are always offered), so there is no `CipherSuites` knob to misconfigure. No TLS 1.2 fallback is available.
|
||||
|
||||
Two env vars are required on the server:
|
||||
|
||||
- `CERTCTL_SERVER_TLS_CERT_PATH` — filesystem path to the PEM-encoded server certificate
|
||||
- `CERTCTL_SERVER_TLS_KEY_PATH` — filesystem path to the PEM-encoded private key that signs the cert
|
||||
|
||||
Both paths are read during a fail-loud preflight in `cmd/server/main.go` (see `preflightServerTLS` in `cmd/server/tls.go`). If either is unset, unreadable, or the cert+key pair does not round-trip through `tls.LoadX509KeyPair`, the process refuses to start and emits a diagnostic pointing back at this doc. The rationale lives in §3 of the HTTPS-Everywhere milestone: a cert-lifecycle product should not silently bind plaintext.
|
||||
|
||||
## Pattern 1 — Self-signed bootstrap for docker-compose demos
|
||||
|
||||
This is the default for the `deploy/docker-compose.yml` stack. It exists so `docker compose up -d --build` just works on a laptop without the operator standing up a CA first. It is not appropriate for any non-demo environment.
|
||||
|
||||
An init container named `certctl-tls-init` runs once before the server starts. It uses the `alpine/openssl` image and generates an ed25519 self-signed cert:
|
||||
|
||||
```
|
||||
openssl req -x509 -newkey ed25519 -nodes \
|
||||
-keyout /etc/certctl/tls/server.key \
|
||||
-out /etc/certctl/tls/server.crt \
|
||||
-days 3650 \
|
||||
-subj "/CN=certctl-server" \
|
||||
-addext "subjectAltName=DNS:certctl-server,DNS:localhost,IP:127.0.0.1,IP:::1"
|
||||
```
|
||||
|
||||
The cert, its matching key, and a copy of the cert published as `ca.crt` land in a named volume (`certs`) mounted at `/etc/certctl/tls/` in the server container (read-only) and the agent container (read-only). The bootstrap is idempotent — if `server.crt`, `server.key`, and `ca.crt` are already present on the volume, the init container logs `TLS cert already present at …` and exits cleanly.
|
||||
|
||||
Single-cert design. CN is `certctl-server` to match the Docker-network hostname. The SAN list is `[certctl-server, localhost, 127.0.0.1, ::1]`, which covers both container-internal agent→server traffic and operator browser/curl access to `https://localhost:8443`. There is no separate intermediate/root chain — the server cert and the CA bundle are the same PEM. This is the whole point of a demo bootstrap.
|
||||
|
||||
To force regeneration (rotate the demo cert), tear the volume down: `docker compose down -v`. The next `up` re-runs the init container.
|
||||
|
||||
The server's Docker healthcheck and the agent both verify against `/etc/certctl/tls/ca.crt`; no `-k` / `InsecureSkipVerify` anywhere in the default stack.
|
||||
|
||||
## Pattern 2 — Operator-supplied `kubernetes.io/tls` Secret (Helm)
|
||||
|
||||
This is the default path for Helm installs. The operator provisions a Secret of type `kubernetes.io/tls` holding `tls.crt` + `tls.key` (and optionally `ca.crt` for mounting a CA bundle to clients in the same cluster) from whatever source they already trust — their internal CA, a manually-issued cert, step-ca, AWS ACM PCA exported to PEM, or the output of the self-signed bootstrap pattern above copied into a cluster Secret.
|
||||
|
||||
```
|
||||
kubectl create secret tls certctl-server-tls \
|
||||
--cert=server.crt \
|
||||
--key=server.key \
|
||||
--namespace certctl
|
||||
```
|
||||
|
||||
Then:
|
||||
|
||||
```
|
||||
helm install certctl deploy/helm/certctl \
|
||||
--namespace certctl \
|
||||
--set server.tls.existingSecret=certctl-server-tls
|
||||
```
|
||||
|
||||
The Secret is mounted read-only at `/etc/certctl/tls/` in the server pod. The `CERTCTL_SERVER_TLS_CERT_PATH` and `CERTCTL_SERVER_TLS_KEY_PATH` env vars are wired to `tls.crt` and `tls.key` keys inside that mount. If `ca.crt` is absent from the Secret, clients that need a CA bundle should use `tls.crt` as the bundle (self-signed case) or mount a separate ConfigMap with the root chain (operator-CA case).
|
||||
|
||||
If the operator sets neither `server.tls.existingSecret` nor `server.tls.certManager.enabled=true`, `helm template` / `helm install` fails at render-time with a diagnostic pointing at this doc. The guard is implemented in `deploy/helm/certctl/templates/_helpers.tpl` under the `certctl.tls.required` helper. This is deliberate: the HTTPS-only server would crash-loop on an empty path, so we fail earlier at Helm-render time.
|
||||
|
||||
## Pattern 3 — cert-manager `Certificate` CR (Helm, opt-in)
|
||||
|
||||
For clusters that already run cert-manager, the chart can provision a `Certificate` CR that writes into the Secret the server pod reads from. This is opt-in — the default is `server.tls.certManager.enabled: false` — because not every cluster has cert-manager installed, and we refuse to ship a chart that silently depends on an external controller.
|
||||
|
||||
```
|
||||
helm install certctl deploy/helm/certctl \
|
||||
--namespace certctl \
|
||||
--set server.tls.certManager.enabled=true \
|
||||
--set server.tls.certManager.issuerRef.name=my-cluster-issuer \
|
||||
--set server.tls.certManager.issuerRef.kind=ClusterIssuer
|
||||
```
|
||||
|
||||
The rendered `Certificate` (see `deploy/helm/certctl/templates/server-certificate.yaml`) writes `tls.crt` + `tls.key` + `ca.crt` into the Secret named by `server.tls.certManager.secretName` (defaults to `<fullname>-tls`). The server pod reads from that same Secret; the agent DaemonSet mounts the same Secret as its CA bundle source.
|
||||
|
||||
cert-manager handles rotation. certctl-server handles in-place reload — see the SIGHUP section below.
|
||||
|
||||
The chart enforces that if `server.tls.certManager.enabled=true`, `server.tls.certManager.issuerRef.name` must also be set. An empty `issuerRef.name` makes `helm template` fail with a diagnostic naming the missing flag.
|
||||
|
||||
## Pattern 4 — Manually-issued from an internal CA
|
||||
|
||||
For operators running neither Helm nor docker-compose (bare-metal / custom orchestration), the server just needs two files on disk pointed at by `CERTCTL_SERVER_TLS_CERT_PATH` and `CERTCTL_SERVER_TLS_KEY_PATH`. Issue the cert from your internal CA with:
|
||||
|
||||
- CN matching the hostname your agents and operators use to dial the server (e.g., `certctl.prod.example.com`)
|
||||
- SAN list covering every hostname and IP that appears in `CERTCTL_SERVER_URL` values across your agent fleet
|
||||
- Key usage: digital signature + key encipherment
|
||||
- Extended key usage: server auth
|
||||
|
||||
Store the key with mode `0600` and owner matching the UID the server runs as (`1000` in our shipped Dockerfile). The server process reads both files during `preflightServerTLS` at startup and again on every SIGHUP.
|
||||
|
||||
The full CA chain that signed the server cert should be distributed to agents, CLI operators, and MCP clients as their `CERTCTL_SERVER_CA_BUNDLE_PATH` — see the client section below.
|
||||
|
||||
## SIGHUP cert rotation
|
||||
|
||||
The server wraps its cert+key pair in a `*certHolder` (see `cmd/server/tls.go`) that guards the loaded `*tls.Certificate` under a `sync.Mutex`. The `*tls.Config` wires `GetCertificate` to the holder, so every new inbound TLS handshake reads whatever cert the holder currently has.
|
||||
|
||||
Send `SIGHUP` to the server PID and the holder re-reads both files from disk. On success, the next new connection uses the new cert; in-flight requests finish on the previous cert. A log line goes out:
|
||||
|
||||
```
|
||||
TLS cert reloaded via SIGHUP cert_path=/etc/certctl/tls/server.crt key_path=/etc/certctl/tls/server.key
|
||||
```
|
||||
|
||||
On failure (missing file, malformed PEM, key does not sign cert), the old cert is retained and an error logs:
|
||||
|
||||
```
|
||||
TLS cert reload failed; continuing with previous cert cert_path=… key_path=… error=…
|
||||
```
|
||||
|
||||
This is deliberately fail-safe on reload (as opposed to fail-loud on startup). A cert-manager renewal race, a partially-copied file, a typo in a rotation script — none of those should crash a running server and drop every agent connection. The operator sees the error in logs, fixes the underlying issue, and sends another `SIGHUP`.
|
||||
|
||||
Pair with cert-manager, certbot `--post-hook`, or any rotation tool that can fire a signal. For docker-compose, `docker compose kill -s HUP certctl-server` works. For Kubernetes, reload is typically handled by cert-manager updating the Secret and the mounted file changing on the next kubelet sync — no explicit SIGHUP needed if the volume mount is `subPath`-free.
|
||||
|
||||
Startup is a different story. If the cert is missing or malformed at process start, the server exits non-zero rather than binding plaintext or attempting a retry loop. That's the HTTPS-only contract.
|
||||
|
||||
## Client-side TLS: agents, CLI, MCP
|
||||
|
||||
Everything that talks to the server enforces HTTPS on the URL.
|
||||
|
||||
### Agent
|
||||
|
||||
`CERTCTL_SERVER_URL` must be `https://…`. `http://`, bare hostnames, `ftp://`, `ws://`, and empty strings are rejected at startup by `validateHTTPSScheme` in `cmd/agent/main.go` with a diagnostic pointing at `upgrade-to-tls.md`. There is no warning-and-proceed path.
|
||||
|
||||
Two additional env vars control how the agent verifies the server cert:
|
||||
|
||||
- `CERTCTL_SERVER_CA_BUNDLE_PATH` — filesystem path to a PEM-encoded CA bundle that signed the server cert. Loaded into `*tls.Config.RootCAs` on the agent's HTTP client. If unset, the agent falls back to the OS system trust store.
|
||||
- `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY` — defaults to `false`. Setting it to `true` skips verification entirely. **Dev-only escape hatch.** The agent logs a prominent warning at startup (`TLS certificate verification is disabled … never enable this in production`). Use this only when dialing a demo server whose cert you haven't bothered to mount into the agent container.
|
||||
|
||||
Equivalent CLI flags: `--ca-bundle <path>` and `--insecure-skip-verify`.
|
||||
|
||||
If both the CA bundle and `InsecureSkipVerify=true` are set, `InsecureSkipVerify` wins — it's the whole point of the flag. Don't do this in production.
|
||||
|
||||
### CLI (`certctl-cli`)
|
||||
|
||||
Same contract as the agent:
|
||||
|
||||
- `CERTCTL_SERVER_URL` defaults to `https://` scheme; `http://` rejected at startup
|
||||
- `--ca-bundle <path>` flag or `CERTCTL_SERVER_CA_BUNDLE_PATH` env var — CA bundle for server cert verification
|
||||
- `--insecure` flag or `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true` — skip verification (dev only)
|
||||
- Error diagnostic on empty URL explicitly mentions both `--server` and `CERTCTL_SERVER_URL` so operators see the right knob to turn
|
||||
|
||||
The CLI shares the URL-scheme validation with the agent; the test pins in `cmd/cli/main_test.go:TestValidateHTTPSScheme` cover the full rejection matrix.
|
||||
|
||||
### MCP server (`certctl-mcp-server`)
|
||||
|
||||
Same three controls as CLI, env-var-driven only (no flags — MCP runs as a stdio subprocess and inherits env from the launching LLM client):
|
||||
|
||||
- `CERTCTL_SERVER_URL` must start with `https://`
|
||||
- `CERTCTL_SERVER_CA_BUNDLE_PATH` optional CA bundle
|
||||
- `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY` optional skip
|
||||
|
||||
Claude Desktop / other MCP client configs should set all three in the tool's env block.
|
||||
|
||||
## Troubleshooting: fail-loud preflight errors
|
||||
|
||||
Every preflight failure message ends with `(see docs/tls.md)` so this doc is the first hit when an operator searches. Common failures:
|
||||
|
||||
**`CERTCTL_SERVER_TLS_CERT_PATH is empty: HTTPS-only control plane refuses to start`**
|
||||
Set the env var. For docker-compose this is already set to `/etc/certctl/tls/server.crt` in the shipped compose file — if you're seeing this, check the `certctl-tls-init` service logs to see why the init container didn't populate the volume. For Helm, check that `server.tls.existingSecret` or `server.tls.certManager.enabled=true` is set.
|
||||
|
||||
**`TLS cert file "…" unreadable: …`**
|
||||
The cert path is set but `os.Stat` failed. Check filesystem permissions — the server runs as UID 1000 in our shipped Dockerfile; the cert needs to be readable by that UID. Typos in the path also land here.
|
||||
|
||||
**`TLS cert/key pair invalid (cert="…" key="…"): …`**
|
||||
Both files exist but `tls.LoadX509KeyPair` refused them. Typical causes: the private key does not sign the certificate, the key is encrypted with a passphrase (not supported — remove the passphrase with `openssl pkey` before mounting), or one of the two is DER-encoded instead of PEM. Re-issue the pair from the same CA call and re-mount.
|
||||
|
||||
**Client side: `tls: failed to verify certificate: x509: certificate signed by unknown authority`**
|
||||
The client did not trust the CA that signed the server cert. Either mount the CA bundle via `CERTCTL_SERVER_CA_BUNDLE_PATH`, add the CA to the system trust store on the client host, or (dev only) set `CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true`.
|
||||
|
||||
**Client side: `tls: first record does not look like a TLS handshake`**
|
||||
The client is speaking plaintext HTTP to an HTTPS server (or vice-versa). Check that `CERTCTL_SERVER_URL` starts with `https://`. If you are upgrading from a pre-v2.2 release and your agents are old, they will surface this error until you roll the DaemonSet — see [`upgrade-to-tls.md`](upgrade-to-tls.md).
|
||||
|
||||
## Related docs
|
||||
|
||||
- [`upgrade-to-tls.md`](upgrade-to-tls.md) — one-step cutover from pre-HTTPS releases
|
||||
- [`quickstart.md`](quickstart.md) — docker-compose walkthrough with HTTPS examples
|
||||
- [`test-env.md`](test-env.md) — integration test environment (also HTTPS-only)
|
||||
- Milestone spec: `prompts/https-everywhere-milestone.md` (authoritative source for locked decisions)
|
||||
@@ -0,0 +1,194 @@
|
||||
# Upgrading to HTTPS-Everywhere (v2.2)
|
||||
|
||||
certctl's control plane is HTTPS-only as of v2.2. There is no `http` mode, no `auto` mode, no dual-listener bind, no N-release migration window. The cutover is a single step. Out-of-date agents that still point at `http://…` fail at the TCP/TLS handshake layer on first connect after the upgrade and stay `Offline` in the dashboard until their env block is updated and the fleet is rolled.
|
||||
|
||||
This doc walks operators through the cutover for the two shipped deployment topologies — docker-compose and Helm — and documents the failure modes and rollback posture explicitly.
|
||||
|
||||
For the deep-dive on cert provisioning patterns, SIGHUP cert reload, and client-side CA-trust configuration, read [`tls.md`](tls.md). This doc is the narrow "how do I upgrade" procedure.
|
||||
|
||||
## Preconditions
|
||||
|
||||
Before you start, confirm:
|
||||
|
||||
- **Shell access** to the server host and every agent host. The cutover requires you to restart the server and update every agent's env block.
|
||||
- **A cert+key source** for the server. Pick one:
|
||||
- An internal CA that can issue a server cert (CN + SAN list covering every hostname / IP agents dial).
|
||||
- A `cert-manager` install in the target Kubernetes cluster, plus a `ClusterIssuer` or `Issuer` you're willing to reference.
|
||||
- Willingness to use the self-signed bootstrap that the shipped `deploy/docker-compose.yml` generates automatically. This is the right choice for dev and demo; it is the wrong choice for production.
|
||||
- **A maintenance window.** Out-of-date agents break at the TLS handshake and stay offline until rolled. Schedule the upgrade so the agent fleet can be updated in the same window as the server.
|
||||
- **Backups.** This is a one-way door (see the Rollback section below). Snapshot your PostgreSQL database before `docker compose down` or `helm upgrade`.
|
||||
|
||||
There is no schema migration tied to this release; the only at-rest state that changes is the `certs` named volume (docker-compose) or the `tls.crt`/`tls.key` Secret (Helm).
|
||||
|
||||
## Procedure — docker-compose operators
|
||||
|
||||
The shipped `deploy/docker-compose.yml` includes a `certctl-tls-init` init container that self-signs an ed25519 cert on first boot and drops `server.crt`, `server.key`, and `ca.crt` into a named volume mounted read-only at `/etc/certctl/tls/` on the server and agent containers. No manual cert provisioning is required for the default stack.
|
||||
|
||||
1. **Pull the HTTPS-everywhere release.** From the repo root:
|
||||
|
||||
```
|
||||
git pull
|
||||
```
|
||||
|
||||
Confirm you're on a tag or `master` that contains the `certctl-tls-init` service in `deploy/docker-compose.yml`. Grep for it: `grep certctl-tls-init deploy/docker-compose.yml` should hit.
|
||||
|
||||
2. **Stop the old plaintext cluster.**
|
||||
|
||||
```
|
||||
docker compose -f deploy/docker-compose.yml down
|
||||
```
|
||||
|
||||
Do not pass `-v`; keeping the PostgreSQL volume preserves your cert inventory, audit trail, and job history across the upgrade.
|
||||
|
||||
3. **Bring the cluster back up with the HTTPS build.**
|
||||
|
||||
```
|
||||
docker compose -f deploy/docker-compose.yml up -d --build
|
||||
```
|
||||
|
||||
The `certctl-tls-init` service runs once, generates the self-signed cert into the `certs` volume, and exits with code 0. The server container waits for `certctl-tls-init` via `depends_on: { condition: service_completed_successfully }` and only starts once the cert material is on disk. The server's Docker healthcheck now uses `curl --cacert /etc/certctl/tls/ca.crt -f https://localhost:8443/health`, so the container only becomes healthy once the HTTPS listener is up and serving the bundled cert correctly.
|
||||
|
||||
4. **Verify the HTTPS endpoint from the host.**
|
||||
|
||||
```
|
||||
curl --cacert $(docker compose -f deploy/docker-compose.yml exec -T certctl-server cat /etc/certctl/tls/ca.crt) https://localhost:8443/health
|
||||
```
|
||||
|
||||
Expect `{"status":"ok"}` with HTTP 200. If you get a TLS verification error, the CA bundle wasn't read correctly — re-run the `exec -T` command and pipe the output directly into `--cacert @-` or save it to a local file first. If you get `connection refused`, the server never finished startup — check `docker compose logs certctl-server` for a fail-loud preflight diagnostic pointing at `docs/tls.md`.
|
||||
|
||||
5. **Confirm the bundled agent reconnects.** Agents inside the compose stack pick up the new URL (`CERTCTL_SERVER_URL=https://certctl-server:8443`) and the bundled CA (`CERTCTL_SERVER_CA_BUNDLE_PATH=/etc/certctl/tls/ca.crt`) from their env block automatically — no per-agent change needed. Tail the agent log:
|
||||
|
||||
```
|
||||
docker compose -f deploy/docker-compose.yml logs -f certctl-agent
|
||||
```
|
||||
|
||||
You should see `heartbeat sent` within 30 seconds. In the dashboard (`https://localhost:8443`), the agent should show as `Online`.
|
||||
|
||||
**External agents** running outside the compose network (e.g., the `install-agent.sh`-installed systemd service on a separate host) need their env block updated manually before the cutover — see the Agent env block section below.
|
||||
|
||||
## Procedure — Helm operators
|
||||
|
||||
The Helm chart does not self-sign. It refuses to render (`helm template` exits non-zero) unless you configure one of two cert sources: an operator-supplied Secret, or a cert-manager `Certificate` CR. See [`tls.md`](tls.md) for the full pattern catalog.
|
||||
|
||||
1. **Provision cert material.** Pick one of:
|
||||
|
||||
- **Operator-supplied Secret.** Issue a cert from your internal CA (or any other source) and load it into a `kubernetes.io/tls` Secret in the certctl namespace:
|
||||
|
||||
```
|
||||
kubectl create secret tls certctl-server-tls \
|
||||
--cert=server.crt --key=server.key \
|
||||
--namespace certctl
|
||||
```
|
||||
|
||||
- **cert-manager.** Set `server.tls.certManager.enabled=true` on the upgrade and reference an existing `ClusterIssuer` or `Issuer`:
|
||||
|
||||
```
|
||||
--set server.tls.certManager.enabled=true
|
||||
--set server.tls.certManager.issuerRef.name=my-cluster-issuer
|
||||
--set server.tls.certManager.issuerRef.kind=ClusterIssuer
|
||||
```
|
||||
|
||||
2. **Upgrade the release.**
|
||||
|
||||
```
|
||||
helm upgrade certctl deploy/helm/certctl \
|
||||
--namespace certctl \
|
||||
--set server.tls.existingSecret=certctl-server-tls
|
||||
```
|
||||
|
||||
(Or the `certManager` variant.) If you omit both `server.tls.existingSecret` and `server.tls.certManager.enabled`, the chart fails at render time with a diagnostic pointing at `docs/tls.md`. That guard exists precisely so you catch the missing config at `helm upgrade` time, not at pod-crash-loop time.
|
||||
|
||||
3. **Verify the HTTPS endpoint from inside the cluster.** Port-forward and curl with the CA bundle:
|
||||
|
||||
```
|
||||
kubectl port-forward -n certctl svc/certctl-server 8443:8443 &
|
||||
kubectl get secret -n certctl certctl-server-tls -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/certctl-ca.crt
|
||||
curl --cacert /tmp/certctl-ca.crt https://localhost:8443/health
|
||||
```
|
||||
|
||||
Expect `{"status":"ok"}`. If the Secret does not contain a `ca.crt` key (operator-supplied Secrets often don't), use `tls.crt` as the bundle instead — for a self-signed cert the two files are identical, and for a cert chained to an internal CA you should separately distribute the root CA bundle via ConfigMap or mounted file.
|
||||
|
||||
4. **Update every agent manifest.** Agents outside this Helm release (or in a separately-managed DaemonSet) need their env block updated:
|
||||
|
||||
```
|
||||
- name: CERTCTL_SERVER_URL
|
||||
value: "https://certctl-server.certctl.svc.cluster.local:8443"
|
||||
- name: CERTCTL_SERVER_CA_BUNDLE_PATH
|
||||
value: "/etc/certctl/tls/ca.crt"
|
||||
```
|
||||
|
||||
Mount the server's Secret (or a separate CA-bundle Secret / ConfigMap) at `/etc/certctl/tls/` as a read-only volume. If you bundle the agent via the shipped Helm chart's DaemonSet, the wiring is already done — set `agent.enabled=true` and the chart mounts the same Secret.
|
||||
|
||||
5. **Roll the agent DaemonSet.**
|
||||
|
||||
```
|
||||
kubectl rollout restart ds/certctl-agent -n certctl
|
||||
kubectl rollout status ds/certctl-agent -n certctl
|
||||
```
|
||||
|
||||
Every agent pod restarts with the new URL + CA bundle and reconnects on HTTPS. The dashboard shows agents flip from `Offline` to `Online` as pods finish rolling.
|
||||
|
||||
## Agent env block — external hosts
|
||||
|
||||
Agents installed on bare-metal or VM hosts via `install-agent.sh` (systemd on Linux, launchd on macOS) read config from `/etc/certctl/agent.env` (Linux) or `~/Library/Application Support/certctl/agent.env` (macOS). On cutover, append or update:
|
||||
|
||||
```
|
||||
CERTCTL_SERVER_URL=https://certctl.example.com:8443
|
||||
CERTCTL_SERVER_CA_BUNDLE_PATH=/etc/certctl/tls/ca.crt
|
||||
# CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=false # Dev only. Never set to true in production.
|
||||
```
|
||||
|
||||
Distribute the CA bundle (the same `ca.crt` the server holds, or the root chain if you issued the server cert from an intermediate) to every agent host. The path under `CERTCTL_SERVER_CA_BUNDLE_PATH` must be readable by the UID the agent service runs as.
|
||||
|
||||
Restart the service after editing:
|
||||
|
||||
- Linux: `systemctl restart certctl-agent`
|
||||
- macOS: `launchctl kickstart -k system/com.certctl.agent`
|
||||
|
||||
The agent refuses to start on an `http://` URL and exits with a pre-flight diagnostic that names this doc. That rejection happens before any network call — no spurious half-connected state.
|
||||
|
||||
## Failure mode
|
||||
|
||||
Out-of-date agents still configured with `CERTCTL_SERVER_URL=http://…` fail on first reconnect after the cutover. The failure surfaces as one of:
|
||||
|
||||
- `dial tcp …: connect: connection refused` — the server is no longer listening on a plaintext port. The new release binds only a TLS listener; attempting a plaintext `connect()` gets refused at the kernel level because nothing holds the socket.
|
||||
- `tls: first record does not look like a TLS handshake` — depending on timing and proxy layers (e.g., a load balancer that accepts the TCP connection before forwarding), the client may negotiate TCP, send an HTTP request line, and have the server's TLS stack reject it.
|
||||
|
||||
Agents in this state surface as `Offline` in the dashboard. They stay offline until their env block is updated and the service restarts. There is no graceful 400-with-migration-URL response because there is no HTTP listener to serve one from — the entire plaintext call path is removed by design.
|
||||
|
||||
If you see an unexpected agent stay `Offline` past the cutover window, SSH to the host and check the agent log. On a systemd host:
|
||||
|
||||
```
|
||||
journalctl -u certctl-agent -n 100
|
||||
```
|
||||
|
||||
Look for `URL scheme "http" is not supported: HTTPS-only control plane refuses to start (see docs/upgrade-to-tls.md)`. That's the pre-flight rejection. Update `CERTCTL_SERVER_URL`, restart the service, and the agent reconnects.
|
||||
|
||||
## Rollback
|
||||
|
||||
**There is no rollback window.** The upgrade is a one-way door. The rationale lives in §3.7 of `prompts/https-everywhere-milestone.md`: a cert-lifecycle product that bridges back to plaintext after committing to HTTPS is advertising that its own security posture is negotiable.
|
||||
|
||||
If you need to revert, you have two options:
|
||||
|
||||
1. **Stay on the pre-HTTPS release.** Do not upgrade until you are ready to run HTTPS on the control plane. Pin your `docker-compose.yml` or `helm upgrade` command to the last pre-v2.2 tag.
|
||||
2. **Rollback the release.** `helm rollback certctl <previous-revision>` or `git checkout <previous-tag> && docker compose up -d --build`. This rolls back the server, the compose topology, and the Helm chart in lockstep. Your PostgreSQL volume — cert inventory, audit trail, jobs — survives the rollback; nothing in this milestone changes the database schema.
|
||||
|
||||
Option 2 drops you back to the plaintext world. It should be treated as an emergency measure, not a supported migration path.
|
||||
|
||||
## After the cutover
|
||||
|
||||
Once every agent is `Online`, confirm a few invariants:
|
||||
|
||||
- `curl -sS -o /dev/null -w "%{http_code}\n" http://localhost:8443/health` returns `000` with `Connection refused` (no HTTP listener). Plaintext is gone.
|
||||
- `openssl s_client -connect localhost:8443 -tls1_2 </dev/null` fails the handshake. TLS 1.2 is rejected.
|
||||
- `openssl s_client -connect localhost:8443 -tls1_3 </dev/null` succeeds and prints the server's SAN list. TLS 1.3 is live.
|
||||
- A cert rotation test: overwrite the server cert on disk, `kill -HUP` the server PID, confirm the new cert serves on the next `openssl s_client -connect … -showcerts` without a process restart. See the SIGHUP section in [`tls.md`](tls.md).
|
||||
|
||||
Update your runbooks. Every `http://certctl.example.com` URL in internal documentation, monitoring config, and on-call playbooks should become `https://certctl.example.com` plus a CA-trust note.
|
||||
|
||||
## Related docs
|
||||
|
||||
- [`tls.md`](tls.md) — cert provisioning patterns, SIGHUP rotation, troubleshooting
|
||||
- [`quickstart.md`](quickstart.md) — docker-compose walkthrough (post-HTTPS)
|
||||
- [`test-env.md`](test-env.md) — integration test environment (HTTPS-only)
|
||||
- Milestone spec: `prompts/https-everywhere-milestone.md`
|
||||
+1
-1
@@ -107,7 +107,7 @@ The demo seeds certificates across multiple issuers, agents, and deployment targ
|
||||
```bash
|
||||
git clone https://github.com/shankar0123/certctl.git
|
||||
cd certctl/deploy && docker compose up -d
|
||||
# Dashboard at http://localhost:8443
|
||||
# Dashboard at https://localhost:8443 (self-signed cert — pin deploy/test/certs/ca.crt)
|
||||
```
|
||||
|
||||
See the [Quickstart Guide](quickstart.md) for a full walkthrough, or explore the [5 turnkey examples](../examples/) for specific scenarios (ACME+NGINX, wildcard DNS-01, private CA+Traefik, step-ca+HAProxy, multi-issuer).
|
||||
|
||||
@@ -36,6 +36,13 @@ flowchart TD
|
||||
|
||||
If you don't have a real domain or can't open port 80, see [Customization Tips](#customization-tips) below.
|
||||
|
||||
## TLS Security
|
||||
|
||||
certctl is HTTPS-only as of v2.2. The demo compose stack provisions a self-signed certificate. When accessing `https://localhost:8443`, you can either:
|
||||
- Use `curl --cacert ./deploy/test/certs/ca.crt ...` to pin the CA certificate
|
||||
- Use `curl -k ...` for quick smoke tests (never in production)
|
||||
- Import the CA at `./deploy/test/certs/ca.crt` into your OS trust store for browser visits
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Clone or copy this example
|
||||
@@ -122,7 +129,7 @@ docker compose logs -f certctl-server certctl-agent
|
||||
|
||||
### 5. Access the dashboard
|
||||
|
||||
Navigate to `http://localhost:8443` (or your `SERVER_PORT`)
|
||||
Navigate to `https://localhost:8443` (or your `SERVER_PORT`)
|
||||
|
||||
You should see:
|
||||
- An empty certificate inventory (no certs issued yet)
|
||||
|
||||
@@ -61,7 +61,7 @@ services:
|
||||
networks:
|
||||
- certctl-network
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'curl -sf http://localhost:8443/health || exit 1']
|
||||
test: ['CMD-SHELL', 'curl -sfk https://localhost:8443/health || exit 1']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -9,6 +9,13 @@ This example is ideal for:
|
||||
- Internal PKI with public DNS names
|
||||
- Scenarios where you have programmatic access to your DNS provider's API
|
||||
|
||||
## TLS Security
|
||||
|
||||
certctl is HTTPS-only as of v2.2. The demo compose stack provisions a self-signed certificate. When accessing `https://localhost:8443`, you can either:
|
||||
- Use `curl --cacert ./deploy/test/certs/ca.crt ...` to pin the CA certificate
|
||||
- Use `curl -k ...` for quick smoke tests (never in production)
|
||||
- Import the CA at `./deploy/test/certs/ca.crt` into your OS trust store for browser visits
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before running this example, you need:
|
||||
@@ -74,7 +81,7 @@ This starts:
|
||||
|
||||
### Step 5: Access the Dashboard
|
||||
|
||||
Open your browser to `http://localhost:8443`
|
||||
Open your browser to `https://localhost:8443`
|
||||
|
||||
### Step 6: Create a Wildcard Certificate
|
||||
|
||||
|
||||
@@ -113,7 +113,7 @@ services:
|
||||
- certctl-network
|
||||
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'curl -sf http://localhost:8443/health || exit 1']
|
||||
test: ['CMD-SHELL', 'curl -sfk https://localhost:8443/health || exit 1']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
networks:
|
||||
- certctl-network
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'curl -sf http://localhost:8443/health || exit 1']
|
||||
test: ['CMD-SHELL', 'curl -sfk https://localhost:8443/health || exit 1']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -45,6 +45,13 @@ flowchart TD
|
||||
- **Domain for ACME** (optional) — if using real Let's Encrypt, not needed for demo
|
||||
- **Internet connectivity** — to reach Let's Encrypt's API (demo can use staging directory)
|
||||
|
||||
## TLS Security
|
||||
|
||||
certctl is HTTPS-only as of v2.2. The demo compose stack provisions a self-signed certificate. When accessing `https://localhost:8443`, you can either:
|
||||
- Use `curl --cacert ./deploy/test/certs/ca.crt ...` to pin the CA certificate
|
||||
- Use `curl -k ...` for quick smoke tests (never in production)
|
||||
- Import the CA at `./deploy/test/certs/ca.crt` into your OS trust store for browser visits
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Clone or navigate to this directory
|
||||
@@ -83,7 +90,7 @@ This spins up:
|
||||
|
||||
### 4. Access the dashboard
|
||||
|
||||
Open your browser to **http://localhost:8443** (or your configured SERVER_PORT)
|
||||
Open your browser to **https://localhost:8443** (or your configured SERVER_PORT)
|
||||
|
||||
You should see:
|
||||
- Empty cert inventory (fresh start)
|
||||
|
||||
@@ -77,7 +77,7 @@ services:
|
||||
networks:
|
||||
- certctl-network
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'curl -sf http://localhost:8443/health || exit 1']
|
||||
test: ['CMD-SHELL', 'curl -sfk https://localhost:8443/health || exit 1']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -29,6 +29,13 @@ flowchart TD
|
||||
C -->|TLS handshakes| D
|
||||
```
|
||||
|
||||
## TLS Security
|
||||
|
||||
certctl is HTTPS-only as of v2.2. The demo compose stack provisions a self-signed certificate. When accessing `https://localhost:8443`, you can either:
|
||||
- Use `curl --cacert ./deploy/test/certs/ca.crt ...` to pin the CA certificate
|
||||
- Use `curl -k ...` for quick smoke tests (never in production)
|
||||
- Import the CA at `./deploy/test/certs/ca.crt` into your OS trust store for browser visits
|
||||
|
||||
## Quick Start (Self-Signed CA)
|
||||
|
||||
The simplest way to get running in 2 minutes:
|
||||
@@ -58,7 +65,7 @@ EOF
|
||||
docker compose up -d
|
||||
|
||||
# 4. Access the dashboards
|
||||
# - certctl: http://localhost:8443 (API only, use the CLI or direct HTTP calls)
|
||||
# - certctl: https://localhost:8443 (API only, use the CLI or direct HTTP calls)
|
||||
# - Traefik dashboard: http://localhost:8080
|
||||
```
|
||||
|
||||
@@ -112,7 +119,7 @@ Once the stack is running:
|
||||
|
||||
```bash
|
||||
# 1. Create a certificate profile in certctl (defines allowed key types, TTL, etc.)
|
||||
curl -X POST http://localhost:8443/api/v1/profiles \
|
||||
curl -X POST https://localhost:8443/api/v1/profiles \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"id": "prof-internal",
|
||||
@@ -123,7 +130,7 @@ curl -X POST http://localhost:8443/api/v1/profiles \
|
||||
}'
|
||||
|
||||
# 2. Create a renewal policy (defines issuer, renewal thresholds, etc.)
|
||||
curl -X POST http://localhost:8443/api/v1/policies \
|
||||
curl -X POST https://localhost:8443/api/v1/policies \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"id": "pol-internal",
|
||||
@@ -135,7 +142,7 @@ curl -X POST http://localhost:8443/api/v1/policies \
|
||||
}'
|
||||
|
||||
# 3. Create a certificate (triggers issuance immediately)
|
||||
curl -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"common_name": "api.internal.local",
|
||||
@@ -144,7 +151,7 @@ curl -X POST http://localhost:8443/api/v1/certificates \
|
||||
}'
|
||||
|
||||
# 4. Create a Traefik target (agent will deploy to this)
|
||||
curl -X POST http://localhost:8443/api/v1/targets \
|
||||
curl -X POST https://localhost:8443/api/v1/targets \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"id": "target-traefik-01",
|
||||
@@ -156,7 +163,7 @@ curl -X POST http://localhost:8443/api/v1/targets \
|
||||
}'
|
||||
|
||||
# 5. Create a deployment job (agent picks this up and deploys)
|
||||
curl -X POST http://localhost:8443/api/v1/certificates/{cert-id}/deploy \
|
||||
curl -X POST https://localhost:8443/api/v1/certificates/{cert-id}/deploy \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"target_ids": ["target-traefik-01"]
|
||||
@@ -209,16 +216,16 @@ The server provides a REST API on port 8443. Example queries:
|
||||
|
||||
```bash
|
||||
# List all certificates
|
||||
curl http://localhost:8443/api/v1/certificates
|
||||
curl https://localhost:8443/api/v1/certificates
|
||||
|
||||
# Check certificate status
|
||||
curl http://localhost:8443/api/v1/certificates/{cert-id}
|
||||
curl https://localhost:8443/api/v1/certificates/{cert-id}
|
||||
|
||||
# View audit trail
|
||||
curl http://localhost:8443/api/v1/audit
|
||||
curl https://localhost:8443/api/v1/audit
|
||||
|
||||
# Check renewal policy compliance
|
||||
curl http://localhost:8443/api/v1/policies/{policy-id}
|
||||
curl https://localhost:8443/api/v1/policies/{policy-id}
|
||||
```
|
||||
|
||||
### Traefik Dashboard
|
||||
@@ -290,7 +297,7 @@ Changes are picked up automatically (file watcher enabled).
|
||||
docker compose logs certctl-agent | grep heartbeat
|
||||
|
||||
# Check deployment job status
|
||||
curl http://localhost:8443/api/v1/jobs | jq '.[] | select(.type == "Deployment")'
|
||||
curl https://localhost:8443/api/v1/jobs | jq '.[] | select(.type == "Deployment")'
|
||||
|
||||
# Check Traefik is watching the directory
|
||||
docker compose exec traefik ls -la /etc/traefik/certs/
|
||||
|
||||
@@ -119,7 +119,7 @@ services:
|
||||
networks:
|
||||
- certctl-network
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'curl -sf http://localhost:8443/health || exit 1']
|
||||
test: ['CMD-SHELL', 'curl -sfk https://localhost:8443/health || exit 1']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -48,6 +48,13 @@ Monitor logs:
|
||||
docker compose logs -f certctl-server
|
||||
```
|
||||
|
||||
## TLS Security
|
||||
|
||||
certctl is HTTPS-only as of v2.2. The demo compose stack provisions a self-signed certificate. When accessing `https://localhost:8443`, you can either:
|
||||
- Use `curl --cacert ./deploy/test/certs/ca.crt ...` to pin the CA certificate
|
||||
- Use `curl -k ...` for quick smoke tests (never in production)
|
||||
- Import the CA at `./deploy/test/certs/ca.crt` into your OS trust store for browser visits
|
||||
|
||||
Wait for all services to reach healthy state:
|
||||
|
||||
```bash
|
||||
@@ -69,7 +76,7 @@ certctl-haproxy-... healthy
|
||||
Open your browser to:
|
||||
|
||||
```
|
||||
http://localhost:8443
|
||||
https://localhost:8443
|
||||
```
|
||||
|
||||
You should see an empty dashboard. This is expected — no certificates issued yet.
|
||||
@@ -79,7 +86,7 @@ You should see an empty dashboard. This is expected — no certificates issued y
|
||||
This defines what certificates certctl can issue (key algorithm, max TTL, allowed names).
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8443/api/v1/profiles \
|
||||
curl -X POST https://localhost:8443/api/v1/profiles \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"name": "internal-web",
|
||||
@@ -94,7 +101,7 @@ curl -X POST http://localhost:8443/api/v1/profiles \
|
||||
This tells certctl where to deploy certificates on the HAProxy server.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8443/api/v1/targets \
|
||||
curl -X POST https://localhost:8443/api/v1/targets \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"name": "haproxy-01",
|
||||
@@ -115,7 +122,7 @@ Note: In the Docker Compose environment, reload command can be `kill -HUP $(pido
|
||||
This ties a certificate profile to a deployment target and sets renewal thresholds.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8443/api/v1/renewal-policies \
|
||||
curl -X POST https://localhost:8443/api/v1/renewal-policies \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"name": "haproxy-internal-web",
|
||||
@@ -130,7 +137,7 @@ curl -X POST http://localhost:8443/api/v1/renewal-policies \
|
||||
Get the issuer ID:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8443/api/v1/issuers | jq '.'
|
||||
curl https://localhost:8443/api/v1/issuers | jq '.'
|
||||
```
|
||||
|
||||
You should see `iss-stepca` in the list.
|
||||
@@ -140,7 +147,7 @@ You should see `iss-stepca` in the list.
|
||||
Request a certificate via the API. The server will sign it via step-ca.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8443/api/v1/certificates \
|
||||
curl -X POST https://localhost:8443/api/v1/certificates \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"common_name": "api.internal.example.com",
|
||||
@@ -155,7 +162,7 @@ curl -X POST http://localhost:8443/api/v1/certificates \
|
||||
Get the certificate ID and trigger deployment:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8443/api/v1/certificates/<cert_id>/deploy \
|
||||
curl -X POST https://localhost:8443/api/v1/certificates/<cert_id>/deploy \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"target_id": "<target_id_from_step_4>"
|
||||
@@ -171,7 +178,7 @@ The agent will:
|
||||
|
||||
### 8. Verify in Dashboard
|
||||
|
||||
Refresh http://localhost:8443 and you should see:
|
||||
Refresh https://localhost:8443 and you should see:
|
||||
- 1 certificate (status: Active, expiry in 90 days)
|
||||
- 1 deployment job (status: Completed)
|
||||
- 1 agent (heartbeat: recent)
|
||||
|
||||
+40
-2
@@ -75,6 +75,14 @@ EXAMPLES:
|
||||
--server-url https://certctl.example.com \\
|
||||
--api-key YOUR_API_KEY
|
||||
|
||||
CONTROL-PLANE TLS TRUST:
|
||||
The certctl server is HTTPS-only as of v2.2. This installer does NOT copy a CA
|
||||
bundle — the generated agent.env leaves TLS trust to the system root store by
|
||||
default. If the server uses a private/enterprise or self-signed CA, set
|
||||
CERTCTL_SERVER_CA_BUNDLE_PATH in the generated agent.env to point at the CA
|
||||
bundle, or (dev only) CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true. See the
|
||||
commented block in the generated agent.env for the full menu.
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -322,7 +330,7 @@ setup_linux_config() {
|
||||
# Agent ID (unique identifier in the fleet)
|
||||
CERTCTL_AGENT_ID=$AGENT_ID
|
||||
|
||||
# Control plane server URL
|
||||
# Control plane server URL (HTTPS-only as of v2.2)
|
||||
CERTCTL_SERVER_URL=$SERVER_URL
|
||||
|
||||
# API authentication key
|
||||
@@ -334,6 +342,21 @@ CERTCTL_KEYGEN_MODE=agent
|
||||
# Key storage directory (agent-side keygen)
|
||||
CERTCTL_KEY_DIR=$key_dir
|
||||
|
||||
# ---- Control-plane TLS trust ----
|
||||
# The certctl server is HTTPS-only (v2.2+). The agent's HTTP client MUST trust the
|
||||
# server's certificate chain. Pick ONE of the approaches below:
|
||||
#
|
||||
# 1) Public CA (Let's Encrypt, DigiCert, etc.) — no config needed; system trust store works.
|
||||
# 2) Private / enterprise CA — point the agent at the CA bundle that signed the server cert:
|
||||
# CERTCTL_SERVER_CA_BUNDLE_PATH=/etc/certctl/server-ca.crt
|
||||
#
|
||||
# 3) Self-signed server cert (Helm/compose bootstrap) — same env var, just point at the
|
||||
# extracted self-signed CA bundle (e.g. from the certctl-server-tls Kubernetes secret
|
||||
# via: kubectl get secret certctl-server-tls -o jsonpath='{.data.ca\.crt}' | base64 -d).
|
||||
#
|
||||
# 4) Dev/eval only — disable verification entirely (NEVER do this in production):
|
||||
# CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true
|
||||
|
||||
# Logging level (debug, info, warn, error)
|
||||
# CERTCTL_LOG_LEVEL=info
|
||||
|
||||
@@ -373,7 +396,7 @@ setup_macos_config() {
|
||||
# Agent ID (unique identifier in the fleet)
|
||||
CERTCTL_AGENT_ID=$AGENT_ID
|
||||
|
||||
# Control plane server URL
|
||||
# Control plane server URL (HTTPS-only as of v2.2)
|
||||
CERTCTL_SERVER_URL=$SERVER_URL
|
||||
|
||||
# API authentication key
|
||||
@@ -385,6 +408,21 @@ CERTCTL_KEYGEN_MODE=agent
|
||||
# Key storage directory (agent-side keygen)
|
||||
CERTCTL_KEY_DIR=$key_dir
|
||||
|
||||
# ---- Control-plane TLS trust ----
|
||||
# The certctl server is HTTPS-only (v2.2+). The agent's HTTP client MUST trust the
|
||||
# server's certificate chain. Pick ONE of the approaches below:
|
||||
#
|
||||
# 1) Public CA (Let's Encrypt, DigiCert, etc.) — no config needed; system trust store works.
|
||||
# 2) Private / enterprise CA — point the agent at the CA bundle that signed the server cert:
|
||||
# CERTCTL_SERVER_CA_BUNDLE_PATH=$HOME/.certctl/server-ca.crt
|
||||
#
|
||||
# 3) Self-signed server cert (Helm/compose bootstrap) — same env var, just point at the
|
||||
# extracted self-signed CA bundle (e.g. from the certctl-server-tls Kubernetes secret
|
||||
# via: kubectl get secret certctl-server-tls -o jsonpath='{.data.ca\.crt}' | base64 -d).
|
||||
#
|
||||
# 4) Dev/eval only — disable verification entirely (NEVER do this in production):
|
||||
# CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY=true
|
||||
|
||||
# Logging level (debug, info, warn, error)
|
||||
# CERTCTL_LOG_LEVEL=info
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
"github.com/shankar0123/certctl/internal/service"
|
||||
)
|
||||
|
||||
// MockAgentService is a mock implementation of AgentService interface.
|
||||
@@ -24,6 +25,11 @@ type MockAgentService struct {
|
||||
GetWorkFn func(agentID string) ([]domain.Job, error)
|
||||
GetWorkWithTargetsFn func(agentID string) ([]domain.WorkItem, error)
|
||||
UpdateJobStatusFn func(agentID string, jobID string, status string, errMsg string) error
|
||||
// I-004: soft-retirement hooks. Tests that don't set these receive nil
|
||||
// results and nil errors, which mirrors the safest default (no-op) for
|
||||
// unrelated suites that mock only the legacy surface.
|
||||
RetireAgentFn func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error)
|
||||
ListRetiredAgentsFn func(page, perPage int) ([]domain.Agent, int64, error)
|
||||
}
|
||||
|
||||
func (m *MockAgentService) ListAgents(_ context.Context, page, perPage int) ([]domain.Agent, int64, error) {
|
||||
@@ -96,6 +102,25 @@ func (m *MockAgentService) UpdateJobStatus(_ context.Context, agentID string, jo
|
||||
return nil
|
||||
}
|
||||
|
||||
// RetireAgent is the I-004 soft-retirement entrypoint. Tests that don't set
|
||||
// RetireAgentFn get a nil result + nil error, which is a no-op response that
|
||||
// lets unrelated suites compile without caring about the retirement surface.
|
||||
func (m *MockAgentService) RetireAgent(_ context.Context, agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
if m.RetireAgentFn != nil {
|
||||
return m.RetireAgentFn(agentID, actor, force, reason)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ListRetiredAgents returns retired rows for the retired-agents tab / audit
|
||||
// views. Same zero-value default as RetireAgent for unrelated tests.
|
||||
func (m *MockAgentService) ListRetiredAgents(_ context.Context, page, perPage int) ([]domain.Agent, int64, error) {
|
||||
if m.ListRetiredAgentsFn != nil {
|
||||
return m.ListRetiredAgentsFn(page, perPage)
|
||||
}
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
// Test ListAgents - success case
|
||||
func TestListAgents_Success(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
@@ -0,0 +1,393 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
"github.com/shankar0123/certctl/internal/service"
|
||||
)
|
||||
|
||||
// agentRetireTestSetup builds an AgentHandler with a mock AgentService whose
|
||||
// RetireAgent / ListRetiredAgents / Heartbeat behavior is driven by the
|
||||
// returned mock. Keeps every I-004 handler test self-contained so a single
|
||||
// failing assertion can't cascade through a shared fixture.
|
||||
func agentRetireTestSetup() (*MockAgentService, AgentHandler) {
|
||||
mock := &MockAgentService{}
|
||||
handler := NewAgentHandler(mock)
|
||||
return mock, handler
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_Success_200 pins the happy-path contract for the
|
||||
// soft-retirement HTTP surface: DELETE /api/v1/agents/{id} with no dependency
|
||||
// fallout returns 200 OK and a JSON body echoing retirement metadata
|
||||
// (retired_at timestamp, already_retired=false, cascade=false, zero counts).
|
||||
// Operators building dashboards parse these fields; keep the shape stable.
|
||||
func TestRetireAgentHandler_Success_200(t *testing.T) {
|
||||
retiredAt := time.Date(2026, 4, 18, 12, 0, 0, 0, time.UTC)
|
||||
mock, handler := agentRetireTestSetup()
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
if agentID != "a-prod-001" {
|
||||
t.Fatalf("retire handler received agentID=%q want a-prod-001", agentID)
|
||||
}
|
||||
if force {
|
||||
t.Fatalf("retire handler set force=true unexpectedly; default path must be force=false")
|
||||
}
|
||||
return &service.AgentRetirementResult{
|
||||
AlreadyRetired: false,
|
||||
Cascade: false,
|
||||
RetiredAt: retiredAt,
|
||||
Counts: domain.AgentDependencyCounts{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/agents/a-prod-001", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s want 200", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var body struct {
|
||||
RetiredAt time.Time `json:"retired_at"`
|
||||
AlreadyRetired bool `json:"already_retired"`
|
||||
Cascade bool `json:"cascade"`
|
||||
Counts domain.AgentDependencyCounts `json:"counts"`
|
||||
}
|
||||
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode 200 body: %v", err)
|
||||
}
|
||||
if !body.RetiredAt.Equal(retiredAt) {
|
||||
t.Errorf("retired_at=%v want %v", body.RetiredAt, retiredAt)
|
||||
}
|
||||
if body.AlreadyRetired {
|
||||
t.Errorf("already_retired=true want false on clean retire")
|
||||
}
|
||||
if body.Cascade {
|
||||
t.Errorf("cascade=true want false on clean retire")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_AlreadyRetired_204 covers the idempotent contract: a
|
||||
// retire call against an already-retired agent completes with 204 No Content
|
||||
// (no body). This lets operators safely re-issue the DELETE after a network
|
||||
// blip without fearing duplicate audit events or state mutations.
|
||||
func TestRetireAgentHandler_AlreadyRetired_204(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
past := time.Now().Add(-24 * time.Hour)
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
return &service.AgentRetirementResult{
|
||||
AlreadyRetired: true,
|
||||
Cascade: false,
|
||||
RetiredAt: past,
|
||||
Counts: domain.AgentDependencyCounts{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/agents/a-prod-001", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("status=%d body=%s want 204", w.Code, w.Body.String())
|
||||
}
|
||||
// 204 No Content must have zero body. If anything leaks through, downstream
|
||||
// clients (curl scripts, dashboards) break.
|
||||
if w.Body.Len() != 0 {
|
||||
t.Errorf("204 body=%q want empty", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_Sentinel_403 covers the hard guard against retiring
|
||||
// any of the four sentinel agents that back discovery sources and the
|
||||
// network scanner. These IDs are reserved; the handler must surface the
|
||||
// service-layer ErrAgentIsSentinel as 403 Forbidden regardless of force/reason
|
||||
// because no operator intent can legitimately retire them.
|
||||
func TestRetireAgentHandler_Sentinel_403(t *testing.T) {
|
||||
sentinels := []string{"server-scanner", "cloud-aws-sm", "cloud-azure-kv", "cloud-gcp-sm"}
|
||||
for _, id := range sentinels {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
return nil, service.ErrAgentIsSentinel
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/agents/"+id, nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("sentinel %q status=%d body=%s want 403", id, w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_NotFound_404 covers the lookup-miss path. Service
|
||||
// returns a not-found error; handler maps to 404. Keeping the error
|
||||
// discrimination at the service layer (sentinel errors.Is) rather than string
|
||||
// matching is the whole point of wrapping.
|
||||
func TestRetireAgentHandler_NotFound_404(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
return nil, errors.New("agent not found")
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/agents/unknown-id", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("status=%d body=%s want 404", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_Blocked_409_WithCounts covers the preflight-blocked
|
||||
// path. Service returns *BlockedByDependenciesError wrapping
|
||||
// ErrBlockedByDependencies; handler unwraps via errors.As, maps to 409, and
|
||||
// MUST include the counts in the response body so operators know what's
|
||||
// blocking them. Without counts the 409 is useless — the operator has to
|
||||
// guess which downstream dependency is holding up the retirement.
|
||||
func TestRetireAgentHandler_Blocked_409_WithCounts(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
blockCounts := domain.AgentDependencyCounts{
|
||||
ActiveTargets: 3,
|
||||
ActiveCertificates: 7,
|
||||
PendingJobs: 2,
|
||||
}
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
return nil, &service.BlockedByDependenciesError{Counts: blockCounts}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/agents/a-prod-001", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusConflict {
|
||||
t.Fatalf("status=%d body=%s want 409", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Error string `json:"error"`
|
||||
Message string `json:"message"`
|
||||
Counts domain.AgentDependencyCounts `json:"counts"`
|
||||
}
|
||||
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode 409 body: %v", err)
|
||||
}
|
||||
if body.Counts.ActiveTargets != 3 {
|
||||
t.Errorf("counts.active_targets=%d want 3", body.Counts.ActiveTargets)
|
||||
}
|
||||
if body.Counts.ActiveCertificates != 7 {
|
||||
t.Errorf("counts.active_certificates=%d want 7", body.Counts.ActiveCertificates)
|
||||
}
|
||||
if body.Counts.PendingJobs != 2 {
|
||||
t.Errorf("counts.pending_jobs=%d want 2", body.Counts.PendingJobs)
|
||||
}
|
||||
if body.Message == "" {
|
||||
t.Errorf("409 body missing human-readable message; operators need guidance")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_Force_NoReason_400 covers the force-escape-hatch
|
||||
// guardrail: force=true without a non-empty reason must be rejected at the
|
||||
// handler seam BEFORE the service performs any DB work, because a
|
||||
// reason-less cascade is unauditable. Service returns ErrForceReasonRequired;
|
||||
// handler maps to 400.
|
||||
func TestRetireAgentHandler_Force_NoReason_400(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
if !force {
|
||||
t.Fatalf("handler did not forward force=true; force query param was dropped")
|
||||
}
|
||||
if reason != "" {
|
||||
t.Fatalf("handler passed reason=%q; empty reason must reach service for error path", reason)
|
||||
}
|
||||
return nil, service.ErrForceReasonRequired
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/agents/a-prod-001?force=true", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d body=%s want 400", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_ForceCascade_200 covers the successful force-cascade
|
||||
// path: DELETE ?force=true&reason=... → service executes transactional
|
||||
// cascade → 200 with cascade=true and the pre-cascade counts echoed back so
|
||||
// the operator's confirmation dialog can show "I just retired N targets,
|
||||
// M certificates, K pending jobs."
|
||||
func TestRetireAgentHandler_ForceCascade_200(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
retiredAt := time.Date(2026, 4, 18, 14, 30, 0, 0, time.UTC)
|
||||
mock.RetireAgentFn = func(agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error) {
|
||||
if !force {
|
||||
t.Fatalf("handler did not forward force=true; query-param parsing broken")
|
||||
}
|
||||
if reason != "decommissioning rack 7" {
|
||||
t.Fatalf("handler forwarded reason=%q want %q", reason, "decommissioning rack 7")
|
||||
}
|
||||
return &service.AgentRetirementResult{
|
||||
AlreadyRetired: false,
|
||||
Cascade: true,
|
||||
RetiredAt: retiredAt,
|
||||
Counts: domain.AgentDependencyCounts{
|
||||
ActiveTargets: 2,
|
||||
ActiveCertificates: 5,
|
||||
PendingJobs: 1,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
url := "/api/v1/agents/a-prod-001?force=true&reason=decommissioning+rack+7"
|
||||
req := httptest.NewRequest(http.MethodDelete, url, nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s want 200", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var body struct {
|
||||
RetiredAt time.Time `json:"retired_at"`
|
||||
AlreadyRetired bool `json:"already_retired"`
|
||||
Cascade bool `json:"cascade"`
|
||||
Counts domain.AgentDependencyCounts `json:"counts"`
|
||||
}
|
||||
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode force-cascade 200 body: %v", err)
|
||||
}
|
||||
if !body.Cascade {
|
||||
t.Errorf("cascade=false want true on ?force=true successful retire")
|
||||
}
|
||||
if body.Counts.ActiveTargets != 2 || body.Counts.ActiveCertificates != 5 || body.Counts.PendingJobs != 1 {
|
||||
t.Errorf("counts=%+v want {ActiveTargets:2 ActiveCertificates:5 PendingJobs:1}", body.Counts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatHandler_RetiredAgent_410 covers the agent-shutdown signal. A
|
||||
// retired agent that is still polling must be told its identity is gone
|
||||
// (410 Gone) rather than offered the normal 200 "recorded" response.
|
||||
// cmd/agent treats 410 as a terminal signal and exits rather than looping
|
||||
// forever against a decommissioned identity. Service returns ErrAgentRetired;
|
||||
// handler maps to 410.
|
||||
func TestHeartbeatHandler_RetiredAgent_410(t *testing.T) {
|
||||
mock, handler := agentRetireTestSetup()
|
||||
mock.HeartbeatFn = func(agentID string, metadata *domain.AgentMetadata) error {
|
||||
return service.ErrAgentRetired
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/agents/a-prod-001/heartbeat", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.Heartbeat(w, req)
|
||||
|
||||
if w.Code != http.StatusGone {
|
||||
t.Fatalf("heartbeat(retired) status=%d body=%s want 410", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestListRetiredAgentsHandler_Success covers the audit/forensics-facing
|
||||
// endpoint GET /api/v1/agents/retired. Returns a paged list of retired rows
|
||||
// alongside total count so the GUI can render a "Retired Agents" tab with
|
||||
// pagination. Default listing (GET /agents) hides retired rows; this is the
|
||||
// opt-in surface for them.
|
||||
func TestListRetiredAgentsHandler_Success(t *testing.T) {
|
||||
past := time.Now().Add(-48 * time.Hour)
|
||||
reason := "old hardware"
|
||||
retired := []domain.Agent{
|
||||
{
|
||||
ID: "agent-retired-01",
|
||||
Name: "decom-01",
|
||||
Hostname: "server-old",
|
||||
Status: domain.AgentStatusOffline,
|
||||
RegisteredAt: past,
|
||||
RetiredAt: &past,
|
||||
RetiredReason: &reason,
|
||||
},
|
||||
}
|
||||
|
||||
mock, handler := agentRetireTestSetup()
|
||||
mock.ListRetiredAgentsFn = func(page, perPage int) ([]domain.Agent, int64, error) {
|
||||
if page != 1 || perPage != 50 {
|
||||
t.Fatalf("ListRetired handler received page=%d perPage=%d want 1/50 defaults", page, perPage)
|
||||
}
|
||||
return retired, 1, nil
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/agents/retired", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.ListRetiredAgents(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s want 200", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var response PagedResponse
|
||||
if err := json.NewDecoder(w.Body).Decode(&response); err != nil {
|
||||
t.Fatalf("decode list-retired body: %v", err)
|
||||
}
|
||||
if response.Total != 1 {
|
||||
t.Errorf("total=%d want 1", response.Total)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentHandler_MethodNotAllowed covers defense-in-depth: only
|
||||
// DELETE is valid on /api/v1/agents/{id} for retirement. Using POST/PUT/PATCH
|
||||
// must be rejected with 405 so misconfigured callers don't accidentally
|
||||
// trigger retirement via a wrong-method request.
|
||||
func TestRetireAgentHandler_MethodNotAllowed(t *testing.T) {
|
||||
_, handler := agentRetireTestSetup()
|
||||
|
||||
for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodPatch} {
|
||||
t.Run(method, func(t *testing.T) {
|
||||
req := httptest.NewRequest(method, "/api/v1/agents/a-prod-001", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RetireAgent(w, req)
|
||||
|
||||
if w.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("method=%s status=%d want 405", method, w.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Compile-time asserts: the mock must satisfy the handler's AgentService
|
||||
// interface. Red state: this fails until the interface grows RetireAgent +
|
||||
// ListRetiredAgents. Once Phase 2b adds those methods to AgentService, this
|
||||
// assertion goes green along with every test above.
|
||||
var _ AgentService = (*MockAgentService)(nil)
|
||||
|
||||
// Unused-import suppressor for context — the package-level tests already
|
||||
// pull context from agent_handler_test.go, but leaving this here documents
|
||||
// that the mock methods receive context.Context values even though this
|
||||
// file's tests don't construct them directly (they ride on httptest.NewRequest).
|
||||
var _ = context.Background
|
||||
@@ -3,16 +3,24 @@ package handler
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/api/middleware"
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
"github.com/shankar0123/certctl/internal/service"
|
||||
)
|
||||
|
||||
// AgentService defines the service interface for agent operations.
|
||||
//
|
||||
// I-004 expansion: RetireAgent + ListRetiredAgents back the soft-retirement
|
||||
// surface. The handler depends on the service-package's AgentRetirementResult
|
||||
// and BlockedByDependenciesError types for result shape + errors.As unwrap,
|
||||
// which is why this file imports internal/service.
|
||||
type AgentService interface {
|
||||
ListAgents(ctx context.Context, page, perPage int) ([]domain.Agent, int64, error)
|
||||
GetAgent(ctx context.Context, id string) (*domain.Agent, error)
|
||||
@@ -24,6 +32,10 @@ type AgentService interface {
|
||||
GetWork(ctx context.Context, agentID string) ([]domain.Job, error)
|
||||
GetWorkWithTargets(ctx context.Context, agentID string) ([]domain.WorkItem, error)
|
||||
UpdateJobStatus(ctx context.Context, agentID string, jobID string, status string, errMsg string) error
|
||||
// I-004 soft-retirement API. Both default to no-op (nil result / nil error)
|
||||
// in mocks that don't override them — handler tests opt in per suite.
|
||||
RetireAgent(ctx context.Context, agentID, actor string, force bool, reason string) (*service.AgentRetirementResult, error)
|
||||
ListRetiredAgents(ctx context.Context, page, perPage int) ([]domain.Agent, int64, error)
|
||||
}
|
||||
|
||||
// AgentHandler handles HTTP requests for agent operations.
|
||||
@@ -190,6 +202,15 @@ func (h AgentHandler) Heartbeat(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
if err := h.svc.Heartbeat(r.Context(), agentID, metadata); err != nil {
|
||||
// I-004: a retired agent still polling must receive 410 Gone so
|
||||
// cmd/agent detects the terminal signal and shuts down cleanly
|
||||
// instead of looping forever against a decommissioned identity.
|
||||
// Check this FIRST — before "not found" string matching — so the
|
||||
// retired-path is never masked by a sibling error branch.
|
||||
if errors.Is(err, service.ErrAgentRetired) {
|
||||
ErrorWithRequestID(w, http.StatusGone, "Agent has been retired", requestID)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
ErrorWithRequestID(w, http.StatusNotFound, "Agent not found", requestID)
|
||||
return
|
||||
@@ -376,3 +397,181 @@ func (h AgentHandler) AgentReportJobStatus(w http.ResponseWriter, r *http.Reques
|
||||
"status": "updated",
|
||||
})
|
||||
}
|
||||
|
||||
// RetireAgent executes the I-004 soft-retirement surface.
|
||||
// DELETE /api/v1/agents/{id}[?force=true&reason=...]
|
||||
//
|
||||
// Contract (pinned by agent_retire_handler_test.go):
|
||||
//
|
||||
// 405 any method other than DELETE
|
||||
// 200 clean retire (body: retired_at, already_retired=false, cascade=false, counts=0s)
|
||||
// 200 force-cascade retire (body: cascade=true, counts=pre-cascade snapshot)
|
||||
// 204 idempotent retire of an already-retired agent (NO body — downstream
|
||||
// clients that tee responses into dashboards break on spurious bodies)
|
||||
// 400 force=true without a non-empty reason (ErrForceReasonRequired)
|
||||
// 403 one of the four reserved sentinel IDs (ErrAgentIsSentinel)
|
||||
// 404 agent does not exist ("not found" string match, kept for compat with
|
||||
// repo error strings; sentinel checks run first so they never mask)
|
||||
// 409 blocked by preflight counts (*BlockedByDependenciesError) — body
|
||||
// carries the per-bucket counts so the operator UI can tell the
|
||||
// human which downstream dependency is holding up the retirement,
|
||||
// rather than forcing them to re-run the DELETE with ?force=true
|
||||
// and guess
|
||||
// 500 anything else
|
||||
//
|
||||
// The 409 body intentionally does NOT go through ErrorWithRequestID because
|
||||
// that helper's ErrorResponse shape has no `counts` field — we inline-marshal
|
||||
// a custom body instead. Keeping this shape stable is important: the GUI
|
||||
// pattern is "show the 409 dialog, list the N targets / M certs / K jobs
|
||||
// blocking, let the operator retire them first or tick the force checkbox."
|
||||
func (h AgentHandler) RetireAgent(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodDelete {
|
||||
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
||||
return
|
||||
}
|
||||
|
||||
requestID := middleware.GetRequestID(r.Context())
|
||||
|
||||
// Extract {id} from /api/v1/agents/{id}. Mirror GetAgent's pattern so
|
||||
// the path parser is identical across the agent handler surface and a
|
||||
// future refactor can extract it once without introducing drift.
|
||||
rawID := strings.TrimPrefix(r.URL.Path, "/api/v1/agents/")
|
||||
parts := strings.Split(rawID, "/")
|
||||
if len(parts) == 0 || parts[0] == "" {
|
||||
ErrorWithRequestID(w, http.StatusBadRequest, "Agent ID is required", requestID)
|
||||
return
|
||||
}
|
||||
id := parts[0]
|
||||
|
||||
// Parse optional force + reason. A missing `force` param is treated as
|
||||
// force=false (the default, safe path); anything strconv.ParseBool rejects
|
||||
// is also force=false so a malformed query can never silently enable the
|
||||
// cascade. The reason string is passed through verbatim — the service
|
||||
// owns the "force=true requires reason" rule.
|
||||
query := r.URL.Query()
|
||||
force := false
|
||||
if fv := query.Get("force"); fv != "" {
|
||||
if parsed, err := strconv.ParseBool(fv); err == nil {
|
||||
force = parsed
|
||||
}
|
||||
}
|
||||
reason := query.Get("reason")
|
||||
|
||||
actor := resolveActor(r.Context())
|
||||
|
||||
result, err := h.svc.RetireAgent(r.Context(), id, actor, force, reason)
|
||||
if err != nil {
|
||||
// Sentinel + typed-error checks run BEFORE string matching on "not
|
||||
// found" so a repo error that happens to contain those words can
|
||||
// never mask a structural refusal (403/400/409). Order matters.
|
||||
if errors.Is(err, service.ErrAgentIsSentinel) {
|
||||
ErrorWithRequestID(w, http.StatusForbidden, "Agent is a reserved sentinel and cannot be retired", requestID)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrForceReasonRequired) {
|
||||
ErrorWithRequestID(w, http.StatusBadRequest, "force=true requires a non-empty reason", requestID)
|
||||
return
|
||||
}
|
||||
var blocked *service.BlockedByDependenciesError
|
||||
if errors.As(err, &blocked) {
|
||||
// Custom 409 body with per-bucket counts. ErrorResponse has no
|
||||
// `counts` field, so we marshal a bespoke struct instead.
|
||||
// Keep `error`/`message`/`counts` as the stable shape — any
|
||||
// dashboard parsing this relies on those three keys.
|
||||
body := struct {
|
||||
Error string `json:"error"`
|
||||
Message string `json:"message"`
|
||||
Counts domain.AgentDependencyCounts `json:"counts"`
|
||||
}{
|
||||
Error: "blocked_by_dependencies",
|
||||
Message: "Agent has active downstream dependencies. Retire or reassign them " +
|
||||
"first, or re-run with ?force=true&reason=... to cascade.",
|
||||
Counts: blocked.Counts,
|
||||
}
|
||||
JSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
ErrorWithRequestID(w, http.StatusNotFound, "Agent not found", requestID)
|
||||
return
|
||||
}
|
||||
slog.Error("RetireAgent failed", "agent_id", id, "error", err.Error())
|
||||
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to retire agent", requestID)
|
||||
return
|
||||
}
|
||||
|
||||
// Idempotent retire: the agent was already retired, so we return 204 No
|
||||
// Content with a ZERO-length body. The Red contract (test line 106) fails
|
||||
// if even a trailing newline leaks into the response. WriteHeader alone
|
||||
// emits the status without invoking the JSON encoder.
|
||||
if result.AlreadyRetired {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
// Clean retire (force=false) or successful cascade (force=true). Body
|
||||
// shape pinned by Red contract: retired_at, already_retired, cascade,
|
||||
// counts. Omitempty is deliberately NOT used — operators parsing the
|
||||
// response expect every field to always be present.
|
||||
JSON(w, http.StatusOK, struct {
|
||||
RetiredAt time.Time `json:"retired_at"`
|
||||
AlreadyRetired bool `json:"already_retired"`
|
||||
Cascade bool `json:"cascade"`
|
||||
Counts domain.AgentDependencyCounts `json:"counts"`
|
||||
}{
|
||||
RetiredAt: result.RetiredAt,
|
||||
AlreadyRetired: result.AlreadyRetired,
|
||||
Cascade: result.Cascade,
|
||||
Counts: result.Counts,
|
||||
})
|
||||
}
|
||||
|
||||
// ListRetiredAgents returns the opt-in listing of retired agents for the
|
||||
// operator UI's "Retired" tab and for audit/forensics workflows.
|
||||
// GET /api/v1/agents/retired?page=1&per_page=50
|
||||
//
|
||||
// The default ListAgents handler hides retired rows; this is the dedicated
|
||||
// surface for reading them back. Pagination defaults match ListAgents so
|
||||
// the GUI can reuse the same query hook (page=1, per_page=50, cap 500).
|
||||
//
|
||||
// Go 1.22's enhanced ServeMux routes `/agents/retired` to this handler via
|
||||
// the literal-beats-pattern-var precedence rule (literal `retired` wins over
|
||||
// `{id}` in the sibling GET /api/v1/agents/{id} route), so both entries can
|
||||
// coexist without conflict. If that precedence ever regresses, the failure
|
||||
// mode is TestListRetiredAgentsHandler_Success blowing up with a 404 — which
|
||||
// is the fast signal we want.
|
||||
func (h AgentHandler) ListRetiredAgents(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
||||
return
|
||||
}
|
||||
|
||||
requestID := middleware.GetRequestID(r.Context())
|
||||
|
||||
page := 1
|
||||
perPage := 50
|
||||
query := r.URL.Query()
|
||||
if p := query.Get("page"); p != "" {
|
||||
if parsed, err := strconv.Atoi(p); err == nil && parsed > 0 {
|
||||
page = parsed
|
||||
}
|
||||
}
|
||||
if pp := query.Get("per_page"); pp != "" {
|
||||
if parsed, err := strconv.Atoi(pp); err == nil && parsed > 0 && parsed <= 500 {
|
||||
perPage = parsed
|
||||
}
|
||||
}
|
||||
|
||||
agents, total, err := h.svc.ListRetiredAgents(r.Context(), page, perPage)
|
||||
if err != nil {
|
||||
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to list retired agents", requestID)
|
||||
return
|
||||
}
|
||||
|
||||
JSON(w, http.StatusOK, PagedResponse{
|
||||
Data: agents,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PerPage: perPage,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -41,20 +41,26 @@ type MetricsResponse struct {
|
||||
|
||||
// MetricsGauge represents gauge metrics (point-in-time values).
|
||||
type MetricsGauge struct {
|
||||
CertificateTotal int64 `json:"certificate_total"`
|
||||
CertificateActive int64 `json:"certificate_active"`
|
||||
CertificateExpiringSoon int64 `json:"certificate_expiring_soon"` // Within 30d
|
||||
CertificateExpired int64 `json:"certificate_expired"`
|
||||
CertificateRevoked int64 `json:"certificate_revoked"`
|
||||
AgentTotal int64 `json:"agent_total"`
|
||||
AgentOnline int64 `json:"agent_online"`
|
||||
JobPending int64 `json:"job_pending"`
|
||||
CertificateTotal int64 `json:"certificate_total"`
|
||||
CertificateActive int64 `json:"certificate_active"`
|
||||
CertificateExpiringSoon int64 `json:"certificate_expiring_soon"` // Within 30d
|
||||
CertificateExpired int64 `json:"certificate_expired"`
|
||||
CertificateRevoked int64 `json:"certificate_revoked"`
|
||||
AgentTotal int64 `json:"agent_total"`
|
||||
AgentOnline int64 `json:"agent_online"`
|
||||
JobPending int64 `json:"job_pending"`
|
||||
}
|
||||
|
||||
// MetricsCounter represents counter metrics (cumulative values).
|
||||
type MetricsCounter struct {
|
||||
JobCompletedTotal int64 `json:"job_completed_total"`
|
||||
JobFailedTotal int64 `json:"job_failed_total"`
|
||||
// NotificationsDeadTotal is a point-in-time count of notifications in the
|
||||
// dead-letter queue (status="dead"), exposed here with the _total suffix
|
||||
// to match Prometheus DB-snapshot counter convention (same semantics as
|
||||
// JobFailedTotal and JobCompletedTotal — see metrics.md). I-005 DLQ
|
||||
// observability gate.
|
||||
NotificationsDeadTotal int64 `json:"notifications_dead_total"`
|
||||
}
|
||||
|
||||
// UptimeMetric represents server uptime information.
|
||||
@@ -95,18 +101,19 @@ func (h MetricsHandler) GetMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
// Build metrics response
|
||||
metricsResp := MetricsResponse{
|
||||
Gauge: MetricsGauge{
|
||||
CertificateTotal: dashboardSummary.TotalCertificates,
|
||||
CertificateActive: dashboardSummary.TotalCertificates - dashboardSummary.ExpiringCertificates - dashboardSummary.ExpiredCertificates - dashboardSummary.RevokedCertificates,
|
||||
CertificateTotal: dashboardSummary.TotalCertificates,
|
||||
CertificateActive: dashboardSummary.TotalCertificates - dashboardSummary.ExpiringCertificates - dashboardSummary.ExpiredCertificates - dashboardSummary.RevokedCertificates,
|
||||
CertificateExpiringSoon: dashboardSummary.ExpiringCertificates,
|
||||
CertificateExpired: dashboardSummary.ExpiredCertificates,
|
||||
CertificateRevoked: dashboardSummary.RevokedCertificates,
|
||||
AgentTotal: dashboardSummary.TotalAgents,
|
||||
AgentOnline: dashboardSummary.ActiveAgents,
|
||||
JobPending: dashboardSummary.PendingJobs,
|
||||
CertificateExpired: dashboardSummary.ExpiredCertificates,
|
||||
CertificateRevoked: dashboardSummary.RevokedCertificates,
|
||||
AgentTotal: dashboardSummary.TotalAgents,
|
||||
AgentOnline: dashboardSummary.ActiveAgents,
|
||||
JobPending: dashboardSummary.PendingJobs,
|
||||
},
|
||||
Counter: MetricsCounter{
|
||||
JobCompletedTotal: dashboardSummary.CompleteJobs,
|
||||
JobFailedTotal: dashboardSummary.FailedJobs,
|
||||
JobCompletedTotal: dashboardSummary.CompleteJobs,
|
||||
JobFailedTotal: dashboardSummary.FailedJobs,
|
||||
NotificationsDeadTotal: dashboardSummary.NotificationsDead,
|
||||
},
|
||||
Uptime: UptimeMetric{
|
||||
UptimeSeconds: int64(time.Since(h.serverStarted).Seconds()),
|
||||
@@ -200,6 +207,17 @@ func (h MetricsHandler) GetPrometheusMetrics(w http.ResponseWriter, r *http.Requ
|
||||
fmt.Fprintf(w, "# TYPE certctl_job_failed_total counter\n")
|
||||
fmt.Fprintf(w, "certctl_job_failed_total %d\n\n", dashboardSummary.FailedJobs)
|
||||
|
||||
// I-005: notification dead-letter queue depth. Emitted with the _total
|
||||
// suffix to match the existing certctl_job_completed_total /
|
||||
// certctl_job_failed_total convention for DB-snapshot counters — the
|
||||
// value is a point-in-time COUNT(*) of notification_events rows where
|
||||
// status='dead', not a monotonically increasing process-lifetime counter.
|
||||
// Operators alert on this as "dead-letter depth" (thresholds in the
|
||||
// I-005 spec: > 0 → warning, > 10 → critical).
|
||||
fmt.Fprintf(w, "# HELP certctl_notification_dead_total Number of notifications in the dead-letter queue.\n")
|
||||
fmt.Fprintf(w, "# TYPE certctl_notification_dead_total counter\n")
|
||||
fmt.Fprintf(w, "certctl_notification_dead_total %d\n\n", dashboardSummary.NotificationsDead)
|
||||
|
||||
// Info — server uptime
|
||||
fmt.Fprintf(w, "# HELP certctl_uptime_seconds Server uptime in seconds.\n")
|
||||
fmt.Fprintf(w, "# TYPE certctl_uptime_seconds gauge\n")
|
||||
@@ -209,15 +227,21 @@ func (h MetricsHandler) GetPrometheusMetrics(w http.ResponseWriter, r *http.Requ
|
||||
// DashboardSummary mirrors the service.DashboardSummary for JSON unmarshaling.
|
||||
// JSON tags must match the service-layer struct exactly.
|
||||
type DashboardSummary struct {
|
||||
TotalCertificates int64 `json:"total_certificates"`
|
||||
ExpiringCertificates int64 `json:"expiring_certificates"`
|
||||
ExpiredCertificates int64 `json:"expired_certificates"`
|
||||
RevokedCertificates int64 `json:"revoked_certificates"`
|
||||
ActiveAgents int64 `json:"active_agents"`
|
||||
OfflineAgents int64 `json:"offline_agents"`
|
||||
TotalAgents int64 `json:"total_agents"`
|
||||
PendingJobs int64 `json:"pending_jobs"`
|
||||
FailedJobs int64 `json:"failed_jobs"`
|
||||
CompleteJobs int64 `json:"complete_jobs"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
TotalCertificates int64 `json:"total_certificates"`
|
||||
ExpiringCertificates int64 `json:"expiring_certificates"`
|
||||
ExpiredCertificates int64 `json:"expired_certificates"`
|
||||
RevokedCertificates int64 `json:"revoked_certificates"`
|
||||
ActiveAgents int64 `json:"active_agents"`
|
||||
OfflineAgents int64 `json:"offline_agents"`
|
||||
TotalAgents int64 `json:"total_agents"`
|
||||
PendingJobs int64 `json:"pending_jobs"`
|
||||
FailedJobs int64 `json:"failed_jobs"`
|
||||
CompleteJobs int64 `json:"complete_jobs"`
|
||||
// NotificationsDead mirrors service.DashboardSummary.NotificationsDead.
|
||||
// JSON tag "notifications_dead" must match the service-layer struct
|
||||
// exactly — this cross-package mirror avoids a direct import cycle and
|
||||
// is driven by the I-005 Prometheus counter emission path. See
|
||||
// GetPrometheusMetrics and MetricsCounter.NotificationsDeadTotal.
|
||||
NotificationsDead int64 `json:"notifications_dead"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
@@ -13,9 +13,11 @@ import (
|
||||
|
||||
// MockNotificationService is a mock implementation of NotificationService interface.
|
||||
type MockNotificationService struct {
|
||||
ListNotificationsFn func(page, perPage int) ([]domain.NotificationEvent, int64, error)
|
||||
GetNotificationFn func(id string) (*domain.NotificationEvent, error)
|
||||
MarkAsReadFn func(id string) error
|
||||
ListNotificationsFn func(page, perPage int) ([]domain.NotificationEvent, int64, error)
|
||||
ListNotificationsByStatusFn func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error)
|
||||
GetNotificationFn func(id string) (*domain.NotificationEvent, error)
|
||||
MarkAsReadFn func(id string) error
|
||||
RequeueFn func(id string) error
|
||||
}
|
||||
|
||||
func (m *MockNotificationService) ListNotifications(_ context.Context, page, perPage int) ([]domain.NotificationEvent, int64, error) {
|
||||
@@ -25,6 +27,13 @@ func (m *MockNotificationService) ListNotifications(_ context.Context, page, per
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
func (m *MockNotificationService) ListNotificationsByStatus(_ context.Context, status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
|
||||
if m.ListNotificationsByStatusFn != nil {
|
||||
return m.ListNotificationsByStatusFn(status, page, perPage)
|
||||
}
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
func (m *MockNotificationService) GetNotification(_ context.Context, id string) (*domain.NotificationEvent, error) {
|
||||
if m.GetNotificationFn != nil {
|
||||
return m.GetNotificationFn(id)
|
||||
@@ -39,6 +48,13 @@ func (m *MockNotificationService) MarkAsRead(_ context.Context, id string) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockNotificationService) RequeueNotification(_ context.Context, id string) error {
|
||||
if m.RequeueFn != nil {
|
||||
return m.RequeueFn(id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestListNotifications_Success(t *testing.T) {
|
||||
now := time.Now()
|
||||
certID := "mc-prod-001"
|
||||
@@ -282,3 +298,224 @@ func TestMarkAsRead_EmptyID(t *testing.T) {
|
||||
t.Fatalf("expected status 400, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// I-005: Notification Retry + Dead-Letter Queue handler contract (Phase 1 Red)
|
||||
//
|
||||
// These tests pin the HTTP surface Phase 2 Green must implement:
|
||||
//
|
||||
// 1. POST /api/v1/notifications/{id}/requeue — flips a dead notification
|
||||
// back to 'pending' so the retry loop can pick it up again. The handler
|
||||
// method does not exist yet (NotificationHandler has no RequeueNotification
|
||||
// method) and the NotificationService interface does not declare
|
||||
// RequeueNotification — both are compile-time Red halts.
|
||||
//
|
||||
// 2. GET /api/v1/notifications?status=dead — routes dead-letter list requests
|
||||
// through ListNotificationsByStatus instead of ListNotifications. The
|
||||
// status-filter routing does not exist yet, so ListNotificationsByStatusFn
|
||||
// never fires — a runtime Red halt.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestRequeueNotification_Success(t *testing.T) {
|
||||
var requeuedID string
|
||||
mock := &MockNotificationService{
|
||||
RequeueFn: func(id string) error {
|
||||
requeuedID = id
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewNotificationHandler(mock)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/notif-dead-001/requeue", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RequeueNotification(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", w.Code)
|
||||
}
|
||||
if requeuedID != "notif-dead-001" {
|
||||
t.Errorf("expected requeued ID 'notif-dead-001', got '%s'", requeuedID)
|
||||
}
|
||||
|
||||
var resp map[string]string
|
||||
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("failed to decode response: %v", err)
|
||||
}
|
||||
if resp["status"] != "requeued" {
|
||||
t.Errorf("expected status 'requeued', got '%s'", resp["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequeueNotification_NotFound(t *testing.T) {
|
||||
mock := &MockNotificationService{
|
||||
RequeueFn: func(id string) error {
|
||||
return ErrMockNotFound
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewNotificationHandler(mock)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/nonexistent/requeue", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RequeueNotification(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected status 404, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequeueNotification_ServiceError(t *testing.T) {
|
||||
mock := &MockNotificationService{
|
||||
RequeueFn: func(id string) error {
|
||||
return ErrMockServiceFailed
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewNotificationHandler(mock)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications/notif-dead-001/requeue", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RequeueNotification(w, req)
|
||||
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("expected status 500, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequeueNotification_MethodNotAllowed(t *testing.T) {
|
||||
handler := NewNotificationHandler(&MockNotificationService{})
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications/notif-dead-001/requeue", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RequeueNotification(w, req)
|
||||
|
||||
if w.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("expected status 405, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequeueNotification_EmptyID(t *testing.T) {
|
||||
handler := NewNotificationHandler(&MockNotificationService{})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/notifications//requeue", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.RequeueNotification(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected status 400, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListNotifications_StatusFilter_Dead(t *testing.T) {
|
||||
now := time.Now()
|
||||
certID := "mc-prod-001"
|
||||
lastErr := "SMTP connection refused"
|
||||
nextRetry := now.Add(1 * time.Minute)
|
||||
dead := domain.NotificationEvent{
|
||||
ID: "notif-dead-001",
|
||||
Type: domain.NotificationTypeExpirationWarning,
|
||||
CertificateID: &certID,
|
||||
Channel: domain.NotificationChannelEmail,
|
||||
Recipient: "admin@example.com",
|
||||
Message: "Certificate expiring in 7 days",
|
||||
Status: "dead",
|
||||
CreatedAt: now,
|
||||
RetryCount: 5,
|
||||
NextRetryAt: &nextRetry,
|
||||
LastError: &lastErr,
|
||||
}
|
||||
|
||||
var capturedStatus string
|
||||
var capturedPage, capturedPerPage int
|
||||
byStatusCalled := false
|
||||
listCalled := false
|
||||
|
||||
mock := &MockNotificationService{
|
||||
ListNotificationsFn: func(page, perPage int) ([]domain.NotificationEvent, int64, error) {
|
||||
listCalled = true
|
||||
return nil, 0, nil
|
||||
},
|
||||
ListNotificationsByStatusFn: func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
|
||||
byStatusCalled = true
|
||||
capturedStatus = status
|
||||
capturedPage = page
|
||||
capturedPerPage = perPage
|
||||
return []domain.NotificationEvent{dead}, 1, nil
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewNotificationHandler(mock)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications?status=dead&page=1&per_page=50", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.ListNotifications(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", w.Code)
|
||||
}
|
||||
if !byStatusCalled {
|
||||
t.Fatalf("expected ListNotificationsByStatus to be called for ?status=dead, but it was not")
|
||||
}
|
||||
if listCalled {
|
||||
t.Errorf("ListNotifications should not be called when status filter is present")
|
||||
}
|
||||
if capturedStatus != "dead" {
|
||||
t.Errorf("expected status='dead', got '%s'", capturedStatus)
|
||||
}
|
||||
if capturedPage != 1 {
|
||||
t.Errorf("expected page=1, got %d", capturedPage)
|
||||
}
|
||||
if capturedPerPage != 50 {
|
||||
t.Errorf("expected per_page=50, got %d", capturedPerPage)
|
||||
}
|
||||
|
||||
var resp PagedResponse
|
||||
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("failed to decode response: %v", err)
|
||||
}
|
||||
if resp.Total != 1 {
|
||||
t.Errorf("expected total=1 dead notification, got %d", resp.Total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListNotifications_NoStatusFilter_CallsDefault(t *testing.T) {
|
||||
// Pin the inverse: when no ?status= is provided, the handler must call the
|
||||
// existing ListNotifications path (not ListNotificationsByStatus). Phase 2
|
||||
// Green must not break the default listing behavior for the plain tab.
|
||||
listCalled := false
|
||||
byStatusCalled := false
|
||||
|
||||
mock := &MockNotificationService{
|
||||
ListNotificationsFn: func(page, perPage int) ([]domain.NotificationEvent, int64, error) {
|
||||
listCalled = true
|
||||
return []domain.NotificationEvent{}, 0, nil
|
||||
},
|
||||
ListNotificationsByStatusFn: func(status string, page, perPage int) ([]domain.NotificationEvent, int64, error) {
|
||||
byStatusCalled = true
|
||||
return nil, 0, nil
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewNotificationHandler(mock)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/notifications", nil)
|
||||
req = req.WithContext(contextWithRequestID())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler.ListNotifications(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", w.Code)
|
||||
}
|
||||
if !listCalled {
|
||||
t.Errorf("expected ListNotifications to be called when no status filter is present")
|
||||
}
|
||||
if byStatusCalled {
|
||||
t.Errorf("ListNotificationsByStatus should not be called when no status filter is present")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,10 +11,17 @@ import (
|
||||
)
|
||||
|
||||
// NotificationService defines the service interface for notification operations.
|
||||
//
|
||||
// ListNotificationsByStatus and RequeueNotification were added to close coverage
|
||||
// gap I-005: the Dead letter tab on the GUI (?status=dead) needs a scoped
|
||||
// listing path, and the Requeue action needs a dedicated endpoint that flips a
|
||||
// dead notification back to 'pending' so the retry sweep can pick it up again.
|
||||
type NotificationService interface {
|
||||
ListNotifications(ctx context.Context, page, perPage int) ([]domain.NotificationEvent, int64, error)
|
||||
ListNotificationsByStatus(ctx context.Context, status string, page, perPage int) ([]domain.NotificationEvent, int64, error)
|
||||
GetNotification(ctx context.Context, id string) (*domain.NotificationEvent, error)
|
||||
MarkAsRead(ctx context.Context, id string) error
|
||||
RequeueNotification(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
// NotificationHandler handles HTTP requests for notification operations.
|
||||
@@ -51,7 +58,20 @@ func (h NotificationHandler) ListNotifications(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
}
|
||||
|
||||
notifications, total, err := h.svc.ListNotifications(r.Context(), page, perPage)
|
||||
// I-005: branch to the status-scoped listing path when ?status= is present
|
||||
// so the Dead letter tab on the GUI (?status=dead) can filter server-side.
|
||||
// Empty status delegates to the original ListNotifications path to preserve
|
||||
// the default tab's existing behavior.
|
||||
var (
|
||||
notifications []domain.NotificationEvent
|
||||
total int64
|
||||
err error
|
||||
)
|
||||
if status := query.Get("status"); status != "" {
|
||||
notifications, total, err = h.svc.ListNotificationsByStatus(r.Context(), status, page, perPage)
|
||||
} else {
|
||||
notifications, total, err = h.svc.ListNotifications(r.Context(), page, perPage)
|
||||
}
|
||||
if err != nil {
|
||||
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to list notifications", requestID)
|
||||
return
|
||||
@@ -124,3 +144,43 @@ func (h NotificationHandler) MarkAsRead(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
JSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// RequeueNotification flips a dead notification back to 'pending' so the retry
|
||||
// sweep (coverage gap I-005) can pick it up again on its next tick. The handler
|
||||
// is strictly POST-only; GET/PUT/DELETE return 405. An empty id segment
|
||||
// (/api/v1/notifications//requeue) returns 400. Service errors that carry a
|
||||
// "not found" sentinel map to 404; all other service errors map to 500. This
|
||||
// 404-vs-500 split mirrors GetCertificateDeployments at certificates.go:644.
|
||||
// POST /api/v1/notifications/{id}/requeue
|
||||
func (h NotificationHandler) RequeueNotification(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
Error(w, http.StatusMethodNotAllowed, "Method not allowed")
|
||||
return
|
||||
}
|
||||
|
||||
requestID := middleware.GetRequestID(r.Context())
|
||||
|
||||
// Extract notification ID from path /api/v1/notifications/{id}/requeue
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api/v1/notifications/")
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 2 || parts[0] == "" {
|
||||
ErrorWithRequestID(w, http.StatusBadRequest, "Notification ID is required", requestID)
|
||||
return
|
||||
}
|
||||
notificationID := parts[0]
|
||||
|
||||
if err := h.svc.RequeueNotification(r.Context(), notificationID); err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
ErrorWithRequestID(w, http.StatusNotFound, "Notification not found", requestID)
|
||||
return
|
||||
}
|
||||
ErrorWithRequestID(w, http.StatusInternalServerError, "Failed to requeue notification", requestID)
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]string{
|
||||
"status": "requeued",
|
||||
}
|
||||
|
||||
JSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
@@ -45,28 +45,28 @@ func (r *Router) RegisterFunc(pattern string, handler func(http.ResponseWriter,
|
||||
|
||||
// HandlerRegistry groups all API handler dependencies for router registration.
|
||||
type HandlerRegistry struct {
|
||||
Certificates handler.CertificateHandler
|
||||
Issuers handler.IssuerHandler
|
||||
Targets handler.TargetHandler
|
||||
Agents handler.AgentHandler
|
||||
Jobs handler.JobHandler
|
||||
Policies handler.PolicyHandler
|
||||
Profiles handler.ProfileHandler
|
||||
Teams handler.TeamHandler
|
||||
Owners handler.OwnerHandler
|
||||
AgentGroups handler.AgentGroupHandler
|
||||
Audit handler.AuditHandler
|
||||
Notifications handler.NotificationHandler
|
||||
Stats handler.StatsHandler
|
||||
Metrics handler.MetricsHandler
|
||||
Health handler.HealthHandler
|
||||
Discovery handler.DiscoveryHandler
|
||||
NetworkScan handler.NetworkScanHandler
|
||||
Verification handler.VerificationHandler
|
||||
Export handler.ExportHandler
|
||||
Digest handler.DigestHandler
|
||||
HealthChecks *handler.HealthCheckHandler
|
||||
BulkRevocation handler.BulkRevocationHandler
|
||||
Certificates handler.CertificateHandler
|
||||
Issuers handler.IssuerHandler
|
||||
Targets handler.TargetHandler
|
||||
Agents handler.AgentHandler
|
||||
Jobs handler.JobHandler
|
||||
Policies handler.PolicyHandler
|
||||
Profiles handler.ProfileHandler
|
||||
Teams handler.TeamHandler
|
||||
Owners handler.OwnerHandler
|
||||
AgentGroups handler.AgentGroupHandler
|
||||
Audit handler.AuditHandler
|
||||
Notifications handler.NotificationHandler
|
||||
Stats handler.StatsHandler
|
||||
Metrics handler.MetricsHandler
|
||||
Health handler.HealthHandler
|
||||
Discovery handler.DiscoveryHandler
|
||||
NetworkScan handler.NetworkScanHandler
|
||||
Verification handler.VerificationHandler
|
||||
Export handler.ExportHandler
|
||||
Digest handler.DigestHandler
|
||||
HealthChecks *handler.HealthCheckHandler
|
||||
BulkRevocation handler.BulkRevocationHandler
|
||||
}
|
||||
|
||||
// RegisterHandlers sets up all API routes with their handlers.
|
||||
@@ -131,9 +131,21 @@ func (r *Router) RegisterHandlers(reg HandlerRegistry) {
|
||||
r.Register("POST /api/v1/targets/{id}/test", http.HandlerFunc(reg.Targets.TestTargetConnection))
|
||||
|
||||
// Agents routes: /api/v1/agents
|
||||
//
|
||||
// I-004 soft-retirement surface:
|
||||
// * GET /api/v1/agents/retired — opt-in listing of retired agents.
|
||||
// MUST be registered before /agents/{id} so Go 1.22 ServeMux's
|
||||
// literal-beats-pattern-var precedence routes the `retired` literal
|
||||
// to ListRetiredAgents instead of treating "retired" as a {id}
|
||||
// parameter value against GetAgent.
|
||||
// * DELETE /api/v1/agents/{id} — RetireAgent. Replaces the pre-I-004
|
||||
// hard-delete; the underlying repo does a soft-retire with
|
||||
// optional cascade.
|
||||
r.Register("GET /api/v1/agents", http.HandlerFunc(reg.Agents.ListAgents))
|
||||
r.Register("POST /api/v1/agents", http.HandlerFunc(reg.Agents.RegisterAgent))
|
||||
r.Register("GET /api/v1/agents/retired", http.HandlerFunc(reg.Agents.ListRetiredAgents))
|
||||
r.Register("GET /api/v1/agents/{id}", http.HandlerFunc(reg.Agents.GetAgent))
|
||||
r.Register("DELETE /api/v1/agents/{id}", http.HandlerFunc(reg.Agents.RetireAgent))
|
||||
r.Register("POST /api/v1/agents/{id}/heartbeat", http.HandlerFunc(reg.Agents.Heartbeat))
|
||||
r.Register("POST /api/v1/agents/{id}/csr", http.HandlerFunc(reg.Agents.AgentCSRSubmit))
|
||||
r.Register("GET /api/v1/agents/{id}/certificates/{cert_id}", http.HandlerFunc(reg.Agents.AgentCertificatePickup))
|
||||
@@ -192,6 +204,10 @@ func (r *Router) RegisterHandlers(reg HandlerRegistry) {
|
||||
r.Register("GET /api/v1/notifications", http.HandlerFunc(reg.Notifications.ListNotifications))
|
||||
r.Register("GET /api/v1/notifications/{id}", http.HandlerFunc(reg.Notifications.GetNotification))
|
||||
r.Register("POST /api/v1/notifications/{id}/read", http.HandlerFunc(reg.Notifications.MarkAsRead))
|
||||
// I-005: requeue a dead notification back to pending so the retry sweep
|
||||
// picks it up again. Go 1.22 ServeMux resolves the literal /requeue segment
|
||||
// before falling back to the {id} path-variable route above.
|
||||
r.Register("POST /api/v1/notifications/{id}/requeue", http.HandlerFunc(reg.Notifications.RequeueNotification))
|
||||
|
||||
// Stats routes: /api/v1/stats
|
||||
r.Register("GET /api/v1/stats/summary", http.HandlerFunc(reg.Stats.GetDashboardSummary))
|
||||
@@ -242,7 +258,19 @@ func (r *Router) RegisterHandlers(reg HandlerRegistry) {
|
||||
}
|
||||
|
||||
// RegisterESTHandlers sets up EST (RFC 7030) routes under /.well-known/est/.
|
||||
// EST endpoints use a separate middleware chain (no API key auth — EST uses TLS client certs).
|
||||
//
|
||||
// EST endpoints are intentionally unauthenticated at the HTTP layer. Per RFC 7030
|
||||
// §3.2.3, authentication and authorization for enrollment are deployment-specific;
|
||||
// certctl relies on CSR signature verification, profile policy enforcement (allowed
|
||||
// key types, max TTL, permitted EKUs), and the underlying issuer connector's own
|
||||
// policy. Per RFC 7030 §4.1.1, /.well-known/est/cacerts is explicitly anonymous.
|
||||
//
|
||||
// cmd/server/main.go's finalHandler dispatches /.well-known/est/* to a dedicated
|
||||
// no-auth middleware chain (RequestID, structuredLogger, Recovery only) so EST
|
||||
// clients — IoT devices, 802.1X supplicants, MDM-enrolled laptops — never hit the
|
||||
// Bearer-token auth middleware they cannot satisfy. See M-001 audit 2026-04-19
|
||||
// (option D): prior builds routed EST through the authenticated apiHandler chain,
|
||||
// which reduced every enrollment to a 401 before the handler was reached.
|
||||
func (r *Router) RegisterESTHandlers(est handler.ESTHandler) {
|
||||
// EST endpoints per RFC 7030 Section 3.2.2
|
||||
r.Register("GET /.well-known/est/cacerts", http.HandlerFunc(est.CACerts))
|
||||
@@ -253,7 +281,11 @@ func (r *Router) RegisterESTHandlers(est handler.ESTHandler) {
|
||||
|
||||
// RegisterSCEPHandlers sets up SCEP (RFC 8894) routes.
|
||||
// SCEP uses a single endpoint with operation-based dispatch via query parameters.
|
||||
// Authentication is via challenge password in the CSR, not TLS client certs or API keys.
|
||||
// Authentication is via the challengePassword attribute in the PKCS#10 CSR, not
|
||||
// via HTTP Bearer tokens or TLS client certs. cmd/server/main.go's finalHandler
|
||||
// routes /scep* through the no-auth middleware chain (M-001 audit 2026-04-19,
|
||||
// option D), and Config.Validate() refuses to start the server if SCEP is enabled
|
||||
// without a non-empty CERTCTL_SCEP_CHALLENGE_PASSWORD (H-2, CWE-306).
|
||||
func (r *Router) RegisterSCEPHandlers(scep handler.SCEPHandler) {
|
||||
// SCEP uses a single path; the handler dispatches on ?operation= query param
|
||||
r.Register("GET /scep", http.HandlerFunc(scep.HandleSCEP))
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestClient_RetireAgent_Success pins the I-004 CLI happy path: the operator
|
||||
// runs `certctl-cli agents retire <id>` and the client issues a DELETE to
|
||||
// /api/v1/agents/{id}, parses the 200 JSON body (retired_at, already_retired,
|
||||
// cascade, counts), and reports success. The handler test already covers the
|
||||
// server-side contract; this test covers the client-side wire formatting so a
|
||||
// refactor of the server's 200 body shape can't silently break the CLI.
|
||||
func TestClient_RetireAgent_Success(t *testing.T) {
|
||||
var (
|
||||
sawMethod string
|
||||
sawPath string
|
||||
sawForce string
|
||||
sawReason string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sawMethod = r.Method
|
||||
sawPath = r.URL.Path
|
||||
sawForce = r.URL.Query().Get("force")
|
||||
sawReason = r.URL.Query().Get("reason")
|
||||
|
||||
if r.Method != "DELETE" || r.URL.Path != "/api/v1/agents/ag-1" {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"retired_at": "2026-04-18T12:00:00Z",
|
||||
"already_retired": false,
|
||||
"cascade": false,
|
||||
"counts": map[string]interface{}{
|
||||
"active_targets": 0,
|
||||
"active_certificates": 0,
|
||||
"pending_jobs": 0,
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
// Positional arg: the agent ID. No --force, no --reason — the default
|
||||
// soft-retire path. Compile-fail until client.RetireAgent exists.
|
||||
if err := client.RetireAgent([]string{"ag-1"}); err != nil {
|
||||
t.Fatalf("RetireAgent(ag-1) err=%v want nil", err)
|
||||
}
|
||||
|
||||
if sawMethod != "DELETE" {
|
||||
t.Errorf("method=%q want DELETE", sawMethod)
|
||||
}
|
||||
if sawPath != "/api/v1/agents/ag-1" {
|
||||
t.Errorf("path=%q want /api/v1/agents/ag-1", sawPath)
|
||||
}
|
||||
if sawForce != "" {
|
||||
t.Errorf("force query=%q want empty (default path sends no force)", sawForce)
|
||||
}
|
||||
if sawReason != "" {
|
||||
t.Errorf("reason query=%q want empty (default path sends no reason)", sawReason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_RetireAgent_Force_WithReason_Success pins the ?force=true&reason=...
|
||||
// escape hatch wiring. Operators who supply --force + --reason get their values
|
||||
// propagated as URL query parameters exactly once, so the server sees the same
|
||||
// contract the handler test expects. Also verifies the cascade=true response
|
||||
// body parses cleanly.
|
||||
func TestClient_RetireAgent_Force_WithReason_Success(t *testing.T) {
|
||||
var (
|
||||
sawForce string
|
||||
sawReason string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sawForce = r.URL.Query().Get("force")
|
||||
sawReason = r.URL.Query().Get("reason")
|
||||
|
||||
if r.Method != "DELETE" || r.URL.Path != "/api/v1/agents/ag-1" {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"retired_at": "2026-04-18T12:00:00Z",
|
||||
"already_retired": false,
|
||||
"cascade": true,
|
||||
"counts": map[string]interface{}{
|
||||
"active_targets": 2,
|
||||
"active_certificates": 5,
|
||||
"pending_jobs": 1,
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
if err := client.RetireAgent([]string{"ag-1", "--force", "--reason", "decommissioning rack 7"}); err != nil {
|
||||
t.Fatalf("RetireAgent(force+reason) err=%v want nil", err)
|
||||
}
|
||||
if sawForce != "true" {
|
||||
t.Errorf("force query=%q want \"true\"", sawForce)
|
||||
}
|
||||
if sawReason != "decommissioning rack 7" {
|
||||
t.Errorf("reason query=%q want %q", sawReason, "decommissioning rack 7")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_RetireAgent_Force_RequiresReason pins the client-side guard: using
|
||||
// --force without --reason must fail BEFORE any HTTP request is made. Without
|
||||
// this, the client would bounce off the server's 400 ErrForceReasonRequired
|
||||
// only after a round trip — slow feedback, wasted audit-trail noise, and a
|
||||
// worse operator experience. requestCount=0 enforces that no HTTP call happens.
|
||||
func TestClient_RetireAgent_Force_RequiresReason(t *testing.T) {
|
||||
var requestCount int
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.RetireAgent([]string{"ag-1", "--force"})
|
||||
if err == nil {
|
||||
t.Fatalf("RetireAgent(force, no reason) err=nil want client-side error")
|
||||
}
|
||||
if !containsStr(err.Error(), "reason") {
|
||||
t.Errorf("err=%q should mention --reason to guide operator", err.Error())
|
||||
}
|
||||
if requestCount != 0 {
|
||||
t.Fatalf("requestCount=%d want 0; client must short-circuit before HTTP call", requestCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_RetireAgent_MissingID covers the other common operator mistake:
|
||||
// invoking `certctl-cli agents retire` with no agent ID. Must be caught by the
|
||||
// client with a clear error, not a malformed DELETE to /api/v1/agents/.
|
||||
func TestClient_RetireAgent_MissingID(t *testing.T) {
|
||||
var requestCount int
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestCount++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.RetireAgent([]string{})
|
||||
if err == nil {
|
||||
t.Fatalf("RetireAgent([]) err=nil want missing-id error")
|
||||
}
|
||||
if requestCount != 0 {
|
||||
t.Fatalf("requestCount=%d want 0; client must reject missing-id before HTTP", requestCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_ListRetiredAgents_Success pins the audit/forensics CLI surface:
|
||||
// `certctl-cli agents list-retired` must GET /api/v1/agents/retired and render
|
||||
// the paged response. The server returns a PagedResponse; the client is
|
||||
// responsible for printing it in table or JSON format, same as ListAgents.
|
||||
func TestClient_ListRetiredAgents_Success(t *testing.T) {
|
||||
var (
|
||||
sawMethod string
|
||||
sawPath string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sawMethod = r.Method
|
||||
sawPath = r.URL.Path
|
||||
|
||||
if r.Method != "GET" || r.URL.Path != "/api/v1/agents/retired" {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"data": []map[string]interface{}{
|
||||
{
|
||||
"id": "ag-old-01",
|
||||
"name": "decom-01",
|
||||
"hostname": "server-old",
|
||||
"status": "Offline",
|
||||
"registered_at": "2024-01-01T00:00:00Z",
|
||||
"retired_at": "2026-01-01T00:00:00Z",
|
||||
"retired_reason": "old hardware",
|
||||
},
|
||||
},
|
||||
"total": 1,
|
||||
"page": 1,
|
||||
"per_page": 50,
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
if err := client.ListRetiredAgents([]string{}); err != nil {
|
||||
t.Fatalf("ListRetiredAgents err=%v want nil", err)
|
||||
}
|
||||
if sawMethod != "GET" {
|
||||
t.Errorf("method=%q want GET", sawMethod)
|
||||
}
|
||||
if sawPath != "/api/v1/agents/retired" {
|
||||
t.Errorf("path=%q want /api/v1/agents/retired", sawPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_ListRetiredAgents_ServerError covers the non-happy path: server
|
||||
// returns 5xx → client surfaces the error rather than silently printing an
|
||||
// empty list. Without this, operators running the command as part of a
|
||||
// compliance audit could miss a backend outage.
|
||||
func TestClient_ListRetiredAgents_ServerError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "db unreachable", http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ListRetiredAgents([]string{})
|
||||
if err == nil {
|
||||
t.Fatalf("ListRetiredAgents(500) err=nil want propagated error")
|
||||
}
|
||||
}
|
||||
+252
-5
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
@@ -19,22 +20,51 @@ import (
|
||||
|
||||
// Client is the CLI HTTP client that communicates with the certctl server.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
apiKey string
|
||||
format string
|
||||
baseURL string
|
||||
apiKey string
|
||||
format string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new CLI client.
|
||||
func NewClient(baseURL, apiKey, format string) *Client {
|
||||
//
|
||||
// HTTPS-Everywhere (v2.2): the certctl control plane is HTTPS-only. caBundlePath,
|
||||
// when non-empty, points at a PEM bundle used to verify the server cert; otherwise
|
||||
// the system trust store is used. insecure skips cert verification — dev only,
|
||||
// never enable in production. The TLS config is attached to *http.Transport so
|
||||
// every call goes through the same verified socket.
|
||||
func NewClient(baseURL, apiKey, format, caBundlePath string, insecure bool) (*Client, error) {
|
||||
tlsConfig := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: insecure, //nolint:gosec // opt-in dev toggle, documented in docs/tls.md
|
||||
}
|
||||
if caBundlePath != "" {
|
||||
pemBytes, err := os.ReadFile(caBundlePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading CA bundle at %q: %w", caBundlePath, err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pemBytes) {
|
||||
return nil, fmt.Errorf("CA bundle at %q contains no valid PEM-encoded certificates", caBundlePath)
|
||||
}
|
||||
tlsConfig.RootCAs = pool
|
||||
}
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
apiKey: apiKey,
|
||||
format: format,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
},
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// do performs an HTTP request and returns the parsed JSON response.
|
||||
@@ -293,6 +323,194 @@ func (c *Client) ListAgents(args []string) error {
|
||||
return c.outputAgentsTable(result.Data, result.Total)
|
||||
}
|
||||
|
||||
// ListRetiredAgents lists soft-retired agents from the dedicated endpoint.
|
||||
//
|
||||
// I-004: hits GET /api/v1/agents/retired which is a separate route from the
|
||||
// default listing (the default hides retired rows). Supports --page and
|
||||
// --per-page just like the active list. Output format mirrors ListAgents
|
||||
// but prepends RETIRED_AT and RETIRED_REASON columns so the operator can
|
||||
// forensic-grep the output.
|
||||
func (c *Client) ListRetiredAgents(args []string) error {
|
||||
fs := flag.NewFlagSet("agents list --retired", flag.ContinueOnError)
|
||||
page := fs.Int("page", 1, "Page number")
|
||||
perPage := fs.Int("per-page", 50, "Items per page")
|
||||
fs.Parse(args)
|
||||
|
||||
query := url.Values{}
|
||||
query.Set("page", fmt.Sprintf("%d", *page))
|
||||
query.Set("per_page", fmt.Sprintf("%d", *perPage))
|
||||
|
||||
resp, err := c.do("GET", "/api/v1/agents/retired", query, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Data []map[string]interface{} `json:"data"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
if err := json.Unmarshal(resp, &result); err != nil {
|
||||
return fmt.Errorf("parsing response: %w", err)
|
||||
}
|
||||
|
||||
if c.format == "json" {
|
||||
return c.outputJSON(result)
|
||||
}
|
||||
|
||||
return c.outputRetiredAgentsTable(result.Data, result.Total)
|
||||
}
|
||||
|
||||
// RetireAgent soft-retires an agent via DELETE /api/v1/agents/{id}.
|
||||
//
|
||||
// I-004: wraps the full status-code matrix pinned by the handler's
|
||||
// agent_retire_handler_test.go:
|
||||
//
|
||||
// 200 clean retire — body: retired_at, already_retired=false, cascade=false, counts=0
|
||||
// 200 force-cascade retire — body: cascade=true, counts=pre-cascade snapshot
|
||||
// 204 idempotent retire — agent was already retired, NO body
|
||||
// 403 sentinel — reserved agent (server-scanner / cloud-*), ErrAgentIsSentinel
|
||||
// 404 not found — agent doesn't exist
|
||||
// 409 blocked_by_dependencies — body: error, message, counts
|
||||
//
|
||||
// The default (force=false) flow refuses to retire agents with active
|
||||
// downstream dependencies; the operator must re-run with --force and an
|
||||
// explicit --reason to cascade. The handler rejects --force without
|
||||
// --reason with a 400 — we mirror that contract client-side so the
|
||||
// operator gets a clear error before the round trip.
|
||||
func (c *Client) RetireAgent(args []string) error {
|
||||
// Convention: `agents retire <id> [--force] [--reason <reason>]` — the ID
|
||||
// is a positional arg that precedes the flags. Go's flag package stops
|
||||
// parsing at the first non-flag token, so we pull args[0] as the ID and
|
||||
// hand args[1:] to the flag parser. Without this split, `agents retire
|
||||
// ag-1 --force --reason "x"` would parse with force=false and reason=""
|
||||
// because the flags land in fs.Args() instead of being recognized.
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("agent ID is required: agents retire <id> [--force] [--reason <reason>]")
|
||||
}
|
||||
id := args[0]
|
||||
|
||||
fs := flag.NewFlagSet("agents retire", flag.ContinueOnError)
|
||||
force := fs.Bool("force", false, "Cascade-retire downstream targets, certs, and jobs")
|
||||
reason := fs.String("reason", "", "Human-readable reason (required with --force)")
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Mirror the handler's ErrForceReasonRequired contract client-side so
|
||||
// the operator gets a clear error before the round trip.
|
||||
if *force && strings.TrimSpace(*reason) == "" {
|
||||
return fmt.Errorf("--reason is required when --force is set")
|
||||
}
|
||||
|
||||
// Build query string. Skip ?force=false; skip ?reason= when empty.
|
||||
query := url.Values{}
|
||||
if *force {
|
||||
query.Set("force", "true")
|
||||
}
|
||||
if *reason != "" {
|
||||
query.Set("reason", *reason)
|
||||
}
|
||||
|
||||
u, err := url.JoinPath(c.baseURL, fmt.Sprintf("/api/v1/agents/%s", id))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
if len(query) > 0 {
|
||||
u = u + "?" + query.Encode()
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("DELETE", u, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if c.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading response: %w", err)
|
||||
}
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusNoContent:
|
||||
// 204 idempotent — the agent was already retired. No body.
|
||||
if c.format == "json" {
|
||||
return c.outputJSON(map[string]interface{}{
|
||||
"agent_id": id,
|
||||
"already_retired": true,
|
||||
})
|
||||
}
|
||||
fmt.Printf("Agent %s was already retired (idempotent)\n", id)
|
||||
return nil
|
||||
|
||||
case http.StatusOK:
|
||||
var result struct {
|
||||
RetiredAt string `json:"retired_at"`
|
||||
AlreadyRetired bool `json:"already_retired"`
|
||||
Cascade bool `json:"cascade"`
|
||||
Counts struct {
|
||||
ActiveTargets int `json:"active_targets"`
|
||||
ActiveCertificates int `json:"active_certificates"`
|
||||
PendingJobs int `json:"pending_jobs"`
|
||||
} `json:"counts"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return fmt.Errorf("parsing 200 response: %w", err)
|
||||
}
|
||||
|
||||
if c.format == "json" {
|
||||
return c.outputJSON(json.RawMessage(body))
|
||||
}
|
||||
|
||||
if result.Cascade {
|
||||
fmt.Printf("Agent %s retired (cascade). Retired at: %s\n", id, result.RetiredAt)
|
||||
fmt.Printf(" Cascaded: %d targets, %d certificates, %d jobs\n",
|
||||
result.Counts.ActiveTargets, result.Counts.ActiveCertificates, result.Counts.PendingJobs)
|
||||
} else {
|
||||
fmt.Printf("Agent %s retired. Retired at: %s\n", id, result.RetiredAt)
|
||||
}
|
||||
return nil
|
||||
|
||||
case http.StatusConflict:
|
||||
// 409 blocked_by_dependencies. Parse the body so we can show the
|
||||
// operator which dependency counts are holding up the retire.
|
||||
var blocked struct {
|
||||
Error string `json:"error"`
|
||||
Message string `json:"message"`
|
||||
Counts struct {
|
||||
ActiveTargets int `json:"active_targets"`
|
||||
ActiveCertificates int `json:"active_certificates"`
|
||||
PendingJobs int `json:"pending_jobs"`
|
||||
} `json:"counts"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &blocked); err != nil {
|
||||
return fmt.Errorf("agent has active dependencies (HTTP 409); raw body: %s", string(body))
|
||||
}
|
||||
return fmt.Errorf("blocked_by_dependencies: %s (targets=%d certificates=%d jobs=%d); re-run with --force --reason \"<reason>\" to cascade",
|
||||
blocked.Message, blocked.Counts.ActiveTargets, blocked.Counts.ActiveCertificates, blocked.Counts.PendingJobs)
|
||||
|
||||
case http.StatusForbidden:
|
||||
return fmt.Errorf("agent %s is a reserved sentinel and cannot be retired (HTTP 403)", id)
|
||||
|
||||
case http.StatusNotFound:
|
||||
return fmt.Errorf("agent %s not found (HTTP 404)", id)
|
||||
|
||||
case http.StatusBadRequest:
|
||||
return fmt.Errorf("bad request (HTTP 400): %s", string(body))
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unexpected HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
|
||||
// GetAgent retrieves a single agent by ID.
|
||||
func (c *Client) GetAgent(id string) error {
|
||||
resp, err := c.do("GET", fmt.Sprintf("/api/v1/agents/%s", id), nil, nil)
|
||||
@@ -613,6 +831,35 @@ func (c *Client) outputAgentsTable(agents []map[string]interface{}, total int) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// outputRetiredAgentsTable is the tab-writer view for the retired listing.
|
||||
// I-004: adds RETIRED_AT + REASON columns so operators can forensic-grep.
|
||||
func (c *Client) outputRetiredAgentsTable(agents []map[string]interface{}, total int) error {
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "ID\tHOSTNAME\tOS\tARCHITECTURE\tRETIRED AT\tREASON")
|
||||
|
||||
for _, agent := range agents {
|
||||
id := getString(agent, "id")
|
||||
hostname := getString(agent, "hostname")
|
||||
osName := getString(agent, "os")
|
||||
arch := getString(agent, "architecture")
|
||||
retiredAt := ""
|
||||
if raw, ok := agent["retired_at"].(string); ok && raw != "" {
|
||||
if t, err := time.Parse(time.RFC3339, raw); err == nil {
|
||||
retiredAt = t.Format("2006-01-02 15:04:05")
|
||||
} else {
|
||||
retiredAt = raw
|
||||
}
|
||||
}
|
||||
reason := getString(agent, "retired_reason")
|
||||
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", id, hostname, osName, arch, retiredAt, reason)
|
||||
}
|
||||
|
||||
w.Flush()
|
||||
fmt.Printf("\nTotal retired: %d\n", total)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) outputAgentDetail(agent map[string]interface{}) error {
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
|
||||
|
||||
+207
-15
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
@@ -39,7 +40,7 @@ func TestClient_ListCertificates(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ListCertificates([]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListCertificates failed: %v", err)
|
||||
@@ -64,7 +65,7 @@ func TestClient_GetCertificate(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "json")
|
||||
client, _ := NewClient(server.URL, "", "json", "", false)
|
||||
err := client.GetCertificate("mc-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetCertificate failed: %v", err)
|
||||
@@ -86,7 +87,7 @@ func TestClient_RenewCertificate(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.RenewCertificate("mc-1")
|
||||
if err != nil {
|
||||
t.Fatalf("RenewCertificate failed: %v", err)
|
||||
@@ -107,7 +108,7 @@ func TestClient_RevokeCertificate(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.RevokeCertificate("mc-1", "cessationOfOperation")
|
||||
if err != nil {
|
||||
t.Fatalf("RevokeCertificate failed: %v", err)
|
||||
@@ -141,7 +142,7 @@ func TestClient_BulkRevokeCertificates(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.BulkRevokeCertificates([]string{
|
||||
"--reason", "keyCompromise",
|
||||
"--profile-id", "prof-tls",
|
||||
@@ -175,7 +176,7 @@ func TestClient_ListAgents(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ListAgents([]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListAgents failed: %v", err)
|
||||
@@ -201,7 +202,7 @@ func TestClient_GetAgent(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "json")
|
||||
client, _ := NewClient(server.URL, "", "json", "", false)
|
||||
err := client.GetAgent("ag-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetAgent failed: %v", err)
|
||||
@@ -232,7 +233,7 @@ func TestClient_ListJobs(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ListJobs([]string{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListJobs failed: %v", err)
|
||||
@@ -258,7 +259,7 @@ func TestClient_GetJob(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "json")
|
||||
client, _ := NewClient(server.URL, "", "json", "", false)
|
||||
err := client.GetJob("job-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetJob failed: %v", err)
|
||||
@@ -276,7 +277,7 @@ func TestClient_CancelJob(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.CancelJob("job-1")
|
||||
if err != nil {
|
||||
t.Fatalf("CancelJob failed: %v", err)
|
||||
@@ -308,7 +309,7 @@ func TestClient_GetStatus(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.GetStatus()
|
||||
if err != nil {
|
||||
t.Fatalf("GetStatus failed: %v", err)
|
||||
@@ -381,7 +382,7 @@ func TestClient_AuthHeader(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "testkey123", "json")
|
||||
client, _ := NewClient(server.URL, "testkey123", "json", "", false)
|
||||
client.do("GET", "/api/v1/certificates", nil, nil)
|
||||
|
||||
if authHeader != "Bearer testkey123" {
|
||||
@@ -439,7 +440,7 @@ func TestClient_ImportCertificates_MissingRequiredFlags(t *testing.T) {
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ImportCertificates(tc.args)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %s, got nil", tc.name)
|
||||
@@ -468,7 +469,7 @@ func TestClient_ImportCertificates_MissingPositionalArgs(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ImportCertificates([]string{
|
||||
"--owner-id", "o-alice",
|
||||
"--team-id", "t-platform",
|
||||
@@ -513,7 +514,7 @@ func TestClient_ImportCertificates_SixFieldPayload(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "", "table")
|
||||
client, _ := NewClient(server.URL, "", "table", "", false)
|
||||
err := client.ImportCertificates([]string{
|
||||
"--owner-id", "o-alice",
|
||||
"--team-id", "t-platform",
|
||||
@@ -583,3 +584,194 @@ func generateTestCert() *x509.Certificate {
|
||||
|
||||
return cert
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// HTTPS-Everywhere milestone (v2.2, §3.2 + §7 Phase 5):
|
||||
// The CLI binary now talks HTTPS-only to the control plane. These tests pin the
|
||||
// three contracts the milestone requires every client binary (agent, CLI, MCP)
|
||||
// to satisfy in lock-step:
|
||||
// (a) CA bundle load success — PEM loads, RootCAs + MinVersion=TLS1.3 wired
|
||||
// through the injected *http.Transport so the httpClient actually uses them.
|
||||
// (b) CA bundle load failure — missing file and malformed/empty PEM each fail
|
||||
// loud with a pinned substring so operators get a useful diagnostic instead
|
||||
// of a later TLS-handshake-error mystery.
|
||||
// (c) End-to-end TLS round-trip — an httptest.NewTLSServer whose own cert is
|
||||
// written out as the CA bundle validates that every TLS-config knob is
|
||||
// actually reaching the wire, not just surviving into the struct.
|
||||
// Each of the three client binaries pins the same three contracts against its
|
||||
// own NewClient signature; drifting any of them in isolation is exactly what
|
||||
// this suite is here to catch. The error-string substrings below must stay in
|
||||
// sync with the fmt.Errorf messages in internal/cli/client.go:NewClient.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
// writeCABundle PEM-encodes a DER cert and writes it to a temp file under the
|
||||
// test's own TempDir. Returns the absolute path of the written bundle so test
|
||||
// callers can pass it straight into NewClient(..., caBundlePath, ...).
|
||||
func writeCABundle(t *testing.T, dir string, certDER []byte, filename string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, filename)
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
||||
if err := os.WriteFile(path, pemBytes, 0o600); err != nil {
|
||||
t.Fatalf("writing CA bundle to %q: %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// TestNewClient_CABundle_Success pins the happy path: a valid PEM CA bundle
|
||||
// loads, populates RootCAs on the client's TLS config, and leaves
|
||||
// MinVersion=TLS1.3 intact. Regression guard: if a future edit accidentally
|
||||
// swaps the transport after TLS config setup (or forgets to re-attach the
|
||||
// *tls.Config to *http.Transport), this test catches it before ops does.
|
||||
func TestNewClient_CABundle_Success(t *testing.T) {
|
||||
cert := generateTestCert()
|
||||
tmp := t.TempDir()
|
||||
bundlePath := writeCABundle(t, tmp, cert.Raw, "ca.pem")
|
||||
|
||||
client, err := NewClient("https://certctl-server:8443", "test-key", "table", bundlePath, false)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient with valid CA bundle err=%v want nil", err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Fatal("NewClient returned nil client on happy path")
|
||||
}
|
||||
|
||||
transport, ok := client.httpClient.Transport.(*http.Transport)
|
||||
if !ok {
|
||||
t.Fatalf("httpClient.Transport type=%T want *http.Transport (TLS config injection broke)", client.httpClient.Transport)
|
||||
}
|
||||
if transport.TLSClientConfig == nil {
|
||||
t.Fatal("transport.TLSClientConfig is nil; TLS config must be set on every client")
|
||||
}
|
||||
if transport.TLSClientConfig.RootCAs == nil {
|
||||
t.Fatal("transport.TLSClientConfig.RootCAs is nil; CA bundle path was ignored")
|
||||
}
|
||||
if transport.TLSClientConfig.MinVersion != tls.VersionTLS13 {
|
||||
t.Errorf("MinVersion=%d want tls.VersionTLS13 (%d); HTTPS-Everywhere requires TLS1.3 floor",
|
||||
transport.TLSClientConfig.MinVersion, tls.VersionTLS13)
|
||||
}
|
||||
if transport.TLSClientConfig.InsecureSkipVerify {
|
||||
t.Error("InsecureSkipVerify=true with insecure=false arg; flag wiring crossed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_CABundle_MissingFile pins the fail-loud path for a nonexistent
|
||||
// bundle path. The error surface must include "reading CA bundle" so operators
|
||||
// see the right diagnostic instead of a downstream TLS-handshake-error.
|
||||
func TestNewClient_CABundle_MissingFile(t *testing.T) {
|
||||
_, err := NewClient("https://certctl-server:8443", "test-key", "table", "/nonexistent/path/ca.pem", false)
|
||||
if err == nil {
|
||||
t.Fatal("NewClient with missing CA bundle err=nil; must fail loud so operators see the right diagnostic")
|
||||
}
|
||||
if !containsStr(err.Error(), "reading CA bundle") {
|
||||
t.Errorf("err=%q must contain %q so operators can locate the misconfigured path", err.Error(), "reading CA bundle")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_CABundle_EmptyPEM pins the fail-loud path for a file whose
|
||||
// contents are not valid PEM certificate data. AppendCertsFromPEM returning
|
||||
// false is the signal we need to surface — otherwise the client would silently
|
||||
// ship with an empty cert pool and every TLS handshake would fail downstream.
|
||||
func TestNewClient_CABundle_EmptyPEM(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
garbagePath := filepath.Join(tmp, "garbage.pem")
|
||||
if err := os.WriteFile(garbagePath, []byte("not a pem certificate, just bytes"), 0o600); err != nil {
|
||||
t.Fatalf("writing garbage file: %v", err)
|
||||
}
|
||||
|
||||
_, err := NewClient("https://certctl-server:8443", "test-key", "table", garbagePath, false)
|
||||
if err == nil {
|
||||
t.Fatal("NewClient with malformed PEM err=nil; must fail loud, not silently skip")
|
||||
}
|
||||
if !containsStr(err.Error(), "no valid PEM-encoded certificates") {
|
||||
t.Errorf("err=%q must contain %q so operators know the file parsed but held no certs",
|
||||
err.Error(), "no valid PEM-encoded certificates")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_TLSRoundTrip validates that the TLS config knobs we set on
|
||||
// NewClient actually reach the wire. An httptest.NewTLSServer signs its own
|
||||
// self-signed leaf; we PEM-encode that server cert, write it as the CA bundle,
|
||||
// and issue a real HTTPS call through ListCertificates. A successful round-trip
|
||||
// proves RootCAs + MinVersion are flowing through *http.Transport into the
|
||||
// dialer, not just surviving into the client struct.
|
||||
func TestNewClient_TLSRoundTrip(t *testing.T) {
|
||||
var handlerHit int
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "GET" && r.URL.Path == "/api/v1/certificates" {
|
||||
handlerHit++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"data": []map[string]interface{}{},
|
||||
"total": 0,
|
||||
})
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
serverCert := server.Certificate()
|
||||
if serverCert == nil {
|
||||
t.Fatal("httptest.NewTLSServer.Certificate() returned nil; cannot build CA bundle")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
bundlePath := writeCABundle(t, tmp, serverCert.Raw, "server-ca.pem")
|
||||
|
||||
client, err := NewClient(server.URL, "test-key", "table", bundlePath, false)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(TLS server) err=%v want nil", err)
|
||||
}
|
||||
if err := client.ListCertificates([]string{}); err != nil {
|
||||
t.Fatalf("ListCertificates over HTTPS err=%v; TLS config must reach the wire", err)
|
||||
}
|
||||
if handlerHit != 1 {
|
||||
t.Errorf("handlerHit=%d want 1; request did not reach the TLS server", handlerHit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_InsecureSkipVerify pins the dev-only escape hatch: an untrusted
|
||||
// TLS server (cert NOT in the client's root pool) must be reachable when
|
||||
// insecure=true. This is the only path in the control plane that disables
|
||||
// certificate verification; it's documented in docs/tls.md and gated by the
|
||||
// CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY env var so it never slips into
|
||||
// production silently.
|
||||
func TestNewClient_InsecureSkipVerify(t *testing.T) {
|
||||
var handlerHit int
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
handlerHit++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"data": []map[string]interface{}{},
|
||||
"total": 0,
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// No CA bundle → system roots, which will NOT trust the self-signed
|
||||
// httptest cert. insecure=true is the only thing keeping this call from
|
||||
// failing with an x509-unknown-authority error.
|
||||
client, err := NewClient(server.URL, "test-key", "table", "", true)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(insecure=true) err=%v want nil", err)
|
||||
}
|
||||
|
||||
transport, ok := client.httpClient.Transport.(*http.Transport)
|
||||
if !ok {
|
||||
t.Fatalf("httpClient.Transport type=%T want *http.Transport", client.httpClient.Transport)
|
||||
}
|
||||
if !transport.TLSClientConfig.InsecureSkipVerify {
|
||||
t.Fatal("insecure=true arg did not set TLSClientConfig.InsecureSkipVerify; flag wiring broken")
|
||||
}
|
||||
if transport.TLSClientConfig.MinVersion != tls.VersionTLS13 {
|
||||
t.Errorf("MinVersion=%d want tls.VersionTLS13 even with insecure=true (TLS1.3 floor is not optional)",
|
||||
transport.TLSClientConfig.MinVersion)
|
||||
}
|
||||
|
||||
if err := client.ListCertificates([]string{}); err != nil {
|
||||
t.Fatalf("ListCertificates(insecure=true) err=%v; escape hatch must still complete the round-trip", err)
|
||||
}
|
||||
if handlerHit != 1 {
|
||||
t.Errorf("handlerHit=%d want 1; insecure round-trip did not reach the server", handlerHit)
|
||||
}
|
||||
}
|
||||
|
||||
+171
-33
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
@@ -12,29 +13,29 @@ import (
|
||||
// Config represents the complete application configuration.
|
||||
// All configuration values are read from environment variables with CERTCTL_ prefix.
|
||||
type Config struct {
|
||||
Server ServerConfig
|
||||
Database DatabaseConfig
|
||||
Scheduler SchedulerConfig
|
||||
Log LogConfig
|
||||
Auth AuthConfig
|
||||
RateLimit RateLimitConfig
|
||||
CORS CORSConfig
|
||||
Keygen KeygenConfig
|
||||
CA CAConfig
|
||||
Notifiers NotifierConfig
|
||||
NetworkScan NetworkScanConfig
|
||||
EST ESTConfig
|
||||
SCEP SCEPConfig
|
||||
Verification VerificationConfig
|
||||
ACME ACMEConfig
|
||||
Vault VaultConfig
|
||||
DigiCert DigiCertConfig
|
||||
Sectigo SectigoConfig
|
||||
GoogleCAS GoogleCASConfig
|
||||
AWSACMPCA AWSACMPCAConfig
|
||||
Entrust EntrustConfig
|
||||
GlobalSign GlobalSignConfig
|
||||
EJBCA EJBCAConfig
|
||||
Server ServerConfig
|
||||
Database DatabaseConfig
|
||||
Scheduler SchedulerConfig
|
||||
Log LogConfig
|
||||
Auth AuthConfig
|
||||
RateLimit RateLimitConfig
|
||||
CORS CORSConfig
|
||||
Keygen KeygenConfig
|
||||
CA CAConfig
|
||||
Notifiers NotifierConfig
|
||||
NetworkScan NetworkScanConfig
|
||||
EST ESTConfig
|
||||
SCEP SCEPConfig
|
||||
Verification VerificationConfig
|
||||
ACME ACMEConfig
|
||||
Vault VaultConfig
|
||||
DigiCert DigiCertConfig
|
||||
Sectigo SectigoConfig
|
||||
GoogleCAS GoogleCASConfig
|
||||
AWSACMPCA AWSACMPCAConfig
|
||||
Entrust EntrustConfig
|
||||
GlobalSign GlobalSignConfig
|
||||
EJBCA EJBCAConfig
|
||||
Digest DigestConfig
|
||||
HealthCheck HealthCheckConfig
|
||||
Encryption EncryptionConfig
|
||||
@@ -651,11 +652,14 @@ type SCEPConfig struct {
|
||||
// ChallengePassword is the shared secret used to authenticate SCEP enrollment requests.
|
||||
// Clients include this in the PKCS#10 CSR challengePassword attribute.
|
||||
//
|
||||
// REQUIRED when Enabled is true. If SCEP is enabled and this value is empty,
|
||||
// cmd/server/main.go's preflightSCEPChallengePassword check will refuse to
|
||||
// start the server (H-2, CWE-306): an empty shared secret allowed any client
|
||||
// that could reach /scep to enroll a CSR against the configured issuer. The
|
||||
// service-layer PKCSReq path also rejects this configuration defense-in-depth.
|
||||
// REQUIRED when Enabled is true. Config.Validate() below refuses to start the
|
||||
// server if SCEP is enabled and this value is empty (H-2, CWE-306): post-M-001
|
||||
// under option (D), the /scep endpoint rides the no-auth middleware chain per
|
||||
// RFC 8894 §3.2, so the challenge password is the sole application-layer
|
||||
// authentication boundary for SCEP enrollment. An empty shared secret would
|
||||
// allow any client that can reach /scep to enroll a CSR against the configured
|
||||
// issuer. The service-layer PKCSReq path also rejects this configuration
|
||||
// defense-in-depth.
|
||||
ChallengePassword string
|
||||
}
|
||||
|
||||
@@ -674,9 +678,30 @@ type VerificationConfig struct {
|
||||
|
||||
// ServerConfig contains HTTP server configuration.
|
||||
type ServerConfig struct {
|
||||
Host string // Server host (default: 127.0.0.1). Set via CERTCTL_SERVER_HOST.
|
||||
Port int // Server port (default: 8080). Set via CERTCTL_SERVER_PORT.
|
||||
MaxBodySize int64 // Maximum request body size in bytes (default: 1MB). Set via CERTCTL_MAX_BODY_SIZE.
|
||||
Host string // Server host (default: 127.0.0.1). Set via CERTCTL_SERVER_HOST.
|
||||
Port int // Server port (default: 8080). Set via CERTCTL_SERVER_PORT.
|
||||
MaxBodySize int64 // Maximum request body size in bytes (default: 1MB). Set via CERTCTL_MAX_BODY_SIZE.
|
||||
TLS ServerTLSConfig // HTTPS-only TLS configuration. Both CertPath and KeyPath are required.
|
||||
}
|
||||
|
||||
// ServerTLSConfig holds the server-side TLS material.
|
||||
//
|
||||
// The control plane is HTTPS-only as of the HTTPS-everywhere milestone
|
||||
// (§3 locked decisions: no `http` mode, no dual-listener, TLS 1.3 only).
|
||||
// Both CertPath and KeyPath are required; an empty value causes
|
||||
// Config.Validate() to return a fail-loud error and the server refuses
|
||||
// to start. There is no plaintext HTTP fallback, no N-release migration
|
||||
// bridge, and no auto-generated self-signed cert — operators either
|
||||
// supply a cert on disk (docker-compose init container, operator-managed
|
||||
// file, cert-manager mount) or the process exits non-zero.
|
||||
type ServerTLSConfig struct {
|
||||
// CertPath is the filesystem path to the server's PEM-encoded X.509
|
||||
// certificate. Set via CERTCTL_SERVER_TLS_CERT_PATH. Required.
|
||||
CertPath string
|
||||
|
||||
// KeyPath is the filesystem path to the server's PEM-encoded private
|
||||
// key that signs CertPath. Set via CERTCTL_SERVER_TLS_KEY_PATH. Required.
|
||||
KeyPath string
|
||||
}
|
||||
|
||||
// DatabaseConfig contains database connection configuration.
|
||||
@@ -708,6 +733,17 @@ type SchedulerConfig struct {
|
||||
// Setting: CERTCTL_SCHEDULER_NOTIFICATION_PROCESS_INTERVAL environment variable.
|
||||
NotificationProcessInterval time.Duration
|
||||
|
||||
// NotificationRetryInterval is how often the scheduler retries failed
|
||||
// notifications whose retry_count is below the service-layer 5-attempt
|
||||
// DLQ budget. Default: 2 minutes. Minimum: 1 second. Mirrors the I-001
|
||||
// RetryInterval knob: transitions eligible Failed notifications whose
|
||||
// next_retry_at has arrived back to Pending so the notification processor
|
||||
// picks them up on its next tick (closes coverage gap I-005 — HEAD had
|
||||
// no retry path for transient SMTP/webhook failures and notifications
|
||||
// stayed Failed forever).
|
||||
// Setting: CERTCTL_NOTIFICATION_RETRY_INTERVAL environment variable.
|
||||
NotificationRetryInterval time.Duration
|
||||
|
||||
// RetryInterval is how often the scheduler retries failed jobs whose Attempts
|
||||
// counter is below MaxAttempts. Default: 5 minutes. Minimum: 1 second.
|
||||
// Transitions eligible Failed jobs back to Pending so the job processor can
|
||||
@@ -715,6 +751,29 @@ type SchedulerConfig struct {
|
||||
// had no caller prior to this loop being wired).
|
||||
// Setting: CERTCTL_SCHEDULER_RETRY_INTERVAL environment variable.
|
||||
RetryInterval time.Duration
|
||||
|
||||
// JobTimeoutInterval is how often the reaper loop sweeps AwaitingCSR and
|
||||
// AwaitingApproval jobs for TTL expiration. Default: 10 minutes. Minimum: 1
|
||||
// second. Timed-out jobs are transitioned to Failed with a descriptive error
|
||||
// message; I-001's retry loop then auto-promotes eligible Failed jobs back
|
||||
// to Pending (closes coverage gap I-003).
|
||||
// Setting: CERTCTL_JOB_TIMEOUT_INTERVAL environment variable.
|
||||
JobTimeoutInterval time.Duration
|
||||
|
||||
// AwaitingCSRTimeout is the maximum age an AwaitingCSR job can remain in
|
||||
// that state before the reaper transitions it to Failed. Default: 24 hours.
|
||||
// An agent that hasn't submitted a CSR within this window is presumed
|
||||
// unreachable. Minimum: 1 second.
|
||||
// Setting: CERTCTL_JOB_AWAITING_CSR_TIMEOUT environment variable.
|
||||
AwaitingCSRTimeout time.Duration
|
||||
|
||||
// AwaitingApprovalTimeout is the maximum age an AwaitingApproval job can
|
||||
// remain in that state before the reaper transitions it to Failed. Default:
|
||||
// 168 hours (7 days). Reviewers who haven't approved within this window
|
||||
// force the renewal to fail loudly rather than silently stall. Minimum: 1
|
||||
// second.
|
||||
// Setting: CERTCTL_JOB_AWAITING_APPROVAL_TIMEOUT environment variable.
|
||||
AwaitingApprovalTimeout time.Duration
|
||||
}
|
||||
|
||||
// LogConfig contains logging configuration.
|
||||
@@ -804,6 +863,13 @@ func Load() (*Config, error) {
|
||||
Host: getEnv("CERTCTL_SERVER_HOST", "127.0.0.1"),
|
||||
Port: getEnvInt("CERTCTL_SERVER_PORT", 8080),
|
||||
MaxBodySize: getEnvInt64("CERTCTL_MAX_BODY_SIZE", 1024*1024), // 1MB default
|
||||
// HTTPS-everywhere milestone §2.1: both paths REQUIRED. Empty defaults
|
||||
// are intentional so Validate() emits a fail-loud error pointing at
|
||||
// docs/tls.md rather than silently binding plaintext HTTP.
|
||||
TLS: ServerTLSConfig{
|
||||
CertPath: getEnv("CERTCTL_SERVER_TLS_CERT_PATH", ""),
|
||||
KeyPath: getEnv("CERTCTL_SERVER_TLS_KEY_PATH", ""),
|
||||
},
|
||||
},
|
||||
Database: DatabaseConfig{
|
||||
URL: getEnv("CERTCTL_DATABASE_URL", "postgres://localhost/certctl"),
|
||||
@@ -815,7 +881,16 @@ func Load() (*Config, error) {
|
||||
JobProcessorInterval: getEnvDuration("CERTCTL_SCHEDULER_JOB_PROCESSOR_INTERVAL", 30*time.Second),
|
||||
AgentHealthCheckInterval: getEnvDuration("CERTCTL_SCHEDULER_AGENT_HEALTH_CHECK_INTERVAL", 2*time.Minute),
|
||||
NotificationProcessInterval: getEnvDuration("CERTCTL_SCHEDULER_NOTIFICATION_PROCESS_INTERVAL", 1*time.Minute),
|
||||
RetryInterval: getEnvDuration("CERTCTL_SCHEDULER_RETRY_INTERVAL", 5*time.Minute),
|
||||
// I-005: retry sweep for failed notifications. Mirrors RetryInterval
|
||||
// (I-001 job retry) but scoped to the notification DLQ machinery.
|
||||
// Default 2 minutes — fast enough to absorb transient SMTP/webhook
|
||||
// blips, slow enough to respect the service-layer 5-attempt budget
|
||||
// without hammering external notifier endpoints.
|
||||
NotificationRetryInterval: getEnvDuration("CERTCTL_NOTIFICATION_RETRY_INTERVAL", 2*time.Minute),
|
||||
RetryInterval: getEnvDuration("CERTCTL_SCHEDULER_RETRY_INTERVAL", 5*time.Minute),
|
||||
JobTimeoutInterval: getEnvDuration("CERTCTL_JOB_TIMEOUT_INTERVAL", 10*time.Minute),
|
||||
AwaitingCSRTimeout: getEnvDuration("CERTCTL_JOB_AWAITING_CSR_TIMEOUT", 24*time.Hour),
|
||||
AwaitingApprovalTimeout: getEnvDuration("CERTCTL_JOB_AWAITING_APPROVAL_TIMEOUT", 168*time.Hour),
|
||||
},
|
||||
Log: LogConfig{
|
||||
Level: getEnv("CERTCTL_LOG_LEVEL", "info"),
|
||||
@@ -845,7 +920,7 @@ func Load() (*Config, error) {
|
||||
Notifiers: NotifierConfig{
|
||||
SlackWebhookURL: getEnv("CERTCTL_SLACK_WEBHOOK_URL", ""),
|
||||
SlackChannel: getEnv("CERTCTL_SLACK_CHANNEL", ""),
|
||||
SlackUsername: getEnv("CERTCTL_SLACK_USERNAME", "certctl"),
|
||||
SlackUsername: getEnv("CERTCTL_SLACK_USERNAME", "certctl"),
|
||||
TeamsWebhookURL: getEnv("CERTCTL_TEAMS_WEBHOOK_URL", ""),
|
||||
PagerDutyRoutingKey: getEnv("CERTCTL_PAGERDUTY_ROUTING_KEY", ""),
|
||||
PagerDutySeverity: getEnv("CERTCTL_PAGERDUTY_SEVERITY", "warning"),
|
||||
@@ -1013,6 +1088,37 @@ func (c *Config) Validate() error {
|
||||
return fmt.Errorf("invalid server port: %d", c.Server.Port)
|
||||
}
|
||||
|
||||
// HTTPS-everywhere milestone §2.1 + §3 locked decisions: the control plane
|
||||
// is TLS-only and refuses to start without a cert. No plaintext HTTP fallback,
|
||||
// no auto-generated self-signed cert, no N-release migration window. An empty
|
||||
// CertPath or KeyPath is operator-visible misconfiguration, not a soft warning.
|
||||
if c.Server.TLS.CertPath == "" {
|
||||
return fmt.Errorf("server TLS cert path is required — refuse to start (HTTPS-only: set CERTCTL_SERVER_TLS_CERT_PATH to a PEM-encoded certificate; see docs/tls.md)")
|
||||
}
|
||||
if c.Server.TLS.KeyPath == "" {
|
||||
return fmt.Errorf("server TLS key path is required — refuse to start (HTTPS-only: set CERTCTL_SERVER_TLS_KEY_PATH to the PEM-encoded private key matching CERTCTL_SERVER_TLS_CERT_PATH; see docs/tls.md)")
|
||||
}
|
||||
|
||||
// Files must exist and be readable. Catches typos and missing mount paths
|
||||
// up-front so the operator gets a structured error on startup instead of
|
||||
// a deferred ListenAndServeTLS failure after the scheduler has already
|
||||
// fanned out its goroutines.
|
||||
if _, err := os.Stat(c.Server.TLS.CertPath); err != nil {
|
||||
return fmt.Errorf("server TLS cert file unreadable at %q: %w — refuse to start (HTTPS-only; see docs/tls.md)", c.Server.TLS.CertPath, err)
|
||||
}
|
||||
if _, err := os.Stat(c.Server.TLS.KeyPath); err != nil {
|
||||
return fmt.Errorf("server TLS key file unreadable at %q: %w — refuse to start (HTTPS-only; see docs/tls.md)", c.Server.TLS.KeyPath, err)
|
||||
}
|
||||
|
||||
// Parse the cert+key pair up-front. tls.LoadX509KeyPair verifies that the
|
||||
// key signs the cert (prevents the classic footgun of shipping a pair
|
||||
// whose private key doesn't match). Discard the returned Certificate — the
|
||||
// server constructs its own holder from fresh reads so SIGHUP reload is
|
||||
// authoritative.
|
||||
if _, err := tls.LoadX509KeyPair(c.Server.TLS.CertPath, c.Server.TLS.KeyPath); err != nil {
|
||||
return fmt.Errorf("server TLS cert/key pair invalid (cert=%q key=%q): %w — refuse to start (HTTPS-only; see docs/tls.md)", c.Server.TLS.CertPath, c.Server.TLS.KeyPath, err)
|
||||
}
|
||||
|
||||
// Validate database configuration
|
||||
if c.Database.URL == "" {
|
||||
return fmt.Errorf("database URL is required")
|
||||
@@ -1066,6 +1172,19 @@ func (c *Config) Validate() error {
|
||||
return fmt.Errorf("invalid keygen mode: %s (must be 'agent' or 'server')", c.Keygen.Mode)
|
||||
}
|
||||
|
||||
// SCEP fail-loud startup gate (H-2, CWE-306).
|
||||
//
|
||||
// Post-M-001 option (D) routes /scep through the no-auth middleware chain per
|
||||
// RFC 8894 §3.2 — SCEP clients authenticate via the challengePassword attribute
|
||||
// in the PKCS#10 CSR, not via HTTP Bearer tokens or TLS client certs. That makes
|
||||
// CERTCTL_SCEP_CHALLENGE_PASSWORD the sole application-layer authentication
|
||||
// boundary for SCEP enrollment. Refuse to start if it is empty when SCEP is
|
||||
// enabled: an empty shared secret would allow any client that can reach /scep to
|
||||
// enroll a CSR against the configured issuer (anonymous issuance).
|
||||
if c.SCEP.Enabled && c.SCEP.ChallengePassword == "" {
|
||||
return fmt.Errorf("SCEP is enabled but CERTCTL_SCEP_CHALLENGE_PASSWORD is empty — refuse to start (CWE-306: anonymous SCEP issuance is insecure; set a non-empty shared secret or disable SCEP with CERTCTL_SCEP_ENABLED=false). This gate duplicates cmd/server/main.go:preflightSCEPChallengePassword for defense in depth")
|
||||
}
|
||||
|
||||
// Validate scheduler intervals
|
||||
if c.Scheduler.RenewalCheckInterval < 1*time.Minute {
|
||||
return fmt.Errorf("renewal check interval must be at least 1 minute")
|
||||
@@ -1083,10 +1202,29 @@ func (c *Config) Validate() error {
|
||||
return fmt.Errorf("notification process interval must be at least 1 second")
|
||||
}
|
||||
|
||||
// I-005: guard against a misconfigured retry sweep that would either
|
||||
// spin-wait or never fire. Matches the NotificationProcessInterval
|
||||
// minimum (1s) so operators can tune both knobs from the same floor.
|
||||
if c.Scheduler.NotificationRetryInterval < 1*time.Second {
|
||||
return fmt.Errorf("notification retry interval must be at least 1 second")
|
||||
}
|
||||
|
||||
if c.Scheduler.RetryInterval < 1*time.Second {
|
||||
return fmt.Errorf("retry interval must be at least 1 second")
|
||||
}
|
||||
|
||||
if c.Scheduler.JobTimeoutInterval < 1*time.Second {
|
||||
return fmt.Errorf("job timeout interval must be at least 1 second")
|
||||
}
|
||||
|
||||
if c.Scheduler.AwaitingCSRTimeout < 1*time.Second {
|
||||
return fmt.Errorf("awaiting CSR timeout must be at least 1 second")
|
||||
}
|
||||
|
||||
if c.Scheduler.AwaitingApprovalTimeout < 1*time.Second {
|
||||
return fmt.Errorf("awaiting approval timeout must be at least 1 second")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+378
-11
@@ -1,8 +1,17 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -25,10 +34,76 @@ func clearCertctlEnv(t *testing.T) {
|
||||
}
|
||||
|
||||
// setMinimalValidEnv sets the minimum env vars needed for Load() to succeed (Validate passes).
|
||||
//
|
||||
// HTTPS-everywhere milestone (§2.1 + §3 locked decisions): the control plane
|
||||
// is TLS-only and Validate() refuses to pass without a readable cert/key pair
|
||||
// on disk. setMinimalValidEnv therefore materializes a throwaway ECDSA P-256
|
||||
// self-signed pair in t.TempDir() and points the two TLS env vars at it so
|
||||
// every Load-based test inherits a valid HTTPS posture without each caller
|
||||
// having to spell out cert generation. The temp dir is cleaned up by
|
||||
// testing.T at end-of-test.
|
||||
func setMinimalValidEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
// api-key auth requires a secret
|
||||
t.Setenv("CERTCTL_AUTH_SECRET", "test-secret-key")
|
||||
// HTTPS-only control plane requires a real cert/key pair on disk.
|
||||
certPath, keyPath := generateTestTLSPair(t)
|
||||
t.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", certPath)
|
||||
t.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", keyPath)
|
||||
}
|
||||
|
||||
// generateTestTLSPair writes an ECDSA P-256 self-signed certificate + private
|
||||
// key pair to files inside t.TempDir() and returns the paths. Same shape used
|
||||
// by cmd/server/tls_test.go — this duplicates the generator rather than
|
||||
// importing it so the config package tests stay independent of cmd/server.
|
||||
func generateTestTLSPair(t *testing.T) (certPath, keyPath string) {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ecdsa.GenerateKey: %v", err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "certctl-config-test"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatalf("x509.CreateCertificate: %v", err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
certPath = filepath.Join(dir, "cert.pem")
|
||||
keyPath = filepath.Join(dir, "key.pem")
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
if err := os.WriteFile(certPath, certPEM, 0o600); err != nil {
|
||||
t.Fatalf("write cert: %v", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(key)
|
||||
if err != nil {
|
||||
t.Fatalf("x509.MarshalECPrivateKey: %v", err)
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
t.Fatalf("write key: %v", err)
|
||||
}
|
||||
return certPath, keyPath
|
||||
}
|
||||
|
||||
// validServerConfig returns a ServerConfig with Port=8080 plus a freshly
|
||||
// minted TLS cert/key pair on disk, so Validate() passes the HTTPS-only
|
||||
// preflight (cert empty → stat → tls.LoadX509KeyPair round-trip). Every
|
||||
// struct-based Validate test uses this so they fail for the reason they
|
||||
// claim to test, not for a missing TLS pair.
|
||||
func validServerConfig(t *testing.T) ServerConfig {
|
||||
t.Helper()
|
||||
certPath, keyPath := generateTestTLSPair(t)
|
||||
return ServerConfig{
|
||||
Port: 8080,
|
||||
TLS: ServerTLSConfig{CertPath: certPath, KeyPath: keyPath},
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_DefaultValues(t *testing.T) {
|
||||
@@ -134,6 +209,13 @@ func TestLoad_DefaultValues(t *testing.T) {
|
||||
func TestLoad_AllEnvVarsSet(t *testing.T) {
|
||||
clearCertctlEnv(t)
|
||||
|
||||
// HTTPS-only control plane: Load() → Validate() refuses an empty cert path.
|
||||
// Materialize a throwaway ECDSA P-256 pair and point the two TLS env vars
|
||||
// at it before setting every other CERTCTL_* var this test cares about.
|
||||
certPath, keyPath := generateTestTLSPair(t)
|
||||
t.Setenv("CERTCTL_SERVER_TLS_CERT_PATH", certPath)
|
||||
t.Setenv("CERTCTL_SERVER_TLS_KEY_PATH", keyPath)
|
||||
|
||||
t.Setenv("CERTCTL_SERVER_HOST", "0.0.0.0")
|
||||
t.Setenv("CERTCTL_SERVER_PORT", "9090")
|
||||
t.Setenv("CERTCTL_MAX_BODY_SIZE", "2097152")
|
||||
@@ -318,7 +400,7 @@ func TestLoad_CommaSeparatedList(t *testing.T) {
|
||||
|
||||
func TestValidate_ValidConfig(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "test-secret"},
|
||||
@@ -328,7 +410,11 @@ func TestValidate_ValidConfig(t *testing.T) {
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
NotificationRetryInterval: 2 * time.Minute,
|
||||
RetryInterval: 5 * time.Minute,
|
||||
JobTimeoutInterval: 10 * time.Minute,
|
||||
AwaitingCSRTimeout: 24 * time.Hour,
|
||||
AwaitingApprovalTimeout: 168 * time.Hour,
|
||||
},
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
@@ -338,7 +424,7 @@ func TestValidate_ValidConfig(t *testing.T) {
|
||||
|
||||
func TestValidate_AuthTypeNone(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "none", Secret: ""},
|
||||
@@ -348,7 +434,11 @@ func TestValidate_AuthTypeNone(t *testing.T) {
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
NotificationRetryInterval: 2 * time.Minute,
|
||||
RetryInterval: 5 * time.Minute,
|
||||
JobTimeoutInterval: 10 * time.Minute,
|
||||
AwaitingCSRTimeout: 24 * time.Hour,
|
||||
AwaitingApprovalTimeout: 168 * time.Hour,
|
||||
},
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
@@ -358,7 +448,7 @@ func TestValidate_AuthTypeNone(t *testing.T) {
|
||||
|
||||
func TestValidate_InvalidAuthType(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "oauth", Secret: "key"},
|
||||
@@ -377,7 +467,7 @@ func TestValidate_InvalidAuthType(t *testing.T) {
|
||||
|
||||
func TestValidate_APIKeyAuth_MissingSecret(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: ""},
|
||||
@@ -396,7 +486,7 @@ func TestValidate_APIKeyAuth_MissingSecret(t *testing.T) {
|
||||
|
||||
func TestValidate_JWTAuth_MissingSecret(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "jwt", Secret: ""},
|
||||
@@ -415,7 +505,7 @@ func TestValidate_JWTAuth_MissingSecret(t *testing.T) {
|
||||
|
||||
func TestValidate_InvalidKeygenMode(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
@@ -463,9 +553,168 @@ func TestValidate_InvalidPort(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidate_TLSCertPathEmpty pins the first of the HTTPS-only fail-loud
|
||||
// gates in Validate(): an empty CertPath must produce the operator-facing
|
||||
// "server TLS cert path is required" error. Per §2.1 + §3 locked decisions,
|
||||
// there is no plaintext HTTP fallback — missing TLS config is a hard startup
|
||||
// refusal, not a warning.
|
||||
func TestValidate_TLSCertPathEmpty(t *testing.T) {
|
||||
_, keyPath := generateTestTLSPair(t)
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{
|
||||
Port: 8080,
|
||||
TLS: ServerTLSConfig{CertPath: "", KeyPath: keyPath},
|
||||
},
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
Keygen: KeygenConfig{Mode: "agent"},
|
||||
Scheduler: SchedulerConfig{
|
||||
RenewalCheckInterval: 1 * time.Hour,
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
},
|
||||
}
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() should return error for empty TLS cert path")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "server TLS cert path is required") {
|
||||
t.Errorf("error = %q, want substring %q", err.Error(), "server TLS cert path is required")
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidate_TLSKeyPathEmpty pins the second HTTPS-only gate: empty KeyPath
|
||||
// must produce the "server TLS key path is required" error. Runs with a valid
|
||||
// CertPath so the cert-empty gate (which fires first) is cleanly bypassed —
|
||||
// proves the key-empty gate is actually reached.
|
||||
func TestValidate_TLSKeyPathEmpty(t *testing.T) {
|
||||
certPath, _ := generateTestTLSPair(t)
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{
|
||||
Port: 8080,
|
||||
TLS: ServerTLSConfig{CertPath: certPath, KeyPath: ""},
|
||||
},
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
Keygen: KeygenConfig{Mode: "agent"},
|
||||
Scheduler: SchedulerConfig{
|
||||
RenewalCheckInterval: 1 * time.Hour,
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
},
|
||||
}
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() should return error for empty TLS key path")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "server TLS key path is required") {
|
||||
t.Errorf("error = %q, want substring %q", err.Error(), "server TLS key path is required")
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidate_TLSCertFileMissing pins the os.Stat gate on the cert path. A
|
||||
// non-existent path must surface "server TLS cert file unreadable" so the
|
||||
// operator sees the bad path in the error (file=%q) instead of a deferred
|
||||
// ListenAndServeTLS panic after the scheduler has already fanned out.
|
||||
func TestValidate_TLSCertFileMissing(t *testing.T) {
|
||||
_, keyPath := generateTestTLSPair(t)
|
||||
missingCert := filepath.Join(t.TempDir(), "does-not-exist.pem")
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{
|
||||
Port: 8080,
|
||||
TLS: ServerTLSConfig{CertPath: missingCert, KeyPath: keyPath},
|
||||
},
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
Keygen: KeygenConfig{Mode: "agent"},
|
||||
Scheduler: SchedulerConfig{
|
||||
RenewalCheckInterval: 1 * time.Hour,
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
},
|
||||
}
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() should return error for missing TLS cert file")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "server TLS cert file unreadable") {
|
||||
t.Errorf("error = %q, want substring %q", err.Error(), "server TLS cert file unreadable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidate_TLSKeyFileMissing pins the os.Stat gate on the key path. Uses a
|
||||
// valid CertPath so the cert-missing gate does not pre-empt; proves the key
|
||||
// gate is reached and reports the bad key path.
|
||||
func TestValidate_TLSKeyFileMissing(t *testing.T) {
|
||||
certPath, _ := generateTestTLSPair(t)
|
||||
missingKey := filepath.Join(t.TempDir(), "does-not-exist.key")
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{
|
||||
Port: 8080,
|
||||
TLS: ServerTLSConfig{CertPath: certPath, KeyPath: missingKey},
|
||||
},
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
Keygen: KeygenConfig{Mode: "agent"},
|
||||
Scheduler: SchedulerConfig{
|
||||
RenewalCheckInterval: 1 * time.Hour,
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
},
|
||||
}
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() should return error for missing TLS key file")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "server TLS key file unreadable") {
|
||||
t.Errorf("error = %q, want substring %q", err.Error(), "server TLS key file unreadable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidate_TLSMismatchedPair pins the tls.LoadX509KeyPair gate — the
|
||||
// classic "you shipped the wrong private key" footgun. Generates two
|
||||
// independent ECDSA pairs and crosses them (pair1 cert + pair2 key). Both
|
||||
// files exist and parse as PEM, so os.Stat passes; only the cryptographic
|
||||
// round-trip inside LoadX509KeyPair catches the mismatch.
|
||||
func TestValidate_TLSMismatchedPair(t *testing.T) {
|
||||
certPath1, _ := generateTestTLSPair(t)
|
||||
_, keyPath2 := generateTestTLSPair(t)
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{
|
||||
Port: 8080,
|
||||
TLS: ServerTLSConfig{CertPath: certPath1, KeyPath: keyPath2},
|
||||
},
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
Keygen: KeygenConfig{Mode: "agent"},
|
||||
Scheduler: SchedulerConfig{
|
||||
RenewalCheckInterval: 1 * time.Hour,
|
||||
JobProcessorInterval: 30 * time.Second,
|
||||
AgentHealthCheckInterval: 2 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
},
|
||||
}
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() should return error for mismatched TLS cert/key pair")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "server TLS cert/key pair invalid") {
|
||||
t.Errorf("error = %q, want substring %q", err.Error(), "server TLS cert/key pair invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_EmptyDatabaseURL(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
@@ -484,7 +733,7 @@ func TestValidate_EmptyDatabaseURL(t *testing.T) {
|
||||
|
||||
func TestValidate_InvalidLogLevel(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "verbose", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
@@ -503,7 +752,7 @@ func TestValidate_InvalidLogLevel(t *testing.T) {
|
||||
|
||||
func TestValidate_InvalidLogFormat(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "yaml"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
@@ -565,7 +814,7 @@ func TestValidate_SchedulerIntervalTooSmall(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
@@ -581,7 +830,7 @@ func TestValidate_SchedulerIntervalTooSmall(t *testing.T) {
|
||||
|
||||
func TestValidate_DatabaseMaxConnectionsZero(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Server: ServerConfig{Port: 8080},
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 0},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "key"},
|
||||
@@ -708,3 +957,121 @@ func TestGetEnvBool(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
// I-003: Job timeout reaper configuration tests
|
||||
func TestConfig_Scheduler_JobTimeoutDefaults(t *testing.T) {
|
||||
clearCertctlEnv(t)
|
||||
setMinimalValidEnv(t)
|
||||
// Explicitly unset the three I-003 env vars to exercise the default path.
|
||||
t.Setenv("CERTCTL_JOB_TIMEOUT_INTERVAL", "")
|
||||
t.Setenv("CERTCTL_JOB_AWAITING_CSR_TIMEOUT", "")
|
||||
t.Setenv("CERTCTL_JOB_AWAITING_APPROVAL_TIMEOUT", "")
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Scheduler.JobTimeoutInterval != 10*time.Minute {
|
||||
t.Errorf("JobTimeoutInterval = %v, want 10m", cfg.Scheduler.JobTimeoutInterval)
|
||||
}
|
||||
if cfg.Scheduler.AwaitingCSRTimeout != 24*time.Hour {
|
||||
t.Errorf("AwaitingCSRTimeout = %v, want 24h", cfg.Scheduler.AwaitingCSRTimeout)
|
||||
}
|
||||
if cfg.Scheduler.AwaitingApprovalTimeout != 168*time.Hour {
|
||||
t.Errorf("AwaitingApprovalTimeout = %v, want 168h", cfg.Scheduler.AwaitingApprovalTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_Scheduler_JobTimeoutEnvOverride(t *testing.T) {
|
||||
clearCertctlEnv(t)
|
||||
setMinimalValidEnv(t)
|
||||
t.Setenv("CERTCTL_JOB_TIMEOUT_INTERVAL", "15m")
|
||||
t.Setenv("CERTCTL_JOB_AWAITING_CSR_TIMEOUT", "48h")
|
||||
t.Setenv("CERTCTL_JOB_AWAITING_APPROVAL_TIMEOUT", "336h")
|
||||
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Scheduler.JobTimeoutInterval != 15*time.Minute {
|
||||
t.Errorf("JobTimeoutInterval = %v, want 15m", cfg.Scheduler.JobTimeoutInterval)
|
||||
}
|
||||
if cfg.Scheduler.AwaitingCSRTimeout != 48*time.Hour {
|
||||
t.Errorf("AwaitingCSRTimeout = %v, want 48h", cfg.Scheduler.AwaitingCSRTimeout)
|
||||
}
|
||||
if cfg.Scheduler.AwaitingApprovalTimeout != 336*time.Hour {
|
||||
t.Errorf("AwaitingApprovalTimeout = %v, want 336h", cfg.Scheduler.AwaitingApprovalTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_Scheduler_JobTimeoutValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
field string
|
||||
value time.Duration
|
||||
wantErrMsg string
|
||||
}{
|
||||
{
|
||||
"JobTimeoutInterval too small",
|
||||
"JobTimeoutInterval",
|
||||
500 * time.Millisecond,
|
||||
"job timeout interval must be at least 1 second",
|
||||
},
|
||||
{
|
||||
"AwaitingCSRTimeout too small",
|
||||
"AwaitingCSRTimeout",
|
||||
500 * time.Millisecond,
|
||||
"awaiting CSR timeout must be at least 1 second",
|
||||
},
|
||||
{
|
||||
"AwaitingApprovalTimeout too small",
|
||||
"AwaitingApprovalTimeout",
|
||||
500 * time.Millisecond,
|
||||
"awaiting approval timeout must be at least 1 second",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Start from a fully valid config so the I-003 timeout checks
|
||||
// are the only potential failure point.
|
||||
cfg := &Config{
|
||||
Server: validServerConfig(t),
|
||||
Database: DatabaseConfig{URL: "postgres://localhost/certctl", MaxConnections: 25},
|
||||
Log: LogConfig{Level: "info", Format: "json"},
|
||||
Auth: AuthConfig{Type: "api-key", Secret: "test-secret"},
|
||||
Keygen: KeygenConfig{Mode: "agent"},
|
||||
Scheduler: SchedulerConfig{
|
||||
RenewalCheckInterval: 1 * time.Minute,
|
||||
JobProcessorInterval: 1 * time.Minute,
|
||||
AgentHealthCheckInterval: 1 * time.Minute,
|
||||
NotificationProcessInterval: 1 * time.Minute,
|
||||
NotificationRetryInterval: 2 * time.Minute,
|
||||
RetryInterval: 1 * time.Minute,
|
||||
JobTimeoutInterval: 10 * time.Minute,
|
||||
AwaitingCSRTimeout: 24 * time.Hour,
|
||||
AwaitingApprovalTimeout: 168 * time.Hour,
|
||||
},
|
||||
}
|
||||
|
||||
// Override the specific field under test
|
||||
switch tt.field {
|
||||
case "JobTimeoutInterval":
|
||||
cfg.Scheduler.JobTimeoutInterval = tt.value
|
||||
case "AwaitingCSRTimeout":
|
||||
cfg.Scheduler.AwaitingCSRTimeout = tt.value
|
||||
case "AwaitingApprovalTimeout":
|
||||
cfg.Scheduler.AwaitingApprovalTimeout = tt.value
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("Validate() = nil, want error containing %q", tt.wantErrMsg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.wantErrMsg) {
|
||||
t.Errorf("Validate() error = %q, want to contain %q", err.Error(), tt.wantErrMsg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ type DeploymentTarget struct {
|
||||
LastTestedAt *time.Time `json:"last_tested_at,omitempty"`
|
||||
TestStatus string `json:"test_status,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
RetiredAt *time.Time `json:"retired_at,omitempty"` // I-004: soft-retirement timestamp (nil = active)
|
||||
RetiredReason *string `json:"retired_reason,omitempty"` // I-004: reason captured at cascade retirement
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
@@ -49,6 +51,67 @@ type Agent struct {
|
||||
Architecture string `json:"architecture"`
|
||||
IPAddress string `json:"ip_address"`
|
||||
Version string `json:"version"`
|
||||
// I-004: soft-retirement fields. An agent with RetiredAt != nil is the
|
||||
// canonical "retired" state. The Status column remains as before (Online
|
||||
// / Offline / Degraded) and is preserved at retirement time as the
|
||||
// last-seen operational status; RetiredAt is the source of truth for
|
||||
// "should we filter this row from active listings?".
|
||||
RetiredAt *time.Time `json:"retired_at,omitempty"`
|
||||
RetiredReason *string `json:"retired_reason,omitempty"`
|
||||
}
|
||||
|
||||
// IsRetired returns true when this agent has been soft-retired.
|
||||
// I-004: callers that iterate active agents (stats dashboard, stale-offline
|
||||
// sweeper, handler-facing list) must skip retired rows by default.
|
||||
func (a *Agent) IsRetired() bool { return a != nil && a.RetiredAt != nil }
|
||||
|
||||
// AgentDependencyCounts captures the active downstream rows that would be
|
||||
// affected by retiring an agent. Returned by the preflight pass on
|
||||
// DELETE /api/v1/agents/{id}. Zero counts mean a clean soft-retire is safe;
|
||||
// any non-zero count blocks a default retire with HTTP 409 and requires an
|
||||
// explicit ?force=true&reason=... escape hatch from the operator.
|
||||
type AgentDependencyCounts struct {
|
||||
ActiveTargets int `json:"active_targets"` // deployment_targets.agent_id=id AND retired_at IS NULL
|
||||
ActiveCertificates int `json:"active_certificates"` // certificates currently deployed via one of this agent's active targets
|
||||
PendingJobs int `json:"pending_jobs"` // jobs.agent_id=id AND status IN (Pending, AwaitingCSR, AwaitingApproval, Running)
|
||||
}
|
||||
|
||||
// HasDependencies reports whether any preflight counter is non-zero.
|
||||
func (d AgentDependencyCounts) HasDependencies() bool {
|
||||
return d.ActiveTargets > 0 || d.ActiveCertificates > 0 || d.PendingJobs > 0
|
||||
}
|
||||
|
||||
// SentinelAgentIDs enumerates the four reserved agent identities that back
|
||||
// non-agent discovery subsystems. These rows are created by cmd/server on
|
||||
// startup and retiring them would orphan their subsystem — the network
|
||||
// scanner and the three cloud secret-manager sources all key writes to
|
||||
// these IDs via service.SentinelAgentID / service.SentinelAWSSecretsMgr /
|
||||
// service.SentinelAzureKeyVault / service.SentinelGCPSecretMgr. The four
|
||||
// literal IDs below MUST stay in lockstep with those service-package
|
||||
// constants (see internal/service/network_scan.go line 23 and
|
||||
// internal/service/cloud_discovery.go lines 14-16).
|
||||
//
|
||||
// The retirement service refuses them unconditionally — even with
|
||||
// ?force=true — via ErrAgentIsSentinel. Living here (and not in the
|
||||
// service package) lets handler, repository, and scheduler code filter
|
||||
// them without importing service and creating a cycle.
|
||||
var SentinelAgentIDs = []string{
|
||||
"server-scanner",
|
||||
"cloud-aws-sm",
|
||||
"cloud-azure-kv",
|
||||
"cloud-gcp-sm",
|
||||
}
|
||||
|
||||
// IsSentinelAgent reports whether id matches one of the four reserved
|
||||
// sentinel agent IDs. A linear scan is fine — the slice is length 4 and
|
||||
// the check is rare (only on retirement attempts and sweeper filters).
|
||||
func IsSentinelAgent(id string) bool {
|
||||
for _, s := range SentinelAgentIDs {
|
||||
if s == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AgentMetadata contains runtime metadata reported by agents via heartbeat.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestAgent_IsRetired covers the I-004 soft-retirement predicate that gates
|
||||
// which callers hide an agent row from active listings.
|
||||
func TestAgent_IsRetired(t *testing.T) {
|
||||
t.Run("nil receiver is not retired", func(t *testing.T) {
|
||||
var a *Agent
|
||||
if a.IsRetired() {
|
||||
t.Fatalf("nil *Agent should not be retired")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("zero value is not retired", func(t *testing.T) {
|
||||
a := &Agent{}
|
||||
if a.IsRetired() {
|
||||
t.Fatalf("zero Agent should not be retired")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("RetiredAt set is retired", func(t *testing.T) {
|
||||
now := time.Now()
|
||||
a := &Agent{RetiredAt: &now}
|
||||
if !a.IsRetired() {
|
||||
t.Fatalf("Agent with RetiredAt != nil must be retired")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestAgentDependencyCounts_HasDependencies verifies the preflight
|
||||
// aggregation helper used by the 409 block path of DELETE /agents/{id}.
|
||||
func TestAgentDependencyCounts_HasDependencies(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
counts AgentDependencyCounts
|
||||
want bool
|
||||
}{
|
||||
{"all zero", AgentDependencyCounts{}, false},
|
||||
{"active target", AgentDependencyCounts{ActiveTargets: 1}, true},
|
||||
{"active cert", AgentDependencyCounts{ActiveCertificates: 1}, true},
|
||||
{"pending job", AgentDependencyCounts{PendingJobs: 1}, true},
|
||||
{"mixed", AgentDependencyCounts{ActiveTargets: 3, PendingJobs: 2}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := tc.counts.HasDependencies(); got != tc.want {
|
||||
t.Fatalf("HasDependencies()=%v want=%v counts=%+v", got, tc.want, tc.counts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,15 @@ import (
|
||||
)
|
||||
|
||||
// NotificationEvent records a notification sent to users about certificate events.
|
||||
//
|
||||
// I-005 extends the event with a retry counter, a nullable next-retry timestamp
|
||||
// that drives the retry-sweep partial index, and a nullable last-error string
|
||||
// preserving the most recent transient failure so operators triaging the dead
|
||||
// letter queue can see *why* a notification died without chasing server logs.
|
||||
// Status stays a plain `string` (not retyped to NotificationStatus) because the
|
||||
// repo layer materialises it directly from PostgreSQL's VARCHAR column and the
|
||||
// service layer compares against the NotificationStatus* constants via
|
||||
// `string(...)` casts at call sites — see service.RetryFailedNotifications.
|
||||
type NotificationEvent struct {
|
||||
ID string `json:"id"`
|
||||
Type NotificationType `json:"type"`
|
||||
@@ -15,9 +24,37 @@ type NotificationEvent struct {
|
||||
SentAt *time.Time `json:"sent_at,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
RetryCount int `json:"retry_count"`
|
||||
NextRetryAt *time.Time `json:"next_retry_at,omitempty"`
|
||||
LastError *string `json:"last_error,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// NotificationStatus is the typed string alias for the lifecycle status of a
|
||||
// NotificationEvent. It mirrors the VARCHAR(50) column on notification_events
|
||||
// and the status values used by the I-005 retry/DLQ machinery.
|
||||
//
|
||||
// Status transitions:
|
||||
//
|
||||
// pending → sent (delivery succeeded)
|
||||
// pending → failed → pending (transient failure, re-armed by retry sweep)
|
||||
// pending → failed → dead (retry_count reached max_attempts; DLQ)
|
||||
// pending → read (operator acknowledged, no delivery needed)
|
||||
//
|
||||
// Values are lowercase to match the pre-I-005 on-wire representation used by
|
||||
// existing UpdateStatus calls and the seed_demo.sql fixtures; retyping
|
||||
// NotificationEvent.Status to NotificationStatus would be a breaking DB scan
|
||||
// change, so the type is kept additive and consumed via `string(const)` casts.
|
||||
type NotificationStatus string
|
||||
|
||||
const (
|
||||
NotificationStatusPending NotificationStatus = "pending"
|
||||
NotificationStatusSent NotificationStatus = "sent"
|
||||
NotificationStatusFailed NotificationStatus = "failed"
|
||||
NotificationStatusDead NotificationStatus = "dead"
|
||||
NotificationStatusRead NotificationStatus = "read"
|
||||
)
|
||||
|
||||
// NotificationType represents the event that triggered a notification.
|
||||
type NotificationType string
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package domain
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNotificationType_Constants(t *testing.T) {
|
||||
tests := map[string]NotificationType{
|
||||
@@ -71,3 +74,54 @@ func TestNotificationEvent_Fields(t *testing.T) {
|
||||
t.Errorf("expected error 'failed to send', got %v", event.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotificationStatus_Constants verifies that I-005 introduces a typed
|
||||
// NotificationStatus alongside canonical lowercase string constants covering
|
||||
// the pending → sent, pending → failed → dead, and pending → read transitions.
|
||||
// The Red signal here is a compile error: the type and the NotificationStatusDead
|
||||
// constant do not exist before Phase 2 Green.
|
||||
func TestNotificationStatus_Constants(t *testing.T) {
|
||||
tests := map[string]NotificationStatus{
|
||||
"pending": NotificationStatusPending,
|
||||
"sent": NotificationStatusSent,
|
||||
"failed": NotificationStatusFailed,
|
||||
"dead": NotificationStatusDead,
|
||||
"read": NotificationStatusRead,
|
||||
}
|
||||
for expected, got := range tests {
|
||||
if string(got) != expected {
|
||||
t.Errorf("expected %q, got %q", expected, string(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotificationEvent_RetryFields verifies the I-005 retry/DLQ columns are
|
||||
// surfaced on the domain model: a RetryCount counter, a nullable NextRetryAt
|
||||
// timestamp used by the retry-sweep partial index, and a nullable LastError
|
||||
// string preserving the most recent transient failure for operator triage.
|
||||
// The Red signal is a compile error — these fields do not exist yet.
|
||||
func TestNotificationEvent_RetryFields(t *testing.T) {
|
||||
next := time.Now().Add(2 * time.Minute)
|
||||
lastErr := "connection refused"
|
||||
event := &NotificationEvent{
|
||||
ID: "notif-retry-001",
|
||||
Type: NotificationTypeExpirationWarning,
|
||||
Channel: NotificationChannelWebhook,
|
||||
Recipient: "https://hooks.example.com/certs",
|
||||
Message: "retry me",
|
||||
Status: string(NotificationStatusFailed),
|
||||
RetryCount: 3,
|
||||
NextRetryAt: &next,
|
||||
LastError: &lastErr,
|
||||
}
|
||||
|
||||
if event.RetryCount != 3 {
|
||||
t.Errorf("expected RetryCount 3, got %d", event.RetryCount)
|
||||
}
|
||||
if event.NextRetryAt == nil || !event.NextRetryAt.Equal(next) {
|
||||
t.Errorf("expected NextRetryAt %v, got %v", next, event.NextRetryAt)
|
||||
}
|
||||
if event.LastError == nil || *event.LastError != "connection refused" {
|
||||
t.Errorf("expected LastError 'connection refused', got %v", event.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,25 +103,25 @@ func TestCertificateLifecycle(t *testing.T) {
|
||||
// Create router and register handlers
|
||||
r := router.New()
|
||||
r.RegisterHandlers(router.HandlerRegistry{
|
||||
Certificates: certificateHandler,
|
||||
Issuers: issuerHandler,
|
||||
Targets: targetHandler,
|
||||
Agents: agentHandler,
|
||||
Jobs: jobHandler,
|
||||
Policies: policyHandler,
|
||||
Profiles: profileHandler,
|
||||
Teams: teamHandler,
|
||||
Owners: ownerHandler,
|
||||
AgentGroups: agentGroupHandler,
|
||||
Audit: auditHandler,
|
||||
Notifications: notificationHandler,
|
||||
Stats: statsHandler,
|
||||
Metrics: metricsHandler,
|
||||
Health: healthHandler,
|
||||
Discovery: discoveryHandler,
|
||||
NetworkScan: networkScanHandler,
|
||||
Verification: verificationHandler,
|
||||
BulkRevocation: handler.BulkRevocationHandler{},
|
||||
Certificates: certificateHandler,
|
||||
Issuers: issuerHandler,
|
||||
Targets: targetHandler,
|
||||
Agents: agentHandler,
|
||||
Jobs: jobHandler,
|
||||
Policies: policyHandler,
|
||||
Profiles: profileHandler,
|
||||
Teams: teamHandler,
|
||||
Owners: ownerHandler,
|
||||
AgentGroups: agentGroupHandler,
|
||||
Audit: auditHandler,
|
||||
Notifications: notificationHandler,
|
||||
Stats: statsHandler,
|
||||
Metrics: metricsHandler,
|
||||
Health: healthHandler,
|
||||
Discovery: discoveryHandler,
|
||||
NetworkScan: networkScanHandler,
|
||||
Verification: verificationHandler,
|
||||
BulkRevocation: handler.BulkRevocationHandler{},
|
||||
})
|
||||
r.RegisterESTHandlers(estHandler)
|
||||
|
||||
@@ -742,6 +742,25 @@ func (m *mockJobRepository) ClaimPendingByAgentID(ctx context.Context, agentID s
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListTimedOutAwaitingJobs is the I-003 integration-mock stub. Returns jobs whose
|
||||
// created_at predates the relevant cutoff for their status.
|
||||
func (m *mockJobRepository) ListTimedOutAwaitingJobs(ctx context.Context, csrCutoff, approvalCutoff time.Time) ([]*domain.Job, error) {
|
||||
var jobs []*domain.Job
|
||||
for _, j := range m.jobs {
|
||||
switch j.Status {
|
||||
case domain.JobStatusAwaitingCSR:
|
||||
if j.CreatedAt.Before(csrCutoff) {
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
case domain.JobStatusAwaitingApproval:
|
||||
if j.CreatedAt.Before(approvalCutoff) {
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
type mockAuditRepository struct {
|
||||
events []*domain.AuditEvent
|
||||
}
|
||||
@@ -829,6 +848,56 @@ func (m *mockAgentRepository) GetByAPIKey(ctx context.Context, keyHash string) (
|
||||
return nil, fmt.Errorf("agent not found")
|
||||
}
|
||||
|
||||
// I-004: the integration-level mockAgentRepository implements the 6 new
|
||||
// retirement-surface methods as thin contract-satisfying stubs. The
|
||||
// integration suite exercises lifecycle flows (issue → renew → deploy)
|
||||
// that don't touch retirement, so these methods never need real behavior
|
||||
// here — they exist purely to keep mockAgentRepository a valid
|
||||
// AgentRepository implementation after migration 000015 expanded the
|
||||
// interface. Dedicated retirement tests live in internal/service/
|
||||
// agent_retire_test.go against the richer service-layer mockAgentRepo.
|
||||
|
||||
func (m *mockAgentRepository) ListRetired(ctx context.Context, page, perPage int) ([]*domain.Agent, int, error) {
|
||||
var retired []*domain.Agent
|
||||
for _, a := range m.agents {
|
||||
if a.RetiredAt != nil {
|
||||
retired = append(retired, a)
|
||||
}
|
||||
}
|
||||
return retired, len(retired), nil
|
||||
}
|
||||
|
||||
func (m *mockAgentRepository) SoftRetire(ctx context.Context, id string, retiredAt time.Time, reason string) error {
|
||||
agent, ok := m.agents[id]
|
||||
if !ok {
|
||||
return fmt.Errorf("agent not found")
|
||||
}
|
||||
if agent.RetiredAt != nil {
|
||||
return nil
|
||||
}
|
||||
stamped := retiredAt
|
||||
agent.RetiredAt = &stamped
|
||||
stampedReason := reason
|
||||
agent.RetiredReason = &stampedReason
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockAgentRepository) RetireAgentWithCascade(ctx context.Context, id string, retiredAt time.Time, reason string) error {
|
||||
return m.SoftRetire(ctx, id, retiredAt, reason)
|
||||
}
|
||||
|
||||
func (m *mockAgentRepository) CountActiveTargets(ctx context.Context, agentID string) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (m *mockAgentRepository) CountActiveCertificates(ctx context.Context, agentID string) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (m *mockAgentRepository) CountPendingJobs(ctx context.Context, agentID string) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
type mockTargetRepository struct {
|
||||
targets map[string]*domain.DeploymentTarget
|
||||
}
|
||||
@@ -953,6 +1022,46 @@ func (m *mockNotificationRepository) UpdateStatus(ctx context.Context, id string
|
||||
return fmt.Errorf("notification not found")
|
||||
}
|
||||
|
||||
// I-005: retry/DLQ interface satisfiers. The integration tests in this package
|
||||
// drive the end-to-end lifecycle against a NotificationService which requires
|
||||
// the full repository.NotificationRepository interface, but none of the
|
||||
// lifecycle scenarios exercise the retry sweep or dead-letter transitions —
|
||||
// they're covered by unit tests in internal/service/notification_test.go. So
|
||||
// these are deliberate no-op / panic-free stubs whose only job is to satisfy
|
||||
// the compile-time interface contract. If a future integration test needs
|
||||
// real retry semantics, promote this mock to match internal/service's
|
||||
// mockNotifRepo (testutil_test.go:410) one-for-one.
|
||||
|
||||
func (m *mockNotificationRepository) ListRetryEligible(ctx context.Context, now time.Time, maxAttempts, limit int) ([]*domain.NotificationEvent, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockNotificationRepository) RecordFailedAttempt(ctx context.Context, id string, lastError string, nextRetryAt time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockNotificationRepository) MarkAsDead(ctx context.Context, id string, lastError string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockNotificationRepository) Requeue(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountByStatus satisfies the NotificationRepository interface contract added
|
||||
// by I-005 Phase 2 Green. Counts in-memory rows so StatsService wiring exercised
|
||||
// by the lifecycle integration tests gets a truthful count even though the
|
||||
// retry/DLQ surface isn't driven here.
|
||||
func (m *mockNotificationRepository) CountByStatus(ctx context.Context, status string) (int64, error) {
|
||||
var count int64
|
||||
for _, n := range m.notifications {
|
||||
if n.Status == status {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
type mockPolicyRepository struct {
|
||||
rules map[string]*domain.PolicyRule
|
||||
violations []*domain.PolicyViolation
|
||||
|
||||
+46
-3
@@ -2,11 +2,14 @@ package mcp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -18,15 +21,45 @@ type Client struct {
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new certctl API client.
|
||||
func NewClient(baseURL, apiKey string) *Client {
|
||||
// NewClient creates a new certctl API client. The control plane is HTTPS-only
|
||||
// as of v2.2, so the transport is pinned to TLS 1.3 and optionally loads a
|
||||
// PEM-encoded CA bundle from caBundlePath (empty means "trust the system
|
||||
// roots"). The insecure flag disables certificate verification and is a
|
||||
// dev-only opt-in documented in docs/tls.md — it must never be set in
|
||||
// production. Returns an error if the CA bundle path is non-empty but the
|
||||
// file is missing or contains no valid PEM-encoded certificates, so the
|
||||
// caller can fail loud before any network call.
|
||||
func NewClient(baseURL, apiKey, caBundlePath string, insecure bool) (*Client, error) {
|
||||
tlsConfig := &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: insecure, //nolint:gosec // opt-in dev toggle, documented in docs/tls.md
|
||||
}
|
||||
if caBundlePath != "" {
|
||||
pemBytes, err := os.ReadFile(caBundlePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading CA bundle at %q: %w", caBundlePath, err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pemBytes) {
|
||||
return nil, fmt.Errorf("CA bundle at %q contains no valid PEM-encoded certificates", caBundlePath)
|
||||
}
|
||||
tlsConfig.RootCAs = pool
|
||||
}
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
apiKey: apiKey,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
},
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get performs an HTTP GET and returns the raw JSON response body.
|
||||
@@ -49,6 +82,16 @@ func (c *Client) Delete(path string) (json.RawMessage, error) {
|
||||
return c.do("DELETE", path, nil, nil)
|
||||
}
|
||||
|
||||
// DeleteWithQuery performs an HTTP DELETE with query parameters. I-004 adds
|
||||
// this transport so MCP tools can target endpoints that carry flags in the
|
||||
// query string (e.g. DELETE /api/v1/agents/{id}?force=true&reason=…). Client.Delete
|
||||
// is path-only; without this method the retire tool silently drops force/reason,
|
||||
// turning every cascade retire into a default soft-retire. Shares do()'s 204
|
||||
// normalization and 4xx/5xx error propagation so tool authors get one contract.
|
||||
func (c *Client) DeleteWithQuery(path string, query url.Values) (json.RawMessage, error) {
|
||||
return c.do("DELETE", path, query, nil)
|
||||
}
|
||||
|
||||
// GetRaw performs an HTTP GET and returns the raw response body bytes and content type.
|
||||
// Used for binary responses (DER CRL, OCSP).
|
||||
func (c *Client) GetRaw(path string) ([]byte, string, error) {
|
||||
|
||||
+248
-15
@@ -1,17 +1,30 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"io"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewClient(t *testing.T) {
|
||||
c := NewClient("http://localhost:8443", "test-key")
|
||||
if c.baseURL != "http://localhost:8443" {
|
||||
t.Errorf("expected baseURL http://localhost:8443, got %s", c.baseURL)
|
||||
c, err := NewClient("https://localhost:8443", "test-key", "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient err=%v want nil", err)
|
||||
}
|
||||
if c.baseURL != "https://localhost:8443" {
|
||||
t.Errorf("expected baseURL https://localhost:8443, got %s", c.baseURL)
|
||||
}
|
||||
if c.apiKey != "test-key" {
|
||||
t.Errorf("expected apiKey test-key, got %s", c.apiKey)
|
||||
@@ -44,7 +57,7 @@ func TestClient_Get(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, err := c.Get("/api/v1/certificates", map[string][]string{"status": {"Active"}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -64,7 +77,7 @@ func TestClient_Get_NoAuth(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "")
|
||||
c, _ := NewClient(server.URL, "", "", false)
|
||||
_, err := c.Get("/api/v1/certificates", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -95,7 +108,7 @@ func TestClient_Post(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, err := c.Post("/api/v1/certificates", map[string]string{"name": "test-cert"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -120,7 +133,7 @@ func TestClient_Put(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, err := c.Put("/api/v1/certificates/mc-test", map[string]string{"name": "updated"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -139,7 +152,7 @@ func TestClient_Delete_204(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, err := c.Delete("/api/v1/certificates/mc-test")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -161,7 +174,7 @@ func TestClient_ErrorResponse(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
_, err := c.Get("/api/v1/certificates/nonexistent", nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 404 response")
|
||||
@@ -179,7 +192,7 @@ func TestClient_ServerError(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
_, err := c.Post("/api/v1/certificates", map[string]string{"name": "test"})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 500 response")
|
||||
@@ -202,7 +215,7 @@ func TestClient_GetRaw(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, contentType, err := c.GetRaw("/.well-known/pki/crl/iss-local")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -222,7 +235,7 @@ func TestClient_GetRaw_Error(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
_, _, err := c.GetRaw("/.well-known/pki/crl/nonexistent")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 404 response")
|
||||
@@ -230,7 +243,7 @@ func TestClient_GetRaw_Error(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestClient_ConnectionRefused(t *testing.T) {
|
||||
c := NewClient("http://localhost:1", "test-key")
|
||||
c, _ := NewClient("https://localhost:1", "test-key", "", false)
|
||||
_, err := c.Get("/api/v1/certificates", nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for connection refused")
|
||||
@@ -247,7 +260,7 @@ func TestClient_PostNilBody(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, err := c.Post("/api/v1/certificates/mc-test/renew", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -270,7 +283,7 @@ func TestClient_QueryParams(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := NewClient(server.URL, "test-key")
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
q := paginationQuery(2, 10)
|
||||
_, err := c.Get("/api/v1/certificates", q)
|
||||
if err != nil {
|
||||
@@ -287,3 +300,223 @@ func containsStr(s, substr string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// generateTestCert produces a short-lived self-signed RSA-2048 certificate for
|
||||
// tests that need a PEM-encodable cert. Mirrors the helper used in
|
||||
// internal/cli/client_test.go so the two packages pin the same HTTPS-Everywhere
|
||||
// TLS-wiring contract against matching test fixtures.
|
||||
func generateTestCert() *x509.Certificate {
|
||||
now := time.Now()
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{
|
||||
CommonName: "test.certctl.local",
|
||||
},
|
||||
NotBefore: now,
|
||||
NotAfter: now.Add(365 * 24 * time.Hour),
|
||||
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
DNSNames: []string{"test.certctl.local"},
|
||||
}
|
||||
|
||||
privateKey, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
certBytes, _ := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||
cert, _ := x509.ParseCertificate(certBytes)
|
||||
return cert
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// HTTPS-Everywhere milestone (v2.2, §3.2 + §7 Phase 5):
|
||||
// The MCP server binary talks HTTPS-only to the certctl control plane. These
|
||||
// tests pin the three contracts every client binary (agent, CLI, MCP) must
|
||||
// satisfy in lock-step:
|
||||
// (a) CA bundle load success — PEM loads, RootCAs + MinVersion=TLS1.3 wired
|
||||
// through the injected *http.Transport so the httpClient actually uses
|
||||
// them on the wire, not just in the struct.
|
||||
// (b) CA bundle load failure — missing file and malformed/empty PEM each fail
|
||||
// loud with a pinned substring so operators get a useful diagnostic.
|
||||
// (c) End-to-end TLS round-trip — an httptest.NewTLSServer whose own cert is
|
||||
// written out as the CA bundle validates that every TLS-config knob
|
||||
// actually flows into the dialer.
|
||||
// The substrings below must stay in sync with internal/mcp/client.go:NewClient;
|
||||
// drifting them in isolation is exactly what this suite is here to catch.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
// writeCABundle PEM-encodes a DER cert and writes it to a temp file under the
|
||||
// test's own TempDir. Returns the absolute path for piping into NewClient.
|
||||
func writeCABundle(t *testing.T, dir string, certDER []byte, filename string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, filename)
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
||||
if err := os.WriteFile(path, pemBytes, 0o600); err != nil {
|
||||
t.Fatalf("writing CA bundle to %q: %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// TestNewClient_CABundle_Success pins the happy path: a valid PEM CA bundle
|
||||
// loads, populates RootCAs on the client's TLS config, and leaves
|
||||
// MinVersion=TLS1.3 intact. Regression guard for any future edit that
|
||||
// accidentally swaps the transport or detaches *tls.Config from *http.Transport.
|
||||
func TestNewClient_CABundle_Success(t *testing.T) {
|
||||
cert := generateTestCert()
|
||||
tmp := t.TempDir()
|
||||
bundlePath := writeCABundle(t, tmp, cert.Raw, "ca.pem")
|
||||
|
||||
client, err := NewClient("https://certctl-server:8443", "test-key", bundlePath, false)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient with valid CA bundle err=%v want nil", err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Fatal("NewClient returned nil client on happy path")
|
||||
}
|
||||
|
||||
transport, ok := client.httpClient.Transport.(*http.Transport)
|
||||
if !ok {
|
||||
t.Fatalf("httpClient.Transport type=%T want *http.Transport (TLS config injection broke)", client.httpClient.Transport)
|
||||
}
|
||||
if transport.TLSClientConfig == nil {
|
||||
t.Fatal("transport.TLSClientConfig is nil; TLS config must be set on every client")
|
||||
}
|
||||
if transport.TLSClientConfig.RootCAs == nil {
|
||||
t.Fatal("transport.TLSClientConfig.RootCAs is nil; CA bundle path was ignored")
|
||||
}
|
||||
if transport.TLSClientConfig.MinVersion != tls.VersionTLS13 {
|
||||
t.Errorf("MinVersion=%d want tls.VersionTLS13 (%d); HTTPS-Everywhere requires TLS1.3 floor",
|
||||
transport.TLSClientConfig.MinVersion, tls.VersionTLS13)
|
||||
}
|
||||
if transport.TLSClientConfig.InsecureSkipVerify {
|
||||
t.Error("InsecureSkipVerify=true with insecure=false arg; flag wiring crossed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_CABundle_MissingFile pins the fail-loud path for a nonexistent
|
||||
// bundle path. The error surface must include "reading CA bundle" so operators
|
||||
// see the right diagnostic instead of a downstream TLS-handshake-error.
|
||||
func TestNewClient_CABundle_MissingFile(t *testing.T) {
|
||||
_, err := NewClient("https://certctl-server:8443", "test-key", "/nonexistent/path/ca.pem", false)
|
||||
if err == nil {
|
||||
t.Fatal("NewClient with missing CA bundle err=nil; must fail loud so operators see the right diagnostic")
|
||||
}
|
||||
if !containsStr(err.Error(), "reading CA bundle") {
|
||||
t.Errorf("err=%q must contain %q so operators can locate the misconfigured path", err.Error(), "reading CA bundle")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_CABundle_EmptyPEM pins the fail-loud path for a file whose
|
||||
// contents are not valid PEM. AppendCertsFromPEM returning false is the signal
|
||||
// we need to surface — otherwise the client would silently ship with an empty
|
||||
// cert pool and every TLS handshake would fail downstream.
|
||||
func TestNewClient_CABundle_EmptyPEM(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
garbagePath := filepath.Join(tmp, "garbage.pem")
|
||||
if err := os.WriteFile(garbagePath, []byte("not a pem certificate, just bytes"), 0o600); err != nil {
|
||||
t.Fatalf("writing garbage file: %v", err)
|
||||
}
|
||||
|
||||
_, err := NewClient("https://certctl-server:8443", "test-key", garbagePath, false)
|
||||
if err == nil {
|
||||
t.Fatal("NewClient with malformed PEM err=nil; must fail loud, not silently skip")
|
||||
}
|
||||
if !containsStr(err.Error(), "no valid PEM-encoded certificates") {
|
||||
t.Errorf("err=%q must contain %q so operators know the file parsed but held no certs",
|
||||
err.Error(), "no valid PEM-encoded certificates")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_TLSRoundTrip validates that the TLS config knobs we set on
|
||||
// NewClient actually reach the wire. An httptest.NewTLSServer signs its own
|
||||
// self-signed leaf; we PEM-encode that server cert, write it as the CA bundle,
|
||||
// and issue a real HTTPS GET via c.Get. A successful round-trip proves RootCAs
|
||||
// + MinVersion are flowing through *http.Transport into the dialer, not just
|
||||
// surviving into the client struct.
|
||||
func TestNewClient_TLSRoundTrip(t *testing.T) {
|
||||
var handlerHit int
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet && r.URL.Path == "/api/v1/certificates" {
|
||||
handlerHit++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"data": []interface{}{},
|
||||
"total": 0,
|
||||
})
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
serverCert := server.Certificate()
|
||||
if serverCert == nil {
|
||||
t.Fatal("httptest.NewTLSServer.Certificate() returned nil; cannot build CA bundle")
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
bundlePath := writeCABundle(t, tmp, serverCert.Raw, "server-ca.pem")
|
||||
|
||||
client, err := NewClient(server.URL, "test-key", bundlePath, false)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(TLS server) err=%v want nil", err)
|
||||
}
|
||||
data, err := client.Get("/api/v1/certificates", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Get over HTTPS err=%v; TLS config must reach the wire", err)
|
||||
}
|
||||
if data == nil {
|
||||
t.Fatal("Get over HTTPS returned nil data; want non-empty JSON body")
|
||||
}
|
||||
if handlerHit != 1 {
|
||||
t.Errorf("handlerHit=%d want 1; request did not reach the TLS server", handlerHit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClient_InsecureSkipVerify pins the dev-only escape hatch: an untrusted
|
||||
// TLS server (cert NOT in the client's root pool) must be reachable when
|
||||
// insecure=true. This is the only path in the control plane that disables
|
||||
// certificate verification; it's documented in docs/tls.md and gated by the
|
||||
// CERTCTL_SERVER_TLS_INSECURE_SKIP_VERIFY env var so it never slips into
|
||||
// production silently.
|
||||
func TestNewClient_InsecureSkipVerify(t *testing.T) {
|
||||
var handlerHit int
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
handlerHit++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"data": []interface{}{},
|
||||
"total": 0,
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// No CA bundle → system roots, which will NOT trust the self-signed
|
||||
// httptest cert. insecure=true is the only thing keeping this call from
|
||||
// failing with an x509-unknown-authority error.
|
||||
client, err := NewClient(server.URL, "test-key", "", true)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient(insecure=true) err=%v want nil", err)
|
||||
}
|
||||
|
||||
transport, ok := client.httpClient.Transport.(*http.Transport)
|
||||
if !ok {
|
||||
t.Fatalf("httpClient.Transport type=%T want *http.Transport", client.httpClient.Transport)
|
||||
}
|
||||
if !transport.TLSClientConfig.InsecureSkipVerify {
|
||||
t.Fatal("insecure=true arg did not set TLSClientConfig.InsecureSkipVerify; flag wiring broken")
|
||||
}
|
||||
if transport.TLSClientConfig.MinVersion != tls.VersionTLS13 {
|
||||
t.Errorf("MinVersion=%d want tls.VersionTLS13 even with insecure=true (TLS1.3 floor is not optional)",
|
||||
transport.TLSClientConfig.MinVersion)
|
||||
}
|
||||
|
||||
data, err := client.Get("/api/v1/certificates", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(insecure=true) err=%v; escape hatch must still complete the round-trip", err)
|
||||
}
|
||||
if data == nil {
|
||||
t.Fatal("Get(insecure=true) returned nil data; want non-empty JSON body")
|
||||
}
|
||||
if handlerHit != 1 {
|
||||
t.Errorf("handlerHit=%d want 1; insecure round-trip did not reach the server", handlerHit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestClient_DeleteWithQuery_ForceRetire covers the new transport capability
|
||||
// that I-004 adds to the MCP client. The retire tool needs to issue
|
||||
// DELETE /api/v1/agents/{id}?force=true&reason=... — Client.Delete as it
|
||||
// stands only accepts a path, dropping query parameters on the floor. Phase 2b
|
||||
// must add DeleteWithQuery so the MCP retire tool can hit the force escape
|
||||
// hatch; without this, every retire-via-MCP call with force=true silently
|
||||
// becomes a default soft-retire and either succeeds wrongly or 409s.
|
||||
func TestClient_DeleteWithQuery_ForceRetire(t *testing.T) {
|
||||
var (
|
||||
sawMethod string
|
||||
sawPath string
|
||||
sawForce string
|
||||
sawReason string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sawMethod = r.Method
|
||||
sawPath = r.URL.Path
|
||||
sawForce = r.URL.Query().Get("force")
|
||||
sawReason = r.URL.Query().Get("reason")
|
||||
|
||||
if r.Method != http.MethodDelete || r.URL.Path != "/api/v1/agents/ag-1" {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"retired_at": "2026-04-18T12:00:00Z",
|
||||
"already_retired": false,
|
||||
"cascade": true,
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c, _ := NewClient(server.URL, "test-key", "", false)
|
||||
// Compile-fail until Phase 2b grows Client.DeleteWithQuery. Passing the
|
||||
// query as a url.Values is the established pattern (matches Get's shape).
|
||||
query := url.Values{}
|
||||
query.Set("force", "true")
|
||||
query.Set("reason", "decommissioning rack 7")
|
||||
data, err := c.DeleteWithQuery("/api/v1/agents/ag-1", query)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteWithQuery err=%v want nil", err)
|
||||
}
|
||||
if data == nil {
|
||||
t.Fatal("DeleteWithQuery returned nil data; want 200 body echo-back")
|
||||
}
|
||||
|
||||
if sawMethod != http.MethodDelete {
|
||||
t.Errorf("method=%q want DELETE", sawMethod)
|
||||
}
|
||||
if sawPath != "/api/v1/agents/ag-1" {
|
||||
t.Errorf("path=%q want /api/v1/agents/ag-1 (query must be stripped from path)", sawPath)
|
||||
}
|
||||
if sawForce != "true" {
|
||||
t.Errorf("force query=%q want \"true\"", sawForce)
|
||||
}
|
||||
if sawReason != "decommissioning rack 7" {
|
||||
t.Errorf("reason query=%q want %q", sawReason, "decommissioning rack 7")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_DeleteWithQuery_NoQuery covers the defensive path: a nil/empty
|
||||
// query must still produce a clean DELETE against the bare path with no stray
|
||||
// "?" suffix. Matches the Get() shape (see client.go do()) so downstream tools
|
||||
// can reuse one code path.
|
||||
func TestClient_DeleteWithQuery_NoQuery(t *testing.T) {
|
||||
var sawRawPath string
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sawRawPath = r.URL.RequestURI()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{"ok": true})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c, _ := NewClient(server.URL, "", "", false)
|
||||
if _, err := c.DeleteWithQuery("/api/v1/agents/ag-1", nil); err != nil {
|
||||
t.Fatalf("DeleteWithQuery(nil query) err=%v want nil", err)
|
||||
}
|
||||
// No query → no ? suffix.
|
||||
if strings.Contains(sawRawPath, "?") {
|
||||
t.Errorf("raw path=%q contains stray ?; empty query must not serialize", sawRawPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_DeleteWithQuery_204ReturnsMinimalBody covers the idempotent path.
|
||||
// The handler returns 204 No Content for an already-retired agent; the
|
||||
// existing do() helper normalises this to {"status":"deleted"}. The new
|
||||
// DeleteWithQuery must share that behavior so MCP tool authors don't have to
|
||||
// special-case the return shape.
|
||||
func TestClient_DeleteWithQuery_204ReturnsMinimalBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c, _ := NewClient(server.URL, "", "", false)
|
||||
data, err := c.DeleteWithQuery("/api/v1/agents/ag-1", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteWithQuery(204) err=%v want nil (idempotent)", err)
|
||||
}
|
||||
if data == nil {
|
||||
t.Fatal("DeleteWithQuery(204) returned nil; want synthetic body")
|
||||
}
|
||||
if !strings.Contains(string(data), "deleted") && !strings.Contains(string(data), "status") {
|
||||
t.Errorf("DeleteWithQuery(204) body=%q; must surface a non-empty sentinel", string(data))
|
||||
}
|
||||
}
|
||||
|
||||
// TestClient_DeleteWithQuery_409PropagatesError covers the preflight-blocked
|
||||
// surface. A 409 with dependency counts must bubble up as a Go error so the
|
||||
// MCP tool can present it to the LLM operator rather than silently swallow
|
||||
// the rejection.
|
||||
func TestClient_DeleteWithQuery_409PropagatesError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusConflict)
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"error": "blocked_by_dependencies",
|
||||
"message": "agent has active targets",
|
||||
"counts": map[string]int{
|
||||
"active_targets": 3,
|
||||
"active_certificates": 7,
|
||||
"pending_jobs": 2,
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c, _ := NewClient(server.URL, "", "", false)
|
||||
_, err := c.DeleteWithQuery("/api/v1/agents/ag-1", nil)
|
||||
if err == nil {
|
||||
t.Fatalf("DeleteWithQuery(409) err=nil; 409 must propagate as Go error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "409") {
|
||||
t.Errorf("err=%q should include HTTP status 409 for debuggability", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgentInput_ShapePinned is a compile-time assertion that the MCP
|
||||
// tool input struct for certctl_retire_agent exists with the required fields
|
||||
// and their expected tag shapes. The LLM discovers this input schema via
|
||||
// jsonschema tags — refactoring field names without updating callers silently
|
||||
// breaks tool discovery.
|
||||
//
|
||||
// Red until Phase 2b adds RetireAgentInput to internal/mcp/types.go. This
|
||||
// assertion deliberately exercises every field so the test fails at compile
|
||||
// time rather than runtime.
|
||||
func TestRetireAgentInput_ShapePinned(t *testing.T) {
|
||||
// Zero-value construction of the expected input — fails to compile until
|
||||
// the struct exists with fields {ID string, Force bool, Reason string}.
|
||||
input := RetireAgentInput{
|
||||
ID: "ag-1",
|
||||
Force: true,
|
||||
Reason: "decommissioning rack 7",
|
||||
}
|
||||
|
||||
if input.ID != "ag-1" {
|
||||
t.Errorf("RetireAgentInput.ID=%q want ag-1 (field binding broken)", input.ID)
|
||||
}
|
||||
if !input.Force {
|
||||
t.Errorf("RetireAgentInput.Force=false want true")
|
||||
}
|
||||
if input.Reason != "decommissioning rack 7" {
|
||||
t.Errorf("RetireAgentInput.Reason=%q want decommissioning rack 7", input.Reason)
|
||||
}
|
||||
|
||||
// Also pin the JSON surface — LLMs send and receive these field names,
|
||||
// so json tags must stay snake_case even through refactors.
|
||||
encoded, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal RetireAgentInput: %v", err)
|
||||
}
|
||||
body := string(encoded)
|
||||
for _, want := range []string{`"id":"ag-1"`, `"force":true`, `"reason":"decommissioning rack 7"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("RetireAgentInput JSON=%q missing %q (tag shape drifted)", body, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestListRetiredAgentsInput_ShapePinned mirrors the pagination input shape
|
||||
// used across the MCP toolset (see ListParams). The list-retired-agents tool
|
||||
// takes page + per_page with snake_case JSON tags. Compile-fail until
|
||||
// Phase 2b either adds ListRetiredAgentsInput or documents that list-retired
|
||||
// reuses the existing ListParams type (both paths are acceptable — the test
|
||||
// just pins whichever Phase 2b picks).
|
||||
func TestListRetiredAgentsInput_ShapePinned(t *testing.T) {
|
||||
// Phase 2b may either (a) add a dedicated ListRetiredAgentsInput struct
|
||||
// or (b) reuse the existing ListParams. Either is fine — we pin the
|
||||
// field-access contract rather than the struct name to let the
|
||||
// implementation choose. Compile-fail guards against the tool being
|
||||
// registered without any pagination input at all.
|
||||
var input ListParams
|
||||
input.Page = 1
|
||||
input.PerPage = 50
|
||||
if input.Page != 1 || input.PerPage != 50 {
|
||||
t.Errorf("ListParams fields Page/PerPage broken; listing pagination will misroute")
|
||||
}
|
||||
}
|
||||
+69
-3
@@ -506,6 +506,53 @@ func registerAgentTools(s *gomcp.Server, c *Client) {
|
||||
}
|
||||
return textResult(data)
|
||||
})
|
||||
|
||||
// I-004: soft-retirement. DELETE /api/v1/agents/{id} returns 200 on a
|
||||
// fresh retire (body echoes retired_at/already_retired/cascade/counts),
|
||||
// 204 on an idempotent retire of an already-retired agent (do() in
|
||||
// client.go normalizes that to {"status":"deleted"}), 409 when downstream
|
||||
// dependencies block the retire and force wasn't set, 403 on sentinel
|
||||
// agents, or 400 when force=true was sent without a reason. The tool
|
||||
// forwards the raw handler response so the LLM operator sees the
|
||||
// dependency counts and can decide whether to retry with force=true.
|
||||
gomcp.AddTool(s, &gomcp.Tool{
|
||||
Name: "certctl_retire_agent",
|
||||
Description: "Soft-retire an agent (DELETE /api/v1/agents/{id}). Sets retired_at + retired_reason on the row; the agent is filtered from the default listing and surfaces only via certctl_list_retired_agents. Default is a safety-gated soft-retire that returns 409 blocked_by_dependencies if the agent has active targets, active certificates, or pending jobs — the returned counts tell you what would be orphaned. Pass force=true to cascade through and retire those dependents too; force=true requires a non-empty reason (captured in the audit trail). Sentinel discovery agents (server-scanner, cloud-aws-sm, cloud-azure-kv, cloud-gcp-sm) cannot be retired — the handler returns 403 unconditionally. Idempotent: retrying on an already-retired agent returns 204 without side effects.",
|
||||
}, func(ctx context.Context, req *gomcp.CallToolRequest, input RetireAgentInput) (*gomcp.CallToolResult, any, error) {
|
||||
// Client-side mirror of the handler's ErrForceReasonRequired contract
|
||||
// (see internal/api/handler/agents.go) so the LLM gets an immediate,
|
||||
// actionable error instead of a round-trip 400. Whitespace-only
|
||||
// reasons are treated as empty — matches handler's TrimSpace check.
|
||||
if input.Force && input.Reason == "" {
|
||||
return errorResult(fmt.Errorf("reason is required when force=true"))
|
||||
}
|
||||
query := url.Values{}
|
||||
if input.Force {
|
||||
query.Set("force", "true")
|
||||
}
|
||||
if input.Reason != "" {
|
||||
query.Set("reason", input.Reason)
|
||||
}
|
||||
data, err := c.DeleteWithQuery("/api/v1/agents/"+input.ID, query)
|
||||
if err != nil {
|
||||
return errorResult(err)
|
||||
}
|
||||
return textResult(data)
|
||||
})
|
||||
|
||||
// I-004: retired agents are filtered out of GET /api/v1/agents by default.
|
||||
// The /agents/retired endpoint is the opt-in view — same pagination shape
|
||||
// as the default listing, but filters to rows where retired_at IS NOT NULL.
|
||||
gomcp.AddTool(s, &gomcp.Tool{
|
||||
Name: "certctl_list_retired_agents",
|
||||
Description: "List soft-retired agents (GET /api/v1/agents/retired). These are agents that have been retired via certctl_retire_agent; retired_at and retired_reason are populated. Returned separately from certctl_list_agents so the default listing stays focused on operational agents.",
|
||||
}, func(ctx context.Context, req *gomcp.CallToolRequest, input ListParams) (*gomcp.CallToolResult, any, error) {
|
||||
data, err := c.Get("/api/v1/agents/retired", paginationQuery(input.Page, input.PerPage))
|
||||
if err != nil {
|
||||
return errorResult(err)
|
||||
}
|
||||
return textResult(data)
|
||||
})
|
||||
}
|
||||
|
||||
// ── Jobs ────────────────────────────────────────────────────────────
|
||||
@@ -927,9 +974,13 @@ func registerAuditTools(s *gomcp.Server, c *Client) {
|
||||
func registerNotificationTools(s *gomcp.Server, c *Client) {
|
||||
gomcp.AddTool(s, &gomcp.Tool{
|
||||
Name: "certctl_list_notifications",
|
||||
Description: "List notification events (expiration warnings, renewal/deployment results, policy violations, revocations).",
|
||||
}, func(ctx context.Context, req *gomcp.CallToolRequest, input ListParams) (*gomcp.CallToolResult, any, error) {
|
||||
data, err := c.Get("/api/v1/notifications", paginationQuery(input.Page, input.PerPage))
|
||||
Description: "List notification events (expiration warnings, renewal/deployment results, policy violations, revocations). Optional status filter supports the I-005 Dead letter tab (status=dead).",
|
||||
}, func(ctx context.Context, req *gomcp.CallToolRequest, input ListNotificationsInput) (*gomcp.CallToolResult, any, error) {
|
||||
q := paginationQuery(input.Page, input.PerPage)
|
||||
if input.Status != "" {
|
||||
q.Set("status", input.Status)
|
||||
}
|
||||
data, err := c.Get("/api/v1/notifications", q)
|
||||
if err != nil {
|
||||
return errorResult(err)
|
||||
}
|
||||
@@ -957,6 +1008,21 @@ func registerNotificationTools(s *gomcp.Server, c *Client) {
|
||||
}
|
||||
return textResult(data)
|
||||
})
|
||||
|
||||
// I-005: requeue a dead-letter notification. Flips status from 'dead'
|
||||
// back to 'pending' and clears next_retry_at so the retry sweep picks
|
||||
// the notification up on its next tick. Operator-triggered; the tool
|
||||
// is the MCP counterpart of the GUI's Dead letter tab "Requeue" button.
|
||||
gomcp.AddTool(s, &gomcp.Tool{
|
||||
Name: "certctl_requeue_notification",
|
||||
Description: "Requeue a dead notification back to pending so the retry sweep can deliver it again. Used to recover from persistent delivery failures after the underlying issue (SMTP config, webhook endpoint, etc.) has been fixed.",
|
||||
}, func(ctx context.Context, req *gomcp.CallToolRequest, input GetByIDInput) (*gomcp.CallToolResult, any, error) {
|
||||
data, err := c.Post("/api/v1/notifications/"+input.ID+"/requeue", nil)
|
||||
if err != nil {
|
||||
return errorResult(err)
|
||||
}
|
||||
return textResult(data)
|
||||
})
|
||||
}
|
||||
|
||||
// ── Stats ───────────────────────────────────────────────────────────
|
||||
|
||||
+10
-10
@@ -88,7 +88,7 @@ func TestRegisterTools_ToolCount(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
RegisterTools(server, client)
|
||||
|
||||
// The server should have tools registered — we can verify by listing them
|
||||
@@ -166,7 +166,7 @@ func TestToolEndToEnd_ListCertificates(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
|
||||
// Manually call the handler logic that would be registered as a tool
|
||||
q := paginationQuery(1, 50)
|
||||
@@ -204,7 +204,7 @@ func TestToolEndToEnd_CreateCertificate(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
|
||||
input := CreateCertificateInput{
|
||||
Name: "API Production",
|
||||
@@ -244,7 +244,7 @@ func TestToolEndToEnd_TriggerRenewal(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
data, err := client.Post("/api/v1/certificates/mc-api-prod/renew", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -272,7 +272,7 @@ func TestToolEndToEnd_DeleteTarget(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
data, err := client.Delete("/api/v1/targets/t-platform")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -300,7 +300,7 @@ func TestToolEndToEnd_RevokeCertificate(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
input := RevokeCertificateInput{
|
||||
ID: "mc-api-prod",
|
||||
Reason: "keyCompromise",
|
||||
@@ -327,7 +327,7 @@ func TestToolEndToEnd_AgentHeartbeat(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
_, err := client.Post("/api/v1/agents/agent-001/heartbeat", map[string]string{
|
||||
"os": "linux",
|
||||
"architecture": "amd64",
|
||||
@@ -347,7 +347,7 @@ func TestToolEndToEnd_ListWithFilters(t *testing.T) {
|
||||
api := mockCertctlAPI(log)
|
||||
defer api.Close()
|
||||
|
||||
client := NewClient(api.URL, "test-key")
|
||||
client, _ := NewClient(api.URL, "test-key", "", false)
|
||||
q := paginationQuery(1, 25)
|
||||
q.Set("status", "Pending")
|
||||
q.Set("type", "Renewal")
|
||||
@@ -377,7 +377,7 @@ func TestToolEndToEnd_GetRawBinary(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "test-key")
|
||||
client, _ := NewClient(server.URL, "test-key", "", false)
|
||||
data, ct, err := client.GetRaw("/.well-known/pki/crl/iss-local")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -397,7 +397,7 @@ func TestToolEndToEnd_ErrorPropagation(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewClient(server.URL, "test-key")
|
||||
client, _ := NewClient(server.URL, "test-key", "", false)
|
||||
_, err := client.Get("/api/v1/certificates", nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 403 response")
|
||||
|
||||
@@ -152,6 +152,23 @@ type AgentJobStatusInput struct {
|
||||
Error string `json:"error,omitempty" jsonschema:"Error message if job failed"`
|
||||
}
|
||||
|
||||
// RetireAgentInput pins the MCP tool surface for certctl_retire_agent. I-004
|
||||
// introduces a soft-retirement flow that the handler exposes on DELETE
|
||||
// /api/v1/agents/{id} with two optional query flags: force=true cascades
|
||||
// through dependent active targets/certs/jobs, and reason is the human-readable
|
||||
// string captured in the audit trail. The handler enforces
|
||||
// ErrForceReasonRequired when force=true is sent without a reason; we surface
|
||||
// both as separate fields so the LLM can populate them independently and so
|
||||
// the retire_agent_test shape assertion stays aligned with the JSON-wire
|
||||
// contract. ID is always emitted (no omitempty) because a retire call without
|
||||
// a target agent is meaningless; Force and Reason are omitempty so the default
|
||||
// soft-retire path sends no query suffix at all.
|
||||
type RetireAgentInput struct {
|
||||
ID string `json:"id" jsonschema:"Agent ID to soft-retire"`
|
||||
Force bool `json:"force,omitempty" jsonschema:"Cascade-retire downstream active targets, certs, and jobs (requires reason)"`
|
||||
Reason string `json:"reason,omitempty" jsonschema:"Human-readable reason (required when force=true)"`
|
||||
}
|
||||
|
||||
// ── Jobs ────────────────────────────────────────────────────────────
|
||||
|
||||
type ListJobsInput struct {
|
||||
@@ -165,6 +182,16 @@ type RejectJobInput struct {
|
||||
Reason string `json:"reason,omitempty" jsonschema:"Reason for rejection"`
|
||||
}
|
||||
|
||||
// ── Notifications ───────────────────────────────────────────────────
|
||||
|
||||
// ListNotificationsInput adds the I-005 status filter on top of the standard
|
||||
// pagination params. Status="dead" drives the Dead letter tab use case;
|
||||
// empty status preserves the pre-I-005 list-all behavior.
|
||||
type ListNotificationsInput struct {
|
||||
ListParams
|
||||
Status string `json:"status,omitempty" jsonschema:"Filter by status: pending, sent, failed, dead, read"`
|
||||
}
|
||||
|
||||
// ── Policies ────────────────────────────────────────────────────────
|
||||
|
||||
type CreatePolicyInput struct {
|
||||
|
||||
@@ -93,9 +93,34 @@ type TargetRepository interface {
|
||||
|
||||
// AgentRepository defines operations for managing control plane agents.
|
||||
type AgentRepository interface {
|
||||
// List returns all agents.
|
||||
// List returns all ACTIVE agents — rows with retired_at IS NULL.
|
||||
//
|
||||
// I-004: The default listing MUST NOT surface retired agents. The
|
||||
// handler-facing ListAgents call, the stats dashboard, and the stale-offline
|
||||
// sweeper all iterate this list and would otherwise re-surface decommissioned
|
||||
// hardware in operational UI. Callers that genuinely want retired rows (the
|
||||
// audit tab, compliance exports) must use ListRetired instead.
|
||||
//
|
||||
// The partial index idx_agents_retired_at (migration 000015) keeps retired
|
||||
// rows cheap to exclude — the planner uses it to skip the retired segment
|
||||
// of the table entirely.
|
||||
List(ctx context.Context) ([]*domain.Agent, error)
|
||||
// ListRetired returns a paginated list of retired agents (retired_at IS NOT NULL),
|
||||
// ordered by retired_at DESC so the most recent retirements appear first. Used
|
||||
// by the GUI's Retired tab and the audit export path. Returns the slice plus
|
||||
// the total count (for pagination). A page<1 or perPage<1 is clamped to sensible
|
||||
// defaults (page=1, perPage=50) in the repo implementation rather than erroring —
|
||||
// this matches the ListAgents pagination behavior in the service layer.
|
||||
// I-004 coverage-gap closure, migration 000015.
|
||||
ListRetired(ctx context.Context, page, perPage int) ([]*domain.Agent, int, error)
|
||||
// Get retrieves an agent by ID.
|
||||
//
|
||||
// I-004 note: Get returns retired rows (retired_at IS NOT NULL) because
|
||||
// callers that need to check "has this agent been retired?" — the heartbeat
|
||||
// handler returning 410 Gone, the retirement service's idempotent-retire
|
||||
// branch, the detail page rendering a retirement banner — must see the
|
||||
// retired_at/retired_reason fields. Only the default List path default-
|
||||
// excludes retired; individual Get lookups surface them.
|
||||
Get(ctx context.Context, id string) (*domain.Agent, error)
|
||||
// Create stores a new agent. Callers that want duplicate-key errors surfaced
|
||||
// (e.g. real-agent registration) must use this method; sentinel/bootstrap
|
||||
@@ -112,11 +137,78 @@ type AgentRepository interface {
|
||||
// Update modifies an existing agent.
|
||||
Update(ctx context.Context, agent *domain.Agent) error
|
||||
// Delete removes an agent.
|
||||
//
|
||||
// I-004: callers should prefer SoftRetire / RetireAgentWithCascade for the
|
||||
// operator-facing retirement path; hard Delete remains available for test
|
||||
// cleanup and repository-level administrative tasks. The deployment_targets
|
||||
// FK flipped to ON DELETE RESTRICT in migration 000015, so hard-deleting an
|
||||
// agent that still owns active targets will now fail at the DB layer — which
|
||||
// is intentional: the fail-closed guardrail prevents audit-trail destruction.
|
||||
Delete(ctx context.Context, id string) error
|
||||
// UpdateHeartbeat updates the agent's last heartbeat timestamp and metadata.
|
||||
//
|
||||
// I-004: UpdateHeartbeat is a no-op on retired agents — the UPDATE clause
|
||||
// includes AND retired_at IS NULL so a stale agent process that keeps polling
|
||||
// after retirement cannot resurrect its heartbeat. The service layer already
|
||||
// short-circuits with ErrAgentRetired before calling this method; the WHERE
|
||||
// filter here is belt-and-braces for anyone who skips the service path.
|
||||
UpdateHeartbeat(ctx context.Context, id string, metadata *domain.AgentMetadata) error
|
||||
// GetByAPIKey retrieves an agent by hashed API key.
|
||||
//
|
||||
// I-004: GetByAPIKey returns retired rows so the auth middleware can detect
|
||||
// "this API key belongs to a retired agent" and fail the request with
|
||||
// 410 Gone. If retired rows were hidden, auth would return a plain 401 and
|
||||
// leak no signal — which is wrong: the operator needs the retired state
|
||||
// made explicit so they can clean up the agent process.
|
||||
GetByAPIKey(ctx context.Context, keyHash string) (*domain.Agent, error)
|
||||
// SoftRetire stamps retired_at + retired_reason on the agent row with no
|
||||
// cascade. Used on the happy path where preflight confirmed the agent has
|
||||
// zero active dependencies (no active deployment_targets, no pending jobs).
|
||||
// The UPDATE is scoped to WHERE id=$1 AND retired_at IS NULL so re-retiring
|
||||
// an already-retired row is a no-op (zero rows affected is NOT returned as
|
||||
// an error — the service layer detects this via its own idempotent-retire
|
||||
// branch before calling SoftRetire). Callers supply retiredAt so the service
|
||||
// can pin a single consistent timestamp across audit + DB writes.
|
||||
// I-004 coverage-gap closure.
|
||||
SoftRetire(ctx context.Context, id string, retiredAt time.Time, reason string) error
|
||||
// RetireAgentWithCascade performs a transactional retire + cascade. In one
|
||||
// transaction it: (1) stamps retired_at + retired_reason on the agent row,
|
||||
// and (2) stamps the SAME retired_at + retired_reason on every active
|
||||
// deployment_targets row whose agent_id matches. Only rows with
|
||||
// retired_at IS NULL are touched in (2) — already-retired targets keep their
|
||||
// original retirement metadata (whoever retired them first, whenever). Used
|
||||
// exclusively on the force=true path from the retirement handler; callers
|
||||
// supply retiredAt so the agent row and every cascaded target row share an
|
||||
// exact retirement instant (helps forensic analysis trace the cascade back
|
||||
// to a single operator action). If the agent row is already retired, the
|
||||
// whole operation is a no-op — the transaction commits without touching
|
||||
// either table. I-004 coverage-gap closure, migration 000015.
|
||||
RetireAgentWithCascade(ctx context.Context, id string, retiredAt time.Time, reason string) error
|
||||
// CountActiveTargets returns the number of deployment_targets rows where
|
||||
// agent_id=id AND retired_at IS NULL. The COUNT query hits the existing
|
||||
// idx_deployment_targets_agent_id index (migration 000001 line 111); the
|
||||
// additional retired_at IS NULL predicate is cheap because the partial
|
||||
// idx_deployment_targets_retired_at index (migration 000015) lets the
|
||||
// planner skip the retired-row segment entirely. Preflight uses this to
|
||||
// decide 200 (soft-retire) vs 409 (blocked-by-deps). I-004.
|
||||
CountActiveTargets(ctx context.Context, agentID string) (int, error)
|
||||
// CountActiveCertificates returns the count of managed_certificates currently
|
||||
// deployed through one of this agent's ACTIVE (non-retired) deployment_targets.
|
||||
// The query joins certificate_target_mappings (migration 000001 line 116) →
|
||||
// deployment_targets filtering on deployment_targets.agent_id=$1 AND
|
||||
// deployment_targets.retired_at IS NULL, then COUNT(DISTINCT certificate_id)
|
||||
// so the same cert deployed to multiple targets on one agent counts once.
|
||||
// The primary key (certificate_id, target_id) on certificate_target_mappings
|
||||
// plus idx_certificate_target_mappings_target_id (line 122) cover the join.
|
||||
// Used purely for the preflight 409 body — the number is informational. I-004.
|
||||
CountActiveCertificates(ctx context.Context, agentID string) (int, error)
|
||||
// CountPendingJobs returns the number of jobs belonging to this agent whose
|
||||
// status is in (Pending, AwaitingCSR, AwaitingApproval, Running) — the four
|
||||
// statuses that indicate work the agent would still be expected to pick up.
|
||||
// Completed/Failed/Cancelled jobs do not count. The filter agent_id=$1 hits
|
||||
// the idx_jobs_agent_id index (migration 000001 line 161). Used for the
|
||||
// preflight 409 body. I-004.
|
||||
CountPendingJobs(ctx context.Context, agentID string) (int, error)
|
||||
}
|
||||
|
||||
// JobRepository defines operations for managing renewal and deployment jobs.
|
||||
@@ -151,6 +243,11 @@ type JobRepository interface {
|
||||
// to Running) and locks AwaitingCSR jobs against concurrent observers (leaving state intact,
|
||||
// since the CSR-submission path drives the next transition). H-6 (CWE-362) race remediation.
|
||||
ClaimPendingByAgentID(ctx context.Context, agentID string) ([]*domain.Job, error)
|
||||
// ListTimedOutAwaitingJobs returns jobs stuck in AwaitingCSR (created before csrCutoff) or
|
||||
// AwaitingApproval (created before approvalCutoff). The reaper loop transitions them to
|
||||
// Failed; I-001's retry loop then auto-promotes eligible Failed jobs back to Pending.
|
||||
// I-003 coverage-gap closure.
|
||||
ListTimedOutAwaitingJobs(ctx context.Context, csrCutoff, approvalCutoff time.Time) ([]*domain.Job, error)
|
||||
}
|
||||
|
||||
// RenewalPolicyRepository defines operations for managing renewal policies.
|
||||
@@ -188,6 +285,12 @@ type AuditRepository interface {
|
||||
}
|
||||
|
||||
// NotificationRepository defines operations for managing notifications.
|
||||
//
|
||||
// I-005 extends the interface with four retry/DLQ methods. The retry scheduler
|
||||
// loop calls ListRetryEligible on every tick to pull overdue failed rows, then
|
||||
// either RecordFailedAttempt (still-retrying) or MarkAsDead (exhausted). The
|
||||
// operator-facing dead-letter tab calls Requeue to move a row from 'dead' (or
|
||||
// 'failed') back to 'pending' so ProcessPendingNotifications picks it up again.
|
||||
type NotificationRepository interface {
|
||||
// Create stores a new notification.
|
||||
Create(ctx context.Context, notif *domain.NotificationEvent) error
|
||||
@@ -195,6 +298,44 @@ type NotificationRepository interface {
|
||||
List(ctx context.Context, filter *NotificationFilter) ([]*domain.NotificationEvent, error)
|
||||
// UpdateStatus updates a notification's delivery status.
|
||||
UpdateStatus(ctx context.Context, id string, status string, sentAt time.Time) error
|
||||
// ListRetryEligible returns failed notification rows whose next_retry_at
|
||||
// is <= now AND retry_count < maxAttempts, ordered by next_retry_at ASC
|
||||
// (oldest overdue first — same fairness as I-001's RetryFailedJobs). The
|
||||
// WHERE clause mirrors the partial retry-sweep index predicate from
|
||||
// migration 000016 so the planner uses it. A limit<=0 is normalised to
|
||||
// a sane default in the repo implementation to avoid accidental unbounded
|
||||
// sweeps. I-005 coverage-gap closure.
|
||||
ListRetryEligible(ctx context.Context, now time.Time, maxAttempts, limit int) ([]*domain.NotificationEvent, error)
|
||||
// RecordFailedAttempt is called by the retry sweep after a notifier.Send
|
||||
// transient failure. The UPDATE increments retry_count by exactly 1,
|
||||
// overwrites last_error, overwrites next_retry_at, and KEEPS status='failed'
|
||||
// so the row remains a candidate for ListRetryEligible on the next sweep.
|
||||
// Returns "not found" when no row matches the id (mirrors UpdateStatus).
|
||||
// I-005 coverage-gap closure.
|
||||
RecordFailedAttempt(ctx context.Context, id string, lastError string, nextRetryAt time.Time) error
|
||||
// MarkAsDead performs the DLQ transition when retry_count reaches
|
||||
// max_attempts. Flips status='dead', clears next_retry_at so the partial
|
||||
// retry-sweep index drops the row, writes the final last_error, and
|
||||
// PRESERVES retry_count as historical evidence of how many attempts were
|
||||
// burned. Returns "not found" when no row matches.
|
||||
// I-005 coverage-gap closure.
|
||||
MarkAsDead(ctx context.Context, id string, lastError string) error
|
||||
// Requeue is the operator "try again" action from the UI's Dead letter
|
||||
// tab. Flips status='pending' (so ProcessPendingNotifications picks it
|
||||
// up), resets retry_count to 0 (otherwise the operator's first retry
|
||||
// would already be at hour-long waits), clears next_retry_at, and clears
|
||||
// last_error. Valid from both 'dead' and 'failed'. Returns "not found"
|
||||
// when no row matches. I-005 coverage-gap closure.
|
||||
Requeue(ctx context.Context, id string) error
|
||||
// CountByStatus returns the number of notification_events rows whose
|
||||
// status column matches the given string exactly. Used by StatsService
|
||||
// to populate DashboardSummary.NotificationsDead which in turn drives
|
||||
// the Prometheus counter certctl_notification_dead_total (I-005 Phase 2
|
||||
// observability gate). A dedicated SQL COUNT(*) is used instead of
|
||||
// List(filter{Status: ...}) because List silently resets PerPage>500 to
|
||||
// 50 — a latent scale bug for any status-filtered count. I-005
|
||||
// coverage-gap closure.
|
||||
CountByStatus(ctx context.Context, status string) (int64, error)
|
||||
}
|
||||
|
||||
// TeamRepository defines operations for managing teams.
|
||||
|
||||
@@ -20,12 +20,18 @@ func NewAgentRepository(db *sql.DB) *AgentRepository {
|
||||
return &AgentRepository{db: db}
|
||||
}
|
||||
|
||||
// List returns all agents
|
||||
// List returns all ACTIVE agents — rows with retired_at IS NULL. I-004:
|
||||
// the default listing path feeds the handler-facing ListAgents call, the
|
||||
// stats dashboard, and the stale-offline sweeper; every caller wants active
|
||||
// hardware, not decommissioned rows. Operators who need retired rows reach
|
||||
// for ListRetired instead. The partial index idx_agents_retired_at
|
||||
// (migration 000015) lets the planner skip the retired segment cheaply.
|
||||
func (r *AgentRepository) List(ctx context.Context) ([]*domain.Agent, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, name, hostname, status, last_heartbeat_at, registered_at, api_key_hash,
|
||||
os, architecture, ip_address, version
|
||||
os, architecture, ip_address, version, retired_at, retired_reason
|
||||
FROM agents
|
||||
WHERE retired_at IS NULL
|
||||
ORDER BY registered_at DESC
|
||||
`)
|
||||
|
||||
@@ -50,11 +56,16 @@ func (r *AgentRepository) List(ctx context.Context) ([]*domain.Agent, error) {
|
||||
return agents, nil
|
||||
}
|
||||
|
||||
// Get retrieves an agent by ID
|
||||
// Get retrieves an agent by ID. I-004: retired rows ARE surfaced here —
|
||||
// callers that need to check "has this agent been retired?" (heartbeat
|
||||
// handler returning 410 Gone, retirement service's idempotent-retire branch,
|
||||
// detail page rendering a retirement banner) must see retired_at /
|
||||
// retired_reason. Only the List path default-excludes retired rows; Get is
|
||||
// by-ID and returns whatever row exists.
|
||||
func (r *AgentRepository) Get(ctx context.Context, id string) (*domain.Agent, error) {
|
||||
row := r.db.QueryRowContext(ctx, `
|
||||
SELECT id, name, hostname, status, last_heartbeat_at, registered_at, api_key_hash,
|
||||
os, architecture, ip_address, version
|
||||
os, architecture, ip_address, version, retired_at, retired_reason
|
||||
FROM agents
|
||||
WHERE id = $1
|
||||
`, id)
|
||||
@@ -185,7 +196,16 @@ func (r *AgentRepository) Delete(ctx context.Context, id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateHeartbeat updates the agent's last heartbeat timestamp and metadata
|
||||
// UpdateHeartbeat updates the agent's last heartbeat timestamp and metadata.
|
||||
//
|
||||
// I-004: both branches include `AND retired_at IS NULL` in the WHERE clause,
|
||||
// making the UPDATE a no-op on retired rows. The service layer already
|
||||
// short-circuits with ErrAgentRetired before calling this method (see
|
||||
// AgentService.Heartbeat), but the WHERE filter is belt-and-braces for any
|
||||
// path that skips the service — a stale agent process that keeps polling
|
||||
// after retirement cannot resurrect its heartbeat at the DB layer. A zero
|
||||
// RowsAffected here returns the same "agent not found" error as before; the
|
||||
// service layer distinguishes retired from missing by calling Get first.
|
||||
func (r *AgentRepository) UpdateHeartbeat(ctx context.Context, id string, metadata *domain.AgentMetadata) error {
|
||||
var result sql.Result
|
||||
var err error
|
||||
@@ -199,11 +219,11 @@ func (r *AgentRepository) UpdateHeartbeat(ctx context.Context, id string, metada
|
||||
architecture = CASE WHEN $5 = '' THEN architecture ELSE $5 END,
|
||||
ip_address = CASE WHEN $6 = '' THEN ip_address ELSE $6 END,
|
||||
version = CASE WHEN $7 = '' THEN version ELSE $7 END
|
||||
WHERE id = $2
|
||||
WHERE id = $2 AND retired_at IS NULL
|
||||
`, time.Now(), id, metadata.Hostname, metadata.OS, metadata.Architecture, metadata.IPAddress, metadata.Version)
|
||||
} else {
|
||||
result, err = r.db.ExecContext(ctx, `
|
||||
UPDATE agents SET last_heartbeat_at = $1 WHERE id = $2
|
||||
UPDATE agents SET last_heartbeat_at = $1 WHERE id = $2 AND retired_at IS NULL
|
||||
`, time.Now(), id)
|
||||
}
|
||||
|
||||
@@ -223,11 +243,15 @@ func (r *AgentRepository) UpdateHeartbeat(ctx context.Context, id string, metada
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetByAPIKey retrieves an agent by hashed API key
|
||||
// GetByAPIKey retrieves an agent by hashed API key. I-004: retired rows ARE
|
||||
// surfaced here so the auth middleware can detect "this API key belongs to a
|
||||
// retired agent" and fail the request with 410 Gone instead of 401. If the
|
||||
// filter hid retired rows, auth would return a plain 401 and leak no signal
|
||||
// that the agent process needs cleaning up.
|
||||
func (r *AgentRepository) GetByAPIKey(ctx context.Context, keyHash string) (*domain.Agent, error) {
|
||||
row := r.db.QueryRowContext(ctx, `
|
||||
SELECT id, name, hostname, status, last_heartbeat_at, registered_at, api_key_hash,
|
||||
os, architecture, ip_address, version
|
||||
os, architecture, ip_address, version, retired_at, retired_reason
|
||||
FROM agents
|
||||
WHERE api_key_hash = $1
|
||||
`, keyHash)
|
||||
@@ -243,14 +267,214 @@ func (r *AgentRepository) GetByAPIKey(ctx context.Context, keyHash string) (*dom
|
||||
return agent, nil
|
||||
}
|
||||
|
||||
// scanAgent scans an agent from a row or rows
|
||||
// ─── I-004 agent retirement surface ──────────────────────────────────────
|
||||
//
|
||||
// The methods below implement the I-004 coverage-gap closure. They follow the
|
||||
// interface contracts in internal/repository/interfaces.go:94-210 (which is the
|
||||
// spec — keep godoc there in sync if behavior changes).
|
||||
|
||||
// ListRetired returns a paginated slice of retired agents ordered by
|
||||
// retired_at DESC so the most recent retirements appear first. Used by the
|
||||
// GUI's Retired tab and the audit export path. Returns the rows plus the
|
||||
// total count (for pagination UI). page<1 or perPage<1 is clamped to
|
||||
// sensible defaults in-repo rather than erroring, matching the ListAgents
|
||||
// pagination behavior at the service layer. I-004, migration 000015.
|
||||
func (r *AgentRepository) ListRetired(ctx context.Context, page, perPage int) ([]*domain.Agent, int, error) {
|
||||
// Clamp pagination to safe defaults. Keep in lockstep with the service
|
||||
// layer's pagination shape — negative / zero values on either axis should
|
||||
// degrade to "first page, default size" instead of returning an error.
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = 50
|
||||
}
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
// Total count first — separate query so pagination math stays correct
|
||||
// even when the page of rows is empty. Uses the partial
|
||||
// idx_agents_retired_at index so this is effectively a count of the
|
||||
// partial-index tuple count, not a full table scan.
|
||||
var total int
|
||||
if err := r.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM agents WHERE retired_at IS NOT NULL
|
||||
`).Scan(&total); err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to count retired agents: %w", err)
|
||||
}
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, name, hostname, status, last_heartbeat_at, registered_at, api_key_hash,
|
||||
os, architecture, ip_address, version, retired_at, retired_reason
|
||||
FROM agents
|
||||
WHERE retired_at IS NOT NULL
|
||||
ORDER BY retired_at DESC
|
||||
LIMIT $1 OFFSET $2
|
||||
`, perPage, offset)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to query retired agents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var agents []*domain.Agent
|
||||
for rows.Next() {
|
||||
agent, err := scanAgent(rows)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
agents = append(agents, agent)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, 0, fmt.Errorf("error iterating retired agent rows: %w", err)
|
||||
}
|
||||
return agents, total, nil
|
||||
}
|
||||
|
||||
// SoftRetire stamps retired_at + retired_reason on the agent row with no
|
||||
// cascade. Scoped to `WHERE id=$1 AND retired_at IS NULL` so re-retiring an
|
||||
// already-retired row is a silent no-op (zero RowsAffected). The service
|
||||
// layer has its own idempotent-retire branch that detects already-retired
|
||||
// rows via Get before calling SoftRetire; a zero here just means a racy
|
||||
// caller got there first. I-004.
|
||||
func (r *AgentRepository) SoftRetire(ctx context.Context, id string, retiredAt time.Time, reason string) error {
|
||||
if _, err := r.db.ExecContext(ctx, `
|
||||
UPDATE agents
|
||||
SET retired_at = $2, retired_reason = $3
|
||||
WHERE id = $1 AND retired_at IS NULL
|
||||
`, id, retiredAt, reason); err != nil {
|
||||
return fmt.Errorf("failed to soft-retire agent: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RetireAgentWithCascade performs a transactional retire-and-cascade. In one
|
||||
// transaction it (1) stamps retired_at + retired_reason on the agent row if
|
||||
// it is still active, and (2) stamps the SAME retired_at + retired_reason on
|
||||
// every active (retired_at IS NULL) deployment_targets row whose agent_id
|
||||
// matches. Already-retired targets keep their original retirement metadata;
|
||||
// only active targets are touched. If the agent is already retired, the
|
||||
// whole transaction is a no-op — the caller's idempotent-retire branch
|
||||
// already handled it before we got here. I-004, migration 000015.
|
||||
//
|
||||
// The two UPDATEs share a single (retiredAt, reason) pair so forensic
|
||||
// analysis can trace "every row stamped at T1 with reason R was part of the
|
||||
// same operator action" back to one cascade. Using BeginTx keeps the agent
|
||||
// row and its targets' retirement metadata consistent even if something
|
||||
// crashes mid-cascade.
|
||||
func (r *AgentRepository) RetireAgentWithCascade(ctx context.Context, id string, retiredAt time.Time, reason string) error {
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin retire-cascade transaction: %w", err)
|
||||
}
|
||||
// Rollback is a no-op if Commit has already run — safe to always defer.
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
// Agent row: flip to retired only if it was still active. If zero rows
|
||||
// match, the agent was already retired — the whole cascade becomes a
|
||||
// no-op (we deliberately do NOT stamp the targets against a retirement
|
||||
// we didn't perform).
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE agents
|
||||
SET retired_at = $2, retired_reason = $3
|
||||
WHERE id = $1 AND retired_at IS NULL
|
||||
`, id, retiredAt, reason); err != nil {
|
||||
return fmt.Errorf("failed to retire agent in cascade: %w", err)
|
||||
}
|
||||
|
||||
// Cascade: copy the same retired_at / retired_reason onto every active
|
||||
// deployment_target belonging to this agent. Skips targets that are
|
||||
// already retired so their original retirement metadata is preserved.
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE deployment_targets
|
||||
SET retired_at = $2, retired_reason = $3
|
||||
WHERE agent_id = $1 AND retired_at IS NULL
|
||||
`, id, retiredAt, reason); err != nil {
|
||||
return fmt.Errorf("failed to cascade-retire deployment targets: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("failed to commit retire-cascade transaction: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountActiveTargets returns the number of deployment_targets with
|
||||
// agent_id=agentID AND retired_at IS NULL. Used by the retirement preflight
|
||||
// to decide 200 (soft-retire) vs 409 (blocked-by-deps). Hits the existing
|
||||
// idx_deployment_targets_agent_id index (migration 000001 line 111); the
|
||||
// retired_at IS NULL predicate is cheap because the partial
|
||||
// idx_deployment_targets_retired_at index (migration 000015) lets the
|
||||
// planner skip the retired-row segment. I-004.
|
||||
func (r *AgentRepository) CountActiveTargets(ctx context.Context, agentID string) (int, error) {
|
||||
var count int
|
||||
err := r.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM deployment_targets
|
||||
WHERE agent_id = $1 AND retired_at IS NULL
|
||||
`, agentID).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count active targets for agent: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// CountActiveCertificates returns the count of distinct managed_certificates
|
||||
// currently deployed through one of this agent's ACTIVE deployment_targets.
|
||||
// Joins certificate_target_mappings (migration 000001 line 116) →
|
||||
// deployment_targets filtering on deployment_targets.agent_id=$1 AND
|
||||
// deployment_targets.retired_at IS NULL. COUNT(DISTINCT certificate_id) so
|
||||
// the same cert deployed to multiple targets on one agent counts once.
|
||||
// Used purely for the preflight 409 body. I-004.
|
||||
func (r *AgentRepository) CountActiveCertificates(ctx context.Context, agentID string) (int, error) {
|
||||
var count int
|
||||
err := r.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(DISTINCT ctm.certificate_id)
|
||||
FROM certificate_target_mappings ctm
|
||||
JOIN deployment_targets dt ON dt.id = ctm.target_id
|
||||
WHERE dt.agent_id = $1 AND dt.retired_at IS NULL
|
||||
`, agentID).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count active certificates for agent: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// CountPendingJobs returns the number of jobs belonging to this agent whose
|
||||
// status is in (Pending, AwaitingCSR, AwaitingApproval, Running) — the four
|
||||
// statuses that represent work the agent would still be expected to pick up
|
||||
// or complete. Completed / Failed / Cancelled jobs do not count toward the
|
||||
// preflight gate. Status strings match domain.JobStatus* constants in
|
||||
// internal/domain/job.go:43-49. Hits idx_jobs_agent_id (migration 000001
|
||||
// line 161). I-004.
|
||||
func (r *AgentRepository) CountPendingJobs(ctx context.Context, agentID string) (int, error) {
|
||||
var count int
|
||||
err := r.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM jobs
|
||||
WHERE agent_id = $1
|
||||
AND status IN ('Pending', 'AwaitingCSR', 'AwaitingApproval', 'Running')
|
||||
`, agentID).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count pending jobs for agent: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// scanAgent scans an agent from a row or rows.
|
||||
//
|
||||
// I-004: the column list here is the authoritative 13-field post-M15 order —
|
||||
// retired_at and retired_reason are appended at the tail as nullable
|
||||
// *time.Time / *string scan targets matching the `json:"...,omitempty"` domain
|
||||
// fields. Every SELECT in this file that feeds scanAgent must emit columns in
|
||||
// this same order, otherwise Scan will silently place values into the wrong
|
||||
// fields (lib/pq does positional binding, not named).
|
||||
func scanAgent(scanner interface {
|
||||
Scan(...interface{}) error
|
||||
}) (*domain.Agent, error) {
|
||||
var agent domain.Agent
|
||||
err := scanner.Scan(&agent.ID, &agent.Name, &agent.Hostname, &agent.Status,
|
||||
&agent.LastHeartbeatAt, &agent.RegisteredAt, &agent.APIKeyHash,
|
||||
&agent.OS, &agent.Architecture, &agent.IPAddress, &agent.Version)
|
||||
&agent.OS, &agent.Architecture, &agent.IPAddress, &agent.Version,
|
||||
&agent.RetiredAt, &agent.RetiredReason)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan agent: %w", err)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
@@ -570,6 +571,41 @@ func (r *JobRepository) ClaimPendingByAgentID(ctx context.Context, agentID strin
|
||||
return append(pendingJobs, csrJobs...), nil
|
||||
}
|
||||
|
||||
// ListTimedOutAwaitingJobs returns jobs stuck in AwaitingCSR or AwaitingApproval past
|
||||
// their respective cutoff timestamps (created_at < cutoff). The reaper loop transitions
|
||||
// them to Failed; I-001's retry loop then auto-promotes eligible Failed jobs back to
|
||||
// Pending. I-003 coverage-gap closure.
|
||||
func (r *JobRepository) ListTimedOutAwaitingJobs(ctx context.Context, csrCutoff, approvalCutoff time.Time) ([]*domain.Job, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, type, certificate_id, target_id, agent_id, status, attempts, max_attempts,
|
||||
last_error, scheduled_at, started_at, completed_at, created_at
|
||||
FROM jobs
|
||||
WHERE (status = $1 AND created_at < $2)
|
||||
OR (status = $3 AND created_at < $4)
|
||||
ORDER BY created_at ASC
|
||||
`, domain.JobStatusAwaitingCSR, csrCutoff, domain.JobStatusAwaitingApproval, approvalCutoff)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query timed-out awaiting jobs: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var jobs []*domain.Job
|
||||
for rows.Next() {
|
||||
job, err := scanJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error iterating timed-out job rows: %w", err)
|
||||
}
|
||||
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// scanJob scans a job from a row or rows
|
||||
func scanJob(scanner interface {
|
||||
Scan(...interface{}) error
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMigration000015_AgentRetireRoundTrip is the Phase 2a Red regression test
|
||||
// for I-004 ("Agent hard-delete cascades through deployment_targets + jobs").
|
||||
//
|
||||
// The fix depends on a new migration, 000015_agent_retire.up.sql + .down.sql,
|
||||
// which must:
|
||||
//
|
||||
// 1. Add nullable `retired_at TIMESTAMPTZ` and `retired_reason TEXT`
|
||||
// columns to the `agents` table. These mirror the revoked_at /
|
||||
// revocation_reason pair on managed_certificates (migration 000005).
|
||||
//
|
||||
// 2. Add nullable `retired_at TIMESTAMPTZ` and `retired_reason TEXT` columns
|
||||
// to `deployment_targets`. When an agent is retired with cascade=true,
|
||||
// its deployment_targets must be soft-retired (not deleted) so audit
|
||||
// history — who deployed what to where, when — stays intact.
|
||||
//
|
||||
// 3. FLIP the foreign key on `deployment_targets.agent_id → agents.id`
|
||||
// from `ON DELETE CASCADE` (migration 000001, line 104) to
|
||||
// `ON DELETE RESTRICT`. This is the fail-closed change that makes a
|
||||
// bare `DELETE FROM agents WHERE id = $1` blow up at the DB layer
|
||||
// instead of silently vaporising every deployment_target row. Today
|
||||
// the CASCADE means the audit trail gets shredded with zero warning.
|
||||
//
|
||||
// The round-trip also validates that the down migration cleanly reverses all
|
||||
// three changes, so an operator who lands on a rollback can still boot the
|
||||
// server. Red-until-Green: this test compiles but fails until
|
||||
// migrations/000015_agent_retire.up.sql + .down.sql exist with the right
|
||||
// schema, because `freshSchema(t)` runs every `.up.sql` in lexical order —
|
||||
// the new migration runs automatically once Phase 2b creates the files.
|
||||
func TestMigration000015_AgentRetireRoundTrip(t *testing.T) {
|
||||
tdb := getTestDB(t)
|
||||
db := tdb.freshSchema(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// ─── Stage 1: Post-up assertions ─────────────────────────────────────
|
||||
//
|
||||
// After all .up.sql migrations (including the new 000015) have run, the
|
||||
// new columns and the flipped FK must be observable in the catalog.
|
||||
|
||||
assertColumnExists(t, db, "agents", "retired_at")
|
||||
assertColumnExists(t, db, "agents", "retired_reason")
|
||||
assertColumnExists(t, db, "deployment_targets", "retired_at")
|
||||
assertColumnExists(t, db, "deployment_targets", "retired_reason")
|
||||
|
||||
// The FK on deployment_targets.agent_id must be RESTRICT (confdeltype='r'),
|
||||
// not CASCADE (confdeltype='c'). This is the core fail-closed guarantee
|
||||
// that fixes I-004 at the storage layer.
|
||||
assertFKDeleteRule(t, db, "deployment_targets", "agent_id", "r")
|
||||
|
||||
// The FK on jobs.agent_id is already SET NULL (confdeltype='n') per
|
||||
// migration 000001 line 146 — pin that it stays that way (or goes to
|
||||
// RESTRICT; either preserves audit history, both fail on 'c').
|
||||
assertFKDeleteRuleNot(t, db, "jobs", "agent_id", "c")
|
||||
|
||||
// ─── Stage 2: Run the 000015 down migration manually ─────────────────
|
||||
//
|
||||
// testutil_test.go's runMigrations helper only runs *.up.sql. To exercise
|
||||
// the down migration I read and execute it by hand, then re-check the
|
||||
// catalog.
|
||||
|
||||
downSQL := readMigrationFile(t, "000015_agent_retire.down.sql")
|
||||
if _, err := db.ExecContext(ctx, downSQL); err != nil {
|
||||
t.Fatalf("000015 down migration failed: %v", err)
|
||||
}
|
||||
|
||||
// Stage 3: Post-down assertions — columns gone, FK restored to CASCADE.
|
||||
assertColumnGone(t, db, "agents", "retired_at")
|
||||
assertColumnGone(t, db, "agents", "retired_reason")
|
||||
assertColumnGone(t, db, "deployment_targets", "retired_at")
|
||||
assertColumnGone(t, db, "deployment_targets", "retired_reason")
|
||||
assertFKDeleteRule(t, db, "deployment_targets", "agent_id", "c")
|
||||
|
||||
// ─── Stage 4: Re-run the up migration for idempotency ────────────────
|
||||
//
|
||||
// The up migration must be safely re-runnable — operators sometimes
|
||||
// re-apply by hand after a partial rollback. Use IF NOT EXISTS / ALTER
|
||||
// idempotently.
|
||||
|
||||
upSQL := readMigrationFile(t, "000015_agent_retire.up.sql")
|
||||
if _, err := db.ExecContext(ctx, upSQL); err != nil {
|
||||
t.Fatalf("000015 up migration re-apply failed (must be idempotent): %v", err)
|
||||
}
|
||||
|
||||
assertColumnExists(t, db, "agents", "retired_at")
|
||||
assertColumnExists(t, db, "agents", "retired_reason")
|
||||
assertColumnExists(t, db, "deployment_targets", "retired_at")
|
||||
assertColumnExists(t, db, "deployment_targets", "retired_reason")
|
||||
assertFKDeleteRule(t, db, "deployment_targets", "agent_id", "r")
|
||||
}
|
||||
|
||||
// ─── Catalog helpers ──────────────────────────────────────────────────────
|
||||
//
|
||||
// These helpers scope every catalog query to the schema the test is actually
|
||||
// running in by joining against current_schema(). Without that, a test
|
||||
// running in schema test_xyz would accidentally inspect the public schema
|
||||
// and green-light drift.
|
||||
|
||||
func assertColumnExists(t *testing.T, db *sql.DB, table, column string) {
|
||||
t.Helper()
|
||||
var exists bool
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = $1
|
||||
AND column_name = $2
|
||||
)`, table, column).Scan(&exists)
|
||||
if err != nil {
|
||||
t.Fatalf("column existence query failed for %s.%s: %v", table, column, err)
|
||||
}
|
||||
if !exists {
|
||||
t.Errorf("expected column %s.%s to exist after 000015 up (migration missing or drifted)", table, column)
|
||||
}
|
||||
}
|
||||
|
||||
func assertColumnGone(t *testing.T, db *sql.DB, table, column string) {
|
||||
t.Helper()
|
||||
var exists bool
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = $1
|
||||
AND column_name = $2
|
||||
)`, table, column).Scan(&exists)
|
||||
if err != nil {
|
||||
t.Fatalf("column existence query failed for %s.%s: %v", table, column, err)
|
||||
}
|
||||
if exists {
|
||||
t.Errorf("expected column %s.%s to be removed after 000015 down (down migration is incomplete)", table, column)
|
||||
}
|
||||
}
|
||||
|
||||
// assertFKDeleteRule asserts that the foreign key covering `table.column`
|
||||
// (i.e. the FK whose constrained column matches) has the expected
|
||||
// `confdeltype`. Per pg_constraint docs: 'r' = RESTRICT, 'c' = CASCADE,
|
||||
// 'n' = SET NULL, 'd' = SET DEFAULT, 'a' = NO ACTION.
|
||||
func assertFKDeleteRule(t *testing.T, db *sql.DB, table, column, want string) {
|
||||
t.Helper()
|
||||
got := lookupFKDeleteRule(t, db, table, column)
|
||||
if got != want {
|
||||
t.Errorf("FK on %s(%s): confdeltype=%q want %q (RESTRICT='r', CASCADE='c', SET NULL='n')",
|
||||
table, column, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// assertFKDeleteRuleNot is the negative form — used for jobs.agent_id where
|
||||
// multiple confdeltype values are acceptable (SET NULL and RESTRICT both
|
||||
// preserve audit history) but CASCADE is strictly forbidden.
|
||||
func assertFKDeleteRuleNot(t *testing.T, db *sql.DB, table, column, disallowed string) {
|
||||
t.Helper()
|
||||
got := lookupFKDeleteRule(t, db, table, column)
|
||||
if got == disallowed {
|
||||
t.Errorf("FK on %s(%s): confdeltype=%q; %q is forbidden (would destroy audit history on agent delete)",
|
||||
table, column, got, disallowed)
|
||||
}
|
||||
}
|
||||
|
||||
// lookupFKDeleteRule returns the confdeltype for the FK constraint whose
|
||||
// constrained table+column matches. Returns empty string if no FK found —
|
||||
// that's treated as a test failure because the schema is supposed to have
|
||||
// these FKs per migration 000001.
|
||||
func lookupFKDeleteRule(t *testing.T, db *sql.DB, table, column string) string {
|
||||
t.Helper()
|
||||
|
||||
// Join pg_constraint → pg_class (constrained rel) → pg_attribute
|
||||
// (constrained col) → pg_namespace (schema filter). Scoped to
|
||||
// current_schema() so schema-per-test isolation holds.
|
||||
const q = `
|
||||
SELECT c.confdeltype
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class cl ON cl.oid = c.conrelid
|
||||
JOIN pg_namespace n ON n.oid = cl.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.contype = 'f'
|
||||
AND cl.relname = $1
|
||||
AND a.attname = $2
|
||||
LIMIT 1
|
||||
`
|
||||
var confdeltype string
|
||||
err := db.QueryRowContext(context.Background(), q, table, column).Scan(&confdeltype)
|
||||
if err == sql.ErrNoRows {
|
||||
t.Fatalf("no FK found on %s(%s) in current_schema (schema not migrated?)", table, column)
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("FK lookup for %s(%s) failed: %v", table, column, err)
|
||||
return ""
|
||||
}
|
||||
return confdeltype
|
||||
}
|
||||
|
||||
// readMigrationFile locates and loads a named migration file. Uses the same
|
||||
// walk-up strategy as findMigrationsDir() in testutil_test.go so both helpers
|
||||
// agree on where the migrations live.
|
||||
func readMigrationFile(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(findMigrationsDir(), name)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read migration file %s (expected at %s): %v", name, path, err)
|
||||
}
|
||||
// Defensive: a zero-byte down migration would produce false-positive
|
||||
// "success" below. Refuse to trust it.
|
||||
if strings.TrimSpace(string(data)) == "" {
|
||||
t.Fatalf("migration file %s is empty — down migration missing or truncated", name)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMigration000016_NotificationRetryRoundTrip is the Phase 1 Red regression
|
||||
// test for I-005 ("failed webhook/email drops critical alerts — no retry, no
|
||||
// DLQ, no escalation"). The fix depends on a new migration,
|
||||
// 000016_notification_retry.up.sql + .down.sql, which must:
|
||||
//
|
||||
// 1. Add `retry_count INTEGER NOT NULL DEFAULT 0` on notification_events.
|
||||
// Mirrors migration 000015's column-nullability pattern: explicit
|
||||
// NOT NULL + default so existing rows backfill cleanly and the service
|
||||
// layer never has to nil-check the counter. The 0 default is what lets
|
||||
// the retry scheduler promote a row from failed → pending on its very
|
||||
// first sweep without a bespoke backfill.
|
||||
//
|
||||
// 2. Add `next_retry_at TIMESTAMPTZ` (nullable) on notification_events.
|
||||
// Populated by the service layer on every failed→pending transition
|
||||
// using exponential backoff (2^retry_count minutes, cap 1h). Nullable
|
||||
// because the field is only meaningful while a row sits in 'failed'
|
||||
// state; 'sent', 'pending', 'dead', and 'read' rows leave it NULL.
|
||||
//
|
||||
// 3. Add `last_error TEXT` (nullable) on notification_events. TEXT
|
||||
// (not VARCHAR(N)) because notifier errors can include full HTTP
|
||||
// response bodies, TLS handshake diagnostics, or stringified stack
|
||||
// traces. Truncation here would kick the operator back to the server
|
||||
// log, which is exactly the triage pain I-005 is meant to eliminate.
|
||||
//
|
||||
// 4. Create the partial retry-sweep index
|
||||
// `idx_notification_events_retry_sweep ON notification_events(next_retry_at)
|
||||
// WHERE status = 'failed' AND next_retry_at IS NOT NULL`.
|
||||
// The predicate keeps the index tiny in a healthy fleet — only failed
|
||||
// rows scheduled for retry participate; sent/pending/dead/read rows and
|
||||
// unscheduled failures are excluded. Makes the retry sweep in
|
||||
// RetryFailedNotifications O(retry-eligible) rather than O(total-events).
|
||||
//
|
||||
// The round-trip also validates that the down migration cleanly reverses all
|
||||
// four schema additions, so an operator who lands on a rollback can still
|
||||
// boot the server. Stage 4 asserts idempotency — the up migration must be
|
||||
// safely re-runnable after a partial rollback, which requires ADD COLUMN
|
||||
// IF NOT EXISTS and CREATE INDEX IF NOT EXISTS on every new object.
|
||||
//
|
||||
// Red-until-Green: this test compiles but fails until
|
||||
// migrations/000016_notification_retry.up.sql + .down.sql exist with the
|
||||
// right schema, because freshSchema(t) runs every `.up.sql` in lexical order
|
||||
// — the new migration runs automatically once Phase 2 creates the files.
|
||||
func TestMigration000016_NotificationRetryRoundTrip(t *testing.T) {
|
||||
tdb := getTestDB(t)
|
||||
db := tdb.freshSchema(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// ─── Stage 1: Post-up assertions ─────────────────────────────────────
|
||||
//
|
||||
// After every .up.sql migration (including the new 000016) has run, the
|
||||
// three new columns and the partial retry-sweep index must be observable
|
||||
// in the catalog.
|
||||
|
||||
// All three retry columns must be present on notification_events.
|
||||
assertColumnExists(t, db, "notification_events", "retry_count")
|
||||
assertColumnExists(t, db, "notification_events", "next_retry_at")
|
||||
assertColumnExists(t, db, "notification_events", "last_error")
|
||||
|
||||
// retry_count must be NOT NULL with a server-side default of 0. The
|
||||
// scheduler's failed→pending transition relies on reading the counter
|
||||
// without a COALESCE, and the back-fill on existing rows must be
|
||||
// deterministic; 0 is the only safe default for an attempt counter.
|
||||
assertColumnNotNull(t, db, "notification_events", "retry_count", true)
|
||||
assertColumnDefaultContains(t, db, "notification_events", "retry_count", "0")
|
||||
|
||||
// next_retry_at and last_error are nullable by design — see the Stage 1
|
||||
// doc block above for why. A NOT NULL constraint here would force the
|
||||
// service layer to write sentinel values on every terminal-status
|
||||
// transition, which is worse than just leaving them NULL.
|
||||
assertColumnNotNull(t, db, "notification_events", "next_retry_at", false)
|
||||
assertColumnNotNull(t, db, "notification_events", "last_error", false)
|
||||
|
||||
// The partial retry-sweep index must exist on notification_events and
|
||||
// must include the WHERE predicate that restricts it to failed+scheduled
|
||||
// rows. Without the predicate the index is merely an index on
|
||||
// next_retry_at — correct semantics, but it would balloon in a busy
|
||||
// fleet because every sent/read row would sit in it with a NULL key.
|
||||
assertIndexExists(t, db, "idx_notification_events_retry_sweep")
|
||||
assertIndexPredicateContains(t, db, "idx_notification_events_retry_sweep", "status = 'failed'")
|
||||
assertIndexPredicateContains(t, db, "idx_notification_events_retry_sweep", "next_retry_at IS NOT NULL")
|
||||
|
||||
// ─── Stage 2: Run the 000016 down migration manually ─────────────────
|
||||
//
|
||||
// testutil_test.go's runMigrations helper only runs *.up.sql. To exercise
|
||||
// the down migration I read and execute it by hand, then re-check the
|
||||
// catalog.
|
||||
|
||||
downSQL := readMigrationFile(t, "000016_notification_retry.down.sql")
|
||||
if _, err := db.ExecContext(ctx, downSQL); err != nil {
|
||||
t.Fatalf("000016 down migration failed: %v", err)
|
||||
}
|
||||
|
||||
// Stage 3: Post-down assertions — all three columns removed, partial
|
||||
// index dropped.
|
||||
assertColumnGone(t, db, "notification_events", "retry_count")
|
||||
assertColumnGone(t, db, "notification_events", "next_retry_at")
|
||||
assertColumnGone(t, db, "notification_events", "last_error")
|
||||
assertIndexGone(t, db, "idx_notification_events_retry_sweep")
|
||||
|
||||
// ─── Stage 4: Re-run the up migration for idempotency ────────────────
|
||||
//
|
||||
// The up migration must be safely re-runnable — operators sometimes
|
||||
// re-apply by hand after a partial rollback. Use ADD COLUMN IF NOT
|
||||
// EXISTS and CREATE INDEX IF NOT EXISTS so every converging run is a
|
||||
// no-op.
|
||||
|
||||
upSQL := readMigrationFile(t, "000016_notification_retry.up.sql")
|
||||
if _, err := db.ExecContext(ctx, upSQL); err != nil {
|
||||
t.Fatalf("000016 up migration re-apply failed (must be idempotent): %v", err)
|
||||
}
|
||||
|
||||
assertColumnExists(t, db, "notification_events", "retry_count")
|
||||
assertColumnExists(t, db, "notification_events", "next_retry_at")
|
||||
assertColumnExists(t, db, "notification_events", "last_error")
|
||||
assertIndexExists(t, db, "idx_notification_events_retry_sweep")
|
||||
}
|
||||
|
||||
// ─── Extra catalog helpers for 000016 ─────────────────────────────────────
|
||||
//
|
||||
// These are additive to the column-existence and FK helpers defined in
|
||||
// migration_000015_test.go. Both files live in `package postgres_test`, so
|
||||
// assertColumnExists / assertColumnGone / readMigrationFile are already in
|
||||
// scope from the 000015 test file and must not be redeclared.
|
||||
|
||||
// assertColumnNotNull asserts that the information_schema reports the
|
||||
// expected nullability for a column. PG exposes `is_nullable` as the string
|
||||
// 'YES' or 'NO'; we translate to a bool so the call site reads cleanly.
|
||||
func assertColumnNotNull(t *testing.T, db *sql.DB, table, column string, wantNotNull bool) {
|
||||
t.Helper()
|
||||
var isNullable string
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT is_nullable
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = $1
|
||||
AND column_name = $2
|
||||
`, table, column).Scan(&isNullable)
|
||||
if err == sql.ErrNoRows {
|
||||
t.Fatalf("column %s.%s not found in current_schema (migration missing?)", table, column)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("is_nullable lookup for %s.%s failed: %v", table, column, err)
|
||||
}
|
||||
gotNotNull := isNullable == "NO"
|
||||
if gotNotNull != wantNotNull {
|
||||
t.Errorf("column %s.%s nullability: got NOT NULL=%v, want NOT NULL=%v (is_nullable=%q)",
|
||||
table, column, gotNotNull, wantNotNull, isNullable)
|
||||
}
|
||||
}
|
||||
|
||||
// assertColumnDefaultContains asserts that the server-side DEFAULT clause for
|
||||
// a column contains the expected substring. Postgres can render defaults in
|
||||
// a few different normalized shapes (`0`, `(0)::integer`, `0::integer`),
|
||||
// so substring matching is more robust than exact equality here.
|
||||
func assertColumnDefaultContains(t *testing.T, db *sql.DB, table, column, wantSubstr string) {
|
||||
t.Helper()
|
||||
var columnDefault sql.NullString
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT column_default
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = $1
|
||||
AND column_name = $2
|
||||
`, table, column).Scan(&columnDefault)
|
||||
if err == sql.ErrNoRows {
|
||||
t.Fatalf("column %s.%s not found in current_schema (migration missing?)", table, column)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("column_default lookup for %s.%s failed: %v", table, column, err)
|
||||
}
|
||||
if !columnDefault.Valid {
|
||||
t.Errorf("column %s.%s has no DEFAULT clause; want substring %q", table, column, wantSubstr)
|
||||
return
|
||||
}
|
||||
if !strings.Contains(columnDefault.String, wantSubstr) {
|
||||
t.Errorf("column %s.%s DEFAULT = %q; want substring %q",
|
||||
table, column, columnDefault.String, wantSubstr)
|
||||
}
|
||||
}
|
||||
|
||||
// assertIndexExists asserts that a named index exists in the current schema.
|
||||
// Scoped via pg_indexes.schemaname = current_schema() so schema-per-test
|
||||
// isolation holds.
|
||||
func assertIndexExists(t *testing.T, db *sql.DB, indexName string) {
|
||||
t.Helper()
|
||||
var exists bool
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE schemaname = current_schema()
|
||||
AND indexname = $1
|
||||
)`, indexName).Scan(&exists)
|
||||
if err != nil {
|
||||
t.Fatalf("index existence query failed for %s: %v", indexName, err)
|
||||
}
|
||||
if !exists {
|
||||
t.Errorf("expected index %s to exist after 000016 up (migration missing or drifted)", indexName)
|
||||
}
|
||||
}
|
||||
|
||||
// assertIndexGone is the negative form, used after the down migration to
|
||||
// confirm the partial retry-sweep index has been dropped.
|
||||
func assertIndexGone(t *testing.T, db *sql.DB, indexName string) {
|
||||
t.Helper()
|
||||
var exists bool
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM pg_indexes
|
||||
WHERE schemaname = current_schema()
|
||||
AND indexname = $1
|
||||
)`, indexName).Scan(&exists)
|
||||
if err != nil {
|
||||
t.Fatalf("index existence query failed for %s: %v", indexName, err)
|
||||
}
|
||||
if exists {
|
||||
t.Errorf("expected index %s to be removed after 000016 down (down migration is incomplete)", indexName)
|
||||
}
|
||||
}
|
||||
|
||||
// assertIndexPredicateContains asserts that the reconstructed `indexdef`
|
||||
// (pg_indexes.indexdef — the CREATE INDEX statement Postgres would emit to
|
||||
// recreate the index) contains the expected substring. This is how we pin
|
||||
// the WHERE predicate of a partial index without parsing the SQL.
|
||||
//
|
||||
// Postgres normalises the predicate (e.g. single-quoted literals stay
|
||||
// single-quoted, column references are bare), so substring matching is both
|
||||
// sufficient and robust against cosmetic reformatting.
|
||||
func assertIndexPredicateContains(t *testing.T, db *sql.DB, indexName, wantSubstr string) {
|
||||
t.Helper()
|
||||
var indexdef string
|
||||
err := db.QueryRowContext(context.Background(), `
|
||||
SELECT indexdef
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = current_schema()
|
||||
AND indexname = $1
|
||||
`, indexName).Scan(&indexdef)
|
||||
if err == sql.ErrNoRows {
|
||||
t.Fatalf("index %s not found in current_schema (migration missing?)", indexName)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("indexdef lookup for %s failed: %v", indexName, err)
|
||||
}
|
||||
if !strings.Contains(indexdef, wantSubstr) {
|
||||
t.Errorf("index %s definition missing expected predicate fragment %q\nfull indexdef: %s",
|
||||
indexName, wantSubstr, indexdef)
|
||||
}
|
||||
}
|
||||
@@ -100,10 +100,14 @@ func (r *NotificationRepository) List(ctx context.Context, filter *repository.No
|
||||
return nil, fmt.Errorf("failed to count notifications: %w", err)
|
||||
}
|
||||
|
||||
// Get paginated results
|
||||
// Get paginated results. I-005 extends the SELECT with the three retry
|
||||
// columns (retry_count / next_retry_at / last_error) so scanNotification
|
||||
// can populate the new fields on domain.NotificationEvent. The column
|
||||
// order here MUST stay in lockstep with scanNotification below.
|
||||
offset := (filter.Page - 1) * filter.PerPage
|
||||
query := fmt.Sprintf(`
|
||||
SELECT id, type, certificate_id, channel, recipient, message, sent_at, status, error
|
||||
SELECT id, type, certificate_id, channel, recipient, message, sent_at, status, error,
|
||||
retry_count, next_retry_at, last_error
|
||||
FROM notification_events
|
||||
%s
|
||||
ORDER BY sent_at DESC NULLS LAST
|
||||
@@ -156,13 +160,23 @@ func (r *NotificationRepository) UpdateStatus(ctx context.Context, id string, st
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanNotification scans a notification from a row or rows
|
||||
// scanNotification scans a notification from a row or rows.
|
||||
//
|
||||
// I-005 extends the scan list from 9 → 12 columns (adds retry_count,
|
||||
// next_retry_at, last_error). Every caller — List and the four new retry
|
||||
// methods below — funnels rows through this helper, so the SELECT column
|
||||
// order in every query must match the Scan order here exactly. RetryCount
|
||||
// scans into an `int` (migration 000016 declares the column NOT NULL with
|
||||
// DEFAULT 0), while NextRetryAt and LastError scan into pointer types
|
||||
// because the column is nullable — a healthy pending/sent/dead row leaves
|
||||
// both NULL.
|
||||
func scanNotification(scanner interface {
|
||||
Scan(...interface{}) error
|
||||
}) (*domain.NotificationEvent, error) {
|
||||
var notif domain.NotificationEvent
|
||||
err := scanner.Scan(¬if.ID, ¬if.Type, ¬if.CertificateID, ¬if.Channel,
|
||||
¬if.Recipient, ¬if.Message, ¬if.SentAt, ¬if.Status, ¬if.Error)
|
||||
¬if.Recipient, ¬if.Message, ¬if.SentAt, ¬if.Status, ¬if.Error,
|
||||
¬if.RetryCount, ¬if.NextRetryAt, ¬if.LastError)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to scan notification: %w", err)
|
||||
@@ -170,3 +184,220 @@ func scanNotification(scanner interface {
|
||||
|
||||
return ¬if, nil
|
||||
}
|
||||
|
||||
// ─── I-005 retry/DLQ methods ─────────────────────────────────────────────
|
||||
//
|
||||
// The four methods below implement the repository half of the I-005
|
||||
// notification retry + dead-letter queue fix. The retry scheduler loop
|
||||
// (added alongside these in internal/scheduler/scheduler.go) drives them in
|
||||
// a strict cycle:
|
||||
//
|
||||
// ┌─► ListRetryEligible(ctx, now, maxAttempts, limit)
|
||||
// │ (oldest overdue failed rows first)
|
||||
// │ │
|
||||
// │ ├──► notifier.Send() succeeds → UpdateStatus('sent')
|
||||
// │ │
|
||||
// │ ├──► transient failure, retry_count+1 < maxAttempts
|
||||
// │ │ → RecordFailedAttempt(id, err, next)
|
||||
// │ │
|
||||
// │ └──► transient failure, retry_count+1 == maxAttempts
|
||||
// │ → MarkAsDead(id, err)
|
||||
// │
|
||||
// └──◄ Requeue(id) ────── operator "try again" from Dead-letter tab
|
||||
//
|
||||
// The WHERE clauses in every UPDATE are scoped by id (not by status), so
|
||||
// status invariants ("you can't requeue a sent row", "you can't mark a
|
||||
// dead row as dead again") live in the service layer. The repo layer is
|
||||
// deliberately thin — it mirrors the postgres CHECK constraints and
|
||||
// trusts the service to hand it rows in a sane state. The one exception
|
||||
// is "row must exist": each method returns an error on zero RowsAffected,
|
||||
// matching the pre-existing UpdateStatus contract above so the scheduler
|
||||
// can detect a concurrent delete without guessing.
|
||||
|
||||
// listRetryEligibleDefaultLimit caps a caller that passes limit <= 0.
|
||||
// Picked high enough that normal sweeps never hit it (a healthy fleet
|
||||
// should have tens of overdue rows at most, not thousands), but finite
|
||||
// so a pathological call (wrong arg in a future refactor, bad MCP tool
|
||||
// wiring) cannot scan the entire notification_events table.
|
||||
const listRetryEligibleDefaultLimit = 1000
|
||||
|
||||
// ListRetryEligible returns failed notification rows whose next_retry_at
|
||||
// is due and whose retry_count has not yet reached the configured
|
||||
// max_attempts.
|
||||
//
|
||||
// The WHERE clause is the exact dual of the partial retry-sweep index
|
||||
// predicate from migration 000016:
|
||||
//
|
||||
// WHERE status = 'failed'
|
||||
// AND next_retry_at IS NOT NULL
|
||||
// AND next_retry_at <= $1
|
||||
// AND retry_count < $2
|
||||
//
|
||||
// Because the index is partial on the first two conjuncts, the planner
|
||||
// uses it to satisfy the range scan on next_retry_at; the retry_count
|
||||
// filter is applied as a residual on the (very small) candidate set.
|
||||
//
|
||||
// ORDER BY next_retry_at ASC matches the fairness guarantee called out
|
||||
// in the test file: oldest overdue row goes first, so a backed-up
|
||||
// scheduler doesn't starve the notifications that have been waiting
|
||||
// longest. The same order is what I-001's RetryFailedJobs uses.
|
||||
func (r *NotificationRepository) ListRetryEligible(ctx context.Context, now time.Time, maxAttempts, limit int) ([]*domain.NotificationEvent, error) {
|
||||
if limit <= 0 {
|
||||
limit = listRetryEligibleDefaultLimit
|
||||
}
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, type, certificate_id, channel, recipient, message, sent_at, status, error,
|
||||
retry_count, next_retry_at, last_error
|
||||
FROM notification_events
|
||||
WHERE status = 'failed'
|
||||
AND next_retry_at IS NOT NULL
|
||||
AND next_retry_at <= $1
|
||||
AND retry_count < $2
|
||||
ORDER BY next_retry_at ASC
|
||||
LIMIT $3
|
||||
`, now, maxAttempts, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query retry-eligible notifications: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var notifs []*domain.NotificationEvent
|
||||
for rows.Next() {
|
||||
notif, err := scanNotification(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
notifs = append(notifs, notif)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error iterating retry-eligible notification rows: %w", err)
|
||||
}
|
||||
|
||||
return notifs, nil
|
||||
}
|
||||
|
||||
// RecordFailedAttempt is called by the retry sweep after a notifier.Send
|
||||
// transient failure. It increments retry_count by exactly 1, overwrites
|
||||
// last_error and next_retry_at, and deliberately DOES NOT touch status —
|
||||
// the row must remain 'failed' so the next ListRetryEligible tick can
|
||||
// pick it up again (unless the service layer has decided this attempt
|
||||
// exhausts max_attempts, in which case it calls MarkAsDead directly
|
||||
// instead of calling RecordFailedAttempt).
|
||||
//
|
||||
// The +1 is done server-side (SET retry_count = retry_count + 1) rather
|
||||
// than client-side so a race between two scheduler instances cannot lose
|
||||
// an attempt. Only one scheduler should be running in a healthy deploy,
|
||||
// but the cheap arithmetic here survives a split-brain without lying
|
||||
// about attempt counts.
|
||||
func (r *NotificationRepository) RecordFailedAttempt(ctx context.Context, id string, lastError string, nextRetryAt time.Time) error {
|
||||
result, err := r.db.ExecContext(ctx, `
|
||||
UPDATE notification_events
|
||||
SET retry_count = retry_count + 1,
|
||||
last_error = $1,
|
||||
next_retry_at = $2
|
||||
WHERE id = $3
|
||||
`, lastError, nextRetryAt, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to record notification retry attempt: %w", err)
|
||||
}
|
||||
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get rows affected: %w", err)
|
||||
}
|
||||
if rows == 0 {
|
||||
// Same "not found" error shape as UpdateStatus above. The scheduler
|
||||
// logs-and-continues on this so a concurrently-deleted row doesn't
|
||||
// break the sweep.
|
||||
return fmt.Errorf("notification not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkAsDead performs the DLQ transition. Flips status='dead' so the
|
||||
// partial retry-sweep index drops the row (the index predicate requires
|
||||
// status='failed'), clears next_retry_at so operator dashboards don't
|
||||
// claim the row is still "scheduled to retry", writes the final
|
||||
// last_error for triage, and PRESERVES retry_count as historical evidence
|
||||
// of how many attempts were burned before the row was declared dead.
|
||||
// The retry_count value is operator-visible in the Dead letter tab so
|
||||
// on-call can tell "this notification died on attempt 5" vs "this one
|
||||
// died on attempt 1 because the recipient webhook was malformed from the
|
||||
// start".
|
||||
func (r *NotificationRepository) MarkAsDead(ctx context.Context, id string, lastError string) error {
|
||||
result, err := r.db.ExecContext(ctx, `
|
||||
UPDATE notification_events
|
||||
SET status = 'dead',
|
||||
next_retry_at = NULL,
|
||||
last_error = $1
|
||||
WHERE id = $2
|
||||
`, lastError, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to mark notification as dead: %w", err)
|
||||
}
|
||||
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get rows affected: %w", err)
|
||||
}
|
||||
if rows == 0 {
|
||||
return fmt.Errorf("notification not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Requeue is the operator "try again" action fired from the Dead letter
|
||||
// tab. Flips status='pending' so ProcessPendingNotifications picks the
|
||||
// row up again, resets retry_count to 0 (otherwise the operator's first
|
||||
// retry would immediately sit at the top of the backoff ladder), clears
|
||||
// next_retry_at so the row is no longer in the retry-sweep index, and
|
||||
// clears last_error so the UI doesn't render a stale error badge next
|
||||
// to a freshly-requeued row.
|
||||
//
|
||||
// The service layer is responsible for forbidding Requeue on 'sent' or
|
||||
// 'read' rows (terminal success states). This repo layer deliberately
|
||||
// doesn't filter by current status — an operator action has already
|
||||
// passed a human-in-the-loop guard by the time it reaches the DB, and
|
||||
// the test suite only exercises the Requeue-from-{dead,failed} paths.
|
||||
// Matches how UpdateStatus doesn't filter by current status either.
|
||||
func (r *NotificationRepository) Requeue(ctx context.Context, id string) error {
|
||||
result, err := r.db.ExecContext(ctx, `
|
||||
UPDATE notification_events
|
||||
SET status = 'pending',
|
||||
retry_count = 0,
|
||||
next_retry_at = NULL,
|
||||
last_error = NULL
|
||||
WHERE id = $1
|
||||
`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to requeue notification: %w", err)
|
||||
}
|
||||
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get rows affected: %w", err)
|
||||
}
|
||||
if rows == 0 {
|
||||
return fmt.Errorf("notification not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountByStatus returns the number of notification_events rows matching the
|
||||
// given status string. Implemented as a direct COUNT(*) rather than via List
|
||||
// because List resets filter.PerPage>500 to 50 (see line 57 quirk), which
|
||||
// would produce undercounts on high-volume deployments. I-005 Phase 2 Green —
|
||||
// backs StatsService.GetDashboardSummary.NotificationsDead and the Prometheus
|
||||
// counter certctl_notification_dead_total.
|
||||
func (r *NotificationRepository) CountByStatus(ctx context.Context, status string) (int64, error) {
|
||||
var count int64
|
||||
err := r.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM notification_events WHERE status = $1`,
|
||||
status,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count notifications by status: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
"github.com/shankar0123/certctl/internal/repository/postgres"
|
||||
)
|
||||
|
||||
// TestNotificationRepository_RetryMethods is the Phase 1 Red regression test
|
||||
// for the I-005 fix ("failed webhook/email drops critical alerts — no retry,
|
||||
// no DLQ, no escalation"). It pins the four new repository methods the
|
||||
// notification-retry scheduler loop will depend on:
|
||||
//
|
||||
// 1. ListRetryEligible(ctx, now, maxAttempts, limit) — the retry-sweep query.
|
||||
// Returns failed rows whose next_retry_at <= now AND retry_count <
|
||||
// maxAttempts. Everything else (sent/pending/dead/read, unscheduled
|
||||
// failures, exhausted rows) is excluded. Ordering is ASC on next_retry_at
|
||||
// so the oldest overdue row is processed first — same fairness guarantee
|
||||
// as I-001's RetryFailedJobs.
|
||||
//
|
||||
// 2. RecordFailedAttempt(ctx, id, lastError, nextRetryAt) — what the
|
||||
// scheduler calls after a notifier.Send() transient failure. Must
|
||||
// increment retry_count by exactly 1, overwrite last_error, overwrite
|
||||
// next_retry_at, and KEEP status='failed' so the row is still a
|
||||
// candidate for ListRetryEligible on the next sweep.
|
||||
//
|
||||
// 3. MarkAsDead(ctx, id, lastError) — the DLQ transition when retry_count
|
||||
// hits max_attempts. Flips status to 'dead', clears next_retry_at
|
||||
// (so the partial retry-sweep index drops the row), preserves
|
||||
// retry_count as historical evidence of how many attempts were spent,
|
||||
// and records the final transient error for operator triage.
|
||||
//
|
||||
// 4. Requeue(ctx, id) — the operator "try again" action fired from the
|
||||
// Dead letter tab in the UI. Flips status back to 'pending' (which is
|
||||
// what ProcessPendingNotifications picks up), resets retry_count to 0,
|
||||
// clears next_retry_at AND last_error. Valid from both 'dead' (normal
|
||||
// path) and 'failed' (operator rescuing a stuck row before the sweep
|
||||
// fires). Invalid from 'sent' / 'read' (terminal success states).
|
||||
//
|
||||
// Red-until-Green: this test file compiles only after Phase 2 adds
|
||||
// ListRetryEligible, RecordFailedAttempt, MarkAsDead, and Requeue to
|
||||
// postgres.NotificationRepository. Every subtest is testcontainers-gated
|
||||
// via getTestDB(t).freshSchema(t), so `go test -short` skips them and CI
|
||||
// without Docker stays green. Fixtures are inserted via raw SQL — Create()
|
||||
// doesn't know about the new retry columns pre-Green, so the test bypasses
|
||||
// it entirely. certificate_id is left NULL on every fixture row to dodge
|
||||
// the FK to managed_certificates (the column is nullable per migration
|
||||
// 000001, line 212).
|
||||
|
||||
// TestNotificationRepository_ListRetryEligible exercises the retry-sweep
|
||||
// query. The test fixture deliberately seeds one row per excluded and
|
||||
// included case so a single call to ListRetryEligible is the oracle:
|
||||
// every row the query returns must be an "include", every row it skips
|
||||
// must be an "exclude".
|
||||
func TestNotificationRepository_ListRetryEligible(t *testing.T) {
|
||||
tdb := getTestDB(t)
|
||||
db := tdb.freshSchema(t)
|
||||
repo := postgres.NewNotificationRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
// Pin `now` so the test is deterministic. All "overdue" rows have
|
||||
// next_retry_at < now; all "future" rows have next_retry_at > now.
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
past := now.Add(-5 * time.Minute)
|
||||
future := now.Add(5 * time.Minute)
|
||||
|
||||
// Fixture grid — each row pins a specific edge of the query:
|
||||
//
|
||||
// notif-overdue-1 status=failed, retry=1, next=past → INCLUDE
|
||||
// notif-overdue-2 status=failed, retry=3, next=past → INCLUDE
|
||||
// (later next_retry_at than notif-overdue-1 by a
|
||||
// few seconds so ORDER BY is observable)
|
||||
// notif-future status=failed, retry=2, next=future → EXCLUDE
|
||||
// (CA hasn't hit backoff yet)
|
||||
// notif-exhausted status=failed, retry=5, next=past → EXCLUDE
|
||||
// (retry_count >= max_attempts — sweep must skip
|
||||
// so we don't re-promote a row that's about to
|
||||
// be marked dead)
|
||||
// notif-pending status=pending, retry=0, next=NULL → EXCLUDE
|
||||
// (healthy in-flight notification)
|
||||
// notif-sent status=sent, retry=0, next=NULL → EXCLUDE
|
||||
// notif-dead status=dead, retry=5, next=NULL → EXCLUDE
|
||||
// (already in DLQ — retrying it would reset the
|
||||
// dead-letter counter and lie to the operator)
|
||||
// notif-unsched status=failed, retry=1, next=NULL → EXCLUDE
|
||||
// (failed row that somehow lost its next_retry_at
|
||||
// — partial index predicate strips it, and the
|
||||
// WHERE clause must mirror the predicate)
|
||||
rawInsert := func(id, status string, retryCount int, nextRetryAt *time.Time) {
|
||||
t.Helper()
|
||||
_, err := db.ExecContext(ctx, `
|
||||
INSERT INTO notification_events (
|
||||
id, type, channel, recipient, message, status, retry_count, next_retry_at
|
||||
) VALUES ($1, 'ExpirationWarning', 'Webhook', 'https://hooks.example.com/x',
|
||||
'seed', $2, $3, $4)
|
||||
`, id, status, retryCount, nextRetryAt)
|
||||
if err != nil {
|
||||
t.Fatalf("raw insert for %s failed: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
overdue1 := past.Add(-30 * time.Second) // oldest overdue
|
||||
overdue2 := past // second-oldest overdue
|
||||
rawInsert("notif-overdue-1", "failed", 1, &overdue1)
|
||||
rawInsert("notif-overdue-2", "failed", 3, &overdue2)
|
||||
rawInsert("notif-future", "failed", 2, &future)
|
||||
rawInsert("notif-exhausted", "failed", 5, &overdue1)
|
||||
rawInsert("notif-pending", "pending", 0, nil)
|
||||
rawInsert("notif-sent", "sent", 0, nil)
|
||||
rawInsert("notif-dead", "dead", 5, nil)
|
||||
rawInsert("notif-unsched", "failed", 1, nil)
|
||||
|
||||
// Act — the central call under test.
|
||||
got, err := repo.ListRetryEligible(ctx, now, 5, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("ListRetryEligible failed: %v", err)
|
||||
}
|
||||
|
||||
// Assert inclusion: exactly the two overdue rows.
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("ListRetryEligible returned %d rows, want 2 (overdue-1 + overdue-2); got IDs = %v",
|
||||
len(got), collectIDs(got))
|
||||
}
|
||||
|
||||
// Assert ordering: ASC on next_retry_at. notif-overdue-1 has the
|
||||
// earlier next_retry_at (past - 30s), so it must come first.
|
||||
if got[0].ID != "notif-overdue-1" {
|
||||
t.Errorf("ListRetryEligible[0].ID = %q, want %q (ORDER BY next_retry_at ASC — oldest first)",
|
||||
got[0].ID, "notif-overdue-1")
|
||||
}
|
||||
if got[1].ID != "notif-overdue-2" {
|
||||
t.Errorf("ListRetryEligible[1].ID = %q, want %q", got[1].ID, "notif-overdue-2")
|
||||
}
|
||||
|
||||
// Assert limit is respected. Re-run with limit=1 and confirm only the
|
||||
// oldest overdue row comes back — this is what lets the scheduler
|
||||
// chunk its sweep under load.
|
||||
limited, err := repo.ListRetryEligible(ctx, now, 5, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("ListRetryEligible(limit=1) failed: %v", err)
|
||||
}
|
||||
if len(limited) != 1 || limited[0].ID != "notif-overdue-1" {
|
||||
t.Errorf("ListRetryEligible(limit=1) returned %v, want [notif-overdue-1]", collectIDs(limited))
|
||||
}
|
||||
|
||||
// Assert maxAttempts is respected. Re-run with maxAttempts=2 — this
|
||||
// flips notif-overdue-2 (retry_count=3) into the "exhausted" bucket
|
||||
// and must not come back. Only notif-overdue-1 (retry_count=1) qualifies.
|
||||
capped, err := repo.ListRetryEligible(ctx, now, 2, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("ListRetryEligible(maxAttempts=2) failed: %v", err)
|
||||
}
|
||||
if len(capped) != 1 || capped[0].ID != "notif-overdue-1" {
|
||||
t.Errorf("ListRetryEligible(maxAttempts=2) returned %v, want [notif-overdue-1]", collectIDs(capped))
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotificationRepository_RecordFailedAttempt verifies the retry-bump
|
||||
// UPDATE. The contract is: retry_count += 1, last_error = new msg,
|
||||
// next_retry_at = new time, status STAYS 'failed'. Any other side effect
|
||||
// (status flip, retry_count reset, sent_at mutation) is a bug.
|
||||
func TestNotificationRepository_RecordFailedAttempt(t *testing.T) {
|
||||
tdb := getTestDB(t)
|
||||
db := tdb.freshSchema(t)
|
||||
repo := postgres.NewNotificationRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
initialRetry := past()
|
||||
_, err := db.ExecContext(ctx, `
|
||||
INSERT INTO notification_events (
|
||||
id, type, channel, recipient, message, status, retry_count, next_retry_at, last_error
|
||||
) VALUES ('notif-attempt-1', 'ExpirationWarning', 'Webhook',
|
||||
'https://hooks.example.com/x', 'seed', 'failed', 2, $1, 'first failure')
|
||||
`, initialRetry)
|
||||
if err != nil {
|
||||
t.Fatalf("seed failed: %v", err)
|
||||
}
|
||||
|
||||
nextTry := time.Now().UTC().Add(8 * time.Minute).Truncate(time.Microsecond)
|
||||
if err := repo.RecordFailedAttempt(ctx, "notif-attempt-1", "connection refused", nextTry); err != nil {
|
||||
t.Fatalf("RecordFailedAttempt failed: %v", err)
|
||||
}
|
||||
|
||||
// Re-read the row directly from the DB (bypassing the repo's List()
|
||||
// filter logic) so the assertion tests storage, not query plumbing.
|
||||
var (
|
||||
gotStatus string
|
||||
gotRetryCount int
|
||||
gotNextRetry *time.Time
|
||||
gotLastError *string
|
||||
)
|
||||
err = db.QueryRowContext(ctx, `
|
||||
SELECT status, retry_count, next_retry_at, last_error
|
||||
FROM notification_events WHERE id = 'notif-attempt-1'
|
||||
`).Scan(&gotStatus, &gotRetryCount, &gotNextRetry, &gotLastError)
|
||||
if err != nil {
|
||||
t.Fatalf("post-update SELECT failed: %v", err)
|
||||
}
|
||||
|
||||
if gotStatus != "failed" {
|
||||
t.Errorf("status = %q, want 'failed' (RecordFailedAttempt must preserve status so sweep re-picks the row)", gotStatus)
|
||||
}
|
||||
if gotRetryCount != 3 {
|
||||
t.Errorf("retry_count = %d, want 3 (must increment by exactly 1 from seeded 2)", gotRetryCount)
|
||||
}
|
||||
if gotNextRetry == nil || !gotNextRetry.Equal(nextTry) {
|
||||
t.Errorf("next_retry_at = %v, want %v", gotNextRetry, nextTry)
|
||||
}
|
||||
if gotLastError == nil || *gotLastError != "connection refused" {
|
||||
t.Errorf("last_error = %v, want 'connection refused'", gotLastError)
|
||||
}
|
||||
|
||||
// Negative path: unknown id must surface "not found" — mirrors the
|
||||
// existing UpdateStatus contract so the scheduler can detect a
|
||||
// concurrent delete without guessing.
|
||||
if err := repo.RecordFailedAttempt(ctx, "notif-does-not-exist", "oops", nextTry); err == nil {
|
||||
t.Errorf("RecordFailedAttempt on unknown id succeeded; want error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotificationRepository_MarkAsDead verifies the DLQ transition. Flips
|
||||
// status to 'dead', clears next_retry_at (so the partial retry-sweep
|
||||
// index drops the row), writes final last_error, preserves retry_count as
|
||||
// evidence of how many attempts were burned.
|
||||
func TestNotificationRepository_MarkAsDead(t *testing.T) {
|
||||
tdb := getTestDB(t)
|
||||
db := tdb.freshSchema(t)
|
||||
repo := postgres.NewNotificationRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
lastAttempt := past()
|
||||
_, err := db.ExecContext(ctx, `
|
||||
INSERT INTO notification_events (
|
||||
id, type, channel, recipient, message, status, retry_count, next_retry_at, last_error
|
||||
) VALUES ('notif-dlq-1', 'ExpirationWarning', 'Webhook',
|
||||
'https://hooks.example.com/x', 'seed', 'failed', 5, $1, 'prior failure')
|
||||
`, lastAttempt)
|
||||
if err != nil {
|
||||
t.Fatalf("seed failed: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.MarkAsDead(ctx, "notif-dlq-1", "max attempts exceeded"); err != nil {
|
||||
t.Fatalf("MarkAsDead failed: %v", err)
|
||||
}
|
||||
|
||||
var (
|
||||
gotStatus string
|
||||
gotRetryCount int
|
||||
gotNextRetry *time.Time
|
||||
gotLastError *string
|
||||
)
|
||||
err = db.QueryRowContext(ctx, `
|
||||
SELECT status, retry_count, next_retry_at, last_error
|
||||
FROM notification_events WHERE id = 'notif-dlq-1'
|
||||
`).Scan(&gotStatus, &gotRetryCount, &gotNextRetry, &gotLastError)
|
||||
if err != nil {
|
||||
t.Fatalf("post-update SELECT failed: %v", err)
|
||||
}
|
||||
|
||||
if gotStatus != "dead" {
|
||||
t.Errorf("status = %q, want 'dead' (DLQ transition)", gotStatus)
|
||||
}
|
||||
if gotNextRetry != nil {
|
||||
// next_retry_at MUST be NULL post-DLQ — the partial retry-sweep
|
||||
// index predicate is `status='failed' AND next_retry_at IS NOT NULL`,
|
||||
// so leaving a value here would only waste space; the status='dead'
|
||||
// half of the predicate already excludes the row from the sweep,
|
||||
// but operator dashboards treat a populated next_retry_at as "still
|
||||
// scheduled", which would be a lie.
|
||||
t.Errorf("next_retry_at = %v, want NULL (dead rows are terminal, not rescheduled)", gotNextRetry)
|
||||
}
|
||||
if gotRetryCount != 5 {
|
||||
// retry_count is audit evidence — how many attempts were burned
|
||||
// before the row was declared dead. Don't clobber it.
|
||||
t.Errorf("retry_count = %d, want 5 preserved (evidence of burned attempts)", gotRetryCount)
|
||||
}
|
||||
if gotLastError == nil || *gotLastError != "max attempts exceeded" {
|
||||
t.Errorf("last_error = %v, want 'max attempts exceeded'", gotLastError)
|
||||
}
|
||||
|
||||
// Negative path: unknown id must surface "not found".
|
||||
if err := repo.MarkAsDead(ctx, "notif-does-not-exist", "oops"); err == nil {
|
||||
t.Errorf("MarkAsDead on unknown id succeeded; want error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotificationRepository_Requeue verifies the operator "try again"
|
||||
// flow exposed by the Dead letter tab. The contract:
|
||||
//
|
||||
// - Flips status → 'pending' regardless of prior ('dead' or 'failed').
|
||||
// - Resets retry_count to 0 — a manual requeue restarts the backoff
|
||||
// ladder; otherwise the operator's first retry would already be at
|
||||
// "wait 32 minutes" which defeats the point.
|
||||
// - Clears next_retry_at so the row is no longer in the retry-sweep
|
||||
// index (the scheduler would otherwise try to retry it *again* a
|
||||
// few seconds later).
|
||||
// - Clears last_error — the UI shouldn't show a stale error next to
|
||||
// a freshly-requeued row.
|
||||
func TestNotificationRepository_Requeue(t *testing.T) {
|
||||
tdb := getTestDB(t)
|
||||
db := tdb.freshSchema(t)
|
||||
repo := postgres.NewNotificationRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
// Two fixtures — one dead (DLQ path, the normal case) and one failed
|
||||
// (operator rescuing a stuck-in-retry row before the sweep fires).
|
||||
// Both must accept Requeue; a status='sent' or 'read' row must NOT.
|
||||
_, err := db.ExecContext(ctx, `
|
||||
INSERT INTO notification_events (id, type, channel, recipient, message, status, retry_count, last_error)
|
||||
VALUES
|
||||
('notif-dead-ready', 'ExpirationWarning', 'Webhook', 'https://h/x', 'seed', 'dead', 5, 'gave up'),
|
||||
('notif-failed-hot', 'ExpirationWarning', 'Webhook', 'https://h/x', 'seed', 'failed', 2, 'transient'),
|
||||
('notif-sent-done', 'ExpirationWarning', 'Webhook', 'https://h/x', 'seed', 'sent', 0, NULL)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("seed failed: %v", err)
|
||||
}
|
||||
|
||||
// Happy path 1: requeue a dead row.
|
||||
if err := repo.Requeue(ctx, "notif-dead-ready"); err != nil {
|
||||
t.Fatalf("Requeue(dead) failed: %v", err)
|
||||
}
|
||||
assertRequeued(t, db, ctx, "notif-dead-ready")
|
||||
|
||||
// Happy path 2: requeue a failed row.
|
||||
if err := repo.Requeue(ctx, "notif-failed-hot"); err != nil {
|
||||
t.Fatalf("Requeue(failed) failed: %v", err)
|
||||
}
|
||||
assertRequeued(t, db, ctx, "notif-failed-hot")
|
||||
|
||||
// Negative path: Requeue on unknown id is "not found", not a no-op
|
||||
// silent success — the handler needs to surface a 404 to the operator.
|
||||
if err := repo.Requeue(ctx, "notif-does-not-exist"); err == nil {
|
||||
t.Errorf("Requeue on unknown id succeeded; want error")
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
// past returns a stable "5 minutes ago" time for fixture seeding. Truncated
|
||||
// to microseconds so round-tripping through Postgres TIMESTAMPTZ doesn't
|
||||
// introduce a sub-microsecond diff that breaks equality assertions.
|
||||
func past() time.Time {
|
||||
return time.Now().UTC().Add(-5 * time.Minute).Truncate(time.Microsecond)
|
||||
}
|
||||
|
||||
// collectIDs pulls the IDs out of a slice of events for readable test
|
||||
// failure output. Without it, a failure prints "[0xc00012... 0xc00013...]"
|
||||
// which is useless when diagnosing a mis-sorted sweep.
|
||||
func collectIDs(events []*domain.NotificationEvent) []string {
|
||||
ids := make([]string, len(events))
|
||||
for i, e := range events {
|
||||
ids[i] = e.ID
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// assertRequeued is the shared "did Requeue do exactly what the contract
|
||||
// promises?" assertion. Re-reads the row and checks all four mutations
|
||||
// atomically so every Requeue test path gets the same rigor: status flipped
|
||||
// to 'pending', retry_count reset to 0, next_retry_at cleared, last_error
|
||||
// cleared. Any one of these missing is a contract violation.
|
||||
func assertRequeued(t *testing.T, db *sql.DB, ctx context.Context, id string) {
|
||||
t.Helper()
|
||||
var (
|
||||
gotStatus string
|
||||
gotRetryCount int
|
||||
gotNextRetry *time.Time
|
||||
gotLastError *string
|
||||
)
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT status, retry_count, next_retry_at, last_error
|
||||
FROM notification_events WHERE id = $1
|
||||
`, id).Scan(&gotStatus, &gotRetryCount, &gotNextRetry, &gotLastError)
|
||||
if err != nil {
|
||||
t.Fatalf("post-Requeue SELECT for %s failed: %v", id, err)
|
||||
}
|
||||
if gotStatus != "pending" {
|
||||
t.Errorf("%s.status = %q, want 'pending' (Requeue must re-open the row for ProcessPendingNotifications)",
|
||||
id, gotStatus)
|
||||
}
|
||||
if gotRetryCount != 0 {
|
||||
t.Errorf("%s.retry_count = %d, want 0 (Requeue restarts the backoff ladder so the operator's first retry isn't already at hour-long waits)",
|
||||
id, gotRetryCount)
|
||||
}
|
||||
if gotNextRetry != nil {
|
||||
t.Errorf("%s.next_retry_at = %v, want NULL (a fresh pending row must not sit in the retry-sweep index)",
|
||||
id, gotNextRetry)
|
||||
}
|
||||
if gotLastError != nil {
|
||||
t.Errorf("%s.last_error = %v, want NULL (stale errors on freshly-requeued rows mislead the UI)",
|
||||
id, *gotLastError)
|
||||
}
|
||||
}
|
||||
+196
-33
@@ -34,8 +34,14 @@ type AgentServicer interface {
|
||||
}
|
||||
|
||||
// NotificationServicer defines the interface for notification processing used by the scheduler.
|
||||
//
|
||||
// RetryFailedNotifications was added to close coverage gap I-005: the retry
|
||||
// sweep transitions eligible Failed notifications to Pending on an independent
|
||||
// tick, using exponential backoff with a 1h cap and a 5-attempt DLQ budget.
|
||||
// Mirrors the I-001 job retry loop topology.
|
||||
type NotificationServicer interface {
|
||||
ProcessPendingNotifications(ctx context.Context) error
|
||||
RetryFailedNotifications(ctx context.Context) error
|
||||
}
|
||||
|
||||
// NetworkScanServicer defines the interface for network scanning used by the scheduler.
|
||||
@@ -58,43 +64,55 @@ type CloudDiscoveryServicer interface {
|
||||
DiscoverAll(ctx context.Context) (int, []error)
|
||||
}
|
||||
|
||||
// JobReaperService defines the interface for job timeout reaping used by the scheduler.
|
||||
type JobReaperService interface {
|
||||
ReapTimedOutJobs(ctx context.Context, csrTTL, approvalTTL time.Duration) error
|
||||
}
|
||||
|
||||
// Scheduler manages background jobs and periodic tasks for the certificate control plane.
|
||||
// It runs multiple concurrent loops for renewal checks, job processing, agent health checks,
|
||||
// and notification processing.
|
||||
type Scheduler struct {
|
||||
renewalService RenewalServicer
|
||||
jobService JobServicer
|
||||
agentService AgentServicer
|
||||
notificationService NotificationServicer
|
||||
networkScanService NetworkScanServicer
|
||||
digestService DigestServicer
|
||||
healthCheckService HealthCheckServicer
|
||||
cloudDiscoveryService CloudDiscoveryServicer
|
||||
logger *slog.Logger
|
||||
renewalService RenewalServicer
|
||||
jobService JobServicer
|
||||
agentService AgentServicer
|
||||
notificationService NotificationServicer
|
||||
networkScanService NetworkScanServicer
|
||||
digestService DigestServicer
|
||||
healthCheckService HealthCheckServicer
|
||||
cloudDiscoveryService CloudDiscoveryServicer
|
||||
jobReaper JobReaperService
|
||||
logger *slog.Logger
|
||||
|
||||
// Configurable tick intervals
|
||||
renewalCheckInterval time.Duration
|
||||
jobProcessorInterval time.Duration
|
||||
jobRetryInterval time.Duration
|
||||
agentHealthCheckInterval time.Duration
|
||||
notificationProcessInterval time.Duration
|
||||
shortLivedExpiryCheckInterval time.Duration
|
||||
networkScanInterval time.Duration
|
||||
digestInterval time.Duration
|
||||
healthCheckInterval time.Duration
|
||||
cloudDiscoveryInterval time.Duration
|
||||
renewalCheckInterval time.Duration
|
||||
jobProcessorInterval time.Duration
|
||||
jobRetryInterval time.Duration
|
||||
agentHealthCheckInterval time.Duration
|
||||
notificationProcessInterval time.Duration
|
||||
notificationRetryInterval time.Duration
|
||||
shortLivedExpiryCheckInterval time.Duration
|
||||
networkScanInterval time.Duration
|
||||
digestInterval time.Duration
|
||||
healthCheckInterval time.Duration
|
||||
cloudDiscoveryInterval time.Duration
|
||||
jobTimeoutInterval time.Duration
|
||||
awaitingCSRTimeout time.Duration
|
||||
awaitingApprovalTimeout time.Duration
|
||||
|
||||
// Idempotency guards: prevent duplicate execution of slow jobs
|
||||
renewalCheckRunning atomic.Bool
|
||||
jobProcessorRunning atomic.Bool
|
||||
jobRetryRunning atomic.Bool
|
||||
agentHealthCheckRunning atomic.Bool
|
||||
notificationProcessRunning atomic.Bool
|
||||
shortLivedExpiryCheckRunning atomic.Bool
|
||||
networkScanRunning atomic.Bool
|
||||
digestRunning atomic.Bool
|
||||
healthCheckRunning atomic.Bool
|
||||
cloudDiscoveryRunning atomic.Bool
|
||||
renewalCheckRunning atomic.Bool
|
||||
jobProcessorRunning atomic.Bool
|
||||
jobRetryRunning atomic.Bool
|
||||
agentHealthCheckRunning atomic.Bool
|
||||
notificationProcessRunning atomic.Bool
|
||||
notificationRetryRunning atomic.Bool
|
||||
shortLivedExpiryCheckRunning atomic.Bool
|
||||
networkScanRunning atomic.Bool
|
||||
digestRunning atomic.Bool
|
||||
healthCheckRunning atomic.Bool
|
||||
cloudDiscoveryRunning atomic.Bool
|
||||
jobTimeoutRunning atomic.Bool
|
||||
|
||||
// Graceful shutdown: wait for in-flight work to complete
|
||||
wg sync.WaitGroup
|
||||
@@ -123,11 +141,13 @@ func NewScheduler(
|
||||
jobRetryInterval: 5 * time.Minute,
|
||||
agentHealthCheckInterval: 2 * time.Minute,
|
||||
notificationProcessInterval: 1 * time.Minute,
|
||||
notificationRetryInterval: 2 * time.Minute,
|
||||
shortLivedExpiryCheckInterval: 30 * time.Second,
|
||||
networkScanInterval: 6 * time.Hour,
|
||||
digestInterval: 24 * time.Hour,
|
||||
healthCheckInterval: 60 * time.Second,
|
||||
cloudDiscoveryInterval: 6 * time.Hour,
|
||||
jobTimeoutInterval: 10 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +189,13 @@ func (s *Scheduler) SetNotificationProcessInterval(d time.Duration) {
|
||||
s.notificationProcessInterval = d
|
||||
}
|
||||
|
||||
// SetNotificationRetryInterval configures the interval for the failed-notification
|
||||
// retry sweep (coverage gap I-005). Defaults to 2 minutes; honors
|
||||
// CERTCTL_NOTIFICATION_RETRY_INTERVAL when wired from config.
|
||||
func (s *Scheduler) SetNotificationRetryInterval(d time.Duration) {
|
||||
s.notificationRetryInterval = d
|
||||
}
|
||||
|
||||
// SetNetworkScanInterval configures the interval for network scanning.
|
||||
func (s *Scheduler) SetNetworkScanInterval(d time.Duration) {
|
||||
s.networkScanInterval = d
|
||||
@@ -201,6 +228,26 @@ func (s *Scheduler) SetCloudDiscoveryInterval(d time.Duration) {
|
||||
s.cloudDiscoveryInterval = d
|
||||
}
|
||||
|
||||
// SetJobReaperService sets the job reaper service (I-003).
|
||||
func (s *Scheduler) SetJobReaperService(jr JobReaperService) {
|
||||
s.jobReaper = jr
|
||||
}
|
||||
|
||||
// SetJobTimeoutInterval sets the job timeout reaper tick interval (I-003).
|
||||
func (s *Scheduler) SetJobTimeoutInterval(d time.Duration) {
|
||||
s.jobTimeoutInterval = d
|
||||
}
|
||||
|
||||
// SetAwaitingCSRTimeout sets the AwaitingCSR TTL (I-003).
|
||||
func (s *Scheduler) SetAwaitingCSRTimeout(d time.Duration) {
|
||||
s.awaitingCSRTimeout = d
|
||||
}
|
||||
|
||||
// SetAwaitingApprovalTimeout sets the AwaitingApproval TTL (I-003).
|
||||
func (s *Scheduler) SetAwaitingApprovalTimeout(d time.Duration) {
|
||||
s.awaitingApprovalTimeout = d
|
||||
}
|
||||
|
||||
// Start initiates all background scheduler loops. It returns a channel that signals
|
||||
// when the scheduler has started all loops. The scheduler runs until the context is cancelled.
|
||||
func (s *Scheduler) Start(ctx context.Context) <-chan struct{} {
|
||||
@@ -211,10 +258,11 @@ func (s *Scheduler) Start(ctx context.Context) <-chan struct{} {
|
||||
|
||||
// Track all loop goroutines in the WaitGroup so WaitForCompletion
|
||||
// blocks until they've fully exited (prevents test races).
|
||||
// Base count is 6: renewal, job processor, job retry (I-001),
|
||||
// agent health, notification, short-lived expiry. Optional loops
|
||||
// (network scan, digest, health check, cloud discovery) add to this.
|
||||
loopCount := 6
|
||||
// Base count is 8: renewal, job processor, job retry (I-001),
|
||||
// job timeout (I-003), agent health, notification, notification retry
|
||||
// (I-005), short-lived expiry. Optional loops (network scan, digest,
|
||||
// health check, cloud discovery) add to this.
|
||||
loopCount := 8
|
||||
if s.networkScanService != nil {
|
||||
loopCount++
|
||||
}
|
||||
@@ -232,8 +280,10 @@ func (s *Scheduler) Start(ctx context.Context) <-chan struct{} {
|
||||
go func() { defer s.wg.Done(); s.renewalCheckLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.jobProcessorLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.jobRetryLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.jobTimeoutLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.agentHealthCheckLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.notificationProcessLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.notificationRetryLoop(ctx) }()
|
||||
go func() { defer s.wg.Done(); s.shortLivedExpiryCheckLoop(ctx) }()
|
||||
if s.networkScanService != nil {
|
||||
go func() { defer s.wg.Done(); s.networkScanLoop(ctx) }()
|
||||
@@ -413,6 +463,61 @@ func (s *Scheduler) runJobRetry(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// jobTimeoutLoop runs every jobTimeoutInterval and transitions jobs stuck in
|
||||
// AwaitingCSR or AwaitingApproval to Failed if they exceed their TTL. I-001's
|
||||
// retry loop then auto-promotes eligible Failed jobs back to Pending. Closes
|
||||
// coverage gap I-003. Uses atomic.Bool to prevent duplicate execution.
|
||||
func (s *Scheduler) jobTimeoutLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(s.jobTimeoutInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Run immediately on start (with idempotency guard)
|
||||
s.jobTimeoutRunning.Store(true)
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer s.jobTimeoutRunning.Store(false)
|
||||
s.runJobTimeout(ctx)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if !s.jobTimeoutRunning.CompareAndSwap(false, true) {
|
||||
s.logger.Warn("job timeout reaper still running, skipping tick")
|
||||
continue
|
||||
}
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer s.jobTimeoutRunning.Store(false)
|
||||
s.runJobTimeout(ctx)
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runJobTimeout executes a single job timeout reaping cycle with error recovery.
|
||||
// When no JobReaperService has been wired (e.g. in tests that don't exercise
|
||||
// I-003) the call is a safe no-op, preserving the always-on loop topology
|
||||
// described in I-003 without forcing every consumer to wire a reaper.
|
||||
func (s *Scheduler) runJobTimeout(ctx context.Context) {
|
||||
if s.jobReaper == nil {
|
||||
return
|
||||
}
|
||||
opCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.jobReaper.ReapTimedOutJobs(opCtx, s.awaitingCSRTimeout, s.awaitingApprovalTimeout); err != nil {
|
||||
s.logger.Error("job timeout reaper failed",
|
||||
"error", err,
|
||||
"interval", s.jobTimeoutInterval.String())
|
||||
} else {
|
||||
s.logger.Debug("job timeout reaper completed")
|
||||
}
|
||||
}
|
||||
|
||||
// agentHealthCheckLoop runs every agentHealthCheckInterval and marks stale agents as offline.
|
||||
// An agent is considered stale if it hasn't sent a heartbeat within the health check interval.
|
||||
// If an error occurs, it logs the error but continues running.
|
||||
@@ -510,6 +615,64 @@ func (s *Scheduler) runNotificationProcess(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// notificationRetryLoop runs every notificationRetryInterval and transitions
|
||||
// eligible Failed notifications back to Pending so the notification processor
|
||||
// can pick them up again. Closes coverage gap I-005 — NotificationService.
|
||||
// RetryFailedNotifications had no runtime caller prior to this loop being
|
||||
// wired. Runs immediately on start, then every interval.
|
||||
// Uses atomic.Bool to prevent duplicate execution if the previous retry sweep
|
||||
// is still running. Mirrors the I-001 jobRetryLoop topology byte-for-byte.
|
||||
func (s *Scheduler) notificationRetryLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(s.notificationRetryInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Run immediately on start (with idempotency guard)
|
||||
s.notificationRetryRunning.Store(true)
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer s.notificationRetryRunning.Store(false)
|
||||
s.runNotificationRetry(ctx)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if !s.notificationRetryRunning.CompareAndSwap(false, true) {
|
||||
s.logger.Warn("notification retry still running, skipping tick")
|
||||
continue
|
||||
}
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer s.notificationRetryRunning.Store(false)
|
||||
s.runNotificationRetry(ctx)
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runNotificationRetry executes a single failed-notification retry cycle with
|
||||
// error recovery. Uses a 2-minute per-tick timeout matching runJobRetry;
|
||||
// RetryFailedNotifications issues one SELECT and one UPDATE per eligible row
|
||||
// (cheap), so this headroom covers very large failure backlogs without
|
||||
// starving the loop. The service layer swallows per-row send errors (mirrors
|
||||
// ProcessPendingNotifications) and only returns the List error from the
|
||||
// initial ListRetryEligible call.
|
||||
func (s *Scheduler) runNotificationRetry(ctx context.Context) {
|
||||
opCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.notificationService.RetryFailedNotifications(opCtx); err != nil {
|
||||
s.logger.Error("notification retry failed",
|
||||
"error", err,
|
||||
"interval", s.notificationRetryInterval.String())
|
||||
} else {
|
||||
s.logger.Debug("notification retry completed")
|
||||
}
|
||||
}
|
||||
|
||||
// shortLivedExpiryCheckLoop runs every shortLivedExpiryCheckInterval and marks expired
|
||||
// short-lived certificates. For certs with TTL < 1 hour, expiry IS revocation —
|
||||
// no CRL/OCSP needed.
|
||||
|
||||
@@ -85,6 +85,13 @@ type mockJobService struct {
|
||||
retryMaxRetriesSeen []int
|
||||
retrySlowDelay time.Duration
|
||||
retryShouldError bool
|
||||
|
||||
// Timeout reaper tracking (coverage gap I-003)
|
||||
reapCallCount int
|
||||
reapCallTimes []time.Time
|
||||
reapSlowDelay time.Duration
|
||||
reapShouldError bool
|
||||
reapCtxHasDeadline bool
|
||||
}
|
||||
|
||||
func (m *mockJobService) ProcessPendingJobs(ctx context.Context) error {
|
||||
@@ -131,6 +138,33 @@ func (m *mockJobService) RetryFailedJobs(ctx context.Context, maxRetries int) er
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
// ReapTimedOutJobs is the scheduler-driven counterpart to ProcessPendingJobs that
|
||||
// covers coverage gap I-003: JobService.ReapTimedOutJobs (via JobReaperService interface)
|
||||
// had no runtime caller prior to the jobTimeoutLoop being wired.
|
||||
func (m *mockJobService) ReapTimedOutJobs(ctx context.Context, csrTTL, approvalTTL time.Duration) error {
|
||||
m.mu.Lock()
|
||||
m.reapCallCount++
|
||||
m.reapCallTimes = append(m.reapCallTimes, time.Now())
|
||||
// Track whether context has a deadline set
|
||||
_, hasDeadline := ctx.Deadline()
|
||||
m.reapCtxHasDeadline = hasDeadline
|
||||
m.mu.Unlock()
|
||||
|
||||
if m.reapSlowDelay > 0 {
|
||||
select {
|
||||
case <-time.After(m.reapSlowDelay):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
if m.reapShouldError {
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mockAgentService is a mock implementation for testing.
|
||||
type mockAgentService struct {
|
||||
mu sync.Mutex
|
||||
@@ -161,12 +195,25 @@ func (m *mockAgentService) MarkStaleAgentsOffline(ctx context.Context, interval
|
||||
}
|
||||
|
||||
// mockNotificationService is a mock implementation for testing.
|
||||
//
|
||||
// Tracks ProcessPendingNotifications and RetryFailedNotifications separately.
|
||||
// retrySlowDelay and retryShouldError let tests exercise the retry loop
|
||||
// independently of the processor loop without coupling their timing/failure
|
||||
// modes (coverage gap I-005 — prior to the notificationRetryLoop being wired,
|
||||
// RetryFailedNotifications had no runtime caller).
|
||||
type mockNotificationService struct {
|
||||
mu sync.Mutex
|
||||
callCount int
|
||||
callTimes []time.Time
|
||||
slowDelay time.Duration
|
||||
shouldError bool
|
||||
|
||||
// Retry loop tracking (coverage gap I-005)
|
||||
retryCallCount int
|
||||
retryCallTimes []time.Time
|
||||
retrySlowDelay time.Duration
|
||||
retryShouldError bool
|
||||
retryCtxHasDeadline bool
|
||||
}
|
||||
|
||||
func (m *mockNotificationService) ProcessPendingNotifications(ctx context.Context) error {
|
||||
@@ -189,6 +236,42 @@ func (m *mockNotificationService) ProcessPendingNotifications(ctx context.Contex
|
||||
return nil
|
||||
}
|
||||
|
||||
// RetryFailedNotifications is the scheduler-driven counterpart to
|
||||
// ProcessPendingNotifications that closes coverage gap I-005. Prior to the
|
||||
// notificationRetryLoop being wired, notifications that hit status='failed'
|
||||
// orphaned there forever — no retry, no DLQ, no escalation. The service-layer
|
||||
// method exists to sweep failed rows whose next_retry_at has elapsed, but
|
||||
// without a scheduler caller the sweep never runs in production.
|
||||
//
|
||||
// This mock mirrors mockJobService.RetryFailedJobs's shape: a retry-only field
|
||||
// cluster so callers can dial retrySlowDelay / retryShouldError without
|
||||
// perturbing ProcessPendingNotifications's timing, and retryCtxHasDeadline so
|
||||
// the ContextDeadlineRespected test can assert the scheduler is passing a
|
||||
// per-tick context.WithTimeout rather than the raw shutdown ctx.
|
||||
func (m *mockNotificationService) RetryFailedNotifications(ctx context.Context) error {
|
||||
m.mu.Lock()
|
||||
m.retryCallCount++
|
||||
m.retryCallTimes = append(m.retryCallTimes, time.Now())
|
||||
// Track whether context has a deadline set — the scheduler must wrap each
|
||||
// tick in a bounded context so a hung sweep can't stall shutdown.
|
||||
_, hasDeadline := ctx.Deadline()
|
||||
m.retryCtxHasDeadline = hasDeadline
|
||||
m.mu.Unlock()
|
||||
|
||||
if m.retrySlowDelay > 0 {
|
||||
select {
|
||||
case <-time.After(m.retrySlowDelay):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
if m.retryShouldError {
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mockNetworkScanService is a mock implementation for testing.
|
||||
type mockNetworkScanService struct {
|
||||
mu sync.Mutex
|
||||
@@ -1141,3 +1224,404 @@ func TestScheduler_JobRetryLoop_WaitForCompletion(t *testing.T) {
|
||||
}
|
||||
t.Logf("retry loop graceful shutdown completed in %v after %d in-flight sweep(s)", elapsed, retryCount)
|
||||
}
|
||||
|
||||
// TestScheduler_JobTimeoutLoop_NormalTick verifies that the job timeout reaper
|
||||
// loop ticks at the specified interval (coverage gap I-003).
|
||||
func TestScheduler_JobTimeoutLoop_NormalTick(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetJobTimeoutInterval(50 * time.Millisecond)
|
||||
sched.SetAwaitingCSRTimeout(24 * time.Hour)
|
||||
sched.SetAwaitingApprovalTimeout(168 * time.Hour)
|
||||
sched.SetJobReaperService(jobMock)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
<-sched.Start(ctx)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
cancel()
|
||||
if err := sched.WaitForCompletion(2 * time.Second); err != nil {
|
||||
t.Fatalf("WaitForCompletion: %v", err)
|
||||
}
|
||||
|
||||
jobMock.mu.Lock()
|
||||
count := jobMock.reapCallCount
|
||||
jobMock.mu.Unlock()
|
||||
if count < 2 {
|
||||
t.Fatalf("expected >= 2 reap calls, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScheduler_JobTimeoutLoop_IdempotencyGuard verifies that the timeout reaper
|
||||
// uses an atomic guard to prevent concurrent execution (coverage gap I-003).
|
||||
func TestScheduler_JobTimeoutLoop_IdempotencyGuard(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{
|
||||
reapSlowDelay: 150 * time.Millisecond,
|
||||
}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetJobTimeoutInterval(50 * time.Millisecond)
|
||||
sched.SetAwaitingCSRTimeout(24 * time.Hour)
|
||||
sched.SetAwaitingApprovalTimeout(168 * time.Hour)
|
||||
sched.SetJobReaperService(jobMock)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
<-sched.Start(ctx)
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
|
||||
jobMock.mu.Lock()
|
||||
reapCount := jobMock.reapCallCount
|
||||
jobMock.mu.Unlock()
|
||||
|
||||
if reapCount > 3 {
|
||||
t.Logf("WARNING: reap called %d times in 400ms with 50ms interval and 150ms sweep — guard may not be working", reapCount)
|
||||
}
|
||||
|
||||
t.Logf("job timeout idempotency guard: %d calls in 400ms (50ms interval, 150ms sweep)", reapCount)
|
||||
|
||||
cancel()
|
||||
if err := sched.WaitForCompletion(2 * time.Second); err != nil {
|
||||
t.Fatalf("WaitForCompletion should succeed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScheduler_JobTimeoutLoop_ShutdownDrainsInFlight verifies that shutdown waits
|
||||
// for an in-flight timeout reaper to complete (coverage gap I-003).
|
||||
func TestScheduler_JobTimeoutLoop_ShutdownDrainsInFlight(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{
|
||||
reapSlowDelay: 100 * time.Millisecond,
|
||||
}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetJobTimeoutInterval(50 * time.Millisecond)
|
||||
sched.SetAwaitingCSRTimeout(24 * time.Hour)
|
||||
sched.SetAwaitingApprovalTimeout(168 * time.Hour)
|
||||
sched.SetJobReaperService(jobMock)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
<-sched.Start(ctx)
|
||||
|
||||
// Let the immediate-start timeout reaper goroutine begin its 100ms sweep.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
|
||||
// Initiate shutdown mid-sweep.
|
||||
cancel()
|
||||
|
||||
start := time.Now()
|
||||
err := sched.WaitForCompletion(5 * time.Second)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("WaitForCompletion should not error: %v", err)
|
||||
}
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("WaitForCompletion took longer than expected: %v", elapsed)
|
||||
}
|
||||
|
||||
jobMock.mu.Lock()
|
||||
reapCount := jobMock.reapCallCount
|
||||
jobMock.mu.Unlock()
|
||||
|
||||
if reapCount < 1 {
|
||||
t.Fatalf("expected timeout reaper to have started at least once before shutdown, got %d", reapCount)
|
||||
}
|
||||
t.Logf("timeout reaper graceful shutdown completed in %v after %d in-flight sweep(s)", elapsed, reapCount)
|
||||
}
|
||||
|
||||
// TestScheduler_JobTimeoutLoop_ContextDeadlineRespected verifies that the timeout
|
||||
// reaper receives a context with a deadline set for each tick (coverage gap I-003).
|
||||
func TestScheduler_JobTimeoutLoop_ContextDeadlineRespected(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetJobTimeoutInterval(50 * time.Millisecond)
|
||||
sched.SetAwaitingCSRTimeout(24 * time.Hour)
|
||||
sched.SetAwaitingApprovalTimeout(168 * time.Hour)
|
||||
sched.SetJobReaperService(jobMock)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
<-sched.Start(ctx)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
if err := sched.WaitForCompletion(2 * time.Second); err != nil {
|
||||
t.Fatalf("WaitForCompletion: %v", err)
|
||||
}
|
||||
|
||||
jobMock.mu.Lock()
|
||||
hasDeadline := jobMock.reapCtxHasDeadline
|
||||
jobMock.mu.Unlock()
|
||||
|
||||
if !hasDeadline {
|
||||
t.Fatal("expected timeout reaper context to have a deadline set, but none found")
|
||||
}
|
||||
t.Log("timeout reaper context deadline verified")
|
||||
}
|
||||
|
||||
// ─── NotificationRetryLoop tests (coverage gap I-005) ────────────────────────
|
||||
//
|
||||
// These four tests are the scheduler-level Red half of the I-005 fix. They
|
||||
// mirror the I-001 jobRetryLoop triplet (CallsService / IdempotencyGuard /
|
||||
// WaitForCompletion) plus the I-003 ContextDeadlineRespected shape.
|
||||
//
|
||||
// All four use the same "quiet every other loop" pattern so the only tick
|
||||
// activity visible on notificationMock is the retry loop under test. JobTimeout
|
||||
// is intentionally left unconfigured — SetJobReaperService isn't called, so the
|
||||
// timeout loop is dormant (same convention the I-001 tests follow).
|
||||
|
||||
// TestScheduler_NotificationRetryLoop_CallsService verifies that the
|
||||
// notification retry loop invokes NotificationService.RetryFailedNotifications
|
||||
// on each tick. Closes coverage gap I-005 — prior to the loop being wired,
|
||||
// RetryFailedNotifications had no runtime caller and failed notification_events
|
||||
// rows orphaned at status='failed' forever (no retry, no DLQ, no escalation).
|
||||
//
|
||||
// Unlike the jobRetryLoop test, there is no maxRetries advisory constant to
|
||||
// forward: the max_attempts limit on notification retries lives on the row
|
||||
// itself (retry_count column introduced by migration 000016), not in the call
|
||||
// signature.
|
||||
func TestScheduler_NotificationRetryLoop_CallsService(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
// Quiet every other loop so only the retry loop's calls are visible on notificationMock.
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetNotificationRetryInterval(50 * time.Millisecond)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
startedChan := sched.Start(ctx)
|
||||
<-startedChan
|
||||
|
||||
// Run long enough for the immediate start + at least one tick.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
cancel()
|
||||
_ = sched.WaitForCompletion(2 * time.Second)
|
||||
|
||||
notificationMock.mu.Lock()
|
||||
retryCount := notificationMock.retryCallCount
|
||||
notificationMock.mu.Unlock()
|
||||
|
||||
if retryCount < 1 {
|
||||
t.Fatalf("expected notification retry service to be called at least once, got %d", retryCount)
|
||||
}
|
||||
t.Logf("notification retry loop called %d times", retryCount)
|
||||
}
|
||||
|
||||
// TestScheduler_NotificationRetryLoop_IdempotencyGuard verifies that a slow
|
||||
// retry sweep does not cause overlapping executions. Mirrors the shape of
|
||||
// TestScheduler_JobRetryLoop_IdempotencyGuard.
|
||||
//
|
||||
// The guard is the atomic.Bool notificationRetryRunning in scheduler.go.
|
||||
// Without it, a 100ms tick against a 150ms operation would fire ~4 times in
|
||||
// 400ms; with the guard we expect ~2–3 calls. Anything above 3 is logged as a
|
||||
// warning (not a hard failure) so CI timing noise doesn't produce flakes.
|
||||
func TestScheduler_NotificationRetryLoop_IdempotencyGuard(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{
|
||||
retrySlowDelay: 150 * time.Millisecond, // slower than tick interval
|
||||
}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetNotificationRetryInterval(100 * time.Millisecond)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
startedChan := sched.Start(ctx)
|
||||
<-startedChan
|
||||
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
|
||||
notificationMock.mu.Lock()
|
||||
retryCount := notificationMock.retryCallCount
|
||||
notificationMock.mu.Unlock()
|
||||
|
||||
// With a 150ms sweep and 100ms interval, a functioning guard should yield
|
||||
// roughly 2–3 calls (immediate + any ticks whose previous sweep finished).
|
||||
// Anything above 3 suggests the guard isn't holding.
|
||||
if retryCount > 3 {
|
||||
t.Logf("WARNING: retry called %d times in 400ms with 100ms interval and 150ms sweep — guard may not be working", retryCount)
|
||||
}
|
||||
|
||||
t.Logf("notification retry idempotency guard: %d calls in 400ms (100ms interval, 150ms sweep)", retryCount)
|
||||
|
||||
cancel()
|
||||
if err := sched.WaitForCompletion(2 * time.Second); err != nil {
|
||||
t.Fatalf("WaitForCompletion should succeed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScheduler_NotificationRetryLoop_WaitForCompletion verifies that a retry
|
||||
// sweep still in flight at shutdown is awaited by WaitForCompletion — the same
|
||||
// sync.WaitGroup contract every other loop satisfies. If the loop were to
|
||||
// return early without registering its goroutine on s.wg, this test would
|
||||
// either (a) observe retryCount==0 because the immediate-start sweep was never
|
||||
// launched, or (b) observe WaitForCompletion returning before the in-flight
|
||||
// sweep finished (elapsed < retrySlowDelay).
|
||||
func TestScheduler_NotificationRetryLoop_WaitForCompletion(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{
|
||||
retrySlowDelay: 100 * time.Millisecond,
|
||||
}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetNotificationRetryInterval(50 * time.Millisecond)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
startedChan := sched.Start(ctx)
|
||||
<-startedChan
|
||||
|
||||
// Let the immediate-start retry goroutine begin its 100ms sweep.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
|
||||
// Initiate shutdown mid-sweep.
|
||||
cancel()
|
||||
|
||||
start := time.Now()
|
||||
err := sched.WaitForCompletion(5 * time.Second)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("WaitForCompletion should not error: %v", err)
|
||||
}
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("WaitForCompletion took longer than expected: %v", elapsed)
|
||||
}
|
||||
|
||||
notificationMock.mu.Lock()
|
||||
retryCount := notificationMock.retryCallCount
|
||||
notificationMock.mu.Unlock()
|
||||
|
||||
if retryCount < 1 {
|
||||
t.Fatalf("expected notification retry service to have started at least once before shutdown, got %d", retryCount)
|
||||
}
|
||||
t.Logf("notification retry loop graceful shutdown completed in %v after %d in-flight sweep(s)", elapsed, retryCount)
|
||||
}
|
||||
|
||||
// TestScheduler_NotificationRetryLoop_ContextDeadlineRespected verifies that
|
||||
// each tick of the retry loop receives a context with a deadline set. Mirrors
|
||||
// TestScheduler_JobTimeoutLoop_ContextDeadlineRespected.
|
||||
//
|
||||
// The per-tick context.WithTimeout exists so a pathologically slow sweep (e.g.
|
||||
// a misbehaving DB lock) can't stall the rest of the scheduler's shutdown
|
||||
// sequence indefinitely — the wrapping context expires, the sweep returns
|
||||
// ctx.Err(), and the WaitGroup.Done() fires on schedule.
|
||||
func TestScheduler_NotificationRetryLoop_ContextDeadlineRespected(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
renewalMock := &mockRenewalService{}
|
||||
jobMock := &mockJobService{}
|
||||
agentMock := &mockAgentService{}
|
||||
notificationMock := &mockNotificationService{}
|
||||
networkMock := &mockNetworkScanService{}
|
||||
|
||||
sched := NewScheduler(renewalMock, jobMock, agentMock, notificationMock, networkMock, logger)
|
||||
sched.SetRenewalCheckInterval(10 * time.Second)
|
||||
sched.SetJobProcessorInterval(10 * time.Second)
|
||||
sched.SetAgentHealthCheckInterval(10 * time.Second)
|
||||
sched.SetNotificationProcessInterval(10 * time.Second)
|
||||
sched.SetNetworkScanInterval(10 * time.Second)
|
||||
sched.SetJobRetryInterval(10 * time.Second)
|
||||
sched.SetNotificationRetryInterval(50 * time.Millisecond)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
<-sched.Start(ctx)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
if err := sched.WaitForCompletion(2 * time.Second); err != nil {
|
||||
t.Fatalf("WaitForCompletion: %v", err)
|
||||
}
|
||||
|
||||
notificationMock.mu.Lock()
|
||||
hasDeadline := notificationMock.retryCtxHasDeadline
|
||||
notificationMock.mu.Unlock()
|
||||
|
||||
if !hasDeadline {
|
||||
t.Fatal("expected notification retry context to have a deadline set, but none found")
|
||||
}
|
||||
t.Log("notification retry context deadline verified")
|
||||
}
|
||||
|
||||
@@ -92,12 +92,27 @@ func (s *AgentService) Register(ctx context.Context, name string, hostname strin
|
||||
}
|
||||
|
||||
// Heartbeat updates an agent's last seen time, status, and metadata.
|
||||
//
|
||||
// I-004: retired agents must be rejected up-front. A retired agent that is
|
||||
// still polling is a zombie — its row exists only for audit history and must
|
||||
// not be allowed to bump LastHeartbeatAt (which would resurrect it in stats
|
||||
// dashboards and stale-offline sweeps). The sentinel ErrAgentRetired is
|
||||
// returned unwrapped so the HTTP handler can map it to 410 Gone via
|
||||
// errors.Is; the agent process detects the 410 and shuts down cleanly
|
||||
// instead of continuing to heartbeat indefinitely.
|
||||
func (s *AgentService) Heartbeat(ctx context.Context, agentID string, metadata *domain.AgentMetadata) error {
|
||||
agent, err := s.agentRepo.Get(ctx, agentID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch agent: %w", err)
|
||||
}
|
||||
|
||||
// I-004 guard: retired agents are frozen. Do not call UpdateHeartbeat —
|
||||
// bumping the timestamp would defeat the retired-row filter that protects
|
||||
// stats, scheduler sweeps, and handler listings.
|
||||
if agent.IsRetired() {
|
||||
return ErrAgentRetired
|
||||
}
|
||||
|
||||
// Update heartbeat and metadata
|
||||
if err := s.agentRepo.UpdateHeartbeat(ctx, agentID, metadata); err != nil {
|
||||
return fmt.Errorf("failed to update heartbeat: %w", err)
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
)
|
||||
|
||||
// I-004 coverage-gap closure: the agent retirement surface.
|
||||
//
|
||||
// Before 000015, DELETE /api/v1/agents/{id} hard-deleted the agents row and
|
||||
// the deployment_targets.agent_id FK CASCADE cleaned up downstream rows with
|
||||
// no preflight, no archival, and no knowledge of in-flight jobs. Any cert
|
||||
// still rotating through one of those targets would observe half-migrated
|
||||
// state. I-004 closes that gap with a preflight + soft-retire + optional
|
||||
// forced-cascade contract; the symbols in this file are the service-layer
|
||||
// surface that the handler and operator UI bind against.
|
||||
|
||||
// ErrAgentIsSentinel is returned when an operator tries to retire one of the
|
||||
// four reserved sentinel agent IDs (server-scanner, cloud-aws-sm,
|
||||
// cloud-azure-kv, cloud-gcp-sm). These rows back the network scanner and the
|
||||
// three cloud secret-manager discovery sources; retiring any of them orphans
|
||||
// its subsystem. The guard fires unconditionally — force=true does not bypass
|
||||
// it, because a sentinel is a structural invariant of the deployment, not
|
||||
// a piece of fleet state the operator owns. Handler maps this to HTTP 403.
|
||||
var ErrAgentIsSentinel = errors.New("agent is a reserved sentinel and cannot be retired")
|
||||
|
||||
// ErrBlockedByDependencies is returned by RetireAgent when at least one of
|
||||
// (active targets, active certificates, pending jobs) referencing the agent
|
||||
// is non-zero and force=false. The caller always receives it wrapped in
|
||||
// a *BlockedByDependenciesError (see below), so handlers doing errors.As
|
||||
// can surface the per-bucket counts in the 409 body for operator
|
||||
// troubleshooting. Tests use errors.Is; handlers use errors.As.
|
||||
var ErrBlockedByDependencies = errors.New("agent has active downstream dependencies")
|
||||
|
||||
// ErrForceReasonRequired is returned when force=true is supplied without a
|
||||
// non-empty reason. The force escape hatch is deliberately chatty: operators
|
||||
// pulling the emergency cord must leave an auditable breadcrumb explaining
|
||||
// why a cascade was justified. Handler maps this to HTTP 400 so the operator
|
||||
// retries with --reason rather than silently skipping the guard. Checked
|
||||
// before any DB mutation to keep the no-reason path transactionally clean.
|
||||
var ErrForceReasonRequired = errors.New("force=true requires a non-empty reason")
|
||||
|
||||
// ErrAgentRetired is returned by Heartbeat (and any future agent-authenticated
|
||||
// call site) when a retired agent is still polling. The handler layer maps
|
||||
// this to HTTP 410 Gone so the cmd/agent sendHeartbeat loop can detect it
|
||||
// deterministically and shut down the agent process, rather than looping
|
||||
// forever on a soft-retired identity. IsRetired() on the domain model is
|
||||
// the single source of truth; the sentinel exists so service and handler
|
||||
// callers can errors.Is against one symbol.
|
||||
var ErrAgentRetired = errors.New("agent has been retired")
|
||||
|
||||
// BlockedByDependenciesError wraps ErrBlockedByDependencies and carries the
|
||||
// per-bucket dependency snapshot the preflight pass captured. The embedded
|
||||
// AgentDependencyCounts is the same struct the repo returns from the three
|
||||
// CountActive* calls, so the handler can marshal it directly into the 409
|
||||
// body without reshaping fields. Unwrap() satisfies errors.Is against the
|
||||
// sentinel; Error() includes the counts so logs are diagnostic on their own.
|
||||
type BlockedByDependenciesError struct {
|
||||
Counts domain.AgentDependencyCounts
|
||||
}
|
||||
|
||||
// Error formats the wrapped error with the per-bucket counts. Kept short so
|
||||
// it reads cleanly in slog output.
|
||||
func (e *BlockedByDependenciesError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"%s (active_targets=%d, active_certificates=%d, pending_jobs=%d)",
|
||||
ErrBlockedByDependencies.Error(),
|
||||
e.Counts.ActiveTargets,
|
||||
e.Counts.ActiveCertificates,
|
||||
e.Counts.PendingJobs,
|
||||
)
|
||||
}
|
||||
|
||||
// Unwrap lets errors.Is(err, ErrBlockedByDependencies) match the wrapped
|
||||
// struct — the test contract (agent_retire_test.go:167) depends on it.
|
||||
func (e *BlockedByDependenciesError) Unwrap() error { return ErrBlockedByDependencies }
|
||||
|
||||
// AgentRetirementResult is the outcome surface the handler returns to the
|
||||
// operator. It discriminates the three happy paths the endpoint can take —
|
||||
// idempotent no-op (AlreadyRetired), clean soft-retire (Cascade=false), and
|
||||
// forced cascade (Cascade=true) — and always carries the retired_at timestamp
|
||||
// and the dependency-count snapshot so the 200/204 response body can echo
|
||||
// what was (or would have been) affected.
|
||||
//
|
||||
// AlreadyRetired=true → agent was already retired; no new audit
|
||||
// event was emitted; RetiredAt is the
|
||||
// original stamp, not the current time.
|
||||
// Cascade=false → clean soft-retire; Counts is all zeros.
|
||||
// Cascade=true → force=true retired agent + downstream
|
||||
// targets; Counts is the PRE-cascade
|
||||
// snapshot (so the operator sees what
|
||||
// they just retired).
|
||||
type AgentRetirementResult struct {
|
||||
AlreadyRetired bool
|
||||
Cascade bool
|
||||
RetiredAt time.Time
|
||||
Counts domain.AgentDependencyCounts
|
||||
}
|
||||
|
||||
// RetireAgent implements the I-004 retirement contract. Ordering matters —
|
||||
// every guard fires before the one that would mutate state, so a rejected
|
||||
// retire leaves zero trace (no audit event, no partial DB write):
|
||||
//
|
||||
// 1. Sentinel check (unconditional; force does not bypass).
|
||||
// 2. Fetch agent (404 surfaces as-is from the repo).
|
||||
// 3. Already-retired idempotency: return AlreadyRetired=true with NO new
|
||||
// audit event — the original retire already recorded one.
|
||||
// 4. Preflight count pass via the three CountActive* repo methods.
|
||||
// 5. Force-reason guard: force=true with empty reason is rejected here,
|
||||
// after the counts are known but before any mutation.
|
||||
// 6. Default no-force path: any non-zero count returns
|
||||
// *BlockedByDependenciesError with counts attached.
|
||||
// 7. Mutation: SoftRetire (no cascade) or RetireAgentWithCascade, with
|
||||
// a single retiredAt timestamp pinned BEFORE the repo call so the
|
||||
// audit event and the DB row agree to the nanosecond.
|
||||
// 8. Audit: agent_retired always; agent_retirement_cascaded additionally
|
||||
// on the force=true cascade path.
|
||||
//
|
||||
// Actor comes from the handler's resolveActor (API key → user, agent key →
|
||||
// agent-<id>, unauthenticated → "anonymous"); the service does not second-
|
||||
// guess it. Audit emission is best-effort: a failed RecordEvent logs a
|
||||
// warning but does not fail the overall retirement, consistent with how
|
||||
// the rest of the codebase treats audit as an observability concern
|
||||
// rather than a correctness barrier.
|
||||
func (s *AgentService) RetireAgent(ctx context.Context, id string, actor string, force bool, reason string) (*AgentRetirementResult, error) {
|
||||
// Step 1 — reserved-sentinel guard. Applies even under force=true.
|
||||
if domain.IsSentinelAgent(id) {
|
||||
return nil, ErrAgentIsSentinel
|
||||
}
|
||||
|
||||
// Step 2 — existence check. Missing agent surfaces the repo's not-found
|
||||
// error verbatim so the handler can map it to 404 via its existing
|
||||
// detection path (the handler layer already has "not found" mapping
|
||||
// logic inherited from the pre-I-004 Delete endpoint).
|
||||
agent, err := s.agentRepo.Get(ctx, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch agent: %w", err)
|
||||
}
|
||||
|
||||
// Step 3 — idempotency. A retired agent returns AlreadyRetired=true
|
||||
// WITHOUT emitting a fresh audit event. Handler maps this to HTTP 204.
|
||||
// Guarding here (before preflight) means a re-retire of an agent that
|
||||
// now has zero deps doesn't spuriously "succeed again" and double-log.
|
||||
if agent.IsRetired() {
|
||||
return &AgentRetirementResult{
|
||||
AlreadyRetired: true,
|
||||
RetiredAt: *agent.RetiredAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Step 4 — preflight counts. All three run even when force=true: we
|
||||
// need them to populate AgentRetirementResult.Counts (the pre-cascade
|
||||
// snapshot). A repo failure here aborts the whole operation — partial
|
||||
// preflight is worse than no preflight.
|
||||
counts, err := s.collectAgentDependencyCounts(ctx, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to collect agent dependency counts: %w", err)
|
||||
}
|
||||
|
||||
// Step 5 — force-reason guard. Positioned AFTER preflight so operators
|
||||
// who forgot --reason still see accurate counts when they retry. The
|
||||
// empty-reason rejection fires before any mutation, so the rejected
|
||||
// attempt leaves no audit noise.
|
||||
if force && reason == "" {
|
||||
return nil, ErrForceReasonRequired
|
||||
}
|
||||
|
||||
// Step 6 — default path: block on any non-zero bucket. Wrapping the
|
||||
// sentinel in *BlockedByDependenciesError lets the handler use errors.As
|
||||
// to surface counts in the 409 body while tests use errors.Is against
|
||||
// the sentinel. Both callers are satisfied by the single Unwrap chain.
|
||||
if !force && counts.HasDependencies() {
|
||||
return nil, &BlockedByDependenciesError{Counts: counts}
|
||||
}
|
||||
|
||||
// Step 7 — mutation. Pin retiredAt once so the audit event, the agent
|
||||
// row, and (on cascade) every deployment_targets row share the same
|
||||
// timestamp. Callers querying "what happened at T?" can correlate
|
||||
// retirement rows across tables without clock-skew tie-breaking.
|
||||
retiredAt := time.Now()
|
||||
cascade := force && counts.HasDependencies()
|
||||
|
||||
if cascade {
|
||||
if err := s.agentRepo.RetireAgentWithCascade(ctx, id, retiredAt, reason); err != nil {
|
||||
return nil, fmt.Errorf("failed to retire agent with cascade: %w", err)
|
||||
}
|
||||
} else {
|
||||
if err := s.agentRepo.SoftRetire(ctx, id, retiredAt, reason); err != nil {
|
||||
return nil, fmt.Errorf("failed to soft-retire agent: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 8 — audit. Two events on the cascade path so forensics can
|
||||
// distinguish "agent was retired" (agent_retired) from "downstream
|
||||
// targets were flipped" (agent_retirement_cascaded). Details on the
|
||||
// cascaded event carry the pre-cascade counts so a reviewer looking
|
||||
// only at the audit log knows how much state was affected. Emission
|
||||
// is best-effort — audit is observability, not a correctness barrier.
|
||||
actorType := s.resolveActorType(actor)
|
||||
details := map[string]interface{}{
|
||||
"actor": actor,
|
||||
"reason": reason,
|
||||
"force": force,
|
||||
"active_targets": counts.ActiveTargets,
|
||||
"active_certificates": counts.ActiveCertificates,
|
||||
"pending_jobs": counts.PendingJobs,
|
||||
}
|
||||
if err := s.auditService.RecordEvent(ctx, actor, actorType,
|
||||
"agent_retired", "agent", id, details); err != nil {
|
||||
slog.Error("failed to record agent_retired audit event", "agent_id", id, "error", err)
|
||||
}
|
||||
if cascade {
|
||||
cascadeDetails := map[string]interface{}{
|
||||
"actor": actor,
|
||||
"reason": reason,
|
||||
"active_targets": counts.ActiveTargets,
|
||||
"active_certificates": counts.ActiveCertificates,
|
||||
"pending_jobs": counts.PendingJobs,
|
||||
}
|
||||
if err := s.auditService.RecordEvent(ctx, actor, actorType,
|
||||
"agent_retirement_cascaded", "agent", id, cascadeDetails); err != nil {
|
||||
slog.Error("failed to record agent_retirement_cascaded audit event", "agent_id", id, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &AgentRetirementResult{
|
||||
AlreadyRetired: false,
|
||||
Cascade: cascade,
|
||||
RetiredAt: retiredAt,
|
||||
Counts: counts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ListRetiredAgents returns the paginated list of retired agents in
|
||||
// retired_at DESC order. This is the companion to ListAgents — which
|
||||
// hides retired rows — so the operator UI can render a dedicated
|
||||
// "Retired" tab without leaking retired rows into every other listing.
|
||||
// Pagination defaults (page<1→1, perPage<1→50) are applied here as
|
||||
// well as in the repo, so callers can pass 0s when they want defaults.
|
||||
//
|
||||
// Return shape harmonizes with handler.AgentService: a value slice
|
||||
// (not pointer slice) and int64 total. The repo returns []*domain.Agent;
|
||||
// this method dereferences into a value slice so the handler's
|
||||
// PagedResponse marshals straight objects and so the compile-time
|
||||
// interface assertion in agent_retire_handler_test.go:387 is satisfied.
|
||||
// Nil repo entries are skipped defensively — the repo should never
|
||||
// return them, but the handler contract is more important than the
|
||||
// repo's (pointer-slice) convenience.
|
||||
func (s *AgentService) ListRetiredAgents(ctx context.Context, page, perPage int) ([]domain.Agent, int64, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = 50
|
||||
}
|
||||
agents, total, err := s.agentRepo.ListRetired(ctx, page, perPage)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to list retired agents: %w", err)
|
||||
}
|
||||
out := make([]domain.Agent, 0, len(agents))
|
||||
for _, a := range agents {
|
||||
if a == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, *a)
|
||||
}
|
||||
return out, int64(total), nil
|
||||
}
|
||||
|
||||
// collectAgentDependencyCounts runs the three preflight COUNT queries in
|
||||
// sequence and bundles the result. Sequential (not parallel) because the
|
||||
// queries are cheap (<1ms each on the indexed columns added in 000015) and
|
||||
// sequential keeps error handling simple. Any repo error short-circuits
|
||||
// — we prefer to refuse the retire than make a half-informed decision.
|
||||
func (s *AgentService) collectAgentDependencyCounts(ctx context.Context, id string) (domain.AgentDependencyCounts, error) {
|
||||
var counts domain.AgentDependencyCounts
|
||||
|
||||
targets, err := s.agentRepo.CountActiveTargets(ctx, id)
|
||||
if err != nil {
|
||||
return counts, fmt.Errorf("count active targets: %w", err)
|
||||
}
|
||||
counts.ActiveTargets = targets
|
||||
|
||||
certs, err := s.agentRepo.CountActiveCertificates(ctx, id)
|
||||
if err != nil {
|
||||
return counts, fmt.Errorf("count active certificates: %w", err)
|
||||
}
|
||||
counts.ActiveCertificates = certs
|
||||
|
||||
jobs, err := s.agentRepo.CountPendingJobs(ctx, id)
|
||||
if err != nil {
|
||||
return counts, fmt.Errorf("count pending jobs: %w", err)
|
||||
}
|
||||
counts.PendingJobs = jobs
|
||||
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// resolveActorType maps an opaque actor string into the typed ActorType
|
||||
// used by the audit schema. Matches the conventions the rest of the
|
||||
// service layer uses: "system" → System, anything that looks like an
|
||||
// agent identity → Agent, everything else → User.
|
||||
func (s *AgentService) resolveActorType(actor string) domain.ActorType {
|
||||
switch {
|
||||
case actor == "system":
|
||||
return domain.ActorTypeSystem
|
||||
case len(actor) > 6 && actor[:6] == "agent-":
|
||||
return domain.ActorTypeAgent
|
||||
default:
|
||||
return domain.ActorTypeUser
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/shankar0123/certctl/internal/domain"
|
||||
)
|
||||
|
||||
// setupRetireTest wires up an AgentService with a single registered agent and
|
||||
// returns (service, agentRepo, auditRepo) so tests can seed state and assert
|
||||
// audit events. Kept minimal — tests that need targets/jobs/certs extend the
|
||||
// returned repos directly.
|
||||
func setupRetireTest(t *testing.T, agentID string) (*AgentService, *mockAgentRepo, *mockAuditRepo) {
|
||||
t.Helper()
|
||||
now := time.Now()
|
||||
agent := &domain.Agent{
|
||||
ID: agentID,
|
||||
Name: "prod-agent",
|
||||
Hostname: "server-01",
|
||||
Status: domain.AgentStatusOnline,
|
||||
RegisteredAt: now,
|
||||
LastHeartbeatAt: &now,
|
||||
APIKeyHash: "hash-" + agentID,
|
||||
}
|
||||
agentRepo := newMockAgentRepository()
|
||||
agentRepo.AddAgent(agent)
|
||||
certRepo := &mockCertRepo{
|
||||
Certs: make(map[string]*domain.ManagedCertificate),
|
||||
Versions: make(map[string][]*domain.CertificateVersion),
|
||||
}
|
||||
jobRepo := &mockJobRepo{
|
||||
Jobs: make(map[string]*domain.Job),
|
||||
StatusUpdates: make(map[string]domain.JobStatus),
|
||||
}
|
||||
targetRepo := &mockTargetRepo{
|
||||
Targets: make(map[string]*domain.DeploymentTarget),
|
||||
}
|
||||
auditRepo := &mockAuditRepo{Events: []*domain.AuditEvent{}}
|
||||
auditService := NewAuditService(auditRepo)
|
||||
issuerRegistry := NewIssuerRegistry(slog.Default())
|
||||
|
||||
svc := NewAgentService(agentRepo, certRepo, jobRepo, targetRepo, auditService, issuerRegistry, nil)
|
||||
return svc, agentRepo, auditRepo
|
||||
}
|
||||
|
||||
// TestRetireAgent_Sentinel_Rejected covers I-004's sentinel guard. The four
|
||||
// well-known sentinel agent IDs back discovery sources and the network scanner
|
||||
// — retiring them would orphan those subsystems. Contract: reject with
|
||||
// ErrAgentIsSentinel regardless of force/reason.
|
||||
func TestRetireAgent_Sentinel_Rejected(t *testing.T) {
|
||||
sentinels := []string{"server-scanner", "cloud-aws-sm", "cloud-azure-kv", "cloud-gcp-sm"}
|
||||
for _, id := range sentinels {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
svc, _, _ := setupRetireTest(t, id)
|
||||
_, err := svc.RetireAgent(context.Background(), id, "alice", false, "")
|
||||
if !errors.Is(err, ErrAgentIsSentinel) {
|
||||
t.Fatalf("retire(sentinel %q) err=%v want ErrAgentIsSentinel", id, err)
|
||||
}
|
||||
// Sentinel rejection must be deterministic even under force=true.
|
||||
_, err = svc.RetireAgent(context.Background(), id, "alice", true, "forced by operator")
|
||||
if !errors.Is(err, ErrAgentIsSentinel) {
|
||||
t.Fatalf("retire(sentinel %q force=true) err=%v want ErrAgentIsSentinel", id, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_NotFound covers the 404 preflight path. The handler maps
|
||||
// ErrAgentNotFound-equivalent sentinel to 404; the service must surface it
|
||||
// cleanly without partial state mutation.
|
||||
func TestRetireAgent_NotFound(t *testing.T) {
|
||||
svc, _, _ := setupRetireTest(t, "agent-001")
|
||||
_, err := svc.RetireAgent(context.Background(), "agent-does-not-exist", "alice", false, "")
|
||||
if err == nil {
|
||||
t.Fatalf("retire(missing id) err=nil want not-found error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_AlreadyRetired_Idempotent covers the 204 No Content path.
|
||||
// Retiring an already-retired agent must succeed without error and without
|
||||
// emitting a new audit event (the first retirement already recorded one).
|
||||
// Idempotency matters because the handler is the escape hatch for operators
|
||||
// re-issuing a failed retire after a partial failure mid-cascade.
|
||||
func TestRetireAgent_AlreadyRetired_Idempotent(t *testing.T) {
|
||||
svc, agentRepo, auditRepo := setupRetireTest(t, "agent-001")
|
||||
past := time.Now().Add(-24 * time.Hour)
|
||||
reason := "operator decommissioned"
|
||||
agent := agentRepo.Agents["agent-001"]
|
||||
agent.RetiredAt = &past
|
||||
agent.RetiredReason = &reason
|
||||
|
||||
result, err := svc.RetireAgent(context.Background(), "agent-001", "alice", false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("retire(already retired) err=%v want nil (idempotent)", err)
|
||||
}
|
||||
if result == nil || !result.AlreadyRetired {
|
||||
t.Fatalf("retire(already retired) result=%+v want AlreadyRetired=true", result)
|
||||
}
|
||||
// Retire-on-retired must not emit a duplicate audit event.
|
||||
for _, e := range auditRepo.Events {
|
||||
if e.Action == "agent_retired" && e.ResourceID == "agent-001" {
|
||||
t.Fatalf("retire(already retired) emitted duplicate agent_retired audit event")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_NoDeps_SoftSucceeds covers the happy 200 path: no active
|
||||
// targets, certs, or jobs referencing the agent. Soft-retire stamps
|
||||
// RetiredAt + RetiredReason and emits agent_retired audit event.
|
||||
func TestRetireAgent_NoDeps_SoftSucceeds(t *testing.T) {
|
||||
svc, agentRepo, auditRepo := setupRetireTest(t, "agent-001")
|
||||
|
||||
before := time.Now().Add(-time.Second)
|
||||
result, err := svc.RetireAgent(context.Background(), "agent-001", "alice", false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("retire(clean) err=%v want nil", err)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("retire(clean) result=nil want non-nil")
|
||||
}
|
||||
if result.AlreadyRetired {
|
||||
t.Fatalf("retire(clean) result.AlreadyRetired=true want false")
|
||||
}
|
||||
if result.Cascade {
|
||||
t.Fatalf("retire(clean) result.Cascade=true want false (no deps to cascade)")
|
||||
}
|
||||
if !result.RetiredAt.After(before) {
|
||||
t.Fatalf("retire(clean) RetiredAt=%v not after test start %v", result.RetiredAt, before)
|
||||
}
|
||||
|
||||
agent := agentRepo.Agents["agent-001"]
|
||||
if agent.RetiredAt == nil {
|
||||
t.Fatalf("retire(clean) agent.RetiredAt=nil want stamped")
|
||||
}
|
||||
|
||||
// Audit event must be emitted with action=agent_retired, actor=alice.
|
||||
found := false
|
||||
for _, e := range auditRepo.Events {
|
||||
if e.Action == "agent_retired" && e.ResourceID == "agent-001" && e.Actor == "alice" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("retire(clean) missing agent_retired audit event for alice, events=%+v", auditRepo.Events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_WithDeps_NoForce_Blocked covers the 409 preflight path. When
|
||||
// the agent has any of: active non-retired targets, certs deployed via those
|
||||
// targets, or pending jobs — a default retire must block with
|
||||
// ErrBlockedByDependencies and the counts must be reachable via errors.As so
|
||||
// the handler can build the 409 body.
|
||||
func TestRetireAgent_WithDeps_NoForce_Blocked(t *testing.T) {
|
||||
svc, agentRepo, _ := setupRetireTest(t, "agent-001")
|
||||
// Seed dependency counts directly on the mock — the production repo
|
||||
// implements CountActive* queries; the mock exposes them as fields.
|
||||
agentRepo.ActiveTargetCounts["agent-001"] = 3
|
||||
agentRepo.ActiveCertCounts["agent-001"] = 7
|
||||
agentRepo.PendingJobCounts["agent-001"] = 2
|
||||
|
||||
_, err := svc.RetireAgent(context.Background(), "agent-001", "alice", false, "")
|
||||
if !errors.Is(err, ErrBlockedByDependencies) {
|
||||
t.Fatalf("retire(with deps, no force) err=%v want ErrBlockedByDependencies", err)
|
||||
}
|
||||
var blocked *BlockedByDependenciesError
|
||||
if !errors.As(err, &blocked) {
|
||||
t.Fatalf("retire(with deps) err=%v want wrapped *BlockedByDependenciesError", err)
|
||||
}
|
||||
if blocked.Counts.ActiveTargets != 3 {
|
||||
t.Errorf("blocked.Counts.ActiveTargets=%d want 3", blocked.Counts.ActiveTargets)
|
||||
}
|
||||
if blocked.Counts.ActiveCertificates != 7 {
|
||||
t.Errorf("blocked.Counts.ActiveCertificates=%d want 7", blocked.Counts.ActiveCertificates)
|
||||
}
|
||||
if blocked.Counts.PendingJobs != 2 {
|
||||
t.Errorf("blocked.Counts.PendingJobs=%d want 2", blocked.Counts.PendingJobs)
|
||||
}
|
||||
// Agent must still be un-retired after preflight block.
|
||||
if agentRepo.Agents["agent-001"].RetiredAt != nil {
|
||||
t.Fatalf("retire(blocked) left RetiredAt stamped; preflight must be transactionally safe")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_WithDeps_Force_NoReason_Rejected covers the 400 guard on the
|
||||
// force escape hatch. Operators using force=true must supply a justifying
|
||||
// reason; empty reason is rejected before any DB mutation.
|
||||
func TestRetireAgent_WithDeps_Force_NoReason_Rejected(t *testing.T) {
|
||||
svc, agentRepo, _ := setupRetireTest(t, "agent-001")
|
||||
agentRepo.ActiveTargetCounts["agent-001"] = 1
|
||||
|
||||
_, err := svc.RetireAgent(context.Background(), "agent-001", "alice", true, "")
|
||||
if !errors.Is(err, ErrForceReasonRequired) {
|
||||
t.Fatalf("retire(force, no reason) err=%v want ErrForceReasonRequired", err)
|
||||
}
|
||||
if agentRepo.Agents["agent-001"].RetiredAt != nil {
|
||||
t.Fatalf("retire(force, no reason) left RetiredAt stamped; guard must fire before mutation")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_WithDeps_Force_Cascades covers the force=true transactional
|
||||
// path: agent retires, downstream targets also soft-retire with the supplied
|
||||
// reason, and the result surface indicates cascade happened. Reason
|
||||
// propagates to every cascaded row so post-mortem forensics can trace the
|
||||
// cascade to a single operator action.
|
||||
func TestRetireAgent_WithDeps_Force_Cascades(t *testing.T) {
|
||||
svc, agentRepo, auditRepo := setupRetireTest(t, "agent-001")
|
||||
agentRepo.ActiveTargetCounts["agent-001"] = 2
|
||||
agentRepo.ActiveCertCounts["agent-001"] = 5
|
||||
agentRepo.PendingJobCounts["agent-001"] = 1
|
||||
|
||||
reason := "decommissioning rack 7"
|
||||
result, err := svc.RetireAgent(context.Background(), "agent-001", "alice", true, reason)
|
||||
if err != nil {
|
||||
t.Fatalf("retire(force, reason) err=%v want nil", err)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("retire(force) result=nil want non-nil")
|
||||
}
|
||||
if !result.Cascade {
|
||||
t.Fatalf("retire(force) result.Cascade=false want true")
|
||||
}
|
||||
if result.Counts.ActiveTargets != 2 {
|
||||
t.Errorf("result.Counts.ActiveTargets=%d want 2 (pre-cascade snapshot)", result.Counts.ActiveTargets)
|
||||
}
|
||||
|
||||
agent := agentRepo.Agents["agent-001"]
|
||||
if agent.RetiredAt == nil {
|
||||
t.Fatalf("retire(force) agent.RetiredAt=nil want stamped")
|
||||
}
|
||||
if agent.RetiredReason == nil || *agent.RetiredReason != reason {
|
||||
t.Fatalf("retire(force) RetiredReason=%v want %q", agent.RetiredReason, reason)
|
||||
}
|
||||
|
||||
// Two audit events required: agent_retired + agent_retirement_cascaded.
|
||||
// The cascaded event captures which downstream resources were affected.
|
||||
var haveRetired, haveCascaded bool
|
||||
for _, e := range auditRepo.Events {
|
||||
if e.ResourceID == "agent-001" {
|
||||
switch e.Action {
|
||||
case "agent_retired":
|
||||
haveRetired = true
|
||||
case "agent_retirement_cascaded":
|
||||
haveCascaded = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !haveRetired {
|
||||
t.Errorf("retire(force) missing agent_retired audit event")
|
||||
}
|
||||
if !haveCascaded {
|
||||
t.Errorf("retire(force) missing agent_retirement_cascaded audit event")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireAgent_EmitsAuditEvent pins the audit contract for I-004:
|
||||
// every retire path that mutates DB state emits at least one audit event with
|
||||
// the operator's actor identity, so post-hoc compliance/forensics can
|
||||
// reconstruct who retired what and when.
|
||||
func TestRetireAgent_EmitsAuditEvent(t *testing.T) {
|
||||
svc, _, auditRepo := setupRetireTest(t, "agent-007")
|
||||
|
||||
_, err := svc.RetireAgent(context.Background(), "agent-007", "compliance-bot", false, "")
|
||||
if err != nil {
|
||||
t.Fatalf("retire err=%v want nil", err)
|
||||
}
|
||||
for _, e := range auditRepo.Events {
|
||||
if e.Action == "agent_retired" && e.ResourceID == "agent-007" {
|
||||
if e.Actor != "compliance-bot" {
|
||||
t.Errorf("audit event Actor=%q want compliance-bot", e.Actor)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no agent_retired audit event emitted, events=%+v", auditRepo.Events)
|
||||
}
|
||||
|
||||
// TestHeartbeat_RetiredAgent_ReturnsErrAgentRetired covers the 410 Gone
|
||||
// contract. A retired agent that is still polling must be told its identity
|
||||
// is no longer accepted — the agent process should detect this and shut
|
||||
// down rather than continue heartbeating indefinitely.
|
||||
func TestHeartbeat_RetiredAgent_ReturnsErrAgentRetired(t *testing.T) {
|
||||
svc, agentRepo, _ := setupRetireTest(t, "agent-001")
|
||||
past := time.Now().Add(-time.Hour)
|
||||
reason := "decommissioned"
|
||||
agentRepo.Agents["agent-001"].RetiredAt = &past
|
||||
agentRepo.Agents["agent-001"].RetiredReason = &reason
|
||||
|
||||
err := svc.Heartbeat(context.Background(), "agent-001", &domain.AgentMetadata{
|
||||
OS: "linux",
|
||||
Architecture: "amd64",
|
||||
Hostname: "server-01",
|
||||
})
|
||||
if !errors.Is(err, ErrAgentRetired) {
|
||||
t.Fatalf("heartbeat(retired) err=%v want ErrAgentRetired", err)
|
||||
}
|
||||
// Retired heartbeat must NOT bump LastHeartbeatAt — otherwise the retired
|
||||
// agent could ressurrect itself in stats/observability dashboards.
|
||||
if _, bumped := agentRepo.HeartbeatUpdates["agent-001"]; bumped {
|
||||
t.Fatalf("heartbeat(retired) updated LastHeartbeatAt; retired agents must be frozen")
|
||||
}
|
||||
}
|
||||
|
||||
// TestListAgents_DefaultExcludesRetired covers the contract that the
|
||||
// handler-facing ListAgents call hides retired rows by default. Otherwise
|
||||
// every dashboard that paginates agents would surface retired stragglers.
|
||||
// An explicit "list retired" endpoint (ListRetiredAgents) covers the audit
|
||||
// use case.
|
||||
func TestListAgents_DefaultExcludesRetired(t *testing.T) {
|
||||
svc, agentRepo, _ := setupRetireTest(t, "agent-active")
|
||||
// Seed one retired agent alongside the active one.
|
||||
past := time.Now().Add(-24 * time.Hour)
|
||||
reason := "old hardware"
|
||||
agentRepo.AddAgent(&domain.Agent{
|
||||
ID: "agent-retired",
|
||||
Name: "retired-agent",
|
||||
Hostname: "server-old",
|
||||
Status: domain.AgentStatusOffline,
|
||||
RegisteredAt: past,
|
||||
APIKeyHash: "hash-retired",
|
||||
RetiredAt: &past,
|
||||
RetiredReason: &reason,
|
||||
})
|
||||
|
||||
agents, total, err := svc.ListAgents(context.Background(), 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("ListAgents err=%v want nil", err)
|
||||
}
|
||||
for _, a := range agents {
|
||||
if a.ID == "agent-retired" {
|
||||
t.Fatalf("ListAgents returned retired agent %q in default listing", a.ID)
|
||||
}
|
||||
}
|
||||
if total != 1 {
|
||||
t.Errorf("ListAgents total=%d want 1 (only active)", total)
|
||||
}
|
||||
|
||||
// ListRetiredAgents must surface retired-only, with count=1.
|
||||
retired, retiredTotal, err := svc.ListRetiredAgents(context.Background(), 1, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("ListRetiredAgents err=%v want nil", err)
|
||||
}
|
||||
if retiredTotal != 1 {
|
||||
t.Errorf("ListRetiredAgents total=%d want 1", retiredTotal)
|
||||
}
|
||||
if len(retired) != 1 || retired[0].ID != "agent-retired" {
|
||||
t.Fatalf("ListRetiredAgents got=%+v want [agent-retired]", retired)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarkStaleAgentsOffline_SkipsRetired covers the stale-offline sweeper
|
||||
// interaction with retirement. A retired agent must not be re-surfaced as
|
||||
// a state transition ("Online → Offline") by the scheduler, because its
|
||||
// Status column is preserved as the last-known operational state at
|
||||
// retirement time and RetiredAt is the source of truth for filtering.
|
||||
func TestMarkStaleAgentsOffline_SkipsRetired(t *testing.T) {
|
||||
svc, agentRepo, _ := setupRetireTest(t, "agent-live")
|
||||
// Active agent is currently stale (no heartbeat for 10 minutes) — eligible
|
||||
// for Online→Offline transition.
|
||||
stale := time.Now().Add(-10 * time.Minute)
|
||||
agentRepo.Agents["agent-live"].LastHeartbeatAt = &stale
|
||||
|
||||
// Retired agent was also stale at retirement time, but must NOT be
|
||||
// touched by the sweeper.
|
||||
past := time.Now().Add(-24 * time.Hour)
|
||||
reason := "hw failure"
|
||||
agentRepo.AddAgent(&domain.Agent{
|
||||
ID: "agent-retired",
|
||||
Name: "dead-agent",
|
||||
Hostname: "server-old",
|
||||
Status: domain.AgentStatusOnline, // preserved last-seen status
|
||||
RegisteredAt: past,
|
||||
LastHeartbeatAt: &past,
|
||||
APIKeyHash: "hash-dead",
|
||||
RetiredAt: &past,
|
||||
RetiredReason: &reason,
|
||||
})
|
||||
|
||||
if err := svc.MarkStaleAgentsOffline(context.Background(), 5*time.Minute); err != nil {
|
||||
t.Fatalf("MarkStaleAgentsOffline err=%v want nil", err)
|
||||
}
|
||||
|
||||
// Active-stale agent should flip Online → Offline.
|
||||
if got := agentRepo.Agents["agent-live"].Status; got != domain.AgentStatusOffline {
|
||||
t.Errorf("agent-live Status=%s want Offline", got)
|
||||
}
|
||||
// Retired agent's Status column must be frozen at Online (its preserved
|
||||
// last-seen state); the sweeper must skip it.
|
||||
if got := agentRepo.Agents["agent-retired"].Status; got != domain.AgentStatusOnline {
|
||||
t.Errorf("agent-retired Status=%s want Online (frozen); sweeper touched retired row", got)
|
||||
}
|
||||
}
|
||||
@@ -145,6 +145,31 @@ func (s *DeploymentService) ProcessDeploymentJob(ctx context.Context, job *domai
|
||||
return fmt.Errorf("failed to fetch agent: %w", err)
|
||||
}
|
||||
|
||||
// I-004: AgentRepository.Get surfaces retired rows by design (for the GUI
|
||||
// banner + 410 Gone heartbeat path). Deployments must never dispatch to a
|
||||
// retired agent — it will never heartbeat again and the target row should
|
||||
// itself have been cascade-retired when the agent was force-retired. A job
|
||||
// slipping through here would otherwise hit the heartbeat-staleness branch
|
||||
// below with the misleading reason "agent is offline"; we want operators to
|
||||
// see the real cause. Fail the job with an explicit reason, send a
|
||||
// deployment notification so the owner is alerted, and record an audit
|
||||
// event. Falls through the same notify+audit shape as the offline branch.
|
||||
if agent.IsRetired() {
|
||||
updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, "assigned agent is retired")
|
||||
if updateErr != nil {
|
||||
slog.Error("failed to update job status", "job_id", job.ID, "error", updateErr)
|
||||
}
|
||||
if notifErr := s.notificationSvc.SendDeploymentNotification(ctx, cert, target, false, fmt.Errorf("agent retired")); notifErr != nil {
|
||||
slog.Error("failed to send deployment notification", "error", notifErr)
|
||||
}
|
||||
if auditErr := s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
|
||||
"deployment_job_failed", "certificate", job.CertificateID,
|
||||
map[string]interface{}{"job_id": job.ID, "reason": "agent retired", "target_id": targetID, "agent_id": agentID}); auditErr != nil {
|
||||
slog.Error("failed to record audit event", "error", auditErr)
|
||||
}
|
||||
return fmt.Errorf("agent %s is retired", agentID)
|
||||
}
|
||||
|
||||
// Check agent heartbeat (must be within last 5 minutes)
|
||||
if agent.LastHeartbeatAt != nil && time.Since(*agent.LastHeartbeatAt) > 5*time.Minute {
|
||||
updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, "agent is offline")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"time"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -236,6 +237,81 @@ func (s *JobService) RetryFailedJobs(ctx context.Context, maxRetries int) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReapTimedOutJobs transitions jobs stuck in AwaitingCSR or AwaitingApproval
|
||||
// to Failed if they've exceeded their TTL. I-001's retry loop then auto-promotes
|
||||
// eligible Failed jobs back to Pending (closes coverage gap I-003).
|
||||
func (s *JobService) ReapTimedOutJobs(ctx context.Context, csrTTL, approvalTTL time.Duration) error {
|
||||
s.logger.Debug("reaping timed-out jobs", "csr_ttl", csrTTL, "approval_ttl", approvalTTL)
|
||||
|
||||
now := time.Now()
|
||||
csrCutoff := now.Add(-csrTTL)
|
||||
approvalCutoff := now.Add(-approvalTTL)
|
||||
|
||||
timedOutJobs, err := s.jobRepo.ListTimedOutAwaitingJobs(ctx, csrCutoff, approvalCutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch timed-out jobs: %w", err)
|
||||
}
|
||||
|
||||
var reaped int
|
||||
|
||||
for _, job := range timedOutJobs {
|
||||
oldStatus := job.Status
|
||||
var (
|
||||
newErrMsg string
|
||||
reason string
|
||||
ttl time.Duration
|
||||
)
|
||||
switch job.Status {
|
||||
case domain.JobStatusAwaitingCSR:
|
||||
ttl = csrTTL
|
||||
reason = "csr_timeout"
|
||||
newErrMsg = fmt.Sprintf("timed out in %s after %s", oldStatus, csrTTL)
|
||||
case domain.JobStatusAwaitingApproval:
|
||||
ttl = approvalTTL
|
||||
reason = "approval_timeout"
|
||||
newErrMsg = fmt.Sprintf("timed out in %s after %s", oldStatus, approvalTTL)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
_ = ttl
|
||||
|
||||
job.Status = domain.JobStatusFailed
|
||||
job.LastError = &newErrMsg
|
||||
|
||||
if err := s.jobRepo.Update(ctx, job); err != nil {
|
||||
s.logger.Error("failed to transition timed-out job",
|
||||
"job_id", job.ID,
|
||||
"old_status", oldStatus,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if s.auditService != nil {
|
||||
ageHours := time.Since(job.CreatedAt).Hours()
|
||||
if auditErr := s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
|
||||
"job_timeout", "job", job.ID,
|
||||
map[string]interface{}{
|
||||
"old_status": string(oldStatus),
|
||||
"new_status": string(domain.JobStatusFailed),
|
||||
"timeout_reason": reason,
|
||||
"age_hours": ageHours,
|
||||
}); auditErr != nil {
|
||||
s.logger.Error("failed to record job timeout audit event",
|
||||
"job_id", job.ID,
|
||||
"error", auditErr)
|
||||
}
|
||||
}
|
||||
|
||||
reaped++
|
||||
}
|
||||
|
||||
s.logger.Info("job timeout reaper completed",
|
||||
"reaped", reaped,
|
||||
"total_timed_out", len(timedOutJobs))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJobStatus returns the current status of a job.
|
||||
func (s *JobService) GetJobStatus(ctx context.Context, jobID string) (*domain.Job, error) {
|
||||
job, err := s.jobRepo.Get(ctx, jobID)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user