diff --git a/cmd/spinifex/cmd/admin_acm.go b/cmd/spinifex/cmd/admin_acm.go new file mode 100644 index 000000000..4ace13f00 --- /dev/null +++ b/cmd/spinifex/cmd/admin_acm.go @@ -0,0 +1,170 @@ +package cmd + +import ( + "bufio" + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/mulgadc/spinifex/spinifex/admin" + handlers_acm "github.com/mulgadc/spinifex/spinifex/handlers/acm" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/spf13/cobra" +) + +var certCreateTenantCACmd = &cobra.Command{ + Use: "create-tenant-ca", + Short: "Create the ACM tenant private CA for PRIVATE_CA certificate issuance", + Long: `Creates the tenant root CA that ACM's PRIVATE_CA validation mode signs leaf +certificates from. This is a separate, independent root from the platform CA +(ca.pem/ca.key generated by 'spx admin init') — it is never used for node-to-node +trust, ipsec, or system VM guests, and installing it into device trust stores +is a one-time, per-deployment action. + +--domain is required and repeatable. The domains given here are baked into the +CA as x509 name constraints and are frozen at creation: PRIVATE_CA certificate +requests are only honoured for these domains (and their subdomains), and +widening the list later requires regenerating the root. + +Running this command against an already-existing tenant CA reports what is +already there rather than silently regenerating — rotating the root invalidates +every device's trust store at once, so that can only ever be an explicit, +operator-driven action via --regenerate.`, + Run: runCertCreateTenantCA, +} + +// certForceRenewCmd reissues a PRIVATE_CA certificate immediately, over NATS +// against the running daemon's renewal worker — distinct from certRenewCmd +// ("spx admin cert renew"), which regenerates the node's own server TLS +// certificate offline and is unrelated to ACM. +var certForceRenewCmd = &cobra.Command{ + Use: "force-renew", + Short: "Reissue an ACM PRIVATE_CA certificate immediately, bypassing its renewal window", + Long: `Reissues the given ACM certificate under its existing ARN right now, instead of +waiting for the renewal worker's proportional window to elapse. Only PRIVATE_CA +certificates (tenant-CA-issued) can be force-renewed; imported and public +certificates are refused. + +This talks to the running daemon over NATS, taking the same per-certificate +lease the scheduled renewal worker uses, so it is safe to run against a live +deployment. It exists primarily to verify the ELBv2 fan-out — that every load +balancer referencing the certificate re-renders with the new leaf — without +waiting out the real renewal window.`, + Run: runCertForceRenew, +} + +func init() { + certCmd.AddCommand(certCreateTenantCACmd) + certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)") + certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)") + certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt") + if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil { + panic(err) + } + + certCmd.AddCommand(certForceRenewCmd) + certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)") + certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account") + if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil { + panic(err) + } +} + +func runCertForceRenew(cmd *cobra.Command, _ []string) { + certArn, _ := cmd.Flags().GetString("arn") + accountID, _ := cmd.Flags().GetString("account") + if accountID == "" { + accountID = admin.DefaultAccountID() + } + + _, nc, err := loadConfigAndConnect() + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + defer nc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + fmt.Printf("Force-renewing %s (account %s)...\n", certArn, accountID) + out, err := utils.NATSRequest[handlers_acm.ForceRenewCertificateOutput]( + ctx, nc, "acm.ForceRenewCertificate", + &handlers_acm.ForceRenewCertificateInput{CertificateArn: certArn}, + 30*time.Second, accountID, + ) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Reissued %s\n", out.CertificateArn) + fmt.Printf(" Serial: %s\n", out.Serial) + fmt.Printf(" NotAfter: %s\n", out.NotAfter.Format(time.RFC3339)) +} + +func runCertCreateTenantCA(cmd *cobra.Command, _ []string) { + domains, _ := cmd.Flags().GetStringSlice("domain") + regenerate, _ := cmd.Flags().GetBool("regenerate") + yes, _ := cmd.Flags().GetBool("yes") + + configDir, _ := cmd.Root().Flags().GetString("config-dir") + certPath := admin.TenantCACertPath(configDir) + keyPath := admin.TenantCAKeyPath(configDir) + + existed := admin.FileExists(certPath) && admin.FileExists(keyPath) + + if existed && regenerate { + fmt.Println("⚠️ About to regenerate the tenant CA. This invalidates trust for every") + fmt.Println(" device that has installed the current root — each one will need the new") + fmt.Println(" certificate reinstalled before it trusts PRIVATE_CA leaves again.") + fmt.Printf(" Current CA: %s\n", certPath) + fmt.Printf(" New domains: %s\n", strings.Join(domains, ", ")) + fmt.Println() + if !yes { + fmt.Print("Type 'yes' to proceed: ") + reader := bufio.NewReader(os.Stdin) + answer, _ := reader.ReadString('\n') + if strings.TrimSpace(answer) != "yes" { + fmt.Println("Aborted.") + return + } + } + // Regeneration is a deliberate, explicit action only reachable via + // --regenerate (plus confirmation) — LoadOrCreateTenantCA itself never + // regenerates over an existing root, so the old files are removed first + // to put it back on the "create" path. + if err := os.Remove(certPath); err != nil { + fmt.Fprintf(os.Stderr, "Error removing existing tenant CA cert: %v\n", err) + os.Exit(1) + } + if err := os.Remove(keyPath); err != nil { + fmt.Fprintf(os.Stderr, "Error removing existing tenant CA key: %v\n", err) + os.Exit(1) + } + } else if existed { + fmt.Println("Tenant CA already exists — reporting current state (use --regenerate to replace it):") + } + + ca, err := handlers_acm.LoadOrCreateTenantCA(certPath, keyPath, domains) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + if existed && !regenerate { + fmt.Println("✅ No changes made.") + } else { + fmt.Println("✅ Tenant CA ready.") + } + fmt.Printf(" Certificate: %s\n", certPath) + fmt.Printf(" Key: %s (0600)\n", keyPath) + fmt.Printf(" Permitted domains: %s\n", strings.Join(ca.PermittedDomains(), ", ")) + fmt.Println() + fmt.Println("Install the certificate into every client's trust store so PRIVATE_CA") + fmt.Println("certificates are trusted. The tenant CA is only loaded at daemon startup, so") + fmt.Println("restart spinifex-daemon on any node that started before this CA existed:") + fmt.Println(" sudo systemctl restart spinifex-daemon") +} diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 5666f0cf6..9d198e949 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -79,6 +79,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Description | |---------|-------|-------------| | `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | +| `spx admin cert create-tenant-ca` | `--domain` (required, repeatable — permitted domains baked into the CA's name constraints), `--regenerate` (replace an existing tenant CA, requires confirmation), `--yes` (skip the `--regenerate` confirmation prompt) | Creates the tenant root CA that ACM's `PRIVATE_CA` validation mode signs leaf certificates from — a separate, independent root from the platform CA. Idempotent: running it again against an existing tenant CA reports the current state rather than regenerating; `--regenerate` is required to replace the root, which invalidates every device's existing trust. | ### Upgrade Management @@ -862,7 +863,32 @@ The default certificate cannot be added/removed via these calls — set it on th ## ACM (AWS Certificate Manager) -Import-only — Spinifex stores externally-issued certificates for ELBv2 listener references; it does not issue certificates or validate domains (`RequestCertificate`). Certs are account-scoped; `describe`/`delete` enforce ownership. +Spinifex both stores externally-issued certificates (`import-certificate`) and +issues its own (`request-certificate`) for ELBv2 listener references. Certs +are account-scoped; `describe`/`delete` enforce ownership, and +`delete-certificate` refuses with `ResourceInUseException` while any load +balancer listener still references the ARN — no force flag, matching AWS. + +`request-certificate` mints a `CertificateArn` immediately and returns +`PENDING_VALIDATION`; it never issues inline. The validation mode is derived +from deployment state, never configured: + +| Mode | Who writes the DNS record | `ResourceRecord` returned | Renewal | Status | +| --- | --- | --- | --- | --- | +| `PROVIDER_API` | Spinifex, via the operator's DNS provider API | none | automatic | request accepted, `PENDING_VALIDATION` — the DNS-01 order is driven by a later worker | +| `MANUAL_TXT` | the operator, by hand or Terraform | TXT, rotates per order | manual — `INELIGIBLE` | request accepted, `PENDING_VALIDATION` — same as above | +| `CNAME_DELEGATION` | operator once, then Spinifex | CNAME, stable | automatic | deferred — never selected yet (northstar cannot serve public authoritative queries); an ARN-stable delegation token is minted on every managed certificate now so this lands as a non-breaking addition | +| `PRIVATE_CA` | nobody — no validation | none | automatic | **DONE** — issues synchronously against the tenant CA, no domain outside its name constraints | + +`PROVIDER_API` is selected when a DNS provider credential is configured; +`MANUAL_TXT` when northstar hosts the zone; otherwise `PRIVATE_CA` — the only +option for a deployment with no real, publicly delegated domain. Terraform's +canonical `aws_acm_certificate` → `aws_route53_record` → +`aws_acm_certificate_validation` → `aws_lb_listener` flow works unmodified in +every mode: where Spinifex owns the record write, no `ResourceRecord` is +emitted, so `for_each` over `domain_validation_options` yields zero records +and `aws_acm_certificate_validation` still blocks correctly by polling until +`ISSUED`. | Command | Implemented Flags | Missing Flags | Status | |---------|-------------------|---------------|--------| @@ -870,8 +896,8 @@ Import-only — Spinifex stores externally-issued certificates for ELBv2 listene | `describe-certificate` | `--certificate-arn` | — | **DONE** | | `list-certificates` | — | `--certificate-statuses`, `--includes`, `--max-items`, `--next-token` | **DONE** | | `delete-certificate` | `--certificate-arn` | — | **DONE** | -| `request-certificate` | — | `--domain-name`, `--validation-method`, `--subject-alternative-names`, `--tags` | **NOT STARTED** | -| `add-tags-to-certificate` / `list-tags-for-certificate` / `remove-tags-from-certificate` | — | `--certificate-arn`, `--tags`/`--tag-keys` | **NOT STARTED** | +| `request-certificate` | `--domain-name`, `--subject-alternative-names`, `--tags` | `--validation-method`, `--certificate-authority-arn`, `--options`, `--idempotency-token` | **PARTIAL** — `PRIVATE_CA` issues synchronously; `PROVIDER_API`/`MANUAL_TXT` are accepted and correctly shaped but stay `PENDING_VALIDATION` until a later issuance worker lands | +| `add-tags-to-certificate` / `list-tags-for-certificate` / `remove-tags-from-certificate` | `--certificate-arn`, `--tags`/`--tag-keys` | — | **DONE** | | `export-certificate` | — | `--certificate-arn`, `--passphrase` | **NOT STARTED** | --- diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index ee3af974b..43bdd21f9 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -257,6 +257,32 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, a return caCertPath } +// tenantCACertFilename and tenantCAKeyFilename name the ACM tenant private +// CA's files on disk. Colocated with the platform CA (ca.pem/ca.key) and +// master.key under the same config directory, but a distinct "tenant-ca" +// basename keeps the two roots from ever being confused: they are +// independent, and the platform CA (trusted by every node, ipsec, EKS/ECS +// guests, ALB microvms and baked catalog images) must never be read, written +// or reused as the tenant CA, or vice versa. +const ( + tenantCACertFilename = "tenant-ca.pem" + tenantCAKeyFilename = "tenant-ca.key" +) + +// TenantCACertPath and TenantCAKeyPath return the on-disk paths for the ACM +// tenant private CA under configDir, mirroring how GenerateCertificatesIfNeeded +// derives the platform CA's paths (filepath.Join(configDir, "ca.pem"/"ca.key")) +// rather than threading them through ConfigSettings: like the platform CA, +// these paths are computed on demand by whoever needs them (the tenant-CA admin +// command, the daemon at startup), not rendered into a config file. +func TenantCACertPath(configDir string) string { + return filepath.Join(configDir, tenantCACertFilename) +} + +func TenantCAKeyPath(configDir string) string { + return filepath.Join(configDir, tenantCAKeyFilename) +} + // GenerateServerCertOnly generates a server certificate signed by an existing CA. // Used by joining nodes that receive the CA from the leader. awsRegion and // internalSuffix add the AWS-parity ECR SANs; empty values omit them. diff --git a/spinifex/awserrors/awserrors.go b/spinifex/awserrors/awserrors.go index 7036e211a..b0ecd7424 100644 --- a/spinifex/awserrors/awserrors.go +++ b/spinifex/awserrors/awserrors.go @@ -378,6 +378,20 @@ var ( ErrorResourceNotFound = "ResourceNotFound" ErrorEKSResourceInUse = "ResourceInUseException" ErrorEKSResourceNotFound = "ResourceNotFoundException" + // ErrorACMResourceInUse aliases ErrorEKSResourceInUse: real ACM's + // DeleteCertificate and real EKS's CreateCluster both legitimately use the + // wire code "ResourceInUseException", so this is a distinct, self-documenting + // name for ACM call sites rather than a second ErrorLookup entry — ErrorLookup + // is keyed by wire code across every service, so it can only hold one message + // per code. Naming it ErrorACMResourceInUse stops an ACM call site reading + // "EKS" while making no claim that the returned message text is ACM-flavored; + // it is still EKS's "cluster already exists" wording until ErrorLookup (or its + // caller in gateway.ErrorHandler) is keyed per-service instead of globally. + ErrorACMResourceInUse = ErrorEKSResourceInUse + // ErrorACMRequestInProgress is what GetCertificate returns for a certificate + // that exists but has not been issued yet (still PENDING_VALIDATION), so a + // client polling for the body can tell "not ready" apart from "not mine". + ErrorACMRequestInProgress = "RequestInProgressException" // ECS JSON-1.1 exception codes. ECS clients key on the "Exception"-suffixed // __type; do not cross-wire with the EC2/ACM not-found codes above. ErrorECSClusterNotFound = "ClusterNotFoundException" @@ -810,6 +824,7 @@ var ErrorLookup = map[string]ErrorMessage{ ErrorInvalidNatGatewayIDNotFound: {HTTPCode: 404, Message: "The specified NAT gateway ID does not exist. Ensure that you specify the AWS Region in which the NAT gateway is located, if it's not in the default Region."}, ErrorInvalidNetworkAclEntryNotFound: {HTTPCode: 404, Message: "The specified network ACL entry does not exist."}, ErrorACMInvalidArn: {HTTPCode: 400, Message: "The requested Amazon Resource Name (ARN) does not refer to an existing resource."}, + ErrorACMRequestInProgress: {HTTPCode: 400, Message: "The certificate request is in process and the certificate in your account is not yet available."}, ErrorInvalidNetworkAclIDNotFound: {HTTPCode: 404, Message: "The specified network ACL does not exist. Ensure that you specify the AWS Region in which the network ACL is located, if it's not in the default Region."}, ErrorInvalidNetworkAclIdMalformed: {HTTPCode: 400, Message: "The specified network ACL ID is malformed. Ensure that you provide the ID in the form acl-xxxxxxxxxxxxxxxxx."}, ErrorInvalidNetworkInterfaceInUse: {HTTPCode: 409, Message: "The specified interface is currently in use and cannot be deleted or attached to another instance. Ensure that you have detached the network interface first. If a network interface is in use, you may also receive the InvalidParameterValue error."}, diff --git a/spinifex/config/config.go b/spinifex/config/config.go index d2cd0120c..3369f861e 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -104,6 +104,7 @@ type Config struct { VPCD VPCDConfig `json:"VPCD" mapstructure:"vpcd"` Northstar NorthstarConfig `json:"Northstar" mapstructure:"northstar"` RDS RDSConfig `json:"RDS" mapstructure:"rds"` + ACM ACMConfig `json:"ACM" mapstructure:"acm"` BaseDir string `json:"BaseDir" mapstructure:"base_dir"` WalDir string `json:"WalDir" mapstructure:"wal_dir"` @@ -200,6 +201,26 @@ type RDSConfig struct { // supernet, so a name-hash collision can never place an RDS subnet in EKS space. const RDSDefaultSystemVPCSupernet = "10.248.0.0/14" +// ACMConfig holds the operator-level ACM certificate-issuance configuration. +// Deliberately small: four keys, nothing deployment-specific and nothing +// derivable. In particular there is no allowed-domains list (public modes are +// authorized by the ACME CA's own validation, PRIVATE_CA by the CA's own name +// constraints), no default validation mode (derived from DNSProvider and +// whether northstar hosts the zone, never configured), no renewal thresholds +// (proportional, hence constants) and no private CA paths (folded into the +// existing CA path handling in admin.ConfigSettings). +type ACMConfig struct { + Enabled bool `json:"Enabled" mapstructure:"enabled"` + DirectoryURL string `json:"DirectoryURL" mapstructure:"directory_url"` + ContactEmail string `json:"ContactEmail" mapstructure:"contact_email"` + // DNSProvider is the lego DNS provider id used for PROVIDER_API DNS-01 + // challenges. Its credentials come from environment variables per lego's + // own convention, never from this file — a provider token in a + // configuration file is a token in every backup. A non-empty value here is + // what selects PROVIDER_API at RequestCertificate time. + DNSProvider string `json:"DNSProvider" mapstructure:"dns_provider"` +} + // ParseEndpoints splits a comma-separated OVSDB endpoint list (NB/SB RAFT // cluster) into individual endpoints, trimming whitespace and dropping empties. // A single endpoint yields a one-element slice; empty input yields nil. Both the diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index ff13f341c..cbf30dea8 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -152,6 +152,7 @@ type Daemon struct { rdsService *handlers_rds.Service rdsReconciler *handlers_rds.Reconciler acmService *handlers_acm.ACMServiceImpl + acmRenewalWorker *handlers_acm.Worker ecrMetaService *handlers_ecr.MetaServiceImpl routeTableService *handlers_ec2_routetable.RouteTableServiceImpl natGatewayService *handlers_ec2_natgw.NatGatewayServiceImpl @@ -1083,16 +1084,24 @@ func (d *Daemon) subscribeAll() error { ) } - // ACM gateway → daemon subscriptions (minimal certificate store). + // ACM gateway → daemon subscriptions (certificate import and managed issuance). if d.acmService != nil { subs = append(subs, natsSub{"acm.ImportCertificate", handleNATSRequest(d.acmService.ImportCertificate), "spinifex-workers"}, + natsSub{"acm.RequestCertificate", handleNATSRequest(d.acmService.RequestCertificate), "spinifex-workers"}, natsSub{"acm.DescribeCertificate", handleNATSRequest(d.acmService.DescribeCertificate), "spinifex-workers"}, + natsSub{"acm.GetCertificate", handleNATSRequest(d.acmService.GetCertificate), "spinifex-workers"}, natsSub{"acm.ListCertificates", handleNATSRequest(d.acmService.ListCertificates), "spinifex-workers"}, natsSub{"acm.DeleteCertificate", handleNATSRequest(d.acmService.DeleteCertificate), "spinifex-workers"}, natsSub{"acm.ListTagsForCertificate", handleNATSRequest(d.acmService.ListTagsForCertificate), "spinifex-workers"}, natsSub{"acm.AddTagsToCertificate", handleNATSRequest(d.acmService.AddTagsToCertificate), "spinifex-workers"}, natsSub{"acm.RemoveTagsFromCertificate", handleNATSRequest(d.acmService.RemoveTagsFromCertificate), "spinifex-workers"}, + // ForceRenewCertificate is not an AWS API — it is the operator- + // triggered path `spx admin cert force-renew` uses to reissue a + // PRIVATE_CA certificate immediately, bypassing the renewal worker's + // window and failure backoff, primarily to verify the ELBv2 fan-out + // without waiting out the proportional window. + natsSub{"acm.ForceRenewCertificate", handleNATSRequest(d.acmRenewalWorker.ForceRenewCertificate), "spinifex-workers"}, ) } @@ -1551,12 +1560,31 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("failed to initialize account settings service: %w", err) } + // ELBv2 resolves listener certificate private keys through its own ACM + // Store over the same JetStream bucket the ACM service writes, so it needs + // the identical master key: unlike EKS/ECS's IAM dependency, there is no + // safe degraded mode for certificate private keys, so a missing key must + // fail daemon startup rather than leave HTTPS listeners uncreatable. d.elbv2Service, err = initServiceWithRetry("ELBv2 service", func() (*handlers_elbv2.ELBv2ServiceImpl, error) { - return handlers_elbv2.NewELBv2ServiceImplWithNATS(d.config, d.natsConn) + masterKey, mkErr := handlers_iam.LoadMasterKey(filepath.Join(filepath.Dir(d.configPath), "master.key")) + if mkErr != nil { + return nil, fmt.Errorf("load ELBv2 master key: %w", mkErr) + } + return handlers_elbv2.NewELBv2ServiceImplWithNATS(d.config, d.natsConn, masterKey) }) if err != nil { return fmt.Errorf("failed to initialize ELBv2 service: %w", err) } + + // The InUseBy index is otherwise only written by listener mutations, so + // listeners created before it existed would never fan out a renewed + // certificate. Not fatal: a stale index degrades renewal, which beats + // refusing to start. Every node reconciles the shared bucket at its own + // startup; the index writes are CAS-guarded, so concurrent runs converge. + if err := d.elbv2Service.ReconcileCertInUseIndex(d.ctx); err != nil { + slog.Error("failed to reconcile the ACM InUseBy index", "err", err) + } + if d.vpcService != nil { d.elbv2Service.VPCService = d.vpcService } @@ -1630,12 +1658,73 @@ func (d *Daemon) startCluster() error { d.rdsReconciler.Run(d.ctx) }) + // ACM certificates hold private keys, so unlike EKS/ECS's IAM dependency + // (which degrades to "feature disabled" without a master key) there is no + // safe degraded mode here: a missing key must fail daemon startup, not + // silently persist keys unencrypted. initServiceWithRetry still retries with + // backoff — the key file can legitimately not be written yet during a + // concurrent boot — but a master key that never arrives fails startCluster + // after the retry window instead of leaving acmService permanently nil. d.acmService, err = initServiceWithRetry("ACM service", func() (*handlers_acm.ACMServiceImpl, error) { - return handlers_acm.NewACMServiceImplWithNATS(d.ctx, d.config, d.natsConn) + masterKey, mkErr := handlers_iam.LoadMasterKey(filepath.Join(filepath.Dir(d.configPath), "master.key")) + if mkErr != nil { + return nil, fmt.Errorf("load ACM master key: %w", mkErr) + } + return handlers_acm.NewACMServiceImplWithNATS(d.ctx, d.config, d.natsConn, masterKey) }) if err != nil { return fmt.Errorf("failed to initialize ACM service: %w", err) } + // Re-importing to an existing ARN must fan out to every load balancer that + // references it; otherwise HAProxy keeps serving the old leaf until it + // expires while the API reports success. ELBv2 is already up (above). + d.acmService.CertMaterialUpdated = d.elbv2Service.UpdateStoredConfigForCert + + // deriveValidationMode needs to know whether northstar hosts a zone for a + // requested domain. handlers/acm deliberately does not import handlers/dns + // (which would pull its S3/northstar-config dependencies into every acm + // test) — wired as a func field instead, the same pattern CertMaterialUpdated + // uses above to keep acm decoupled from elbv2. + d.acmService.NorthstarHostsZone = func(domain string) bool { + return handlers_dns.HostsZone(d.config, domain) + } + + // The tenant private CA is optional, unlike the master key above: a + // deployment issuing only public (PROVIDER_API/MANUAL_TXT) certificates has + // no reason to have one, so its absence must not block daemon startup. + // LoadTenantCA (load-only — see privateca.go) leaves TenantCA nil rather + // than creating a root here, because creation needs an explicit permitted- + // domains list that has no safe default; a PRIVATE_CA RequestCertificate + // against a nil TenantCA already fails loudly with an actionable error + // (see ACMServiceImpl.RequestCertificate). Both states are logged here so an + // operator can tell at a glance, from daemon startup logs alone, which one + // they are in. + configDir := filepath.Dir(d.configPath) + tenantCA, tenantCAErr := handlers_acm.LoadTenantCA(admin.TenantCACertPath(configDir), admin.TenantCAKeyPath(configDir)) + if tenantCAErr != nil { + slog.Warn("ACM: tenant private CA not found; PRIVATE_CA certificate requests will fail until one is created", + "err", tenantCAErr) + } else { + d.acmService.TenantCA = tenantCA + slog.Info("ACM: tenant private CA wired", "permitted_domains", tenantCA.PermittedDomains()) + } + + // The renewal worker scans for PRIVATE_CA certificates past their + // proportional renewal window and reissues them under their existing ARN. + // It reads d.acmService.TenantCA/CertMaterialUpdated live on every scan + // rather than a snapshot taken here, so it starts unconditionally: a + // tenant CA loaded later (or never, on a public-certificates-only + // deployment) is not a startup dependency — the worker just finds + // nothing to renew until one is wired. + d.acmRenewalWorker = handlers_acm.NewWorker(d.acmService, d.node) + d.shutdownWg.Go(func() { + defer func() { + if r := recover(); r != nil { + slog.Error("ACM renewal worker goroutine panicked", "recover", r) + } + }() + d.acmRenewalWorker.Run(d.ctx) + }) // ECR metadata service: owns per-account JetStream KV for repos, tags, // manifest records and upload-state CAS. Disabled (gateway returns NATS diff --git a/spinifex/daemon/daemon_handlers.go b/spinifex/daemon/daemon_handlers.go index 40daf76f4..8d17b7a9e 100644 --- a/spinifex/daemon/daemon_handlers.go +++ b/spinifex/daemon/daemon_handlers.go @@ -52,6 +52,19 @@ func respondWithError(msg *nats.Msg, errCode string) { } } +// respondWithServiceError sends the sanitized error code AND the handler's +// message, so a caller sees the actionable reason rather than a bare code. Use +// it for any error originating in a service call: the code alone collapses a +// specific refusal ("only PRIVATE_CA certificates can be force-renewed") into +// an opaque ServerInternal, leaving the reason visible only in the daemon log. +// Mirrors utils.ServeNATSRequestCtx, which has always preserved the message. +func respondWithServiceError(msg *nats.Msg, err error) { + payload := utils.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), err.Error()) + if respErr := msg.Respond(payload); respErr != nil { + slog.Error("Failed to respond to NATS request", "err", respErr) + } +} + // respondWithJSON marshals data to JSON and sends it as a NATS response. // On marshal failure it responds with an internal server error. func respondWithJSON(msg *nats.Msg, data any) { @@ -91,7 +104,7 @@ func handleNATSRequest[I any, O any](serviceFn func(context.Context, *I, string) // in the journal too. slog.ErrorContext(ctx, "handleNATSRequest: service call failed", "subject", msg.Subject, "err", err) utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } respondWithJSON(msg, output) @@ -120,7 +133,7 @@ func handleNATSRequestWithPrincipal[I any, O any](serviceFn func(context.Context output, err := serviceFn(ctx, input, accountID, principalARN) if err != nil { utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } respondWithJSON(msg, output) @@ -183,7 +196,7 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) { endOpSpan(opSpan, err) if err != nil { utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil { @@ -195,7 +208,7 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) { endOpSpan(opSpan, err) if err != nil { utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if err := msg.Respond([]byte(`{}`)); err != nil { @@ -211,7 +224,7 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) { endOpSpan(opSpan, err) if err != nil { utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if err := msg.Respond([]byte(`{}`)); err != nil { diff --git a/spinifex/daemon/daemon_handlers_capacityreservation.go b/spinifex/daemon/daemon_handlers_capacityreservation.go index 7e6b77126..f3db6be0a 100644 --- a/spinifex/daemon/daemon_handlers_capacityreservation.go +++ b/spinifex/daemon/daemon_handlers_capacityreservation.go @@ -58,7 +58,7 @@ func (d *Daemon) handleEC2CreateCapacityReservation(msg *nats.Msg) { } if err := d.resourceMgr.CreateReservation(rec); err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } diff --git a/spinifex/daemon/daemon_handlers_eni.go b/spinifex/daemon/daemon_handlers_eni.go index 2cb7f1213..ce6240500 100644 --- a/spinifex/daemon/daemon_handlers_eni.go +++ b/spinifex/daemon/daemon_handlers_eni.go @@ -39,7 +39,7 @@ func (d *Daemon) handleAttachNetworkInterface(ctx context.Context, msg *nats.Msg record, err := d.vpcService.GetENIRecord(accountID, eniID) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if record.Status == "in-use" && record.InstanceId != instance.ID { @@ -49,7 +49,7 @@ func (d *Daemon) handleAttachNetworkInterface(ctx context.Context, msg *nats.Msg attachmentID, err := d.vpcService.AttachENI(accountID, eniID, instance.ID, deviceIndex) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if err := d.vpcService.UpdateENI(accountID, eniID, func(r *handlers_ec2_vpc.ENIRecord) { @@ -117,7 +117,7 @@ func (d *Daemon) handleDetachNetworkInterface(ctx context.Context, msg *nats.Msg record, err := d.vpcService.FindENIByAttachment(accountID, attachmentID) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if record.InstanceId != instance.ID { diff --git a/spinifex/daemon/daemon_handlers_error_test.go b/spinifex/daemon/daemon_handlers_error_test.go new file mode 100644 index 000000000..ea9e93c7f --- /dev/null +++ b/spinifex/daemon/daemon_handlers_error_test.go @@ -0,0 +1,85 @@ +package daemon + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests drive the real NATS transport rather than calling a handler +// function directly, because the defect they pin lived entirely in the response +// encoding: the handler already returned a well-worded error and only the wire +// payload dropped it, so a test calling the service function would have passed +// throughout. The neighbouring TestHandleNATSRequest_* cases assert Code only, +// which is why Message needs pinning separately. + +// requestErrorEnvelope drives fn over NATS on subject and decodes the error +// envelope the handler wrote. +func requestErrorEnvelope(t *testing.T, subject string, fn func(context.Context, *testInput, string) (*testOutput, error)) map[string]any { + t.Helper() + + nc, err := nats.Connect(sharedNATSURL) + require.NoError(t, err) + defer nc.Close() + + sub, err := nc.Subscribe(subject, handleNATSRequest(fn)) + require.NoError(t, err) + defer func() { _ = sub.Unsubscribe() }() + + reqData, err := json.Marshal(testInput{Name: "world"}) + require.NoError(t, err) + reply, err := nc.Request(subject, reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + require.NoError(t, json.Unmarshal(reply.Data, &errResp)) + return errResp +} + +// An unregistered error sanitizes to ServerInternal, but its reason must still +// reach the caller. This is the ACM force-renew refusal: without the message the +// CLI prints a bare "ServerInternal" and the operator has to read the daemon log +// to learn that only PRIVATE_CA certificates can be force-renewed. +func TestHandleNATSRequest_UncodedErrorCarriesMessage(t *testing.T) { + const reason = "acm renewal: arn:aws:acm:ap-southeast-2:000000000001:certificate/abc is a AMAZON_ISSUED certificate; only PRIVATE_CA certificates can be force-renewed" + + errResp := requestErrorEnvelope(t, "test.err.msg.uncoded", func(context.Context, *testInput, string) (*testOutput, error) { + return nil, errors.New(reason) + }) + + assert.Equal(t, awserrors.ErrorServerInternal, errResp["Code"]) + assert.Equal(t, reason, errResp["Message"], + "the actionable reason must survive the transport, not only the daemon log") +} + +// A wrapped coded error keeps the real code (so the gateway maps the right HTTP +// status) and carries the wrapping context alongside it. +func TestHandleNATSRequest_WrappedCodedErrorCarriesMessage(t *testing.T) { + errResp := requestErrorEnvelope(t, "test.err.msg.wrapped", func(context.Context, *testInput, string) (*testOutput, error) { + return nil, fmt.Errorf("vol-123 is attached to i-456: %w", errors.New(awserrors.ErrorVolumeInUse)) + }) + + assert.Equal(t, awserrors.ErrorVolumeInUse, errResp["Code"]) + assert.Contains(t, errResp["Message"], "vol-123 is attached to i-456") +} + +// A bare code carries no message. Handlers returning errors.New(awserrors.X) are +// the common case, and echoing the code back as its own message is noise. +func TestHandleNATSRequest_BareCodeOmitsMessage(t *testing.T) { + errResp := requestErrorEnvelope(t, "test.err.msg.bare", func(context.Context, *testInput, string) (*testOutput, error) { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + }) + + assert.Equal(t, awserrors.ErrorInvalidParameterValue, errResp["Code"]) + // The field is always present on the wire (no omitempty); left unset it + // serializes as null rather than repeating the code back. + assert.Nil(t, errResp["Message"]) +} diff --git a/spinifex/daemon/daemon_handlers_iam_profile.go b/spinifex/daemon/daemon_handlers_iam_profile.go index 5dd868f11..126fda883 100644 --- a/spinifex/daemon/daemon_handlers_iam_profile.go +++ b/spinifex/daemon/daemon_handlers_iam_profile.go @@ -3,7 +3,6 @@ package daemon import ( "context" - "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/vm" "github.com/nats-io/nats.go" @@ -15,7 +14,7 @@ import ( func (d *Daemon) handleAssociateIamInstanceProfile(ctx context.Context, msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) { result, err := d.instanceService.AssociateIamInstanceProfile(ctx, instance, command) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } respondWithJSON(msg, result) diff --git a/spinifex/daemon/daemon_handlers_image.go b/spinifex/daemon/daemon_handlers_image.go index 251801b5e..b03ce4935 100644 --- a/spinifex/daemon/daemon_handlers_image.go +++ b/spinifex/daemon/daemon_handlers_image.go @@ -130,7 +130,7 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) { output, err := d.imageService.CreateImageFromInstance(params, accountID) if err != nil { slog.Error("CreateImage: service failed", "instanceId", instanceID, "err", err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } diff --git a/spinifex/daemon/daemon_handlers_instance.go b/spinifex/daemon/daemon_handlers_instance.go index 5dd24fc73..d60afd9f2 100644 --- a/spinifex/daemon/daemon_handlers_instance.go +++ b/spinifex/daemon/daemon_handlers_instance.go @@ -69,7 +69,7 @@ func (d *Daemon) handleSetInstanceTags(ctx context.Context, msg *nats.Msg, comma return true }) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if !found { @@ -117,7 +117,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) { if reservationID != "" { if it, ok := d.resourceMgr.instanceTypes[aws.StringValue(input.InstanceType)]; ok { if vErr := d.resourceMgr.ValidateReservationTarget(reservationID, accountID, it); vErr != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(vErr)) + respondWithServiceError(msg, vErr) return } } @@ -127,7 +127,7 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) { reservation, instances, instanceType, err := d.instanceService.PrepareRunInstances(ctx, input, accountID, reservationID) endOpSpan(prepSpan, err) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } diff --git a/spinifex/daemon/daemon_handlers_spotinstance.go b/spinifex/daemon/daemon_handlers_spotinstance.go index 356540f8d..e23366e20 100644 --- a/spinifex/daemon/daemon_handlers_spotinstance.go +++ b/spinifex/daemon/daemon_handlers_spotinstance.go @@ -27,7 +27,7 @@ func (d *Daemon) handleSetSpotLineage(ctx context.Context, msg *nats.Msg, comman return true }) if err != nil { - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } if !found { diff --git a/spinifex/daemon/daemon_handlers_volume.go b/spinifex/daemon/daemon_handlers_volume.go index 798505d87..fd231a915 100644 --- a/spinifex/daemon/daemon_handlers_volume.go +++ b/spinifex/daemon/daemon_handlers_volume.go @@ -235,7 +235,7 @@ func (d *Daemon) handleEC2ModifyVolume(msg *nats.Msg) { if err != nil { slog.ErrorContext(ctx, "handleEC2ModifyVolume service.ModifyVolume failed", "err", err) utils.MarkSpanError(span, err) - respondWithError(msg, awserrors.ValidErrorCodeFromError(err)) + respondWithServiceError(msg, err) return } diff --git a/spinifex/daemon/daemon_system_instance_test.go b/spinifex/daemon/daemon_system_instance_test.go index 0c90874e8..ef19b71fc 100644 --- a/spinifex/daemon/daemon_system_instance_test.go +++ b/spinifex/daemon/daemon_system_instance_test.go @@ -15,6 +15,7 @@ import ( handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/network/external" "github.com/mulgadc/spinifex/spinifex/tags" "github.com/mulgadc/spinifex/spinifex/testutil" @@ -597,7 +598,9 @@ func newRefreshTestDaemon(t *testing.T) (*Daemon, *nats.Conn, string) { tmpDir := t.TempDir() t.Setenv("XDG_RUNTIME_DIR", tmpDir) _, nc, _ := testutil.StartTestJetStream(t) - svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc, masterKey) require.NoError(t, err) t.Cleanup(svc.Close) return &Daemon{vmMgr: vm.NewManager(), elbv2Service: svc}, nc, tmpDir diff --git a/spinifex/daemon/daemon_wire_lb_test.go b/spinifex/daemon/daemon_wire_lb_test.go index 34c2e7200..8f9971268 100644 --- a/spinifex/daemon/daemon_wire_lb_test.go +++ b/spinifex/daemon/daemon_wire_lb_test.go @@ -7,6 +7,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" @@ -19,7 +20,9 @@ func newWireLBTestDaemon(t *testing.T, cfg *config.Config) *Daemon { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc, masterKey) require.NoError(t, err) t.Cleanup(func() { svc.Close() }) @@ -250,7 +253,9 @@ func TestWireLBAgentConfig_GatewayURL_SingleNodeBindMatchesAdvertise(t *testing. // newSubscribeTestDaemon creates a minimal Daemon wired enough for subscribeAll. func newSubscribeTestDaemon(t *testing.T, nc *nats.Conn, gatewayURL, accessKey string) *Daemon { t.Helper() - svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(nil, nc, masterKey) require.NoError(t, err) t.Cleanup(func() { svc.Close() }) svc.GatewayURL = gatewayURL diff --git a/spinifex/gateway/acm.go b/spinifex/gateway/acm.go index 0131e8fec..a6fab4e54 100644 --- a/spinifex/gateway/acm.go +++ b/spinifex/gateway/acm.go @@ -2,16 +2,25 @@ package gateway import ( "context" + "encoding/json" "errors" "io" "log/slog" "net/http" "strings" + "time" + "github.com/aws/aws-sdk-go/service/acm" "github.com/mulgadc/spinifex/spinifex/awserrors" gateway_acm "github.com/mulgadc/spinifex/spinifex/gateway/acm" + "github.com/mulgadc/spinifex/spinifex/utils" ) +// acmNATSTimeout bounds the gateway's wait for a daemon-side ACM response. +// RequestCertificate itself returns immediately (issuance is asynchronous), +// so this is a network/queueing budget, not an issuance one. +const acmNATSTimeout = 30 * time.Second + // acmHandler invokes a per-action ACM gateway function. type acmHandler func(ctx context.Context, gw *GatewayConfig, accountID string, body []byte) (any, error) @@ -20,9 +29,25 @@ var acmActions = map[string]acmHandler{ "ImportCertificate": func(ctx context.Context, gw *GatewayConfig, acct string, b []byte) (any, error) { return gateway_acm.ImportCertificate(ctx, gw.NATSConn, acct, b) }, + // RequestCertificate calls the daemon over NATS directly (rather than + // through package gateway_acm, which does not yet have a helper for it) — + // the same "acm.RequestCertificate" subject and request/response shape + // utils.NATSRequest uses everywhere else in this table. + "RequestCertificate": func(ctx context.Context, gw *GatewayConfig, acct string, b []byte) (any, error) { + input := new(acm.RequestCertificateInput) + if len(b) > 0 { + if err := json.Unmarshal(b, input); err != nil { + return nil, errors.New(awserrors.ErrorInvalidParameterValue) + } + } + return utils.NATSRequest[acm.RequestCertificateOutput](ctx, gw.NATSConn, "acm.RequestCertificate", input, acmNATSTimeout, acct) + }, "DescribeCertificate": func(ctx context.Context, gw *GatewayConfig, acct string, b []byte) (any, error) { return gateway_acm.DescribeCertificate(ctx, gw.NATSConn, acct, b) }, + "GetCertificate": func(ctx context.Context, gw *GatewayConfig, acct string, b []byte) (any, error) { + return gateway_acm.GetCertificate(ctx, gw.NATSConn, acct, b) + }, "ListCertificates": func(ctx context.Context, gw *GatewayConfig, acct string, b []byte) (any, error) { return gateway_acm.ListCertificates(ctx, gw.NATSConn, acct, b) }, diff --git a/spinifex/gateway/acm/certificates.go b/spinifex/gateway/acm/certificates.go index f050386b8..7d4b352ad 100644 --- a/spinifex/gateway/acm/certificates.go +++ b/spinifex/gateway/acm/certificates.go @@ -28,6 +28,15 @@ func DescribeCertificate(ctx context.Context, natsConn *nats.Conn, accountID str return handlers_acm.NewNATSACMService(natsConn).DescribeCertificate(ctx, input, accountID) } +// GetCertificate — CertificateManager.GetCertificate. +func GetCertificate(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.GetCertificateOutput, error) { + input := new(acm.GetCertificateInput) + if err := unmarshalIfBody(body, input); err != nil { + return nil, errors.New(awserrors.ErrorInvalidParameter) + } + return handlers_acm.NewNATSACMService(natsConn).GetCertificate(ctx, input, accountID) +} + // ListCertificates — CertificateManager.ListCertificates. func ListCertificates(ctx context.Context, natsConn *nats.Conn, accountID string, body []byte) (*acm.ListCertificatesOutput, error) { input := new(acm.ListCertificatesInput) diff --git a/spinifex/gateway/acm_test.go b/spinifex/gateway/acm_test.go index 90a3295e9..2c7e984d5 100644 --- a/spinifex/gateway/acm_test.go +++ b/spinifex/gateway/acm_test.go @@ -32,11 +32,13 @@ func TestACMActionsMap_AllActionsRegistered(t *testing.T) { expected := []string{ "ImportCertificate", "DescribeCertificate", + "GetCertificate", "ListCertificates", "DeleteCertificate", "ListTagsForCertificate", "AddTagsToCertificate", "RemoveTagsFromCertificate", + "RequestCertificate", } for _, action := range expected { _, ok := acmActions[action] @@ -56,7 +58,7 @@ func TestACMRequest_MissingTarget(t *testing.T) { func TestACMRequest_UnknownAction(t *testing.T) { gw := &GatewayConfig{DisableLogging: true} w := httptest.NewRecorder() - err := gw.ACM_Request(w, setupACMRequest("CertificateManager.RequestCertificate", "{}")) + err := gw.ACM_Request(w, setupACMRequest("CertificateManager.BogusAction", "{}")) require.Error(t, err) assert.Equal(t, awserrors.ErrorInvalidAction, err.Error()) } diff --git a/spinifex/handlers/acm/getcertificate_test.go b/spinifex/handlers/acm/getcertificate_test.go new file mode 100644 index 000000000..24f9ef92e --- /dev/null +++ b/spinifex/handlers/acm/getcertificate_test.go @@ -0,0 +1,201 @@ +package handlers_acm + +import ( + "context" + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/acm" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetCertificate_ReturnsBodyAndChain(t *testing.T) { + svc := setupACMService(t) + leafPEM, keyPEM := genCert(t, "get.example.com") + chainPEM, _ := genCert(t, "issuer.example.com") + + imported, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: leafPEM, + PrivateKey: keyPEM, + CertificateChain: chainPEM, + }, testAccountID) + require.NoError(t, err) + + out, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{ + CertificateArn: imported.CertificateArn, + }, testAccountID) + require.NoError(t, err) + assert.Equal(t, string(leafPEM), aws.StringValue(out.Certificate)) + assert.Equal(t, string(chainPEM), aws.StringValue(out.CertificateChain)) +} + +// A self-signed leaf has no chain; the field is left absent rather than an +// empty string a client would have to distinguish from a real chain. +func TestGetCertificate_OmitsAbsentChain(t *testing.T) { + svc := setupACMService(t) + leafPEM, keyPEM := genCert(t, "nochain.example.com") + + imported, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: leafPEM, + PrivateKey: keyPEM, + }, testAccountID) + require.NoError(t, err) + + out, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{ + CertificateArn: imported.CertificateArn, + }, testAccountID) + require.NoError(t, err) + assert.Equal(t, string(leafPEM), aws.StringValue(out.Certificate)) + assert.Nil(t, out.CertificateChain) +} + +func TestGetCertificate_UnknownArnNotFound(t *testing.T) { + svc := setupACMService(t) + + _, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{ + CertificateArn: aws.String("arn:aws:acm:ap-southeast-2:000000000001:certificate/does-not-exist"), + }, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorResourceNotFound) +} + +// Another account's certificate is indistinguishable from a missing one — the +// body is the material itself, so a cross-account read must not confirm the +// certificate even exists. +func TestGetCertificate_CrossAccountNotFound(t *testing.T) { + svc := setupACMService(t) + leafPEM, keyPEM := genCert(t, "other.example.com") + + imported, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: leafPEM, + PrivateKey: keyPEM, + }, testAccountID) + require.NoError(t, err) + + _, err = svc.GetCertificate(context.Background(), &acm.GetCertificateInput{ + CertificateArn: imported.CertificateArn, + }, "000000000002") + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorResourceNotFound) +} + +// A requested certificate has a record before it has a body. That is retryable +// and distinct from not-found, so a client polling for issuance can tell them +// apart. +func TestGetCertificate_PendingIssuanceIsRequestInProgress(t *testing.T) { + svc := setupACMService(t) + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/pending-1" + require.NoError(t, svc.store.PutCert(t.Context(), &CertRecord{ + CertificateArn: arn, + AccountID: testAccountID, + DomainName: "pending.example.com", + Status: acm.CertificateStatusPendingValidation, + })) + + _, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{ + CertificateArn: aws.String(arn), + }, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorACMRequestInProgress) +} + +func TestGetCertificate_EmptyArnRejected(t *testing.T) { + svc := setupACMService(t) + + _, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{}, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorACMInvalidArn) +} + +// putUndecryptableCert writes a record whose stored PrivateKey is neither valid +// ciphertext nor a PEM key, bypassing PutCert (which would encrypt it). Any read +// path that decrypts fails on this record, which is what makes it a detector: +// a path that succeeds provably never touched the key. +func putUndecryptableCert(t *testing.T, store *Store, arn, domain string) { + t.Helper() + data, err := json.Marshal(&CertRecord{ + CertificateArn: arn, + AccountID: testAccountID, + DomainName: domain, + Certificate: "-----BEGIN CERTIFICATE-----\nbody\n-----END CERTIFICATE-----\n", + PrivateKey: "not-pem-and-not-ciphertext", + Status: acm.CertificateStatusIssued, + }) + require.NoError(t, err) + _, err = store.kv.Put(t.Context(), certKey(arn), data) + require.NoError(t, err) +} + +// Pins that the summary paths never decrypt key material, by consequence rather +// than instrumentation: an undecryptable key is fatal to a decrypting read, so a +// list that still returns the certificate cannot have decrypted it. +func TestListPath_NeverTouchesKeyMaterial(t *testing.T) { + svc := setupACMService(t) + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/nokeyread-1" + putUndecryptableCert(t, svc.store, arn, "nokeyread.example.com") + + // Control: the decrypting read does fail on this record, so the assertions + // below are testing the absence of a decrypt and not a benign key. + _, err := svc.store.GetCert(t.Context(), arn) + require.Error(t, err) + + list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID) + require.NoError(t, err) + require.Len(t, list.CertificateSummaryList, 1) + assert.Equal(t, arn, aws.StringValue(list.CertificateSummaryList[0].CertificateArn)) + + // Same for the single-record metadata reads layered on GetCertMetadata. + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{ + CertificateArn: aws.String(arn), + }, testAccountID) + require.NoError(t, err) + assert.Equal(t, "nokeyread.example.com", aws.StringValue(desc.Certificate.DomainName)) + + body, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{ + CertificateArn: aws.String(arn), + }, testAccountID) + require.NoError(t, err) + assert.Contains(t, aws.StringValue(body.Certificate), "BEGIN CERTIFICATE") +} + +// The metadata reads clear PrivateKey rather than handing back ciphertext, so a +// caller cannot mistake ciphertext for usable key material in a field that is +// plaintext PEM everywhere else. +func TestMetadataReads_ClearPrivateKey(t *testing.T) { + store := setupACMStore(t) + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/cleared-1" + _, keyPEM := genCert(t, "cleared.example.com") + require.NoError(t, store.PutCert(t.Context(), &CertRecord{ + CertificateArn: arn, + AccountID: testAccountID, + DomainName: "cleared.example.com", + PrivateKey: string(keyPEM), + })) + + one, err := store.GetCertMetadata(t.Context(), arn) + require.NoError(t, err) + require.NotNil(t, one) + assert.Empty(t, one.PrivateKey) + + recs, err := store.ListCertMetadata(t.Context(), testAccountID) + require.NoError(t, err) + require.Len(t, recs, 1) + assert.Empty(t, recs[0].PrivateKey) + + // The key is still there and still readable through the decrypting read — + // the metadata reads withhold it, they do not lose it. + full, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + assert.Equal(t, string(keyPEM), full.PrivateKey) +} + +func TestGetCertMetadata_AbsentReturnsNil(t *testing.T) { + store := setupACMStore(t) + + rec, err := store.GetCertMetadata(t.Context(), "arn:aws:acm:ap-southeast-2:000000000001:certificate/missing") + require.NoError(t, err) + assert.Nil(t, rec) +} diff --git a/spinifex/handlers/acm/privateca.go b/spinifex/handlers/acm/privateca.go new file mode 100644 index 000000000..04fb73f9a --- /dev/null +++ b/spinifex/handlers/acm/privateca.go @@ -0,0 +1,499 @@ +package handlers_acm + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "slices" + "sort" + "strings" + "time" +) + +// caKeyBits and leafKeyBits are RSA key sizes, kept as package-level vars so +// tests can lower them for faster key generation, mirroring +// admin.certKeyBits. Production keeps the 4096/2048 defaults. +var ( + caKeyBits = 4096 + leafKeyBits = 2048 +) + +const ( + // tenantCAValidity is long because the root is a one-time trust-store + // install on every client device: rotating it invalidates all of them at + // once, so it should outlive many leaf-renewal cycles. + tenantCAValidity = 10 * 365 * 24 * time.Hour + + // leafValidity is short because issuance and renewal are instant, offline + // operations for this CA — unlike the public ACME path, there is no + // rate-limit reason to prefer a long-lived leaf, so a short lifetime is + // pure upside: it bounds the exposure window of the fresh key IssueLeaf + // generates on every call. + leafValidity = 90 * 24 * time.Hour + + // leafClockSkewBackdate absorbs a small amount of clock drift between + // this node and whatever client validates the issued leaf. + leafClockSkewBackdate = 5 * time.Minute +) + +// TenantCA signs leaf certificates for tenant domains from an independent +// root, generated specifically for this purpose. +// +// It is deliberately not the platform CA (admin.GenerateCACert / +// admin.GenerateCertificatesIfNeeded). That CA is trusted by every node, the +// ipsec charon store, EKS and ECS guests, ALB microvms, and EKS addon pods, +// and is baked into catalog images — signing tenant leaves from it would turn +// any tenant certificate request into a control-plane-wide MITM primitive. +// Nor is it a subordinate beneath the platform CA: that CA is pathlen=0, +// which forbids a subordinate outright, and even if it were not, a shared +// hierarchy would still let a tenant leaf request touch platform trust. +// +// The root's x509 name constraints (see createTenantCA) are the actual +// security boundary, not the AccountID/domain checks in the API layer above +// this file: a client that enforces RFC 5280 constraints rejects an +// out-of-scope leaf even if every server-side check here is bypassed or +// buggy. Go, OpenSSL and browsers all honour constraints; some embedded TLS +// stacks do not, so treat this as defense in depth, not a guarantee. +type TenantCA struct { + root *x509.Certificate + key *rsa.PrivateKey + rootPEM string // cached PEM encoding of root, returned as the chain for every issued leaf. +} + +var _ CertAuthority = (*TenantCA)(nil) + +// tenantCACreateCommandHint names the admin command that creates the tenant +// CA, quoted into every "no tenant CA here" error so an operator lands on the +// fix without having to go spelunking through docs. +const tenantCACreateCommandHint = "spx admin cert create-tenant-ca --domain " + +// LoadOrCreateTenantCA loads the tenant root from certPath/keyPath, creating +// it on first use with permittedDomains baked in as x509 name constraints. +// +// Constraints are frozen at creation. If a root already exists at these +// paths, its PermittedDNSDomains must cover every entry in permittedDomains +// or this returns an error rather than regenerating: rotating the root +// invalidates every device's trust store at once, so that can only ever be a +// deliberate, operator-driven action. +func LoadOrCreateTenantCA(certPath, keyPath string, permittedDomains []string) (*TenantCA, error) { + normalized, err := normalizeDomains(permittedDomains) + if err != nil { + return nil, fmt.Errorf("privateca: %w", err) + } + + certExists := fileExists(certPath) + keyExists := fileExists(keyPath) + + switch { + case certExists && keyExists: + ca, err := readTenantCAFiles(certPath, keyPath) + if err != nil { + return nil, err + } + if uncovered := uncoveredDomains(normalized, ca.root.PermittedDNSDomains); len(uncovered) > 0 { + return nil, fmt.Errorf( + "privateca: tenant CA at %s does not cover domain(s) %s; name constraints are frozen at creation — regenerate the tenant CA (delete %s and %s) and redistribute the new root to every client trust store", + certPath, strings.Join(uncovered, ", "), certPath, keyPath, + ) + } + return ca, nil + case certExists != keyExists: + // One file present without its counterpart is an ambiguous, corrupt + // state. Guessing wrong here (e.g. regenerating over a surviving key) + // is worse than refusing, so this is never auto-repaired. + return nil, fmt.Errorf( + "privateca: tenant CA at %s is incomplete (cert present=%v, key present=%v); restore the missing file from backup or remove both to regenerate", + certPath, certExists, keyExists, + ) + default: + return createTenantCA(certPath, keyPath, normalized) + } +} + +// LoadTenantCA loads an existing tenant root from certPath/keyPath. Unlike +// LoadOrCreateTenantCA, it never creates one — there is no permitted-domains +// argument to bake in, because the daemon (this function's only caller) has +// paths but not a domain list to create with. The permitted domains come from +// the certificate itself once loaded (see PermittedDomains), not from a +// caller-supplied list, so there is nothing to drift-check here. +// +// Absent or half-present files return a clearly-worded, actionable error +// naming the admin command that creates the CA, rather than a bare "file not +// found" — this is the first thing an operator sees when PRIVATE_CA issuance +// is unreachable. +func LoadTenantCA(certPath, keyPath string) (*TenantCA, error) { + certExists := fileExists(certPath) + keyExists := fileExists(keyPath) + + switch { + case certExists && keyExists: + return readTenantCAFiles(certPath, keyPath) + case certExists != keyExists: + return nil, fmt.Errorf( + "privateca: tenant CA at %s is incomplete (cert present=%v, key present=%v); restore the missing file from backup or remove both and recreate with %q", + certPath, certExists, keyExists, tenantCACreateCommandHint, + ) + default: + return nil, fmt.Errorf( + "privateca: no tenant CA found at %s; PRIVATE_CA certificate issuance is unavailable until one is created — run %q", + certPath, tenantCACreateCommandHint, + ) + } +} + +// readTenantCAFiles reads and parses an existing root and key pair from disk, +// verifying the key matches the certificate's public key. It performs no +// domain-coverage check; callers that need one (LoadOrCreateTenantCA) apply it +// themselves against the freshly loaded root's own PermittedDNSDomains. +func readTenantCAFiles(certPath, keyPath string) (*TenantCA, error) { + certPEMBytes, err := os.ReadFile(certPath) + if err != nil { + return nil, fmt.Errorf("privateca: read tenant CA cert: %w", err) + } + certBlock, _ := pem.Decode(certPEMBytes) + if certBlock == nil { + return nil, fmt.Errorf("privateca: no PEM certificate block in %s", certPath) + } + root, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("privateca: parse tenant CA cert: %w", err) + } + + keyPEMBytes, err := os.ReadFile(keyPath) + if err != nil { + return nil, fmt.Errorf("privateca: read tenant CA key: %w", err) + } + keyBlock, _ := pem.Decode(keyPEMBytes) + if keyBlock == nil { + return nil, fmt.Errorf("privateca: no PEM key block in %s", keyPath) + } + parsedKey, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes) + if err != nil { + // Never fold key bytes into an error string — this only reports the + // parse failure, not the key material. + return nil, fmt.Errorf("privateca: parse tenant CA key: %w", err) + } + key, ok := parsedKey.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("privateca: tenant CA key at %s is not RSA", keyPath) + } + if !key.PublicKey.Equal(root.PublicKey) { + return nil, fmt.Errorf("privateca: tenant CA key at %s does not match the public key in cert at %s", keyPath, certPath) + } + + return &TenantCA{root: root, key: key, rootPEM: string(certPEMBytes)}, nil +} + +// createTenantCA generates a fresh, self-signed tenant root with +// permittedDomains baked in as x509 name constraints, and persists it to +// certPath/keyPath. +func createTenantCA(certPath, keyPath string, permittedDomains []string) (*TenantCA, error) { + if len(permittedDomains) == 0 { + // An empty PermittedDNSDomains list is not "constrained to nothing" — + // Go's constraint matcher (crypto/x509/constraints.go) treats a nil + // permitted-DNS constraint as "no restriction", so this would silently + // create an unconstrained root. Refuse instead of building something + // that looks constrained but is not. + return nil, fmt.Errorf("privateca: refusing to create a tenant CA with no permitted domains — it would be unconstrained") + } + + key, err := rsa.GenerateKey(rand.Reader, caKeyBits) + if err != nil { + return nil, fmt.Errorf("privateca: generate tenant CA key: %w", err) + } + + serial, err := randomSerial() + if err != nil { + return nil, fmt.Errorf("privateca: generate tenant CA serial: %w", err) + } + + // 0.0.0.0/0 and ::/0 as excluded ranges, with no permitted IP ranges, + // make every possible IP address excluded — this root can never validate + // a leaf carrying an IP SAN. + _, allV4, err := net.ParseCIDR("0.0.0.0/0") + if err != nil { + return nil, fmt.Errorf("privateca: parse IPv4 exclusion range: %w", err) + } + _, allV6, err := net.ParseCIDR("::/0") + if err != nil { + return nil, fmt.Errorf("privateca: parse IPv6 exclusion range: %w", err) + } + + notBefore := time.Now() + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: "Spinifex Tenant CA", + Organization: []string{"Spinifex Tenant"}, + }, + NotBefore: notBefore, + NotAfter: notBefore.Add(tenantCAValidity), + + // This root only ever signs and revokes tenant leaves — it never + // authenticates a TLS connection itself, so KeyUsage is limited to + // certificate and CRL signing. + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + // MaxPathLen 0 with MaxPathLenZero true (Go treats a bare zero value + // as "unset" otherwise) forbids any subordinate CA beneath this root: + // it may only ever sign end-entity leaves. + MaxPathLen: 0, + MaxPathLenZero: true, + + // Name constraints are the cryptographic security boundary described + // on TenantCA: PermittedDNSDomains in apex form covers every + // subdomain, so adding a service never touches the CA. + PermittedDNSDomains: permittedDomains, + // RFC 5280 4.2.1.10: "conforming CAs MUST mark this extension as + // critical." A client that does not understand name constraints must + // then reject the certificate outright rather than silently ignore + // the restriction. + PermittedDNSDomainsCritical: true, + ExcludedIPRanges: []*net.IPNet{allV4, allV6}, + + // Go's constraint matcher (crypto/x509/constraints.go, + // newDNSConstraints) treats a single zero-length entry in a + // domain-shaped constraint list as "match every name" — that + // function backs DNS, email and URI constraints alike, keyed on the + // domain portion of the address/URI. RFC 5280 defines no dedicated + // "exclude everything" wildcard for rfc822Name/URI; this relies + // specifically on Go's interpretation (consistent with errata 5997) + // of an empty constraint value, verified against the go1.26.5 + // stdlib. A single "" entry in each list therefore excludes every + // possible email address and URI outright — this root never expects + // to sign either. + // + // This CA is installed in device trust stores, so it is validated by + // non-Go verifiers too. Checked against OpenSSL 3.5.6: `openssl x509 + // -text` parses a root built this way without error, printing empty + // "email:"/"URI:" excluded-subtree lines rather than rejecting the + // extension, and `openssl verify -CAfile` both accepts a valid + // in-scope leaf and rejects a leaf forged outside + // PermittedDNSDomains with "error 47: permitted subtree violation" — + // so the zero-length exclusion neither breaks parsing nor changes + // the DNS-constraint outcome on a non-Go client. If a future + // verifier turns out to disagree, drop these two fields rather than + // fight it: PermittedDNSDomains plus ExcludedIPRanges already carry + // the security argument, and IssueLeaf never emits an email or URI + // SAN, so the exclusions are redundant defense in depth, not load + // bearing. + ExcludedEmailAddresses: []string{""}, + ExcludedURIDomains: []string{""}, + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return nil, fmt.Errorf("privateca: create tenant CA cert: %w", err) + } + root, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("privateca: parse newly created tenant CA cert: %w", err) + } + + certPEMBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + // The cert is public material (distributed to every client trust store), + // so it is written with the default create mode, matching + // admin.GenerateCACert's cert file. + certOut, err := os.Create(certPath) + if err != nil { + return nil, fmt.Errorf("privateca: create tenant CA cert file: %w", err) + } + defer certOut.Close() + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { + return nil, fmt.Errorf("privateca: write tenant CA cert: %w", err) + } + + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("privateca: marshal tenant CA key: %w", err) + } + // 0600: this key signs every tenant certificate in the deployment — + // getting this wrong compromises every tenant certificate at once. + keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return nil, fmt.Errorf("privateca: create tenant CA key file: %w", err) + } + defer keyOut.Close() + if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}); err != nil { + return nil, fmt.Errorf("privateca: write tenant CA key: %w", err) + } + + return &TenantCA{root: root, key: key, rootPEM: string(certPEMBytes)}, nil +} + +// IssueLeaf signs a leaf certificate for domain plus sans, generating a +// fresh private key. Returns PEM-encoded leaf, chain and key. +// +// domain and every entry in sans must fall within the root's permitted +// domains (see Authorized). This is enforced here, independently of any +// authorization check the caller already performed, as a second line of +// defense — the same reasoning that motivates baking the constraints into +// the certificate at all. +func (c *TenantCA) IssueLeaf(domain string, sans []string) (certPEM, chainPEM, keyPEM string, err error) { + if domain == "" { + return "", "", "", fmt.Errorf("privateca: domain is required") + } + names := dedupeStrings(append([]string{domain}, sans...)) + for _, n := range names { + if !c.Authorized(n) { + return "", "", "", fmt.Errorf("privateca: domain %q is outside the tenant CA's permitted domains %v", n, c.root.PermittedDNSDomains) + } + } + + leafKey, err := rsa.GenerateKey(rand.Reader, leafKeyBits) + if err != nil { + return "", "", "", fmt.Errorf("privateca: generate leaf key: %w", err) + } + + serial, err := randomSerial() + if err != nil { + return "", "", "", fmt.Errorf("privateca: generate leaf serial: %w", err) + } + + notBefore := time.Now().Add(-leafClockSkewBackdate) + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: domain}, + NotBefore: notBefore, + NotAfter: notBefore.Add(leafValidity + leafClockSkewBackdate), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: false, + DNSNames: names, + } + + der, err := x509.CreateCertificate(rand.Reader, template, c.root, &leafKey.PublicKey, c.key) + if err != nil { + return "", "", "", fmt.Errorf("privateca: sign leaf: %w", err) + } + + leafPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + + keyDER, err := x509.MarshalPKCS8PrivateKey(leafKey) + if err != nil { + // As above: report the failure, never the key material. + return "", "", "", fmt.Errorf("privateca: marshal leaf key: %w", err) + } + leafKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + + return string(leafPEM), c.rootPEM, string(leafKeyPEM), nil +} + +// PermittedDomains returns the name constraints baked into the loaded root, +// read from the certificate itself rather than a parallel list so the two +// cannot drift. The returned slice is a copy; mutating it has no effect on +// the CA. +func (c *TenantCA) PermittedDomains() []string { + return slices.Clone(c.root.PermittedDNSDomains) +} + +// Authorized reports whether domain falls within the root's permitted +// domains. domain may be an apex, subdomain, or wildcard name; comparison is +// case-insensitive and mirrors the RFC 5280 apex-domain match Go's own +// verifier applies at certificate-verification time, so a domain authorized +// here is guaranteed to actually chain once a leaf is issued for it. +func (c *TenantCA) Authorized(domain string) bool { + return domainCovered(domain, c.root.PermittedDNSDomains) +} + +// randomSerial returns a random 128-bit certificate serial number, matching +// admin.GenerateCACert's convention. +func randomSerial() (*big.Int, error) { + return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) +} + +// fileExists reports whether path names a regular, readable file. +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +// normalizeDomains lowercases, strips wildcard/leading-dot prefixes, dedupes +// and sorts domains into apex form suitable for PermittedDNSDomains. It +// rejects any entry that normalizes to the empty string. +func normalizeDomains(domains []string) ([]string, error) { + seen := make(map[string]struct{}, len(domains)) + out := make([]string, 0, len(domains)) + for _, d := range domains { + nd := normalizeDomainQuery(d) + if nd == "" { + return nil, fmt.Errorf("empty or invalid domain %q", d) + } + if _, ok := seen[nd]; ok { + continue + } + seen[nd] = struct{}{} + out = append(out, nd) + } + sort.Strings(out) + return out, nil +} + +// normalizeDomainQuery lowercases domain and strips a trailing dot and a +// leading "*." or "." so apex, subdomain and wildcard forms all compare +// against PermittedDNSDomains the same way. +func normalizeDomainQuery(domain string) string { + d := strings.ToLower(strings.TrimSuffix(domain, ".")) + d = strings.TrimPrefix(d, "*.") + d = strings.TrimPrefix(d, ".") + return d +} + +// domainCovered reports whether name (apex, subdomain, or wildcard) is +// exactly one of permitted or a subdomain of one of permitted, matching the +// RFC 5280 dNSName constraint semantics for an apex-form (no leading dot) +// constraint: the apex itself and every subdomain are covered. +func domainCovered(name string, permitted []string) bool { + nd := normalizeDomainQuery(name) + if nd == "" { + return false + } + for _, p := range permitted { + np := strings.ToLower(strings.TrimSuffix(p, ".")) + if np == "" { + continue + } + if nd == np || strings.HasSuffix(nd, "."+np) { + return true + } + } + return false +} + +// uncoveredDomains returns the entries of requested (already normalized) not +// covered by existing, preserving requested's order. +func uncoveredDomains(requested, existing []string) []string { + var out []string + for _, d := range requested { + if !domainCovered(d, existing) { + out = append(out, d) + } + } + return out +} + +// dedupeStrings returns names with duplicates removed, preserving the first +// occurrence's order. +func dedupeStrings(names []string) []string { + seen := make(map[string]struct{}, len(names)) + out := make([]string, 0, len(names)) + for _, n := range names { + if n == "" { + continue + } + if _, ok := seen[n]; ok { + continue + } + seen[n] = struct{}{} + out = append(out, n) + } + return out +} diff --git a/spinifex/handlers/acm/privateca_test.go b/spinifex/handlers/acm/privateca_test.go new file mode 100644 index 000000000..38a29e4ad --- /dev/null +++ b/spinifex/handlers/acm/privateca_test.go @@ -0,0 +1,465 @@ +package handlers_acm + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io" + "math/big" + "net" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMain lowers the RSA key sizes for the whole privateca test file. 2048 +// (CA) and 1024 (leaf) bits are still valid RSA keys and generate several +// times faster than the production defaults, and the tests here care about +// certificate structure, not cryptographic strength — mirrors +// admin.certKeyBits's test seam. +func TestMain(m *testing.M) { + caKeyBits = 2048 + leafKeyBits = 1024 + os.Exit(m.Run()) +} + +func tenantCAPaths(t *testing.T) (certPath, keyPath string) { + t.Helper() + dir := t.TempDir() + return filepath.Join(dir, "tenant-ca.pem"), filepath.Join(dir, "tenant-ca-key.pem") +} + +func TestLoadOrCreateTenantCA_CreatesConstrainedRoot(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + require.NotNil(t, ca) + + root := ca.root + assert.True(t, root.IsCA) + assert.True(t, root.BasicConstraintsValid) + assert.Equal(t, 0, root.MaxPathLen) + assert.True(t, root.MaxPathLenZero) + assert.Equal(t, x509.KeyUsageCertSign|x509.KeyUsageCRLSign, root.KeyUsage) + assert.Equal(t, []string{"home.example.com"}, root.PermittedDNSDomains) + assert.True(t, root.PermittedDNSDomainsCritical) + + // 0.0.0.0/0 and ::/0 must both be present in ExcludedIPRanges so this + // root can never validate an IP SAN. + var haveV4, haveV6 bool + for _, r := range root.ExcludedIPRanges { + ones, bits := r.Mask.Size() + if ones == 0 && bits == 32 { + haveV4 = true + } + if ones == 0 && bits == 128 { + haveV6 = true + } + } + assert.True(t, haveV4, "expected 0.0.0.0/0 in ExcludedIPRanges") + assert.True(t, haveV6, "expected ::/0 in ExcludedIPRanges") + + assert.Equal(t, []string{""}, root.ExcludedEmailAddresses) + assert.Equal(t, []string{""}, root.ExcludedURIDomains) + + // Files must exist on disk with the key locked down to 0600. + certInfo, err := os.Stat(certPath) + require.NoError(t, err) + assert.False(t, certInfo.IsDir()) + + keyInfo, err := os.Stat(keyPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), keyInfo.Mode().Perm()) +} + +func TestLoadOrCreateTenantCA_RefusesEmptyPermittedDomains(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + _, err := LoadOrCreateTenantCA(certPath, keyPath, nil) + require.Error(t, err) + assert.NoFileExists(t, certPath) + assert.NoFileExists(t, keyPath) +} + +func TestLoadOrCreateTenantCA_Idempotent(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + first, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + keyInfoBefore, err := os.Stat(keyPath) + require.NoError(t, err) + + second, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + // Same root: identical serial number proves the second call loaded + // rather than regenerated. + assert.Equal(t, 0, first.root.SerialNumber.Cmp(second.root.SerialNumber)) + assert.Equal(t, first.rootPEM, second.rootPEM) + + keyInfoAfter, err := os.Stat(keyPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), keyInfoAfter.Mode().Perm()) + assert.Equal(t, keyInfoBefore.ModTime(), keyInfoAfter.ModTime(), "second call must not rewrite the key file") +} + +func TestLoadOrCreateTenantCA_DriftIsRejectedAndLeavesRootUntouched(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + _, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + certBefore, err := os.ReadFile(certPath) + require.NoError(t, err) + keyBefore, err := os.ReadFile(keyPath) + require.NoError(t, err) + + _, err = LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com", "evil.example.org"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "evil.example.org") + assert.Contains(t, err.Error(), "regenerate") + + certAfter, err := os.ReadFile(certPath) + require.NoError(t, err) + keyAfter, err := os.ReadFile(keyPath) + require.NoError(t, err) + assert.Equal(t, certBefore, certAfter, "drift must not touch the on-disk cert") + assert.Equal(t, keyBefore, keyAfter, "drift must not touch the on-disk key") +} + +func TestLoadOrCreateTenantCA_SubdomainRequestCoveredByExistingApex(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + _, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"example.com"}) + require.NoError(t, err) + + // A narrower request (a subdomain of an already-covered apex) must load + // cleanly — only a domain the root does NOT cover is drift. + _, err = LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) +} + +func TestLoadOrCreateTenantCA_IncompletePairIsRejected(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + _, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + require.NoError(t, os.Remove(keyPath)) + + _, err = LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "incomplete") + assert.NoFileExists(t, keyPath, "must not regenerate a key next to a surviving cert") +} + +func TestIssueLeaf_ChainsToRootAndCoversWildcard(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + certPEM, chainPEM, keyPEM, err := ca.IssueLeaf("*.home.example.com", []string{"home.example.com"}) + require.NoError(t, err) + require.NotEmpty(t, certPEM) + require.NotEmpty(t, chainPEM) + require.NotEmpty(t, keyPEM) + + leaf := parseCertPEM(t, certPEM) + + pool := x509.NewCertPool() + require.True(t, pool.AppendCertsFromPEM([]byte(chainPEM))) + + opts := x509.VerifyOptions{ + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + // Exact apex SAN. + opts.DNSName = "home.example.com" + _, err = leaf.Verify(opts) + require.NoError(t, err) + + // Wildcard SAN covering an arbitrary subdomain. + opts.DNSName = "foo.home.example.com" + _, err = leaf.Verify(opts) + require.NoError(t, err) +} + +func TestIssueLeaf_RejectsDomainOutsidePermittedDomains(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + _, _, _, err = ca.IssueLeaf("evil.example.org", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "evil.example.org") +} + +// TestLeafOutsidePermittedDomains_FailsVerify is the test that proves the +// name constraints on the tenant root are cryptographically real rather than +// a policy check IssueLeaf happens to apply. It signs a leaf for a +// domain outside PermittedDNSDomains directly against the CA's own key, +// bypassing IssueLeaf's Authorized guard entirely — simulating exactly the +// "our server-side check is bypassed or buggy" scenario the design doc +// calls out. If this leaf were to verify, the constraints baked into the +// root would be decorative, not enforced. +func TestLeafOutsidePermittedDomains_FailsVerify(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + leafPEM := forgeLeaf(t, ca, &x509.Certificate{ + DNSNames: []string{"evil.example.org"}, + }) + leaf := parseCertPEM(t, leafPEM) + + pool := x509.NewCertPool() + pool.AddCert(ca.root) + + _, err = leaf.Verify(x509.VerifyOptions{ + DNSName: "evil.example.org", + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + require.Error(t, err, "a leaf for a domain outside PermittedDNSDomains must fail verification") + var constraintErr x509.CertificateInvalidError + if assert.ErrorAs(t, err, &constraintErr) { + assert.Equal(t, x509.CANotAuthorizedForThisName, constraintErr.Reason) + } +} + +// TestLeafWithIPSAN_FailsVerify proves the excluded 0.0.0.0/0 and ::/0 +// ranges actually block IP SANs, again by forging a leaf directly against +// the CA key so the test does not depend on IssueLeaf ever choosing to +// offer an IP SAN option. +func TestLeafWithIPSAN_FailsVerify(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + leafPEM := forgeLeaf(t, ca, &x509.Certificate{ + DNSNames: []string{"home.example.com"}, + IPAddresses: []net.IP{net.ParseIP("10.0.0.1")}, + }) + leaf := parseCertPEM(t, leafPEM) + + pool := x509.NewCertPool() + pool.AddCert(ca.root) + + _, err = leaf.Verify(x509.VerifyOptions{ + // VerifyHostname matches an IP-address-shaped DNSName against + // IPAddresses SANs (RFC 6125 Appendix B.2) — this is the standard way + // to exercise IP SAN validation through VerifyOptions. + DNSName: "10.0.0.1", + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + require.Error(t, err, "a leaf carrying an IP SAN must fail verification against this root") +} + +// TestTenantRootCannotSignSubordinateCA proves MaxPathLen 0 actually forbids +// a subordinate CA: it forges an intermediate CA cert directly beneath the +// tenant root (again bypassing the public API, which offers no way to mint +// one), uses that intermediate to sign a leaf, and asserts the resulting +// chain fails verification even though every individual signature is valid. +func TestTenantRootCannotSignSubordinateCA(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + subKey := generateTestRSAKey(t) + subTemplate := &x509.Certificate{ + SerialNumber: testSerial(t), + Subject: pkixCN(t, "Tenant Sub CA"), + NotBefore: ca.root.NotBefore, + NotAfter: ca.root.NotAfter, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + } + subDER, err := x509.CreateCertificate(rngForTest(), subTemplate, ca.root, &subKey.PublicKey, ca.key) + require.NoError(t, err) + subCert, err := x509.ParseCertificate(subDER) + require.NoError(t, err) + + leafKey := generateTestRSAKey(t) + leafTemplate := &x509.Certificate{ + SerialNumber: testSerial(t), + Subject: pkixCN(t, "home.example.com"), + NotBefore: ca.root.NotBefore, + NotAfter: ca.root.NotAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{"home.example.com"}, + } + leafDER, err := x509.CreateCertificate(rngForTest(), leafTemplate, subCert, &leafKey.PublicKey, subKey) + require.NoError(t, err) + leaf, err := x509.ParseCertificate(leafDER) + require.NoError(t, err) + + roots := x509.NewCertPool() + roots.AddCert(ca.root) + intermediates := x509.NewCertPool() + intermediates.AddCert(subCert) + + _, err = leaf.Verify(x509.VerifyOptions{ + DNSName: "home.example.com", + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + require.Error(t, err, "a leaf issued through a subordinate CA beneath a pathlen-0 root must fail verification") +} + +func TestAuthorized(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + cases := []struct { + name string + domain string + want bool + }{ + {"apex", "home.example.com", true}, + {"subdomain", "foo.home.example.com", true}, + {"deep subdomain", "a.b.home.example.com", true}, + {"wildcard", "*.home.example.com", true}, + {"case-insensitive apex", "HOME.EXAMPLE.COM", true}, + {"case-insensitive subdomain", "Foo.Home.Example.Com", true}, + {"unrelated domain", "example.org", false}, + {"superstring, not a subdomain", "nothome.example.com", false}, + {"parent of permitted apex", "example.com", false}, + {"empty", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ca.Authorized(tc.domain)) + }) + } +} + +func TestLoadTenantCA_MissingReturnsActionableError(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + _, err := LoadTenantCA(certPath, keyPath) + require.Error(t, err) + assert.Contains(t, err.Error(), tenantCACreateCommandHint, "error must name the admin command that creates the CA") +} + +func TestLoadTenantCA_IncompletePairReturnsActionableError(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + _, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + require.NoError(t, os.Remove(keyPath)) + + _, err = LoadTenantCA(certPath, keyPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "incomplete") + assert.Contains(t, err.Error(), tenantCACreateCommandHint) +} + +func TestLoadTenantCA_RoundTripsCreatedCA(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + + created, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com", "lab.example.org"}) + require.NoError(t, err) + + loaded, err := LoadTenantCA(certPath, keyPath) + require.NoError(t, err) + + // Same root: identical serial number and PEM prove LoadTenantCA loaded + // rather than fabricated anything. + assert.Equal(t, 0, created.root.SerialNumber.Cmp(loaded.root.SerialNumber)) + assert.Equal(t, created.rootPEM, loaded.rootPEM) + assert.Equal(t, []string{"home.example.com", "lab.example.org"}, loaded.PermittedDomains()) + + // PermittedDomains reflects exactly what was baked in at creation, read + // back from the certificate rather than a parallel list. + assert.True(t, loaded.Authorized("home.example.com")) + assert.True(t, loaded.Authorized("foo.lab.example.org")) + assert.False(t, loaded.Authorized("evil.example.net")) +} + +func TestPermittedDomains_ReturnsDefensiveCopy(t *testing.T) { + certPath, keyPath := tenantCAPaths(t) + ca, err := LoadOrCreateTenantCA(certPath, keyPath, []string{"home.example.com"}) + require.NoError(t, err) + + got := ca.PermittedDomains() + require.Equal(t, []string{"home.example.com"}, got) + + got[0] = "mutated.example.com" + assert.Equal(t, []string{"home.example.com"}, ca.PermittedDomains(), "mutating the returned slice must not affect the CA") +} + +// -- test helpers -------------------------------------------------------- + +// forgeLeaf signs overrides directly against ca's root and key, filling in +// only the fields a real caller of IssueLeaf would not control, so tests can +// construct leaves IssueLeaf itself would refuse to produce. +func forgeLeaf(t *testing.T, ca *TenantCA, overrides *x509.Certificate) string { + t.Helper() + key := generateTestRSAKey(t) + overrides.SerialNumber = testSerial(t) + if overrides.Subject.CommonName == "" && len(overrides.DNSNames) > 0 { + overrides.Subject = pkixCN(t, overrides.DNSNames[0]) + } + overrides.NotBefore = ca.root.NotBefore + overrides.NotAfter = ca.root.NotAfter + overrides.KeyUsage = x509.KeyUsageDigitalSignature + overrides.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + overrides.BasicConstraintsValid = true + + der, err := x509.CreateCertificate(rngForTest(), overrides, ca.root, &key.PublicKey, ca.key) + require.NoError(t, err) + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) +} + +func parseCertPEM(t *testing.T, certPEM string) *x509.Certificate { + t.Helper() + block, _ := pem.Decode([]byte(certPEM)) + require.NotNil(t, block) + cert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + return cert +} + +// generateTestRSAKey returns a key sized for fast test signing — not +// production-strength, and never used outside this test file. +func generateTestRSAKey(t *testing.T) *rsa.PrivateKey { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, leafKeyBits) + require.NoError(t, err) + return key +} + +// testSerial returns a random 128-bit serial number for a forged test +// certificate. +func testSerial(t *testing.T) *big.Int { + t.Helper() + serial, err := randomSerial() + require.NoError(t, err) + return serial +} + +// pkixCN builds a minimal Subject carrying only a CommonName, for forged +// test certificates that do not go through IssueLeaf. +func pkixCN(t *testing.T, cn string) pkix.Name { + t.Helper() + return pkix.Name{CommonName: cn} +} + +// rngForTest is crypto/rand.Reader, named for readability at x509.CreateCertificate call sites in this file. +func rngForTest() io.Reader { + return rand.Reader +} diff --git a/spinifex/handlers/acm/renewal.go b/spinifex/handlers/acm/renewal.go new file mode 100644 index 000000000..9c2fda8d3 --- /dev/null +++ b/spinifex/handlers/acm/renewal.go @@ -0,0 +1,404 @@ +package handlers_acm + +// renewal.go implements the PRIVATE_CA renewal worker described in the ACM +// design doc's "Renewal" section: a scan-and-lease worker, sharing the +// per-certificate lease pattern reserved on CertRecord, that reissues a +// tenant-CA-signed certificate under its existing ARN once it crosses a +// proportional fraction of its lifetime. Re-signing from the tenant CA is +// fully offline, so this worker never touches ACME, lego, or the network — +// unlike PROVIDER_API/MANUAL_TXT renewal, which is deferred along with the +// rest of public issuance. + +import ( + "context" + "errors" + "fmt" + "hash/fnv" + "log/slog" + "time" + + "github.com/aws/aws-sdk-go/service/acm" + "github.com/mulgadc/spinifex/spinifex/awserrors" +) + +const ( + // renewalLifetimeFraction is the proportion of a certificate's total + // lifetime (NotAfter - NotBefore) that must elapse before renewal is due. + // Proportional rather than a fixed calendar window: a 30-day window is + // correct for a 90-day certificate and wrong for a short-lived one, where + // it degenerates into "renew immediately, forever". Two-thirds self-tunes + // to whatever lifetime a certificate actually carries. + renewalLifetimeFraction = 2.0 / 3.0 + + // minRenewalWindow floors the computed window so a very short-lived + // certificate does not renew in a near-constant loop — two-thirds of a + // ten-minute lifetime is under seven minutes, which is thrash, not + // maintenance. + minRenewalWindow = 1 * time.Hour + + // renewalJitterSpread bounds the deterministic per-ARN offset added on + // top of the renewal window, so certificates minted together in one + // `apply` do not all become due in the same scan tick a lifetime later. + renewalJitterSpread = 10 * time.Minute + + // renewalFailureBackoff is the minimum time between renewal attempts for + // a certificate whose most recent attempt failed. A certificate outside + // the tenant CA's name constraints, or issued while no tenant CA was + // wired, cannot be fixed by retrying — the fix is an operator action + // (regenerate the CA, or create one) — so retrying every scan tick would + // just spin. An hour, well over the default scan interval, still notices + // promptly once the operator has actually fixed it. + renewalFailureBackoff = 1 * time.Hour + + // defaultRenewalScanInterval is how often the worker looks for + // certificates past their renewal window. + defaultRenewalScanInterval = 5 * time.Minute + + // defaultRenewalLeaseTTL bounds how long a per-certificate lease survives + // without being released, so a holder that crashes mid-renewal does not + // block that certificate forever — the next scan (here or on another + // node) reclaims it once the TTL passes. + defaultRenewalLeaseTTL = 2 * time.Minute +) + +// renewalWindow returns the elapsed-lifetime duration after which a +// certificate with the given total lifetime becomes due for renewal: +// two-thirds of lifetime, floored at minRenewalWindow. +func renewalWindow(lifetime time.Duration) time.Duration { + window := time.Duration(float64(lifetime) * renewalLifetimeFraction) + if window < minRenewalWindow { + return minRenewalWindow + } + return window +} + +// renewalJitter derives a deterministic offset in [0, renewalJitterSpread) +// from certArn by hashing it with FNV-1a. Deterministic, not random: the same +// ARN yields the same offset on every call, including across process +// restarts, while distinct ARNs spread across the jitter range instead of +// all becoming due on the same scan tick. +func renewalJitter(certArn string) time.Duration { + h := fnv.New64a() + _, _ = h.Write([]byte(certArn)) // hash.Hash.Write never errors + return time.Duration(h.Sum64() % uint64(renewalJitterSpread)) +} + +// renewalDueAt returns the instant at which rec becomes due for renewal: +// NotBefore, plus the proportional (floored) window, plus this ARN's +// deterministic jitter offset. +func renewalDueAt(rec *CertRecord) time.Time { + lifetime := rec.NotAfter.Sub(rec.NotBefore) + return rec.NotBefore.Add(renewalWindow(lifetime)).Add(renewalJitter(rec.CertificateArn)) +} + +// renewable reports whether rec is a candidate for automatic renewal at now: +// a PRIVATE (tenant-CA-issued) certificate, ISSUED, marked ELIGIBLE, past its +// renewal window, and — if its last attempt failed — past the failure +// backoff. Imported certificates (Type IMPORTED) and MANUAL_TXT-validated +// certificates are never renewable (see renewalEligibilityForMode in +// service_impl.go); AMAZON_ISSUED (PROVIDER_API) renewal is out of scope for +// this worker, which drives PRIVATE_CA reissuance only. +func renewable(rec *CertRecord, now time.Time) bool { + if rec.Type != acm.CertificateTypePrivate { + return false + } + if rec.Status != certStatusIssued { + return false + } + if rec.RenewalEligibility != acm.RenewalEligibilityEligible { + return false + } + if rec.RenewalSummary != nil && rec.RenewalSummary.RenewalStatus == acm.RenewalStatusFailed && + now.Sub(rec.RenewalSummary.UpdatedAt) < renewalFailureBackoff { + return false + } + return !renewalDueAt(rec).After(now) +} + +// Worker scans stored certificates for PRIVATE_CA-issued ones past their +// renewal window and reissues them under their existing ARN with a fresh +// leaf key, then pushes the new material to every load balancer that +// references the ARN. +// +// One Worker runs per node (wired alongside svc in the daemon), mirroring the +// issuance-worker model in the ACM design doc: every node is a candidate +// driver, and the per-certificate lease in svc's Store ensures only one of +// them actually renews a given certificate. This costs nothing extra on a +// single-node deployment and needs no leader election. +// +// Worker holds svc rather than a snapshot of its TenantCA/CertMaterialUpdated +// fields so that a tenant CA loaded after the worker starts (the daemon does +// not treat a missing CA as fatal) is picked up on the next scan without a +// restart. +type Worker struct { + svc *ACMServiceImpl + holderID string + now func() time.Time + + scanInterval time.Duration + leaseTTL time.Duration +} + +// WorkerOption tunes a Worker's scan cadence, lease TTL, or clock. All three +// have sane production defaults; options exist for tests (which cannot wait +// out real renewal windows) and are not exposed as config-file or +// environment-variable knobs. +type WorkerOption func(*Worker) + +// WithRenewalScanInterval overrides how often the worker scans for +// certificates past their renewal window. +func WithRenewalScanInterval(d time.Duration) WorkerOption { + return func(w *Worker) { w.scanInterval = d } +} + +// WithRenewalLeaseTTL overrides the per-certificate lease TTL. +func WithRenewalLeaseTTL(d time.Duration) WorkerOption { + return func(w *Worker) { w.leaseTTL = d } +} + +// WithRenewalClock overrides the worker's notion of "now", so a test can move +// a certificate across its renewal window without waiting for real time to +// pass. +func WithRenewalClock(now func() time.Time) WorkerOption { + return func(w *Worker) { w.now = now } +} + +// NewWorker builds a renewal worker over svc. holderID identifies this node +// in the per-certificate lease (the daemon passes its node ID, mirroring +// handlers_ecs.NewScheduler's holder identity). +func NewWorker(svc *ACMServiceImpl, holderID string, opts ...WorkerOption) *Worker { + w := &Worker{ + svc: svc, + holderID: holderID, + now: time.Now, + scanInterval: defaultRenewalScanInterval, + leaseTTL: defaultRenewalLeaseTTL, + } + for _, o := range opts { + o(w) + } + return w +} + +// Run scans on WithRenewalScanInterval's cadence until ctx is cancelled. The +// daemon starts one Run per node in its own goroutine and stops it by +// cancelling ctx on shutdown. +func (w *Worker) Run(ctx context.Context) { + ticker := time.NewTicker(w.scanInterval) + defer ticker.Stop() + + // A scan with nothing due logs nothing, so without this an operator cannot + // distinguish a running worker from one that never started. + slog.InfoContext(ctx, "ACM renewal worker started", "scan_interval", w.scanInterval) + + w.scanOnce(ctx) // don't wait a full interval before the first pass + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + w.scanOnce(ctx) + } + } +} + +// scanOnce lists every certificate in the deployment and attempts to renew +// each one renewable at the current time. Per-certificate errors (lease loss, +// CA authorization failure, store errors) are logged and do not stop the +// scan — one bad certificate must not block renewal of the rest. +func (w *Worker) scanOnce(ctx context.Context) { + if w.svc.TenantCA == nil { + // No tenant CA wired: nothing this worker can renew. An operator + // using only public certificates has no reason to own one, so this + // is an expected steady state, not a fault — logged at Debug. + slog.DebugContext(ctx, "acm renewal: no tenant CA wired, skipping scan") + return + } + // Metadata only: the predicate below never reads PrivateKey, and the + // winner is re-fetched (decrypted) via GetCert in renewOne once it has + // actually won the lease — see ListAllCertMetadata. + recs, err := w.svc.store.ListAllCertMetadata(ctx) + if err != nil { + slog.WarnContext(ctx, "acm renewal: list certificates failed", "err", err) + return + } + now := w.now() + for _, rec := range recs { + if !renewable(rec, now) { + continue + } + if _, err := w.renewWithLease(ctx, rec.CertificateArn); err != nil { + slog.WarnContext(ctx, "acm renewal: renewal attempt failed", "arn", rec.CertificateArn, "err", err) + } + } +} + +// renewWithLease acquires certArn's per-certificate lease, renews it, and +// releases the lease, all under holderID. acquired reports whether the lease +// was won: false with a nil error means another node currently holds it, a +// case the scheduled scan treats as "someone else has this" (not logged) and +// ForceRenewCertificate treats as an error to report to the caller. +func (w *Worker) renewWithLease(ctx context.Context, certArn string) (acquired bool, err error) { + acquired, err = w.svc.store.AcquireLease(ctx, certArn, w.holderID, w.leaseTTL, w.now()) + if err != nil { + return false, fmt.Errorf("acquire lease: %w", err) + } + if !acquired { + return false, nil + } + defer func() { + if relErr := w.svc.store.ReleaseLease(ctx, certArn, w.holderID); relErr != nil { + // The TTL reaps it either way; this just means another node + // waits out the TTL instead of picking it up immediately. + slog.WarnContext(ctx, "acm renewal: lease release failed (TTL will reap)", "arn", certArn, "err", relErr) + } + }() + return true, w.renewOne(ctx, certArn) +} + +// renewOne re-fetches certArn under the caller's lease, verifies it is still +// PRIVATE_CA-authorized, and reissues its leaf from the tenant CA under the +// same ARN, domain and SANs with a fresh key — then pushes the new material +// out via CertMaterialUpdated so every load balancer referencing the ARN +// re-renders. Terminal authorization failures (no tenant CA, or a domain that +// no longer falls within the CA's name constraints — e.g. the operator +// regenerated the root with a narrower domain list) are recorded on +// RenewalSummary via recordRenewalFailure rather than retried immediately. +func (w *Worker) renewOne(ctx context.Context, certArn string) error { + rec, err := w.svc.store.GetCert(ctx, certArn) + if err != nil { + return fmt.Errorf("get cert: %w", err) + } + if rec == nil { + return nil // deleted since the scan listed it; nothing to do + } + + if w.svc.TenantCA == nil { + return w.recordRenewalFailure(ctx, rec, acm.FailureReasonOther, + fmt.Sprintf("no tenant CA wired; run %q", tenantCACreateCommandHint)) + } + domains := uniqueDomains(rec.DomainName, rec.SubjectAltNames) + for _, d := range domains { + if !w.svc.TenantCA.Authorized(d) { + return w.recordRenewalFailure(ctx, rec, acm.FailureReasonPcaNameConstraintsValidation, + fmt.Sprintf("domain %q is outside the tenant CA's permitted domains", d)) + } + } + + certPEM, chainPEM, keyPEM, err := w.svc.TenantCA.IssueLeaf(rec.DomainName, rec.SubjectAltNames) + if err != nil { + return w.recordRenewalFailure(ctx, rec, acm.FailureReasonOther, err.Error()) + } + leaf, err := parseLeaf([]byte(certPEM)) + if err != nil { + return w.recordRenewalFailure(ctx, rec, acm.FailureReasonOther, "issued leaf was unparseable: "+err.Error()) + } + + // Same ARN, domain and SANs; everything derived from the fresh leaf + // changes. rec.DomainName/SubjectAltNames are left untouched. + rec.Certificate = certPEM + rec.CertificateChain = chainPEM + rec.PrivateKey = keyPEM + rec.Serial = leaf.SerialNumber.Text(16) + rec.Subject = leaf.Subject.String() + rec.Issuer = leaf.Issuer.String() + rec.KeyAlgorithm = keyAlgorithm(leaf) + rec.NotBefore = leaf.NotBefore + rec.NotAfter = leaf.NotAfter + rec.RenewalSummary = &RenewalSummaryRecord{ + RenewalStatus: acm.RenewalStatusSuccess, + UpdatedAt: w.now(), + } + + if err := w.svc.store.PutCert(ctx, rec); err != nil { + return fmt.Errorf("store renewed cert: %w", err) + } + slog.InfoContext(ctx, "acm renewal: reissued", "arn", certArn, "domain", rec.DomainName, "not_after", rec.NotAfter) + + if w.svc.CertMaterialUpdated != nil { + // The store write already succeeded; a fan-out failure is logged, + // never propagated, mirroring ImportCertificate's re-import fan-out — + // a rendering problem on one load balancer must not fail the + // renewal that already succeeded. + if fanErr := w.svc.CertMaterialUpdated(ctx, certArn); fanErr != nil { + slog.ErrorContext(ctx, "acm renewal: fan-out to load balancers failed", "arn", certArn, "err", fanErr) + } + } + return nil +} + +// recordRenewalFailure persists a FAILED RenewalSummary with reason on rec — +// the sole diagnostic surface DescribeCertificate exposes for a certificate +// that issued cleanly but can no longer renew — and returns an error +// describing the failure for the caller/log. It does not touch +// RenewalEligibility: a certificate that is temporarily outside the CA's +// constraints (an operator can regenerate the CA to cover it again) is still +// eligible in principle, just not renewable right now. +func (w *Worker) recordRenewalFailure(ctx context.Context, rec *CertRecord, reason, detail string) error { + rec.RenewalSummary = &RenewalSummaryRecord{ + RenewalStatus: acm.RenewalStatusFailed, + RenewalStatusReason: reason, + UpdatedAt: w.now(), + } + if err := w.svc.store.PutCert(ctx, rec); err != nil { + return fmt.Errorf("store renewal failure: %w", err) + } + slog.WarnContext(ctx, "acm renewal: failed", "arn", rec.CertificateArn, "reason", reason, "detail", detail) + return fmt.Errorf("acm renewal: %s: %s", reason, detail) +} + +// ForceRenewCertificateInput names the certificate to renew immediately. +type ForceRenewCertificateInput struct { + CertificateArn string `json:"certificate_arn"` +} + +// ForceRenewCertificateOutput reports the reissued certificate's new serial +// and expiry. +type ForceRenewCertificateOutput struct { + CertificateArn string `json:"certificate_arn"` + Serial string `json:"serial"` + NotAfter time.Time `json:"not_after"` +} + +// ForceRenewCertificate reissues input.CertificateArn immediately, bypassing +// the renewal window and failure backoff — but not lease contention or CA +// authorization. This is the operator-triggered path (wired to `spx admin +// cert force-renew` over NATS subject "acm.ForceRenewCertificate") used to +// verify the ELBv2 fan-out live without waiting for the proportional window +// to elapse. +func (w *Worker) ForceRenewCertificate(ctx context.Context, input *ForceRenewCertificateInput, accountID string) (*ForceRenewCertificateOutput, error) { + if input == nil || input.CertificateArn == "" { + return nil, errors.New(awserrors.ErrorACMInvalidArn) + } + rec, err := w.svc.lookupOwned(ctx, input.CertificateArn, accountID) + if err != nil { + return nil, err + } + if rec.Type != acm.CertificateTypePrivate { + return nil, fmt.Errorf("acm renewal: %s is a %s certificate; only PRIVATE_CA certificates can be force-renewed", input.CertificateArn, rec.Type) + } + + acquired, err := w.renewWithLease(ctx, input.CertificateArn) + if err != nil { + return nil, err + } + if !acquired { + return nil, fmt.Errorf("acm renewal: %s is currently leased by another node; try again shortly", input.CertificateArn) + } + + // Re-read for the post-renewal outcome only: serial, validity and renewal + // status. The new key was already written by the reissue above and is not + // read here, so this takes the metadata read. + renewed, err := w.svc.store.GetCertMetadata(ctx, input.CertificateArn) + if err != nil { + return nil, fmt.Errorf("get renewed cert: %w", err) + } + if renewed.RenewalSummary != nil && renewed.RenewalSummary.RenewalStatus == acm.RenewalStatusFailed { + return nil, fmt.Errorf("acm renewal: %s: %s", renewed.RenewalSummary.RenewalStatusReason, input.CertificateArn) + } + return &ForceRenewCertificateOutput{ + CertificateArn: renewed.CertificateArn, + Serial: renewed.Serial, + NotAfter: renewed.NotAfter, + }, nil +} diff --git a/spinifex/handlers/acm/renewal_test.go b/spinifex/handlers/acm/renewal_test.go new file mode 100644 index 000000000..3f56e7f75 --- /dev/null +++ b/spinifex/handlers/acm/renewal_test.go @@ -0,0 +1,388 @@ +package handlers_acm + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/acm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --- Proportional renewal window ------------------------------------------ + +// TestRenewable_ProportionalWindow exercises the four cases the renewal +// predicate must get right: not yet due, exactly at the threshold, overdue, +// and a lifetime short enough that the two-thirds fraction is floored. +// jitter is computed from the fixed test ARN so the boundary cases land +// exactly on renewalDueAt rather than depending on it landing on a whole +// unit of time. +func TestRenewable_ProportionalWindow(t *testing.T) { + const arn = "arn:aws:acm:ap-southeast-2:1:certificate/window-test" + jitter := renewalJitter(arn) + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + newRec := func(lifetime, elapsedBeforeNow time.Duration) *CertRecord { + notBefore := now.Add(-elapsedBeforeNow) + return &CertRecord{ + CertificateArn: arn, + Type: acm.CertificateTypePrivate, + Status: certStatusIssued, + RenewalEligibility: acm.RenewalEligibilityEligible, + NotBefore: notBefore, + NotAfter: notBefore.Add(lifetime), + } + } + + t.Run("not yet due", func(t *testing.T) { + lifetime := 90 * 24 * time.Hour + window := renewalWindow(lifetime) + rec := newRec(lifetime, window+jitter-time.Minute) + assert.False(t, renewable(rec, now)) + }) + + t.Run("exactly at threshold", func(t *testing.T) { + lifetime := 90 * 24 * time.Hour + window := renewalWindow(lifetime) + rec := newRec(lifetime, window+jitter) + assert.True(t, renewable(rec, now)) + }) + + t.Run("overdue", func(t *testing.T) { + lifetime := 90 * 24 * time.Hour + window := renewalWindow(lifetime) + rec := newRec(lifetime, window+jitter+24*time.Hour) + assert.True(t, renewable(rec, now)) + }) + + t.Run("very short lifetime hits the floor", func(t *testing.T) { + lifetime := 10 * time.Minute + window := renewalWindow(lifetime) + require.Equal(t, minRenewalWindow, window, + "a 10-minute lifetime's raw two-thirds fraction (~6m40s) must be floored to minRenewalWindow") + + // Past the raw 2/3 fraction (~6m40s elapsed) but short of the floor: + // must not be renewable yet, or the floor would be pointless. + notDue := newRec(lifetime, 7*time.Minute) + assert.False(t, renewable(notDue, now)) + + due := newRec(lifetime, window+jitter) + assert.True(t, renewable(due, now)) + }) +} + +// TestRenewable_GatesOnTypeStatusEligibility asserts every non-window gate: +// only a PRIVATE, ISSUED, ELIGIBLE certificate is ever renewable, regardless +// of how overdue it is. +func TestRenewable_GatesOnTypeStatusEligibility(t *testing.T) { + now := time.Now() + overdue := func() *CertRecord { + return &CertRecord{ + CertificateArn: "arn:aws:acm:ap-southeast-2:1:certificate/gate-test", + Type: acm.CertificateTypePrivate, + Status: certStatusIssued, + RenewalEligibility: acm.RenewalEligibilityEligible, + NotBefore: now.Add(-365 * 24 * time.Hour), + NotAfter: now.Add(-time.Hour), + } + } + + tests := []struct { + name string + mutate func(*CertRecord) + }{ + {"imported certificate", func(r *CertRecord) { r.Type = certTypeImported }}, + {"amazon-issued (public) certificate", func(r *CertRecord) { r.Type = acm.CertificateTypeAmazonIssued }}, + {"pending validation status", func(r *CertRecord) { r.Status = acm.CertificateStatusPendingValidation }}, + {"renewal ineligible", func(r *CertRecord) { r.RenewalEligibility = acm.RenewalEligibilityIneligible }}, + {"eligibility unset (legacy record)", func(r *CertRecord) { r.RenewalEligibility = "" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := overdue() + tt.mutate(rec) + assert.False(t, renewable(rec, now)) + }) + } +} + +// --- Deterministic per-ARN jitter ------------------------------------------- + +func TestRenewalJitter_DeterministicPerARN(t *testing.T) { + const arn = "arn:aws:acm:ap-southeast-2:1:certificate/jitter-test" + first := renewalJitter(arn) + second := renewalJitter(arn) + assert.Equal(t, first, second, "the same ARN must yield the same offset on every call, including across restarts") + assert.GreaterOrEqual(t, first, time.Duration(0)) + assert.Less(t, first, renewalJitterSpread) +} + +func TestRenewalJitter_SpreadsAcrossDistinctARNs(t *testing.T) { + offsets := make(map[time.Duration]bool) + for i := range 8 { + arn := fmt.Sprintf("arn:aws:acm:ap-southeast-2:1:certificate/cert-%d", i) + offsets[renewalJitter(arn)] = true + } + assert.Greater(t, len(offsets), 1, "eight distinct ARNs issued together must not all collapse onto the same offset") +} + +// --- Worker: renews the right certificates, leaves the rest alone ---------- + +// backdateToOverdue rewrites certArn's NotBefore/NotAfter so it is well past +// its renewal window, without waiting for real time to pass. +func backdateToOverdue(t *testing.T, svc *ACMServiceImpl, certArn string) { + t.Helper() + rec, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + rec.NotBefore = time.Now().Add(-100 * 24 * time.Hour) + rec.NotAfter = rec.NotBefore.Add(90 * 24 * time.Hour) + require.NoError(t, svc.store.PutCert(context.Background(), rec)) +} + +// TestWorker_RenewOne_PreservesIdentityChangesKeyAndSerial is the core +// renewal contract: same ARN, same domain and SANs, fresh key and serial. +func TestWorker_RenewOne_PreservesIdentityChangesKeyAndSerial(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{ + "renew.example.com": true, + "www.renew.example.com": true, + }} + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("renew.example.com"), + SubjectAlternativeNames: aws.StringSlice([]string{"www.renew.example.com"}), + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + backdateToOverdue(t, svc, certArn) + + before, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + + worker := NewWorker(svc, "node-1") + acquired, err := worker.renewWithLease(context.Background(), certArn) + require.NoError(t, err) + require.True(t, acquired) + + after, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + + assert.Equal(t, certArn, after.CertificateArn) + assert.Equal(t, "renew.example.com", after.DomainName) + assert.ElementsMatch(t, []string{"www.renew.example.com"}, after.SubjectAltNames) + assert.NotEqual(t, before.Serial, after.Serial, "renewal must mint a fresh serial") + assert.NotEqual(t, before.PrivateKey, after.PrivateKey, "renewal must mint a fresh leaf key") + assert.True(t, after.NotAfter.After(before.NotAfter), "the renewed certificate must have a later expiry") + require.NotNil(t, after.RenewalSummary) + assert.Equal(t, acm.RenewalStatusSuccess, after.RenewalSummary.RenewalStatus) +} + +// TestWorker_CertMaterialUpdated_InvokedExactlyOnceOnSuccess asserts the +// ELBv2 fan-out hook fires exactly once per successful renewal — the +// mandatory push described in the ACM design doc's ELBv2 propagation +// section. +func TestWorker_CertMaterialUpdated_InvokedExactlyOnceOnSuccess(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"fanout.example.com": true}} + var calls int + var gotArn string + svc.CertMaterialUpdated = func(_ context.Context, certArn string) error { + calls++ + gotArn = certArn + return nil + } + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("fanout.example.com"), + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + backdateToOverdue(t, svc, certArn) + + worker := NewWorker(svc, "node-1") + acquired, err := worker.renewWithLease(context.Background(), certArn) + require.NoError(t, err) + require.True(t, acquired) + + assert.Equal(t, 1, calls, "CertMaterialUpdated must be invoked exactly once on a successful renewal") + assert.Equal(t, certArn, gotArn) +} + +// TestWorker_ScanOnce_SkipsIneligibleAndImported_RenewsEligible drives a full +// scan over a mix of certificate shapes and asserts only the eligible, +// overdue PRIVATE_CA certificate is renewed. +func TestWorker_ScanOnce_SkipsIneligibleAndImported_RenewsEligible(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{ + "eligible.example.com": true, + }} + var renewedArns []string + svc.CertMaterialUpdated = func(_ context.Context, certArn string) error { + renewedArns = append(renewedArns, certArn) + return nil + } + + // MANUAL_TXT: INELIGIBLE by construction (challenge token rotates per order). + svc.NorthstarHostsZone = func(string) bool { return true } + manualOut, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("manual-skip.example.com"), + }, testAccountID) + require.NoError(t, err) + + // Imported: also INELIGIBLE, and Type IMPORTED regardless. + c, k := genCert(t, "imported-skip.example.com") + impOut, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c, PrivateKey: k}, testAccountID) + require.NoError(t, err) + + // Eligible PRIVATE_CA certificate, backdated past its renewal window. + svc.NorthstarHostsZone = nil + eligibleOut, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("eligible.example.com"), + }, testAccountID) + require.NoError(t, err) + eligibleArn := aws.StringValue(eligibleOut.CertificateArn) + backdateToOverdue(t, svc, eligibleArn) + + worker := NewWorker(svc, "node-1") + worker.scanOnce(context.Background()) + + assert.Equal(t, []string{eligibleArn}, renewedArns, + "only the eligible, overdue PRIVATE_CA certificate may be renewed") + + manualRec, err := svc.store.GetCert(context.Background(), aws.StringValue(manualOut.CertificateArn)) + require.NoError(t, err) + assert.Nil(t, manualRec.RenewalSummary, "an ineligible certificate must never be touched by the renewal worker") + + impRec, err := svc.store.GetCert(context.Background(), aws.StringValue(impOut.CertificateArn)) + require.NoError(t, err) + assert.Nil(t, impRec.RenewalSummary, "an imported certificate must never be touched by the renewal worker") +} + +// TestWorker_ScanOnce_DomainOutsideConstraints_RecordsFailureAndBacksOff +// simulates an operator regenerating the tenant CA with a narrower domain +// list after a certificate was issued: renewal must record the failure on +// RenewalSummary (the sole DescribeCertificate diagnostic surface) and must +// not re-attempt on the very next scan. +func TestWorker_ScanOnce_DomainOutsideConstraints_RecordsFailureAndBacksOff(t *testing.T) { + svc := setupACMService(t) + ca := &fakeCertAuthority{permitted: map[string]bool{"narrowed.example.com": true}} + svc.TenantCA = ca + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("narrowed.example.com"), + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + backdateToOverdue(t, svc, certArn) + + // The root now excludes narrowed.example.com. + ca.permitted = map[string]bool{} + + fixedNow := time.Now() + worker := NewWorker(svc, "node-1", WithRenewalClock(func() time.Time { return fixedNow })) + worker.scanOnce(context.Background()) + + rec, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + require.NotNil(t, rec.RenewalSummary) + assert.Equal(t, acm.RenewalStatusFailed, rec.RenewalSummary.RenewalStatus) + assert.Equal(t, acm.FailureReasonPcaNameConstraintsValidation, rec.RenewalSummary.RenewalStatusReason) + firstUpdatedAt := rec.RenewalSummary.UpdatedAt + + // Same fixed clock, so a second scan lands inside the failure backoff + // window — it must be a no-op, not a second attempt. + worker.scanOnce(context.Background()) + rec2, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + assert.Equal(t, firstUpdatedAt, rec2.RenewalSummary.UpdatedAt, + "a scan inside the failure backoff window must not re-attempt renewal") +} + +// TestWorker_LeaseContention_OnlyOneNodeRenews asserts the per-certificate +// lease actually excludes a second worker: node-a's live lease must stop +// node-b's worker from reissuing the same certificate underneath it. +func TestWorker_LeaseContention_OnlyOneNodeRenews(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"contended.example.com": true}} + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("contended.example.com"), + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + backdateToOverdue(t, svc, certArn) + + before, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + + acquired, err := svc.store.AcquireLease(context.Background(), certArn, "node-a", 5*time.Minute, time.Now()) + require.NoError(t, err) + require.True(t, acquired, "node-a must win the lease on an unheld certificate") + + workerB := NewWorker(svc, "node-b") + won, err := workerB.renewWithLease(context.Background(), certArn) + require.NoError(t, err) + assert.False(t, won, "node-b must not win a lease node-a already holds live") + + after, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + assert.Equal(t, before.Serial, after.Serial, "the certificate must not be reissued while another node's lease is live") +} + +// --- Operator-triggered forced renewal -------------------------------------- + +// TestForceRenewCertificate_BypassesRenewalWindow is the live-verification +// path: a freshly issued certificate, nowhere near its renewal window, must +// still reissue on demand. +func TestForceRenewCertificate_BypassesRenewalWindow(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"force.example.com": true}} + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("force.example.com"), + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + + before, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + require.False(t, renewable(before, time.Now()), "a freshly issued certificate must not be naturally due") + + worker := NewWorker(svc, "node-1") + result, err := worker.ForceRenewCertificate(context.Background(), &ForceRenewCertificateInput{CertificateArn: certArn}, testAccountID) + require.NoError(t, err) + assert.Equal(t, certArn, result.CertificateArn) + + after, err := svc.store.GetCert(context.Background(), certArn) + require.NoError(t, err) + assert.NotEqual(t, before.Serial, after.Serial, "force-renew must reissue even though the certificate is not yet due") +} + +// TestForceRenewCertificate_RejectsNonPrivateType asserts force-renew refuses +// an imported certificate rather than silently no-op'ing or crashing — this +// worker only ever drives PRIVATE_CA reissuance. +func TestForceRenewCertificate_RejectsNonPrivateType(t *testing.T) { + svc := setupACMService(t) + c, k := genCert(t, "force-import.example.com") + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c, PrivateKey: k}, testAccountID) + require.NoError(t, err) + + worker := NewWorker(svc, "node-1") + _, err = worker.ForceRenewCertificate(context.Background(), &ForceRenewCertificateInput{CertificateArn: aws.StringValue(out.CertificateArn)}, testAccountID) + require.Error(t, err) +} + +// TestForceRenewCertificate_UnknownArnRejected mirrors DescribeCertificate's +// ownership check: an ARN that does not exist, or belongs to another +// account, must be rejected rather than disclosing anything about it. +func TestForceRenewCertificate_UnknownArnRejected(t *testing.T) { + svc := setupACMService(t) + worker := NewWorker(svc, "node-1") + _, err := worker.ForceRenewCertificate(context.Background(), &ForceRenewCertificateInput{ + CertificateArn: "arn:aws:acm:ap-southeast-2:1:certificate/does-not-exist", + }, testAccountID) + require.Error(t, err) +} diff --git a/spinifex/handlers/acm/service.go b/spinifex/handlers/acm/service.go index 14f21f171..83d0091f1 100644 --- a/spinifex/handlers/acm/service.go +++ b/spinifex/handlers/acm/service.go @@ -10,6 +10,7 @@ import ( type ACMService interface { ImportCertificate(ctx context.Context, input *acm.ImportCertificateInput, accountID string) (*acm.ImportCertificateOutput, error) DescribeCertificate(ctx context.Context, input *acm.DescribeCertificateInput, accountID string) (*acm.DescribeCertificateOutput, error) + GetCertificate(ctx context.Context, input *acm.GetCertificateInput, accountID string) (*acm.GetCertificateOutput, error) ListCertificates(ctx context.Context, input *acm.ListCertificatesInput, accountID string) (*acm.ListCertificatesOutput, error) DeleteCertificate(ctx context.Context, input *acm.DeleteCertificateInput, accountID string) (*acm.DeleteCertificateOutput, error) ListTagsForCertificate(ctx context.Context, input *acm.ListTagsForCertificateInput, accountID string) (*acm.ListTagsForCertificateOutput, error) diff --git a/spinifex/handlers/acm/service_impl.go b/spinifex/handlers/acm/service_impl.go index 4a4c4ba91..d6b25b9a8 100644 --- a/spinifex/handlers/acm/service_impl.go +++ b/spinifex/handlers/acm/service_impl.go @@ -29,26 +29,161 @@ const ( certTypeImported = "IMPORTED" ) +// CertAuthority issues leaf certificates from the tenant private CA and +// authorizes domains against the CA's own x509 name constraints. Satisfied by +// *TenantCA (handlers/acm/privateca.go), which is implemented separately; +// defined here as a narrow interface so this package builds and is testable +// with a fake independently of that file. +type CertAuthority interface { + // IssueLeaf signs a new leaf certificate for domain (with sans as + // additional SANs) from the tenant CA, returning the leaf, chain and a + // freshly generated private key as PEM. + IssueLeaf(domain string, sans []string) (certPEM, chainPEM, keyPEM string, err error) + // Authorized reports whether domain is within the CA's own + // permittedDNSDomains name constraints — the sole authorization source for + // a PRIVATE_CA request, so config and the CA can never drift apart. + Authorized(domain string) bool +} + // ACMServiceImpl is the local, KV-backed implementation of ACMService. type ACMServiceImpl struct { store *Store region string + + // dnsProviderConfigured drives deriveValidationMode. Wired from + // config.Config at construction (cfg.ACM.DNSProvider != ""); tests in this + // package may set it directly to exercise PROVIDER_API without a full + // config. + dnsProviderConfigured bool + + // TenantCA issues PRIVATE_CA leaves and authorizes domains against the + // tenant CA's name constraints. Nil until wired by the daemon + // post-construction (mirroring CertMaterialUpdated below); RequestCertificate + // fails loudly rather than silently skipping issuance when PRIVATE_CA is + // derived and no CA is wired. + TenantCA CertAuthority + + // NorthstarHostsZone, when set, reports whether northstar hosts a zone + // covering domain — consulted once per requested domain (primary + every + // SAN) by deriveValidationMode. Wired by the daemon post-construction + // (mirroring CertMaterialUpdated below) to handlers/dns.HostsZone. A func + // field stands in for a direct import of handlers/dns, the same pattern + // CertMaterialUpdated uses for handlers/elbv2, so this package and its + // tests stay decoupled from handlers/dns's S3/northstar-config + // dependencies. Nil means "northstar hosts nothing" — the safe default for + // tests and any deployment that never wires it, so an unwired service can + // never accidentally select MANUAL_TXT for a zone it cannot actually see. + NorthstarHostsZone func(domain string) bool + + // CertMaterialUpdated, when set, is invoked after new certificate material + // is written under an existing ARN so the caller can re-render every load + // balancer that references it (see handlers/elbv2.UpdateStoredConfigForCert). + // Wired by the daemon post-construction to avoid an elbv2 -> acm import + // cycle; nil-safe — skipped in tests and whenever ELBv2 isn't wired up. + // Fan-out errors are logged, never propagated: a rendering problem on one + // load balancer must not fail the certificate write that already succeeded. + CertMaterialUpdated func(ctx context.Context, certArn string) error } var _ ACMService = (*ACMServiceImpl)(nil) // NewACMServiceImplWithNATS builds an ACM service backed by a JetStream KV // store. cfg may be nil (tests); region then falls back to the default. -func NewACMServiceImplWithNATS(ctx context.Context, cfg *config.Config, nc *nats.Conn) (*ACMServiceImpl, error) { - store, err := NewStore(ctx, nc) +// masterKey encrypts certificate private keys at rest and is required: unlike +// EKS/ECS's IAM dependency (which degrades to "feature disabled" when the key +// is missing), ACM has no safe degraded mode for a value this sensitive, so a +// missing or malformed key fails construction outright rather than starting +// the service without at-rest protection. +func NewACMServiceImplWithNATS(ctx context.Context, cfg *config.Config, nc *nats.Conn, masterKey []byte) (*ACMServiceImpl, error) { + if len(masterKey) == 0 { + return nil, fmt.Errorf("ACM service requires a master key to encrypt certificate private keys at rest; none provided") + } + store, err := NewStore(ctx, nc, masterKey) if err != nil { return nil, err } - region := defaultRegion - if cfg != nil && cfg.Region != "" { - region = cfg.Region + svc := &ACMServiceImpl{store: store, region: defaultRegion} + if cfg != nil { + if cfg.Region != "" { + svc.region = cfg.Region + } + svc.dnsProviderConfigured = cfg.ACM.DNSProvider != "" } - return &ACMServiceImpl{store: store, region: region}, nil + return svc, nil +} + +// deriveValidationMode selects a validation mode from deployment state and the +// full set of requested domains (primary + every SAN), never from +// configuration alone or from the primary domain in isolation. +// +// Order, and why: +// 1. PRIVATE_CA, when a tenant CA is wired and its name constraints authorize +// every requested domain. A name-constrained tenant CA is a definite, +// operator-declared local capability, so it is preferred even over +// northstar hosting the zone: northstar hosting a zone is evidence the +// zone is internal, and an internal zone can never complete MANUAL_TXT (a +// public CA cannot resolve it there) — PRIVATE_CA is the only mode that +// can actually issue for it. +// 2. PROVIDER_API, when a DNS provider credential is configured. An operator +// who has wired provider credentials wants public trust driven +// automatically; that must not be downgraded just because northstar also +// happens to host the zone. +// 3. MANUAL_TXT, when northstar hosts a zone covering every requested domain +// and neither of the above applied. This is last, not first: MANUAL_TXT +// means "the operator hand-publishes an ACME TXT record for a public CA to +// resolve", which can never complete for an internal zone — it is only +// offered once PRIVATE_CA, the mode that actually works there, has been +// ruled out. +// 4. PRIVATE_CA otherwise — the same terminal fallback as before, which then +// fails the authorization check in RequestCertificate loudly when no +// tenant CA is wired or it does not cover the domain. +// +// Coverage in (1) and (3) is all-or-nothing across every domain: a request +// with mixed coverage (some domains covered, some not) must not select a mode +// it cannot actually serve for the domains it misses. +// CNAME_DELEGATION is never selected here — it is deferred until northstar can +// serve public authoritative queries, so northstar hosting the zone takes the +// manual-TXT branch of that family for now. +func (s *ACMServiceImpl) deriveValidationMode(domains []string) string { + switch { + case s.TenantCA != nil && allAuthorized(s.TenantCA, domains): + return ValidationModePrivateCA + case s.dnsProviderConfigured: + return ValidationModeProviderAPI + case s.northstarHostsAll(domains): + return ValidationModeManualTXT + default: + return ValidationModePrivateCA + } +} + +// allAuthorized reports whether ca authorizes every domain in domains. ca must +// be non-nil; domains must be non-empty (an empty set authorizes nothing). +func allAuthorized(ca CertAuthority, domains []string) bool { + if len(domains) == 0 { + return false + } + for _, d := range domains { + if !ca.Authorized(d) { + return false + } + } + return true +} + +// northstarHostsAll reports whether NorthstarHostsZone covers every domain in +// domains. A nil NorthstarHostsZone (unwired service, or a test that never set +// it) means northstar hosts nothing. +func (s *ACMServiceImpl) northstarHostsAll(domains []string) bool { + if s.NorthstarHostsZone == nil || len(domains) == 0 { + return false + } + for _, d := range domains { + if !s.NorthstarHostsZone(d) { + return false + } + } + return true } // mintCertificateArn generates an ACM-style certificate ARN for accountID. @@ -75,6 +210,8 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor } certArn := aws.StringValue(input.CertificateArn) + var inUseBy []string + reimport := certArn != "" if certArn == "" { certArn = s.mintCertificateArn(accountID) } else { @@ -86,6 +223,9 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor if existing == nil || existing.AccountID != accountID { return nil, errors.New(awserrors.ErrorResourceNotFound) } + // Carry the InUseBy index forward — new material under the same ARN + // must not silently drop the load balancers that reference it. + inUseBy = existing.InUseBy } rec := &CertRecord{ @@ -104,16 +244,217 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor NotAfter: leaf.NotAfter, ImportedAt: time.Now().UTC(), Tags: tagsToMap(input.Tags), + InUseBy: inUseBy, + Type: certTypeImported, + Status: certStatusIssued, + // Imported material has no validation and is never auto-renewed — + // re-importing (and thus re-proving control) is the operator's job. + RenewalEligibility: acm.RenewalEligibilityIneligible, } if err := s.store.PutCert(ctx, rec); err != nil { slog.ErrorContext(ctx, "ImportCertificate: store failed", "err", err) return nil, errors.New(awserrors.ErrorInternalError) } + // Fan the new material out to every load balancer already referencing this + // ARN so HAProxy picks it up on its next poll instead of silently serving + // the old leaf until it expires. Zero load balancers in use is a no-op. + if reimport && len(inUseBy) > 0 && s.CertMaterialUpdated != nil { + if fanErr := s.CertMaterialUpdated(ctx, certArn); fanErr != nil { + slog.ErrorContext(ctx, "ImportCertificate: fan-out to load balancers failed", "arn", certArn, "err", fanErr) + } + } + slog.InfoContext(ctx, "ImportCertificate: stored", "arn", certArn, "domain", rec.DomainName, "account", accountID) return &acm.ImportCertificateOutput{CertificateArn: aws.String(certArn)}, nil } +// RequestCertificate mints a CertificateArn and returns immediately with the +// certificate in PENDING_VALIDATION — it never performs issuance inline, +// matching AWS. The validation mode is derived (deriveValidationMode), not +// read from the request. Only PRIVATE_CA is driven to completion in this +// slice: it has no domain validation, so it issues synchronously against +// TenantCA and reaches ISSUED before this call returns. PROVIDER_API and +// MANUAL_TXT requests are stored correctly shaped in PENDING_VALIDATION and +// left for a later issuance worker to drive. +func (s *ACMServiceImpl) RequestCertificate(ctx context.Context, input *acm.RequestCertificateInput, accountID string) (*acm.RequestCertificateOutput, error) { + if input == nil || aws.StringValue(input.DomainName) == "" { + return nil, errors.New(awserrors.ErrorInvalidParameter) + } + domain := aws.StringValue(input.DomainName) + sans := aws.StringValueSlice(input.SubjectAlternativeNames) + allDomains := uniqueDomains(domain, sans) + + mode := s.deriveValidationMode(allDomains) + + // PRIVATE_CA has no ACME CA to prove domain control, so request-time + // authorization is read from the tenant CA's own name constraints — the + // sole source, so it and this check can never drift apart. Public modes + // need no equivalent check: the ACME CA's own validation is a stronger + // proof of control than anything Spinifex could assert itself. + if mode == ValidationModePrivateCA { + if s.TenantCA == nil { + slog.ErrorContext(ctx, "RequestCertificate: PRIVATE_CA mode derived but no tenant CA is wired", "domain", domain) + return nil, errors.New(awserrors.ErrorInternalError) + } + for _, d := range allDomains { + if !s.TenantCA.Authorized(d) { + slog.InfoContext(ctx, "RequestCertificate: domain outside tenant CA permitted domains", "domain", d) + return nil, errors.New(awserrors.ErrorInvalidParameter) + } + } + } + + certArn := s.mintCertificateArn(accountID) + rec := &CertRecord{ + CertificateArn: certArn, + AccountID: accountID, + DomainName: domain, + SubjectAltNames: sans, + Status: acm.CertificateStatusPendingValidation, + ValidationMethod: mode, + DomainValidationOptions: buildDomainValidationOptions(mode, allDomains), + RenewalEligibility: renewalEligibilityForMode(mode), + // Minted now regardless of mode so CNAME_DELEGATION, when it lands, has + // a stable target without changing an existing certificate's validation. + DelegationToken: uuid.NewString(), + Tags: tagsToMap(input.Tags), + } + + if mode == ValidationModePrivateCA { + rec.Type = acm.CertificateTypePrivate + if err := s.issuePrivateCALeaf(ctx, rec, domain, sans); err != nil { + return nil, err + } + } else { + rec.Type = acm.CertificateTypeAmazonIssued + } + + if err := s.store.PutCert(ctx, rec); err != nil { + slog.ErrorContext(ctx, "RequestCertificate: store failed", "err", err) + return nil, errors.New(awserrors.ErrorInternalError) + } + + slog.InfoContext(ctx, "RequestCertificate: stored", "arn", certArn, "domain", domain, "mode", mode, "status", rec.Status) + return &acm.RequestCertificateOutput{CertificateArn: aws.String(certArn)}, nil +} + +// issuePrivateCALeaf signs rec's leaf from TenantCA and advances rec straight +// to ISSUED — PRIVATE_CA has no validation step to wait on. Caller has already +// checked TenantCA is non-nil and every domain is authorized. +func (s *ACMServiceImpl) issuePrivateCALeaf(ctx context.Context, rec *CertRecord, domain string, sans []string) error { + certPEM, chainPEM, keyPEM, err := s.TenantCA.IssueLeaf(domain, sans) + if err != nil { + slog.ErrorContext(ctx, "RequestCertificate: private CA issuance failed", "domain", domain, "err", err) + return errors.New(awserrors.ErrorInternalError) + } + leaf, err := parseLeaf([]byte(certPEM)) + if err != nil { + slog.ErrorContext(ctx, "RequestCertificate: private CA returned an unparseable leaf", "domain", domain, "err", err) + return errors.New(awserrors.ErrorInternalError) + } + + rec.Certificate = certPEM + rec.CertificateChain = chainPEM + rec.PrivateKey = keyPEM + rec.Serial = leaf.SerialNumber.Text(16) + rec.Subject = leaf.Subject.String() + rec.Issuer = leaf.Issuer.String() + rec.KeyAlgorithm = keyAlgorithm(leaf) + rec.NotBefore = leaf.NotBefore + rec.NotAfter = leaf.NotAfter + rec.Status = acm.CertificateStatusIssued + for i := range rec.DomainValidationOptions { + rec.DomainValidationOptions[i].ValidationStatus = acm.DomainStatusSuccess + } + return nil +} + +// uniqueDomains returns domain plus sans, deduplicated and order-preserved — +// the full set of names a certificate's per-domain validation state and +// authorization checks must cover. +func uniqueDomains(domain string, sans []string) []string { + out := make([]string, 0, 1+len(sans)) + seen := make(map[string]bool, 1+len(sans)) + for _, d := range append([]string{domain}, sans...) { + if d == "" || seen[d] { + continue + } + seen[d] = true + out = append(out, d) + } + return out +} + +// buildDomainValidationOptions shapes one DomainValidationEntry per domain for +// the derived mode. Only MANUAL_TXT gets a ResourceRecord: PROVIDER_API and +// PRIVATE_CA both leave it empty because Spinifex (or, for PRIVATE_CA, nobody) +// owns the record write, and the caller has nothing to publish. The TXT +// record's Name is deterministic ("_acme-challenge.."), but its Value +// is the per-order ACME challenge token, which does not exist until a later +// slice's worker places the order — so it stays empty here. +func buildDomainValidationOptions(mode string, domains []string) []DomainValidationEntry { + entries := make([]DomainValidationEntry, 0, len(domains)) + for _, d := range domains { + e := DomainValidationEntry{DomainName: d, ValidationStatus: acm.DomainStatusPendingValidation} + // PRIVATE_CA validates nothing, so it carries no method — matching real + // ACM on a private certificate, where the AWS provider reads the method + // back as NONE whatever the request asked for. + if mode != ValidationModePrivateCA { + e.ValidationMethod = acm.ValidationMethodDns + } + if mode == ValidationModeManualTXT { + e.RecordType = "TXT" + e.RecordName = "_acme-challenge." + d + "." + } + entries = append(entries, e) + } + return entries +} + +// renewalEligibilityForMode reports whether the future renewal worker may +// reissue a managed certificate under its existing ARN. MANUAL_TXT is +// INELIGIBLE because its challenge token rotates per order, making unattended +// renewal impossible; marking it now keeps that worker from thrashing against +// a certificate it can never renew. +func renewalEligibilityForMode(mode string) string { + if mode == ValidationModeManualTXT { + return acm.RenewalEligibilityIneligible + } + return acm.RenewalEligibilityEligible +} + +// domainValidationOptionsToAWS converts stored DomainValidationEntry records +// to the AWS-shaped []*acm.DomainValidation. An entry with no RecordType +// carries DomainName, ValidationMethod and ValidationStatus but no +// ResourceRecord — real ACM omits the field the same way until it has +// something to report, so this is a legitimate AWS-shaped state rather than a +// partial one. +func domainValidationOptionsToAWS(entries []DomainValidationEntry) []*acm.DomainValidation { + if len(entries) == 0 { + return nil + } + out := make([]*acm.DomainValidation, 0, len(entries)) + for _, e := range entries { + dv := &acm.DomainValidation{ + DomainName: aws.String(e.DomainName), + ValidationStatus: aws.String(e.ValidationStatus), + } + if e.ValidationMethod != "" { + dv.ValidationMethod = aws.String(e.ValidationMethod) + } + if e.RecordType != "" { + dv.ResourceRecord = &acm.ResourceRecord{ + Name: aws.String(e.RecordName), + Type: aws.String(e.RecordType), + Value: aws.String(e.RecordValue), + } + } + out = append(out, dv) + } + return out +} + // DescribeCertificate returns the CertificateDetail for an owned ARN. func (s *ACMServiceImpl) DescribeCertificate(ctx context.Context, input *acm.DescribeCertificateInput, accountID string) (*acm.DescribeCertificateOutput, error) { if input == nil || aws.StringValue(input.CertificateArn) == "" { @@ -126,9 +467,37 @@ func (s *ACMServiceImpl) DescribeCertificate(ctx context.Context, input *acm.Des return &acm.DescribeCertificateOutput{Certificate: recordToDetail(rec)}, nil } +// GetCertificate returns the PEM-encoded certificate body and chain for an +// owned ARN. This is the only ACM read that hands back the certificate itself +// rather than metadata about it — it is what a client pinning the issuing chain +// fetches, and what Terraform reads to feed a certificate into another +// resource. It returns no key material: ACM discloses none through any API. +func (s *ACMServiceImpl) GetCertificate(ctx context.Context, input *acm.GetCertificateInput, accountID string) (*acm.GetCertificateOutput, error) { + if input == nil || aws.StringValue(input.CertificateArn) == "" { + return nil, errors.New(awserrors.ErrorACMInvalidArn) + } + rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID) + if err != nil { + return nil, err + } + // A requested certificate has a record before it has a body. Returning an + // empty string here would read as a successfully fetched empty certificate, + // so a not-yet-issued cert is a distinct retryable error, as in AWS. + if rec.Certificate == "" { + return nil, errors.New(awserrors.ErrorACMRequestInProgress) + } + out := &acm.GetCertificateOutput{Certificate: aws.String(rec.Certificate)} + // Left absent rather than empty for a self-signed leaf with no chain. + if rec.CertificateChain != "" { + out.CertificateChain = aws.String(rec.CertificateChain) + } + return out, nil +} + // ListCertificates returns summaries for every cert owned by accountID. +// Summaries carry no key material, so this takes the metadata-only accessor. func (s *ACMServiceImpl) ListCertificates(ctx context.Context, input *acm.ListCertificatesInput, accountID string) (*acm.ListCertificatesOutput, error) { - recs, err := s.store.ListCerts(ctx, accountID) + recs, err := s.store.ListCertMetadata(ctx, accountID) if err != nil { return nil, errors.New(awserrors.ErrorInternalError) } @@ -137,26 +506,35 @@ func (s *ACMServiceImpl) ListCertificates(ctx context.Context, input *acm.ListCe out.CertificateSummaryList = append(out.CertificateSummaryList, &acm.CertificateSummary{ CertificateArn: aws.String(rec.CertificateArn), DomainName: aws.String(rec.DomainName), - Status: aws.String(certStatusIssued), - Type: aws.String(certTypeImported), + Status: aws.String(certStatusOrDefault(rec)), + Type: aws.String(certTypeOrDefault(rec)), KeyAlgorithm: aws.String(rec.KeyAlgorithm), NotBefore: timePtr(rec.NotBefore), NotAfter: timePtr(rec.NotAfter), ImportedAt: timePtr(rec.ImportedAt), - InUse: aws.Bool(false), + InUse: aws.Bool(len(rec.InUseBy) > 0), }) } return out, nil } // DeleteCertificate removes an owned cert; unknown ARN → ResourceNotFound. +// Matches AWS: a certificate still referenced by a load balancer listener +// (InUseBy non-empty) is refused with ResourceInUseException, no force flag. +// Without this, a destroy on a spine workspace could remove a shared +// certificate out from under every HTTPS listener still using it. func (s *ACMServiceImpl) DeleteCertificate(ctx context.Context, input *acm.DeleteCertificateInput, accountID string) (*acm.DeleteCertificateOutput, error) { if input == nil || aws.StringValue(input.CertificateArn) == "" { return nil, errors.New(awserrors.ErrorACMInvalidArn) } - if _, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID); err != nil { + rec, err := s.lookupOwned(ctx, aws.StringValue(input.CertificateArn), accountID) + if err != nil { return nil, err } + if len(rec.InUseBy) > 0 { + slog.InfoContext(ctx, "DeleteCertificate: refused, certificate in use", "arn", rec.CertificateArn, "in_use_by", rec.InUseBy) + return nil, errors.New(awserrors.ErrorACMResourceInUse) + } if _, err := s.store.DeleteCert(ctx, aws.StringValue(input.CertificateArn)); err != nil { return nil, errors.New(awserrors.ErrorInternalError) } @@ -165,8 +543,12 @@ func (s *ACMServiceImpl) DeleteCertificate(ctx context.Context, input *acm.Delet // lookupOwned fetches a cert by ARN, returning ResourceNotFound when absent or // owned by a different account (no cross-account disclosure). +// +// Takes the metadata read: no caller of this helper reads PrivateKey, so an +// ownership check must not decrypt one. A caller that does need the key +// fetches it explicitly via store.GetCert once ownership is established. func (s *ACMServiceImpl) lookupOwned(ctx context.Context, certArn, accountID string) (*CertRecord, error) { - rec, err := s.store.GetCert(ctx, certArn) + rec, err := s.store.GetCertMetadata(ctx, certArn) if err != nil { return nil, errors.New(awserrors.ErrorInternalError) } @@ -176,27 +558,78 @@ func (s *ACMServiceImpl) lookupOwned(ctx context.Context, certArn, accountID str return rec, nil } +// recordToDetail is the only surface on which Terraform, the CLI and +// spinifex-ui learn why a certificate is stuck, so Type/Status/ +// DomainValidationOptions/FailureReason/RenewalEligibility/RenewalSummary are +// functional requirements, not cosmetics. It never sets a PrivateKey field — +// CertificateDetail has none; ACM never returns key material from Describe. func recordToDetail(rec *CertRecord) *acm.CertificateDetail { detail := &acm.CertificateDetail{ - CertificateArn: aws.String(rec.CertificateArn), - DomainName: aws.String(rec.DomainName), - Serial: aws.String(rec.Serial), - Subject: aws.String(rec.Subject), - Issuer: aws.String(rec.Issuer), - KeyAlgorithm: aws.String(rec.KeyAlgorithm), - Status: aws.String(certStatusIssued), - Type: aws.String(certTypeImported), - NotBefore: timePtr(rec.NotBefore), - NotAfter: timePtr(rec.NotAfter), - ImportedAt: timePtr(rec.ImportedAt), - InUseBy: []*string{}, + CertificateArn: aws.String(rec.CertificateArn), + DomainName: aws.String(rec.DomainName), + Serial: aws.String(rec.Serial), + Subject: aws.String(rec.Subject), + Issuer: aws.String(rec.Issuer), + KeyAlgorithm: aws.String(rec.KeyAlgorithm), + Status: aws.String(certStatusOrDefault(rec)), + Type: aws.String(certTypeOrDefault(rec)), + NotBefore: timePtr(rec.NotBefore), + NotAfter: timePtr(rec.NotAfter), + ImportedAt: timePtr(rec.ImportedAt), + InUseBy: aws.StringSlice(rec.InUseBy), + RenewalEligibility: aws.String(renewalEligibilityOrDefault(rec)), } if len(rec.SubjectAltNames) > 0 { detail.SubjectAlternativeNames = aws.StringSlice(rec.SubjectAltNames) } + if len(rec.DomainValidationOptions) > 0 { + detail.DomainValidationOptions = domainValidationOptionsToAWS(rec.DomainValidationOptions) + } + if rec.FailureReason != "" { + detail.FailureReason = aws.String(rec.FailureReason) + } + if rec.RenewalSummary != nil { + summary := &acm.RenewalSummary{ + DomainValidationOptions: domainValidationOptionsToAWS(rec.DomainValidationOptions), + RenewalStatus: aws.String(rec.RenewalSummary.RenewalStatus), + UpdatedAt: timePtr(rec.RenewalSummary.UpdatedAt), + } + if rec.RenewalSummary.RenewalStatusReason != "" { + summary.RenewalStatusReason = aws.String(rec.RenewalSummary.RenewalStatusReason) + } + detail.RenewalSummary = summary + } return detail } +// certStatusOrDefault and certTypeOrDefault fall back to the pre-managed- +// issuance defaults (ISSUED/IMPORTED) for records written before Type/Status +// existed on CertRecord — every certificate stored before this change was an +// immediately-issued import. +func certStatusOrDefault(rec *CertRecord) string { + if rec.Status != "" { + return rec.Status + } + return certStatusIssued +} + +func certTypeOrDefault(rec *CertRecord) string { + if rec.Type != "" { + return rec.Type + } + return certTypeImported +} + +// renewalEligibilityOrDefault falls back to INELIGIBLE: unset covers both +// legacy imported records (predating this field) and freshly imported ones, +// and imported certificates are never auto-renewed either way. +func renewalEligibilityOrDefault(rec *CertRecord) string { + if rec.RenewalEligibility != "" { + return rec.RenewalEligibility + } + return acm.RenewalEligibilityIneligible +} + func timePtr(t time.Time) *time.Time { if t.IsZero() { return nil diff --git a/spinifex/handlers/acm/service_impl_test.go b/spinifex/handlers/acm/service_impl_test.go index ee48511da..72c67324a 100644 --- a/spinifex/handlers/acm/service_impl_test.go +++ b/spinifex/handlers/acm/service_impl_test.go @@ -7,14 +7,20 @@ import ( "crypto/rand" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "encoding/pem" + "errors" + "log/slog" "math/big" + "path/filepath" + "strings" "testing" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/acm" "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,7 +31,9 @@ const testAccountID = "000000000001" func setupACMService(t *testing.T) *ACMServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewACMServiceImplWithNATS(t.Context(), nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := NewACMServiceImplWithNATS(t.Context(), nil, nc, masterKey) require.NoError(t, err) return svc } @@ -154,6 +162,95 @@ func TestDeleteCertificate(t *testing.T) { assert.Contains(t, err.Error(), awserrors.ErrorResourceNotFound) } +func TestImportCertificate_ReimportPreservesInUseBy(t *testing.T) { + svc := setupACMService(t) + c1, k1 := genCert(t, "inuse.example.com") + + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + + require.NoError(t, svc.store.AddInUseBy(context.Background(), certArn, "arn:lb/one")) + + // Re-import different material under the same ARN. + c2, k2 := genCert(t, "inuse-rotated.example.com") + _, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: c2, + PrivateKey: k2, + CertificateArn: aws.String(certArn), + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: aws.String(certArn)}, testAccountID) + require.NoError(t, err) + assert.Equal(t, "inuse-rotated.example.com", aws.StringValue(desc.Certificate.DomainName), "material must be replaced") + assert.Equal(t, []string{"arn:lb/one"}, aws.StringValueSlice(desc.Certificate.InUseBy), "InUseBy must survive re-import") +} + +func TestImportCertificate_ReimportInvokesFanoutHook(t *testing.T) { + svc := setupACMService(t) + c1, k1 := genCert(t, "hook.example.com") + + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + require.NoError(t, svc.store.AddInUseBy(context.Background(), certArn, "arn:lb/one")) + + var calledWith string + svc.CertMaterialUpdated = func(ctx context.Context, arn string) error { + calledWith = arn + return nil + } + + c2, k2 := genCert(t, "hook-rotated.example.com") + _, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: c2, + PrivateKey: k2, + CertificateArn: aws.String(certArn), + }, testAccountID) + require.NoError(t, err) + assert.Equal(t, certArn, calledWith, "re-import to an in-use ARN must invoke the fan-out hook") +} + +func TestImportCertificate_ReimportZeroInUseBy_HookNotCalled(t *testing.T) { + svc := setupACMService(t) + c1, k1 := genCert(t, "unused.example.com") + + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + + called := false + svc.CertMaterialUpdated = func(ctx context.Context, arn string) error { + called = true + return nil + } + + // Re-import with zero load balancers referencing the ARN — a no-op fan-out. + c2, k2 := genCert(t, "unused-rotated.example.com") + _, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: c2, + PrivateKey: k2, + CertificateArn: aws.String(certArn), + }, testAccountID) + require.NoError(t, err) + assert.False(t, called, "a certificate referenced by zero load balancers must not trigger a fan-out") +} + +func TestImportCertificate_FreshImportDoesNotInvokeHook(t *testing.T) { + svc := setupACMService(t) + called := false + svc.CertMaterialUpdated = func(ctx context.Context, arn string) error { + called = true + return nil + } + + c1, k1 := genCert(t, "fresh.example.com") + _, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + assert.False(t, called, "a brand-new ARN cannot yet be in use by anything") +} + func TestImportCertificate_ReimportUnknownArnRejected(t *testing.T) { svc := setupACMService(t) c1, k1 := genCert(t, "re.example.com") @@ -165,3 +262,388 @@ func TestImportCertificate_ReimportUnknownArnRejected(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), awserrors.ErrorResourceNotFound) } + +// TestImportCertificate_PrivateKeyEncryptedAtRest is the core regression for +// mulga-ubkoz: the raw bytes ImportCertificate lands in JetStream KV must not +// contain the plaintext PEM key, and reading it back through the Store must +// still recover the original plaintext. +func TestImportCertificate_PrivateKeyEncryptedAtRest(t *testing.T) { + svc := setupACMService(t) + certPEM, keyPEM := genCert(t, "atrest.example.com") + + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: certPEM, + PrivateKey: keyPEM, + }, testAccountID) + require.NoError(t, err) + + entry, err := svc.store.kv.Get(context.Background(), certKey(aws.StringValue(out.CertificateArn))) + require.NoError(t, err) + + var raw CertRecord + require.NoError(t, json.Unmarshal(entry.Value(), &raw)) + assert.NotEqual(t, string(keyPEM), raw.PrivateKey, "private key must not be stored as plaintext PEM") + assert.NotContains(t, string(entry.Value()), "PRIVATE KEY", "raw KV bytes must not contain a PEM private key marker") + + got, err := svc.store.GetCert(context.Background(), aws.StringValue(out.CertificateArn)) + require.NoError(t, err) + assert.Equal(t, string(keyPEM), got.PrivateKey, "GetCert must decrypt back to the original plaintext") +} + +// TestDescribeCertificate_DoesNotLeakPrivateKey asserts the public +// CertificateDetail response can never carry key material, however the +// record is stored. Precedent: TestSensitiveDataNotLogged_MasterKey in +// handlers/iam/service_impl_test.go. +func TestDescribeCertificate_DoesNotLeakPrivateKey(t *testing.T) { + svc := setupACMService(t) + certPEM, keyPEM := genCert(t, "noleak.example.com") + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: certPEM, + PrivateKey: keyPEM, + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{ + CertificateArn: out.CertificateArn, + }, testAccountID) + require.NoError(t, err) + + b, err := json.Marshal(desc) + require.NoError(t, err) + assert.NotContains(t, string(b), "PRIVATE KEY", "DescribeCertificate response must never carry key material") +} + +// TestSensitiveDataNotLogged_ACMPrivateKey asserts ImportCertificate and +// DescribeCertificate never place the raw private key PEM in log output. +// Precedent: TestSensitiveDataNotLogged_MasterKey in +// handlers/iam/service_impl_test.go. +// fakeCertAuthority is a minimal CertAuthority for testing RequestCertificate's +// PRIVATE_CA path without the real tenant CA implementation (handlers/acm/privateca.go +// lives on a separate branch). +type fakeCertAuthority struct { + permitted map[string]bool + issueErr error +} + +func (f *fakeCertAuthority) Authorized(domain string) bool { return f.permitted[domain] } + +func (f *fakeCertAuthority) IssueLeaf(domain string, sans []string) (certPEM, chainPEM, keyPEM string, err error) { + if f.issueErr != nil { + return "", "", "", f.issueErr + } + c, k, err := genLeafPEM(domain, append([]string{domain}, sans...)) + if err != nil { + return "", "", "", err + } + return c, "-----BEGIN CERTIFICATE-----\nfake-chain\n-----END CERTIFICATE-----\n", k, nil +} + +// genLeafPEM generates a self-signed leaf certificate + private key as PEM, +// independent of *testing.T so it can be called from fakeCertAuthority.IssueLeaf. +func genLeafPEM(cn string, dnsNames []string) (certPEM, keyPEM string, err error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", "", err + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: cn}, + DNSNames: dnsNames, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return "", "", err + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return "", "", err + } + certBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyBytes := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return string(certBytes), string(keyBytes), nil +} + +func TestRequestCertificate_ProviderAPI_PendingValidationNoResourceRecord(t *testing.T) { + svc := setupACMService(t) + svc.dnsProviderConfigured = true + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("api.example.com"), + }, testAccountID) + require.NoError(t, err) + require.NotNil(t, out.CertificateArn) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + d := desc.Certificate + assert.Equal(t, acm.CertificateStatusPendingValidation, aws.StringValue(d.Status)) + assert.Equal(t, acm.CertificateTypeAmazonIssued, aws.StringValue(d.Type)) + require.Len(t, d.DomainValidationOptions, 1) + assert.Equal(t, "api.example.com", aws.StringValue(d.DomainValidationOptions[0].DomainName)) + assert.Nil(t, d.DomainValidationOptions[0].ResourceRecord, "PROVIDER_API: Spinifex owns the record write, nothing for the caller to publish") + // The AWS provider reads this back to decide whether the configured + // aws_acm_certificate.validation_method still matches; omitting it makes + // every plan see drift and replace the certificate. + assert.Equal(t, acm.ValidationMethodDns, aws.StringValue(d.DomainValidationOptions[0].ValidationMethod)) + assert.Equal(t, acm.RenewalEligibilityEligible, aws.StringValue(d.RenewalEligibility)) +} + +func TestRequestCertificate_ManualTXT_EmitsTXTResourceRecordAndIneligible(t *testing.T) { + svc := setupACMService(t) + svc.NorthstarHostsZone = func(domain string) bool { return true } + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("manual.example.com"), + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + d := desc.Certificate + assert.Equal(t, acm.CertificateStatusPendingValidation, aws.StringValue(d.Status)) + require.Len(t, d.DomainValidationOptions, 1) + rr := d.DomainValidationOptions[0].ResourceRecord + require.NotNil(t, rr, "MANUAL_TXT: the operator must publish the challenge record by hand") + assert.Equal(t, "TXT", aws.StringValue(rr.Type)) + assert.Equal(t, "_acme-challenge.manual.example.com.", aws.StringValue(rr.Name)) + assert.Equal(t, acm.ValidationMethodDns, aws.StringValue(d.DomainValidationOptions[0].ValidationMethod)) + assert.Equal(t, acm.RenewalEligibilityIneligible, aws.StringValue(d.RenewalEligibility), + "MANUAL_TXT's challenge token rotates per order, so unattended renewal is impossible") +} + +// TestRequestCertificate_PrivateCA_WinsOverNorthstarHostedZone is the +// demo-critical regression: a homelab where northstar hosts the requested +// zone AND the operator has a tenant CA constrained to it must issue +// PRIVATE_CA (automatic, offline, renewable), not MANUAL_TXT (which can never +// complete against an internal zone a public CA cannot resolve). +func TestRequestCertificate_PrivateCA_WinsOverNorthstarHostedZone(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"home.example.com": true}} + svc.NorthstarHostsZone = func(domain string) bool { return domain == "home.example.com" } + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("home.example.com"), + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + d := desc.Certificate + assert.Equal(t, acm.CertificateStatusIssued, aws.StringValue(d.Status), "PRIVATE_CA issues synchronously, unlike MANUAL_TXT") + assert.Equal(t, acm.CertificateTypePrivate, aws.StringValue(d.Type)) + assert.Equal(t, acm.RenewalEligibilityEligible, aws.StringValue(d.RenewalEligibility), + "PRIVATE_CA is renewable; MANUAL_TXT, which this must not select, would not be") +} + +func TestRequestCertificate_PrivateCA_IssuesSynchronously(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"private.example.com": true}} + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("private.example.com"), + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + d := desc.Certificate + assert.Equal(t, acm.CertificateStatusIssued, aws.StringValue(d.Status)) + assert.Equal(t, acm.CertificateTypePrivate, aws.StringValue(d.Type)) + assert.NotEmpty(t, aws.StringValue(d.Serial), "the leaf from the injected CA must be parsed and stored") + require.Len(t, d.DomainValidationOptions, 1) + assert.Nil(t, d.DomainValidationOptions[0].ResourceRecord, "PRIVATE_CA: no validation, nothing to publish") + assert.Nil(t, d.DomainValidationOptions[0].ValidationMethod, + "PRIVATE_CA validates nothing, and real ACM omits the method on a private certificate") + assert.Equal(t, acm.DomainStatusSuccess, aws.StringValue(d.DomainValidationOptions[0].ValidationStatus)) + assert.Equal(t, acm.RenewalEligibilityEligible, aws.StringValue(d.RenewalEligibility)) +} + +// TestRequestCertificate_PrivateCA_RealTenantCA_IssuesAndChains exercises the +// PRIVATE_CA path against a real *TenantCA (handlers/acm/privateca.go), not +// fakeCertAuthority above — this is what daemon wiring actually plugs in, and +// the daemon-wiring slice adds no coverage for the seam between +// RequestCertificate and the real implementation without a test like this one. +func TestRequestCertificate_PrivateCA_RealTenantCA_IssuesAndChains(t *testing.T) { + svc := setupACMService(t) + dir := t.TempDir() + ca, err := LoadOrCreateTenantCA( + filepath.Join(dir, "tenant-ca.pem"), + filepath.Join(dir, "tenant-ca.key"), + []string{"real.example.com"}, + ) + require.NoError(t, err) + svc.TenantCA = ca + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("leaf.real.example.com"), + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + d := desc.Certificate + assert.Equal(t, acm.CertificateStatusIssued, aws.StringValue(d.Status)) + assert.Equal(t, acm.CertificateTypePrivate, aws.StringValue(d.Type)) + + rec, err := svc.store.GetCert(context.Background(), aws.StringValue(out.CertificateArn)) + require.NoError(t, err) + leafBlock, _ := pem.Decode([]byte(rec.Certificate)) + require.NotNil(t, leafBlock) + leaf, err := x509.ParseCertificate(leafBlock.Bytes) + require.NoError(t, err) + + roots := x509.NewCertPool() + roots.AddCert(ca.root) + _, err = leaf.Verify(x509.VerifyOptions{ + DNSName: "leaf.real.example.com", + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + require.NoError(t, err, "the leaf issued through RequestCertificate must chain to the real tenant CA") +} + +func TestRequestCertificate_PrivateCA_DomainOutsidePermittedRejected(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"allowed.example.com": true}} + + _, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("evil.example.com"), + }, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameter) +} + +func TestRequestCertificate_PrivateCA_SANOutsidePermittedRejected(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"allowed.example.com": true}} + + _, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("allowed.example.com"), + SubjectAlternativeNames: aws.StringSlice([]string{"other.example.com"}), + }, testAccountID) + require.Error(t, err, "a SAN outside permitted domains must be rejected, not just the primary domain name") + assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameter) +} + +func TestRequestCertificate_PrivateCA_NoTenantCAWired_FailsLoudly(t *testing.T) { + svc := setupACMService(t) + // TenantCA deliberately left nil: PRIVATE_CA is derived (no dns provider, no + // northstar zone) but nothing can drive it. + _, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("x.example.com"), + }, testAccountID) + require.Error(t, err, "must fail loudly rather than silently skip issuance") +} + +func TestRequestCertificate_PrivateCA_IssueLeafErrorNotStored(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{ + permitted: map[string]bool{"broken.example.com": true}, + issueErr: errors.New("ca offline"), + } + + _, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("broken.example.com"), + }, testAccountID) + require.Error(t, err) + + list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID) + require.NoError(t, err) + assert.Empty(t, list.CertificateSummaryList, "a failed private-CA issuance must not leave a dangling record") +} + +func TestRequestCertificate_NoDomainName_Rejected(t *testing.T) { + svc := setupACMService(t) + _, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{}, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorInvalidParameter) +} + +func TestRequestCertificate_PrivateCA_DescribeDoesNotLeakPrivateKey(t *testing.T) { + svc := setupACMService(t) + svc.TenantCA = &fakeCertAuthority{permitted: map[string]bool{"noleak-priv.example.com": true}} + + out, err := svc.RequestCertificate(context.Background(), &acm.RequestCertificateInput{ + DomainName: aws.String("noleak-priv.example.com"), + }, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + + b, err := json.Marshal(desc) + require.NoError(t, err) + assert.NotContains(t, string(b), "PRIVATE KEY", "DescribeCertificate response must never carry key material") +} + +func TestImportCertificate_RenewalEligibilityIneligible(t *testing.T) { + svc := setupACMService(t) + c1, k1 := genCert(t, "imported-ineligible.example.com") + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + + desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + assert.Equal(t, acm.RenewalEligibilityIneligible, aws.StringValue(desc.Certificate.RenewalEligibility), + "imported certificates are never auto-renewed") +} + +func TestDeleteCertificate_RefusesWhenInUse(t *testing.T) { + svc := setupACMService(t) + c1, k1 := genCert(t, "inuse-delete.example.com") + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(out.CertificateArn) + require.NoError(t, svc.store.AddInUseBy(context.Background(), certArn, "arn:lb/one")) + + _, err = svc.DeleteCertificate(context.Background(), &acm.DeleteCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorACMResourceInUse) + + // Refused, not partially applied: the certificate must still be there. + _, err = svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) +} + +func TestDeleteCertificate_SucceedsWhenNotInUse(t *testing.T) { + svc := setupACMService(t) + c1, k1 := genCert(t, "notinuse-delete.example.com") + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID) + require.NoError(t, err) + + _, err = svc.DeleteCertificate(context.Background(), &acm.DeleteCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.NoError(t, err) + + _, err = svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: out.CertificateArn}, testAccountID) + require.Error(t, err) + assert.Contains(t, err.Error(), awserrors.ErrorResourceNotFound) +} + +func TestSensitiveDataNotLogged_ACMPrivateKey(t *testing.T) { + var buf strings.Builder + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + slog.SetDefault(logger) + t.Cleanup(func() { + slog.SetDefault(slog.New(slog.DiscardHandler)) + }) + + svc := setupACMService(t) + certPEM, keyPEM := genCert(t, "quiet.example.com") + + out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: certPEM, + PrivateKey: keyPEM, + }, testAccountID) + require.NoError(t, err) + + _, err = svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{ + CertificateArn: out.CertificateArn, + }, testAccountID) + require.NoError(t, err) + + assert.NotContains(t, buf.String(), string(keyPEM), "raw private key PEM must not appear in log output") +} diff --git a/spinifex/handlers/acm/service_nats.go b/spinifex/handlers/acm/service_nats.go index fb237a4b0..ab25ec48a 100644 --- a/spinifex/handlers/acm/service_nats.go +++ b/spinifex/handlers/acm/service_nats.go @@ -31,6 +31,10 @@ func (s *NATSACMService) DescribeCertificate(ctx context.Context, input *acm.Des return utils.NATSRequest[acm.DescribeCertificateOutput](ctx, s.natsConn, "acm.DescribeCertificate", input, defaultTimeout, accountID) } +func (s *NATSACMService) GetCertificate(ctx context.Context, input *acm.GetCertificateInput, accountID string) (*acm.GetCertificateOutput, error) { + return utils.NATSRequest[acm.GetCertificateOutput](ctx, s.natsConn, "acm.GetCertificate", input, defaultTimeout, accountID) +} + func (s *NATSACMService) ListCertificates(ctx context.Context, input *acm.ListCertificatesInput, accountID string) (*acm.ListCertificatesOutput, error) { return utils.NATSRequest[acm.ListCertificatesOutput](ctx, s.natsConn, "acm.ListCertificates", input, defaultTimeout, accountID) } diff --git a/spinifex/handlers/acm/store.go b/spinifex/handlers/acm/store.go index 8e70da34f..91250884d 100644 --- a/spinifex/handlers/acm/store.go +++ b/spinifex/handlers/acm/store.go @@ -3,12 +3,16 @@ package handlers_acm import ( "context" "encoding/json" + "encoding/pem" "errors" "fmt" "log/slog" + "math/rand/v2" + "slices" "strings" "time" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -22,8 +26,68 @@ const ( KeyPrefixCert = "cert." ) -// CertRecord is the stored representation of an imported certificate. -// AccountID scopes ownership so list/describe never cross account boundaries. +// Validation modes for a managed (RequestCertificate) certificate. The mode is +// derived from deployment state at request time, never configured, and stored +// on CertRecord.ValidationMethod so DescribeCertificate and the future +// issuance/renewal workers can see how a certificate is being validated +// without re-deriving it. +const ( + // ValidationModeProviderAPI: Spinifex writes the DNS-01 challenge via the + // operator's DNS provider API. No ResourceRecord is returned — there is + // nothing for the caller to publish. Automatic renewal. + ValidationModeProviderAPI = "PROVIDER_API" + // ValidationModeManualTXT: the operator publishes a rotating TXT challenge + // record by hand or via Terraform. Renewal is INELIGIBLE because the + // challenge token rotates per order, so unattended renewal cannot succeed. + ValidationModeManualTXT = "MANUAL_TXT" + // ValidationModeCNAMEDelegation: the operator publishes one stable CNAME + // once, and Spinifex answers every subsequent challenge behind it. Deferred + // until northstar can serve public authoritative queries; never selected by + // deriveValidationMode today, but CertRecord.DelegationToken is minted for + // every managed certificate now so this mode is a non-breaking addition + // later. + ValidationModeCNAMEDelegation = "CNAME_DELEGATION" + // ValidationModePrivateCA: signed from the tenant root CA with no domain + // validation at all. The only mode that issues synchronously. + ValidationModePrivateCA = "PRIVATE_CA" +) + +// DomainValidationEntry is the resumable per-domain validation state for one +// name on a managed certificate. Converted to the AWS-shaped +// []*acm.DomainValidation in recordToDetail. RecordType/RecordName/RecordValue +// stay empty for any mode where Spinifex (or, for PRIVATE_CA, nobody) owns the +// record write — that omission is what makes DescribeCertificate AWS-shaped +// without fabricating a ResourceRecord the caller has nothing to do with. +type DomainValidationEntry struct { + DomainName string `json:"domain_name"` + // RecordType/RecordName/RecordValue mirror acm.ResourceRecord. RecordType + // empty means "no ResourceRecord" in the AWS-shaped output. + RecordType string `json:"record_type,omitempty"` + RecordName string `json:"record_name,omitempty"` + RecordValue string `json:"record_value,omitempty"` + // ValidationStatus is one of the ACM DomainStatus values (PENDING_VALIDATION, + // SUCCESS, FAILED). + ValidationStatus string `json:"validation_status"` + // ValidationMethod is DNS for the DNS-based modes and empty for PRIVATE_CA, + // which validates nothing. Real ACM omits it on a private certificate too, + // and the AWS provider reads it to decide whether the configured + // validation_method still matches. + ValidationMethod string `json:"validation_method,omitempty"` +} + +// RenewalSummaryRecord is the stored form of acm.RenewalSummary, populated by +// the future renewal worker. RenewalStatusReason is what lets a certificate +// that issued cleanly but can no longer renew surface before it expires +// rather than after. +type RenewalSummaryRecord struct { + RenewalStatus string `json:"renewal_status"` + RenewalStatusReason string `json:"renewal_status_reason,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// CertRecord is the stored representation of a certificate, whether imported +// or issued by RequestCertificate. AccountID scopes ownership so list/describe +// never cross account boundaries. type CertRecord struct { CertificateArn string `json:"certificate_arn"` AccountID string `json:"account_id"` @@ -40,16 +104,76 @@ type CertRecord struct { NotAfter time.Time `json:"not_after"` ImportedAt time.Time `json:"imported_at"` Tags map[string]string `json:"tags,omitempty"` + // InUseBy is the set of load balancer ARNs whose listeners currently + // reference this certificate. Maintained by handlers/elbv2 as listeners are + // created, modified and deleted; also surfaces as the public + // CertificateDetail.InUseBy field. + InUseBy []string `json:"in_use_by,omitempty"` + + // Type is the ACM certificate type: AMAZON_ISSUED, PRIVATE or IMPORTED. + Type string `json:"type,omitempty"` + // Status is the ACM certificate status (PENDING_VALIDATION, ISSUED, + // FAILED, ...). + Status string `json:"status,omitempty"` + // ValidationMethod is the derived validation mode (see the + // ValidationMode* constants above). Empty for imported certificates, which + // have no validation. + ValidationMethod string `json:"validation_method,omitempty"` + // DomainValidationOptions carries one entry per name on the certificate + // (DomainName plus any SubjectAltNames). + DomainValidationOptions []DomainValidationEntry `json:"domain_validation_options,omitempty"` + // FailureReason is populated on terminal issuance failure, using the ACM + // FailureReason enum. + FailureReason string `json:"failure_reason,omitempty"` + // RenewalEligibility reports whether the renewal worker may reissue this + // certificate under its existing ARN. + RenewalEligibility string `json:"renewal_eligibility,omitempty"` + // RenewalSummary is nil until a renewal has been attempted. + RenewalSummary *RenewalSummaryRecord `json:"renewal_summary,omitempty"` + // DelegationToken is minted once at RequestCertificate time and persisted + // for the certificate's lifetime, so that CNAME_DELEGATION mode, when it + // lands, has a stable CNAME target that never changes retroactively. Unused + // by any mode today. + DelegationToken string `json:"delegation_token,omitempty"` + + // --- Resumable ACME order state, populated by the future issuance worker. + // Persisted so a crashed lease holder's work resumes elsewhere instead of + // minting a fresh order (and burning rate-limit budget) on every retry. + ACMEOrderURL string `json:"acme_order_url,omitempty"` + ACMEAuthzURLs []string `json:"acme_authz_urls,omitempty"` + ACMEChallengeToken string `json:"acme_challenge_token,omitempty"` + ACMEAttemptCount int `json:"acme_attempt_count,omitempty"` + ACMENextAttemptAt time.Time `json:"acme_next_attempt_at"` + + // --- Per-certificate issuance lease (see handlers/eks/cluster_reconciler.go + // for the pattern this follows). Only the holder may drive this + // certificate's order; empty/zero when unleased. + LeaseHolder string `json:"lease_holder,omitempty"` + LeaseExpiresAt time.Time `json:"lease_expires_at"` } // Store provides CRUD for ACM certificate records backed by JetStream KV. type Store struct { kv jetstream.KeyValue + // masterKey encrypts/decrypts CertRecord.PrivateKey at rest (AES-256-GCM via + // handlers_iam.EncryptSecret/DecryptSecret). Every Store over the ACM bucket + // — the ACM service and ELBv2's independent read-only Store alike — must be + // constructed with the same deployment key: a keyed writer and an unkeyed (or + // differently keyed) reader of the same bucket disagree silently, with the + // reader getting ciphertext where it expects PEM. NewStore therefore requires + // a non-empty key; there is no unkeyed Store. + masterKey []byte } // NewStore creates an ACM store using the provided NATS connection. ctx bounds -// the bucket get-or-create only; each operation carries its own. -func NewStore(ctx context.Context, nc *nats.Conn) (*Store, error) { +// the bucket get-or-create only; each operation carries its own. masterKey +// encrypts CertRecord.PrivateKey on write and decrypts it on read; it must be +// non-empty so every caller sharing the bucket agrees on the same key. +func NewStore(ctx context.Context, nc *nats.Conn, masterKey []byte) (*Store, error) { + if len(masterKey) == 0 { + return nil, fmt.Errorf("ACM store requires a master key to encrypt certificate private keys at rest; none provided") + } + js, err := jetstream.New(nc) if err != nil { return nil, fmt.Errorf("failed to get JetStream context: %w", err) @@ -61,7 +185,7 @@ func NewStore(ctx context.Context, nc *nats.Conn) (*Store, error) { } slog.Info("ACM store initialized", "bucket", KVBucketACM) - return &Store{kv: kv}, nil + return &Store{kv: kv, masterKey: masterKey}, nil } // certKey derives the KV key from a certificate ARN using the UUID after "certificate/". @@ -73,9 +197,21 @@ func certKey(certArn string) string { return KeyPrefixCert + id } -// PutCert stores (or replaces) a certificate record. +// PutCert stores (or replaces) a certificate record. PrivateKey is +// AES-256-GCM encrypted before it is written; a plaintext record read back +// via the legacy-passthrough path in decryptPrivateKey is therefore +// re-encrypted the next time it is put. func (s *Store) PutCert(ctx context.Context, rec *CertRecord) error { - data, err := json.Marshal(rec) + ciphertext, err := handlers_iam.EncryptSecret(rec.PrivateKey, s.masterKey) + if err != nil { + return fmt.Errorf("encrypt private key: %w", err) + } + // Encrypt a copy so the caller's in-memory record keeps holding + // plaintext — ImportCertificate and the tag handlers reuse *rec after + // this call. + clone := *rec + clone.PrivateKey = ciphertext + data, err := json.Marshal(&clone) if err != nil { return fmt.Errorf("marshal cert: %w", err) } @@ -83,8 +219,35 @@ func (s *Store) PutCert(ctx context.Context, rec *CertRecord) error { return err } -// GetCert retrieves a certificate by ARN, returning (nil, nil) when absent. +// GetCert retrieves a certificate by ARN with PrivateKey decrypted to +// plaintext PEM, returning (nil, nil) when absent. Use it only where the key +// itself is needed (HAProxy fan-out, reissue); everything else wants +// GetCertMetadata. func (s *Store) GetCert(ctx context.Context, certArn string) (*CertRecord, error) { + return s.getCert(ctx, certArn, true) +} + +// GetCertMetadata retrieves a certificate by ARN with PrivateKey left empty +// rather than decrypted, returning (nil, nil) when absent. +// +// This is the right read for every caller that answers a question about a +// certificate rather than serving it: ownership checks, Describe, Delete, tag +// reads and GetCertificate all ignore the key, and decrypting it for them +// produces plaintext of exactly the material this store exists to protect +// with no caller to consume it. +// +// A record whose key cannot be decrypted is still returned here, since +// nothing on this path reads it — an unreadable key must not make a +// certificate undescribable or undeletable. +func (s *Store) GetCertMetadata(ctx context.Context, certArn string) (*CertRecord, error) { + return s.getCert(ctx, certArn, false) +} + +// getCert reads and unmarshals one certificate record, resolving PrivateKey to +// plaintext PEM when decrypt is true and clearing it to "" otherwise, so a +// caller cannot mistake ciphertext for usable key material in a field typed as +// plaintext PEM everywhere else in this package. +func (s *Store) getCert(ctx context.Context, certArn string, decrypt bool) (*CertRecord, error) { entry, err := s.kv.Get(ctx, certKey(certArn)) if err != nil { if errors.Is(err, jetstream.ErrKeyNotFound) { @@ -96,9 +259,62 @@ func (s *Store) GetCert(ctx context.Context, certArn string) (*CertRecord, error if err := json.Unmarshal(entry.Value(), &rec); err != nil { return nil, fmt.Errorf("unmarshal cert: %w", err) } + if !decrypt { + rec.PrivateKey = "" + return &rec, nil + } + if err := s.decryptPrivateKey(&rec); err != nil { + return nil, err + } return &rec, nil } +// legacyPrivateKeyPEMTypes are the PEM block types ImportCertificate accepts +// for a private key (see tls.X509KeyPair / crypto/x509 parsing in +// service_impl.go). Gates the legacy-plaintext passthrough in +// decryptPrivateKey below. +var legacyPrivateKeyPEMTypes = map[string]bool{ + "RSA PRIVATE KEY": true, + "EC PRIVATE KEY": true, + "PRIVATE KEY": true, // PKCS#8 +} + +// isPlaintextPrivateKeyPEM reports whether raw is unambiguously a single +// PEM-encoded private key block: pem.Decode must consume the entire string +// (no trailing bytes) and the block type must be a recognized private-key +// type. Deliberately strict — this is the sole gate that lets decryptPrivateKey +// treat a decrypt failure as "pre-encryption legacy record" rather than +// "corrupt or tampered ciphertext", so loosening it would turn the fallback +// into a downgrade oracle that trusts arbitrary bytes as plaintext. +func isPlaintextPrivateKeyPEM(raw string) bool { + block, rest := pem.Decode([]byte(raw)) + if block == nil || len(strings.TrimSpace(string(rest))) != 0 { + return false + } + return legacyPrivateKeyPEMTypes[block.Type] +} + +// decryptPrivateKey resolves rec.PrivateKey to plaintext PEM in place. +// +// Attempts AES-256-GCM decryption first. A successful decrypt is the common +// case once a record has been through PutCert under this Store. A failed +// decrypt falls back to legacy-plaintext ONLY when the raw value is +// unambiguously a PEM private key (isPlaintextPrivateKeyPEM) — i.e. a record +// written before encryption was wired up. It is re-encrypted automatically +// the next time PutCert runs. Anything else (wrong key, truncated/tampered +// ciphertext, garbage) is a hard error rather than a silent plaintext +// fallback. +func (s *Store) decryptPrivateKey(rec *CertRecord) error { + if plaintext, err := handlers_iam.DecryptSecret(rec.PrivateKey, s.masterKey); err == nil { + rec.PrivateKey = plaintext + return nil + } + if isPlaintextPrivateKeyPEM(rec.PrivateKey) { + return nil + } + return fmt.Errorf("cert %s: private key is neither valid ciphertext nor a recognizable PEM key", rec.CertificateArn) +} + // DeleteCert removes a certificate by ARN. Returns (false, nil) when absent. func (s *Store) DeleteCert(ctx context.Context, certArn string) (bool, error) { if _, err := s.kv.Get(ctx, certKey(certArn)); err != nil { @@ -113,8 +329,47 @@ func (s *Store) DeleteCert(ctx context.Context, certArn string) (bool, error) { return true, nil } -// ListCerts returns all certificate records owned by accountID. -func (s *Store) ListCerts(ctx context.Context, accountID string) ([]*CertRecord, error) { +// ListCertMetadata returns metadata for every certificate owned by accountID, +// with PrivateKey left empty rather than decrypted. +// +// Listing is a summary operation: ListCertificates, its only caller, reads +// ARN/domain/status/type/algorithm/validity/InUseBy and never the key. A +// decrypting variant existed here and cost an AES-GCM open per stored +// certificate on every list call, scaling with certificate count, to produce +// plaintext of exactly the material this store exists to protect and that no +// caller then read. A caller needing key material fetches the one certificate +// it wants via GetCert. +func (s *Store) ListCertMetadata(ctx context.Context, accountID string) ([]*CertRecord, error) { + return s.listCerts(ctx, func(rec *CertRecord) bool { return rec.AccountID == accountID }, false) +} + +// ListAllCertMetadata returns every certificate record's metadata, across +// every account, with PrivateKey left empty rather than decrypted. +// +// This exists for the renewal worker's scan, which only ever reads +// Type/Status/RenewalEligibility/NotBefore/NotAfter/RenewalSummary/ +// CertificateArn to decide what is due — decrypting every stored private key +// on every scan tick, forever, to answer a question that never looks at the +// key is needless AES-GCM work that scales with certificate count, and +// needless plaintext exposure of exactly the material this store exists to +// protect. The worker re-fetches the one certificate it actually renews via +// GetCert (which does decrypt) once it has won that certificate's lease. +// +// PrivateKey is cleared to "" here rather than left as ciphertext, so a +// caller cannot mistake ciphertext for usable key material in a field typed +// as plaintext PEM everywhere else in this package. +func (s *Store) ListAllCertMetadata(ctx context.Context) ([]*CertRecord, error) { + return s.listCerts(ctx, func(*CertRecord) bool { return true }, false) +} + +// listCerts walks every cert key in the bucket, keeping only the records for +// which keep returns true. When decrypt is true, PrivateKey is resolved to +// plaintext PEM as GetCert does, and a record whose key cannot be decrypted +// is skipped and logged rather than failing the whole scan. When decrypt is +// false, PrivateKey is cleared to "" instead of decrypted — see +// ListAllCertMetadata — and an undecryptable key is not a reason to skip the +// record, since nothing here reads it. +func (s *Store) listCerts(ctx context.Context, keep func(*CertRecord) bool, decrypt bool) ([]*CertRecord, error) { keys, err := s.kv.Keys(ctx) if err != nil { if errors.Is(err, jetstream.ErrNoKeysFound) { @@ -135,10 +390,198 @@ func (s *Store) ListCerts(ctx context.Context, accountID string) ([]*CertRecord, if err := json.Unmarshal(entry.Value(), &rec); err != nil { continue } - if rec.AccountID != accountID { + if !keep(&rec) { continue } + if decrypt { + if err := s.decryptPrivateKey(&rec); err != nil { + slog.Warn("listCerts: skipping cert with undecryptable private key", "arn", rec.CertificateArn, "err", err) + continue + } + } else { + rec.PrivateKey = "" + } out = append(out, &rec) } return out, nil } + +// maxInUseByCASAttempts bounds the AddInUseBy/RemoveInUseBy optimistic- +// concurrency retry loop so a pathologically contended key fails loudly +// instead of retrying forever. Set high enough to ride out a burst of +// listeners referencing the same certificate all writing at once (e.g. a +// load balancer created with several HTTPS listeners in short succession). +const maxInUseByCASAttempts = 50 + +// inUseByCASBackoffBase is the base delay between CAS retries. A small +// jittered backoff spreads out contending writers instead of having every +// retry immediately re-collide on the same revision. +const inUseByCASBackoffBase = 2 * time.Millisecond + +// AddInUseBy adds resourceArn (a load balancer ARN) to certArn's InUseBy set. +// No-op if the certificate does not exist or already lists resourceArn. +// +// InUseBy is the sole mechanism by which a re-imported certificate reaches +// the data plane (see UpdateStoredConfigForCert in handlers/elbv2), so a lost +// update here is not a cosmetic race: it silently drops a load balancer from +// fan-out, and that load balancer's certificate expires in HAProxy while ACM +// still reports it as renewed. Two listeners on different load balancers can +// legitimately attach the same certificate concurrently, so this uses +// JetStream KV revision-based compare-and-swap rather than a plain +// get/mutate/put, retrying on a conflicting concurrent writer. +func (s *Store) AddInUseBy(ctx context.Context, certArn, resourceArn string) error { + return s.updateInUseByCAS(ctx, certArn, func(cur []string) []string { + if slices.Contains(cur, resourceArn) { + return nil // no-op: already present + } + next := append(slices.Clone(cur), resourceArn) + slices.Sort(next) + return next + }) +} + +// RemoveInUseBy removes resourceArn from certArn's InUseBy set. No-op if the +// certificate or the entry does not exist. See AddInUseBy for why this uses +// CAS rather than get/mutate/put. +func (s *Store) RemoveInUseBy(ctx context.Context, certArn, resourceArn string) error { + return s.updateInUseByCAS(ctx, certArn, func(cur []string) []string { + idx := slices.Index(cur, resourceArn) + if idx == -1 { + return nil // no-op: not present + } + return slices.Delete(slices.Clone(cur), idx, idx+1) + }) +} + +// updateInUseByCAS applies mutate to certArn's current InUseBy set and writes +// the result back with a revision-checked kv.Update, retrying against the +// latest revision whenever a concurrent writer wins the race. mutate returns +// nil to mean "no change needed", in which case nothing is written. Returns +// nil (no-op, no retry) if the certificate does not exist. +func (s *Store) updateInUseByCAS(ctx context.Context, certArn string, mutate func(cur []string) []string) error { + key := certKey(certArn) + for attempt := range maxInUseByCASAttempts { + entry, err := s.kv.Get(ctx, key) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + return err + } + var rec CertRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return fmt.Errorf("unmarshal cert: %w", err) + } + + next := mutate(rec.InUseBy) + if next == nil { + return nil + } + rec.InUseBy = next + + data, err := json.Marshal(&rec) + if err != nil { + return fmt.Errorf("marshal cert: %w", err) + } + if _, err := s.kv.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { + // Another writer updated the record between our Get and + // Update; back off briefly (jittered, so contending writers + // don't all re-collide on the same revision) and retry + // against whatever is there now. + backoff := inUseByCASBackoffBase * time.Duration(attempt+1) + jitter := time.Duration(rand.Int64N(int64(backoff))) //nolint:gosec // jitter, not cryptographic + select { + case <-time.After(backoff/2 + jitter): + case <-ctx.Done(): + return ctx.Err() + } + continue + } + return err + } + return nil + } + return fmt.Errorf("acm store: exceeded %d CAS attempts updating InUseBy for %s", maxInUseByCASAttempts, certArn) +} + +// AcquireLease attempts to take the per-certificate issuance/renewal lease on +// certArn for holderID, valid until now+ttl. The lease lives on the record +// itself (CertRecord.LeaseHolder/LeaseExpiresAt) rather than in a separate +// leader-election bucket — contrast handlers/eks/cluster_reconciler.go, which +// leases a whole cluster for the life of a Run loop — because the unit of +// work here is one certificate, reissued and released in a single pass. +// +// Succeeds (true) when the lease is unheld, expired, or already held by +// holderID, so a worker that restarts mid-renewal with the same holderID +// does not need to release first. Returns (false, nil), not an error, when a +// different holder's lease is still live or a concurrent acquirer won the +// CAS race — the caller should skip this certificate and let the current +// holder finish or the lease expire. +func (s *Store) AcquireLease(ctx context.Context, certArn, holderID string, ttl time.Duration, now time.Time) (bool, error) { + key := certKey(certArn) + entry, err := s.kv.Get(ctx, key) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + return false, err + } + var rec CertRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return false, fmt.Errorf("unmarshal cert: %w", err) + } + if rec.LeaseHolder != "" && rec.LeaseHolder != holderID && rec.LeaseExpiresAt.After(now) { + return false, nil + } + rec.LeaseHolder = holderID + rec.LeaseExpiresAt = now.Add(ttl) + data, err := json.Marshal(&rec) + if err != nil { + return false, fmt.Errorf("marshal cert: %w", err) + } + if _, err := s.kv.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { + // Lost the race to a concurrent acquirer between Get and Update; + // the caller skips this tick rather than retrying immediately. + return false, nil + } + return false, err + } + return true, nil +} + +// ReleaseLease clears certArn's lease if it is still held by holderID, so the +// next scan (on this node or another) can pick the certificate up immediately +// instead of waiting out the TTL. Best-effort: a failed release just leaves +// the lease for LeaseExpiresAt to reap. +func (s *Store) ReleaseLease(ctx context.Context, certArn, holderID string) error { + key := certKey(certArn) + entry, err := s.kv.Get(ctx, key) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + return err + } + var rec CertRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return fmt.Errorf("unmarshal cert: %w", err) + } + if rec.LeaseHolder != holderID { + return nil // already released, expired, or taken over by another holder + } + rec.LeaseHolder = "" + rec.LeaseExpiresAt = time.Time{} + data, err := json.Marshal(&rec) + if err != nil { + return fmt.Errorf("marshal cert: %w", err) + } + if _, err := s.kv.Update(ctx, key, data, entry.Revision()); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { + return nil // record changed concurrently; nothing to clean up + } + return err + } + return nil +} diff --git a/spinifex/handlers/acm/store_test.go b/spinifex/handlers/acm/store_test.go new file mode 100644 index 000000000..1722fb243 --- /dev/null +++ b/spinifex/handlers/acm/store_test.go @@ -0,0 +1,347 @@ +package handlers_acm + +import ( + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupACMStore returns a Store wired with a master key, mirroring how +// NewACMServiceImplWithNATS constructs one for the ACM service. A Store is +// never unkeyed, so there is no plain variant. +func setupACMStore(t *testing.T) *Store { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + store, err := NewStore(t.Context(), nc, masterKey) + require.NoError(t, err) + return store +} + +func TestAddInUseBy_AddsAndDedupes(t *testing.T) { + store := setupACMStore(t) + rec := &CertRecord{CertificateArn: "arn:aws:acm:ap-southeast-2:000000000001:certificate/inuse-1", AccountID: testAccountID} + require.NoError(t, store.PutCert(t.Context(), rec)) + + require.NoError(t, store.AddInUseBy(t.Context(), rec.CertificateArn, "arn:lb/one")) + got, err := store.GetCert(t.Context(), rec.CertificateArn) + require.NoError(t, err) + assert.Equal(t, []string{"arn:lb/one"}, got.InUseBy) + + // Adding the same LB again is a no-op, not a duplicate entry. + require.NoError(t, store.AddInUseBy(t.Context(), rec.CertificateArn, "arn:lb/one")) + got, err = store.GetCert(t.Context(), rec.CertificateArn) + require.NoError(t, err) + assert.Equal(t, []string{"arn:lb/one"}, got.InUseBy) + + // A second, distinct LB is appended. + require.NoError(t, store.AddInUseBy(t.Context(), rec.CertificateArn, "arn:lb/two")) + got, err = store.GetCert(t.Context(), rec.CertificateArn) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"arn:lb/one", "arn:lb/two"}, got.InUseBy) +} + +func TestAddInUseBy_NoopOnMissingCert(t *testing.T) { + store := setupACMStore(t) + // No cert exists under this ARN — must not error and must not create one. + require.NoError(t, store.AddInUseBy(t.Context(), "arn:aws:acm:ap-southeast-2:000000000001:certificate/missing", "arn:lb/one")) + + got, err := store.GetCert(t.Context(), "arn:aws:acm:ap-southeast-2:000000000001:certificate/missing") + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestRemoveInUseBy_RemovesEntry(t *testing.T) { + store := setupACMStore(t) + rec := &CertRecord{ + CertificateArn: "arn:aws:acm:ap-southeast-2:000000000001:certificate/inuse-2", + AccountID: testAccountID, + InUseBy: []string{"arn:lb/one", "arn:lb/two"}, + } + require.NoError(t, store.PutCert(t.Context(), rec)) + + require.NoError(t, store.RemoveInUseBy(t.Context(), rec.CertificateArn, "arn:lb/one")) + got, err := store.GetCert(t.Context(), rec.CertificateArn) + require.NoError(t, err) + assert.Equal(t, []string{"arn:lb/two"}, got.InUseBy) +} + +// TestAddInUseBy_ConcurrentDistinctResourcesAllSurvive is the correctness +// regression test for the InUseBy index: it asserts final state, not merely +// "no error/panic". A plain get-modify-put loses concurrent writers silently +// — every goroutine reads the same starting record, mutates its own copy, +// and the last Put wins, discarding the others' additions with no error +// raised anywhere. That failure mode is exactly what makes this bug +// dangerous: the record still looks well-formed, just short an entry, and +// the missing load balancer never gets a fan-out re-render when its +// certificate is renewed — it silently expires in HAProxy while ACM reports +// a healthy, renewed certificate. Only asserting the exact final set +// catches that; asserting "no errors" does not. +func TestAddInUseBy_ConcurrentDistinctResourcesAllSurvive(t *testing.T) { + store := setupACMStore(t) + certArn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/inuse-concurrent" + require.NoError(t, store.PutCert(t.Context(), &CertRecord{CertificateArn: certArn, AccountID: testAccountID})) + + const n = 20 + want := make([]string, n) + var wg sync.WaitGroup + errs := make(chan error, n) + for i := range n { + want[i] = fmt.Sprintf("arn:lb/concurrent-%02d", i) + wg.Add(1) + go func(resourceArn string) { + defer wg.Done() + if err := store.AddInUseBy(t.Context(), certArn, resourceArn); err != nil { + errs <- err + } + }(want[i]) + } + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("AddInUseBy failed: %v", err) + } + + got, err := store.GetCert(t.Context(), certArn) + require.NoError(t, err) + assert.ElementsMatch(t, want, got.InUseBy, "every concurrent AddInUseBy call must survive, none silently lost") +} + +func TestRemoveInUseBy_NoopWhenAbsentOrMissingCert(t *testing.T) { + store := setupACMStore(t) + rec := &CertRecord{ + CertificateArn: "arn:aws:acm:ap-southeast-2:000000000001:certificate/inuse-3", + AccountID: testAccountID, + InUseBy: []string{"arn:lb/one"}, + } + require.NoError(t, store.PutCert(t.Context(), rec)) + + // Removing an LB that was never in the set is a no-op. + require.NoError(t, store.RemoveInUseBy(t.Context(), rec.CertificateArn, "arn:lb/never-there")) + got, err := store.GetCert(t.Context(), rec.CertificateArn) + require.NoError(t, err) + assert.Equal(t, []string{"arn:lb/one"}, got.InUseBy) + + // Removing against a nonexistent cert must not error. + require.NoError(t, store.RemoveInUseBy(t.Context(), "arn:aws:acm:ap-southeast-2:000000000001:certificate/missing", "arn:lb/one")) +} + +func TestStore_PutCert_EncryptsPrivateKeyAtRest(t *testing.T) { + store := setupACMStore(t) + const plaintext = "-----BEGIN EC PRIVATE KEY-----\nZm9v\n-----END EC PRIVATE KEY-----\n" + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/enc" + rec := &CertRecord{CertificateArn: arn, AccountID: "000000000001", PrivateKey: plaintext} + require.NoError(t, store.PutCert(t.Context(), rec)) + + // The raw bytes landed in KV must not contain the plaintext key. + entry, err := store.kv.Get(t.Context(), certKey(arn)) + require.NoError(t, err) + assert.NotContains(t, string(entry.Value()), "BEGIN EC PRIVATE KEY") + + // GetCert decrypts back to the original plaintext. + got, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + assert.Equal(t, plaintext, got.PrivateKey) + + // PutCert must not mutate the caller's record in place. + assert.Equal(t, plaintext, rec.PrivateKey) +} + +func TestStore_GetCert_LegacyPlaintextPassthroughAndReencrypt(t *testing.T) { + store := setupACMStore(t) + const plaintext = "-----BEGIN RSA PRIVATE KEY-----\nYmFy\n-----END RSA PRIVATE KEY-----\n" + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/legacy" + + // Simulate a pre-encryption record written before this Store ever had a + // master key: plaintext PEM landed directly in KV via json.Marshal, the + // same shape PutCert produced before this change. + legacy := &CertRecord{CertificateArn: arn, AccountID: "000000000001", PrivateKey: plaintext} + data, err := json.Marshal(legacy) + require.NoError(t, err) + _, err = store.kv.Put(t.Context(), certKey(arn), data) + require.NoError(t, err) + + got, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + assert.Equal(t, plaintext, got.PrivateKey, "legacy plaintext record must still be readable with no migration step") + + // The next write re-encrypts it — no operator action required. + require.NoError(t, store.PutCert(t.Context(), got)) + entry, err := store.kv.Get(t.Context(), certKey(arn)) + require.NoError(t, err) + assert.NotContains(t, string(entry.Value()), "BEGIN RSA PRIVATE KEY", "record must be re-encrypted after the next PutCert") +} + +// TestStore_GetCert_RejectsGarbageAsLegacyPlaintext is the downgrade-vector +// guard: a value that decrypts under no key and does not unambiguously parse +// as a PEM private key block must be a hard error, never silently trusted as +// legacy plaintext. +func TestStore_GetCert_RejectsGarbageAsLegacyPlaintext(t *testing.T) { + store := setupACMStore(t) + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/garbage" + + garbage := &CertRecord{CertificateArn: arn, AccountID: "000000000001", PrivateKey: "not-pem-and-not-ciphertext"} + data, err := json.Marshal(garbage) + require.NoError(t, err) + _, err = store.kv.Put(t.Context(), certKey(arn), data) + require.NoError(t, err) + + _, err = store.GetCert(t.Context(), arn) + require.Error(t, err, "garbage that is neither valid ciphertext nor PEM must not be silently treated as plaintext") +} + +// TestNewStore_RequiresMasterKey guards against a Store ever being +// constructed unkeyed: a keyed writer and an unkeyed reader of the same +// bucket would disagree silently, with the reader getting ciphertext where +// it expects PEM. +func TestNewStore_RequiresMasterKey(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + _, err := NewStore(t.Context(), nc, nil) + require.Error(t, err) +} + +// TestStore_PutGetCert_RoundTripsManagedIssuanceFields asserts the +// RequestCertificate-era fields on CertRecord — validation state, the +// delegation token, and the resumable ACME/lease state — survive a +// PutCert/GetCert round trip unchanged, alongside the encrypted PrivateKey. +func TestStore_PutGetCert_RoundTripsManagedIssuanceFields(t *testing.T) { + store := setupACMStore(t) + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/managed-1" + rec := &CertRecord{ + CertificateArn: arn, + AccountID: testAccountID, + DomainName: "managed.example.com", + Type: "AMAZON_ISSUED", + Status: "PENDING_VALIDATION", + ValidationMethod: ValidationModeManualTXT, + DomainValidationOptions: []DomainValidationEntry{ + { + DomainName: "managed.example.com", + RecordType: "TXT", + RecordName: "_acme-challenge.managed.example.com.", + ValidationStatus: "PENDING_VALIDATION", + }, + }, + RenewalEligibility: "INELIGIBLE", + DelegationToken: "delegation-token-abc", + ACMEOrderURL: "https://acme.example/order/1", + LeaseHolder: "node-a", + } + require.NoError(t, store.PutCert(t.Context(), rec)) + + got, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, rec.ValidationMethod, got.ValidationMethod) + assert.Equal(t, rec.DomainValidationOptions, got.DomainValidationOptions) + assert.Equal(t, rec.DelegationToken, got.DelegationToken, "the delegation token must be ARN-stable across reads") + assert.Equal(t, rec.ACMEOrderURL, got.ACMEOrderURL) + assert.Equal(t, rec.LeaseHolder, got.LeaseHolder) + assert.Equal(t, rec.RenewalEligibility, got.RenewalEligibility) +} + +// TestListAllCertMetadata_DoesNotDecryptPrivateKey is the renewal worker's +// scan path (Worker.scanOnce calls this directly): it must return metadata +// without ever touching AES-GCM or materialising plaintext key PEM, because +// the renewal predicate never reads PrivateKey and the scan runs on every +// tick for every certificate in the store. +func TestListAllCertMetadata_DoesNotDecryptPrivateKey(t *testing.T) { + store := setupACMStore(t) + const plaintext = "-----BEGIN EC PRIVATE KEY-----\nZm9v\n-----END EC PRIVATE KEY-----\n" + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/metadata-only" + require.NoError(t, store.PutCert(t.Context(), &CertRecord{ + CertificateArn: arn, AccountID: testAccountID, DomainName: "metadata.example.com", PrivateKey: plaintext, + })) + + recs, err := store.ListAllCertMetadata(t.Context()) + require.NoError(t, err) + require.Len(t, recs, 1) + assert.Equal(t, "metadata.example.com", recs[0].DomainName, "metadata fields must still be populated") + assert.Empty(t, recs[0].PrivateKey, "ListAllCertMetadata must not return the plaintext key") + assert.NotContains(t, recs[0].PrivateKey, "BEGIN EC PRIVATE KEY") + + // And GetCert on the same record still decrypts fine — proves the scan + // path never touched (or corrupted) the stored ciphertext. + got, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + assert.Equal(t, plaintext, got.PrivateKey) +} + +// TestListAllCertMetadata_CrossesAccounts asserts the renewal worker's scan +// sees every account's certificates, unlike the account-scoped ListCerts used +// by the ListCertificates API. +func TestListAllCertMetadata_CrossesAccounts(t *testing.T) { + store := setupACMStore(t) + require.NoError(t, store.PutCert(t.Context(), &CertRecord{ + CertificateArn: "arn:aws:acm:ap-southeast-2:1:certificate/acct-a", AccountID: "000000000001", + })) + require.NoError(t, store.PutCert(t.Context(), &CertRecord{ + CertificateArn: "arn:aws:acm:ap-southeast-2:2:certificate/acct-b", AccountID: "000000000002", + })) + + recs, err := store.ListAllCertMetadata(t.Context()) + require.NoError(t, err) + assert.Len(t, recs, 2) +} + +// TestAcquireLease_LeavesPrivateKeyEncrypted mirrors +// TestAddInUseBy_LeavesPrivateKeyEncrypted: AcquireLease is the same +// raw-CAS-write shape (get, mutate a couple of fields, marshal, CAS Update), +// and that shape has already produced one silent-plaintext-write bug in this +// package, so every write path over CertRecord gets this guard. +func TestAcquireLease_LeavesPrivateKeyEncrypted(t *testing.T) { + store := setupACMStore(t) + const plaintext = "-----BEGIN EC PRIVATE KEY-----\nZm9v\n-----END EC PRIVATE KEY-----\n" + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/lease-encrypted" + require.NoError(t, store.PutCert(t.Context(), &CertRecord{ + CertificateArn: arn, AccountID: testAccountID, PrivateKey: plaintext, + })) + + acquired, err := store.AcquireLease(t.Context(), arn, "node-a", 5*time.Minute, time.Now()) + require.NoError(t, err) + require.True(t, acquired) + + entry, err := store.kv.Get(t.Context(), certKey(arn)) + require.NoError(t, err) + assert.NotContains(t, string(entry.Value()), "BEGIN EC PRIVATE KEY", + "AcquireLease must not write the private key back in plaintext") + + got, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + assert.Equal(t, plaintext, got.PrivateKey, "the key must still decrypt after a lease acquire") + assert.Equal(t, "node-a", got.LeaseHolder) +} + +// TestReleaseLease_LeavesPrivateKeyEncrypted is AcquireLease's counterpart +// for the release path — same raw-CAS shape, same guard. +func TestReleaseLease_LeavesPrivateKeyEncrypted(t *testing.T) { + store := setupACMStore(t) + const plaintext = "-----BEGIN EC PRIVATE KEY-----\nZm9v\n-----END EC PRIVATE KEY-----\n" + arn := "arn:aws:acm:ap-southeast-2:000000000001:certificate/lease-release-encrypted" + require.NoError(t, store.PutCert(t.Context(), &CertRecord{ + CertificateArn: arn, AccountID: testAccountID, PrivateKey: plaintext, + })) + acquired, err := store.AcquireLease(t.Context(), arn, "node-a", 5*time.Minute, time.Now()) + require.NoError(t, err) + require.True(t, acquired) + + require.NoError(t, store.ReleaseLease(t.Context(), arn, "node-a")) + + entry, err := store.kv.Get(t.Context(), certKey(arn)) + require.NoError(t, err) + assert.NotContains(t, string(entry.Value()), "BEGIN EC PRIVATE KEY", + "ReleaseLease must not write the private key back in plaintext") + + got, err := store.GetCert(t.Context(), arn) + require.NoError(t, err) + assert.Equal(t, plaintext, got.PrivateKey, "the key must still decrypt after a lease release") + assert.Empty(t, got.LeaseHolder, "a released lease must clear the holder") +} diff --git a/spinifex/handlers/acm/validation_mode_test.go b/spinifex/handlers/acm/validation_mode_test.go new file mode 100644 index 000000000..a39cbaa98 --- /dev/null +++ b/spinifex/handlers/acm/validation_mode_test.go @@ -0,0 +1,104 @@ +package handlers_acm + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// hostsZoneFake returns a NorthstarHostsZone func that reports true only for +// the exact domains listed — no suffix matching, so a test that never lists a +// parent zone can never accidentally pass through it. +func hostsZoneFake(hosted ...string) func(string) bool { + set := make(map[string]bool, len(hosted)) + for _, h := range hosted { + set[h] = true + } + return func(domain string) bool { return set[domain] } +} + +// TestDeriveValidationMode exercises the precedence directly against +// deriveValidationMode, independent of RequestCertificate's surrounding +// plumbing. +func TestDeriveValidationMode(t *testing.T) { + tenantCA := &fakeCertAuthority{permitted: map[string]bool{ + "private.example.com": true, + }} + + cases := []struct { + name string + tenantCA CertAuthority + dnsProviderConfigured bool + northstarHostsZone func(string) bool + domains []string + want string + }{ + { + name: "no signals at all falls back to PRIVATE_CA", + domains: []string{"unknown.example.com"}, + want: ValidationModePrivateCA, + }, + { + name: "domain outside every hosted zone does not select MANUAL_TXT", + northstarHostsZone: hostsZoneFake("other.example.org"), + domains: []string{"unrelated.example.com"}, + want: ValidationModePrivateCA, + }, + { + name: "domain inside a hosted zone selects MANUAL_TXT when nothing else covers it", + northstarHostsZone: hostsZoneFake("lab.example.com"), + domains: []string{"lab.example.com"}, + want: ValidationModeManualTXT, + }, + { + name: "tenant CA coverage wins over a northstar-hosted zone (demo-critical)", + tenantCA: tenantCA, + northstarHostsZone: hostsZoneFake("private.example.com"), + domains: []string{"private.example.com"}, + want: ValidationModePrivateCA, + }, + { + name: "mixed SAN coverage under the tenant CA falls through instead of selecting PRIVATE_CA via rule 1", + // tenantCA only permits "private.example.com", not the SAN, so rule 1 + // must not fire; northstar hosts both domains, so the correct result + // (falling through to rule 3) is MANUAL_TXT, not PRIVATE_CA. + tenantCA: tenantCA, + northstarHostsZone: hostsZoneFake("private.example.com", "other.example.com"), + domains: []string{"private.example.com", "other.example.com"}, + want: ValidationModeManualTXT, + }, + { + name: "mixed SAN coverage under northstar does not select MANUAL_TXT", + // Only one of the two domains is hosted; a buggy "any" check would + // select MANUAL_TXT here instead of falling through to PRIVATE_CA. + northstarHostsZone: hostsZoneFake("hosted.example.com"), + domains: []string{"hosted.example.com", "unhosted.example.com"}, + want: ValidationModePrivateCA, + }, + { + name: "dnsProviderConfigured still wins over a northstar-hosted zone when no tenant CA covers the domain", + dnsProviderConfigured: true, + northstarHostsZone: hostsZoneFake("provider.example.com"), + domains: []string{"provider.example.com"}, + want: ValidationModeProviderAPI, + }, + { + name: "dnsProviderConfigured wins even when the tenant CA is wired but does not cover the domain", + tenantCA: tenantCA, + dnsProviderConfigured: true, + domains: []string{"provider-only.example.com"}, + want: ValidationModeProviderAPI, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc := &ACMServiceImpl{ + TenantCA: tc.tenantCA, + dnsProviderConfigured: tc.dnsProviderConfigured, + NorthstarHostsZone: tc.northstarHostsZone, + } + assert.Equal(t, tc.want, svc.deriveValidationMode(tc.domains)) + }) + } +} diff --git a/spinifex/handlers/dns/zone_lookup.go b/spinifex/handlers/dns/zone_lookup.go new file mode 100644 index 000000000..503d853b6 --- /dev/null +++ b/spinifex/handlers/dns/zone_lookup.go @@ -0,0 +1,50 @@ +package dns + +import ( + "log/slog" + "strings" + + nsconfig "github.com/mulgadc/northstar/pkg/config" + "github.com/mulgadc/spinifex/spinifex/config" +) + +// HostsZone reports whether northstar hosts a zone covering domain — the zone +// for domain itself, or for one of its parent labels (so a request for +// "jellyfin.lab.example.com" matches a zone hosted at "lab.example.com" +// without requiring an exact-match zone per host). +// +// Used by ACM's deriveValidationMode to tell an internal zone (only a private +// CA or a hand-published TXT record can prove control over it) from a public +// one (a real ACME CA's own validation applies). Returns false, not an error, +// whenever the answer cannot be determined: northstar S3 zone storage is not +// configured (zoneS3Config fails), or every candidate zone lookup errors — a +// certificate request must not hard-fail because a zone lookup was +// unavailable, it must just decline to select a northstar-answered mode. +func HostsZone(cfg *config.Config, domain string) bool { + zoneCfg, ok := zoneS3Config(cfg) + if !ok { + return false + } + + domain = strings.ToLower(strings.TrimSpace(domain)) + domain = strings.TrimSuffix(domain, ".") + if domain == "" { + return false + } + labels := strings.Split(domain, ".") + + // Walk parent labels longest-first: the most specific zone that exists + // wins. Stop once fewer than two labels remain so a bare TLD ("com") is + // never queried as a candidate zone on its own. + for len(labels) >= 2 { + zone := strings.Join(labels, ".") + _, exists, err := nsconfig.ReadZoneRaw(zoneCfg.s3, zone) + if err != nil { + slog.Warn("dns: HostsZone lookup failed, treating as not hosted", "zone", zone, "error", err) + } else if exists { + return true + } + labels = labels[1:] + } + return false +} diff --git a/spinifex/handlers/dns/zone_lookup_test.go b/spinifex/handlers/dns/zone_lookup_test.go new file mode 100644 index 000000000..973b1e649 --- /dev/null +++ b/spinifex/handlers/dns/zone_lookup_test.go @@ -0,0 +1,161 @@ +package dns + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// minimalZoneTOML renders just enough of the zone TOML format for +// nsconfig.ReadZoneRaw to parse successfully — HostsZone only cares that the +// object exists and decodes, not its record contents. +func minimalZoneTOML(domain string) string { + return fmt.Sprintf(`version = 1.0 +[domain] +domain = %q +active = true +soa = "ns1.%s." +[defaults] +ttl = 300 +type = 1 +class = 1 +`, domain, domain) +} + +// fakeZoneS3 is a mutable path-style S3 endpoint (GET only) backing HostsZone +// lookups. Keys listed in errorKeys return 500 instead of the usual 200/404, +// simulating a transient S3 failure for that specific zone object. requested +// records every key fetched, letting a test assert which candidate zones were +// actually queried. +func fakeZoneS3(t *testing.T, bucket string, objects map[string]string, errorKeys map[string]bool) (endpoint string, requested *[]string) { + t.Helper() + var mu sync.Mutex + var log []string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/"+bucket+"/") + mu.Lock() + log = append(log, key) + mu.Unlock() + if errorKeys[key] { + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + body, ok := objects[key] + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv.URL, &log +} + +// hostsZoneTestConfig builds a *config.Config wired at endpoint with the given +// hosted zone objects and per-key error injection, mirroring newTestWriter's +// northstar.toml construction. +func hostsZoneTestConfig(t *testing.T, objects map[string]string, errorKeys map[string]bool) (*config.Config, *[]string) { + t.Helper() + endpoint, requested := fakeZoneS3(t, "northstar", objects, errorKeys) + tomlBody := fmt.Sprintf(`listen = "0.0.0.0:5300" +default_domain = "spx3.net" +[s3] +endpoint = %q +bucket = "northstar" +region = "us-east-1" +access_key = "READONLY" +secret_key = "READONLY" +`, endpoint) + configPath := filepath.Join(t.TempDir(), "northstar.toml") + require.NoError(t, os.WriteFile(configPath, []byte(tomlBody), 0o600)) + + cfg := &config.Config{ + Predastore: config.PredastoreConfig{AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET"}, + Northstar: config.NorthstarConfig{ConfigPath: configPath}, + } + return cfg, requested +} + +func TestHostsZone(t *testing.T) { + objects := map[string]string{ + "lab.example.com.toml": minimalZoneTOML("lab.example.com"), + } + cfg, _ := hostsZoneTestConfig(t, objects, nil) + + cases := []struct { + name string + domain string + want bool + }{ + {"exact match on the hosted zone", "lab.example.com", true}, + {"subdomain walks up to the hosted parent zone", "jellyfin.lab.example.com", true}, + {"deep subdomain walks up to the hosted parent zone", "a.b.lab.example.com", true}, + {"domain outside every hosted zone", "unrelated.example.org", false}, + {"superstring, not a subdomain, must not string-suffix match", "notlab.example.com", false}, + {"trailing dot and mixed case are normalised", "JELLYFIN.LAB.EXAMPLE.COM.", true}, + {"empty domain", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, HostsZone(cfg, tc.domain)) + }) + } +} + +// TestHostsZone_NeverQueriesBareTLD asserts the walk stops once fewer than two +// labels remain, so a lookup for "example.com" (or anything under it) never +// issues a GET for "com.toml". +func TestHostsZone_NeverQueriesBareTLD(t *testing.T) { + cfg, requested := hostsZoneTestConfig(t, map[string]string{}, nil) + + got := HostsZone(cfg, "sub.example.com") + assert.False(t, got) + for _, key := range *requested { + assert.NotEqual(t, "com.toml", key, "must never query the bare TLD as a candidate zone") + } +} + +// TestHostsZone_S3ErrorAtOneLevelContinuesToParent asserts a read error on the +// most specific candidate does not abort the walk — the error is logged and +// treated as "not hosted" for that candidate only, so a real parent zone one +// level up is still found. +func TestHostsZone_S3ErrorAtOneLevelContinuesToParent(t *testing.T) { + objects := map[string]string{ + "example.com.toml": minimalZoneTOML("example.com"), + } + errorKeys := map[string]bool{"lab.example.com.toml": true} + cfg, _ := hostsZoneTestConfig(t, objects, errorKeys) + + assert.True(t, HostsZone(cfg, "lab.example.com"), "a transient error on one candidate must not hide a real parent zone") +} + +// TestHostsZone_S3ErrorEverywhereReturnsFalse asserts a lookup failure with no +// good answer anywhere degrades to "not hosted" rather than propagating an +// error — a certificate request must not hard-fail because a zone lookup was +// unavailable. +func TestHostsZone_S3ErrorEverywhereReturnsFalse(t *testing.T) { + errorKeys := map[string]bool{"lab.example.com.toml": true, "example.com.toml": true} + cfg, _ := hostsZoneTestConfig(t, map[string]string{}, errorKeys) + + assert.False(t, HostsZone(cfg, "lab.example.com")) +} + +// TestHostsZone_NorthstarNotConfigured asserts an unconfigured (or +// misconfigured) northstar S3 backend is "not hosted", not an error — the same +// degrade-safe default zoneS3Config's other callers (Writer, Reconciler) use. +func TestHostsZone_NorthstarNotConfigured(t *testing.T) { + assert.False(t, HostsZone(&config.Config{}, "lab.example.com")) + assert.False(t, HostsZone(nil, "lab.example.com")) +} diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index 3b8fdabc8..72415e94a 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go/service/elbv2" "github.com/mulgadc/spinifex/spinifex/config" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -132,7 +133,9 @@ func setupMicrovmTestService(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc DevNetworking: true, }, } - elbv2Svc, err := NewELBv2ServiceImplWithNATS(cfg, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + elbv2Svc, err := NewELBv2ServiceImplWithNATS(cfg, nc, masterKey) require.NoError(t, err) elbv2Svc.VPCService = vpcSvc elbv2Svc.GatewayURL = "https://10.20.0.5:9999" diff --git a/spinifex/handlers/elbv2/cert_inuse_test.go b/spinifex/handlers/elbv2/cert_inuse_test.go new file mode 100644 index 000000000..adee05016 --- /dev/null +++ b/spinifex/handlers/elbv2/cert_inuse_test.go @@ -0,0 +1,486 @@ +package handlers_elbv2 + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/acm" + "github.com/aws/aws-sdk-go/service/elbv2" + handlers_acm "github.com/mulgadc/spinifex/spinifex/handlers/acm" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// activeLB persists a LoadBalancerRecord directly with an InstanceID set, +// mirroring the LBAgentHeartbeat test helpers, so updateStoredConfig actually +// renders instead of no-opping on a still-provisioning LB. +func activeLB(t *testing.T, svc *ELBv2ServiceImpl, id, name string) *LoadBalancerRecord { + t.Helper() + lb := &LoadBalancerRecord{ + LoadBalancerArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/" + name + "/" + id, + LoadBalancerID: id, + Name: name, + State: StateActive, + InstanceID: "i-sys-" + id, + AccountID: testAccountID, + } + require.NoError(t, svc.store.PutLoadBalancer(context.Background(), lb)) + return lb +} + +func httpsListenerInput(lbArn, certArn string, port int64) *elbv2.CreateListenerInput { + return &elbv2.CreateListenerInput{ + LoadBalancerArn: aws.String(lbArn), + Protocol: aws.String(ProtocolHTTPS), + Port: aws.Int64(port), + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(certArn)}}, + DefaultActions: fixedResponseAction(), + } +} + +func TestCreateListener_AddsToInUseByIndex(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx1", "idx-lb1") + + _, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec.InUseBy) +} + +func TestCreateListener_TwoListenersSameCert_SingleInUseByEntry(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx2", "idx-lb2") + + _, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + _, err = svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 8443), testAccountID) + require.NoError(t, err) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec.InUseBy, + "two listeners on the same LB referencing the same cert must dedupe to one entry") +} + +func TestDeleteListener_RemovesFromInUseByIndex(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx3", "idx-lb3") + + out, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + + _, err = svc.DeleteListener(context.Background(), &elbv2.DeleteListenerInput{ + ListenerArn: out.Listeners[0].ListenerArn, + }, testAccountID) + require.NoError(t, err) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Empty(t, rec.InUseBy, "deleting the sole listener must clear the index entry") +} + +func TestDeleteListener_KeepsInUseByWhenAnotherListenerSharesCert(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx4", "idx-lb4") + + out1, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + _, err = svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 8443), testAccountID) + require.NoError(t, err) + + _, err = svc.DeleteListener(context.Background(), &elbv2.DeleteListenerInput{ + ListenerArn: out1.Listeners[0].ListenerArn, + }, testAccountID) + require.NoError(t, err) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec.InUseBy, "the other listener still references the cert") +} + +func TestModifyListener_SwitchesCertInInUseByIndex(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx5", "idx-lb5") + + out, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + + _, err = svc.ModifyListener(context.Background(), &elbv2.ModifyListenerInput{ + ListenerArn: out.Listeners[0].ListenerArn, + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + oldRec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Empty(t, oldRec.InUseBy, "old cert must drop the LB once no listener references it") + + newRec, err := svc.acmStore.GetCert(context.Background(), testCertArn2) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, newRec.InUseBy) +} + +func TestDeleteLoadBalancer_RemovesFromInUseByIndex(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx6", "idx-lb6") + + _, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + + _, err = svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{ + LoadBalancerArn: aws.String(lb.LoadBalancerArn), + }, testAccountID) + require.NoError(t, err) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Empty(t, rec.InUseBy, "deleting the LB must remove it from the index via the listener cascade") +} + +func TestAddRemoveListenerCertificates_MaintainInUseByIndex(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-idx7", "idx-lb7") + + out, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + listenerArn := out.Listeners[0].ListenerArn + + _, err = svc.AddListenerCertificates(context.Background(), &elbv2.AddListenerCertificatesInput{ + ListenerArn: listenerArn, + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + rec2, err := svc.acmStore.GetCert(context.Background(), testCertArn2) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec2.InUseBy) + + _, err = svc.RemoveListenerCertificates(context.Background(), &elbv2.RemoveListenerCertificatesInput{ + ListenerArn: listenerArn, + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + rec2, err = svc.acmStore.GetCert(context.Background(), testCertArn2) + require.NoError(t, err) + assert.Empty(t, rec2.InUseBy) + + // The default cert (testCertArn) is still referenced. + rec1, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec1.InUseBy) +} + +func TestUpdateStoredConfigForCert_ZeroInUseBy_NoOp(t *testing.T) { + svc := setupTestService(t) + require.NoError(t, svc.UpdateStoredConfigForCert(context.Background(), testCertArn)) +} + +func TestUpdateStoredConfigForCert_UnknownCert_NoOp(t *testing.T) { + svc := setupTestService(t) + require.NoError(t, svc.UpdateStoredConfigForCert(context.Background(), "arn:aws:acm:ap-southeast-2:000000000001:certificate/does-not-exist")) +} + +// TestUpdateStoredConfigForCert_SkipsMissingOrInactiveLB covers both halves of +// the "must not fail the certificate write" edge case: an InUseBy entry whose +// LB was deleted out from under the index, and one whose LB exists but never +// went active (empty InstanceID, so updateStoredConfig itself no-ops). +func TestUpdateStoredConfigForCert_SkipsMissingOrInactiveLB(t *testing.T) { + svc := setupTestService(t) + + require.NoError(t, svc.acmStore.AddInUseBy(context.Background(), testCertArn, + "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/gone/lb-gone")) + + provisioning := &LoadBalancerRecord{ + LoadBalancerArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/prov/lb-prov1", + LoadBalancerID: "lb-prov1", + Name: "prov-lb", + State: StateProvisioning, + AccountID: testAccountID, + } + require.NoError(t, svc.store.PutLoadBalancer(context.Background(), provisioning)) + require.NoError(t, svc.acmStore.AddInUseBy(context.Background(), testCertArn, provisioning.LoadBalancerArn)) + + require.NoError(t, svc.UpdateStoredConfigForCert(context.Background(), testCertArn)) + + stored, err := svc.store.GetLoadBalancer(context.Background(), "lb-prov1") + require.NoError(t, err) + assert.Empty(t, stored.ConfigHash, "a still-provisioning LB must not get a config built for it") +} + +// clearInUseBy strips a certificate's index entry for lbArn without touching +// its listeners, reproducing a listener that predates the index. +func clearInUseBy(t *testing.T, svc *ELBv2ServiceImpl, certArn, lbArn string) { + t.Helper() + require.NoError(t, svc.acmStore.RemoveInUseBy(context.Background(), certArn, lbArn)) + rec, err := svc.acmStore.GetCert(context.Background(), certArn) + require.NoError(t, err) + require.Empty(t, rec.InUseBy, "precondition: the index must start empty") +} + +func TestReconcileCertInUseIndex_BackfillsPreExistingListener(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-rec1", "rec-lb1") + + _, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + clearInUseBy(t, svc, testCertArn, lb.LoadBalancerArn) + + require.NoError(t, svc.ReconcileCertInUseIndex(context.Background())) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec.InUseBy, + "the listener still references the cert, so the rebuild must restore the entry") +} + +func TestReconcileCertInUseIndex_IsIdempotent(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-rec2", "rec-lb2") + + _, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + clearInUseBy(t, svc, testCertArn, lb.LoadBalancerArn) + + require.NoError(t, svc.ReconcileCertInUseIndex(context.Background())) + require.NoError(t, svc.ReconcileCertInUseIndex(context.Background())) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec.InUseBy, "a second run must not duplicate the entry") +} + +func TestReconcileCertInUseIndex_RemovesEntryWithNoListener(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-rec3", "rec-lb3") + + // The LB exists but no listener references the cert, so the entry is stale. + require.NoError(t, svc.acmStore.AddInUseBy(context.Background(), testCertArn, lb.LoadBalancerArn)) + + require.NoError(t, svc.ReconcileCertInUseIndex(context.Background())) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Empty(t, rec.InUseBy, "a rebuild must drop an entry no listener justifies") +} + +func TestReconcileCertInUseIndex_RemovesEntryForDeletedLB(t *testing.T) { + svc := setupTestService(t) + + require.NoError(t, svc.acmStore.AddInUseBy(context.Background(), testCertArn, + "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/gone/lb-rec-gone")) + + require.NoError(t, svc.ReconcileCertInUseIndex(context.Background())) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Empty(t, rec.InUseBy, "an entry naming a load balancer that no longer exists must be dropped") +} + +func TestReconcileCertInUseIndex_TwoListenersSameLB_SingleEntry(t *testing.T) { + svc := setupTestService(t) + lb := activeLB(t, svc, "lb-rec4", "rec-lb4") + + _, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + _, err = svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 8443), testAccountID) + require.NoError(t, err) + clearInUseBy(t, svc, testCertArn, lb.LoadBalancerArn) + + require.NoError(t, svc.ReconcileCertInUseIndex(context.Background())) + + rec, err := svc.acmStore.GetCert(context.Background(), testCertArn) + require.NoError(t, err) + assert.Equal(t, []string{lb.LoadBalancerArn}, rec.InUseBy, "two listeners on one LB must dedupe to one entry") +} + +// A nil acmStore means HTTPS is unavailable, not that the index is broken, so +// the reconcile has to no-op rather than panic on daemon startup. +func TestReconcileCertInUseIndex_NilACMStore_NoOp(t *testing.T) { + require.NoError(t, (&ELBv2ServiceImpl{}).ReconcileCertInUseIndex(context.Background())) +} + +// TestReconcileCertInUseIndex_RestoresRenewalFanOut is the regression test for +// the reported symptom rather than the mechanism: with the index emptied, a +// re-import reaches no load balancer and the served leaf goes stale silently. +// After a reconcile the same re-import must fan out again. +func TestReconcileCertInUseIndex_RestoresRenewalFanOut(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + + elbv2Svc, err := NewELBv2ServiceImplWithNATS(nil, nc, masterKey) + require.NoError(t, err) + acmSvc, err := handlers_acm.NewACMServiceImplWithNATS(context.Background(), nil, nc, masterKey) + require.NoError(t, err) + acmSvc.CertMaterialUpdated = elbv2Svc.UpdateStoredConfigForCert + + origLeaf, origKey := genLeafCertPEM(t, "stale.example.com") + importOut, err := acmSvc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: origLeaf, + PrivateKey: origKey, + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(importOut.CertificateArn) + + lb := activeLB(t, elbv2Svc, "lb-rec5", "rec-lb5") + _, err = elbv2Svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, certArn, 443), testAccountID) + require.NoError(t, err) + clearInUseBy(t, elbv2Svc, certArn, lb.LoadBalancerArn) + + stale, err := elbv2Svc.store.GetLoadBalancer(context.Background(), "lb-rec5") + require.NoError(t, err) + + // Without an index entry the re-import succeeds but reaches nothing. + midLeaf, midKey := genLeafCertPEM(t, "ignored.example.com") + _, err = acmSvc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: midLeaf, + PrivateKey: midKey, + CertificateArn: aws.String(certArn), + }, testAccountID) + require.NoError(t, err) + + unchanged, err := elbv2Svc.store.GetLoadBalancer(context.Background(), "lb-rec5") + require.NoError(t, err) + require.Equal(t, stale.ConfigHash, unchanged.ConfigHash, + "precondition: an empty index is exactly why renewal goes unnoticed") + + require.NoError(t, elbv2Svc.ReconcileCertInUseIndex(context.Background())) + + newLeaf, newKey := genLeafCertPEM(t, "renewed.example.com") + _, err = acmSvc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: newLeaf, + PrivateKey: newKey, + CertificateArn: aws.String(certArn), + }, testAccountID) + require.NoError(t, err) + + after, err := elbv2Svc.store.GetLoadBalancer(context.Background(), "lb-rec5") + require.NoError(t, err) + assert.NotEqual(t, unchanged.ConfigHash, after.ConfigHash, "the reconciled index must let the fan-out through") + + var afterPEM string + for _, pemContent := range after.CertFiles { + afterPEM = pemContent + } + assert.Contains(t, afterPEM, string(newLeaf), "the rendered cert file must carry the renewed leaf") +} + +// genLeafCertPEM returns a self-signed leaf certificate + private key as PEM, +// mirroring handlers_acm's own test helper (unexported, different package). +// dnsNames become SANs, which is how a renderer learns the names a cert serves. +func genLeafCertPEM(t *testing.T, cn string, dnsNames ...string) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: dnsNames, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +// TestACMReimport_FansOutToInUseLoadBalancer is the end-to-end regression test +// for the silent re-import bug: ACMServiceImpl.ImportCertificate on an +// existing, in-use ARN must fan out to every load balancer referencing it and +// re-render its stored config, exactly as wired in daemon.go. It exercises +// the real ImportCertificate path (not a direct store write), the real +// CreateListener path (which builds the InUseBy index), and the real +// UpdateStoredConfigForCert fan-out. +func TestACMReimport_FansOutToInUseLoadBalancer(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + // Both services read the same bucket, so they must share one key or the + // reader gets ciphertext where it expects PEM. + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + + elbv2Svc, err := NewELBv2ServiceImplWithNATS(nil, nc, masterKey) + require.NoError(t, err) + acmSvc, err := handlers_acm.NewACMServiceImplWithNATS(context.Background(), nil, nc, masterKey) + require.NoError(t, err) + // Mirrors the daemon.go wiring between the two services. + acmSvc.CertMaterialUpdated = elbv2Svc.UpdateStoredConfigForCert + + origLeaf, origKey := genLeafCertPEM(t, "orig.example.com") + importOut, err := acmSvc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: origLeaf, + PrivateKey: origKey, + }, testAccountID) + require.NoError(t, err) + certArn := aws.StringValue(importOut.CertificateArn) + + lb := activeLB(t, elbv2Svc, "lb-reimport1", "reimport-lb") + + _, err = elbv2Svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, certArn, 443), testAccountID) + require.NoError(t, err) + + before, err := elbv2Svc.store.GetLoadBalancer(context.Background(), "lb-reimport1") + require.NoError(t, err) + require.NotEmpty(t, before.ConfigHash, "CreateListener must have rendered a config") + + var beforePEM string + for _, pemContent := range before.CertFiles { + beforePEM = pemContent + } + assert.Contains(t, beforePEM, string(origLeaf)) + + // Re-import new leaf material under the same ARN — the bug being fixed. + newLeaf, newKey := genLeafCertPEM(t, "rotated.example.com") + _, err = acmSvc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{ + Certificate: newLeaf, + PrivateKey: newKey, + CertificateArn: aws.String(certArn), + }, testAccountID) + require.NoError(t, err) + + after, err := elbv2Svc.store.GetLoadBalancer(context.Background(), "lb-reimport1") + require.NoError(t, err) + assert.NotEqual(t, before.ConfigHash, after.ConfigHash, "ConfigHash must change after re-import fans out") + + var afterPEM string + for _, pemContent := range after.CertFiles { + afterPEM = pemContent + } + assert.Contains(t, afterPEM, string(newLeaf), "the rendered cert file must carry the new leaf") + assert.NotContains(t, afterPEM, string(origLeaf), "the old leaf must not still be served") +} + +// Concurrency coverage for the InUseBy index itself — N goroutines each +// adding a distinct resource ARN to the same certificate, asserting the +// final set contains exactly all N — lives at the store layer, where the +// property actually belongs: see +// TestAddInUseBy_ConcurrentDistinctResourcesAllSurvive in +// handlers/acm/store_test.go. An earlier version of this test asserted only +// "no errors/panics" across a concurrent re-import/ModifyListener race, +// which cannot detect a lost index entry: a dropped update produces a +// correct-looking record with a missing entry, not an error, so that +// assertion passed whether or not the underlying race was fixed. diff --git a/spinifex/handlers/elbv2/haproxy.go b/spinifex/handlers/elbv2/haproxy.go index 7a2dc90dd..10b469ae9 100644 --- a/spinifex/handlers/elbv2/haproxy.go +++ b/spinifex/handlers/elbv2/haproxy.go @@ -29,7 +29,7 @@ defaults timeout server 60s {{range .Frontends}} frontend {{.Name}} - bind *:{{.Port}}{{if .CertPath}} ssl crt {{.CertPath}}{{end}} + bind *:{{.Port}}{{if .CertPaths}} ssl{{range .CertPaths}} crt {{.}}{{end}}{{end}} {{- range .Rules}} {{- range .ACLs}} acl {{.Name}} {{.Expr}} @@ -70,11 +70,14 @@ type HAProxyConfig struct { // HAProxyFrontend represents a listener frontend. type HAProxyFrontend struct { - Name string - BindAddr string - Port int64 - Protocol string // listener protocol (HTTP/HTTPS/TCP/TLS); drives TLS termination - CertPath string // combined-PEM path for `ssl crt`; empty for non-secure frontends + Name string + BindAddr string + Port int64 + Protocol string // listener protocol (HTTP/HTTPS/TCP/TLS); drives TLS termination + // CertPaths are the combined-PEM paths for `ssl crt`, default certificate + // first, then each attached SNI certificate. Empty for non-secure frontends. + // HAProxy accepts crt more than once on a bind line and picks by SNI itself. + CertPaths []string DefaultBackend string Rules []HAProxyRule } @@ -258,14 +261,14 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB DefaultBackend: defaultBackendName, } - // Resolve the default cert and stage it for delivery to the agent. + // Resolve every attached cert and stage each for delivery to the agent. if protocolRequiresCert(l.Protocol) { - if pem, path := resolveFrontendCert(lb, l, certPEMByArn); pem != "" { - frontend.CertPath = path + for _, c := range resolveFrontendCerts(lbagent.CertDir, lb, l, certPEMByArn) { + frontend.CertPaths = append(frontend.CertPaths, c.Path) if cfg.CertFiles == nil { cfg.CertFiles = make(map[string]string) } - cfg.CertFiles[path] = pem + cfg.CertFiles[c.Path] = c.PEM } } @@ -299,29 +302,69 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB return cfg, nil } -// frontendCertPath returns the stable absolute PEM path under the agent's cert dir. -func frontendCertPath(lb *LoadBalancerRecord, l *ListenerRecord) string { - return filepath.Join(lbagent.CertDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID)) +// frontendCertPath returns the stable absolute PEM path under certDir, which is +// engine-specific: HAProxy and nginx read from different directories. +func frontendCertPath(certDir string, lb *LoadBalancerRecord, l *ListenerRecord) string { + return filepath.Join(certDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID)) } -// resolveFrontendCert returns (pem, path) for the listener's default certificate, -// or ("", "") when absent or unresolved. -func resolveFrontendCert(lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn map[string]string) (pem, path string) { +// frontendCert is one staged PEM: the path the agent writes it to, and the +// combined cert+chain+key it writes there. +type frontendCert struct { + Path string + PEM string +} + +// resolveFrontendCerts returns every resolvable certificate attached to the +// listener, default first so it is the one the engine falls back to when a client +// sends no SNI. Certificates whose material cannot be resolved are skipped +// rather than failing the render: a listener must keep serving its other names. +func resolveFrontendCerts(certDir string, lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn map[string]string) []frontendCert { if len(l.Certificates) == 0 || certPEMByArn == nil { - return "", "" + return nil + } + + out := make([]frontendCert, 0, len(l.Certificates)) + defaultArn := defaultCertArn(l) + if pem := certPEMByArn[defaultArn]; pem != "" { + out = append(out, frontendCert{Path: frontendCertPath(certDir, lb, l), PEM: pem}) + } + + for _, c := range l.Certificates { + if c.CertificateArn == defaultArn { + continue + } + pem, ok := certPEMByArn[c.CertificateArn] + if !ok || pem == "" { + continue + } + out = append(out, frontendCert{Path: sniCertPath(certDir, lb, l, c.CertificateArn), PEM: pem}) + } + return out +} + +// defaultCertArn returns the listener's default certificate ARN: the one flagged +// IsDefault, else the first attached. +func defaultCertArn(l *ListenerRecord) string { + if len(l.Certificates) == 0 { + return "" } - arn := l.Certificates[0].CertificateArn for _, c := range l.Certificates { if c.IsDefault { - arn = c.CertificateArn - break + return c.CertificateArn } } - pem, ok := certPEMByArn[arn] - if !ok || pem == "" { - return "", "" - } - return pem, frontendCertPath(lb, l) + return l.Certificates[0].CertificateArn +} + +// sniCertPath is the stable absolute PEM path for a non-default certificate. +// Keyed on the certificate so it is distinct per cert and stable across renders; +// the default cert keeps frontendCertPath's name so an existing deployment's +// file does not move. sanitizeName also guarantees no path separators, which the +// agent's write path rejects. +func sniCertPath(certDir string, lb *LoadBalancerRecord, l *ListenerRecord, certArn string) string { + return filepath.Join(certDir, + fmt.Sprintf("%s-%s-%s.pem", lb.LoadBalancerID, l.ListenerID, sanitizeName("cert", certArn))) } // registerRuleBackend registers the backend for a rule's single action and returns its name. diff --git a/spinifex/handlers/elbv2/haproxy_certs_test.go b/spinifex/handlers/elbv2/haproxy_certs_test.go index 43c502526..4180c3cb3 100644 --- a/spinifex/handlers/elbv2/haproxy_certs_test.go +++ b/spinifex/handlers/elbv2/haproxy_certs_test.go @@ -2,14 +2,22 @@ package handlers_elbv2 import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" "encoding/xml" "path/filepath" + "strings" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_acm "github.com/mulgadc/spinifex/spinifex/handlers/acm" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/lbagent" + "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -121,6 +129,51 @@ func TestResolveCertPEM_ConcatOrderAndOwnership(t *testing.T) { require.Error(t, err) } +// TestResolveCertPEM_CrossesACMServiceKeyBoundary writes a certificate through +// a Store constructed the way the ACM service constructs its own — NewStore +// wired with the shared deployment master key — then resolves it through +// ELBv2's independently-constructed Store via resolveCertPEM. Every other +// cert test in this file goes through putTestCert, which writes and reads +// via svc.acmStore itself, so it never crosses from one Store instance to +// another; this test exercises the real daemon topology, where +// ACMServiceImpl and ELBv2ServiceImpl each open their own Store over the +// same JetStream bucket and must agree on the same key. +func TestResolveCertPEM_CrossesACMServiceKeyBoundary(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + + svc, err := NewELBv2ServiceImplWithNATS(nil, nc, masterKey) + require.NoError(t, err) + + // ACM-service-style writer: its own Store over the same bucket, wired with + // the same deployment key rather than svc's own acmStore. + acmStore, err := handlers_acm.NewStore(t.Context(), nc, masterKey) + require.NoError(t, err) + + // A realistic EC private key PEM, not a placeholder — a non-PEM value + // would take the legacy-plaintext guard's rejection path instead of the + // real decrypt path this test exercises. + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + keyPEM := string(pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})) + + arn := "arn:aws:acm:ap-southeast-2:123456789012:certificate/cross-boundary" + require.NoError(t, acmStore.PutCert(t.Context(), &handlers_acm.CertRecord{ + CertificateArn: arn, + AccountID: testAccountID, + Certificate: "LEAF", + PrivateKey: keyPEM, + })) + + got, err := svc.resolveCertPEM(t.Context(), arn, testAccountID) + require.NoError(t, err) + assert.Equal(t, "LEAF\n"+strings.TrimRight(keyPEM, "\n")+"\n", got) +} + func TestValidateListenerCerts_RejectsUnknownAndWrongAccount(t *testing.T) { svc := setupTestService(t) putTestCert(t, svc, certTestArn, testAccountID, "LEAF", "", "KEY") diff --git a/spinifex/handlers/elbv2/nginx.go b/spinifex/handlers/elbv2/nginx.go index d1d6e115f..305c2ab06 100644 --- a/spinifex/handlers/elbv2/nginx.go +++ b/spinifex/handlers/elbv2/nginx.go @@ -1,8 +1,10 @@ package handlers_elbv2 import ( + "crypto/x509" + "encoding/pem" "fmt" - "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -11,6 +13,10 @@ import ( "github.com/mulgadc/spinifex/spinifex/lbagent" ) +// maxDNSNameLen is the longest legal DNS name, so a map key cannot be padded out +// to an unreasonable size by a certificate subject. +const maxDNSNameLen = 253 + // NLB (L4) data plane. NLB listeners render an nginx `stream` config because HAProxy // has no UDP support; ALBs stay on HAProxy. Health checks are passive only // (max_fails/fail_timeout) — OSS nginx stream has no active upstream probing. @@ -27,6 +33,15 @@ events { } stream { +{{- range .CertMaps}} + map $ssl_server_name ${{.Var}} { + hostnames; + default {{.DefaultPath}}; +{{- range .Entries}} + "{{.Host}}" {{.Path}}; +{{- end}} + } +{{end}} {{- range .Upstreams}} upstream {{.Name}} { {{- range .Servers}} @@ -40,8 +55,8 @@ stream { server { listen {{.Port}}{{if .UDP}} udp{{end}}{{if .SSL}} ssl{{end}}; {{- if .SSL}} - ssl_certificate {{.CertPath}}; - ssl_certificate_key {{.CertPath}}; + ssl_certificate {{.CertRef}}; + ssl_certificate_key {{.CertRef}}; {{- end}} {{- if .UDP}} proxy_responses 1; @@ -59,10 +74,29 @@ var nginxStreamTmpl = template.Must( // nginxConfig is the render model for the NLB nginx `stream` config. type nginxConfig struct { PIDPath string + CertMaps []nginxCertMap Upstreams []nginxUpstream Servers []nginxServer } +// nginxCertMap picks one TLS listener's certificate by SNI name. nginx allows a +// single ssl_certificate per key type in a server block, so several names are +// served by giving the directive a variable and mapping $ssl_server_name to a +// path. `hostnames` in the rendered map is what makes a *.example.com key match +// a subdomain, so a wildcard certificate's SAN works as written. +type nginxCertMap struct { + Var string + DefaultPath string + Entries []nginxCertMapEntry +} + +// nginxCertMapEntry is one SNI name → PEM path row. Host is lower-cased and +// validated as a hostname before it reaches the config. +type nginxCertMapEntry struct { + Host string + Path string +} + // nginxUpstream is one target group's backend pool. An empty pool renders a // single permanently-down placeholder so nginx (which requires >=1 server) // still validates. @@ -81,10 +115,12 @@ type nginxUpstreamServer struct { // nginxServer is one listener's `server` block. TCP_UDP listeners expand to two // blocks on the same port (one TCP, one UDP). type nginxServer struct { - Port int64 - UDP bool - SSL bool - CertPath string + Port int64 + UDP bool + SSL bool + // CertRef is what ssl_certificate is given: a literal PEM path for a + // single-certificate listener, or `$var` naming this listener's cert map. + CertRef string Upstream string } @@ -140,15 +176,18 @@ func generateNLBStreamConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord } upstream := addUpstream(da.TargetGroupArn) - var certPath string + var certRef string if protocolRequiresCert(l.Protocol) { - pem, _ := resolveFrontendCert(lb, l, certPEMByArn) - if pem != "" { - certPath = nginxCertPath(lb, l) + ref, certMap, files := buildNginxCerts(lb, l, certPEMByArn) + certRef = ref + if certMap != nil { + cfg.CertMaps = append(cfg.CertMaps, *certMap) + } + for path, pem := range files { if certFiles == nil { certFiles = make(map[string]string) } - certFiles[certPath] = pem + certFiles[path] = pem } } @@ -162,8 +201,8 @@ func generateNLBStreamConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord default: // TCP, TLS cfg.Servers = append(cfg.Servers, nginxServer{ Port: l.Port, - SSL: certPath != "", - CertPath: certPath, + SSL: certRef != "", + CertRef: certRef, Upstream: upstream, }) } @@ -176,10 +215,111 @@ func generateNLBStreamConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord return buf.String(), certFiles, nil } -// nginxCertPath is the stable absolute PEM path for a TLS listener under the -// agent's nginx cert dir. Daemon-rendered and agent-written paths must match. -func nginxCertPath(lb *LoadBalancerRecord, l *ListenerRecord) string { - return filepath.Join(lbagent.NginxCertDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID)) +// buildNginxCerts resolves a TLS listener's certificates into the value for +// ssl_certificate, an optional SNI map, and the PEMs to deliver. One certificate +// renders as a literal path: a variable path makes nginx read the PEM on every +// handshake, which the single-certificate case should not pay for. +func buildNginxCerts(lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn map[string]string) (string, *nginxCertMap, map[string]string) { + certs := resolveFrontendCerts(lbagent.NginxCertDir, lb, l, certPEMByArn) + if len(certs) == 0 { + return "", nil, nil + } + + files := map[string]string{certs[0].Path: certs[0].PEM} + if len(certs) == 1 { + return certs[0].Path, nil, files + } + + cm := nginxCertMap{Var: nginxCertVar(l), DefaultPath: certs[0].Path} + claimed := make(map[string]bool) + for i, c := range certs { + names := certDNSNames(c.PEM) + if i == 0 { + // The default already answers every unmatched name, so it needs no + // rows — but its own names are claimed so no later certificate can + // take them. nginx rejects a duplicate map key outright. + for _, h := range names { + claimed[h] = true + } + continue + } + var selectable bool + for _, h := range names { + if claimed[h] { + continue + } + claimed[h] = true + cm.Entries = append(cm.Entries, nginxCertMapEntry{Host: h, Path: c.Path}) + selectable = true + } + // A certificate no SNI name can select would only strand its private key + // on the load balancer, so it is not delivered. + if selectable { + files[c.Path] = c.PEM + } + } + + if len(cm.Entries) == 0 { + return cm.DefaultPath, nil, files + } + return "$" + cm.Var, &cm, files +} + +// nginxCertVar names the per-listener map variable. Scoped to the listener +// because a map lives at stream level, and two listeners may attach different +// certificates for the same name. nginx variables allow only [A-Za-z0-9_], which +// sanitizeName's hyphens would break. +func nginxCertVar(l *ListenerRecord) string { + return strings.ReplaceAll(sanitizeName("sslcert", l.ListenerID), "-", "_") +} + +// nginxDNSNameRegex matches a hostname usable as a map key, optionally wildcarded +// in the leftmost label. Deliberately stricter than the rule-condition host regex: +// these come from a caller-supplied certificate subject and land in the config. +var nginxDNSNameRegex = regexp.MustCompile(`^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$`) + +// certDNSNames returns the names a combined PEM's leaf can be selected by: its +// DNS SANs, or the subject CN when it has none. Anything that is not plainly a +// hostname is dropped rather than rendered. +func certDNSNames(combinedPEM string) []string { + leaf := parseLeafCert(combinedPEM) + if leaf == nil { + return nil + } + names := leaf.DNSNames + if len(names) == 0 && leaf.Subject.CommonName != "" { + names = []string{leaf.Subject.CommonName} + } + + out := make([]string, 0, len(names)) + for _, n := range names { + n = strings.ToLower(strings.TrimSpace(n)) + if len(n) <= maxDNSNameLen && nginxDNSNameRegex.MatchString(n) { + out = append(out, n) + } + } + return out +} + +// parseLeafCert returns the first certificate in a combined cert+chain+key PEM, +// which is the leaf by convention of how the material is assembled. +func parseLeafCert(combinedPEM string) *x509.Certificate { + rest := []byte(combinedPEM) + for { + block, remainder := pem.Decode(rest) + if block == nil { + return nil + } + rest = remainder + if block.Type != "CERTIFICATE" { + continue + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil + } + return leaf + } } // buildNLBHealthTargets returns backends the agent actively probes (nginx stream diff --git a/spinifex/handlers/elbv2/nginx_test.go b/spinifex/handlers/elbv2/nginx_test.go index 5b2976f3d..8af034112 100644 --- a/spinifex/handlers/elbv2/nginx_test.go +++ b/spinifex/handlers/elbv2/nginx_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/mulgadc/spinifex/spinifex/lbagent" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -186,3 +187,110 @@ func TestBuildNLBHealthTargets_ExplicitHCPortAndHTTP(t *testing.T) { assert.Equal(t, ProtocolHTTP, got[0].Protocol) assert.Equal(t, "/healthz", got[0].Path) } + +// tlsListenerWithCerts is a TLS NLB listener whose first certificate is the +// default and the rest are SNI extras, matching AddListenerCertificates. +func tlsListenerWithCerts(listenerID, tgArn string, certArns ...string) *ListenerRecord { + lst := tcpListener("arn:"+listenerID, listenerID, ProtocolTLS, 443, tgArn) + for i, arn := range certArns { + lst.Certificates = append(lst.Certificates, ListenerCertificate{CertificateArn: arn, IsDefault: i == 0}) + } + return lst +} + +// nlbTLSFixture is an NLB with one healthy target behind a TLS listener. +func nlbTLSFixture(listenerID string, certArns ...string) (*LoadBalancerRecord, *ListenerRecord, map[string]*TargetGroupRecord) { + lb := &LoadBalancerRecord{LoadBalancerID: "lb-sni", Type: LoadBalancerTypeNetwork} + tgArn := "arn:tg-sni" + return lb, tlsListenerWithCerts(listenerID, tgArn, certArns...), map[string]*TargetGroupRecord{ + tgArn: nlbTG(tgArn, Target{Id: "i-1", Port: 8443, PrivateIP: "10.0.4.10", HealthState: TargetHealthHealthy}), + } +} + +// combinedPEM mirrors resolveCertPEM's output: leaf, then key. +func combinedPEM(t *testing.T, cn string, dnsNames ...string) string { + t.Helper() + leaf, key := genLeafCertPEM(t, cn, dnsNames...) + return string(leaf) + string(key) +} + +func TestGenerateNLBStream_SNICertGetsItsOwnMapEntry(t *testing.T) { + lb, lst, tgByArn := nlbTLSFixture("lst-sni", "arn:cert-1", "arn:cert-2") + certPEM := map[string]string{ + "arn:cert-1": combinedPEM(t, "default.example.com", "default.example.com"), + "arn:cert-2": combinedPEM(t, "shop.example.com", "shop.example.com", "www.shop.example.com"), + } + + config, certs, err := GenerateHAProxyConfigWithCerts(lb, []*ListenerRecord{lst}, tgByArn, nil, "10.0.4.1", certPEM) + require.NoError(t, err) + + defaultPath := frontendCertPath(lbagent.NginxCertDir, lb, lst) + sniPath := sniCertPath(lbagent.NginxCertDir, lb, lst, "arn:cert-2") + + // The map is what makes nginx serve more than one cert on a listener; the + // directive must reference the variable, not a fixed path. + assert.Contains(t, config, "map $ssl_server_name $sslcert_lst_sni {") + assert.Contains(t, config, "hostnames;") + assert.Contains(t, config, "default "+defaultPath+";") + assert.Contains(t, config, `"shop.example.com" `+sniPath+";") + assert.Contains(t, config, `"www.shop.example.com" `+sniPath+";") + assert.Contains(t, config, "ssl_certificate $sslcert_lst_sni;") + assert.Contains(t, config, "ssl_certificate_key $sslcert_lst_sni;") + + // Both PEMs must be staged, or the map points at a file that never arrives. + require.Contains(t, certs, defaultPath) + require.Contains(t, certs, sniPath) + assert.Equal(t, certPEM["arn:cert-2"], certs[sniPath]) +} + +func TestGenerateNLBStream_WildcardSANBecomesMapKey(t *testing.T) { + lb, lst, tgByArn := nlbTLSFixture("lst-wild", "arn:cert-1", "arn:cert-2") + certPEM := map[string]string{ + "arn:cert-1": combinedPEM(t, "example.com", "example.com"), + "arn:cert-2": combinedPEM(t, "*.apps.example.com", "*.apps.example.com"), + } + + config, _, err := GenerateHAProxyConfigWithCerts(lb, []*ListenerRecord{lst}, tgByArn, nil, "10.0.4.1", certPEM) + require.NoError(t, err) + + assert.Contains(t, config, `"*.apps.example.com" `+sniCertPath(lbagent.NginxCertDir, lb, lst, "arn:cert-2")+";") +} + +// A single certificate must keep rendering a literal path: a variable makes nginx +// re-read the PEM on every handshake. +func TestGenerateNLBStream_SingleCertRendersLiteralPath(t *testing.T) { + lb, lst, tgByArn := nlbTLSFixture("lst-one", "arn:cert-1") + certPEM := map[string]string{"arn:cert-1": combinedPEM(t, "example.com", "example.com")} + + config, _, err := GenerateHAProxyConfigWithCerts(lb, []*ListenerRecord{lst}, tgByArn, nil, "10.0.4.1", certPEM) + require.NoError(t, err) + + assert.Contains(t, config, "ssl_certificate "+frontendCertPath(lbagent.NginxCertDir, lb, lst)+";") + assert.NotContains(t, config, "map $ssl_server_name") +} + +// The default certificate already answers unmatched names, so a second cert that +// only repeats them cannot be selected — and its key must not be shipped. +func TestGenerateNLBStream_UnselectableSNICertNotDelivered(t *testing.T) { + lb, lst, tgByArn := nlbTLSFixture("lst-dup", "arn:cert-1", "arn:cert-2") + certPEM := map[string]string{ + "arn:cert-1": combinedPEM(t, "example.com", "example.com"), + "arn:cert-2": combinedPEM(t, "example.com", "example.com"), + } + + config, certs, err := GenerateHAProxyConfigWithCerts(lb, []*ListenerRecord{lst}, tgByArn, nil, "10.0.4.1", certPEM) + require.NoError(t, err) + + assert.NotContains(t, config, "map $ssl_server_name") + assert.Contains(t, config, "ssl_certificate "+frontendCertPath(lbagent.NginxCertDir, lb, lst)+";") + assert.NotContains(t, certs, sniCertPath(lbagent.NginxCertDir, lb, lst, "arn:cert-2")) +} + +// A subject is caller-supplied on import, so a name that could break the config +// must never reach a map key. +func TestCertDNSNames_DropsNamesThatAreNotHostnames(t *testing.T) { + pemText := combinedPEM(t, "ok.example.com", "ok.example.com", "bad name; }", "-leading-hyphen.com", "UPPER.example.com") + + assert.Equal(t, []string{"ok.example.com", "upper.example.com"}, certDNSNames(pemText)) + assert.Nil(t, certDNSNames("not a pem"), "unparseable material yields no names") +} diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index 0b22aca41..3fa38147d 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -143,7 +143,10 @@ type ELBv2ServiceImpl struct { func (s *ELBv2ServiceImpl) WaitLaunches() { s.launchWG.Wait() } // NewELBv2ServiceImplWithNATS creates an ELBv2 service backed by JetStream KV. -func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ELBv2ServiceImpl, error) { +// masterKey is the same deployment key the ACM service is constructed with; +// ELBv2 needs it to decrypt CertRecord.PrivateKey when resolving listener +// certificates, since it opens its own Store over the shared ACM bucket. +func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn, masterKey []byte) (*ELBv2ServiceImpl, error) { region := "us-east-1" nodeID := "" if cfg != nil { @@ -164,11 +167,14 @@ func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ELBv2Servi } // ACM store shares JetStream KV; its bucket opens under the service lifetime - // context. Non-fatal: failure only disables HTTPS termination. - acmStore, acmErr := handlers_acm.NewStore(ctx, nc) + // context. Fatal: a nil acmStore is indistinguishable from "certificate not + // found" everywhere it is consulted (resolveCertPEM, validateListenerCerts), + // so a construction failure here must fail the whole service rather than + // silently degrade every HTTPS listener. + acmStore, acmErr := handlers_acm.NewStore(ctx, nc, masterKey) if acmErr != nil { - slog.Warn("ELBv2: ACM store unavailable, HTTPS listeners cannot resolve certs", "err", acmErr) - acmStore = nil + cancel() + return nil, fmt.Errorf("failed to create ELBv2 ACM store: %w", acmErr) } hc := newHealthChecker(store) @@ -847,6 +853,175 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(ctx context.Context, return nil } +// UpdateStoredConfigForCert re-renders every load balancer that references +// certArn, per the ACM InUseBy index, after new material has been written +// under that ARN. Mirrors updateStoredConfigForTargetGroup: a load balancer +// that no longer exists is skipped, and updateStoredConfig itself no-ops for +// one that hasn't gone active (lb.InstanceID == ""), so neither can fail the +// certificate write that triggered the fan-out. Zero load balancers in the +// index is a no-op. +func (s *ELBv2ServiceImpl) UpdateStoredConfigForCert(ctx context.Context, certArn string) error { + if s.acmStore == nil { + return nil + } + rec, err := s.acmStore.GetCert(ctx, certArn) + if err != nil { + slog.ErrorContext(ctx, "UpdateStoredConfigForCert: failed to load cert", "certArn", certArn, "err", err) + return fmt.Errorf("get cert: %w", err) + } + if rec == nil { + return nil + } + + // InUseBy is already a de-duplicated set of LB ARNs, but re-key it here + // too so two listeners on the same LB never trigger a double render. + lbArns := make(map[string]bool, len(rec.InUseBy)) + for _, arn := range rec.InUseBy { + lbArns[arn] = true + } + + for lbArn := range lbArns { + lb, lbErr := s.store.GetLoadBalancerByArn(ctx, lbArn) + if lbErr != nil || lb == nil { + continue + } + if err := s.updateStoredConfig(ctx, lb); err != nil { + return err + } + } + return nil +} + +// certsUsedByLB returns the set of certificate ARNs referenced by any +// listener currently persisted for lbArn. Used to reconcile the ACM InUseBy +// index against listener mutations. +func (s *ELBv2ServiceImpl) certsUsedByLB(ctx context.Context, lbArn string) (map[string]bool, error) { + listeners, err := s.store.ListListenersByLB(ctx, lbArn) + if err != nil { + return nil, err + } + used := make(map[string]bool, len(listeners)) + for _, l := range listeners { + for _, c := range l.Certificates { + used[c.CertificateArn] = true + } + } + return used, nil +} + +// syncCertInUseIndex reconciles the ACM InUseBy index for lbArn against a +// pre-mutation certificate-usage snapshot (from certsUsedByLB), adding or +// removing lbArn from each affected certificate's InUseBy set so the index +// keeps tracking listeners as currently persisted. Best-effort: errors are +// logged, not returned, since a listener mutation must not fail because +// bookkeeping on an unrelated index failed. +func (s *ELBv2ServiceImpl) syncCertInUseIndex(ctx context.Context, lbArn string, before map[string]bool) { + if s.acmStore == nil { + return + } + after, err := s.certsUsedByLB(ctx, lbArn) + if err != nil { + slog.ErrorContext(ctx, "syncCertInUseIndex: failed to list listeners", "lbArn", lbArn, "err", err) + return + } + for arn := range before { + if after[arn] { + continue + } + if err := s.acmStore.RemoveInUseBy(ctx, arn, lbArn); err != nil { + slog.ErrorContext(ctx, "syncCertInUseIndex: failed to remove InUseBy entry", "certArn", arn, "lbArn", lbArn, "err", err) + } + } + for arn := range after { + if before[arn] { + continue + } + if err := s.acmStore.AddInUseBy(ctx, arn, lbArn); err != nil { + slog.ErrorContext(ctx, "syncCertInUseIndex: failed to add InUseBy entry", "certArn", arn, "lbArn", lbArn, "err", err) + } + } +} + +// ReconcileCertInUseIndex rebuilds the ACM InUseBy index from the listener +// records currently persisted. syncCertInUseIndex only ever runs on a listener +// mutation, so a listener that predates the index has no entry, and a +// certificate with an empty index neither fans out on renewal nor trips +// DeleteCertificate's in-use guard. Safe to call repeatedly: it writes only +// differences, so a converged index costs zero writes. +func (s *ELBv2ServiceImpl) ReconcileCertInUseIndex(ctx context.Context) error { + if s.acmStore == nil { + return nil + } + + lbs, err := s.store.ListLoadBalancers(ctx) + if err != nil { + return fmt.Errorf("list load balancers: %w", err) + } + + // certArn → the LB ARNs that should be indexed against it. + desired := make(map[string]map[string]bool) + for _, lb := range lbs { + used, usedErr := s.certsUsedByLB(ctx, lb.LoadBalancerArn) + if usedErr != nil { + // Skipping one LB would silently drop its entries, which is the bug + // being fixed, so give up rather than write a partial index. + return fmt.Errorf("list listeners for %s: %w", lb.LoadBalancerArn, usedErr) + } + for arn := range used { + if desired[arn] == nil { + desired[arn] = make(map[string]bool) + } + desired[arn][lb.LoadBalancerArn] = true + } + } + + certs, err := s.acmStore.ListAllCertMetadata(ctx) + if err != nil { + return fmt.Errorf("list certificates: %w", err) + } + + // Walking every certificate rather than only the referenced ones is what + // makes this a rebuild and not just a backfill: it also drops entries naming + // a load balancer that no longer exists. + added, removed := 0, 0 + for _, rec := range certs { + want := desired[rec.CertificateArn] + have := make(map[string]bool, len(rec.InUseBy)) + for _, arn := range rec.InUseBy { + have[arn] = true + } + + for arn := range have { + if want[arn] { + continue + } + if err := s.acmStore.RemoveInUseBy(ctx, rec.CertificateArn, arn); err != nil { + slog.ErrorContext(ctx, "ReconcileCertInUseIndex: failed to remove InUseBy entry", + "certArn", rec.CertificateArn, "lbArn", arn, "err", err) + continue + } + removed++ + } + for arn := range want { + if have[arn] { + continue + } + if err := s.acmStore.AddInUseBy(ctx, rec.CertificateArn, arn); err != nil { + slog.ErrorContext(ctx, "ReconcileCertInUseIndex: failed to add InUseBy entry", + "certArn", rec.CertificateArn, "lbArn", arn, "err", err) + continue + } + added++ + } + } + + if added > 0 || removed > 0 { + slog.InfoContext(ctx, "ReconcileCertInUseIndex: rebuilt ACM InUseBy index", + "certificates", len(certs), "loadBalancers", len(lbs), "added", added, "removed", removed) + } + return nil +} + // LBAgentHeartbeat processes a heartbeat from an LB agent. On first heartbeat // transitions LB provisioning→active, processes health, and returns the config hash. func (s *ELBv2ServiceImpl) LBAgentHeartbeat(ctx context.Context, input *LBAgentHeartbeatInput, accountID string) (*LBAgentHeartbeatOutput, error) { @@ -2495,10 +2670,18 @@ func (s *ELBv2ServiceImpl) CreateListener(ctx context.Context, input *elbv2.Crea Tags: tags, } + // Snapshot cert usage before persisting so the InUseBy index can be + // reconciled against it once the new listener is in the store. + certsBefore, certsBeforeErr := s.certsUsedByLB(ctx, lb.LoadBalancerArn) + if certsBeforeErr != nil { + slog.ErrorContext(ctx, "CreateListener: failed to snapshot cert usage", "lbArn", lb.LoadBalancerArn, "err", certsBeforeErr) + } + if err := s.store.PutListener(ctx, record); err != nil { slog.ErrorContext(ctx, "CreateListener: failed to persist record", "listenerId", listenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } + s.syncCertInUseIndex(ctx, lb.LoadBalancerArn, certsBefore) // Open the listener port on the NLB's managed front-end SG so inbound // traffic from the configured client CIDRs is admitted by the OVN ACL @@ -2551,6 +2734,15 @@ func (s *ELBv2ServiceImpl) rollbackListener(ctx context.Context, record *Listene // DeleteListener and DeleteLoadBalancer so LB teardown never bypasses the rule // cascade and leaves orphan rules that pin a target group as ResourceInUse. func (s *ELBv2ServiceImpl) deleteListenerCascade(ctx context.Context, listener *ListenerRecord) error { + // Snapshot cert usage before the listener disappears so the InUseBy index + // can be reconciled once it's gone. Shared by DeleteListener, DeleteLoadBalancer's + // per-listener loop, and rollbackListener, so this is the single point where + // listener removal always keeps the index from drifting. + certsBefore, certsBeforeErr := s.certsUsedByLB(ctx, listener.LoadBalancerArn) + if certsBeforeErr != nil { + slog.ErrorContext(ctx, "deleteListenerCascade: failed to snapshot cert usage", "lbArn", listener.LoadBalancerArn, "err", certsBeforeErr) + } + rules, err := s.store.ListRulesByListener(ctx, listener.ListenerArn) if err != nil { return fmt.Errorf("list rules for listener %s: %w", listener.ListenerArn, err) @@ -2563,6 +2755,8 @@ func (s *ELBv2ServiceImpl) deleteListenerCascade(ctx context.Context, listener * if err := s.store.DeleteListener(ctx, listener.ListenerID); err != nil { return fmt.Errorf("delete listener %s: %w", listener.ListenerID, err) } + + s.syncCertInUseIndex(ctx, listener.LoadBalancerArn, certsBefore) return nil } @@ -2634,6 +2828,13 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi return nil, errors.New(awserrors.ErrorELBv2LoadBalancerNotFound) } + // Snapshot cert usage before any mutation below so the InUseBy index can be + // reconciled once the change is persisted. + certsBefore, certsBeforeErr := s.certsUsedByLB(ctx, lb.LoadBalancerArn) + if certsBeforeErr != nil { + slog.ErrorContext(ctx, "ModifyListener: failed to snapshot cert usage", "lbArn", lb.LoadBalancerArn, "err", certsBeforeErr) + } + updated := *listener if input.Protocol != nil && *input.Protocol != "" { @@ -2753,6 +2954,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(ctx context.Context, input *elbv2.Modi slog.ErrorContext(ctx, "ModifyListener: failed to persist record", "listenerId", updated.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } + s.syncCertInUseIndex(ctx, lb.LoadBalancerArn, certsBefore) if err := s.updateStoredConfig(ctx, lb); err != nil { slog.ErrorContext(ctx, "ModifyListener: failed to update config", "listenerArn", updated.ListenerArn, "err", err) diff --git a/spinifex/handlers/elbv2/service_impl_health_test.go b/spinifex/handlers/elbv2/service_impl_health_test.go index c7bf78f9a..4c337b888 100644 --- a/spinifex/handlers/elbv2/service_impl_health_test.go +++ b/spinifex/handlers/elbv2/service_impl_health_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elbv2" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -23,7 +24,9 @@ func setupTestServiceWithInstance(t *testing.T, instanceID, instanceIP string) ( vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), nil, nc) require.NoError(t, err) - elbv2Svc, err := NewELBv2ServiceImplWithNATS(nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + elbv2Svc, err := NewELBv2ServiceImplWithNATS(nil, nc, masterKey) require.NoError(t, err) elbv2Svc.VPCService = vpcSvc diff --git a/spinifex/handlers/elbv2/service_impl_listener_certs.go b/spinifex/handlers/elbv2/service_impl_listener_certs.go index ae27f37c3..4bb638954 100644 --- a/spinifex/handlers/elbv2/service_impl_listener_certs.go +++ b/spinifex/handlers/elbv2/service_impl_listener_certs.go @@ -66,10 +66,24 @@ func (s *ELBv2ServiceImpl) AddListenerCertificates(ctx context.Context, input *e return nil, err } + // Snapshot cert usage before persisting so the InUseBy index can be + // reconciled against it once the added certs are in the store. + certsBefore, certsBeforeErr := s.certsUsedByLB(ctx, listener.LoadBalancerArn) + if certsBeforeErr != nil { + slog.ErrorContext(ctx, "AddListenerCertificates: failed to snapshot cert usage", "lbArn", listener.LoadBalancerArn, "err", certsBeforeErr) + } + if err := s.store.PutListener(ctx, &updated); err != nil { slog.ErrorContext(ctx, "AddListenerCertificates: failed to persist record", "listenerId", updated.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } + s.syncCertInUseIndex(ctx, listener.LoadBalancerArn, certsBefore) + + // The SNI set lives in the rendered config, so persisting the listener is + // only half the change: without this the certificate never reaches HAProxy. + if err := s.reloadListenerLB(ctx, &updated); err != nil { + return nil, err + } return &elbv2.AddListenerCertificatesOutput{ Certificates: listenerCertsToSDK(certs), @@ -118,10 +132,24 @@ func (s *ELBv2ServiceImpl) RemoveListenerCertificates(ctx context.Context, input } updated.Certificates = kept + // Snapshot cert usage before persisting so the InUseBy index can be + // reconciled against it once the removed certs are gone from the store. + certsBefore, certsBeforeErr := s.certsUsedByLB(ctx, listener.LoadBalancerArn) + if certsBeforeErr != nil { + slog.ErrorContext(ctx, "RemoveListenerCertificates: failed to snapshot cert usage", "lbArn", listener.LoadBalancerArn, "err", certsBeforeErr) + } + if err := s.store.PutListener(ctx, &updated); err != nil { slog.ErrorContext(ctx, "RemoveListenerCertificates: failed to persist record", "listenerId", updated.ListenerID, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } + s.syncCertInUseIndex(ctx, listener.LoadBalancerArn, certsBefore) + + // Re-render so the detached certificate stops being served. Skipping this + // leaves HAProxy answering SNI for a certificate the listener no longer has. + if err := s.reloadListenerLB(ctx, &updated); err != nil { + return nil, err + } return &elbv2.RemoveListenerCertificatesOutput{}, nil } diff --git a/spinifex/handlers/elbv2/service_impl_listener_certs_test.go b/spinifex/handlers/elbv2/service_impl_listener_certs_test.go index 813d1ec41..3efc69632 100644 --- a/spinifex/handlers/elbv2/service_impl_listener_certs_test.go +++ b/spinifex/handlers/elbv2/service_impl_listener_certs_test.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/lbagent" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -316,3 +317,97 @@ func TestDescribeSSLPolicies(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), awserrors.ErrorELBv2SSLPolicyNotFound) } + +// attachedCertListener puts an active LB and an HTTPS listener on it, returning +// the LB's short ID and the listener ARN. An active LB is required: +// updateStoredConfig no-ops while InstanceID is empty, which would make a +// missing re-render indistinguishable from a suppressed one. +func attachedCertListener(t *testing.T, svc *ELBv2ServiceImpl, id, name string) (string, string) { + t.Helper() + lb := activeLB(t, svc, id, name) + out, err := svc.CreateListener(context.Background(), httpsListenerInput(lb.LoadBalancerArn, testCertArn, 443), testAccountID) + require.NoError(t, err) + return lb.LoadBalancerID, *out.Listeners[0].ListenerArn +} + +func TestAddListenerCertificates_RerendersConfig(t *testing.T) { + svc := setupTestService(t) + lbID, listenerArn := attachedCertListener(t, svc, "lb-rerender1", "rerender-lb1") + + before, err := svc.store.GetLoadBalancer(context.Background(), lbID) + require.NoError(t, err) + require.NotEmpty(t, before.ConfigHash) + require.Len(t, before.CertFiles, 1, "only the default cert is staged before the SNI cert is added") + + _, err = svc.AddListenerCertificates(context.Background(), &elbv2.AddListenerCertificatesInput{ + ListenerArn: aws.String(listenerArn), + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + after, err := svc.store.GetLoadBalancer(context.Background(), lbID) + require.NoError(t, err) + assert.NotEqual(t, before.ConfigHash, after.ConfigHash, "attaching an SNI cert must re-render the config") + assert.Len(t, after.CertFiles, 2, "the added cert must be staged for delivery to the agent") + + listener, err := svc.store.GetListenerByArn(context.Background(), listenerArn) + require.NoError(t, err) + sniPath := sniCertPath(lbagent.CertDir, after, listener, testCertArn2) + assert.Contains(t, after.CertFiles, sniPath, "the added cert needs its own PEM path") + assert.Contains(t, after.ConfigText, "crt "+sniPath, "HAProxy must be told to load the added cert") +} + +func TestRemoveListenerCertificates_RerendersConfig(t *testing.T) { + svc := setupTestService(t) + lbID, listenerArn := attachedCertListener(t, svc, "lb-rerender2", "rerender-lb2") + + _, err := svc.AddListenerCertificates(context.Background(), &elbv2.AddListenerCertificatesInput{ + ListenerArn: aws.String(listenerArn), + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + added, err := svc.store.GetLoadBalancer(context.Background(), lbID) + require.NoError(t, err) + listener, err := svc.store.GetListenerByArn(context.Background(), listenerArn) + require.NoError(t, err) + sniPath := sniCertPath(lbagent.CertDir, added, listener, testCertArn2) + require.Contains(t, added.CertFiles, sniPath) + + _, err = svc.RemoveListenerCertificates(context.Background(), &elbv2.RemoveListenerCertificatesInput{ + ListenerArn: aws.String(listenerArn), + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + after, err := svc.store.GetLoadBalancer(context.Background(), lbID) + require.NoError(t, err) + assert.NotEqual(t, added.ConfigHash, after.ConfigHash, "detaching an SNI cert must re-render the config") + assert.NotContains(t, after.CertFiles, sniPath, "the detached cert must stop being staged") + assert.NotContains(t, after.ConfigText, "crt "+sniPath, "HAProxy must stop being told to load it") + assert.Len(t, after.CertFiles, 1, "the default cert stays") +} + +// The default certificate keeps frontendCertPath's name so an upgrade does not +// move an existing deployment's PEM, and it must render first so a client that +// sends no SNI still gets it. +func TestListenerCerts_DefaultCertRendersFirstAndKeepsItsPath(t *testing.T) { + svc := setupTestService(t) + lbID, listenerArn := attachedCertListener(t, svc, "lb-rerender3", "rerender-lb3") + + _, err := svc.AddListenerCertificates(context.Background(), &elbv2.AddListenerCertificatesInput{ + ListenerArn: aws.String(listenerArn), + Certificates: []*elbv2.Certificate{{CertificateArn: aws.String(testCertArn2)}}, + }, testAccountID) + require.NoError(t, err) + + lb, err := svc.store.GetLoadBalancer(context.Background(), lbID) + require.NoError(t, err) + listener, err := svc.store.GetListenerByArn(context.Background(), listenerArn) + require.NoError(t, err) + + defaultPath := frontendCertPath(lbagent.CertDir, lb, listener) + assert.Contains(t, lb.CertFiles, defaultPath) + assert.Contains(t, lb.ConfigText, "ssl crt "+defaultPath, + "the default cert must be the first crt on the bind line") +} diff --git a/spinifex/handlers/elbv2/service_impl_test.go b/spinifex/handlers/elbv2/service_impl_test.go index 18d3d2b07..2fb9ea053 100644 --- a/spinifex/handlers/elbv2/service_impl_test.go +++ b/spinifex/handlers/elbv2/service_impl_test.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/stretchr/testify/assert" @@ -22,7 +23,9 @@ func setupTestService(t *testing.T) *ELBv2ServiceImpl { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - svc, err := NewELBv2ServiceImplWithNATS(nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + svc, err := NewELBv2ServiceImplWithNATS(nil, nc, masterKey) require.NoError(t, err) // Seed the certificate ARNs the listener tests attach so the fail-closed // cert validation in Create/Modify/AddListenerCertificates resolves them. diff --git a/spinifex/handlers/elbv2/service_impl_vpc_test.go b/spinifex/handlers/elbv2/service_impl_vpc_test.go index 451710c65..b7d19ff35 100644 --- a/spinifex/handlers/elbv2/service_impl_vpc_test.go +++ b/spinifex/handlers/elbv2/service_impl_vpc_test.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/service/elbv2" "github.com/mulgadc/spinifex/spinifex/config" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" @@ -31,7 +32,9 @@ func setupTestServiceWithVPC(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc // Create ELBv2 service with VPC wired in. // Use DevNetworking=true so single-subnet tests aren't blocked by multi-AZ validation. cfg := &config.Config{Daemon: config.DaemonConfig{DevNetworking: true}} - elbv2Svc, err := NewELBv2ServiceImplWithNATS(cfg, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + elbv2Svc, err := NewELBv2ServiceImplWithNATS(cfg, nc, masterKey) require.NoError(t, err) elbv2Svc.VPCService = vpcSvc diff --git a/spinifex/handlers/elbv2/service_nats_e2e_test.go b/spinifex/handlers/elbv2/service_nats_e2e_test.go index 7b18a9105..65e95f737 100644 --- a/spinifex/handlers/elbv2/service_nats_e2e_test.go +++ b/spinifex/handlers/elbv2/service_nats_e2e_test.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elbv2" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" @@ -21,7 +22,9 @@ func setupNATSELBv2Test(t *testing.T) (ELBv2Service, *ELBv2ServiceImpl) { _, nc, _ := testutil.StartTestJetStream(t) - backend, err := NewELBv2ServiceImplWithNATS(nil, nc) + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + backend, err := NewELBv2ServiceImplWithNATS(nil, nc, masterKey) require.NoError(t, err) // Wire backend as NATS subscriber (simulates daemon subscriptions) diff --git a/spinifex/lbagent/agent.go b/spinifex/lbagent/agent.go index 06ff07d31..c354119ab 100644 --- a/spinifex/lbagent/agent.go +++ b/spinifex/lbagent/agent.go @@ -328,6 +328,16 @@ func (a *Agent) fetchAndApplyConfig() error { return fmt.Errorf("reload %s: %w", engine, err) } + // Only after a successful reload: until then the engine is still serving the + // previous config, which references the files being removed. A delivered PEM + // carries a private key, so undelivered ones left in place strand key + // material on disk for every certificate ever detached from a listener. + if err := a.pruneCertFiles(certDir, resp.CertFiles); err != nil { + // Not fatal — the config is live and correct, and a failed prune leaves a + // stale file rather than breaking traffic. + slog.Error("Failed to prune stale TLS certs", "certDir", certDir, "err", err) + } + a.engine = engine a.healthTargets = resp.HealthTargets a.localConfigHash = resp.ConfigHash @@ -393,6 +403,37 @@ func (a *Agent) writeCertFiles(certDir string, certs []certFile) error { return nil } +// pruneCertFiles removes .pem files under certDir that are not in the delivered +// set. An LB VM hosts exactly one load balancer, so every PEM there belongs to +// this agent and anything undelivered is stale. Scoped to .pem so it cannot +// touch an engine's own files, and to certDir's immediate entries so it cannot +// follow a path out. +func (a *Agent) pruneCertFiles(certDir string, certs []certFile) error { + entries, err := os.ReadDir(certDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read cert dir: %w", err) + } + + keep := make(map[string]bool, len(certs)) + for _, c := range certs { + keep[filepath.Base(filepath.Clean(c.Path))] = true + } + + for _, e := range entries { + if e.IsDir() || filepath.Ext(e.Name()) != ".pem" || keep[e.Name()] { + continue + } + if err := os.Remove(filepath.Join(certDir, e.Name())); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale cert %q: %w", e.Name(), err) + } + slog.Info("Removed stale TLS cert", "path", filepath.Join(certDir, e.Name())) + } + return nil +} + // WriteConfig atomically writes an HAProxy config file. // It writes to a temp file first, then renames for atomicity. func WriteConfig(path, content string) error { diff --git a/spinifex/lbagent/agent_certs_test.go b/spinifex/lbagent/agent_certs_test.go index 8c343c16a..02afae66b 100644 --- a/spinifex/lbagent/agent_certs_test.go +++ b/spinifex/lbagent/agent_certs_test.go @@ -61,3 +61,45 @@ func TestWriteCertFiles_RejectsEmptyPEM(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "empty PEM") } + +func TestPruneCertFiles_RemovesUndeliveredPEM(t *testing.T) { + a := newCertTestAgent(t) + kept := filepath.Join(a.certDir, "lb-1-lst-1.pem") + stale := filepath.Join(a.certDir, "lb-1-lst-1-cert_bbbb.pem") + require.NoError(t, a.writeCertFiles(a.certDir, []certFile{ + {Path: kept, PEM: "LEAF\nKEY\n"}, + {Path: stale, PEM: "OTHER\nKEY\n"}, + })) + + // The next poll delivers only the default cert, as after a detach. + require.NoError(t, a.pruneCertFiles(a.certDir, []certFile{{Path: kept, PEM: "LEAF\nKEY\n"}})) + + assert.FileExists(t, kept) + _, err := os.Stat(stale) + assert.True(t, os.IsNotExist(err), "a detached certificate's private key must not be left on disk") +} + +// A listener switched from HTTPS to HTTP delivers no certs at all, which is the +// case writeCertFiles short-circuits — pruning still has to run. +func TestPruneCertFiles_EmptyDeliverySetClearsDir(t *testing.T) { + a := newCertTestAgent(t) + stale := filepath.Join(a.certDir, "lb-1-lst-1.pem") + require.NoError(t, a.writeCertFiles(a.certDir, []certFile{{Path: stale, PEM: "LEAF\nKEY\n"}})) + + require.NoError(t, a.pruneCertFiles(a.certDir, nil)) + + _, err := os.Stat(stale) + assert.True(t, os.IsNotExist(err)) +} + +func TestPruneCertFiles_LeavesNonPEMAndMissingDirAlone(t *testing.T) { + a := newCertTestAgent(t) + other := filepath.Join(a.certDir, "haproxy.cfg") + require.NoError(t, os.WriteFile(other, []byte("x"), 0o600)) + + require.NoError(t, a.pruneCertFiles(a.certDir, nil)) + assert.FileExists(t, other, "pruning must not touch an engine's own files") + + a.certDir = filepath.Join(t.TempDir(), "does-not-exist") + assert.NoError(t, a.pruneCertFiles(a.certDir, nil), "a dir that was never created is not an error") +}