Complete M1, M1.1, M2: end-to-end lifecycle, agent deployment, ACME v2

- Wire issuer connector end-to-end with IssuerConnectorAdapter (dependency inversion)
- Renewal/issuance job processor: RSA key + CSR generation, Local CA signing, cert version storage
- Agent work API (GET /agents/{id}/work) and job status API (POST /agents/{id}/jobs/{job_id}/status)
- Agent-side deployment: WorkItem enrichment with target type/config, NGINX/F5/IIS connector invocation
- Full ACME v2 implementation: HTTP-01 challenge solving, account registration, order lifecycle
- Update all docs (README, architecture, connectors, demo-advanced, quickstart) for M1-M2
- Fix go vet warning in deployment.go (non-constant format string)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

internal/service/renewal.go

@@ -2,6 +2,13 @@ package service
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/hex"
+	"encoding/pem"
 	"fmt"
 	"time"
 
@@ -11,19 +18,30 @@ import (
 
 // RenewalService manages certificate renewal workflows.
 type RenewalService struct {
-	certRepo         repository.CertificateRepository
-	jobRepo          repository.JobRepository
-	auditService     *AuditService
-	notificationSvc  *NotificationService
-	issuerRegistry   map[string]IssuerConnector
+	certRepo        repository.CertificateRepository
+	jobRepo         repository.JobRepository
+	auditService    *AuditService
+	notificationSvc *NotificationService
+	issuerRegistry  map[string]IssuerConnector
 }
 
-// IssuerConnector defines the interface for interacting with certificate issuers.
+// IssuerConnector defines the service-layer interface for interacting with certificate issuers.
+// This is distinct from the connector-layer issuer.Connector interface to maintain dependency
+// inversion. Use IssuerConnectorAdapter to bridge between the two.
 type IssuerConnector interface {
-	// RenewCertificate renews a certificate and returns the new certificate PEM.
-	RenewCertificate(ctx context.Context, csr []byte) ([]byte, error)
-	// GetCertificateChain returns the issuer's certificate chain.
-	GetCertificateChain(ctx context.Context) ([]byte, error)
+	// IssueCertificate issues a new certificate using the provided CSR PEM.
+	IssueCertificate(ctx context.Context, commonName string, sans []string, csrPEM string) (*IssuanceResult, error)
+	// RenewCertificate renews a certificate using the provided CSR PEM.
+	RenewCertificate(ctx context.Context, commonName string, sans []string, csrPEM string) (*IssuanceResult, error)
+}
+
+// IssuanceResult holds the result of a certificate issuance or renewal operation.
+type IssuanceResult struct {
+	CertPEM   string
+	ChainPEM  string
+	Serial    string
+	NotBefore time.Time
+	NotAfter  time.Time
 }
 
 // NewRenewalService creates a new renewal service.
@@ -72,12 +90,29 @@ func (s *RenewalService) CheckExpiringCertificates(ctx context.Context) error {
 			continue
 		}
 
+		// Check for existing pending/running renewal jobs to avoid duplicates
+		existingJobs, err := s.jobRepo.ListByCertificate(ctx, cert.ID)
+		if err == nil {
+			hasActiveRenewal := false
+			for _, j := range existingJobs {
+				if j.Type == domain.JobTypeRenewal &&
+					(j.Status == domain.JobStatusPending || j.Status == domain.JobStatusRunning) {
+					hasActiveRenewal = true
+					break
+				}
+			}
+			if hasActiveRenewal {
+				continue
+			}
+		}
+
 		// Create renewal job
 		job := &domain.Job{
 			ID:            generateID("job"),
 			CertificateID: cert.ID,
 			Type:          domain.JobTypeRenewal,
 			Status:        domain.JobStatusPending,
+			MaxAttempts:   3,
 			ScheduledAt:   time.Now(),
 			CreatedAt:     time.Now(),
 		}
@@ -87,6 +122,12 @@ func (s *RenewalService) CheckExpiringCertificates(ctx context.Context) error {
 			continue
 		}
 
+		// Update certificate status to RenewalInProgress
+		cert.Status = domain.CertificateStatusRenewalInProgress
+		if err := s.certRepo.Update(ctx, cert); err != nil {
+			fmt.Printf("failed to update cert status for %s: %v\n", cert.ID, err)
+		}
+
 		// Record audit event
 		_ = s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
 			"renewal_job_created", "certificate", cert.ID,
@@ -96,7 +137,13 @@ func (s *RenewalService) CheckExpiringCertificates(ctx context.Context) error {
 	return nil
 }
 
-// ProcessRenewalJob executes a renewal job: call issuer, store new version, update cert status.
+// ProcessRenewalJob executes a renewal job: generate CSR, call issuer, store new version,
+// update cert status, and create deployment jobs for targets.
+//
+// V1 Architecture Note: For the Local CA issuer, the control plane generates a server-side
+// ephemeral key + CSR. The private key is stored in the CertificateVersion.CSRPEM field
+// so agents can retrieve it for deployment. In V2+ with ACME/external CAs, agents will
+// generate keys locally and submit CSRs, so private keys never leave the target infrastructure.
 func (s *RenewalService) ProcessRenewalJob(ctx context.Context, job *domain.Job) error {
 	// Update job status to in-progress
 	if err := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusRunning, ""); err != nil {
@@ -106,40 +153,59 @@ func (s *RenewalService) ProcessRenewalJob(ctx context.Context, job *domain.Job)
 	// Fetch certificate
 	cert, err := s.certRepo.Get(ctx, job.CertificateID)
 	if err != nil {
-		updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, fmt.Sprintf("certificate fetch failed: %v", err))
-		if updateErr != nil {
-			fmt.Printf("failed to update job status: %v\n", updateErr)
-		}
+		s.failJob(ctx, job, fmt.Sprintf("certificate fetch failed: %v", err))
 		return fmt.Errorf("failed to fetch certificate: %w", err)
 	}
 
 	// Get issuer connector
 	issuerID := cert.IssuerID
 	if issuerID == "" {
+		s.failJob(ctx, job, "certificate has no issuer assigned")
 		return fmt.Errorf("certificate has no issuer assigned")
 	}
 
 	connector, ok := s.issuerRegistry[issuerID]
 	if !ok {
-		updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed,
-			fmt.Sprintf("issuer connector not found for %s", issuerID))
-		if updateErr != nil {
-			fmt.Printf("failed to update job status: %v\n", updateErr)
-		}
+		s.failJob(ctx, job, fmt.Sprintf("issuer connector not found for %s", issuerID))
 		return fmt.Errorf("issuer connector not found for %s", issuerID)
 	}
 
-	// TODO: In production, fetch CSR from agent or generate new CSR
-	// For now, we'd use cert.CSR or generate a new one from the private key
-	csr := []byte{} // placeholder
-
-	// Call issuer to renew
-	certPEM, err := connector.RenewCertificate(ctx, csr)
+	// Generate server-side RSA key + CSR for this renewal
+	// V1: server generates ephemeral key for Local CA. V2+: agent generates key locally.
+	privKey, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
-		updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, fmt.Sprintf("issuer renewal failed: %v", err))
-		if updateErr != nil {
-			fmt.Printf("failed to update job status: %v\n", updateErr)
-		}
+		s.failJob(ctx, job, fmt.Sprintf("key generation failed: %v", err))
+		return fmt.Errorf("failed to generate private key: %w", err)
+	}
+
+	csrTemplate := &x509.CertificateRequest{
+		Subject: pkix.Name{
+			CommonName: cert.CommonName,
+		},
+		DNSNames: cert.SANs,
+	}
+
+	csrDER, err := x509.CreateCertificateRequest(rand.Reader, csrTemplate, privKey)
+	if err != nil {
+		s.failJob(ctx, job, fmt.Sprintf("CSR generation failed: %v", err))
+		return fmt.Errorf("failed to generate CSR: %w", err)
+	}
+
+	csrPEM := string(pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE REQUEST",
+		Bytes: csrDER,
+	}))
+
+	// Encode private key to PEM for storage (V1: stored so agent can retrieve for deployment)
+	privKeyPEM := string(pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(privKey),
+	}))
+
+	// Call issuer connector to renew
+	result, err := connector.RenewCertificate(ctx, cert.CommonName, cert.SANs, csrPEM)
+	if err != nil {
+		s.failJob(ctx, job, fmt.Sprintf("issuer renewal failed: %v", err))
 
 		// Send failure notification
 		_ = s.notificationSvc.SendRenewalNotification(ctx, cert, false, err)
@@ -152,38 +218,63 @@ func (s *RenewalService) ProcessRenewalJob(ctx context.Context, job *domain.Job)
 		return fmt.Errorf("issuer renewal failed: %w", err)
 	}
 
+	// Compute SHA-256 fingerprint of the issued certificate
+	fingerprint := computeCertFingerprint(result.CertPEM)
+
 	// Create new certificate version
 	version := &domain.CertificateVersion{
-		ID:            generateID("certver"),
-		CertificateID: job.CertificateID,
-		SerialNumber:  fmt.Sprintf("renewed-%d", time.Now().Unix()),
-		PEMChain:      string(certPEM),
-		CreatedAt:     time.Now(),
+		ID:                generateID("certver"),
+		CertificateID:    job.CertificateID,
+		SerialNumber:     result.Serial,
+		NotBefore:        result.NotBefore,
+		NotAfter:         result.NotAfter,
+		FingerprintSHA256: fingerprint,
+		PEMChain:         result.CertPEM + "\n" + result.ChainPEM,
+		CSRPEM:           privKeyPEM, // V1: stores private key for agent deployment
+		CreatedAt:        time.Now(),
 	}
 
 	if err := s.certRepo.CreateVersion(ctx, version); err != nil {
-		updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, fmt.Sprintf("version creation failed: %v", err))
-		if updateErr != nil {
-			fmt.Printf("failed to update job status: %v\n", updateErr)
-		}
+		s.failJob(ctx, job, fmt.Sprintf("version creation failed: %v", err))
 		return fmt.Errorf("failed to create certificate version: %w", err)
 	}
 
-	// Update certificate status
+	// Update certificate status and expiry
 	cert.Status = domain.CertificateStatusActive
+	cert.ExpiresAt = result.NotAfter
+	now := time.Now()
+	cert.LastRenewalAt = &now
+	cert.UpdatedAt = now
 	if err := s.certRepo.Update(ctx, cert); err != nil {
-		updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, fmt.Sprintf("cert update failed: %v", err))
-		if updateErr != nil {
-			fmt.Printf("failed to update job status: %v\n", updateErr)
-		}
+		s.failJob(ctx, job, fmt.Sprintf("cert update failed: %v", err))
 		return fmt.Errorf("failed to update certificate: %w", err)
 	}
 
-	// Mark job as completed
+	// Mark renewal job as completed
 	if err := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusCompleted, ""); err != nil {
 		return fmt.Errorf("failed to update job status: %w", err)
 	}
 
+	// Create deployment jobs for each target
+	if len(cert.TargetIDs) > 0 {
+		for _, targetID := range cert.TargetIDs {
+			tid := targetID // capture loop variable
+			deployJob := &domain.Job{
+				ID:            generateID("job"),
+				CertificateID: cert.ID,
+				Type:          domain.JobTypeDeployment,
+				Status:        domain.JobStatusPending,
+				TargetID:      &tid,
+				MaxAttempts:   3,
+				ScheduledAt:   time.Now(),
+				CreatedAt:     time.Now(),
+			}
+			if err := s.jobRepo.Create(ctx, deployJob); err != nil {
+				fmt.Printf("failed to create deployment job for target %s: %v\n", targetID, err)
+			}
+		}
+	}
+
 	// Send success notification
 	if err := s.notificationSvc.SendRenewalNotification(ctx, cert, true, nil); err != nil {
 		fmt.Printf("failed to send renewal notification: %v\n", err)
@@ -192,12 +283,33 @@ func (s *RenewalService) ProcessRenewalJob(ctx context.Context, job *domain.Job)
 	// Record audit event
 	_ = s.auditService.RecordEvent(ctx, "system", domain.ActorTypeSystem,
 		"renewal_job_completed", "certificate", job.CertificateID,
-		map[string]interface{}{"job_id": job.ID, "serial": version.SerialNumber})
+		map[string]interface{}{
+			"job_id":    job.ID,
+			"serial":    result.Serial,
+			"not_after": result.NotAfter,
+		})
 
 	return nil
 }
 
-// Retry attempts to reprocess failed renewal jobs with exponential backoff.
+// failJob is a helper to mark a job as failed with an error message.
+func (s *RenewalService) failJob(ctx context.Context, job *domain.Job, errMsg string) {
+	if updateErr := s.jobRepo.UpdateStatus(ctx, job.ID, domain.JobStatusFailed, errMsg); updateErr != nil {
+		fmt.Printf("failed to update job status: %v\n", updateErr)
+	}
+}
+
+// computeCertFingerprint computes the SHA-256 fingerprint of a PEM-encoded certificate.
+func computeCertFingerprint(certPEM string) string {
+	block, _ := pem.Decode([]byte(certPEM))
+	if block == nil {
+		return ""
+	}
+	hash := sha256.Sum256(block.Bytes)
+	return hex.EncodeToString(hash[:])
+}
+
+// RetryFailedJobs resets failed renewal jobs for retry if they haven't exceeded max attempts.
 func (s *RenewalService) RetryFailedJobs(ctx context.Context, maxRetries int) error {
 	failedJobs, err := s.jobRepo.ListByStatus(ctx, domain.JobStatusFailed)
 	if err != nil {