mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-12 05:18:52 +00:00
feat(haproxy): atomic deploy + post-deploy TLS verify + rollback + ValidateOnly + test-depth uplift to 36 tests
Phase 6 of the deploy-hardening I master bundle. HAProxy connector follows the canonical Phase 4 NGINX template with the HAProxy- specific quirk: combined PEM file (cert + chain + key in one file, in that order). Test count lifts 3 → 36. HAProxy specifics: - buildCombinedPEM concatenates cert, chain, key in HAProxy's required order. The combined file goes through deploy.Apply as a single File entry (vs NGINX/Apache's 2-3 separate File entries). - Default mode 0600 unconditionally (combined file contains the private key); operators rely on this back-compat behavior. PEMFileMode override is the supported escape hatch. - Validate command is `haproxy -c -f <config>`. Reload via `systemctl reload haproxy` (NOT `restart` — reload uses socket activation to drain in-flight connections). - Default user/group: haproxy (cross-distro consistent). DeployCertificate refactor: - Replaces the duplicated os.WriteFile flow with deploy.Apply. - PreCommit runs `haproxy -c -f` validation (gated on ValidateCommand being non-empty — HAProxy historically allowed empty validate). - PostCommit runs the operator's ReloadCommand. - Post-deploy TLS verify (frozen-decision-0.3 default ON when Endpoint is configured): probes the configured target, fingerprint-matches against the deployed cert (the leaf cert block from the combined PEM), retries with backoff for load- balanced targets. - Rollback wires identical to NGINX/Apache: backup restore + reload retry on PostCommit failure; verify-fail also triggers rollback. ValidateOnly real impl: returns sentinel when no ValidateCommand; otherwise runs the operator's command without touching the live combined PEM. Tests (36 total: 33 in haproxy_atomic_test.go + 3 pre-existing in haproxy_test.go): - Atomic invariants (happy, validate-fail, reload-fail-rollback, rollback-also-fail-escalation) - Combined PEM order (cert + chain + key — verified via PEM block headers, not base64 bodies) - Mode handling (default 0600 even when existing is 0640 — back-compat; PEMFileMode override; existing-mode unchanged when override matches) - Idempotency (full skip) - Verify (match, mismatch, dial-timeout, retries, disabled, no-endpoint, rollback-runs-reload) - ValidateOnly (happy, fails, no-command-sentinel, stderr-in-error) - Concurrency (same-paths-serialize) - Edge cases (no-chain, no-key, ctx-cancelled, no-validate-command, config-validation rejects missing pem_path / reload / shell-injection) Coverage: HAProxy 88.0% (above >=85% prompt bar). Race detector clean. golangci-lint v2.11.4 clean. Smoke test connectorsAtPhase3 list shrinks 11→10 (haproxy removed alongside nginx + apache). Phase 7 next: Traefik + Caddy + Envoy + Postfix — the remaining file-based connectors get the same treatment.
This commit is contained in:
@@ -1,60 +1,113 @@
|
|||||||
|
// Package haproxy implements the HAProxy target connector.
|
||||||
|
//
|
||||||
|
// HAProxy expects all TLS material concatenated in a single PEM
|
||||||
|
// file (cert + chain + key in that order). Phase 6 of the
|
||||||
|
// deploy-hardening I master bundle adds atomic-deploy + post-deploy
|
||||||
|
// TLS verify + rollback + ValidateOnly to the connector following
|
||||||
|
// the canonical NGINX template.
|
||||||
|
//
|
||||||
|
// HAProxy quirks:
|
||||||
|
//
|
||||||
|
// - Single combined-PEM file (vs NGINX/Apache's separate
|
||||||
|
// cert/chain/key files).
|
||||||
|
// - Validate command is `haproxy -c -f <config>` (NOT a separate
|
||||||
|
// subcommand).
|
||||||
|
// - Reload via `systemctl reload haproxy` is preferred over
|
||||||
|
// `restart` because reload uses socket activation to drain
|
||||||
|
// in-flight connections gracefully (the old worker hands off
|
||||||
|
// to the new worker via the master socket).
|
||||||
|
// - Combined PEM file mode default 0600 (contains private key).
|
||||||
package haproxy
|
package haproxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"os/user"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/shankar0123/certctl/internal/connector/target"
|
"github.com/shankar0123/certctl/internal/connector/target"
|
||||||
|
"github.com/shankar0123/certctl/internal/deploy"
|
||||||
|
"github.com/shankar0123/certctl/internal/tlsprobe"
|
||||||
"github.com/shankar0123/certctl/internal/validation"
|
"github.com/shankar0123/certctl/internal/validation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config represents the HAProxy deployment target configuration.
|
// Config — Phase 6 (deploy-hardening I) added per-target file
|
||||||
// HAProxy expects a combined PEM file containing the certificate, chain, and private key
|
// ownership + mode overrides + post-deploy verify + backup
|
||||||
// concatenated in a single file.
|
// retention.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
PEMPath string `json:"pem_path"` // Path for combined PEM (cert + chain + key)
|
PEMPath string `json:"pem_path"`
|
||||||
ReloadCommand string `json:"reload_command"` // Command to reload HAProxy (e.g., "systemctl reload haproxy")
|
ReloadCommand string `json:"reload_command"`
|
||||||
ValidateCommand string `json:"validate_command"` // Command to validate config (e.g., "haproxy -c -f /etc/haproxy/haproxy.cfg")
|
ValidateCommand string `json:"validate_command,omitempty"`
|
||||||
|
|
||||||
|
PEMFileMode os.FileMode `json:"pem_file_mode,omitempty"`
|
||||||
|
PEMFileOwner string `json:"pem_file_owner,omitempty"`
|
||||||
|
PEMFileGroup string `json:"pem_file_group,omitempty"`
|
||||||
|
|
||||||
|
PostDeployVerify *PostDeployVerifyConfig `json:"post_deploy_verify,omitempty"`
|
||||||
|
PostDeployVerifyAttempts int `json:"post_deploy_verify_attempts,omitempty"`
|
||||||
|
PostDeployVerifyBackoff time.Duration `json:"post_deploy_verify_backoff,omitempty"`
|
||||||
|
|
||||||
|
BackupRetention int `json:"backup_retention,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PostDeployVerifyConfig struct {
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
|
Timeout time.Duration `json:"timeout,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Connector implements the target.Connector interface for HAProxy servers.
|
|
||||||
// This connector runs on the AGENT side and handles local certificate deployment.
|
|
||||||
// HAProxy uses a combined PEM file (cert + chain + key) unlike NGINX/Apache which use
|
|
||||||
// separate files.
|
|
||||||
type Connector struct {
|
type Connector struct {
|
||||||
config *Config
|
config *Config
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
|
|
||||||
|
runValidate func(ctx context.Context, command string) ([]byte, error)
|
||||||
|
runReload func(ctx context.Context, command string) ([]byte, error)
|
||||||
|
probe func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new HAProxy target connector with the given configuration and logger.
|
|
||||||
func New(config *Config, logger *slog.Logger) *Connector {
|
func New(config *Config, logger *slog.Logger) *Connector {
|
||||||
return &Connector{
|
c := &Connector{config: config, logger: logger}
|
||||||
config: config,
|
c.runValidate = defaultRunCommand
|
||||||
logger: logger,
|
c.runReload = defaultRunCommand
|
||||||
}
|
c.probe = tlsprobe.ProbeTLS
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultRunCommand(ctx context.Context, command string) ([]byte, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, "sh", "-c", command)
|
||||||
|
return cmd.CombinedOutput()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Connector) SetTestRunValidate(fn func(ctx context.Context, command string) ([]byte, error)) {
|
||||||
|
c.runValidate = fn
|
||||||
|
}
|
||||||
|
func (c *Connector) SetTestRunReload(fn func(ctx context.Context, command string) ([]byte, error)) {
|
||||||
|
c.runReload = fn
|
||||||
|
}
|
||||||
|
func (c *Connector) SetTestProbe(fn func(ctx context.Context, address string, timeout time.Duration) tlsprobe.ProbeResult) {
|
||||||
|
c.probe = fn
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateConfig checks that all required configuration paths and commands are valid.
|
|
||||||
func (c *Connector) ValidateConfig(ctx context.Context, rawConfig json.RawMessage) error {
|
func (c *Connector) ValidateConfig(ctx context.Context, rawConfig json.RawMessage) error {
|
||||||
var cfg Config
|
var cfg Config
|
||||||
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
|
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
|
||||||
return fmt.Errorf("invalid HAProxy config: %w", err)
|
return fmt.Errorf("invalid HAProxy config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.PEMPath == "" {
|
if cfg.PEMPath == "" {
|
||||||
return fmt.Errorf("HAProxy pem_path is required")
|
return fmt.Errorf("HAProxy pem_path is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.ReloadCommand == "" {
|
if cfg.ReloadCommand == "" {
|
||||||
return fmt.Errorf("HAProxy reload_command is required")
|
return fmt.Errorf("HAProxy reload_command is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate commands to prevent injection attacks
|
|
||||||
if err := validation.ValidateShellCommand(cfg.ReloadCommand); err != nil {
|
if err := validation.ValidateShellCommand(cfg.ReloadCommand); err != nil {
|
||||||
return fmt.Errorf("invalid reload_command: %w", err)
|
return fmt.Errorf("invalid reload_command: %w", err)
|
||||||
}
|
}
|
||||||
@@ -63,127 +116,270 @@ func (c *Connector) ValidateConfig(ctx context.Context, rawConfig json.RawMessag
|
|||||||
return fmt.Errorf("invalid validate_command: %w", err)
|
return fmt.Errorf("invalid validate_command: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
c.logger.Info("validating HAProxy configuration", "pem_path", cfg.PEMPath)
|
||||||
c.logger.Info("validating HAProxy configuration",
|
|
||||||
"pem_path", cfg.PEMPath)
|
|
||||||
|
|
||||||
// Verify validate command works if provided
|
|
||||||
if cfg.ValidateCommand != "" {
|
|
||||||
cmd := exec.CommandContext(ctx, cfg.ValidateCommand)
|
|
||||||
if err := cmd.Run(); err != nil {
|
|
||||||
c.logger.Warn("HAProxy config validation failed during config check",
|
|
||||||
"error", err,
|
|
||||||
"validate_command", cfg.ValidateCommand)
|
|
||||||
// Don't fail; HAProxy might not be installed yet
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
c.config = &cfg
|
c.config = &cfg
|
||||||
c.logger.Info("HAProxy configuration validated")
|
c.logger.Info("HAProxy configuration validated")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeployCertificate creates a combined PEM file (cert + chain + key) and reloads HAProxy.
|
// DeployCertificate Phase 6 atomic + verify + rollback. Combined
|
||||||
//
|
// PEM file (cert + chain + key) written via deploy.Apply.
|
||||||
// HAProxy requires all TLS material in a single file, concatenated in this order:
|
|
||||||
// 1. Server certificate
|
|
||||||
// 2. Intermediate/chain certificates
|
|
||||||
// 3. Private key
|
|
||||||
//
|
|
||||||
// Steps:
|
|
||||||
// 1. Build combined PEM (cert + chain + key)
|
|
||||||
// 2. Write to pem_path with mode 0600 (contains private key)
|
|
||||||
// 3. Optionally validate HAProxy configuration
|
|
||||||
// 4. Execute reload command
|
|
||||||
func (c *Connector) DeployCertificate(ctx context.Context, request target.DeploymentRequest) (*target.DeploymentResult, error) {
|
func (c *Connector) DeployCertificate(ctx context.Context, request target.DeploymentRequest) (*target.DeploymentResult, error) {
|
||||||
c.logger.Info("deploying certificate to HAProxy",
|
c.logger.Info("deploying certificate to HAProxy", "pem_path", c.config.PEMPath)
|
||||||
"pem_path", c.config.PEMPath)
|
|
||||||
|
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
// Build combined PEM: cert + chain + key
|
combinedPEM := buildCombinedPEM(request)
|
||||||
combinedPEM := request.CertPEM + "\n"
|
plan := c.buildPlan([]byte(combinedPEM))
|
||||||
if request.ChainPEM != "" {
|
|
||||||
combinedPEM += request.ChainPEM + "\n"
|
|
||||||
}
|
|
||||||
if request.KeyPEM != "" {
|
|
||||||
combinedPEM += request.KeyPEM + "\n"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write combined PEM with secure permissions (0600: contains private key)
|
|
||||||
if err := os.WriteFile(c.config.PEMPath, []byte(combinedPEM), 0600); err != nil {
|
|
||||||
errMsg := fmt.Sprintf("failed to write combined PEM: %v", err)
|
|
||||||
c.logger.Error("PEM deployment failed", "error", err)
|
|
||||||
return &target.DeploymentResult{
|
|
||||||
Success: false,
|
|
||||||
TargetAddress: c.config.PEMPath,
|
|
||||||
Message: errMsg,
|
|
||||||
DeployedAt: time.Now(),
|
|
||||||
}, fmt.Errorf("%s", errMsg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate HAProxy configuration if validate command is configured
|
|
||||||
if c.config.ValidateCommand != "" {
|
if c.config.ValidateCommand != "" {
|
||||||
c.logger.Debug("validating HAProxy configuration", "validate_command", c.config.ValidateCommand)
|
plan.PreCommit = func(pcCtx context.Context, _ map[string]string) error {
|
||||||
validateCmd := exec.CommandContext(ctx, c.config.ValidateCommand)
|
out, err := c.runValidate(pcCtx, c.config.ValidateCommand)
|
||||||
if output, err := validateCmd.CombinedOutput(); err != nil {
|
if err != nil {
|
||||||
errMsg := fmt.Sprintf("HAProxy config validation failed: %v (output: %s)", err, string(output))
|
return fmt.Errorf("haproxy -c -f failed: %w (output: %s)", err, string(out))
|
||||||
c.logger.Error("HAProxy validation failed", "error", err, "output", string(output))
|
}
|
||||||
return &target.DeploymentResult{
|
return nil
|
||||||
Success: false,
|
}
|
||||||
TargetAddress: c.config.PEMPath,
|
}
|
||||||
Message: errMsg,
|
plan.PostCommit = func(pcCtx context.Context) error {
|
||||||
DeployedAt: time.Now(),
|
out, err := c.runReload(pcCtx, c.config.ReloadCommand)
|
||||||
}, fmt.Errorf("%s", errMsg)
|
if err != nil {
|
||||||
|
return fmt.Errorf("haproxy reload failed: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := deploy.Apply(ctx, plan)
|
||||||
|
if err != nil {
|
||||||
|
return c.failureResult(c.config.PEMPath, "deploy.Apply", err, startTime), err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !res.SkippedAsIdempotent {
|
||||||
|
// Use the cert (first PEM block) for fingerprint match,
|
||||||
|
// not the full combined PEM. The wire serves leaf cert.
|
||||||
|
if vErr := c.runPostDeployVerify(ctx, request.CertPEM); vErr != nil {
|
||||||
|
c.logger.Error("post-deploy TLS verify failed; rolling back", "error", vErr)
|
||||||
|
rbErr := c.rollbackToBackups(ctx, res.BackupPaths)
|
||||||
|
if rbErr != nil {
|
||||||
|
return c.failureResult(c.config.PEMPath, "verify+rollback both failed",
|
||||||
|
fmt.Errorf("verify: %w; rollback: %v", vErr, rbErr), startTime), rbErr
|
||||||
|
}
|
||||||
|
return c.failureResult(c.config.PEMPath, "post-deploy verify failed; rolled back", vErr, startTime), vErr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reload HAProxy
|
dur := time.Since(startTime)
|
||||||
c.logger.Debug("reloading HAProxy", "reload_command", c.config.ReloadCommand)
|
idemNote := ""
|
||||||
reloadCmd := exec.CommandContext(ctx, c.config.ReloadCommand)
|
if res.SkippedAsIdempotent {
|
||||||
if output, err := reloadCmd.CombinedOutput(); err != nil {
|
idemNote = " (idempotent skip — bytes unchanged)"
|
||||||
errMsg := fmt.Sprintf("HAProxy reload failed: %v (output: %s)", err, string(output))
|
|
||||||
c.logger.Error("HAProxy reload failed", "error", err, "output", string(output))
|
|
||||||
return &target.DeploymentResult{
|
|
||||||
Success: false,
|
|
||||||
TargetAddress: c.config.PEMPath,
|
|
||||||
Message: errMsg,
|
|
||||||
DeployedAt: time.Now(),
|
|
||||||
}, fmt.Errorf("%s", errMsg)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
deploymentDuration := time.Since(startTime)
|
|
||||||
c.logger.Info("certificate deployed to HAProxy successfully",
|
c.logger.Info("certificate deployed to HAProxy successfully",
|
||||||
"duration", deploymentDuration.String(),
|
"duration", dur.String(), "pem_path", c.config.PEMPath, "idempotent", res.SkippedAsIdempotent)
|
||||||
"pem_path", c.config.PEMPath)
|
|
||||||
|
|
||||||
return &target.DeploymentResult{
|
return &target.DeploymentResult{
|
||||||
Success: true,
|
Success: true,
|
||||||
TargetAddress: c.config.PEMPath,
|
TargetAddress: c.config.PEMPath,
|
||||||
DeploymentID: fmt.Sprintf("haproxy-%d", time.Now().Unix()),
|
DeploymentID: fmt.Sprintf("haproxy-%d", time.Now().Unix()),
|
||||||
Message: "Combined PEM deployed and HAProxy reloaded successfully",
|
Message: "Certificate deployed and HAProxy reloaded successfully" + idemNote,
|
||||||
DeployedAt: time.Now(),
|
DeployedAt: time.Now(),
|
||||||
Metadata: map[string]string{
|
Metadata: map[string]string{
|
||||||
"pem_path": c.config.PEMPath,
|
"pem_path": c.config.PEMPath,
|
||||||
"duration_ms": fmt.Sprintf("%d", deploymentDuration.Milliseconds()),
|
"duration_ms": fmt.Sprintf("%d", dur.Milliseconds()),
|
||||||
|
"idempotent": fmt.Sprintf("%t", res.SkippedAsIdempotent),
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateDeployment verifies that the deployed certificate is valid and accessible.
|
// ValidateOnly real impl — Phase 6 replaces the stub.
|
||||||
|
func (c *Connector) ValidateOnly(ctx context.Context, request target.DeploymentRequest) error {
|
||||||
|
if c.config == nil || c.config.ValidateCommand == "" {
|
||||||
|
return target.ErrValidateOnlyNotSupported
|
||||||
|
}
|
||||||
|
out, err := c.runValidate(ctx, c.config.ValidateCommand)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("haproxy -c -f (ValidateOnly): %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildCombinedPEM concatenates cert + chain + key in the order
|
||||||
|
// HAProxy requires.
|
||||||
|
func buildCombinedPEM(request target.DeploymentRequest) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(request.CertPEM)
|
||||||
|
b.WriteString("\n")
|
||||||
|
if request.ChainPEM != "" {
|
||||||
|
b.WriteString(request.ChainPEM)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
if request.KeyPEM != "" {
|
||||||
|
b.WriteString(request.KeyPEM)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Connector) buildPlan(combined []byte) deploy.Plan {
|
||||||
|
mode := c.config.PEMFileMode
|
||||||
|
if mode == 0 {
|
||||||
|
mode = 0600 // combined file contains the private key
|
||||||
|
}
|
||||||
|
return deploy.Plan{
|
||||||
|
Files: []deploy.File{{
|
||||||
|
Path: c.config.PEMPath,
|
||||||
|
Bytes: combined,
|
||||||
|
Mode: mode,
|
||||||
|
Owner: c.config.PEMFileOwner,
|
||||||
|
Group: c.config.PEMFileGroup,
|
||||||
|
}},
|
||||||
|
Defaults: deploy.FileDefaults{
|
||||||
|
Mode: 0600,
|
||||||
|
Owner: pickFirstExistingUser("haproxy"),
|
||||||
|
Group: pickFirstExistingGroup("haproxy"),
|
||||||
|
},
|
||||||
|
BackupRetention: c.config.BackupRetention,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Connector) runPostDeployVerify(ctx context.Context, deployedCertPEM string) error {
|
||||||
|
verify := c.config.PostDeployVerify
|
||||||
|
if verify != nil && !verify.Enabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
endpoint := ""
|
||||||
|
timeout := 10 * time.Second
|
||||||
|
if verify != nil {
|
||||||
|
endpoint = verify.Endpoint
|
||||||
|
if verify.Timeout > 0 {
|
||||||
|
timeout = verify.Timeout
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if endpoint == "" {
|
||||||
|
c.logger.Warn("post-deploy verify enabled but no endpoint configured; skipping")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
want, err := certPEMToFingerprint(deployedCertPEM)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("compute deployed cert fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
attempts := c.config.PostDeployVerifyAttempts
|
||||||
|
if attempts <= 0 {
|
||||||
|
attempts = 3
|
||||||
|
}
|
||||||
|
backoff := c.config.PostDeployVerifyBackoff
|
||||||
|
if backoff <= 0 {
|
||||||
|
backoff = 2 * time.Second
|
||||||
|
}
|
||||||
|
var lastErr error
|
||||||
|
for i := 0; i < attempts; i++ {
|
||||||
|
if i > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(backoff):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
res := c.probe(ctx, endpoint, timeout)
|
||||||
|
if !res.Success {
|
||||||
|
lastErr = fmt.Errorf("TLS probe failed: %s", res.Error)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
got := strings.ToLower(res.Fingerprint)
|
||||||
|
want = strings.ToLower(want)
|
||||||
|
if got == want {
|
||||||
|
c.logger.Info("post-deploy TLS verify succeeded",
|
||||||
|
"endpoint", endpoint, "fingerprint", got, "attempt", i+1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
lastErr = fmt.Errorf("post-deploy TLS verify SHA-256 mismatch: got %s, want %s", got, want)
|
||||||
|
}
|
||||||
|
return lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Connector) rollbackToBackups(ctx context.Context, backupPaths map[string]string) error {
|
||||||
|
for finalPath, backupPath := range backupPaths {
|
||||||
|
if backupPath == "" {
|
||||||
|
if err := os.Remove(finalPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("rollback remove %s: %w", finalPath, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bytes, err := os.ReadFile(backupPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("rollback read backup %s: %w", backupPath, err)
|
||||||
|
}
|
||||||
|
if _, err := deploy.AtomicWriteFile(ctx, finalPath, bytes, deploy.WriteOptions{
|
||||||
|
SkipIdempotent: true,
|
||||||
|
BackupRetention: -1,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("rollback write %s: %w", finalPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out, err := c.runReload(ctx, c.config.ReloadCommand)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("rollback reload failed: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Connector) failureResult(addr, stage string, err error, startTime time.Time) *target.DeploymentResult {
|
||||||
|
return &target.DeploymentResult{
|
||||||
|
Success: false,
|
||||||
|
TargetAddress: addr,
|
||||||
|
Message: fmt.Sprintf("%s: %v", stage, err),
|
||||||
|
DeployedAt: time.Now(),
|
||||||
|
Metadata: map[string]string{
|
||||||
|
"stage": stage,
|
||||||
|
"duration_ms": fmt.Sprintf("%d", time.Since(startTime).Milliseconds()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func certPEMToFingerprint(pemBytes string) (string, error) {
|
||||||
|
begin := "-----BEGIN CERTIFICATE-----"
|
||||||
|
end := "-----END CERTIFICATE-----"
|
||||||
|
beginIdx := strings.Index(pemBytes, begin)
|
||||||
|
if beginIdx < 0 {
|
||||||
|
return "", fmt.Errorf("no CERTIFICATE PEM block")
|
||||||
|
}
|
||||||
|
rest := pemBytes[beginIdx+len(begin):]
|
||||||
|
endIdx := strings.Index(rest, end)
|
||||||
|
if endIdx < 0 {
|
||||||
|
return "", fmt.Errorf("PEM block not terminated")
|
||||||
|
}
|
||||||
|
body := strings.TrimSpace(rest[:endIdx])
|
||||||
|
body = strings.ReplaceAll(body, "\n", "")
|
||||||
|
body = strings.ReplaceAll(body, "\r", "")
|
||||||
|
body = strings.ReplaceAll(body, " ", "")
|
||||||
|
der, err := base64.StdEncoding.DecodeString(body)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("base64 decode: %w", err)
|
||||||
|
}
|
||||||
|
h := sha256.Sum256(der)
|
||||||
|
return hex.EncodeToString(h[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pickFirstExistingUser(candidates ...string) string {
|
||||||
|
for _, name := range candidates {
|
||||||
|
if _, err := user.Lookup(name); err == nil {
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
func pickFirstExistingGroup(candidates ...string) string {
|
||||||
|
for _, name := range candidates {
|
||||||
|
if _, err := user.LookupGroup(name); err == nil {
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Connector) ValidateDeployment(ctx context.Context, request target.ValidationRequest) (*target.ValidationResult, error) {
|
func (c *Connector) ValidateDeployment(ctx context.Context, request target.ValidationRequest) (*target.ValidationResult, error) {
|
||||||
c.logger.Info("validating HAProxy deployment",
|
c.logger.Info("validating HAProxy deployment",
|
||||||
"certificate_id", request.CertificateID,
|
"certificate_id", request.CertificateID, "serial", request.Serial)
|
||||||
"serial", request.Serial)
|
|
||||||
|
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
// Validate HAProxy configuration if command provided
|
|
||||||
if c.config.ValidateCommand != "" {
|
if c.config.ValidateCommand != "" {
|
||||||
validateCmd := exec.CommandContext(ctx, c.config.ValidateCommand)
|
if _, err := c.runValidate(ctx, c.config.ValidateCommand); err != nil {
|
||||||
if output, err := validateCmd.CombinedOutput(); err != nil {
|
errMsg := fmt.Sprintf("HAProxy config validation failed: %v", err)
|
||||||
errMsg := fmt.Sprintf("HAProxy config validation failed: %v (output: %s)", err, string(output))
|
|
||||||
c.logger.Error("validation failed", "error", err)
|
|
||||||
return &target.ValidationResult{
|
return &target.ValidationResult{
|
||||||
Valid: false,
|
Valid: false,
|
||||||
Serial: request.Serial,
|
Serial: request.Serial,
|
||||||
@@ -193,11 +389,8 @@ func (c *Connector) ValidateDeployment(ctx context.Context, request target.Valid
|
|||||||
}, fmt.Errorf("%s", errMsg)
|
}, fmt.Errorf("%s", errMsg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify combined PEM file exists and is readable
|
|
||||||
if _, err := os.Stat(c.config.PEMPath); os.IsNotExist(err) {
|
if _, err := os.Stat(c.config.PEMPath); os.IsNotExist(err) {
|
||||||
errMsg := fmt.Sprintf("combined PEM file not found: %s", c.config.PEMPath)
|
errMsg := fmt.Sprintf("PEM file not found: %s", c.config.PEMPath)
|
||||||
c.logger.Error("validation failed", "error", err)
|
|
||||||
return &target.ValidationResult{
|
return &target.ValidationResult{
|
||||||
Valid: false,
|
Valid: false,
|
||||||
Serial: request.Serial,
|
Serial: request.Serial,
|
||||||
@@ -206,20 +399,17 @@ func (c *Connector) ValidateDeployment(ctx context.Context, request target.Valid
|
|||||||
ValidatedAt: time.Now(),
|
ValidatedAt: time.Now(),
|
||||||
}, fmt.Errorf("%s", errMsg)
|
}, fmt.Errorf("%s", errMsg)
|
||||||
}
|
}
|
||||||
|
dur := time.Since(startTime)
|
||||||
validationDuration := time.Since(startTime)
|
c.logger.Info("HAProxy deployment validated successfully", "duration", dur.String())
|
||||||
c.logger.Info("HAProxy deployment validated successfully",
|
|
||||||
"duration", validationDuration.String())
|
|
||||||
|
|
||||||
return &target.ValidationResult{
|
return &target.ValidationResult{
|
||||||
Valid: true,
|
Valid: true,
|
||||||
Serial: request.Serial,
|
Serial: request.Serial,
|
||||||
TargetAddress: c.config.PEMPath,
|
TargetAddress: c.config.PEMPath,
|
||||||
Message: "HAProxy configuration valid and PEM accessible",
|
Message: "HAProxy PEM file present and config valid",
|
||||||
ValidatedAt: time.Now(),
|
ValidatedAt: time.Now(),
|
||||||
Metadata: map[string]string{
|
Metadata: map[string]string{
|
||||||
"pem_path": c.config.PEMPath,
|
"validate_command": c.config.ValidateCommand,
|
||||||
"duration_ms": fmt.Sprintf("%d", validationDuration.Milliseconds()),
|
"duration_ms": fmt.Sprintf("%d", dur.Milliseconds()),
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,594 @@
|
|||||||
|
package haproxy_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/shankar0123/certctl/internal/connector/target"
|
||||||
|
"github.com/shankar0123/certctl/internal/connector/target/haproxy"
|
||||||
|
"github.com/shankar0123/certctl/internal/deploy"
|
||||||
|
"github.com/shankar0123/certctl/internal/tlsprobe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Phase 6 of the deploy-hardening I master bundle: ≥30 tests on
|
||||||
|
// the HAProxy connector. HAProxy's quirk vs NGINX/Apache: a single
|
||||||
|
// combined PEM (cert + chain + key) instead of separate files.
|
||||||
|
// Test count lifts 3 → 30+.
|
||||||
|
|
||||||
|
const (
|
||||||
|
certA = "-----BEGIN CERTIFICATE-----\nQUxQSEEtQ0VSVA==\n-----END CERTIFICATE-----\n"
|
||||||
|
chain = "-----BEGIN CERTIFICATE-----\nSU5UQ0hBSU4=\n-----END CERTIFICATE-----\n"
|
||||||
|
keyA = "-----BEGIN PRIVATE KEY-----\nZmFrZS1rZXk=\n-----END PRIVATE KEY-----\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
func quietLogger() *slog.Logger {
|
||||||
|
return slog.New(slog.NewTextHandler(os.NewFile(0, os.DevNull), &slog.HandlerOptions{Level: slog.LevelError}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func fingerprintOfPEM(t *testing.T, pem string) string {
|
||||||
|
t.Helper()
|
||||||
|
beg := strings.Index(pem, "-----BEGIN CERTIFICATE-----") + len("-----BEGIN CERTIFICATE-----")
|
||||||
|
body := pem[beg:]
|
||||||
|
end := strings.Index(body, "-----END CERTIFICATE-----")
|
||||||
|
body = strings.TrimSpace(body[:end])
|
||||||
|
body = strings.ReplaceAll(body, "\n", "")
|
||||||
|
der, _ := base64.StdEncoding.DecodeString(body)
|
||||||
|
h := sha256.Sum256(der)
|
||||||
|
return hex.EncodeToString(h[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func okProbe(fp string) func(context.Context, string, time.Duration) tlsprobe.ProbeResult {
|
||||||
|
return func(_ context.Context, addr string, _ time.Duration) tlsprobe.ProbeResult {
|
||||||
|
return tlsprobe.ProbeResult{Address: addr, Success: true, Fingerprint: fp}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func failProbe(reason string) func(context.Context, string, time.Duration) tlsprobe.ProbeResult {
|
||||||
|
return func(_ context.Context, addr string, _ time.Duration) tlsprobe.ProbeResult {
|
||||||
|
return tlsprobe.ProbeResult{Address: addr, Success: false, Error: reason}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func noopRun(context.Context, string) ([]byte, error) { return nil, nil }
|
||||||
|
func failRun(reason string) func(context.Context, string) ([]byte, error) {
|
||||||
|
return func(context.Context, string) ([]byte, error) {
|
||||||
|
return []byte(reason), errors.New(reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newC(_ *testing.T, cfg *haproxy.Config) *haproxy.Connector {
|
||||||
|
c := haproxy.New(cfg, quietLogger())
|
||||||
|
c.SetTestRunValidate(noopRun)
|
||||||
|
c.SetTestRunReload(noopRun)
|
||||||
|
c.SetTestProbe(okProbe("ignored"))
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func basicCfg(dir string) *haproxy.Config {
|
||||||
|
return &haproxy.Config{
|
||||||
|
PEMPath: filepath.Join(dir, "haproxy.pem"),
|
||||||
|
ReloadCommand: "systemctl reload haproxy",
|
||||||
|
ValidateCommand: "haproxy -c -f /etc/haproxy/haproxy.cfg",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Happy
|
||||||
|
func TestHAProxy_Happy(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
c := newC(t, basicCfg(dir))
|
||||||
|
res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain, KeyPEM: keyA})
|
||||||
|
if err != nil || !res.Success {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, _ := os.ReadFile(filepath.Join(dir, "haproxy.pem"))
|
||||||
|
if !strings.Contains(string(body), "ALPHA") || !strings.Contains(string(body), "INTCHAIN") || !strings.Contains(string(body), "fake-key") {
|
||||||
|
// (decoded base64 not visible in body; check headers instead)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), "BEGIN CERTIFICATE") {
|
||||||
|
t.Errorf("PEM not written: %s", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(body), "BEGIN PRIVATE KEY") {
|
||||||
|
t.Errorf("key not in combined PEM: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Validate fails
|
||||||
|
func TestHAProxy_ValidateFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.SetTestRunValidate(failRun("config error"))
|
||||||
|
_, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if !errors.Is(err, deploy.ErrValidateFailed) {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(pem); string(got) != "ORIG" {
|
||||||
|
t.Error("PEM modified")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Reload fails → rollback
|
||||||
|
func TestHAProxy_ReloadFails_Rollback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
var n int32
|
||||||
|
c.SetTestRunReload(func(_ context.Context, _ string) ([]byte, error) {
|
||||||
|
if atomic.AddInt32(&n, 1) == 1 {
|
||||||
|
return nil, errors.New("reload failed")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
_, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if !errors.Is(err, deploy.ErrReloadFailed) {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(pem); string(got) != "ORIG" {
|
||||||
|
t.Error("rollback didn't restore")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Rollback also fails
|
||||||
|
func TestHAProxy_RollbackAlsoFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.SetTestRunReload(failRun("wedged"))
|
||||||
|
_, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if !errors.Is(err, deploy.ErrRollbackFailed) {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Verify mismatch → rollback
|
||||||
|
func TestHAProxy_VerifyMismatch_Rollback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerifyAttempts = 1
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: true, Endpoint: "h:443"}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.SetTestProbe(okProbe("0000"))
|
||||||
|
_, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "SHA-256 mismatch") {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. Verify match → success
|
||||||
|
func TestHAProxy_VerifyMatch_Success(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerifyAttempts = 1
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: true, Endpoint: "h:443"}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.SetTestProbe(okProbe(fingerprintOfPEM(t, certA)))
|
||||||
|
res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err != nil || !res.Success {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 7. Idempotency
|
||||||
|
func TestHAProxy_Idempotency(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
combined := certA + "\n" + chain + "\n" + keyA + "\n"
|
||||||
|
os.WriteFile(pem, []byte(combined), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
var v, r int32
|
||||||
|
c.SetTestRunValidate(func(_ context.Context, _ string) ([]byte, error) {
|
||||||
|
atomic.AddInt32(&v, 1)
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
c.SetTestRunReload(func(_ context.Context, _ string) ([]byte, error) {
|
||||||
|
atomic.AddInt32(&r, 1)
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain, KeyPEM: keyA})
|
||||||
|
if v != 0 || r != 0 {
|
||||||
|
t.Errorf("v=%d r=%d", v, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 8. Combined PEM has correct order: cert + chain + key. Search
|
||||||
|
// by PEM block headers (the b64 bodies are opaque; check the
|
||||||
|
// structural markers instead).
|
||||||
|
func TestHAProxy_CombinedPEM_Order(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain, KeyPEM: keyA})
|
||||||
|
body, _ := os.ReadFile(cfg.PEMPath)
|
||||||
|
s := string(body)
|
||||||
|
// Two CERTIFICATE blocks (cert + chain); one PRIVATE KEY block.
|
||||||
|
firstCert := strings.Index(s, "BEGIN CERTIFICATE")
|
||||||
|
secondCert := strings.Index(s[firstCert+1:], "BEGIN CERTIFICATE") + firstCert + 1
|
||||||
|
keyHdr := strings.Index(s, "BEGIN PRIVATE KEY")
|
||||||
|
if !(firstCert >= 0 && secondCert > firstCert && keyHdr > secondCert) {
|
||||||
|
t.Errorf("PEM order broken: firstCert=%d secondCert=%d key=%d", firstCert, secondCert, keyHdr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 9. Default mode 0600
|
||||||
|
func TestHAProxy_DefaultMode_0600(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, KeyPEM: keyA})
|
||||||
|
stat, _ := os.Stat(cfg.PEMPath)
|
||||||
|
if stat.Mode().Perm() != 0600 {
|
||||||
|
t.Errorf("mode = %#o", stat.Mode().Perm())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 10. Mode override
|
||||||
|
func TestHAProxy_ModeOverride(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PEMFileMode = 0640
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
stat, _ := os.Stat(cfg.PEMPath)
|
||||||
|
if stat.Mode().Perm() != 0640 {
|
||||||
|
t.Errorf("mode = %#o", stat.Mode().Perm())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 11. Default 0600 wins over existing mode for HAProxy. Unlike
|
||||||
|
// NGINX/Apache (where preservation is the safer default), HAProxy
|
||||||
|
// historically wrote 0600 unconditionally — operators rely on
|
||||||
|
// that. Mode override via PEMFileMode is the supported escape
|
||||||
|
// hatch. Test pins the back-compat behavior.
|
||||||
|
func TestHAProxy_DefaultsTo0600_EvenWhenExistingIs0640(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("OLD"), 0640)
|
||||||
|
os.Chmod(pem, 0640)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
stat, _ := os.Stat(pem)
|
||||||
|
if stat.Mode().Perm() != 0600 {
|
||||||
|
t.Errorf("mode = %#o, want 0600 (HAProxy back-compat default)", stat.Mode().Perm())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 12. Backup retention
|
||||||
|
func TestHAProxy_BackupRetention(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("V0"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.BackupRetention = 2
|
||||||
|
c := newC(t, cfg)
|
||||||
|
for i := 1; i <= 5; i++ {
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: fmt.Sprintf("V%d-CERT", i)})
|
||||||
|
time.Sleep(2 * time.Millisecond)
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(dir)
|
||||||
|
cnt := 0
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.Contains(e.Name(), deploy.BackupSuffix) {
|
||||||
|
cnt++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cnt != 2 {
|
||||||
|
t.Errorf("count = %d", cnt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 13. ValidateOnly happy
|
||||||
|
func TestHAProxy_ValidateOnly_Happy(t *testing.T) {
|
||||||
|
c := newC(t, basicCfg(t.TempDir()))
|
||||||
|
if err := c.ValidateOnly(context.Background(), target.DeploymentRequest{}); err != nil {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 14. ValidateOnly fails
|
||||||
|
func TestHAProxy_ValidateOnly_Fails(t *testing.T) {
|
||||||
|
c := newC(t, basicCfg(t.TempDir()))
|
||||||
|
c.SetTestRunValidate(failRun("config err"))
|
||||||
|
if err := c.ValidateOnly(context.Background(), target.DeploymentRequest{}); err == nil {
|
||||||
|
t.Error("expected error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 15. ValidateOnly no command
|
||||||
|
func TestHAProxy_ValidateOnly_NoCommand(t *testing.T) {
|
||||||
|
c := haproxy.New(&haproxy.Config{}, quietLogger())
|
||||||
|
if err := c.ValidateOnly(context.Background(), target.DeploymentRequest{}); !errors.Is(err, target.ErrValidateOnlyNotSupported) {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 16. Verify disabled
|
||||||
|
func TestHAProxy_VerifyDisabled(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: false, Endpoint: "h:443"}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
var n int32
|
||||||
|
c.SetTestProbe(func(_ context.Context, _ string, _ time.Duration) tlsprobe.ProbeResult {
|
||||||
|
atomic.AddInt32(&n, 1)
|
||||||
|
return tlsprobe.ProbeResult{Success: true, Fingerprint: "x"}
|
||||||
|
})
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if n != 0 {
|
||||||
|
t.Error("probe called")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 17. Verify no endpoint
|
||||||
|
func TestHAProxy_VerifyNoEndpoint(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: true}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err != nil || !res.Success {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 18. Verify retries
|
||||||
|
func TestHAProxy_VerifyRetries(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerifyAttempts = 3
|
||||||
|
cfg.PostDeployVerifyBackoff = 1 * time.Millisecond
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: true, Endpoint: "h:443"}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
want := fingerprintOfPEM(t, certA)
|
||||||
|
var n int32
|
||||||
|
c.SetTestProbe(func(_ context.Context, _ string, _ time.Duration) tlsprobe.ProbeResult {
|
||||||
|
if atomic.AddInt32(&n, 1) < 3 {
|
||||||
|
return tlsprobe.ProbeResult{Success: true, Fingerprint: "stale"}
|
||||||
|
}
|
||||||
|
return tlsprobe.ProbeResult{Success: true, Fingerprint: want}
|
||||||
|
})
|
||||||
|
res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err != nil || !res.Success {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n != 3 {
|
||||||
|
t.Errorf("n = %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 19. Concurrent serializes
|
||||||
|
func TestHAProxy_ConcurrentSerializes(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
var inFlight, maxIF int32
|
||||||
|
c.SetTestRunReload(func(_ context.Context, _ string) ([]byte, error) {
|
||||||
|
n := atomic.AddInt32(&inFlight, 1)
|
||||||
|
for {
|
||||||
|
m := atomic.LoadInt32(&maxIF)
|
||||||
|
if n <= m || atomic.CompareAndSwapInt32(&maxIF, m, n) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Millisecond)
|
||||||
|
atomic.AddInt32(&inFlight, -1)
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
const N = 4
|
||||||
|
done := make(chan struct{}, N)
|
||||||
|
for i := 0; i < N; i++ {
|
||||||
|
go func(idx int) {
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: fmt.Sprintf("CERT-%d-%s", idx, certA)})
|
||||||
|
done <- struct{}{}
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
for i := 0; i < N; i++ {
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
if maxIF > 1 {
|
||||||
|
t.Errorf("max in flight = %d", maxIF)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 20. No chain → still works
|
||||||
|
func TestHAProxy_NoChain(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
res, _ := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, KeyPEM: keyA})
|
||||||
|
if !res.Success {
|
||||||
|
t.Error("not success")
|
||||||
|
}
|
||||||
|
body, _ := os.ReadFile(cfg.PEMPath)
|
||||||
|
if strings.Contains(string(body), "INTCHAIN") {
|
||||||
|
t.Error("chain in PEM despite empty ChainPEM")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 21. No key
|
||||||
|
func TestHAProxy_NoKey(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA, ChainPEM: chain})
|
||||||
|
body, _ := os.ReadFile(cfg.PEMPath)
|
||||||
|
if strings.Contains(string(body), "BEGIN PRIVATE KEY") {
|
||||||
|
t.Error("key in PEM despite empty KeyPEM")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 22. Backup created
|
||||||
|
func TestHAProxy_BackupCreated(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
entries, _ := os.ReadDir(dir)
|
||||||
|
found := false
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.Contains(e.Name(), deploy.BackupSuffix) {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("no backup")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 23. Backup disabled
|
||||||
|
func TestHAProxy_BackupDisabled(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.BackupRetention = -1
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
entries, _ := os.ReadDir(dir)
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.Contains(e.Name(), deploy.BackupSuffix) {
|
||||||
|
t.Error("backup despite -1")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 24. ValidateOnly stderr in error
|
||||||
|
func TestHAProxy_ValidateOnly_Stderr(t *testing.T) {
|
||||||
|
c := newC(t, basicCfg(t.TempDir()))
|
||||||
|
c.SetTestRunValidate(failRun("[ALERT] backend has no server"))
|
||||||
|
err := c.ValidateOnly(context.Background(), target.DeploymentRequest{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "ALERT") {
|
||||||
|
t.Errorf("got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 25. Ctx cancelled
|
||||||
|
func TestHAProxy_CtxCancelled(t *testing.T) {
|
||||||
|
cfg := basicCfg(t.TempDir())
|
||||||
|
c := newC(t, cfg)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
_, err := c.DeployCertificate(ctx, target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err == nil {
|
||||||
|
t.Error("expected ctx error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 26. Verify rollback re-runs reload
|
||||||
|
func TestHAProxy_VerifyRollback_RunsReload(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerifyAttempts = 1
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: true, Endpoint: "h:443"}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.SetTestProbe(okProbe("0000"))
|
||||||
|
var r int32
|
||||||
|
c.SetTestRunReload(func(_ context.Context, _ string) ([]byte, error) {
|
||||||
|
atomic.AddInt32(&r, 1)
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if r != 2 {
|
||||||
|
t.Errorf("reload calls = %d", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 27. DeploymentID has haproxy prefix
|
||||||
|
func TestHAProxy_DeploymentID(t *testing.T) {
|
||||||
|
c := newC(t, basicCfg(t.TempDir()))
|
||||||
|
res, _ := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if !strings.HasPrefix(res.DeploymentID, "haproxy-") {
|
||||||
|
t.Errorf("ID = %q", res.DeploymentID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 28. Metadata populated
|
||||||
|
func TestHAProxy_Metadata(t *testing.T) {
|
||||||
|
c := newC(t, basicCfg(t.TempDir()))
|
||||||
|
res, _ := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
for _, k := range []string{"pem_path", "duration_ms", "idempotent"} {
|
||||||
|
if _, ok := res.Metadata[k]; !ok {
|
||||||
|
t.Errorf("missing %q", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 29. Verify dial timeout
|
||||||
|
func TestHAProxy_VerifyDialTimeout(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pem := filepath.Join(dir, "haproxy.pem")
|
||||||
|
os.WriteFile(pem, []byte("ORIG"), 0600)
|
||||||
|
cfg := basicCfg(dir)
|
||||||
|
cfg.PostDeployVerifyAttempts = 1
|
||||||
|
cfg.PostDeployVerify = &haproxy.PostDeployVerifyConfig{Enabled: true, Endpoint: "h:443"}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
c.SetTestProbe(failProbe("dial timeout"))
|
||||||
|
_, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err == nil {
|
||||||
|
t.Error("expected timeout err")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 30. Validate empty (no validate command) → only reload runs, no
|
||||||
|
// PreCommit gate
|
||||||
|
func TestHAProxy_NoValidateCommand_OK(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cfg := &haproxy.Config{
|
||||||
|
PEMPath: filepath.Join(dir, "haproxy.pem"),
|
||||||
|
ReloadCommand: "systemctl reload haproxy",
|
||||||
|
}
|
||||||
|
c := newC(t, cfg)
|
||||||
|
res, err := c.DeployCertificate(context.Background(), target.DeploymentRequest{CertPEM: certA})
|
||||||
|
if err != nil || !res.Success {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 31. ValidateConfig rejects missing pem_path
|
||||||
|
func TestHAProxy_ValidateConfig_MissingPEMPath(t *testing.T) {
|
||||||
|
c := haproxy.New(&haproxy.Config{}, quietLogger())
|
||||||
|
err := c.ValidateConfig(context.Background(), []byte(`{"reload_command":"x"}`))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("expected error for missing pem_path")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 32. ValidateConfig rejects missing reload_command
|
||||||
|
func TestHAProxy_ValidateConfig_MissingReload(t *testing.T) {
|
||||||
|
c := haproxy.New(&haproxy.Config{}, quietLogger())
|
||||||
|
err := c.ValidateConfig(context.Background(), []byte(`{"pem_path":"/tmp/x"}`))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("expected error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 33. ValidateConfig rejects shell injection in reload command
|
||||||
|
func TestHAProxy_ValidateConfig_RejectsInjection(t *testing.T) {
|
||||||
|
c := haproxy.New(&haproxy.Config{}, quietLogger())
|
||||||
|
err := c.ValidateConfig(context.Background(), []byte(`{"pem_path":"/tmp/x","reload_command":"reload; rm -rf /"}`))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("expected injection error")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
package haproxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"github.com/shankar0123/certctl/internal/connector/target"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ValidateOnly is the default Phase 3 stub for the deploy-hardening
|
|
||||||
// I master bundle: returns ErrValidateOnlyNotSupported so existing
|
|
||||||
// connectors compile against the extended target.Connector interface
|
|
||||||
// without changing behavior. Phase haproxy dry-run support arrives when
|
|
||||||
// the connector's atomic-deploy implementation lands (NGINX in
|
|
||||||
// Phase 4, Apache in Phase 5, etc.); each phase replaces this stub
|
|
||||||
// with a real validate-with-the-target implementation.
|
|
||||||
func (c *Connector) ValidateOnly(ctx context.Context, request target.DeploymentRequest) error {
|
|
||||||
return target.ErrValidateOnlyNotSupported
|
|
||||||
}
|
|
||||||
@@ -25,7 +25,7 @@ import (
|
|||||||
"github.com/shankar0123/certctl/internal/connector/target/caddy"
|
"github.com/shankar0123/certctl/internal/connector/target/caddy"
|
||||||
"github.com/shankar0123/certctl/internal/connector/target/envoy"
|
"github.com/shankar0123/certctl/internal/connector/target/envoy"
|
||||||
"github.com/shankar0123/certctl/internal/connector/target/f5"
|
"github.com/shankar0123/certctl/internal/connector/target/f5"
|
||||||
"github.com/shankar0123/certctl/internal/connector/target/haproxy"
|
// haproxy removed Phase 6 — real ValidateOnly implementation now in haproxy.go.
|
||||||
"github.com/shankar0123/certctl/internal/connector/target/iis"
|
"github.com/shankar0123/certctl/internal/connector/target/iis"
|
||||||
"github.com/shankar0123/certctl/internal/connector/target/javakeystore"
|
"github.com/shankar0123/certctl/internal/connector/target/javakeystore"
|
||||||
"github.com/shankar0123/certctl/internal/connector/target/k8ssecret"
|
"github.com/shankar0123/certctl/internal/connector/target/k8ssecret"
|
||||||
@@ -69,7 +69,8 @@ var connectorsAtPhase3 = []struct {
|
|||||||
{"caddy", func() target.Connector { return &caddy.Connector{} }},
|
{"caddy", func() target.Connector { return &caddy.Connector{} }},
|
||||||
{"envoy", func() target.Connector { return &envoy.Connector{} }},
|
{"envoy", func() target.Connector { return &envoy.Connector{} }},
|
||||||
{"f5", func() target.Connector { return &f5.Connector{} }},
|
{"f5", func() target.Connector { return &f5.Connector{} }},
|
||||||
{"haproxy", func() target.Connector { return &haproxy.Connector{} }},
|
// haproxy removed Phase 6 — its ValidateOnly is now real;
|
||||||
|
// tested in haproxy/haproxy_atomic_test.go.
|
||||||
{"iis", func() target.Connector { return &iis.Connector{} }},
|
{"iis", func() target.Connector { return &iis.Connector{} }},
|
||||||
{"javakeystore", func() target.Connector { return &javakeystore.Connector{} }},
|
{"javakeystore", func() target.Connector { return &javakeystore.Connector{} }},
|
||||||
{"k8ssecret", func() target.Connector { return &k8ssecret.Connector{} }},
|
{"k8ssecret", func() target.Connector { return &k8ssecret.Connector{} }},
|
||||||
@@ -85,7 +86,7 @@ var connectorsAtPhase3 = []struct {
|
|||||||
func TestEveryConnectorDefaultsToSentinel(t *testing.T) {
|
func TestEveryConnectorDefaultsToSentinel(t *testing.T) {
|
||||||
// Expected list size shrinks as Phases 4-9 land their real
|
// Expected list size shrinks as Phases 4-9 land their real
|
||||||
// ValidateOnly implementations. Phase 4 removed nginx.
|
// ValidateOnly implementations. Phase 4 removed nginx.
|
||||||
const expectedAtCurrentPhase = 11
|
const expectedAtCurrentPhase = 10
|
||||||
if len(connectorsAtPhase3) != expectedAtCurrentPhase {
|
if len(connectorsAtPhase3) != expectedAtCurrentPhase {
|
||||||
t.Fatalf("connectors-at-phase list = %d entries, want %d (drift in the 13-connector inventory)", len(connectorsAtPhase3), expectedAtCurrentPhase)
|
t.Fatalf("connectors-at-phase list = %d entries, want %d (drift in the 13-connector inventory)", len(connectorsAtPhase3), expectedAtCurrentPhase)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user