mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
3420fcdee4
The 2026-09-01 Patrol assessment could not answer four questions from telemetry: what share of Patrol installs run local versus cloud models, what Patrol costs an install per month (issue 1789 estimates ~104k input tokens per full run at ~5.5 runs a day, above the Pro fee on non-Flash tiers), how the 240 of 255 fleet investigations that produced no plan actually ended, and which effective Patrol mode an install runs. Schema v17 adds closed buckets only. ai_provider_class classifies the Patrol model route (none, local, cloud_byok, cloud_subscription, hosted_quickstart, unknown) with a syntactic private-host check on custom OpenAI-compatible endpoints; it never resolves DNS or carries a provider, model, endpoint, or account. pulse_intelligence_patrol_autonomy_level is the effective level after licence and Autopilot gating. The two token fields bucket the existing local usage ledger's Patrol events, so exact counts and prices stay on the install. Thirteen investigation outcome counters partition the findings already counted as investigated, one bucket per finding, adding no finding, resource, or session identity. The strings are never omitted, so an empty value can only mean a pre-v17 sender; the receiver stores those as unknown. PRIVACY.md, its shipped mirror, the Settings preview interface, and the security-privacy and api-contracts subsystem notes carry the disclosure, and the coverage gap is registered in status.json. The agent-lifecycle and storage-recovery contracts name internal/api under their extension points, so both record that this telemetry is adjacent adoption analytics and never agent or storage state. The stable E2E tier pins the public schema version, so the disclosure spec moves to 17, and the Settings preview interface change carries a browser receipt from a scratch build of this tree at 1280x800 and 390x844. status.json also drops an identical duplicate of the ai-provider-guided-setup coverage-gap record that two merges each appended, which the private governance audit rejects. The bucket for a legacy config still pointing at the retired Pulse-hosted route is named hosted_legacy: the shipped privacy document discloses the vocabulary verbatim and the frontend copy contract keeps that document free of the retired hosted quickstart wording, so the telemetry package now pins the vocabulary at the source.
1259 lines
45 KiB
Go
1259 lines
45 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"crypto/tls"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"path"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/api"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/crypto"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/hosted"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/telemetry"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/updates"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/audit"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/extensions"
|
|
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/reporting"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// Version information
|
|
var (
|
|
MetricsPort = 9091
|
|
)
|
|
|
|
// BusinessHooks allows enterprise features to hook into the server lifecycle.
|
|
type BusinessHooks struct {
|
|
// OnMetricsStoreReady is called when the metrics store is initialized.
|
|
// This allows enterprise features to access metrics for reporting.
|
|
OnMetricsStoreReady func(store *metrics.Store)
|
|
|
|
// BindRBACAdminEndpoints allows enterprise modules to replace or decorate
|
|
// RBAC admin endpoints without importing internal API packages.
|
|
BindRBACAdminEndpoints extensions.BindRBACAdminEndpointsFunc
|
|
|
|
// BindAuditAdminEndpoints allows enterprise modules to replace or decorate
|
|
// audit admin endpoints without importing internal API packages.
|
|
BindAuditAdminEndpoints extensions.BindAuditAdminEndpointsFunc
|
|
|
|
// ResolveAuditStoreConfig allows enterprise runtimes to configure the
|
|
// canonical audit store without opening a competing database connection.
|
|
ResolveAuditStoreConfig extensions.ResolveAuditStoreConfigFunc
|
|
|
|
// BindSSOAdminEndpoints allows enterprise modules to replace or decorate
|
|
// SSO admin endpoints without importing internal API packages.
|
|
BindSSOAdminEndpoints extensions.BindSSOAdminEndpointsFunc
|
|
|
|
// BindReportingAdminEndpoints allows enterprise modules to replace or decorate
|
|
// reporting admin endpoints without importing internal API packages.
|
|
BindReportingAdminEndpoints extensions.BindReportingAdminEndpointsFunc
|
|
|
|
// BindAIAutoFixEndpoints allows enterprise modules to replace or decorate
|
|
// safe remediation endpoints (investigation, remediation, autonomy, fix execution).
|
|
BindAIAutoFixEndpoints extensions.BindAIAutoFixEndpointsFunc
|
|
|
|
// BindAIAlertAnalysisEndpoints allows enterprise modules to replace or decorate
|
|
// AI alert analysis endpoints (alert investigation, Kubernetes analysis).
|
|
BindAIAlertAnalysisEndpoints extensions.BindAIAlertAnalysisEndpointsFunc
|
|
|
|
// AIInvestigationEnabled controls whether premium AI investigation and
|
|
// remediation components are created at runtime. When nil or returns false,
|
|
// patrol runs in monitor-only mode (findings reported but never investigated,
|
|
// no remediation plans generated). Enterprise sets this to return true.
|
|
AIInvestigationEnabled func() bool
|
|
|
|
// CreateRemediationEngine creates the premium remediation engine.
|
|
// Returns nil in OSS (no implementation available). Enterprise provides
|
|
// the concrete engine implementation wrapped behind the interface.
|
|
CreateRemediationEngine func(cfg aicontracts.EngineConfig) aicontracts.RemediationEngine
|
|
|
|
// CreateInvestigationStore creates the premium investigation session store.
|
|
// Returns nil in OSS. Enterprise provides a concrete implementation.
|
|
CreateInvestigationStore func(dataDir string) aicontracts.InvestigationStore
|
|
|
|
// CreateInvestigationOrchestrator creates the premium investigation orchestrator.
|
|
// Returns nil in OSS. Enterprise provides a concrete implementation.
|
|
CreateInvestigationOrchestrator func(deps aicontracts.OrchestratorDeps) aicontracts.InvestigationOrchestrator
|
|
|
|
// CreateAlertAnalyzer creates the premium alert-triggered analyzer.
|
|
// Returns nil in OSS. Enterprise provides a concrete implementation.
|
|
CreateAlertAnalyzer func(deps aicontracts.AlertAnalyzerDeps) aicontracts.AlertAnalyzer
|
|
|
|
// ResolveMonitoredSystemAdmissionPolicy lets commercial runtimes own
|
|
// monitored-system admission policy while the public server owns inventory
|
|
// projection and the shared API contract.
|
|
ResolveMonitoredSystemAdmissionPolicy func(context.Context, extensions.MonitoredSystemAdmissionInput) extensions.MonitoredSystemAdmissionDecision
|
|
}
|
|
|
|
var (
|
|
globalHooks BusinessHooks
|
|
globalHooksMu sync.Mutex
|
|
)
|
|
|
|
// SetBusinessHooks registers hooks for the server.
|
|
func SetBusinessHooks(h BusinessHooks) {
|
|
globalHooksMu.Lock()
|
|
defer globalHooksMu.Unlock()
|
|
globalHooks = h
|
|
}
|
|
|
|
func runtimeIdentityForBusinessHooks(h BusinessHooks) pkglicensing.RuntimeIdentity {
|
|
if h.BindAuditAdminEndpoints != nil ||
|
|
h.ResolveAuditStoreConfig != nil ||
|
|
h.BindRBACAdminEndpoints != nil ||
|
|
h.BindSSOAdminEndpoints != nil ||
|
|
h.BindReportingAdminEndpoints != nil ||
|
|
h.BindAIAutoFixEndpoints != nil ||
|
|
h.BindAIAlertAnalysisEndpoints != nil ||
|
|
h.AIInvestigationEnabled != nil ||
|
|
h.CreateRemediationEngine != nil ||
|
|
h.CreateInvestigationStore != nil ||
|
|
h.CreateInvestigationOrchestrator != nil ||
|
|
h.CreateAlertAnalyzer != nil ||
|
|
h.ResolveMonitoredSystemAdmissionPolicy != nil {
|
|
return pkglicensing.ProRuntimeIdentity()
|
|
}
|
|
return pkglicensing.CommunityRuntimeIdentity()
|
|
}
|
|
|
|
func bindRuntimeVersion(version string) {
|
|
// The server package is also the entry point for wrapper binaries such as
|
|
// Pulse Pro. Keep the update/version subsystem bound to the version supplied
|
|
// by the executing binary so /api/version cannot fall back to a VERSION file
|
|
// inherited from its container image.
|
|
updates.BuildVersion = strings.TrimSpace(version)
|
|
}
|
|
|
|
// Run starts the Pulse monitoring server.
|
|
func Run(ctx context.Context, version string) (runErr error) {
|
|
bindRuntimeVersion(version)
|
|
|
|
// Initialize logger with baseline defaults for early startup logs
|
|
logging.Init(logging.Config{
|
|
Format: "auto",
|
|
Level: "info",
|
|
Component: "pulse",
|
|
})
|
|
defer logging.Shutdown()
|
|
|
|
applyRuntimeMemoryLimit()
|
|
|
|
if err := api.ValidateTrustedProxyCIDRsFromEnv(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Check for auto-import on first startup
|
|
if ShouldAutoImport() {
|
|
if err := PerformAutoImport(); err != nil {
|
|
log.Error().Err(err).Msg("Auto-import failed, continuing with normal startup")
|
|
}
|
|
}
|
|
|
|
// Load unified configuration
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load configuration: %w", err)
|
|
}
|
|
|
|
// Re-initialize logging with configuration-driven settings
|
|
logging.Init(logging.Config{
|
|
Format: cfg.LogFormat,
|
|
Level: cfg.LogLevel,
|
|
Component: "pulse",
|
|
FilePath: cfg.LogFile,
|
|
MaxSizeMB: cfg.LogMaxSize,
|
|
MaxAgeDays: cfg.LogMaxAge,
|
|
Compress: cfg.LogCompress,
|
|
})
|
|
|
|
// Initialize license public key for Pro feature validation
|
|
pkglicensing.InitEmbeddedPublicKey()
|
|
|
|
// Resolve telemetry identity before binding so a listener/startup failure can
|
|
// still report the same privacy-bounded service-health contract before exit.
|
|
mtPersistence := config.NewMultiTenantPersistence(cfg.DataPath)
|
|
baseDataDir := mtPersistence.BaseDataDir()
|
|
isDocker := os.Getenv("PULSE_DOCKER") == "true"
|
|
failureTelemetryCfg := telemetry.Config{
|
|
Version: version,
|
|
DataDir: baseDataDir,
|
|
IsDocker: isDocker,
|
|
Enabled: cfg.TelemetryEnabled,
|
|
}
|
|
failureCategory := telemetry.ServiceHealthFailureStartup
|
|
defer func() {
|
|
if runErr == nil {
|
|
return
|
|
}
|
|
reportCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
observation := telemetry.ServiceHealthObservation{
|
|
Observed: true,
|
|
FailureCategory: failureCategory,
|
|
}
|
|
if err := telemetry.SendServiceHealthEvent(reportCtx, failureTelemetryCfg, "startup", observation); err != nil {
|
|
log.Debug().Err(err).Str("category", failureCategory).Msg("Could not send bounded service-health failure telemetry")
|
|
}
|
|
}()
|
|
|
|
mainAddr := fmt.Sprintf("%s:%d", cfg.BindAddress, cfg.FrontendPort)
|
|
mainListener, err := net.Listen("tcp", mainAddr)
|
|
if err != nil {
|
|
failureCategory = telemetry.ServiceHealthFailureListener
|
|
return fmt.Errorf("failed to bind UI/API server on %s: %w", mainAddr, err)
|
|
}
|
|
defer mainListener.Close()
|
|
|
|
// Optional dedicated agent-control listener. Reports, command admission,
|
|
// version checks, and bootstrap downloads share this network boundary;
|
|
// the web UI and management API are not exposed on it.
|
|
var agentListener net.Listener
|
|
if cfg.AgentIngestPort > 0 {
|
|
agentAddr := fmt.Sprintf("%s:%d", cfg.BindAddress, cfg.AgentIngestPort)
|
|
agentListener, err = net.Listen("tcp", agentAddr)
|
|
if err != nil {
|
|
failureCategory = telemetry.ServiceHealthFailureListener
|
|
return fmt.Errorf("failed to bind agent ingest server on %s: %w", agentAddr, err)
|
|
}
|
|
defer agentListener.Close()
|
|
}
|
|
|
|
// Run multi-tenant data migration only when the feature is explicitly enabled.
|
|
// This prevents any on-disk layout changes for default (single-tenant) users.
|
|
if api.IsMultiTenantEnabled() {
|
|
if err := config.RunMigrationIfNeeded(baseDataDir); err != nil {
|
|
log.Error().Err(err).Msg("Multi-tenant data migration failed")
|
|
// Continue anyway - migration failure shouldn't block startup
|
|
}
|
|
}
|
|
if err := ensureDefaultOrgOwnerMembership(mtPersistence, cfg.AuthUser); err != nil {
|
|
log.Warn().Err(err).Msg("Failed to ensure default organization owner membership")
|
|
}
|
|
|
|
// Always capture audit events to SQLite (defense in depth). Read/export endpoints are license-gated.
|
|
// For the default org, TenantLoggerManager routes to the global logger, so initialize it as SQLite too.
|
|
globalHooksMu.Lock()
|
|
resolveAuditStoreConfig := globalHooks.ResolveAuditStoreConfig
|
|
globalHooksMu.Unlock()
|
|
resolvedAuditStore := extensions.AuditStoreConfig{}
|
|
if resolveAuditStoreConfig != nil {
|
|
resolvedAuditStore = resolveAuditStoreConfig(baseDataDir)
|
|
}
|
|
var globalCrypto audit.CryptoEncryptor
|
|
if cm, err := crypto.NewCryptoManagerAt(baseDataDir); err != nil {
|
|
log.Warn().Err(err).Str("data_dir", baseDataDir).Msg("Failed to initialize crypto manager for audit signing; signatures will be disabled")
|
|
} else {
|
|
globalCrypto = cm
|
|
}
|
|
if sqliteLogger, err := audit.NewSQLiteLogger(audit.SQLiteLoggerConfig{
|
|
DataDir: baseDataDir,
|
|
AuditDir: resolvedAuditStore.Directory,
|
|
CryptoMgr: globalCrypto,
|
|
SigningKey: resolvedAuditStore.SigningKey,
|
|
RetentionDays: resolvedAuditStore.RetentionDays,
|
|
RetentionConfigured: resolvedAuditStore.RetentionConfigured,
|
|
CleanupInterval: resolvedAuditStore.CleanupInterval,
|
|
}); err != nil {
|
|
log.Warn().Err(err).Str("data_dir", baseDataDir).Msg("Failed to initialize global SQLite audit logger; falling back to console logger")
|
|
} else {
|
|
audit.SetLogger(sqliteLogger)
|
|
}
|
|
|
|
// Initialize tenant audit manager for per-tenant audit logging
|
|
tenantAuditManager := audit.NewTenantLoggerManager(baseDataDir, &audit.SQLiteLoggerFactory{
|
|
// Prefer per-tenant crypto managers so each org has its own .encryption.key.
|
|
CryptoMgrForDataDir: func(dataDir string) (audit.CryptoEncryptor, error) {
|
|
return crypto.NewCryptoManagerAt(dataDir)
|
|
},
|
|
// Fallback for environments where per-tenant crypto initialization fails.
|
|
CryptoMgr: globalCrypto,
|
|
RetentionDays: resolvedAuditStore.RetentionDays,
|
|
RetentionConfigured: resolvedAuditStore.RetentionConfigured,
|
|
CleanupInterval: resolvedAuditStore.CleanupInterval,
|
|
})
|
|
api.SetTenantAuditManager(tenantAuditManager)
|
|
log.Info().Msg("Tenant audit manager initialized")
|
|
|
|
// Enable async audit logging to avoid request latency on audit writes.
|
|
if !strings.EqualFold(os.Getenv("PULSE_AUDIT_ASYNC"), "false") {
|
|
audit.EnableAsyncLogging(audit.AsyncLoggerConfig{BufferSize: 4096})
|
|
log.Info().Msg("Async audit logging enabled")
|
|
}
|
|
|
|
log.Info().Msg("Starting Pulse monitoring server")
|
|
|
|
// TODO: Validate agent binaries are available for download
|
|
// updates.EnsureHostAgentBinaries(version)
|
|
|
|
// Create derived context that cancels on interrupt
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
// Metrics uses a dedicated bind address so bearer-token scrape credentials
|
|
// do not inherit the public UI/API listener exposure by default.
|
|
metricsAddr := metricsListenAddress(cfg, MetricsPort)
|
|
if err := startMetricsServer(ctx, metricsAddr, cfg.MetricsToken, cfg.MetricsAllowInsecureRemote); err != nil {
|
|
log.Error().Err(err).Str("addr", metricsAddr).Msg("Metrics server disabled")
|
|
}
|
|
|
|
// Initialize WebSocket hub first
|
|
wsHub := websocket.NewHub(nil)
|
|
// Gate X-Forwarded-* trust in checkOrigin on the same trusted-proxy list
|
|
// used by the main API's auth.go / security.go.
|
|
wsHub.SetTrustedProxyChecker(api.IsTrustedProxyIP)
|
|
// Set allowed origins from configuration
|
|
if cfg.AllowedOrigins != "" {
|
|
if cfg.AllowedOrigins == "*" {
|
|
// Explicit wildcard - allow all origins (less secure)
|
|
wsHub.SetAllowedOrigins([]string{"*"})
|
|
} else {
|
|
// Use configured origins
|
|
wsHub.SetAllowedOrigins(strings.Split(cfg.AllowedOrigins, ","))
|
|
}
|
|
} else {
|
|
// Default: don't set any specific origins
|
|
wsHub.SetAllowedOrigins([]string{})
|
|
}
|
|
go wsHub.Run()
|
|
defer wsHub.Stop()
|
|
|
|
// Initialize reloadable monitoring system
|
|
reloadableMonitor, err := monitoring.NewReloadableMonitor(cfg, mtPersistence, wsHub)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to initialize monitoring system: %w", err)
|
|
}
|
|
|
|
// Trigger enterprise hooks if registered
|
|
globalHooksMu.Lock()
|
|
onMetricsStoreReady := globalHooks.OnMetricsStoreReady
|
|
bindRBACAdminEndpoints := globalHooks.BindRBACAdminEndpoints
|
|
bindAuditAdminEndpoints := globalHooks.BindAuditAdminEndpoints
|
|
bindSSOAdminEndpoints := globalHooks.BindSSOAdminEndpoints
|
|
bindReportingAdminEndpoints := globalHooks.BindReportingAdminEndpoints
|
|
bindAIAutoFixEndpoints := globalHooks.BindAIAutoFixEndpoints
|
|
bindAIAlertAnalysisEndpoints := globalHooks.BindAIAlertAnalysisEndpoints
|
|
aiInvestigationEnabled := globalHooks.AIInvestigationEnabled
|
|
createRemediationEngine := globalHooks.CreateRemediationEngine
|
|
createInvestigationStore := globalHooks.CreateInvestigationStore
|
|
createInvestigationOrchestrator := globalHooks.CreateInvestigationOrchestrator
|
|
createAlertAnalyzer := globalHooks.CreateAlertAnalyzer
|
|
runtimeIdentity := runtimeIdentityForBusinessHooks(globalHooks)
|
|
globalHooksMu.Unlock()
|
|
|
|
api.SetAIInvestigationEnabled(aiInvestigationEnabled)
|
|
api.SetCreateRemediationEngine(createRemediationEngine)
|
|
api.SetCreateInvestigationStore(createInvestigationStore)
|
|
api.SetCreateInvestigationOrchestrator(createInvestigationOrchestrator)
|
|
api.SetCreateAlertAnalyzer(createAlertAnalyzer)
|
|
api.SetRBACAdminEndpointsBinder(bindRBACAdminEndpoints)
|
|
api.SetAuditAdminEndpointsBinder(bindAuditAdminEndpoints)
|
|
api.SetSSOAdminEndpointsBinder(bindSSOAdminEndpoints)
|
|
api.SetReportingAdminEndpointsBinder(bindReportingAdminEndpoints)
|
|
api.SetAIAutoFixEndpointsBinder(bindAIAutoFixEndpoints)
|
|
api.SetAIAlertAnalysisEndpointsBinder(bindAIAlertAnalysisEndpoints)
|
|
|
|
if onMetricsStoreReady != nil {
|
|
func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
log.Error().Interface("panic", r).Msg("Enterprise OnMetricsStoreReady hook panicked")
|
|
}
|
|
}()
|
|
if store := reloadableMonitor.GetMonitor().GetMetricsStore(); store != nil {
|
|
onMetricsStoreReady(store)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Initialize reporting engine if not already set by enterprise hooks.
|
|
// This ensures Pro license holders get reporting even with the standard binary.
|
|
// Uses a dynamic store getter so the engine always queries the current monitor's
|
|
// metrics store, even after monitor reloads (which close and recreate the store).
|
|
if reporting.GetEngine() == nil {
|
|
if store := reloadableMonitor.GetMonitor().GetMetricsStore(); store != nil {
|
|
engine := reporting.NewReportEngine(reporting.EngineConfig{
|
|
MetricsStoreGetter: func() *metrics.Store {
|
|
return reloadableMonitor.GetMonitor().GetMetricsStore()
|
|
},
|
|
})
|
|
reporting.SetEngine(engine)
|
|
log.Info().Msg("Advanced Infrastructure Reporting (PDF/CSV) initialized")
|
|
}
|
|
}
|
|
|
|
// WebSocket state ownership is tenant-aware by construction, including default-org state.
|
|
wsHub.SetStateGetter(func(orgID string) interface{} {
|
|
if orgID == "" || orgID == "default" {
|
|
return reloadableMonitor.GetMonitor().BuildFrontendState()
|
|
}
|
|
|
|
mtMonitor := reloadableMonitor.GetMultiTenantMonitor()
|
|
if mtMonitor == nil {
|
|
// Security: never expose default-org state to non-default org clients.
|
|
log.Warn().Str("org_id", orgID).Msg("Tenant monitor unavailable for org state request")
|
|
return models.EmptyStateFrontend()
|
|
}
|
|
monitor, err := mtMonitor.GetMonitor(orgID)
|
|
if err != nil || monitor == nil {
|
|
log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to get tenant monitor for org state request")
|
|
return models.EmptyStateFrontend()
|
|
}
|
|
return monitor.BuildFrontendState()
|
|
})
|
|
|
|
// Set org authorization checker for WebSocket connections
|
|
// This ensures clients can only subscribe to orgs they have access to
|
|
orgLoader := api.NewMultiTenantOrganizationLoader(mtPersistence)
|
|
wsHub.SetOrgAuthChecker(api.NewAuthorizationChecker(orgLoader))
|
|
|
|
// Set multi-tenant checker for WebSocket connections
|
|
// This ensures the feature flag and license are checked before allowing non-default org connections
|
|
hostedMode := os.Getenv("PULSE_HOSTED_MODE") == "true"
|
|
wsHub.SetMultiTenantChecker(api.NewMultiTenantChecker(hostedMode))
|
|
|
|
// Wire up Prometheus metrics for alert lifecycle
|
|
alerts.SetMetricHooks(
|
|
metrics.RecordAlertFired,
|
|
metrics.RecordAlertResolved,
|
|
metrics.RecordAlertSuppressed,
|
|
metrics.RecordAlertAcknowledged,
|
|
)
|
|
log.Info().Msg("Alert metrics hooks registered")
|
|
|
|
// Start monitoring
|
|
reloadableMonitor.Start(ctx)
|
|
|
|
// Initialize API server with reload function
|
|
var router *api.Router
|
|
refreshRouterAfterMonitorReload := func() {
|
|
if router != nil {
|
|
router.SetMonitor(reloadableMonitor.GetMonitor())
|
|
router.SetMultiTenantMonitor(reloadableMonitor.GetMultiTenantMonitor())
|
|
if cfg := reloadableMonitor.GetConfig(); cfg != nil {
|
|
router.SetConfig(cfg)
|
|
}
|
|
// Re-apply system settings (e.g. webhook private CIDR allowlist)
|
|
// to the freshly recreated notification manager. Without this,
|
|
// instance-wide settings stored inside the notification manager
|
|
// are lost on every monitor reload triggered by auto-registration.
|
|
router.ReloadSystemSettings()
|
|
}
|
|
}
|
|
reloadFunc := func() error {
|
|
if err := reloadableMonitor.Reload(); err != nil {
|
|
return err
|
|
}
|
|
refreshRouterAfterMonitorReload()
|
|
return nil
|
|
}
|
|
router = api.NewRouter(cfg, reloadableMonitor.GetMonitor(), reloadableMonitor.GetMultiTenantMonitor(), wsHub, reloadFunc, version)
|
|
router.SetLicenseRuntimeIdentity(runtimeIdentity)
|
|
router.StartBackgroundWorkers()
|
|
|
|
// Inject resource store into monitor for WebSocket broadcasts
|
|
router.SetMonitor(reloadableMonitor.GetMonitor())
|
|
// Wire multi-tenant monitor to resource handlers for tenant-aware state
|
|
router.SetMultiTenantMonitor(reloadableMonitor.GetMultiTenantMonitor())
|
|
|
|
// Start AI patrol service for background infrastructure monitoring
|
|
router.StartPatrol(ctx)
|
|
|
|
// Start AI chat service
|
|
router.StartAIChat(ctx)
|
|
|
|
// Start hosted tenant reaper for automatic soft-delete cleanup
|
|
if os.Getenv("PULSE_HOSTED_MODE") == "true" {
|
|
reaper := hosted.NewReaper(mtPersistence, mtPersistence, 5*time.Minute, true)
|
|
reaper.OnBeforeDelete = func(orgID string) error {
|
|
return router.CleanupTenant(ctx, orgID)
|
|
}
|
|
go func() {
|
|
if err := reaper.Run(ctx); err != nil {
|
|
log.Error().Err(err).Msg("Hosted tenant reaper exited with error")
|
|
}
|
|
}()
|
|
log.Info().Msg("Hosted tenant reaper started")
|
|
}
|
|
|
|
// Start relay client for mobile remote access
|
|
router.StartRelay(ctx)
|
|
|
|
// Wire alert-triggered AI analysis
|
|
router.WireAlertTriggeredAI()
|
|
|
|
// Start pseudonymous telemetry (enabled by default; opt out via PULSE_TELEMETRY=false or Settings toggle).
|
|
// Persistence is created once here (outside the closure) to avoid NewConfigPersistence's
|
|
// fatal-on-error path running inside the telemetry goroutine.
|
|
telemetryPersistence := config.NewConfigPersistence(baseDataDir)
|
|
telemetryCfg := telemetry.Config{
|
|
Version: version,
|
|
DataDir: baseDataDir,
|
|
IsDocker: isDocker,
|
|
Enabled: cfg.TelemetryEnabled,
|
|
GetServiceHealth: newServiceHealthProbe(mainListener, cfg.HTTPSEnabled && cfg.TLSCertFile != "" && cfg.TLSKeyFile != ""),
|
|
GetSnapshot: func() telemetry.Snapshot {
|
|
// Use the latest config (may have been swapped by a reload).
|
|
currentCfg := cfg
|
|
if reloaded := reloadableMonitor.GetConfig(); reloaded != nil {
|
|
currentCfg = reloaded
|
|
}
|
|
|
|
snap := telemetry.Snapshot{
|
|
MultiTenant: currentCfg.MultiTenantEnabled,
|
|
HasAPITokens: currentCfg.HasAPITokens(),
|
|
AuthConfigured: currentCfg.AuthUser != "" || currentCfg.AuthPass != "" || currentCfg.HasAPITokens() || currentCfg.ProxyAuthSecret != "",
|
|
ConfiguredConnections: len(currentCfg.PVEInstances) + len(currentCfg.PBSInstances) + len(currentCfg.PMGInstances),
|
|
}
|
|
if truenas, err := telemetryPersistence.LoadTrueNASConfig(); err == nil {
|
|
snap.ConfiguredConnections += len(truenas)
|
|
}
|
|
if vmware, err := telemetryPersistence.LoadVMwareConfig(); err == nil {
|
|
snap.ConfiguredConnections += len(vmware)
|
|
}
|
|
if targets, err := telemetryPersistence.LoadAvailabilityTargets(); err == nil {
|
|
snap.ConfiguredConnections += len(targets)
|
|
probeAgents := make(map[string]struct{})
|
|
for _, target := range targets {
|
|
if agentID := strings.TrimSpace(target.ProbeAgentID); agentID != "" {
|
|
snap.AvailabilityProbeTargets++
|
|
probeAgents[agentID] = struct{}{}
|
|
}
|
|
}
|
|
snap.AvailabilityProbeAgents = len(probeAgents)
|
|
}
|
|
|
|
// Resource counts come from the tenant-aware monitor aggregate, not the
|
|
// default-org compatibility shim.
|
|
counts := reloadableMonitor.AggregateInstallSnapshotCounts()
|
|
snap.PVENodes = counts.PVENodes
|
|
snap.PBSInstances = counts.PBSInstances
|
|
snap.PMGInstances = counts.PMGInstances
|
|
snap.VMs = counts.VMs
|
|
snap.Containers = counts.Containers
|
|
snap.AgentHosts = counts.AgentHosts
|
|
snap.DockerHosts = counts.DockerHosts
|
|
snap.DockerContainers = counts.DockerContainers
|
|
snap.KubernetesClusters = counts.KubernetesClusters
|
|
snap.KubernetesNodes = counts.KubernetesNodes
|
|
snap.KubernetesPods = counts.KubernetesPods
|
|
snap.KubernetesDeployments = counts.KubernetesDeployments
|
|
snap.StoragePools = counts.StoragePools
|
|
snap.PhysicalDisks = counts.PhysicalDisks
|
|
snap.CephClusters = counts.CephClusters
|
|
snap.NetworkShares = counts.NetworkShares
|
|
snap.TrueNASSystems = counts.TrueNASSystems
|
|
snap.TrueNASVMs = counts.TrueNASVMs
|
|
snap.TrueNASApps = counts.TrueNASApps
|
|
snap.VMwareHosts = counts.VMwareHosts
|
|
snap.VMwareVMs = counts.VMwareVMs
|
|
snap.VMwareDatastores = counts.VMwareDatastores
|
|
snap.AvailabilityTargets = counts.AvailabilityTargets
|
|
snap.ActiveAlerts = counts.ActiveAlerts
|
|
snap.AlertsFired30d = counts.AlertsFired30d
|
|
snap.AlertsAcknowledged30d = counts.AlertsAcknowledged30d
|
|
snap.AlertsResolved30d = counts.AlertsResolved30d
|
|
snap.ActiveAlertsInfo = counts.AlertQuality.ActiveInfo
|
|
snap.ActiveAlertsWarning = counts.AlertQuality.ActiveWarning
|
|
snap.ActiveAlertsCritical = counts.AlertQuality.ActiveCritical
|
|
snap.ActiveAlertsAgeUnder1h = counts.AlertQuality.ActiveAgeUnder1h
|
|
snap.ActiveAlertsAge1h24h = counts.AlertQuality.ActiveAge1h24h
|
|
snap.ActiveAlertsAge1d7d = counts.AlertQuality.ActiveAge1d7d
|
|
snap.ActiveAlertsAge7dPlus = counts.AlertQuality.ActiveAge7dPlus
|
|
snap.AlertsFiredInfo30d = counts.AlertQuality.FiredInfo30d
|
|
snap.AlertsFiredWarning30d = counts.AlertQuality.FiredWarning30d
|
|
snap.AlertsFiredCritical30d = counts.AlertQuality.FiredCritical30d
|
|
snap.AlertsResolvedInfo30d = counts.AlertQuality.ResolvedInfo30d
|
|
snap.AlertsResolvedWarning30d = counts.AlertQuality.ResolvedWarning30d
|
|
snap.AlertsResolvedCritical30d = counts.AlertQuality.ResolvedCritical30d
|
|
snap.AlertsResolutionUnder15m30d = counts.AlertQuality.ResolutionUnder15m30d
|
|
snap.AlertsResolution15m1h30d = counts.AlertQuality.Resolution15m1h30d
|
|
snap.AlertsResolution1h24h30d = counts.AlertQuality.Resolution1h24h30d
|
|
snap.AlertsResolution1d7d30d = counts.AlertQuality.Resolution1d7d30d
|
|
snap.AlertsResolution7dPlus30d = counts.AlertQuality.Resolution7dPlus30d
|
|
snap.AlertsRepeatOccurrences30d = counts.AlertQuality.RepeatOccurrences30d
|
|
snap.AlertsSnoozedOccurrences30d = counts.AlertQuality.SnoozedOccurrences30d
|
|
snap.AlertsResolvedWhileSnoozed30d = counts.AlertQuality.ResolvedWhileSnoozed30d
|
|
snap.AlertManagerTenants = counts.AlertQuality.ManagerTenants
|
|
snap.AlertDeliveryActiveTenants = counts.AlertQuality.DeliveryActiveTenants
|
|
snap.AlertFlappingEnabledTenants = counts.AlertQuality.FlappingEnabledTenants
|
|
snap.AlertIntentPolicyConfiguredTenants = counts.AlertQuality.IntentPolicyTenants
|
|
snap.AlertEventHistoryAuthoritativeTenants = counts.AlertQuality.EventHistoryHealthyTenants
|
|
snap.AlertActiveStateAuthoritativeTenants = counts.AlertQuality.ActiveStateHealthyTenants
|
|
snap.AlertActiveStatePersistenceDegradedTenants = counts.AlertQuality.ActiveStateDegradedTenants
|
|
snap.NotificationAttempts7d = counts.NotificationAttempts7d
|
|
snap.NotificationDeliveries7d = counts.NotificationDeliveries7d
|
|
snap.NotificationFailures7d = counts.NotificationFailures7d
|
|
snap.NotificationFailuresAuthentication7d = counts.NotificationFailuresAuthentication7d
|
|
snap.NotificationFailuresRateLimited7d = counts.NotificationFailuresRateLimited7d
|
|
snap.NotificationFailuresConnectivity7d = counts.NotificationFailuresConnectivity7d
|
|
snap.NotificationFailuresTLS7d = counts.NotificationFailuresTLS7d
|
|
snap.NotificationFailuresConfiguration7d = counts.NotificationFailuresConfiguration7d
|
|
snap.NotificationFailuresRejected7d = counts.NotificationFailuresRejected7d
|
|
snap.NotificationFailuresServerError7d = counts.NotificationFailuresServerError7d
|
|
snap.NotificationFailuresUnknown7d = counts.NotificationFailuresUnknown7d
|
|
snap.DiscoveryEnabled = currentCfg.DiscoveryEnabled
|
|
|
|
// Feature flags from persisted config (using pre-created persistence).
|
|
snap.AIProviderClass = telemetry.AIProviderClassNone
|
|
if aiCfg, err := telemetryPersistence.LoadAIConfig(); err == nil && aiCfg != nil {
|
|
snap.AIEnabled = aiCfg.Enabled
|
|
// Closed route class only; provider, model, endpoint, and
|
|
// account identity never enter the snapshot.
|
|
snap.AIProviderClass = telemetry.ClassifyAIProviderClass(aiCfg)
|
|
snap.PatrolEnabled = aiCfg.IsPatrolEnabled()
|
|
snap.DiscoveryEnabled = snap.DiscoveryEnabled || aiCfg.IsDiscoveryEnabled()
|
|
snap.AIActionsEnabled = aiCfg.IsControlEnabled()
|
|
snap.AlertAIEnabled = aiCfg.IsAlertTriggeredAnalysisEnabled()
|
|
}
|
|
if relayCfg, err := telemetryPersistence.LoadRelayConfig(); err == nil {
|
|
snap.RelayEnabled = relayCfg.Enabled
|
|
}
|
|
|
|
applyLicensedFeatureConfigSnapshot(&snap, telemetryPersistence, time.Now().UTC())
|
|
|
|
// SSO/OIDC status.
|
|
if ssoCfg, err := telemetryPersistence.LoadSSOConfig(); err == nil && ssoCfg != nil {
|
|
snap.SSOEnabled = ssoCfg.HasEnabledProviders()
|
|
snap.AuthConfigured = snap.AuthConfigured || snap.SSOEnabled
|
|
}
|
|
if emailCfg, err := telemetryPersistence.LoadEmailConfig(); err == nil && emailCfg != nil && emailCfg.Enabled {
|
|
snap.NotificationsEnabled = true
|
|
}
|
|
if !snap.NotificationsEnabled {
|
|
if appriseCfg, err := telemetryPersistence.LoadAppriseConfig(); err == nil && appriseCfg != nil && appriseCfg.Enabled {
|
|
snap.NotificationsEnabled = true
|
|
}
|
|
}
|
|
if !snap.NotificationsEnabled {
|
|
if webhooks, err := telemetryPersistence.LoadWebhooks(); err == nil {
|
|
for _, webhook := range webhooks {
|
|
if webhook.Enabled {
|
|
snap.NotificationsEnabled = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Coarse commercial posture only; telemetry does not send exact tiers.
|
|
if router != nil && router.GetLicenseHandlers() != nil {
|
|
if svc := router.GetLicenseHandlers().Service(context.Background()); svc != nil {
|
|
if lic := svc.Current(); lic != nil {
|
|
snap.PaidLicense = lic.Claims.Tier != pkglicensing.TierFree
|
|
}
|
|
}
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
if router != nil {
|
|
router.ApplyUpdateTelemetrySnapshot(&snap, now)
|
|
router.ApplyLicensedFeatureTelemetrySnapshot(&snap, now)
|
|
router.ApplyWorkloadHistoryTelemetrySnapshot(&snap, now)
|
|
}
|
|
applyNodeTestTelemetrySnapshot(&snap, telemetryPersistence, now)
|
|
var actionSnapshot telemetry.PulseIntelligenceActionSnapshot
|
|
if router != nil {
|
|
actionSnapshot = router.GetPulseIntelligenceActionTelemetry(now.Add(-telemetry.PulseIntelligenceTelemetryWindow))
|
|
}
|
|
applyPulseIntelligenceTelemetrySnapshot(&snap, telemetryPersistence, currentCfg, actionSnapshot, now)
|
|
|
|
return snap
|
|
},
|
|
}
|
|
failureTelemetryCfg = telemetryCfg
|
|
telemetry.Start(ctx, telemetryCfg)
|
|
defer telemetry.Stop()
|
|
|
|
// Wire live telemetry toggle so Settings changes take effect immediately.
|
|
router.SetTelemetryToggleFunc(func(enabled bool) {
|
|
if enabled {
|
|
telemetryCfg.Enabled = true
|
|
telemetry.Start(ctx, telemetryCfg)
|
|
log.Info().Msg("Telemetry re-enabled via settings (live toggle)")
|
|
} else {
|
|
telemetryCfg.Enabled = false
|
|
telemetry.Stop()
|
|
log.Info().Msg("Telemetry disabled via settings (live toggle)")
|
|
}
|
|
})
|
|
router.SetTelemetryPreviewFunc(func() (telemetry.Ping, error) {
|
|
return telemetry.BuildPreview(telemetryCfg)
|
|
})
|
|
router.SetTelemetryResetFunc(func() (telemetry.Ping, error) {
|
|
if _, err := telemetry.ResetInstallID(baseDataDir); err != nil {
|
|
return telemetry.Ping{}, err
|
|
}
|
|
if telemetryCfg.Enabled {
|
|
telemetry.Start(ctx, telemetryCfg)
|
|
}
|
|
return telemetry.BuildPreview(telemetryCfg)
|
|
})
|
|
|
|
// Create HTTP server with unified configuration
|
|
srv := &http.Server{
|
|
Addr: mainAddr,
|
|
Handler: withGzip(router.Handler()),
|
|
ReadHeaderTimeout: 15 * time.Second,
|
|
WriteTimeout: 0, // Disabled to support SSE/streaming
|
|
IdleTimeout: 120 * time.Second,
|
|
TLSConfig: &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
},
|
|
}
|
|
|
|
// The dedicated listener serves the complete agent control plane, while
|
|
// excluding the web UI and management REST API.
|
|
var agentSrv *http.Server
|
|
if agentListener != nil {
|
|
agentSrv = &http.Server{
|
|
Handler: agentIngestHandler(router.Handler()),
|
|
ReadHeaderTimeout: 15 * time.Second,
|
|
WriteTimeout: 0, // Disabled to support SSE/streaming
|
|
IdleTimeout: 120 * time.Second,
|
|
TLSConfig: &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Start config watcher for .env file changes
|
|
configWatcher, err := config.NewConfigWatcher(cfg)
|
|
if err != nil {
|
|
log.Warn().Err(err).Msg("Failed to create config watcher, .env changes will require restart")
|
|
} else {
|
|
configWatcher.SetMockReloadCallback(func() {
|
|
log.Info().Msg(".env mock settings changed, reloading monitor")
|
|
if err := reloadableMonitor.Reload(); err != nil {
|
|
log.Error().Err(err).Msg("Failed to reload monitor after .env mock setting change")
|
|
} else {
|
|
refreshRouterAfterMonitorReload()
|
|
}
|
|
})
|
|
|
|
configWatcher.SetAPITokenReloadCallback(func() {
|
|
if monitor := reloadableMonitor.GetMonitor(); monitor != nil {
|
|
monitor.RebuildTokenBindings()
|
|
}
|
|
})
|
|
|
|
if err := configWatcher.Start(); err != nil {
|
|
log.Warn().Err(err).Msg("Failed to start config watcher")
|
|
}
|
|
defer configWatcher.Stop()
|
|
}
|
|
|
|
// Start HTTP→HTTPS redirect server when HTTPS is active and redirect port is configured.
|
|
var redirectSrv *http.Server
|
|
if cfg.HTTPSEnabled && cfg.TLSCertFile != "" && cfg.TLSKeyFile != "" && cfg.HTTPRedirectPort > 0 {
|
|
if cfg.HTTPRedirectPort == cfg.FrontendPort {
|
|
log.Error().
|
|
Int("redirect_port", cfg.HTTPRedirectPort).
|
|
Int("frontend_port", cfg.FrontendPort).
|
|
Msg("HTTP_REDIRECT_PORT must differ from FRONTEND_PORT; skipping redirect server")
|
|
} else {
|
|
httpsPort := cfg.FrontendPort
|
|
redirectSrv = &http.Server{
|
|
Addr: fmt.Sprintf("%s:%d", cfg.BindAddress, cfg.HTTPRedirectPort),
|
|
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Host == "" {
|
|
http.Error(w, "Bad Request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
// Extract the hostname, stripping any port. net.SplitHostPort
|
|
// handles IPv6 bracket notation (e.g. [::1]:80 → "::1").
|
|
// If SplitHostPort fails, the Host has no port — strip any
|
|
// stray brackets that a bare IPv6 literal may carry.
|
|
host := r.Host
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
} else {
|
|
host = strings.TrimPrefix(strings.TrimSuffix(host, "]"), "[")
|
|
}
|
|
// Rebuild the authority with the HTTPS port.
|
|
// net.JoinHostPort handles IPv6 bracketing automatically.
|
|
var authority string
|
|
if httpsPort != 443 {
|
|
authority = net.JoinHostPort(host, strconv.Itoa(httpsPort))
|
|
} else if strings.Contains(host, ":") {
|
|
authority = "[" + host + "]"
|
|
} else {
|
|
authority = host
|
|
}
|
|
target := "https://" + authority + r.URL.RequestURI()
|
|
http.Redirect(w, r, target, http.StatusMovedPermanently)
|
|
}),
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
}
|
|
go func() {
|
|
log.Info().
|
|
Str("host", cfg.BindAddress).
|
|
Int("port", cfg.HTTPRedirectPort).
|
|
Int("https_port", httpsPort).
|
|
Msg("HTTP→HTTPS redirect server listening")
|
|
if err := redirectSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
log.Error().Err(err).Msg("Failed to start HTTP redirect server")
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// Start server
|
|
serverErr := make(chan error, 2)
|
|
go func() {
|
|
var err error
|
|
if cfg.HTTPSEnabled && cfg.TLSCertFile != "" && cfg.TLSKeyFile != "" {
|
|
log.Info().
|
|
Str("host", cfg.BindAddress).
|
|
Int("port", cfg.FrontendPort).
|
|
Str("protocol", "HTTPS").
|
|
Msg("Server listening")
|
|
err = srv.ServeTLS(mainListener, cfg.TLSCertFile, cfg.TLSKeyFile)
|
|
} else {
|
|
if cfg.HTTPSEnabled {
|
|
log.Warn().Msg("HTTPS_ENABLED is true but TLS_CERT_FILE or TLS_KEY_FILE not configured, falling back to HTTP")
|
|
}
|
|
log.Info().
|
|
Str("host", cfg.BindAddress).
|
|
Int("port", cfg.FrontendPort).
|
|
Str("protocol", "HTTP").
|
|
Msg("Server listening")
|
|
err = srv.Serve(mainListener)
|
|
}
|
|
if err != nil && err != http.ErrServerClosed {
|
|
serverErr <- err
|
|
}
|
|
}()
|
|
failureCategory = telemetry.ServiceHealthFailureRuntime
|
|
|
|
if agentSrv != nil {
|
|
go func() {
|
|
var err error
|
|
if cfg.HTTPSEnabled && cfg.TLSCertFile != "" && cfg.TLSKeyFile != "" {
|
|
log.Info().
|
|
Str("host", cfg.BindAddress).
|
|
Int("port", cfg.AgentIngestPort).
|
|
Str("protocol", "HTTPS").
|
|
Msg("Agent ingest server listening")
|
|
err = agentSrv.ServeTLS(agentListener, cfg.TLSCertFile, cfg.TLSKeyFile)
|
|
} else {
|
|
log.Info().
|
|
Str("host", cfg.BindAddress).
|
|
Int("port", cfg.AgentIngestPort).
|
|
Str("protocol", "HTTP").
|
|
Msg("Agent ingest server listening")
|
|
err = agentSrv.Serve(agentListener)
|
|
}
|
|
if err != nil && err != http.ErrServerClosed {
|
|
serverErr <- err
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Setup signal handlers
|
|
sigChan := make(chan os.Signal, 1)
|
|
reloadChan := make(chan os.Signal, 1)
|
|
|
|
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
|
signal.Notify(reloadChan, syscall.SIGHUP)
|
|
defer signal.Stop(sigChan)
|
|
defer signal.Stop(reloadChan)
|
|
|
|
for {
|
|
select {
|
|
case err := <-serverErr:
|
|
runErr = fmt.Errorf("UI/API server stopped unexpectedly: %w", err)
|
|
log.Error().Err(runErr).Msg("Shutting down after UI/API server failure")
|
|
goto shutdown
|
|
|
|
case <-ctx.Done():
|
|
log.Info().Msg("Context cancelled, shutting down...")
|
|
goto shutdown
|
|
|
|
case <-reloadChan:
|
|
log.Info().Msg("Received SIGHUP, reloading configuration...")
|
|
if configWatcher != nil {
|
|
configWatcher.ReloadConfig()
|
|
}
|
|
|
|
if err := reloadFunc(); err != nil {
|
|
log.Error().Err(err).Msg("Failed to reload monitor after SIGHUP")
|
|
} else {
|
|
log.Info().Msg("Runtime configuration reloaded")
|
|
}
|
|
|
|
case <-sigChan:
|
|
log.Info().Msg("Shutting down server...")
|
|
goto shutdown
|
|
}
|
|
}
|
|
|
|
shutdown:
|
|
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer shutdownCancel()
|
|
|
|
if redirectSrv != nil {
|
|
if err := redirectSrv.Shutdown(shutdownCtx); err != nil {
|
|
log.Error().Err(err).Msg("HTTP redirect server shutdown error")
|
|
}
|
|
}
|
|
if err := srv.Shutdown(shutdownCtx); err != nil {
|
|
log.Error().Err(err).Msg("Server shutdown error")
|
|
}
|
|
if agentSrv != nil {
|
|
if err := agentSrv.Shutdown(shutdownCtx); err != nil {
|
|
log.Error().Err(err).Msg("Agent ingest server shutdown error")
|
|
}
|
|
}
|
|
|
|
// Stop license grant refresh loops
|
|
router.StopGrantRefresh()
|
|
|
|
// Close organization RBAC databases and clear the canonical global manager.
|
|
router.ShutdownRBAC()
|
|
|
|
// Gracefully stop AI intelligence services (patrol, investigations, triggers)
|
|
router.ShutdownAIIntelligence()
|
|
|
|
// Stop relay client
|
|
router.StopRelay()
|
|
|
|
// Stop AI chat service (kills sidecar process group)
|
|
router.StopAIChat(shutdownCtx)
|
|
|
|
// Ensure mock-mode background update ticker is stopped before process exit.
|
|
if mock.IsMockEnabled() {
|
|
if err := mock.SetEnabled(false); err != nil {
|
|
log.Warn().Err(err).Msg("Failed to disable mock mode during shutdown")
|
|
}
|
|
}
|
|
|
|
cancel()
|
|
reloadableMonitor.Stop()
|
|
|
|
if configWatcher != nil {
|
|
configWatcher.Stop()
|
|
}
|
|
|
|
// Close tenant audit loggers
|
|
tenantAuditManager.Close()
|
|
if err := audit.Close(); err != nil {
|
|
log.Error().Err(err).Msg("Failed to close audit logger")
|
|
}
|
|
log.Info().Msg("Server stopped")
|
|
return runErr
|
|
}
|
|
|
|
// agentIngestHandler restricts a handler to the complete agent control plane.
|
|
// Reports, command admission, version checks, and bootstrap downloads must
|
|
// share one reachable listener; management APIs and the web UI remain absent.
|
|
func agentIngestHandler(inner http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if !isAgentControlPlanePath(r.URL.Path) {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
inner.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
func isAgentControlPlanePath(requestPath string) bool {
|
|
if requestPath == "" || path.Clean(requestPath) != requestPath {
|
|
return false
|
|
}
|
|
if strings.HasPrefix(requestPath, "/api/agents/") {
|
|
return true
|
|
}
|
|
switch requestPath {
|
|
case "/api/agent/ws",
|
|
"/api/agent/version",
|
|
"/api/server/info",
|
|
"/install.sh",
|
|
"/install.ps1",
|
|
"/download/pulse-agent":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// startMetricsServer starts the Prometheus /metrics endpoint. When metricsToken
|
|
// is non-empty, requests must include a matching Authorization: Bearer <token> header.
|
|
func startMetricsServer(ctx context.Context, addr string, metricsToken string, allowInsecureRemote bool) error {
|
|
if metricsToken != "" && !allowInsecureRemote && !metricsAddressIsLoopback(addr) {
|
|
return fmt.Errorf("metrics bearer token over remote plaintext HTTP is disabled; bind metrics to loopback or set PULSE_METRICS_ALLOW_INSECURE_REMOTE=true")
|
|
}
|
|
|
|
handler := promhttp.Handler()
|
|
if metricsToken != "" {
|
|
inner := handler
|
|
handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
auth := r.Header.Get("Authorization")
|
|
const prefix = "Bearer "
|
|
providedToken := ""
|
|
if len(auth) >= len(prefix) && strings.EqualFold(auth[:len(prefix)], prefix) {
|
|
providedToken = auth[len(prefix):]
|
|
}
|
|
if len(providedToken) != len(metricsToken) ||
|
|
subtle.ConstantTimeCompare([]byte(providedToken), []byte(metricsToken)) != 1 {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
inner.ServeHTTP(w, r)
|
|
})
|
|
log.Info().Msg("Metrics endpoint requires bearer token authentication")
|
|
}
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/metrics", handler)
|
|
|
|
srv := &http.Server{
|
|
Addr: addr,
|
|
Handler: mux,
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
}
|
|
|
|
go func() {
|
|
log.Info().Str("addr", addr).Msg("Metrics server listening")
|
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
|
log.Error().Err(err).Msg("Metrics server failed")
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
_ = srv.Shutdown(shutdownCtx)
|
|
}()
|
|
|
|
return nil
|
|
}
|
|
|
|
func ShouldAutoImport() bool {
|
|
configPath := config.ResolveRuntimeDataDir("")
|
|
|
|
if _, err := os.Stat(filepath.Join(configPath, "nodes.enc")); err == nil {
|
|
return false
|
|
}
|
|
|
|
return os.Getenv("PULSE_INIT_CONFIG_DATA") != "" ||
|
|
os.Getenv("PULSE_INIT_CONFIG_FILE") != ""
|
|
}
|
|
|
|
func PerformAutoImport() error {
|
|
configData := os.Getenv("PULSE_INIT_CONFIG_DATA")
|
|
configFile := os.Getenv("PULSE_INIT_CONFIG_FILE")
|
|
configPass := os.Getenv("PULSE_INIT_CONFIG_PASSPHRASE")
|
|
source := "none"
|
|
if configFile != "" {
|
|
source = "file"
|
|
} else if configData != "" {
|
|
source = "env_data"
|
|
}
|
|
|
|
logAudit := func(success bool, reason string) {
|
|
details := "source=" + source
|
|
if reason != "" {
|
|
details += " reason=" + reason
|
|
}
|
|
audit.Log("config_auto_import", "system", "", "/startup/auto-import", success, details)
|
|
}
|
|
|
|
if configPass == "" {
|
|
logAudit(false, "missing_passphrase")
|
|
return fmt.Errorf("PULSE_INIT_CONFIG_PASSPHRASE is required for auto-import")
|
|
}
|
|
|
|
var encryptedData string
|
|
|
|
if configFile != "" {
|
|
data, err := os.ReadFile(configFile)
|
|
if err != nil {
|
|
logAudit(false, "read_config_file_failed")
|
|
return fmt.Errorf("failed to read config file: %w", err)
|
|
}
|
|
payload, err := NormalizeImportPayload(data)
|
|
if err != nil {
|
|
logAudit(false, "normalize_payload_failed")
|
|
return err
|
|
}
|
|
encryptedData = payload
|
|
} else if configData != "" {
|
|
payload, err := NormalizeImportPayload([]byte(configData))
|
|
if err != nil {
|
|
logAudit(false, "normalize_payload_failed")
|
|
return err
|
|
}
|
|
encryptedData = payload
|
|
} else {
|
|
logAudit(false, "missing_payload")
|
|
return fmt.Errorf("no config data provided")
|
|
}
|
|
|
|
configPath := config.ResolveRuntimeDataDir("")
|
|
|
|
persistence := config.NewConfigPersistence(configPath)
|
|
if err := persistence.ImportConfig(encryptedData, configPass); err != nil {
|
|
logAudit(false, "import_failed")
|
|
return fmt.Errorf("failed to import configuration: %w", err)
|
|
}
|
|
|
|
logAudit(true, "")
|
|
log.Info().Msg("Configuration auto-imported successfully")
|
|
return nil
|
|
}
|
|
|
|
func NormalizeImportPayload(raw []byte) (string, error) {
|
|
trimmed := strings.TrimSpace(string(raw))
|
|
if trimmed == "" {
|
|
return "", fmt.Errorf("configuration payload is empty")
|
|
}
|
|
|
|
if decoded, err := base64.StdEncoding.DecodeString(trimmed); err == nil {
|
|
decodedTrimmed := strings.TrimSpace(string(decoded))
|
|
if LooksLikeBase64(decodedTrimmed) {
|
|
return decodedTrimmed, nil
|
|
}
|
|
return trimmed, nil
|
|
}
|
|
|
|
return base64.StdEncoding.EncodeToString(raw), nil
|
|
}
|
|
|
|
func LooksLikeBase64(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
compact := strings.Map(func(r rune) rune {
|
|
switch r {
|
|
case '\n', '\r', '\t', ' ':
|
|
return -1
|
|
default:
|
|
return r
|
|
}
|
|
}, s)
|
|
|
|
if compact == "" || len(compact)%4 != 0 {
|
|
return false
|
|
}
|
|
for i := 0; i < len(compact); i++ {
|
|
c := compact[i]
|
|
isAlphaNum := (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')
|
|
if isAlphaNum || c == '+' || c == '/' || c == '=' {
|
|
continue
|
|
}
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func ensureDefaultOrgOwnerMembership(mtp *config.MultiTenantPersistence, adminUser string) error {
|
|
if mtp == nil {
|
|
return nil
|
|
}
|
|
adminUser = strings.TrimSpace(adminUser)
|
|
if adminUser == "" {
|
|
return nil
|
|
}
|
|
|
|
org, err := mtp.LoadOrganization("default")
|
|
if err != nil {
|
|
return fmt.Errorf("load default organization: %w", err)
|
|
}
|
|
if org == nil {
|
|
org = &models.Organization{}
|
|
}
|
|
|
|
changed := false
|
|
now := time.Now().UTC()
|
|
if strings.TrimSpace(org.ID) == "" {
|
|
org.ID = "default"
|
|
changed = true
|
|
}
|
|
if strings.TrimSpace(org.DisplayName) == "" {
|
|
org.DisplayName = "default"
|
|
changed = true
|
|
}
|
|
if org.CreatedAt.IsZero() {
|
|
org.CreatedAt = now
|
|
changed = true
|
|
}
|
|
if strings.TrimSpace(org.OwnerUserID) == "" {
|
|
org.OwnerUserID = adminUser
|
|
changed = true
|
|
}
|
|
|
|
if ensureOrgOwnerMembership(org, adminUser, now) {
|
|
changed = true
|
|
}
|
|
|
|
if ownerUserID := strings.TrimSpace(org.OwnerUserID); ownerUserID != "" && ownerUserID != adminUser {
|
|
if ensureOrgOwnerMembership(org, ownerUserID, now) {
|
|
changed = true
|
|
}
|
|
}
|
|
|
|
if !changed {
|
|
return nil
|
|
}
|
|
return mtp.SaveOrganization(org)
|
|
}
|
|
|
|
func ensureOrgOwnerMembership(org *models.Organization, userID string, now time.Time) bool {
|
|
userID = strings.TrimSpace(userID)
|
|
if org == nil || userID == "" {
|
|
return false
|
|
}
|
|
|
|
for i := range org.Members {
|
|
if strings.TrimSpace(org.Members[i].UserID) != userID {
|
|
continue
|
|
}
|
|
changed := false
|
|
if org.Members[i].Role != models.OrgRoleOwner {
|
|
org.Members[i].Role = models.OrgRoleOwner
|
|
changed = true
|
|
}
|
|
if org.Members[i].AddedAt.IsZero() {
|
|
org.Members[i].AddedAt = now
|
|
changed = true
|
|
}
|
|
if strings.TrimSpace(org.Members[i].AddedBy) == "" {
|
|
org.Members[i].AddedBy = userID
|
|
changed = true
|
|
}
|
|
return changed
|
|
}
|
|
|
|
org.Members = append(org.Members, models.OrganizationMember{
|
|
UserID: userID,
|
|
Role: models.OrgRoleOwner,
|
|
AddedAt: now,
|
|
AddedBy: userID,
|
|
})
|
|
return true
|
|
}
|