/* Copyright NetFoundry Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "bytes" "crypto/tls" "crypto/x509" "fmt" "github.com/hashicorp/go-hclog" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v2" "github.com/openziti/identity" "github.com/openziti/storage/boltz" "github.com/openziti/transport/v2" transporttls "github.com/openziti/transport/v2/tls" "github.com/openziti/ziti/common/config" "github.com/openziti/ziti/common/pb/ctrl_pb" "github.com/openziti/ziti/common/pb/mgmt_pb" "github.com/openziti/ziti/controller/command" "github.com/openziti/ziti/controller/db" "github.com/openziti/ziti/controller/network" "github.com/openziti/ziti/controller/raft" "github.com/openziti/ziti/router/xgress" "github.com/pkg/errors" "gopkg.in/yaml.v2" "math" "os" "strings" "time" ) const ( DefaultProfileMemoryInterval = 15 * time.Second DefaultHealthChecksBoltCheckInterval = 30 * time.Second DefaultHealthChecksBoltCheckTimeout = 20 * time.Second DefaultHealthChecksBoltCheckInitialDelay = 30 * time.Second DefaultRaftCommandHandlerMaxQueueSize = 1000 // DefaultTlsHandshakeRateLimiterEnabled is whether the tls handshake rate limiter is enabled by default DefaultTlsHandshakeRateLimiterEnabled = false // TlsHandshakeRateLimiterMinSizeValue is the minimum size that can be configured for the tls handshake rate limiter // window range TlsHandshakeRateLimiterMinSizeValue = 5 // TlsHandshakeRateLimiterMaxSizeValue is the maximum size that can be configured for the tls handshake rate limiter // window range TlsHandshakeRateLimiterMaxSizeValue = 10000 // TlsHandshakeRateLimiterMetricOutstandingCount is the name of the metric tracking how many tasks are in process TlsHandshakeRateLimiterMetricOutstandingCount = "tls_handshake_limiter.in_process" // TlsHandshakeRateLimiterMetricCurrentWindowSize is the name of the metric tracking the current window size TlsHandshakeRateLimiterMetricCurrentWindowSize = "tls_handshake_limiter.window_size" // TlsHandshakeRateLimiterMetricWorkTimer is the name of the metric tracking how long successful tasks are taking to complete TlsHandshakeRateLimiterMetricWorkTimer = "tls_handshake_limiter.work_timer" // DefaultTlsHandshakeRateLimiterMaxWindow is the default max size for the tls handshake rate limiter DefaultTlsHandshakeRateLimiterMaxWindow = 1000 ) type Config struct { Id *identity.TokenId Raft *raft.Config Network *network.Options Db boltz.Db Trace struct { Handler *channel.TraceHandler } Profile struct { Memory struct { Path string Interval time.Duration } CPU struct { Path string } } Ctrl struct { Listener transport.Address Options *CtrlOptions } HealthChecks struct { BoltCheck struct { Interval time.Duration Timeout time.Duration InitialDelay time.Duration } } CommandRateLimiter command.RateLimiterConfig TlsHandshakeRateLimiter command.AdaptiveRateLimiterConfig src map[interface{}]interface{} } // CtrlOptions extends channel.Options to include support for additional, non-channel specific options // (e.g. NewListener) type CtrlOptions struct { *channel.Options NewListener *transport.Address AdvertiseAddress *transport.Address RouterHeartbeatOptions *channel.HeartbeatOptions PeerHeartbeatOptions *channel.HeartbeatOptions } func (config *Config) Configure(sub config.Subconfig) error { return sub.LoadConfig(config.src) } func LoadConfig(path string) (*Config, error) { cfgBytes, err := os.ReadFile(path) if err != nil { return nil, err } cfgmap := make(map[interface{}]interface{}) if err = yaml.NewDecoder(bytes.NewReader(cfgBytes)).Decode(&cfgmap); err != nil { return nil, err } config.InjectEnv(cfgmap) if value, found := cfgmap["v"]; found { if value.(int) != 3 { panic("config version mismatch: see docs for information on config updates") } } else { panic("no config version: see docs for information on config") } var identityConfig *identity.Config if value, found := cfgmap["identity"]; found { subMap := value.(map[interface{}]interface{}) identityConfig, err = identity.NewConfigFromMapWithPathContext(subMap, "identity") if err != nil { return nil, fmt.Errorf("could not parse root identity: %v", err) } if identityConfig.ServerCert == "" && identityConfig.ServerKey == "" { identityConfig.ServerCert = identityConfig.Cert identityConfig.ServerKey = identityConfig.Key } } else { return nil, fmt.Errorf("identity section not found") } controllerConfig := &Config{ Network: network.DefaultOptions(), src: cfgmap, } if id, err := identity.LoadIdentity(*identityConfig); err != nil { return nil, fmt.Errorf("unable to load identity (%s)", err) } else { controllerConfig.Id = identity.NewIdentity(id) } if value, found := cfgmap["network"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if options, err := network.LoadOptions(submap); err == nil { controllerConfig.Network = options } else { return nil, fmt.Errorf("invalid 'network' stanza (%s)", err) } } else { pfxlog.Logger().Warn("invalid or empty 'network' stanza") } } if value, found := cfgmap["raft"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { controllerConfig.Raft = &raft.Config{} controllerConfig.Raft.CommandHandlerOptions.MaxQueueSize = DefaultRaftCommandHandlerMaxQueueSize if value, found := submap["dataDir"]; found { controllerConfig.Raft.DataDir = value.(string) } else { return nil, errors.Errorf("raft dataDir configuration missing") } if value, found := submap["minClusterSize"]; found { controllerConfig.Raft.MinClusterSize = uint32(value.(int)) } if value, found := submap["bootstrapMembers"]; found { if lst, ok := value.([]interface{}); ok { for idx, val := range lst { if member, ok := val.(string); ok { controllerConfig.Raft.BootstrapMembers = append(controllerConfig.Raft.BootstrapMembers, member) } else { return nil, errors.Errorf("invalid bootstrapMembers value '%v'at index %v, should be array", idx, val) } } } else { return nil, errors.New("invalid bootstrapMembers value, should be array") } } if value, found := submap["snapshotInterval"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.Raft.SnapshotInterval = &val } else { return nil, errors.Wrapf(err, "failed to parse raft.snapshotInterval value '%v", value) } } if value, found := submap["commitTimeout"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.Raft.CommitTimeout = &val } else { return nil, errors.Wrapf(err, "failed to parse raft.commitTimeout value '%v", value) } } if value, found := submap["electionTimeout"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.Raft.ElectionTimeout = &val } else { return nil, errors.Wrapf(err, "failed to parse raft.electionTimeout value '%v", value) } } if value, found := submap["heartbeatTimeout"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.Raft.HeartbeatTimeout = &val } else { return nil, errors.Wrapf(err, "failed to parse raft.heartbeatTimeout value '%v", value) } } if value, found := submap["leaderLeaseTimeout"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.Raft.LeaderLeaseTimeout = &val } else { return nil, errors.Wrapf(err, "failed to parse raft.leaderLeaseTimeout value '%v", value) } } if value, found := submap["snapshotThreshold"]; found { val := uint32(value.(int)) controllerConfig.Raft.SnapshotThreshold = &val } if value, found := submap["maxAppendEntries"]; found { val := uint32(value.(int)) controllerConfig.Raft.MaxAppendEntries = &val } if value, found := submap["trailingLogs"]; found { val := uint32(value.(int)) controllerConfig.Raft.TrailingLogs = &val } if value, found := submap["logLevel"]; found { val := fmt.Sprintf("%v", value) if hclog.LevelFromString(val) == hclog.NoLevel { return nil, errors.Errorf("invalid value for raft.logLevel [%v]", val) } controllerConfig.Raft.LogLevel = &val } if value, found := submap["logFile"]; found { val := fmt.Sprintf("%v", value) options := *hclog.DefaultOptions f, err := os.Create(val) if err != nil { return nil, errors.Wrapf(err, "unable to open raft log file [%v]", val) } options.Output = f if controllerConfig.Raft.LogLevel != nil { options.Level = hclog.LevelFromString(*controllerConfig.Raft.LogLevel) } controllerConfig.Raft.Logger = hclog.New(&options) } if value, found := cfgmap["commandHandler"]; found { if chSubMap, ok := value.(map[interface{}]interface{}); ok { if value, found := chSubMap["maxQueueSize"]; found { controllerConfig.Raft.CommandHandlerOptions.MaxQueueSize = uint16(value.(int)) } } else { return nil, errors.New("invalid commandHandler value, should be map") } } } else { return nil, errors.Errorf("invalid raft configuration") } } else if value, found := cfgmap["db"]; found { str, err := db.Open(value.(string)) if err != nil { return nil, err } controllerConfig.Db = str } else { panic("controllerConfig must provide [db] or [raft]") } if value, found := cfgmap["trace"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if value, found := submap["path"]; found { handler, err := channel.NewTraceHandler(value.(string), controllerConfig.Id.Token) if err != nil { return nil, err } handler.AddDecoder(&channel.Decoder{}) handler.AddDecoder(&ctrl_pb.Decoder{}) handler.AddDecoder(&xgress.Decoder{}) handler.AddDecoder(&mgmt_pb.Decoder{}) controllerConfig.Trace.Handler = handler } } } if value, found := cfgmap["profile"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if value, found := submap["memory"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if value, found := submap["path"]; found { controllerConfig.Profile.Memory.Path = value.(string) } if value, found := submap["intervalMs"]; found { controllerConfig.Profile.Memory.Interval = time.Duration(value.(int)) * time.Millisecond } else { controllerConfig.Profile.Memory.Interval = DefaultProfileMemoryInterval } } } if value, found := submap["cpu"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if value, found := submap["path"]; found { controllerConfig.Profile.CPU.Path = value.(string) } } } } } if value, found := cfgmap["ctrl"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if value, found := submap["listener"]; found { listener, err := transport.ParseAddress(value.(string)) if err != nil { return nil, err } controllerConfig.Ctrl.Listener = listener } else { panic("controllerConfig must provide [ctrl/listener]") } controllerConfig.Ctrl.Options = &CtrlOptions{ Options: channel.DefaultOptions(), PeerHeartbeatOptions: channel.DefaultHeartbeatOptions(), RouterHeartbeatOptions: channel.DefaultHeartbeatOptions(), } if value, found := submap["options"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { options, err := channel.LoadOptions(submap) if err != nil { return nil, err } controllerConfig.Ctrl.Options.Options = options if val, found := submap["newListener"]; found { if newListener, ok := val.(string); ok { if newListener != "" { if addr, err := transport.ParseAddress(newListener); err == nil { controllerConfig.Ctrl.Options.NewListener = &addr if err := verifyNewListenerInServerCert(controllerConfig, addr); err != nil { return nil, err } } else { return nil, fmt.Errorf("error loading newListener for [ctrl/options] (%v)", err) } } } else { return nil, errors.New("error loading newAddress for [ctrl/options] (must be a string)") } } if val, found := submap["advertiseAddress"]; found { if advertiseAddr, ok := val.(string); ok { if advertiseAddr != "" { addr, err := transport.ParseAddress(advertiseAddr) if err != nil { return nil, errors.Wrapf(err, "error parsing value '%v' for [ctrl/options/advertiseAddress]", advertiseAddr) } controllerConfig.Ctrl.Options.AdvertiseAddress = &addr if controllerConfig.Raft != nil { controllerConfig.Raft.AdvertiseAddress = addr } } } else { return nil, errors.New("error loading advertiseAddress for [ctrl/options] (must be a string)") } } if value, found := submap["routerHeartbeats"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { options, err := channel.LoadHeartbeatOptions(submap) if err != nil { return nil, err } controllerConfig.Ctrl.Options.RouterHeartbeatOptions = options } } if value, found := submap["peerHeartbeats"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { options, err := channel.LoadHeartbeatOptions(submap) if err != nil { return nil, err } controllerConfig.Ctrl.Options.PeerHeartbeatOptions = options } } if err := controllerConfig.Ctrl.Options.Validate(); err != nil { return nil, fmt.Errorf("error loading channel options for [ctrl/options] (%v)", err) } } } if controllerConfig.Raft != nil && controllerConfig.Raft.AdvertiseAddress == nil { return nil, errors.New("[ctrl/options/advertiseAddress] is required when raft is enabled") } } else { panic("controllerConfig [ctrl] section in unexpected format") } } else { panic("controllerConfig must provide [ctrl]") } controllerConfig.HealthChecks.BoltCheck.Interval = DefaultHealthChecksBoltCheckInterval controllerConfig.HealthChecks.BoltCheck.Timeout = DefaultHealthChecksBoltCheckTimeout controllerConfig.HealthChecks.BoltCheck.InitialDelay = DefaultHealthChecksBoltCheckInitialDelay if value, found := cfgmap["healthChecks"]; found { if healthChecksMap, ok := value.(map[interface{}]interface{}); ok { if value, found := healthChecksMap["boltCheck"]; found { if boltMap, ok := value.(map[interface{}]interface{}); ok { if value, found := boltMap["interval"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.HealthChecks.BoltCheck.Interval = val } else { return nil, errors.Wrapf(err, "failed to parse healthChecks.bolt.interval value '%v", value) } } if value, found := boltMap["timeout"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.HealthChecks.BoltCheck.Timeout = val } else { return nil, errors.Wrapf(err, "failed to parse healthChecks.bolt.timeout value '%v", value) } } if value, found := boltMap["initialDelay"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { controllerConfig.HealthChecks.BoltCheck.InitialDelay = val } else { return nil, errors.Wrapf(err, "failed to parse healthChecks.bolt.initialDelay value '%v", value) } } } else { pfxlog.Logger().Warn("invalid [healthChecks.bolt] stanza") } } } else { pfxlog.Logger().Warn("invalid [healthChecks] stanza") } } controllerConfig.CommandRateLimiter.Enabled = true controllerConfig.CommandRateLimiter.QueueSize = command.DefaultLimiterSize if value, found := cfgmap["commandRateLimiter"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if value, found := submap["enabled"]; found { controllerConfig.CommandRateLimiter.Enabled = strings.EqualFold("true", fmt.Sprintf("%v", value)) } if value, found := submap["maxQueued"]; found { if intVal, ok := value.(int); ok { v := int64(intVal) if v < command.MinLimiterSize { return nil, errors.Errorf("invalid value %v for commandRateLimiter, must be at least %v", value, command.MinLimiterSize) } if v > math.MaxUint32 { return nil, errors.Errorf("invalid value %v for commandRateLimiter, must be at most %v", value, int64(math.MaxUint32)) } controllerConfig.CommandRateLimiter.QueueSize = uint32(v) } else { return nil, errors.Errorf("invalid value %v for commandRateLimiter, must be integer value", value) } } } } controllerConfig.TlsHandshakeRateLimiter.SetDefaults() controllerConfig.TlsHandshakeRateLimiter.Enabled = DefaultTlsHandshakeRateLimiterEnabled controllerConfig.TlsHandshakeRateLimiter.MaxSize = DefaultTlsHandshakeRateLimiterMaxWindow controllerConfig.TlsHandshakeRateLimiter.QueueSizeMetric = TlsHandshakeRateLimiterMetricOutstandingCount controllerConfig.TlsHandshakeRateLimiter.WindowSizeMetric = TlsHandshakeRateLimiterMetricCurrentWindowSize controllerConfig.TlsHandshakeRateLimiter.WorkTimerMetric = TlsHandshakeRateLimiterMetricWorkTimer if value, found := cfgmap["tls"]; found { if tlsMap, ok := value.(map[interface{}]interface{}); ok { if value, found := tlsMap["handshakeTimeout"]; found { if val, err := time.ParseDuration(fmt.Sprintf("%v", value)); err == nil { transporttls.SetSharedListenerHandshakeTimeout(val) } else { return nil, errors.Wrapf(err, "failed to parse tls.handshakeTimeout value '%v", value) } } if err = loadTlsHandshakeRateLimiterConfig(&controllerConfig.TlsHandshakeRateLimiter, tlsMap); err != nil { return nil, err } } } return controllerConfig, nil } func loadTlsHandshakeRateLimiterConfig(rateLimitConfig *command.AdaptiveRateLimiterConfig, cfgmap map[interface{}]interface{}) error { if value, found := cfgmap["rateLimiter"]; found { if submap, ok := value.(map[interface{}]interface{}); ok { if err := command.LoadAdaptiveRateLimiterConfig(rateLimitConfig, submap); err != nil { return err } if rateLimitConfig.MaxSize < TlsHandshakeRateLimiterMinSizeValue { return errors.Errorf("invalid value %v for tls.rateLimiter.maxSize, must be at least %v", rateLimitConfig.MaxSize, TlsHandshakeRateLimiterMinSizeValue) } if rateLimitConfig.MaxSize > TlsHandshakeRateLimiterMaxSizeValue { return errors.Errorf("invalid value %v for tls.rateLimiter.maxSize, must be at most %v", rateLimitConfig.MaxSize, TlsHandshakeRateLimiterMaxSizeValue) } if rateLimitConfig.MinSize < TlsHandshakeRateLimiterMinSizeValue { return errors.Errorf("invalid value %v for tls.rateLimiter.minSize, must be at least %v", rateLimitConfig.MinSize, TlsHandshakeRateLimiterMinSizeValue) } if rateLimitConfig.MinSize > TlsHandshakeRateLimiterMaxSizeValue { return errors.Errorf("invalid value %v for tls.rateLimiter.minSize, must be at most %v", rateLimitConfig.MinSize, TlsHandshakeRateLimiterMaxSizeValue) } } else { return errors.Errorf("invalid type for tls.rateLimiter, should be map instead of %T", value) } } return nil } // verifyNewListenerInServerCert verifies that the hostname (ip/dns) for addr is present as an IP/DNS SAN in the first // certificate provided in the controller's identity server certificates. This is to avoid scenarios where // newListener propagated to routers who will never be able to verify the controller's certificates due to SAN issues. func verifyNewListenerInServerCert(controllerConfig *Config, addr transport.Address) error { addrSplits := strings.Split(addr.String(), ":") if len(addrSplits) < 3 { return errors.New("could not determine newListener's host value, expected at least three segments") } host := addrSplits[1] serverCerts := controllerConfig.Id.Identity.ServerCert() if len(serverCerts) == 0 { return errors.New("could not verify newListener value, server certificate for identity contains no certificates") } hostFound := false for _, serverCert := range serverCerts { for _, dnsName := range serverCert.Leaf.DNSNames { if dnsName == host { hostFound = true break } } if hostFound { break } if !hostFound { for _, ipAddresses := range serverCert.Leaf.IPAddresses { if host == ipAddresses.String() { hostFound = true break } } } if hostFound { break } } if !hostFound { return fmt.Errorf("could not find newListener [%s] host value [%s] in first certificate for controller identity", addr.String(), host) } return nil } type CertValidatingIdentity struct { identity.Identity } func (self *CertValidatingIdentity) ClientTLSConfig() *tls.Config { cfg := self.Identity.ClientTLSConfig() cfg.VerifyConnection = self.VerifyConnection return cfg } func (self *CertValidatingIdentity) ServerTLSConfig() *tls.Config { cfg := self.Identity.ServerTLSConfig() cfg.VerifyConnection = self.VerifyConnection return cfg } func (self *CertValidatingIdentity) VerifyConnection(state tls.ConnectionState) error { if len(state.PeerCertificates) == 0 { return errors.New("no peer certificates provided") } log := pfxlog.Logger() for _, cert := range state.PeerCertificates { log.Infof("cert provided: CN: %v IsCA: %v", cert.Subject.CommonName, cert.IsCA) } options := x509.VerifyOptions{ Roots: self.Identity.CA(), Intermediates: x509.NewCertPool(), } for _, cert := range state.PeerCertificates[1:] { options.Intermediates.AddCert(cert) } result, err := state.PeerCertificates[0].Verify(options) if err != nil { pfxlog.Logger().WithError(err).Error("got error validating cert") return err } log.Infof("got result: %v", result) return nil }