diff --git a/CHANGELOG.md b/CHANGELOG.md index edc68fa7d..cf748b743 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,53 @@ -# Release 0.23.2 +# Release 0.24.0 + +## Breaking Changes + +* ziti-fabric-gw has been removed since the fabric now has its own REST API +* ziti-fabric-test is no longer being built by default and won't be included in future release bundles. Use `go build --tags all ./...` to build it +* ziti-fabric has been deprecated. Most of its features are now available in the `ziti` CLI under `ziti fabric` ## What's New +* Feature: Fabric REST API +* Performance: Additional route selection work +* Bug Fix: Fix controller deadlock which can happen if a control channel is closed while controller is responding * Bug fix: Fix panic for UDP-only tproxy intercepts +## Fabric REST API + +The fabric now has a REST API in addition to the channel2 management API. To enable it, add the fabric binding to the apis section off the xweb config, as follows: + +``` + apis: + # binding - required + # Specifies an API to bind to this webListener. Built-in APIs are + # - health-checks + - binding: fabric +``` + +If running without the edge, the fabric API uses client certificates for authorization, much like the existing channel2 mgmt based API does. If running with the edge, the edge provides authentication/authorization for the fabric REST APIs. + +### Supported Operations + +These operations are supported in the REST API. The ziti CLI has been updated to use this in the new `ziti fabric` sub-command. + +* Services: create/read/update/delete +* Routers: create/read/update/delete +* Terminators: create/read/update/delete +* Links: read/update +* Circuits: read/delete + +### Unsupported Operations + +Some operations from ziti-fabric aren't get supported: + +* Stream metrics/traces/circuits + * This feature may be re-implemented in terms of websockets, or may be left as-is, or may be dropped +* Inspect (get stackdumps) + * This will be ported to `ziti fabric` +* Decode trace files + * This may be ported to `ziti-ops` + # Release 0.23.1 ## What's New diff --git a/go.mod b/go.mod index c2158edd9..c60c8b477 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ replace go.etcd.io/bbolt => github.com/openziti/bbolt v1.3.6-0.20210317142109-54 require ( github.com/Jeffail/gabs v1.4.0 + github.com/Jeffail/gabs/v2 v2.6.1 github.com/MakeNowJust/heredoc v1.0.0 github.com/MichaelMure/go-term-markdown v0.1.4 github.com/alecthomas/chroma v0.9.2 @@ -30,7 +31,6 @@ require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.6 - github.com/gorilla/mux v1.8.0 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19 github.com/michaelquigley/pfxlog v0.6.3 @@ -41,7 +41,6 @@ require ( github.com/pborman/uuid v1.2.0 github.com/pkg/errors v0.9.1 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 - github.com/rs/cors v1.7.0 github.com/russross/blackfriday v1.5.2 github.com/shirou/gopsutil v2.20.9+incompatible github.com/sirupsen/logrus v1.8.1 @@ -49,7 +48,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 - github.com/urfave/negroni v1.0.0 github.com/valyala/fasttemplate v1.2.1 go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 golang.org/x/net v0.0.0-20211029224645-99673261e6eb @@ -98,6 +96,7 @@ require ( github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/go.sum b/go.sum index 56e7930b7..9f95042be 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= +github.com/Jeffail/gabs/v2 v2.6.1 h1:wwbE6nTQTwIMsMxzi6XFQQYRZ6wDc1mSdxoAN+9U4Gk= github.com/Jeffail/gabs/v2 v2.6.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -841,8 +842,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -944,8 +943,6 @@ github.com/transip/gotransip/v6 v6.2.0/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= diff --git a/ziti-fabric/main.go b/ziti-fabric/main.go index 2f0c1f027..f64783cac 100644 --- a/ziti-fabric/main.go +++ b/ziti-fabric/main.go @@ -17,6 +17,7 @@ package main import ( + "fmt" "github.com/michaelquigley/pfxlog" "github.com/openziti/foundation/transport" "github.com/openziti/foundation/transport/quic" @@ -41,5 +42,6 @@ func init() { } func main() { + fmt.Println("DEPRECATION WARNING: ziti-fabric is deprecated and will be removed in a future release. Use ziti fabric instead.") subcmd.Execute() } diff --git a/ziti-ops/logs/common.go b/ziti-ops/logs/common.go new file mode 100644 index 000000000..68d56779a --- /dev/null +++ b/ziti-ops/logs/common.go @@ -0,0 +1,275 @@ +package logs + +import ( + "bufio" + "fmt" + "github.com/Jeffail/gabs/v2" + "github.com/pkg/errors" + "os" + "sort" + "strings" + "time" +) + +type ParseContext struct { + path string + lineNumber int + eof bool + line string +} + +func ScanLines(ctx *ParseContext, callback func(ctx *ParseContext) error) error { + file, err := os.Open(ctx.path) + if err != nil { + return err + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + ctx.line = line + if err := callback(ctx); err != nil { + return errors.Wrapf(err, "error parsing %v on line %v", ctx.path, ctx.lineNumber) + } + ctx.lineNumber++ + } + ctx.eof = true + return callback(ctx) +} + +type JsonParseContext struct { + ParseContext + entry *gabs.Container +} + +func (self *JsonParseContext) RequiredString(path ...string) (string, error) { + v := self.entry.Search(path...) + if v == nil || v.Data() == nil { + return "", errors.Errorf("%v not found in log entry", strings.Join(path, ".")) + } + s, ok := v.Data().(string) + if !ok { + return "", errors.Errorf("%v is not a string", strings.Join(path, ".")) + } + return s, nil +} + +func (self *JsonParseContext) ParseJsonEntry() error { + input := self.line + idx := strings.IndexByte(self.line, '{') + if idx < 0 { + fmt.Printf("WARN: non-json line: %v", self.line) + return nil + } + if idx > 0 { + input = input[idx:] + } + entry, err := gabs.ParseJSON([]byte(input)) + if err != nil { + return err + } + self.entry = entry + return nil +} + +func ScanJsonLines(ctx *JsonParseContext, callback func(ctx *JsonParseContext) error) error { + return ScanLines(&ctx.ParseContext, func(*ParseContext) error { + if ctx.eof { + return callback(ctx) + } + if err := ctx.ParseJsonEntry(); err != nil { + return err + } + if ctx.entry != nil { + return callback(ctx) + } + return nil + }) +} + +type LogMatcher interface { + Matches(ctx *JsonParseContext) (bool, error) +} + +type LogFilter interface { + LogMatcher + Id() string + Label() string + Desc() string +} + +type filter struct { + LogMatcher + id string + label string + desc string +} + +func (self *filter) Id() string { + return self.id +} + +func (self *filter) Label() string { + return self.label +} + +func (self *filter) Desc() string { + return self.desc +} + +type JsonLogsParser struct { + bucketSize time.Duration + currentBucket time.Time + filters []LogFilter + bucketMatches map[LogFilter]int + unmatched int +} + +func (self *JsonLogsParser) examineLogEntry(ctx *JsonParseContext) error { + if ctx.eof { + self.dumpBucket() + return nil + } + if err := self.bucket(ctx); err != nil { + return err + } + + for _, filter := range self.filters { + match, err := filter.Matches(ctx) + if err != nil { + return err + } + + if match { + current := self.bucketMatches[filter] + self.bucketMatches[filter] = current + 1 + return nil + } + } + + self.unmatched++ + if self.unmatched == 1 { + fmt.Printf("WARN: unmatched line: %v\n\n", ctx.line) + } + + return nil +} + +func (self *JsonLogsParser) bucket(ctx *JsonParseContext) error { + s, err := ctx.RequiredString("time") + if err != nil { + return err + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return errors.Errorf("time is in an unexpected format: %v", s) + } + interval := t.Truncate(self.bucketSize) + if interval != self.currentBucket { + if !self.currentBucket.IsZero() { + self.dumpBucket() + } + self.currentBucket = interval + self.bucketMatches = map[LogFilter]int{} + self.unmatched = 0 + } + return nil +} + +func (self *JsonLogsParser) dumpBucket() { + if len(self.bucketMatches) == 0 { + return + } + var filters []LogFilter + for k := range self.bucketMatches { + filters = append(filters, k) + } + sort.Slice(filters, func(i, j int) bool { + return filters[i].Label() < filters[j].Label() + }) + fmt.Printf("%v\n---------------------------------------------------\n", self.currentBucket.Format(time.RFC3339)) + for _, filter := range filters { + fmt.Printf(" %v (%v): %0000v\n", filter.Id(), filter.Label(), self.bucketMatches[filter]) + } + if self.unmatched > 0 { + fmt.Printf(" unmatched: %0000v\n", self.unmatched) + } + fmt.Println() +} + +func FieldStartsWith(field, substring string) LogMatcher { + return &EntryFieldStartsWithMatcher{ + field: field, + prefix: substring, + } +} + +type EntryFieldStartsWithMatcher struct { + field string + prefix string +} + +func (self *EntryFieldStartsWithMatcher) Matches(ctx *JsonParseContext) (bool, error) { + fieldValue, err := ctx.RequiredString(self.field) + if err != nil { + return false, err + } + return strings.HasPrefix(fieldValue, self.prefix), nil +} + +func FieldContains(field, substring string) LogMatcher { + return &EntryFieldContainsMatcher{ + field: field, + substring: substring, + } +} + +type EntryFieldContainsMatcher struct { + field string + substring string +} + +func (self *EntryFieldContainsMatcher) Matches(ctx *JsonParseContext) (bool, error) { + fieldValue, err := ctx.RequiredString(self.field) + if err != nil { + return false, err + } + return strings.Contains(fieldValue, self.substring), nil +} + +func FieldEquals(field, substring string) LogMatcher { + return &EntryFieldContainsMatcher{ + field: field, + substring: substring, + } +} + +type EntryFieldEqualsMatcher struct { + field string + value string +} + +func (self *EntryFieldEqualsMatcher) Matches(ctx *JsonParseContext) (bool, error) { + fieldValue, err := ctx.RequiredString(self.field) + if err != nil { + return false, err + } + return fieldValue == self.value, nil +} + +func AndMatchers(matchers ...LogMatcher) LogMatcher { + return &AndMatcher{matchers: matchers} +} + +type AndMatcher struct { + matchers []LogMatcher +} + +func (self *AndMatcher) Matches(ctx *JsonParseContext) (bool, error) { + for _, matcher := range self.matchers { + result, err := matcher.Matches(ctx) + if !result || err != nil { + return result, err + } + } + return true, nil +} diff --git a/ziti-ops/logs/router_logs.go b/ziti-ops/logs/router_logs.go new file mode 100644 index 000000000..bb5a10151 --- /dev/null +++ b/ziti-ops/logs/router_logs.go @@ -0,0 +1,128 @@ +package logs + +import ( + "github.com/spf13/cobra" + "time" +) + +func NewLogsCommand() *cobra.Command { + logsCmd := &cobra.Command{ + Use: "logs", + Short: "commands related to analyzing various ziti component logs", + } + + parseRouterLogs := &ParseRouterLogs{} + parseRouterLogs.Init() + + parseRouterLogsCmd := &cobra.Command{ + Use: "router", + Short: "Parse router logs", + Args: cobra.ExactArgs(1), + RunE: parseRouterLogs.run, + } + parseRouterLogsCmd.Flags().DurationVarP(&parseRouterLogs.bucketSize, "interval", "n", time.Hour, "Interval for which to aggregate log messages") + + logsCmd.AddCommand(parseRouterLogsCmd) + + return logsCmd +} + +type ParseRouterLogs struct { + JsonLogsParser +} + +func (self *ParseRouterLogs) Init() { + self.filters = append(self.filters, + &filter{ + id: "IDLE", + label: "idle circuit scanner", + desc: "a circuit has been idle for at least one minute and the controller will be checked to see if the circuit is still valid", + LogMatcher: AndMatchers( + FieldContains("file", "forwarder/scanner.go"), + FieldContains("msg", " idle after "), + )}, + &filter{ + id: "CONF", + label: "circuit confirmation send", + desc: "controller has been notified of idle circuits and can respond if they are no longer valid", + LogMatcher: AndMatchers( + FieldContains("msg", "sent confirmation for "), + FieldContains("file", "forwarder/scanner.go"), + )}, + &filter{ + id: "TSCX", + label: "tunnel dial success", + desc: "a router embedded tunneler has made a successful connection to a server hosting an application", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "successful connection "), + FieldContains("file", "xgress_edge_tunnel/dialer.go"), + )}, + &filter{ + id: "DSTX", + label: "routing destination exists", + desc: "attempting to establish a circuit, but a previous dial attempt already established the egress connection", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "destination exists for "), + FieldContains("file", "handler_ctrl/route.go"), + )}, + &filter{ + id: "XGRF", + label: "xgress read failure", + desc: "read failure on the client or server side of the circuit; will cause the circuit to be torn down", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "read failed"), + FieldContains("file", "xgress/xgress.go"), + )}, + &filter{ + id: "XGWF", + label: "xgress write failure", + desc: "write failure on the client or server side of the circuit; will cause the circuit to be torn down", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "write failed"), + FieldContains("file", "xgress/xgress.go"), + )}, + &filter{ + id: "XGFB", + label: "xgress can't buffer", + desc: "while a circuit was being closed, data was received from the client or server", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "failure to buffer payload"), + FieldEquals("error", "payload buffer closed"), + FieldContains("file", "xgress/xgress.go"), + )}, + &filter{ + id: "CSTO", + label: "circuit not started in time", + desc: "the terminating side of the circuit didn't receive a start from the initiating side", + LogMatcher: AndMatchers( + FieldEquals("msg", "xgress circuit not started in time, closing"), + FieldContains("file", "xgress/xgress.go"), + )}, + &filter{ + id: "TLSC", + label: "no tls certification provided", + desc: "a connection attempt was made to a TLS listener but no certificate was provided", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "error receiving hello from "), + FieldContains("msg", "tls: client didn't provide a certificate"), + FieldContains("file", "channel2/classic_listener.go"), + )}, + &filter{ + id: "LCYT", + label: "latency timeout", + desc: "a latency ping was sent, but no response was received within the timeout", + LogMatcher: AndMatchers( + FieldStartsWith("msg", "latency timeout after "), + FieldContains("file", "metrics/latency.go"), + )}, + ) +} + +func (self *ParseRouterLogs) run(_ *cobra.Command, args []string) error { + ctx := &JsonParseContext{ + ParseContext: ParseContext{ + path: args[0], + }, + } + return ScanJsonLines(ctx, self.examineLogEntry) +} diff --git a/ziti-ops/main.go b/ziti-ops/main.go new file mode 100644 index 000000000..761043245 --- /dev/null +++ b/ziti-ops/main.go @@ -0,0 +1,78 @@ +//go:build all + +/* + Copyright NetFoundry, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/fabric/build" + "github.com/openziti/ziti/common/version" + "github.com/openziti/ziti/ziti-ops/logs" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var verbose bool +var logFormatter string + +func init() { + options := pfxlog.DefaultOptions().SetTrimPrefix("github.com/openziti/").NoColor() + pfxlog.GlobalInit(logrus.InfoLevel, options) + build.InitBuildInfo(version.GetCmdBuildInfo()) + + root.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging") + root.PersistentFlags().StringVar(&logFormatter, "log-formatter", "", "Specify log formatter [json|pfxlog|text]") + + root.AddCommand(&cobra.Command{ + Use: "version", + Short: "Show component version", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(version.GetBuildMetadata(verbose)) + }, + }) + + root.AddCommand(logs.NewLogsCommand()) +} + +var root = &cobra.Command{ + Use: "ziti-ops", + Short: "Ziti Ops Tools", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + if verbose { + logrus.SetLevel(logrus.DebugLevel) + } + + switch logFormatter { + case "pfxlog": + pfxlog.SetFormatter(pfxlog.NewFormatter(pfxlog.DefaultOptions().SetTrimPrefix("github.com/openziti/").StartingToday())) + case "json": + pfxlog.SetFormatter(&logrus.JSONFormatter{TimestampFormat: "2006-01-02T15:04:05.000Z"}) + case "text": + pfxlog.SetFormatter(&logrus.TextFormatter{}) + default: + // let logrus do its own thing + } + }, +} + +func main() { + if err := root.Execute(); err != nil { + fmt.Printf("error: %s\n", err) + } +} diff --git a/ziti-router/subcmd/run.go b/ziti-router/subcmd/run.go index e38417e9e..bc8ce3594 100644 --- a/ziti-router/subcmd/run.go +++ b/ziti-router/subcmd/run.go @@ -135,6 +135,6 @@ func waitForShutdown(r *router.Router, config *router.Config) { pfxlog.Logger().Info("shutting down ziti-router") if err := r.Shutdown(); err != nil { - pfxlog.Logger().Info("error encountered during shutdown: %v", err) + pfxlog.Logger().WithError(err).Info("error encountered during shutdown") } } diff --git a/ziti/cmd/ziti/cmd/lets_encrypt/revoke.go b/ziti/cmd/ziti/cmd/lets_encrypt/revoke.go index c257d41ff..8be0d9fe4 100644 --- a/ziti/cmd/ziti/cmd/lets_encrypt/revoke.go +++ b/ziti/cmd/ziti/cmd/lets_encrypt/revoke.go @@ -93,7 +93,7 @@ func runRevoke(options *leOptions) (err error) { return err } - log.Infof("Certificate was archived for domain:", options.domain) + log.Infof("Certificate was archived for domain: %v", options.domain) return nil