From c9365b834060ad4a4810603011096e493108d913 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Wed, 29 Jul 2020 15:07:13 -0400 Subject: [PATCH] Add db integrity checks. Fixes GH-163 --- doc/release-notes.md | 32 ++++++++ go.mod | 6 +- go.sum | 8 +- ziti/cmd/ziti/cmd/edge_controller/db.go | 25 ++++++ .../cmd/edge_controller/db_check_integrity.go | 79 +++++++++++++++++++ .../{create_snapshot.go => db_snapshot.go} | 12 ++- ziti/cmd/ziti/cmd/edge_controller/root.go | 2 +- 7 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 ziti/cmd/ziti/cmd/edge_controller/db.go create mode 100644 ziti/cmd/ziti/cmd/edge_controller/db_check_integrity.go rename ziti/cmd/ziti/cmd/edge_controller/{create_snapshot.go => db_snapshot.go} (81%) diff --git a/doc/release-notes.md b/doc/release-notes.md index 13f1fad8b..5dbcc4344 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -6,11 +6,20 @@ * [e2e Service Configuration & Router Termination](https://github.com/openziti/edge/issues/173) * Router Scaling Issues * [Add worker pools for link and xgress dials](https://github.com/openziti/fabric/issues/109) +* Model Performance Improvements + * [Denormalize policy links for performance](https://github.com/openziti/edge/issues/256) +* Datastore Integrity Checker + * [foundation#107](https://github.com/openziti/foundation/issues/107) + * [edge#258](https://github.com/openziti/edge/issues/258) + * [#163](https://github.com/openziti/ziti/issues/163) * Bug Fixes: * [#152](https://github.com/openziti/ziti/issues/152) - Fix ziti-router enroll exit code on failure * [#156](https://github.com/openziti/ziti/issues/156) - fix display of policies with empty roles lists +* Backwards Compatibility + * The `ziti edge snapshot-db` command is now `ziti edge db snapshot` + ## End-To-End Encryption Enhancements ### E2E Encryption Router Termination A new xgress module has been added specifically for handling Ziti Edge e2e to handle SDK to Router Termination scenarios. @@ -102,6 +111,29 @@ forwarder: linkDialWorkerCount: 10 ``` +## Model Performance Improvements +Policy relationships are now stored in a denormalized fashion. This means that checking if an entity is tied to another entity via a policy is now a direct lookup, and much faster. This means that the Ziti controller should scale very well in cases where we have many identities, services and/or edge routers. Performance was tested against the APIs used by the SDKs. + +See for more detail: + +* [Denormalized Policies](https://github.com/openziti/edge/wiki/Denormalized-Policies) +* [Characterization (Pure Model Tests)](https://github.com/openziti/ziti/wiki/Characterization#pure-model-tests) + +## Data Integrity Checking Framework +The bbolt datastore used by Ziti provides simple key/value storage with nesting. Ziti has implemented some basic relational constructs on top of bbolt, such as indexed values, foreign key indexes, many to many collections and reference counted collections (for policy denormalization). This release adds a data integrity checking framework which allows us to verify that constraint assumptions are valid, and allows fixing issues when they are found (if possible). This work was done in part to validate that the policy denormalization code is working correctly, and to provide a rememdy if issues are found. + +There are two new REST APIs available + +* GET `/database/check-data-integrity` + * https://github.com/openziti/edge/blob/master/specs/swagger.yml#L2916 +* POST `/database/fix-data-integrity` + * https://github.com/openziti/edge/blob/master/specs/swagger.yml#L2930 + +These APIs can be used from the ziti CLI. + +* `ziti edge db check-integrity` - to report on data integrity issues +* `ziti edge db check-integrity -f` - to report on data integrity issues and attempt to fix any that are found + # Release 0.15.2 * What's New: diff --git a/go.mod b/go.mod index b09ab6a27..266319c69 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,9 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19 github.com/michaelquigley/pfxlog v0.0.0-20190813191113-2be43bd0dccc - github.com/openziti/edge v0.15.31 - github.com/openziti/fabric v0.11.60 - github.com/openziti/foundation v0.11.18 + github.com/openziti/edge v0.15.32 + github.com/openziti/fabric v0.11.61 + github.com/openziti/foundation v0.11.19 github.com/openziti/sdk-golang v0.13.27 github.com/pborman/uuid v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 0e339f794..ab5622e61 100644 --- a/go.sum +++ b/go.sum @@ -388,13 +388,17 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/openziti/edge v0.15.31 h1:3uXUplVbbzO5O9dex+X1nfA2gi5Tnwvkn2ASSf+H5Iw= -github.com/openziti/edge v0.15.31/go.mod h1:bd7ImcHOkAAtbDB7t05vYHLeV9uO10EePC2qvUm5kuo= +github.com/openziti/edge v0.15.32 h1:7QDWtjp9DPCGS3xkkV7y+se8sm9/zzGV8MxSCBefTBk= +github.com/openziti/edge v0.15.32/go.mod h1:IaslRUgJGfkphC6+jVuKbLCnhB9Xcob8HxQ/hOA0HME= github.com/openziti/fabric v0.11.60 h1:Hr/fyvcjByhFN3gvohHwyaDYGFPbisWDcTQJqOKqJts= github.com/openziti/fabric v0.11.60/go.mod h1:fvB4uE9Ht1CIAads1wv6HOQqrbFcSeAUeEiiCMXco7g= +github.com/openziti/fabric v0.11.61 h1:yKSVejRjbgfQ+KaMh9gpPYRvFAP0+ZkO/LU62xUh/dw= +github.com/openziti/fabric v0.11.61/go.mod h1:m0wfc70b7f3x7xj4zs+WtJzXsPjraCKy8NwdKCk21js= github.com/openziti/foundation v0.11.16/go.mod h1:4Jgs3HGDiP1aVk5nNzXJqwJwWhrjOYy1F+4Xtxcgxvo= github.com/openziti/foundation v0.11.18 h1:DhUKJAqon7DNL/fWpqogr4D7abHLPpZdfOh+YzRUL7M= github.com/openziti/foundation v0.11.18/go.mod h1:qYViHuLb9+F2Sf8Fp82hvx+oVyP3nWV2czTN/n+zHCc= +github.com/openziti/foundation v0.11.19 h1:yKhkIK8Pwv/BVXTcEu0SBrGB6LknuTWKpiLflSOaqKo= +github.com/openziti/foundation v0.11.19/go.mod h1:qYViHuLb9+F2Sf8Fp82hvx+oVyP3nWV2czTN/n+zHCc= github.com/openziti/sdk-golang v0.13.27 h1:J0LSHHplqZiJLI6f22bo40+94n3k4QKQHSjvBM9DTX8= github.com/openziti/sdk-golang v0.13.27/go.mod h1:leIaRUZzYSt+AHnwXlzH/XSX0hOhcjjJ/rMDTkieV/8= github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw= diff --git a/ziti/cmd/ziti/cmd/edge_controller/db.go b/ziti/cmd/ziti/cmd/edge_controller/db.go new file mode 100644 index 000000000..3f9b20d48 --- /dev/null +++ b/ziti/cmd/ziti/cmd/edge_controller/db.go @@ -0,0 +1,25 @@ +package edge_controller + +import ( + cmdutil "github.com/openziti/ziti/ziti/cmd/ziti/cmd/factory" + cmdhelper "github.com/openziti/ziti/ziti/cmd/ziti/cmd/helpers" + "github.com/spf13/cobra" + "io" +) + +// newListCmd creates a command object for the "controller list" command +func newDbCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "db", + Short: "Database management operations for the Ziti Edge Controller", + Run: func(cmd *cobra.Command, args []string) { + err := cmd.Help() + cmdhelper.CheckErr(err) + }, + } + + cmd.AddCommand(newDbSnapshotCmd(f, out, errOut)) + cmd.AddCommand(newDbCheckIntegrityCmd(f, out, errOut)) + + return cmd +} diff --git a/ziti/cmd/ziti/cmd/edge_controller/db_check_integrity.go b/ziti/cmd/ziti/cmd/edge_controller/db_check_integrity.go new file mode 100644 index 000000000..bbd8782b1 --- /dev/null +++ b/ziti/cmd/ziti/cmd/edge_controller/db_check_integrity.go @@ -0,0 +1,79 @@ +package edge_controller + +import ( + "fmt" + "github.com/Jeffail/gabs" + "github.com/openziti/ziti/ziti/cmd/ziti/cmd/common" + cmdutil "github.com/openziti/ziti/ziti/cmd/ziti/cmd/factory" + cmdhelper "github.com/openziti/ziti/ziti/cmd/ziti/cmd/helpers" + "github.com/openziti/ziti/ziti/cmd/ziti/util" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "io" + "net/http" +) + +type dbCheckIntegrityOptions struct { + commonOptions + fix bool +} + +func newDbCheckIntegrityCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command { + options := &dbCheckIntegrityOptions{ + commonOptions: commonOptions{ + CommonOptions: common.CommonOptions{Factory: f, Out: out, Err: errOut}, + }, + } + + cmd := &cobra.Command{ + Use: "check-integrity", + Short: "checks integrity of database references and constraints", + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + options.Cmd = cmd + options.Args = args + err := runCheckIntegrityDb(options) + cmdhelper.CheckErr(err) + }, + SuggestFor: []string{}, + } + + // allow interspersing positional args and flags + cmd.Flags().SetInterspersed(true) + cmd.Flags().BoolVarP(&options.fix, "fix-errors", "f", false, "Attempt to fix any detected errors") + options.AddCommonFlags(cmd) + + return cmd +} + +func runCheckIntegrityDb(o *dbCheckIntegrityOptions) error { + var err error + var body *gabs.Container + + if o.fix { + body, err = util.EdgeControllerUpdate("database/fix-data-integrity", "", o.Out, http.MethodPost, o.OutputJSONRequest, o.OutputJSONResponse) + } else { + body, err = util.EdgeControllerList("database/check-data-integrity", nil, o.OutputJSONResponse, o.Out) + } + + if err != nil { + return err + } + + data := body.S("data") + children, err := data.Children() + if len(children) == 0 || errors.Is(gabs.ErrNotObjOrArray, err) { + _, err = fmt.Fprintln(o.Out, "no data integrity errors found") + } + + for _, child := range children { + desc := child.S("description").Data().(string) + fixed := child.S("fixed").Data().(bool) + + if _, err = fmt.Fprintf(o.Out, "Issue: %v. Fixed: %v\n", desc, fixed); err != nil { + return err + } + } + + return err +} diff --git a/ziti/cmd/ziti/cmd/edge_controller/create_snapshot.go b/ziti/cmd/ziti/cmd/edge_controller/db_snapshot.go similarity index 81% rename from ziti/cmd/ziti/cmd/edge_controller/create_snapshot.go rename to ziti/cmd/ziti/cmd/edge_controller/db_snapshot.go index 0e43b31e3..e12ec0993 100644 --- a/ziti/cmd/ziti/cmd/edge_controller/create_snapshot.go +++ b/ziti/cmd/ziti/cmd/edge_controller/db_snapshot.go @@ -26,20 +26,19 @@ import ( "net/http" ) -type snapshotDbOptions struct { +type dbSnapshotOptions struct { commonOptions } -// newSnapshotDbCmd creates the 'edge controller create service-policy' command -func newSnapshotDbCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command { - options := &snapshotDbOptions{ +func newDbSnapshotCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command { + options := &dbSnapshotOptions{ commonOptions: commonOptions{ CommonOptions: common.CommonOptions{Factory: f, Out: out, Err: errOut}, }, } cmd := &cobra.Command{ - Use: "snapshot-db", + Use: "snapshot", Short: "creates a database snapshot", Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { @@ -58,8 +57,7 @@ func newSnapshotDbCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra return cmd } -// runSnapshotDb create a new config on the Ziti Edge Controller -func runSnapshotDb(o *snapshotDbOptions) error { +func runSnapshotDb(o *dbSnapshotOptions) error { _, err := util.EdgeControllerUpdate("database/snapshot", "", o.Out, http.MethodPost, false, false) return err } diff --git a/ziti/cmd/ziti/cmd/edge_controller/root.go b/ziti/cmd/ziti/cmd/edge_controller/root.go index cc895a6a0..8be20fca3 100644 --- a/ziti/cmd/ziti/cmd/edge_controller/root.go +++ b/ziti/cmd/ziti/cmd/edge_controller/root.go @@ -63,7 +63,7 @@ func populateEdgeCommands(f cmdutil.Factory, out io.Writer, errOut io.Writer, cm cmd.AddCommand(newVersionCmd(f, out, errOut)) cmd.AddCommand(newPolicyAdivsorCmd(f, out, errOut)) cmd.AddCommand(newVerifyCmd(f, out, errOut)) - cmd.AddCommand(newSnapshotDbCmd(f, out, errOut)) + cmd.AddCommand(newDbCmd(f, out, errOut)) return cmd }