Merge pull request #164 from openziti/update-edge

Add db integrity checks. Fixes GH-163

doc/release-notes.md

@@ -6,11 +6,20 @@
   * [e2e Service Configuration & Router Termination](https://github.com/openziti/edge/issues/173)
 * Router Scaling Issues
   * [Add worker pools for link and xgress dials](https://github.com/openziti/fabric/issues/109)
+* Model Performance Improvements
+  * [Denormalize policy links for performance](https://github.com/openziti/edge/issues/256)
+* Datastore Integrity Checker
+  * [foundation#107](https://github.com/openziti/foundation/issues/107)
+  * [edge#258](https://github.com/openziti/edge/issues/258)  
+  * [#163](https://github.com/openziti/ziti/issues/163)
 
 * Bug Fixes:
   * [#152](https://github.com/openziti/ziti/issues/152) - Fix ziti-router enroll exit code on failure
   * [#156](https://github.com/openziti/ziti/issues/156) - fix display of policies with empty roles lists
 
+* Backwards Compatibility 
+  * The `ziti edge snapshot-db` command is now `ziti edge db snapshot`
+
 ## End-To-End Encryption Enhancements
 ### E2E Encryption Router Termination
 A new xgress module has been added specifically for handling Ziti Edge e2e to handle SDK to Router Termination scenarios.
@@ -102,6 +111,29 @@ forwarder:
   linkDialWorkerCount: 10
 ```
 
+## Model Performance Improvements
+Policy relationships are now stored in a denormalized fashion. This means that checking if an entity is tied to another entity via a policy is now a direct lookup, and much faster. This means that the Ziti controller should scale very well in cases where we have many identities, services and/or edge routers. Performance was tested against the APIs used by the SDKs. 
+
+See for more detail:
+
+* [Denormalized Policies](https://github.com/openziti/edge/wiki/Denormalized-Policies)
+* [Characterization (Pure Model Tests)](https://github.com/openziti/ziti/wiki/Characterization#pure-model-tests)
+
+## Data Integrity Checking Framework
+The bbolt datastore used by Ziti provides simple key/value storage with nesting. Ziti has implemented some basic relational constructs on top of bbolt, such as indexed values, foreign key indexes, many to many collections and reference counted collections (for policy denormalization). This release adds a data integrity checking framework which allows us to verify that constraint assumptions are valid, and allows fixing issues when they are found (if possible). This work was done in part to validate that the policy denormalization code is working correctly, and to provide a rememdy if issues are found.
+
+There are two new REST APIs available
+
+* GET `/database/check-data-integrity`
+    * https://github.com/openziti/edge/blob/master/specs/swagger.yml#L2916
+* POST `/database/fix-data-integrity`
+    * https://github.com/openziti/edge/blob/master/specs/swagger.yml#L2930
+
+These APIs can be used from the ziti CLI.
+
+* `ziti edge db check-integrity` - to report on data integrity issues
+* `ziti edge db check-integrity -f` - to report on data integrity issues and attempt to fix any that are found
+
 # Release 0.15.2
 
 * What's New:

go.mod

@@ -27,9 +27,9 @@ require (
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
 	github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19
 	github.com/michaelquigley/pfxlog v0.0.0-20190813191113-2be43bd0dccc
-	github.com/openziti/edge v0.15.31
-	github.com/openziti/fabric v0.11.60
-	github.com/openziti/foundation v0.11.18
+	github.com/openziti/edge v0.15.32
+	github.com/openziti/fabric v0.11.61
+	github.com/openziti/foundation v0.11.19
 	github.com/openziti/sdk-golang v0.13.27
 	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -388,13 +388,17 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/openziti/edge v0.15.31 h1:3uXUplVbbzO5O9dex+X1nfA2gi5Tnwvkn2ASSf+H5Iw=
-github.com/openziti/edge v0.15.31/go.mod h1:bd7ImcHOkAAtbDB7t05vYHLeV9uO10EePC2qvUm5kuo=
+github.com/openziti/edge v0.15.32 h1:7QDWtjp9DPCGS3xkkV7y+se8sm9/zzGV8MxSCBefTBk=
+github.com/openziti/edge v0.15.32/go.mod h1:IaslRUgJGfkphC6+jVuKbLCnhB9Xcob8HxQ/hOA0HME=
 github.com/openziti/fabric v0.11.60 h1:Hr/fyvcjByhFN3gvohHwyaDYGFPbisWDcTQJqOKqJts=
 github.com/openziti/fabric v0.11.60/go.mod h1:fvB4uE9Ht1CIAads1wv6HOQqrbFcSeAUeEiiCMXco7g=
+github.com/openziti/fabric v0.11.61 h1:yKSVejRjbgfQ+KaMh9gpPYRvFAP0+ZkO/LU62xUh/dw=
+github.com/openziti/fabric v0.11.61/go.mod h1:m0wfc70b7f3x7xj4zs+WtJzXsPjraCKy8NwdKCk21js=
 github.com/openziti/foundation v0.11.16/go.mod h1:4Jgs3HGDiP1aVk5nNzXJqwJwWhrjOYy1F+4Xtxcgxvo=
 github.com/openziti/foundation v0.11.18 h1:DhUKJAqon7DNL/fWpqogr4D7abHLPpZdfOh+YzRUL7M=
 github.com/openziti/foundation v0.11.18/go.mod h1:qYViHuLb9+F2Sf8Fp82hvx+oVyP3nWV2czTN/n+zHCc=
+github.com/openziti/foundation v0.11.19 h1:yKhkIK8Pwv/BVXTcEu0SBrGB6LknuTWKpiLflSOaqKo=
+github.com/openziti/foundation v0.11.19/go.mod h1:qYViHuLb9+F2Sf8Fp82hvx+oVyP3nWV2czTN/n+zHCc=
 github.com/openziti/sdk-golang v0.13.27 h1:J0LSHHplqZiJLI6f22bo40+94n3k4QKQHSjvBM9DTX8=
 github.com/openziti/sdk-golang v0.13.27/go.mod h1:leIaRUZzYSt+AHnwXlzH/XSX0hOhcjjJ/rMDTkieV/8=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=

ziti/cmd/ziti/cmd/edge_controller/db.go

@@ -0,0 +1,25 @@
+package edge_controller
+
+import (
+	cmdutil "github.com/openziti/ziti/ziti/cmd/ziti/cmd/factory"
+	cmdhelper "github.com/openziti/ziti/ziti/cmd/ziti/cmd/helpers"
+	"github.com/spf13/cobra"
+	"io"
+)
+
+// newListCmd creates a command object for the "controller list" command
+func newDbCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "db",
+		Short: "Database management operations for the Ziti Edge Controller",
+		Run: func(cmd *cobra.Command, args []string) {
+			err := cmd.Help()
+			cmdhelper.CheckErr(err)
+		},
+	}
+
+	cmd.AddCommand(newDbSnapshotCmd(f, out, errOut))
+	cmd.AddCommand(newDbCheckIntegrityCmd(f, out, errOut))
+
+	return cmd
+}

ziti/cmd/ziti/cmd/edge_controller/db_check_integrity.go

@@ -0,0 +1,79 @@
+package edge_controller
+
+import (
+	"fmt"
+	"github.com/Jeffail/gabs"
+	"github.com/openziti/ziti/ziti/cmd/ziti/cmd/common"
+	cmdutil "github.com/openziti/ziti/ziti/cmd/ziti/cmd/factory"
+	cmdhelper "github.com/openziti/ziti/ziti/cmd/ziti/cmd/helpers"
+	"github.com/openziti/ziti/ziti/cmd/ziti/util"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"io"
+	"net/http"
+)
+
+type dbCheckIntegrityOptions struct {
+	commonOptions
+	fix bool
+}
+
+func newDbCheckIntegrityCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
+	options := &dbCheckIntegrityOptions{
+		commonOptions: commonOptions{
+			CommonOptions: common.CommonOptions{Factory: f, Out: out, Err: errOut},
+		},
+	}
+
+	cmd := &cobra.Command{
+		Use:   "check-integrity",
+		Short: "checks integrity of database references and constraints",
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			options.Cmd = cmd
+			options.Args = args
+			err := runCheckIntegrityDb(options)
+			cmdhelper.CheckErr(err)
+		},
+		SuggestFor: []string{},
+	}
+
+	// allow interspersing positional args and flags
+	cmd.Flags().SetInterspersed(true)
+	cmd.Flags().BoolVarP(&options.fix, "fix-errors", "f", false, "Attempt to fix any detected errors")
+	options.AddCommonFlags(cmd)
+
+	return cmd
+}
+
+func runCheckIntegrityDb(o *dbCheckIntegrityOptions) error {
+	var err error
+	var body *gabs.Container
+
+	if o.fix {
+		body, err = util.EdgeControllerUpdate("database/fix-data-integrity", "", o.Out, http.MethodPost, o.OutputJSONRequest, o.OutputJSONResponse)
+	} else {
+		body, err = util.EdgeControllerList("database/check-data-integrity", nil, o.OutputJSONResponse, o.Out)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	data := body.S("data")
+	children, err := data.Children()
+	if len(children) == 0 || errors.Is(gabs.ErrNotObjOrArray, err) {
+		_, err = fmt.Fprintln(o.Out, "no data integrity errors found")
+	}
+
+	for _, child := range children {
+		desc := child.S("description").Data().(string)
+		fixed := child.S("fixed").Data().(bool)
+
+		if _, err = fmt.Fprintf(o.Out, "Issue: %v. Fixed: %v\n", desc, fixed); err != nil {
+			return err
+		}
+	}
+
+	return err
+}

ziti/cmd/ziti/cmd/edge_controller/db_snapshot.go

@@ -26,20 +26,19 @@ import (
 	"net/http"
 )
 
-type snapshotDbOptions struct {
+type dbSnapshotOptions struct {
 	commonOptions
 }
 
-// newSnapshotDbCmd creates the 'edge controller create service-policy' command
-func newSnapshotDbCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
-	options := &snapshotDbOptions{
+func newDbSnapshotCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Command {
+	options := &dbSnapshotOptions{
 		commonOptions: commonOptions{
 			CommonOptions: common.CommonOptions{Factory: f, Out: out, Err: errOut},
 		},
 	}
 
 	cmd := &cobra.Command{
-		Use:   "snapshot-db",
+		Use:   "snapshot",
 		Short: "creates a database snapshot",
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
@@ -58,8 +57,7 @@ func newSnapshotDbCmd(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra
 	return cmd
 }
 
-// runSnapshotDb create a new config on the Ziti Edge Controller
-func runSnapshotDb(o *snapshotDbOptions) error {
+func runSnapshotDb(o *dbSnapshotOptions) error {
 	_, err := util.EdgeControllerUpdate("database/snapshot", "", o.Out, http.MethodPost, false, false)
 	return err
 }

ziti/cmd/ziti/cmd/edge_controller/root.go

@@ -63,7 +63,7 @@ func populateEdgeCommands(f cmdutil.Factory, out io.Writer, errOut io.Writer, cm
 	cmd.AddCommand(newVersionCmd(f, out, errOut))
 	cmd.AddCommand(newPolicyAdivsorCmd(f, out, errOut))
 	cmd.AddCommand(newVerifyCmd(f, out, errOut))
-	cmd.AddCommand(newSnapshotDbCmd(f, out, errOut))
+	cmd.AddCommand(newDbCmd(f, out, errOut))
 	return cmd
 }