Merge pull request openziti/fabric#289 from openziti/routing-perf

Add routing perf benchmark and improve router perf

controller/network/link_controller.go

@@ -26,22 +26,21 @@ import (
 )
 
 type linkController struct {
-	linkTable      *linkTable
-	adjacencyTable *adjacencyTable
-	sequence       *sequence.Sequence
+	linkTable *linkTable
+	sequence  *sequence.Sequence
 }
 
 func newLinkController() *linkController {
 	return &linkController{
-		linkTable:      newLinkTable(),
-		adjacencyTable: newAdjacencyTable(),
-		sequence:       sequence.NewSequence(),
+		linkTable: newLinkTable(),
+		sequence:  sequence.NewSequence(),
 	}
 }
 
 func (linkController *linkController) add(link *Link) {
 	linkController.linkTable.add(link)
-	linkController.adjacencyTable.add(link)
+	link.Src.routerLinks.Add(link)
+	link.Dst.routerLinks.Add(link)
 }
 
 func (linkController *linkController) has(link *Link) bool {
@@ -59,20 +58,14 @@ func (linkController *linkController) all() []*Link {
 
 func (linkController *linkController) remove(link *Link) {
 	linkController.linkTable.remove(link)
-	linkController.adjacencyTable.remove(link)
-}
-
-func (linkController *linkController) allLinksForRouter(routerId string) []*Link {
-	if rlt, found := linkController.adjacencyTable.get(routerId); found {
-		return rlt.allLinksForAllRouters()
-	}
-	return nil
+	link.Src.routerLinks.Remove(link)
+	link.Dst.routerLinks.Remove(link)
 }
 
 func (linkController *linkController) connectedNeighborsOfRouter(router *Router) []*Router {
 	neighborMap := make(map[string]*Router)
 
-	links := linkController.allLinksForRouter(router.Id)
+	links := router.routerLinks.GetLinks()
 	for _, link := range links {
 		currentState := link.CurrentState()
 		if currentState != nil && currentState.Mode == Connected && !link.Down {
@@ -96,7 +89,7 @@ func (linkController *linkController) leastExpensiveLink(a, b *Router) (*Link, b
 	var selected *Link
 	var cost int64 = math.MaxInt64
 
-	links := linkController.allLinksForRouter(a.Id)
+	links := a.routerLinks.GetLinks()
 	for _, link := range links {
 		currentState := link.CurrentState()
 		if currentState != nil && currentState.Mode == Connected && !link.Down {
@@ -156,15 +149,12 @@ func (linkController *linkController) hasLink(a, b *Router, pendingLimit int64)
 }
 
 func (linkController *linkController) hasDirectedLink(a, b *Router, pendingLimit int64) bool {
-	if rlt, found := linkController.adjacencyTable.get(a.Id); found {
-		if links, found := rlt.allLinksForRouter(b.Id); found {
-			for _, link := range links {
-				state := link.CurrentState()
-				if link.Src == a && link.Dst == b && state != nil {
-					if state.Mode == Connected || (state.Mode == Pending && state.Timestamp > pendingLimit) {
-						return true
-					}
-				}
+	links := a.routerLinks.GetLinks()
+	for _, link := range links {
+		state := link.CurrentState()
+		if link.Src == a && link.Dst == b && state != nil {
+			if state.Mode == Connected || (state.Mode == Pending && state.Timestamp > pendingLimit) {
+				return true
 			}
 		}
 	}
@@ -328,19 +318,13 @@ func (rlt *routerLinksTable) allLinksForRouter(routerId string) ([]*Link, bool)
 }
 
 func (rlt *routerLinksTable) allLinksForAllRouters() []*Link {
-	linksMap := make(map[string]*Link)
+	allLinks := make([]*Link, 0, rlt.routerLinks.Count())
 	for i := range rlt.routerLinks.IterBuffered() {
 		links := i.Val.([]*Link)
 		for _, link := range links {
-			linksMap[link.Id.Token] = link
+			allLinks = append(allLinks, link)
 		}
 	}
-
-	allLinks := make([]*Link, 0)
-	for _, v := range linksMap {
-		allLinks = append(allLinks, v)
-	}
-
 	return allLinks
 }
 

controller/network/link_controller_test.go

@@ -36,21 +36,21 @@ func TestLifecycle(t *testing.T) {
 	linkController.add(l0)
 	assert.True(t, linkController.has(l0))
 
-	links := linkController.allLinksForRouter(r0.Id)
+	links := r0.routerLinks.GetLinks()
 	assert.Equal(t, 1, len(links))
 	assert.Equal(t, l0, links[0])
 
-	links = linkController.allLinksForRouter(r1.Id)
+	links = r1.routerLinks.GetLinks()
 	assert.Equal(t, 1, len(links))
 	assert.Equal(t, l0, links[0])
 
 	linkController.remove(l0)
 	assert.False(t, linkController.has(l0))
 
-	links = linkController.allLinksForRouter(r0.Id)
+	links = r0.routerLinks.GetLinks()
 	assert.Equal(t, 0, len(links))
 
-	links = linkController.allLinksForRouter(r1.Id)
+	links = r1.routerLinks.GetLinks()
 	assert.Equal(t, 0, len(links))
 }
 

controller/network/network.go

@@ -193,7 +193,11 @@ func (network *Network) GetAllLinks() []*Link {
 }
 
 func (network *Network) GetAllLinksForRouter(routerId string) []*Link {
-	return network.linkController.allLinksForRouter(routerId)
+	r := network.GetConnectedRouter(routerId)
+	if r == nil {
+		return nil
+	}
+	return r.routerLinks.GetLinks()
 }
 
 func (network *Network) GetCircuit(circuitId string) (*Circuit, bool) {
@@ -291,7 +295,7 @@ func (network *Network) ValidateTerminators(r *Router) {
 
 func (network *Network) DisconnectRouter(r *Router) {
 	// 1: remove Links for Router
-	for _, l := range network.linkController.allLinksForRouter(r.Id) {
+	for _, l := range r.routerLinks.GetLinks() {
 		network.linkController.remove(l)
 		network.LinkChanged(l)
 	}
@@ -930,13 +934,13 @@ func (network *Network) GetServiceCache() Cache {
 
 func (network *Network) NotifyRouterRenamed(id, name string) {
 	if cached, _ := network.Routers.cache.Get(id); cached != nil {
-		if cachedRouter, ok := cached.(*db.Router); ok {
+		if cachedRouter, ok := cached.(*Router); ok {
 			cachedRouter.Name = name
 		}
 	}
 
 	if cached, _ := network.Routers.connected.Get(id); cached != nil {
-		if cachedRouter, ok := cached.(*db.Router); ok {
+		if cachedRouter, ok := cached.(*Router); ok {
 			cachedRouter.Name = name
 		}
 	}

controller/network/path.go

@@ -176,6 +176,9 @@ func (network *Network) shortestPath(srcR *Router, dstR *Router) ([]*Router, int
 
 	for len(unvisited) > 0 {
 		u := minCost(unvisited, dist)
+		if u == dstR { // if the dest router is the lowest cost next link, we can stop evaluating
+			break
+		}
 		delete(unvisited, u)
 
 		neighbors := network.linkController.connectedNeighborsOfRouter(u)

controller/network/router.go

@@ -28,6 +28,8 @@ import (
 	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 	"reflect"
+	"sync"
+	"sync/atomic"
 )
 
 type Router struct {
@@ -38,6 +40,7 @@ type Router struct {
 	Control            channel2.Channel
 	Connected          concurrenz.AtomicBoolean
 	VersionInfo        *common.VersionInfo
+	routerLinks        RouterLinks
 }
 
 func (entity *Router) fillFrom(_ Controller, _ *bbolt.Tx, boltEntity boltz.Entity) error {
@@ -63,11 +66,67 @@ func NewRouter(id, name, fingerprint string) *Router {
 	if name == "" {
 		name = id
 	}
-	return &Router{
+	result := &Router{
 		BaseEntity:  models.BaseEntity{Id: id},
 		Name:        name,
 		Fingerprint: &fingerprint,
 	}
+	result.routerLinks.Store([]*Link{})
+	return result
+}
+
+type routerCopyOnWriteMap struct {
+	sync.Mutex
+	atomic.Value
+}
+
+func newRouterCopyOnWriteMap() *routerCopyOnWriteMap {
+	result := &routerCopyOnWriteMap{}
+	result.Store(map[string]*Router{})
+	return result
+}
+
+func (self *routerCopyOnWriteMap) Has(id string) bool {
+	_, found := self.getCurrentMap()[id]
+	return found
+}
+
+func (self *routerCopyOnWriteMap) Count() int {
+	return len(self.getCurrentMap())
+}
+
+func (self *routerCopyOnWriteMap) Get(id string) (*Router, bool) {
+	r, found := self.getCurrentMap()[id]
+	return r, found
+}
+
+func (self *routerCopyOnWriteMap) Set(id string, router *Router) {
+	self.Lock()
+	defer self.Unlock()
+	m := self.getMapCopy()
+	m[id] = router
+	self.Store(m)
+}
+
+func (self *routerCopyOnWriteMap) Remove(id string) {
+	self.Lock()
+	defer self.Unlock()
+	m := self.getMapCopy()
+	delete(m, id)
+	self.Store(m)
+}
+
+func (self *routerCopyOnWriteMap) getCurrentMap() map[string]*Router {
+	return self.Load().(map[string]*Router)
+}
+
+func (self *routerCopyOnWriteMap) getMapCopy() map[string]*Router {
+	m := self.getCurrentMap()
+	mapCopy := map[string]*Router{}
+	for k, v := range m {
+		mapCopy[k] = v
+	}
+	return mapCopy
 }
 
 type RouterController struct {
@@ -100,6 +159,7 @@ func (ctrl *RouterController) markConnected(r *Router) {
 func (ctrl *RouterController) markDisconnected(r *Router) {
 	r.Connected.Set(false)
 	ctrl.connected.Remove(r.Id)
+	r.routerLinks.Clear()
 }
 
 func (ctrl *RouterController) IsConnected(id string) bool {
@@ -115,8 +175,8 @@ func (ctrl *RouterController) getConnected(id string) *Router {
 
 func (ctrl *RouterController) allConnected() []*Router {
 	var routers []*Router
-	for i := range ctrl.connected.IterBuffered() {
-		routers = append(routers, i.Val.(*Router))
+	for v := range ctrl.connected.IterBuffered() {
+		routers = append(routers, v.Val.(*Router))
 	}
 	return routers
 }
@@ -179,3 +239,45 @@ func (ctrl *RouterController) UpdateCachedFingerprint(id, fingerprint string) {
 		}
 	}
 }
+
+type RouterLinks struct {
+	sync.Mutex
+	atomic.Value
+}
+
+func (self *RouterLinks) GetLinks() []*Link {
+	result := self.Load()
+	if result == nil {
+		return nil
+	}
+	return result.([]*Link)
+}
+
+func (self *RouterLinks) Add(link *Link) {
+	self.Lock()
+	defer self.Unlock()
+	links := self.GetLinks()
+	newLinks := make([]*Link, 0, len(links)+1)
+	for _, l := range links {
+		newLinks = append(newLinks, l)
+	}
+	newLinks = append(newLinks, link)
+	self.Store(newLinks)
+}
+
+func (self *RouterLinks) Remove(link *Link) {
+	self.Lock()
+	defer self.Unlock()
+	links := self.GetLinks()
+	newLinks := make([]*Link, 0, len(links)+1)
+	for _, l := range links {
+		if link != l {
+			newLinks = append(newLinks, l)
+		}
+	}
+	self.Store(newLinks)
+}
+
+func (self *RouterLinks) Clear() {
+	self.Store([]*Link{})
+}

controller/network/util_test.go

@@ -123,7 +123,7 @@ func debugNetwork(n *Network) {
 
 	for rIdx, router := range routers {
 		debugf("%v router: %v\n", rIdx, router.Id)
-		links := n.linkController.allLinksForRouter(router.Id)
+		links := router.routerLinks.GetLinks()
 		sort.Slice(links, func(i, j int) bool {
 			return links[i].Id.Token < links[j].Id.Token
 		})