Track shared routes (#24)

* handle signals from within the service poller to enable selective route cleanup

* update iptables dependency

* use refcounts to track shared addresses

* add single entry to intercept LUT per service

* tidy

* don't panic when intercept address isn't in LUT

* deal with multiple intercept addresses per service.

* don't preallocate array.

* use syscall package for all signals.

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/achanda/go-sysctl v0.0.0-20160222034550-6be7678c45d2
 	github.com/antlr/antlr4 v0.0.0-20191115170859-54daca92f7b0
-	github.com/coreos/go-iptables v0.4.3
+	github.com/coreos/go-iptables v0.4.5
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1 // indirect

go.sum

@@ -29,8 +29,8 @@ github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wX
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-iptables v0.4.3 h1:jJg1aFuhCqWbgBl1VTqgTHG5faPM60A5JDMjQ2HYv+A=
-github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38=
+github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -169,32 +169,10 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/netfoundry/ziti-fabric v0.0.0-20191203170918-f312601f55c9 h1:36Fudtih7iNab1XISQr+DszW4ot2dtDr4o3VQc2ZyWM=
-github.com/netfoundry/ziti-fabric v0.0.0-20191203170918-f312601f55c9/go.mod h1:ktqE4LYK1Z4MGBpP29ylnUvEsSNwk7jXBfq2K/kq4nw=
-github.com/netfoundry/ziti-fabric v0.0.0-20191203174113-af4eb6e60864 h1:GGMpVeTIqeN+ZFC2MJX3qQNX+V9Zkda9lquhw1Toh+k=
-github.com/netfoundry/ziti-fabric v0.0.0-20191203174113-af4eb6e60864/go.mod h1:ktqE4LYK1Z4MGBpP29ylnUvEsSNwk7jXBfq2K/kq4nw=
-github.com/netfoundry/ziti-fabric v0.0.0-20191209202137-4205e73d614e h1:EdTaD5+hjMJpKRQW6yelbeqzrnSfpvXPb45XA0zUCe4=
-github.com/netfoundry/ziti-fabric v0.0.0-20191209202137-4205e73d614e/go.mod h1:vJc7LEWv4KWvWkLqpWekH6aNe9uK5z0BrQ8NARlY21Q=
-github.com/netfoundry/ziti-fabric v0.0.0-20191213150256-ce7a3ba09e7f h1:UG529yOXSiaL69e5Dd+r2/VvaWWKf2X6Hwxiebup52E=
-github.com/netfoundry/ziti-fabric v0.0.0-20191213150256-ce7a3ba09e7f/go.mod h1:4wVl/a+B+k2gEboApyC2wXLRT/6Vy0Rn8Ciy4VUXmZg=
 github.com/netfoundry/ziti-fabric v0.0.0-20191213215600-dcc209b66c31 h1:qlh09aBS5mmi6gWxsn6peSrrpZYfHCaOReBNMq8P6Lg=
 github.com/netfoundry/ziti-fabric v0.0.0-20191213215600-dcc209b66c31/go.mod h1:C06MeM/NC8eIDHWnhisFzEcneMx+dyrewB3wu/KzSn4=
-github.com/netfoundry/ziti-foundation v0.0.0-20191203170231-5c1204e0e65d h1:V6VZeV8JVUipJhZgDUNUhaJ2L6bqmosAf+SNUj6C1f0=
-github.com/netfoundry/ziti-foundation v0.0.0-20191203170231-5c1204e0e65d/go.mod h1:27kvN9RZk7cpt2xba6owuXktNphq5hPK62CAiKnIS10=
-github.com/netfoundry/ziti-foundation v0.0.0-20191209180049-dd15781cc4aa h1:6Ekd22eDTyTskrvQj2m+uTJ8rf3kDq1vrDWVLwF+oC0=
-github.com/netfoundry/ziti-foundation v0.0.0-20191209180049-dd15781cc4aa/go.mod h1:27kvN9RZk7cpt2xba6owuXktNphq5hPK62CAiKnIS10=
-github.com/netfoundry/ziti-foundation v0.0.0-20191212214941-04587c9d27b9 h1:ksLQi1ZMR427L7Y+U7G0zKsoQPiK8jfBIz8/yTLlEQ8=
-github.com/netfoundry/ziti-foundation v0.0.0-20191212214941-04587c9d27b9/go.mod h1:27kvN9RZk7cpt2xba6owuXktNphq5hPK62CAiKnIS10=
-github.com/netfoundry/ziti-foundation v0.0.0-20191213140847-b961be5acac5 h1:SEqF/OO7Be4Jq/4oCsSYDSRsa3D6R9qUMvj16FCAABo=
-github.com/netfoundry/ziti-foundation v0.0.0-20191213140847-b961be5acac5/go.mod h1:27kvN9RZk7cpt2xba6owuXktNphq5hPK62CAiKnIS10=
 github.com/netfoundry/ziti-foundation v0.0.0-20191213201903-e68267fe89e6 h1:6mEfpYgZurDpmv/uTRbTWS9AXc+CBDjyoLM5ugijYn4=
 github.com/netfoundry/ziti-foundation v0.0.0-20191213201903-e68267fe89e6/go.mod h1:27kvN9RZk7cpt2xba6owuXktNphq5hPK62CAiKnIS10=
-github.com/netfoundry/ziti-sdk-golang v0.0.0-20191206202611-43555d2de4f9 h1:kIt+dOCtn7o3KOU8yBu9CpwXZA0V2qumvthnRNzKtag=
-github.com/netfoundry/ziti-sdk-golang v0.0.0-20191206202611-43555d2de4f9/go.mod h1:WJ49orNIo3R813cG4hfB4mww8NU4mCzQBsKQldqs/ZQ=
-github.com/netfoundry/ziti-sdk-golang v0.0.0-20191209205522-dd68e7c5ed49 h1:40QUJE6ow9GzW8N0kPaJ1ptYmt4axkFD7982IyVZSZQ=
-github.com/netfoundry/ziti-sdk-golang v0.0.0-20191209205522-dd68e7c5ed49/go.mod h1:HaMjEeFnxNo91ew0RbWJ5IZRS04kD4QliFboTVFcLuQ=
-github.com/netfoundry/ziti-sdk-golang v0.0.0-20191213150244-c2fe09d13455 h1:pT09Ndhh5FCUavsiT5Wzl30W789TGdVSLxGHXVreDVg=
-github.com/netfoundry/ziti-sdk-golang v0.0.0-20191213150244-c2fe09d13455/go.mod h1:PnJrHWf+i+SVoe2EOfuJZBzYQubh+lXSb7BXhnaJM7I=
 github.com/netfoundry/ziti-sdk-golang v0.0.0-20191217143926-85a9668710ec h1:3xvlOZqkT9PS6/DH2VKF2mrvgVLxPYIuDfKMt9BSiSc=
 github.com/netfoundry/ziti-sdk-golang v0.0.0-20191217143926-85a9668710ec/go.mod h1:swcAExEHVuvgvtf6zPO+v9gHww8GpYt4GBVFu6ZHxQI=
 github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY=

tunnel/intercept/interceptor.go

@@ -136,14 +136,15 @@ func (m LUT) GetByAddress(addr net.Addr) (*InterceptedService, error) {
 	return &s, nil
 }
 
-func (m LUT) GetByName(serviceName string) (*InterceptedService, error) {
+func (m LUT) GetByName(serviceName string) []InterceptedService {
+	var r []InterceptedService
 	for _, service := range m {
 		if serviceName == service.Name {
-			return &service, nil
+			r = append(r, service)
 		}
 	}
 
-	return nil, fmt.Errorf("service %s is not being intercepted", serviceName)
+	return r
 }
 
 func (m LUT) Put(addr interceptAddress, serviceName string, data interface{}) error {

tunnel/intercept/svcpoll.go

@@ -21,58 +21,86 @@ import (
 	"github.com/netfoundry/ziti-edge/tunnel/dns"
 	"github.com/netfoundry/ziti-sdk-golang/ziti"
 	"github.com/netfoundry/ziti-sdk-golang/ziti/edge"
-	"github.com/sirupsen/logrus"
 	"io"
 	"net"
+	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 	"time"
 )
 
 func ServicePoller(context ziti.Context, interceptor Interceptor, resolver dns.Resolver, pollRate time.Duration) {
 	interceptor.Start(context)
 	knownServices := make(map[string]edge.Service)
+	log := pfxlog.Logger()
+
+	sig := make(chan os.Signal)
+	signal.Notify(sig)
 
 	if pollRate < time.Second {
 		pollRate = 15 * time.Second
 	}
 
-	for ; ; time.Sleep(pollRate) {
+	for {
 		edgeServices, err := context.GetServices()
 		if err != nil {
-			logrus.Errorf("failed to get ziti services: %v", err)
+			log.Errorf("failed to get ziti services: %v", err)
 			if err.Error() == "unauthorized" {
 				if err := context.Authenticate(); err != nil {
-					logrus.WithError(err).Error("could not re-authenticate, session lost")
-					return
+					log.WithError(err).Error("could not re-authenticate, session lost")
+					break
 				}
 			}
+		}
+		added, removed := diffServices(edgeServices, knownServices)
+		updateServices(context, interceptor, resolver, added, removed, knownServices)
 
+		select {
+		case <-time.After(pollRate):
 			continue
-		}
-
-		// find new services
-		addedServices := make(map[string]edge.Service)
-		edgeServiceIds := make(map[string]struct{})
-		for _, edgeSvc := range edgeServices {
-			// get edge service IDs for efficiently finding removed services
-			edgeServiceIds[edgeSvc.Id] = struct{}{}
-			if _, ok := knownServices[edgeSvc.Id]; !ok {
-				addedServices[edgeSvc.Id] = edgeSvc
-				knownServices[edgeSvc.Id] = edgeSvc
+		case s := <-sig:
+			log.Debugf("caught signal %v", s)
+			switch s {
+			case syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM:
+				log.Debugf("caught signal %v", s)
+				goto done
 			}
 		}
-
-		// look for removed services
-		removedServices := make(map[string]edge.Service)
-		for id, knownSvc := range knownServices {
-			if _, ok := edgeServiceIds[id]; !ok {
-				removedServices[knownSvc.Id] = knownSvc
-				delete(knownServices, id)
-			}
-		}
-
-		updateServices(context, interceptor, resolver, addedServices, removedServices, knownServices)
 	}
+
+done:
+	// use `updateServices` to stop each intercepted service one-at-a-time.
+	updateServices(context, interceptor, resolver, nil, knownServices, knownServices)
+	interceptor.Stop()
+}
+
+// compare a list of services from the edge controller against the services that we are currently
+// familiar with to determine which (if any) services have been added or removed since the last
+// time `knownServices` was updated.
+func diffServices(edgeServices []edge.Service, knownServices map[string]edge.Service) (added, removed map[string]edge.Service) {
+	// find new services
+	added = make(map[string]edge.Service)
+	edgeServiceIds := make(map[string]struct{})
+	for _, edgeSvc := range edgeServices {
+		// get edge service IDs for efficiently finding removed services
+		edgeServiceIds[edgeSvc.Id] = struct{}{}
+		if _, ok := knownServices[edgeSvc.Id]; !ok {
+			added[edgeSvc.Id] = edgeSvc
+			knownServices[edgeSvc.Id] = edgeSvc
+		}
+	}
+
+	// look for removed services
+	removed = make(map[string]edge.Service)
+	for id, knownSvc := range knownServices {
+		if _, ok := edgeServiceIds[id]; !ok {
+			removed[knownSvc.Id] = knownSvc
+			delete(knownServices, id)
+		}
+	}
+
+	return
 }
 
 func updateServices(context ziti.Context, interceptor Interceptor, resolver dns.Resolver, added, removed, all map[string]edge.Service) {
@@ -96,22 +124,23 @@ func updateServices(context ziti.Context, interceptor Interceptor, resolver dns.
 		}
 	}
 
-	// build map of all in-use address strings, so we know when a route needs to be removed from the tun
-	allAddrs := make(map[string]*struct{}, len(all))
+	// build map of all in-use address strings, so we know when a route needs to be removed
+	allAddrs := make(map[string]int, len(all))
 	for _, svc := range all {
 		addr := svc.Dns.Hostname
 		if _, ok := allAddrs[addr]; !ok {
-			allAddrs[addr] = nil
+			allAddrs[addr] += 1
 		}
 	}
 
 	for _, svc := range removed {
 		log.Infof("stopping tunnel for unavailable service: %s", svc.Name)
-		_, sharedAddress := allAddrs[svc.Dns.Hostname]
-		err := interceptor.StopIntercepting(svc.Name, !sharedAddress)
+		useCnt := allAddrs[svc.Dns.Hostname]
+		err := interceptor.StopIntercepting(svc.Name, useCnt == 1)
 		if err != nil {
 			log.Errorf("failed to stop intercepting: %v", err)
 		}
+		allAddrs[svc.Dns.Hostname] -= 1
 	}
 }
 

tunnel/intercept/tproxy/tproxy_linux.go

@@ -155,8 +155,10 @@ func (t *tProxyInterceptor) accept(context ziti.Context) {
 			}
 			log.Infof("received connection: %s --> %s", client.LocalAddr().String(), client.RemoteAddr().String())
 			service, err := t.interceptLUT.GetByAddress(client.LocalAddr())
-			if err != nil {
-				log.Debugf("received connection for %s, which does not map to an intercepted service", client.LocalAddr().String())
+			if service == nil {
+				log.Warnf("received connection for %s, which does not map to an intercepted service", client.LocalAddr().String())
+				client.Close()
+				continue
 			}
 			go tunnel.Run(context, service.Name, client)
 		}
@@ -334,22 +336,28 @@ func (t *tProxyInterceptor) intercept(service edge.Service, resolver dns.Resolve
 }
 
 func (t *tProxyInterceptor) StopIntercepting(serviceName string, removeRoute bool) error {
-	service, err := t.interceptLUT.GetByName(serviceName)
-	if err != nil {
+	services := t.interceptLUT.GetByName(serviceName)
+	if len(services) == 0 {
 		return fmt.Errorf("service %s not found in intercept LUT", serviceName)
 	}
-	defer t.interceptLUT.Remove(service.Addr)
-
-	err = t.ipt.Delete(mangleTable, dstChain, service.Data.([]string)...)
-	if err != nil {
-		return fmt.Errorf("failed to remove iptables rule for service %s: %v", serviceName, err)
+	// keep track of routes used by all intercepts. use a map to avoid duplicates
+	routes := map[string]net.IPNet{}
+	for _, service := range services {
+		defer t.interceptLUT.Remove(service.Addr)
+		err := t.ipt.Delete(mangleTable, dstChain, service.Data.([]string)...)
+		if err != nil {
+			return fmt.Errorf("failed to remove iptables rule for service %s: %v", serviceName, err)
+		}
+		ipn := service.Addr.IpNet()
+		routes[ipn.String()] = ipn
 	}
 
 	if removeRoute {
-		ipNet := service.Addr.IpNet()
-		err := router.RemoveLocalAddress(&ipNet, "lo")
-		if err != nil {
-			return fmt.Errorf("failed to remove route for service %s: %v", serviceName, err)
+		for _, ipNet := range routes {
+			err := router.RemoveLocalAddress(&ipNet, "lo")
+			if err != nil {
+				return fmt.Errorf("failed to remove route for service %s: %v", serviceName, err)
+			}
 		}
 	}
 

tunnel/intercept/tun/tun.go

@@ -102,20 +102,26 @@ func (t *tunInterceptor) Intercept(service edge.Service, resolver dns.Resolver)
 }
 
 func (t *tunInterceptor) StopIntercepting(serviceName string, removeRoute bool) error {
-	service, err := t.interceptLUT.GetByName(serviceName)
-	if err != nil {
+	services := t.interceptLUT.GetByName(serviceName)
+	if len(services) == 0 {
 		return fmt.Errorf("service %s not found in intercept LUT", serviceName)
 	}
-	defer t.interceptLUT.Remove(service.Addr)
-
+	// keep track of routes used by all intercepts. use a map to avoid duplicates
+	routes := map[string]net.IPNet{}
+	for _, service := range services {
+		defer t.interceptLUT.Remove(service.Addr)
+		ipn := service.Addr.IpNet()
+		routes[ipn.String()] = ipn
+	}
 	tcp.UnregisterService(serviceName)
 	t.udpManager.UnregisterService(serviceName)
 
 	if removeRoute {
-		ipNet := service.Addr.IpNet()
-		err = router.RemovePointToPointAddress(t.localPrefix.IP, &ipNet, t.tunIf.iFace.Name)
-		if err != nil {
-			return fmt.Errorf("failed to delete route: %v", err)
+		for _, ipNet := range routes {
+			err := router.RemovePointToPointAddress(t.localPrefix.IP, &ipNet, t.tunIf.iFace.Name)
+			if err != nil {
+				return fmt.Errorf("failed to delete route: %v", err)
+			}
 		}
 	}