Merge pull request #577 from openziti/fix-flaky-test

Fix flaky test by clamping times to nearest 100ms

health/checks.go

@@ -107,7 +107,7 @@ func (self *actionImpl) Matches(result *health.Result) bool {
 	logger := pfxlog.Logger()
 	logger.WithField("action", self.Description()).Debug("evaluating")
 
-	now := time.Now()
+	now := roundToClosest(time.Now(), timeClamp)
 	if result.IsHealthy() {
 		if self.consectivePasses == 0 {
 			self.passingSince = now
@@ -138,6 +138,9 @@ func (self *actionImpl) Matches(result *health.Result) bool {
 		if result.IsHealthy() && now.Sub(self.passingSince) < *self.Duration {
 			return false
 		}
+		if result.TimeOfFirstFailure != nil {
+			fmt.Printf("current fail duration: %v, required duration: %v\n", now.Sub(*result.TimeOfFirstFailure), *self.Duration)
+		}
 		if !result.IsHealthy() && now.Sub(*result.TimeOfFirstFailure) < *self.Duration {
 			return false
 		}

health/health.go

@@ -9,6 +9,11 @@ import (
 	"github.com/openziti/sdk-golang/ziti/edge"
 	"reflect"
 	"sync"
+	"time"
+)
+
+const (
+	timeClamp = 100 * time.Millisecond
 )
 
 type ServiceUpdater interface {
@@ -121,6 +126,10 @@ func (self *manager) handleResults() {
 }
 
 func (self *manager) handleResult(result *result) {
+	if result.TimeOfFirstFailure != nil {
+		rounded := roundToClosest(*result.TimeOfFirstFailure, timeClamp)
+		result.TimeOfFirstFailure = &rounded
+	}
 	if val, ok := self.checks.Load(result.name); ok {
 		if check, ok := val.(*checkContext); ok {
 			for _, action := range check.actions {
@@ -228,3 +237,11 @@ type result struct {
 	name string
 	health.Result
 }
+
+func roundToClosest(t time.Time, interval time.Duration) time.Time {
+	roundDown := t.Truncate(interval)
+	if t.Sub(roundDown) >= interval/2 {
+		return roundDown.Add(interval)
+	}
+	return roundDown
+}

health/round_test.go

@@ -0,0 +1,28 @@
+package health
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestRound(t *testing.T) {
+	interval := time.Millisecond * 100
+
+	for i := 0; i < 100; i++ {
+		now := time.Now()
+		val := roundToClosest(now, interval)
+		lower := now.Truncate(interval)
+		upper := lower.Add(interval)
+		lowerDelta := val.Sub(lower)
+		upperDelta := upper.Sub(val)
+		if lowerDelta < upperDelta {
+			assert.Equal(t, val, lower)
+		} else {
+			assert.Equal(t, val, upper)
+		}
+		fmt.Printf("now: %v rounded: %v\n", now, val)
+		time.Sleep(25 * time.Millisecond)
+	}
+}