v1.2.2: improved change detection

watchdog.go

@@ -33,6 +33,12 @@ func (s Status) String() string {
 	}
 }
 
+const (
+	MessageDown   = "went down"
+	MessageUp     = "came back up"
+	MessageHiccup = "hiccupped"
+)
+
 type Dog struct {
 	Watchdog      string
 	Name          string
@@ -47,15 +53,15 @@ type Dog struct {
 	status        Status
 	changed       bool
 	error         error
-	failures      int
+	//failures      int
-	passes        int
+	//passes        int
-	lastFailed    time.Time
+	//lastFailed    time.Time
-	lastPassed    time.Time
+	//lastPassed    time.Time
-	lastNotified  time.Time
+	//lastNotified time.Time
 }
 
 func New(d *Dog) *Dog {
-	d.lastPassed = time.Now().Add(-5 * time.Minute)
+	//d.lastPassed = time.Now().Add(-5 * time.Minute)
 	d.status = StatusUp
 	d.changed = false
 	return d
@@ -75,90 +81,82 @@ func (d *Dog) Watch() {
 func (d *Dog) watch() {
 	d.Logger <- fmt.Sprintf("Check: '%s'", d.Name)
 
-	err := d.check()
+	// This may be up or down
+	err := d.hardcheck()
 	if nil == err {
+		d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
+		// if it's down, coming up, notify
 		if d.changed {
-			d.notify("came back up")
+			d.notify(MessageUp)
 		}
 		return
 	}
 
-	changed := d.changed
+	// If being down is a change, check to see if it's just a hiccup
-	time.Sleep(time.Duration(5) * time.Second)
+	if d.changed {
-
+		time.Sleep(time.Duration(5) * time.Second)
-	err2 := d.check()
+		err2 := d.softcheck()
-	if nil != err2 {
+		if nil != err2 {
-		d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
+			// it's really down
-	} else {
+			d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
-		d.notify("hiccuped")
+		} else {
-		d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
+			// it's not really down, so reset the change info
-		return
+			d.changed = false
+			d.status = StatusUp
+			// and notify of the hiccup
+			d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
+			d.notify(MessageHiccup)
+			return
+		}
 	}
 
+	// TODO what if the server is flip-flopping rapidly?
+	// how to rate limit?
+	// "{{ .Server }} is on cooldown for 30 minutes"
+
+	// * We've had success since the last notification
+	// * It's been at least 5 minutes since the last notification
+	//fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
+	//if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
+	//}
+
 	t := 10
 	for {
+		// try to recover, then backoff exponentially
 		d.recover()
 		time.Sleep(time.Duration(t) * time.Second)
-		// backoff
 		t *= 2
-		err := d.check()
+		if t > 120 {
-		if nil != err {
+			t = 120
-			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
 		}
 
-		// We should notify if
+		err := d.softcheck()
-		// * The status has changed
+		if nil != err {
-		//
+			// this is down, and we know it's down
-		// TODO what if the server is flip-flopping rapidly?
+			d.status = StatusDown
-		// how to rate limit?
+			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
-		// "{{ .Server }} is on cooldown for 30 minutes"
+			if d.changed {
-		if changed || d.changed {
+				d.changed = false
-			changed = false
+				d.notify(MessageDown)
-			if StatusUp == d.status {
-				d.notify("came back up")
-				break
-			} else {
-				d.notify("went down")
 			}
-
+		} else {
-			// * We've had success since the last notification
+			// it came back up
-			// * It's been at least 5 minutes since the last notification
+			d.status = StatusUp
-			//fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
+			d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
-			//if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
+			if d.changed {
-			//}
+				// and the downtime was short - just a recovery
-			//if !failure || d.failures >= 5 {
+				d.notify(MessageHiccup)
-			// go back to the main 5-minute loop
+			} else {
-			//	break
+				// and the downtime was some time
-			//}
+				d.notify(MessageUp)
+			}
+			d.changed = false
+			break
 		}
 	}
 }
 
-func (d *Dog) check() error {
+func (d *Dog) softcheck() error {
-	previousStatus := d.status
-
-	var err error
-	defer func() {
-		// Are we up, or down?
-		if nil != err {
-			d.status = StatusDown
-			d.failures += 1
-			d.lastFailed = time.Now()
-		} else {
-			d.status = StatusUp
-			d.lastPassed = time.Now()
-			d.passes += 1
-			d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
-		}
-
-		// Has that changed?
-		if previousStatus != d.status {
-			d.changed = true
-		} else {
-			d.changed = false
-		}
-	}()
-
 	client := NewHTTPClient()
 	response, err := client.Get(d.CheckURL)
 	if nil != err {
@@ -192,6 +190,32 @@ func (d *Dog) check() error {
 	return nil
 }
 
+func (d *Dog) hardcheck() error {
+	previousStatus := d.status
+
+	err := d.softcheck()
+
+	// Are we up, or down?
+	if nil != err {
+		d.status = StatusDown
+		//d.failures += 1
+		//d.lastFailed = time.Now()
+	} else {
+		d.status = StatusUp
+		//d.lastPassed = time.Now()
+		//d.passes += 1
+	}
+
+	// Has that changed?
+	if previousStatus != d.status {
+		d.changed = true
+	} else {
+		d.changed = false
+	}
+
+	return err
+}
+
 func (d *Dog) recover() {
 	if "" == d.Recover {
 		return
@@ -221,7 +245,7 @@ func (d *Dog) recover() {
 
 func (d *Dog) notify(msg string) {
 	d.Logger <- fmt.Sprintf("Notifying the authorities of %s's status change", d.Name)
-	d.lastNotified = time.Now()
+	//d.lastNotified = time.Now()
 
 	for i := range d.Webhooks {
 		name := d.Webhooks[i]