v1.2.2: improved change detection

This commit is contained in:
AJ ONeal 2019-07-04 19:54:51 -06:00
parent e3de4a2ef6
commit 5a0382e8a3
1 changed files with 95 additions and 71 deletions

View File

@ -33,6 +33,12 @@ func (s Status) String() string {
} }
} }
const (
MessageDown = "went down"
MessageUp = "came back up"
MessageHiccup = "hiccupped"
)
type Dog struct { type Dog struct {
Watchdog string Watchdog string
Name string Name string
@ -47,15 +53,15 @@ type Dog struct {
status Status status Status
changed bool changed bool
error error error error
failures int //failures int
passes int //passes int
lastFailed time.Time //lastFailed time.Time
lastPassed time.Time //lastPassed time.Time
lastNotified time.Time //lastNotified time.Time
} }
func New(d *Dog) *Dog { func New(d *Dog) *Dog {
d.lastPassed = time.Now().Add(-5 * time.Minute) //d.lastPassed = time.Now().Add(-5 * time.Minute)
d.status = StatusUp d.status = StatusUp
d.changed = false d.changed = false
return d return d
@ -75,90 +81,82 @@ func (d *Dog) Watch() {
func (d *Dog) watch() { func (d *Dog) watch() {
d.Logger <- fmt.Sprintf("Check: '%s'", d.Name) d.Logger <- fmt.Sprintf("Check: '%s'", d.Name)
err := d.check() // This may be up or down
err := d.hardcheck()
if nil == err { if nil == err {
d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
// if it's down, coming up, notify
if d.changed { if d.changed {
d.notify("came back up") d.notify(MessageUp)
} }
return return
} }
changed := d.changed // If being down is a change, check to see if it's just a hiccup
if d.changed {
time.Sleep(time.Duration(5) * time.Second) time.Sleep(time.Duration(5) * time.Second)
err2 := d.softcheck()
err2 := d.check()
if nil != err2 { if nil != err2 {
// it's really down
d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2) d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
} else { } else {
d.notify("hiccuped") // it's not really down, so reset the change info
d.changed = false
d.status = StatusUp
// and notify of the hiccup
d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err) d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
d.notify(MessageHiccup)
return return
} }
t := 10
for {
d.recover()
time.Sleep(time.Duration(t) * time.Second)
// backoff
t *= 2
err := d.check()
if nil != err {
d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
} }
// We should notify if
// * The status has changed
//
// TODO what if the server is flip-flopping rapidly? // TODO what if the server is flip-flopping rapidly?
// how to rate limit? // how to rate limit?
// "{{ .Server }} is on cooldown for 30 minutes" // "{{ .Server }} is on cooldown for 30 minutes"
if changed || d.changed {
changed = false
if StatusUp == d.status {
d.notify("came back up")
break
} else {
d.notify("went down")
}
// * We've had success since the last notification // * We've had success since the last notification
// * It's been at least 5 minutes since the last notification // * It's been at least 5 minutes since the last notification
//fiveMinutesAgo := time.Now().Add(-5 * time.Minute) //fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
//if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) { //if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
//} //}
//if !failure || d.failures >= 5 {
// go back to the main 5-minute loop t := 10
// break for {
//} // try to recover, then backoff exponentially
} d.recover()
} time.Sleep(time.Duration(t) * time.Second)
t *= 2
if t > 120 {
t = 120
} }
func (d *Dog) check() error { err := d.softcheck()
previousStatus := d.status
var err error
defer func() {
// Are we up, or down?
if nil != err { if nil != err {
// this is down, and we know it's down
d.status = StatusDown d.status = StatusDown
d.failures += 1 d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
d.lastFailed = time.Now() if d.changed {
} else {
d.status = StatusUp
d.lastPassed = time.Now()
d.passes += 1
d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
}
// Has that changed?
if previousStatus != d.status {
d.changed = true
} else {
d.changed = false d.changed = false
d.notify(MessageDown)
}
} else {
// it came back up
d.status = StatusUp
d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
if d.changed {
// and the downtime was short - just a recovery
d.notify(MessageHiccup)
} else {
// and the downtime was some time
d.notify(MessageUp)
}
d.changed = false
break
}
}
} }
}()
func (d *Dog) softcheck() error {
client := NewHTTPClient() client := NewHTTPClient()
response, err := client.Get(d.CheckURL) response, err := client.Get(d.CheckURL)
if nil != err { if nil != err {
@ -192,6 +190,32 @@ func (d *Dog) check() error {
return nil return nil
} }
func (d *Dog) hardcheck() error {
previousStatus := d.status
err := d.softcheck()
// Are we up, or down?
if nil != err {
d.status = StatusDown
//d.failures += 1
//d.lastFailed = time.Now()
} else {
d.status = StatusUp
//d.lastPassed = time.Now()
//d.passes += 1
}
// Has that changed?
if previousStatus != d.status {
d.changed = true
} else {
d.changed = false
}
return err
}
func (d *Dog) recover() { func (d *Dog) recover() {
if "" == d.Recover { if "" == d.Recover {
return return
@ -221,7 +245,7 @@ func (d *Dog) recover() {
func (d *Dog) notify(msg string) { func (d *Dog) notify(msg string) {
d.Logger <- fmt.Sprintf("Notifying the authorities of %s's status change", d.Name) d.Logger <- fmt.Sprintf("Notifying the authorities of %s's status change", d.Name)
d.lastNotified = time.Now() //d.lastNotified = time.Now()
for i := range d.Webhooks { for i := range d.Webhooks {
name := d.Webhooks[i] name := d.Webhooks[i]