From 8248c33607a488b58570d74143f48bf3682d8ce8 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sat, 8 Jun 2019 03:11:42 -0600 Subject: [PATCH] made useful, nay, usable! --- README.md | 40 ++++-- watchdog.go | 341 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 238 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index f98bef4..0601cd5 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,34 @@ go run ./watchdog.go -c dog.json ```json { - "watches": [ - { - "name": "Example Site", - "url": "https://example.com/", - "webhook": "twilio", - "keywords": "My Site", - "recover_script": "systemctl restart example-site" - } - ], + "watches": [ + { + "name": "Example Site", + "url": "https://example.com/", + "webhooks": ["twilio"], + "keywords": "My Site", + "recover_script": "systemctl restart example-site" + } + ], + "webhooks": [ + { + "name": "twilio", + "url": "https://api.twilio.com/2010-04-01/Accounts/AC00000000000000000000000000000000/Messages.json", + "auth": { + "user": "AC00000000000000000000000000000000", + "pass": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + "form": { + "To": "+1 801 555 1234", + "From": "+1 800 555 4321", + "Body": "[{{ .Name }}] The system is down. The system is down." + } + } + ] +} +``` + + diff --git a/watchdog.go b/watchdog.go index 29aa617..67e1390 100644 --- a/watchdog.go +++ b/watchdog.go @@ -9,8 +9,10 @@ import ( "log" "net" "net/http" + "net/url" "os" "os/exec" + "strings" "time" ) @@ -54,17 +56,29 @@ func main() { done := make(chan struct{}, 1) + allWebhooks := make(map[string]ConfigWebhook) + + for i := range config.Webhooks { + h := config.Webhooks[i] + allWebhooks[h.Name] = h + } + + logQueue := make(chan string, 10) + go logger(logQueue) for i := range config.Watches { c := config.Watches[i] - fmt.Printf("Watching '%s'", c.Name) + logQueue <- fmt.Sprintf("Watching '%s'", c.Name) go func(c ConfigWatch) { - w := &Dog{ - Name: c.Name, - CheckURL: c.URL, - Keywords: c.Keywords, - Recover: c.RecoverScript, - } - w.Watch() + d := New(&Dog{ + Name: c.Name, + CheckURL: c.URL, + Keywords: c.Keywords, + Recover: c.RecoverScript, + Webhooks: c.Webhooks, + AllWebhooks: allWebhooks, + logger: logQueue, + }) + d.Watch() }(config.Watches[i]) } @@ -81,6 +95,9 @@ type Dog struct { CheckURL string Keywords string Recover string + Webhooks []string + AllWebhooks map[string]ConfigWebhook + logger chan string error error failures int passes int @@ -89,46 +106,24 @@ type Dog struct { lastNotified time.Time } -func (w *Dog) execRecover() { - if "" == w.Recover { - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - cmd := exec.CommandContext(ctx, "bash") - pipe, err := cmd.StdinPipe() - pipe.Write([]byte(w.Recover)) - if nil != err { - fmt.Fprintf(os.Stderr, "Could not write to bash '%s': %s\n", w.Recover, err) - } - err = cmd.Start() - if nil != err { - fmt.Fprintf(os.Stderr, "Could not start '%s': %s\n", w.Recover, err) - } - err = pipe.Close() - if nil != err { - fmt.Fprintf(os.Stderr, "Could not close '%s': %s\n", w.Recover, err) - } - err = cmd.Wait() - cancel() - if nil != err { - fmt.Fprintf(os.Stderr, "'%s' failed: %s\n", w.Recover, err) - } +func New(d *Dog) *Dog { + d.lastPassed = time.Now().Add(-5 * time.Minute) + return d } -func (w *Dog) Watch() { - w.watch() +func (d *Dog) Watch() { + d.watch() for { // TODO set cancellable callback ? time.Sleep(5 * time.Minute) - w.watch() + d.watch() } } -func (w *Dog) watch() { - fmt.Println("Running a check") +func (d *Dog) watch() { + d.logger <- fmt.Sprintf("Check: '%s'", d.Name) - err := w.check() + err := d.check() if nil == err { return } @@ -136,30 +131,193 @@ func (w *Dog) watch() { failure := false t := 10 for { - w.execRecover() + d.recover() time.Sleep(time.Duration(t) * time.Second) // backoff t *= 2 - err := w.check() + err := d.check() if nil != err { failure = true } // We should notify if // * We've had success since the last notification // * It's been at least 5 minutes since the last notification - if w.lastPassed.After(w.lastNotified) && w.lastNotified.Before(time.Now().Add(-5*time.Minute)) { - err := w.notify(failure) - if nil != err { - fmt.Println("Notify:", err) - } + fiveMinutesAgo := time.Now().Add(-5 * time.Minute) + if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) { + d.notify(failure) } - if w.failures >= 5 { + if d.failures >= 5 { // go back to the main 5-minute loop break } } } +func (d *Dog) check() error { + var err error + defer func() { + if nil != err { + d.failures += 1 + d.lastFailed = time.Now() + } else { + d.lastPassed = time.Now() + d.passes += 1 + } + }() + + client := NewHTTPClient() + response, err := client.Get(d.CheckURL) + if nil != err { + d.error = fmt.Errorf("Connection Failure: " + err.Error()) + return err + } + + b, err := ioutil.ReadAll(response.Body) + if nil != err { + d.error = fmt.Errorf("Network Failure: " + err.Error()) + return err + } + + if !bytes.Contains(b, []byte(d.Keywords)) { + err = fmt.Errorf("Down: '%s' Not Found for '%s'", d.Keywords, d.Name) + d.logger <- fmt.Sprintf("%s", err) + d.error = err + return err + } else { + d.logger <- fmt.Sprintf("Up: '%s'", d.Name) + } + + return nil +} + +func (d *Dog) recover() { + if "" == d.Recover { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + cmd := exec.CommandContext(ctx, "bash") + pipe, err := cmd.StdinPipe() + pipe.Write([]byte(d.Recover)) + if nil != err { + d.logger <- fmt.Sprintf("[Recover] Could not write to bash '%s': %s", d.Recover, err) + } + err = cmd.Start() + if nil != err { + d.logger <- fmt.Sprintf("[Recover] Could not start '%s': %s", d.Recover, err) + } + err = pipe.Close() + if nil != err { + d.logger <- fmt.Sprintf("[Recover] Could not close '%s': %s", d.Recover, err) + } + err = cmd.Wait() + cancel() + if nil != err { + d.logger <- fmt.Sprintf("[Recover] '%s' failed: %s", d.Recover, err) + } +} + +func (d *Dog) notify(hardFail bool) { + d.logger <- fmt.Sprintf("Notifying the authorities of %s's failure", d.Name) + d.lastNotified = time.Now() + + for i := range d.Webhooks { + name := d.Webhooks[i] + if "" == name { + continue + } + + h, ok := d.AllWebhooks[name] + if !ok { + // TODO check in main when config is read + d.Webhooks[i] = "" + d.logger <- fmt.Sprintf("[Warning] Could not find webhook '%s' for '%s'", name, h.Name) + continue + } + + // TODO do this in main on config init + if "" == h.Method { + h.Method = "POST" + } + + var body *strings.Reader + if 0 != len(h.Form) { + form := url.Values{} + for k := range h.Form { + v := h.Form[k] + // TODO real templates + v = strings.Replace(v, "{{ .Name }}", d.Name, -1) + form.Set(k, v) + } + body = strings.NewReader(form.Encode()) + } + + client := NewHTTPClient() + req, err := http.NewRequest(h.Method, h.URL, body) + if nil != err { + log.Println("[Notify] HTTP Client Network Error:", err) + continue + } + + if 0 != len(h.Form) { + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + } + + if 0 != len(h.Auth) { + user := h.Auth["user"] + pass := h.Auth["pass"] + req.SetBasicAuth(user, pass) + } + + resp, err := client.Do(req) + if nil != err { + d.logger <- fmt.Sprintf("[Notify] HTTP Client Error: %s", err) + continue + } + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + d.logger <- fmt.Sprintf("[Notify] Response Error: %s", resp.Status) + continue + } + + // TODO json vs xml vs txt + var data map[string]interface{} + req.Header.Add("Accept", "application/json") + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&data) + if err != nil { + d.logger <- fmt.Sprintf("[Notify] Response Body Error: %s", resp.Status) + continue + } + + // TODO some sort of way to determine if data is successful (keywords) + d.logger <- fmt.Sprintf("[Notify] Success? %#v", data) + } +} + +type Config struct { + Watches []ConfigWatch `json:"watches"` + Webhooks []ConfigWebhook `json:"webhooks"` +} + +type ConfigWatch struct { + Name string `json:"name"` + URL string `json:"url"` + Keywords string `json:"keywords"` + Webhooks []string `json:"webhooks"` + RecoverScript string `json:"recover_script"` +} + +type ConfigWebhook struct { + Name string `json:"name"` + Method string `json:"method"` + URL string `json:"url"` + Auth map[string]string `json:"auth"` + Form map[string]string `json:"form"` + Config map[string]string `json:"config"` + Configs []map[string]string `json:"configs"` +} + func NewHTTPClient() *http.Client { transport := &http.Transport{ Dial: (&net.Dialer{ @@ -174,92 +332,11 @@ func NewHTTPClient() *http.Client { return client } -func (w *Dog) check() error { - var err error - defer func() { - if nil != err { - w.failures += 1 - w.lastFailed = time.Now() - } else { - w.lastPassed = time.Now() - w.passes += 1 - } - }() - - client := NewHTTPClient() - response, err := client.Get(w.CheckURL) - if nil != err { - w.error = fmt.Errorf("Connection Failure: " + err.Error()) - return err +func logger(msgs chan string) { + for { + msg := <-msgs + log.Println(msg) } - - b, err := ioutil.ReadAll(response.Body) - if nil != err { - w.error = fmt.Errorf("Network Failure: " + err.Error()) - return err - } - - if !bytes.Contains(b, []byte(w.Keywords)) { - err = fmt.Errorf("Keywords Not Found: " + w.Keywords) - fmt.Println(err) - w.error = err - return err - } else { - fmt.Println("Happy day!") - } - - return nil -} - -func (w *Dog) notify(hardFail bool) error { - w.lastNotified = time.Now() - fmt.Println("Notifying the authorities of a failure") - return nil - /* - urlStr := "https://api.twilio.com/2010-04-01/Accounts/" + accountSid + "/Messages.json" - msgData := url.Values{} - msgData.Set("To", "+1 555 555 5555") - msgData.Set("From", "+1 555 555 1234") - if hardFail { - msgData.Set("Body", fmt.Sprintf("[%s] The system is down. The system is down.", w.Name)) - } else { - msgData.Set("Body", fmt.Sprintf("[%s] had a hiccup.", w.Name)) - } - msgDataReader := *strings.NewReader(msgData.Encode()) - client := NewHTTPClient() - req, err := http.NewRequest("POST", urlStr, &msgDataReader) - if nil != err { - fmt.Fprintf(os.Stderr, "Failed to text: %s\n", err) - } - req.SetBasicAuth(accountSid, authToken) - req.Header.Add("Accept", "application/json") - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - resp, _ := client.Do(req) - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - var data map[string]interface{} - decoder := json.NewDecoder(resp.Body) - err := decoder.Decode(&data) - if err == nil { - fmt.Println(data["sid"]) - } - } else { - fmt.Println("Error:", resp.Status) - } - - return nil - */ -} - -type Config struct { - Watches []ConfigWatch `json:"watches"` -} - -type ConfigWatch struct { - Name string `json:"name"` - URL string `json:"url"` - Keywords string `json:"keywords"` - Webhook string `json:"webhook"` - RecoverScript string `json:"recover_script"` } /*