v1.2.2: improved change detection

.gitignore

@@ -0,0 +1,4 @@
+/watchdog
+/cmd/watchdog/watchdog
+xversion.go
+*.json

README.md

@@ -11,17 +11,117 @@ Can work with email, text (sms), push notifications, etc.
 
 # Install
 
+## Downloads
+
+### MacOS
+
+MacOS (darwin): [64-bit Download ](https://rootprojects.org/watchdog/dist/darwin/amd64/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/darwin/amd64/watchdog -o watchdog
+```
+
+### Windows
+
+<details>
+<summary>See download options</summary>
+Windows 10: [64-bit Download](https://rootprojects.org/watchdog/dist/windows/amd64/watchdog.exe)
+
+```
+powershell.exe $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest https://rootprojects.org/watchdog/dist/windows/amd64/watchdog.exe -OutFile watchdog.exe
+```
+
+Windows 7: [32-bit Download](https://rootprojects.org/watchdog/dist/windows/386/watchdog.exe)
+
+```
+powershell.exe "(New-Object Net.WebClient).DownloadFile('https://rootprojects.org/watchdog/dist/windows/386/watchdog.exe', 'watchdog.exe')"
+```
+
+</details>
+
+### Linux
+
+<details>
+<summary>See download options</summary>
+
+Linux (64-bit): [Download](https://rootprojects.org/watchdog/dist/linux/amd64/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/amd64/watchdog -o watchdog
+```
+
+Linux (32-bit): [Download](https://rootprojects.org/watchdog/dist/linux/386/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/386/watchdog -o watchdog
+```
+
+</details>
+
+### Raspberry Pi (Linux ARM)
+
+<details>
+<summary>See download options</summary>
+
+RPi 4 (64-bit armv8): [Download](https://rootprojects.org/watchdog/dist/linux/armv8/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv8/watchdog -o watchdog`
+```
+
+RPi 3 (armv7): [Download](https://rootprojects.org/watchdog/dist/linux/armv7/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv7/watchdog -o watchdog
+```
+
+ARMv6: [Download](https://rootprojects.org/watchdog/dist/linux/armv6/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv6/watchdog -o watchdog
+```
+
+RPi Zero (armv5): [Download](https://rootprojects.org/watchdog/dist/linux/armv5/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv5/watchdog -o watchdog
+```
+
+</details>
+
+## Git:
+
 ```bash
 git clone https://git.coolaj86.com/coolaj86/watchdog.go.git
+pushd watchdog.go/
+go generate -mod=vendor ./...
+pushd cmd/watchdog
+go build -mod=vendor
 ```
 
 # Usage
 
+Mac, Linux:
+
 ```bash
-pushd watchdog.go/
-go run ./watchdog.go -c dog.json
+./watchdog -c config.json
 ```
 
+Windows:
+
+```bash
+watchdog.exe -c config.json
+```
+
+# Changelog
+
+- v1.2.0
+  - report when sites come back up
+  - and more template vars
+  - and localization for status
+- v1.1.0 support `json` request bodies (for Pushbullet)
+- v1.0.0 support Twilio and Mailgun
+
 # Getting Started
 
 <details>
@@ -52,6 +152,23 @@ Be careful of "smart quotes" and HTML entities:
 - `We’re Open!` is not `We're Open!`
 - Neither is `We&apos;re Open!` nor `We&#39;re Open!`
 
+Leave empty for No Content pages, such as redirects.
+
+### `badwords`
+
+The opposite of `keywords`.
+
+If a literal, exact match of badwords exists as part of the response, the site is considered to be down.
+
+Ignored if empty.
+
+### `localizations`
+
+Normally `{{ .Status }}` will be `"up"` or `"down"` and `{{ .Message }}` will be `"is down"` or `"came back up"`.
+Localizations allow you to swap that out for something else.
+
+I added this so that I could use "🔥🔥🔥" and "👍" for myself without imposing upon others.
+
 ### `webhooks`
 
 This references the arbitrary `name` of a webhook in the `webhooks` array.
@@ -88,7 +205,10 @@ command="systemctl restart foo.service",no-port-forwarding,no-x11-forwarding,no-
 <details>
   <summary>{{ .Name }} and other template variables</summary>
 
-`{{ .Name }}` is the only template variable right now.
+- `{{ .Name }}` is the name of your site.
+- `{{ .Message }}` is either `went down` or `came back up`.
+- `{{ .Status }}` is either `up` or `down`.
+- `{{ .Watchdog }}` is the name of your watchdog (useful if you have multiple).
 
 It refers to the name of the watch, which is "Example Site" in the sample config below.
 
@@ -123,6 +243,35 @@ The `from` address can be _any_ domain in your mailgun account.
 
 `subject` is the plain-text subject and `text` must be plain-text (not html) email contents.
 
+</details>
+<details>
+  <summary>How to use with Pushbullet</summary>
+
+Pushbullet is a push notification service that I found pretty easy to work with.
+I logged in with my iPhone through facebook, as well as in a web browser,
+grabbed the API token, and I was able to test the documentation all in just a few minutes.
+
+### `my_pushbullet`
+
+Replace `my_pushbullet` with whatever name you like, or leave it the same.
+
+It's an arbitrary name for you to reference.
+For example, you may have different Pushbullet configurations for different domains.
+
+### `xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`
+
+Replace `xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx` in the HTTP Access-Token header
+with your API token.
+
+You'll find this on the Pushbullet dashboard under Settings, Account, Access Tokens,
+and you'll have to click to create it:
+
+- <https://www.pushbullet.com/#settings/account>
+
+The example was taken from the Pushbullet API documentation:
+
+- <https://docs.pushbullet.com/#create-push>
+
 </details>
 <details>
   <summary>How to use with Twilio</summary>
@@ -164,7 +313,7 @@ All phone numbers should have the country code prefix (`+1` for USA) attached.
 <details>
   <summary>How to use with Nexmo, Mailjet, and other webhook-enabled services</summary>
 
-See the examples of Twilio and Mailgun.
+See the examples of Mailgun, Pushbullet, Twilio.
 
 Look for "curl" in the documentation of the service that you're using.
 It should be fairly easy to just look at the headers that are being set and repeat.
@@ -175,16 +324,18 @@ It should be fairly easy to just look at the headers that are being set and repe
 
 You can set notifications for _any_ service that supports HTTPS webhooks.
 
-The examples below are shown with Twilio and Mailgun, as taken from their `curl` documentation.
+The examples below are shown with Mailgun, Pushbullet, and Twilio, as taken from their `curl` documentation.
 
 ```json
 {
+  "watchdog": "Monitor A",
   "watches": [
     {
       "name": "Example Site",
       "url": "https://example.com/",
       "keywords": "My Site",
-      "webhooks": ["my_mailgun", "my_twilio"],
+      "badwords": "Could not connect to database.",
+      "webhooks": ["my_mailgun", "my_pushbullet", "my_twilio"],
       "recover_script": "systemctl restart example-site"
     }
   ],
@@ -203,8 +354,22 @@ The examples below are shown with Twilio and Mailgun, as taken from their `curl`
       "form": {
         "from": "Watchdog <watchdog@my.example.com>",
         "to": "jon.doe@gmail.com",
-        "subject": "{{ .Name }} is down.",
-        "text": "The system is down. Check up on {{ .Name }} ASAP."
+        "subject": "[{{ .Watchdog }}] {{ .Name }} {{ .Message }}.",
+        "text": "{{ .Name }} {{ .Message }}. Reported by {{ .Watchdog }}."
+      }
+    },
+    {
+      "name": "my_pushbullet",
+      "method": "POST",
+      "url": "https://api.pushbullet.com/v2/pushes",
+      "headers": {
+        "Access-Token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+        "User-Agent": "Watchdog/1.0"
+      },
+      "json": {
+        "body": "The system {{ .Message }}. Check up on {{ .Name }} ASAP.",
+        "title": "{{ .Name }} {{ .Message }}.",
+        "type": "note"
       }
     },
     {
@@ -224,7 +389,11 @@ The examples below are shown with Twilio and Mailgun, as taken from their `curl`
         "Body": "[{{ .Name }}] The system is down. The system is down."
       }
     }
-  ]
+  ],
+  "localizations": {
+    "up": "👍",
+    "down": "🔥🔥🔥"
+  }
 }
 ```
 

build-all.sh

@@ -0,0 +1,48 @@
+#GOOS=windows GOARCH=amd64 go install
+#go tool dist list
+
+# TODO move this into tools/build.go
+
+export CGO_ENABLED=0
+exe=watchdog
+distpre=../..
+gocmd=.
+
+echo ""
+go generate -mod=vendor ./...
+
+pushd cmd/${exe}
+echo ""
+echo "Windows amd64"
+#GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.exe -ldflags "-H=windowsgui" $gocmd
+#GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.debug.exe
+GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.exe
+echo "Windows 386"
+#GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.exe -ldflags "-H=windowsgui" $gocmd
+#GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.debug.exe
+GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.exe
+
+echo ""
+echo "Darwin (macOS) amd64"
+GOOS=darwin GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/darwin/amd64/${exe} $gocmd
+
+echo ""
+echo "Linux amd64"
+GOOS=linux GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/linux/amd64/${exe} $gocmd
+echo "Linux 386"
+GOOS=linux GOARCH=386 go build -mod=vendor -o ${distpre}/dist/linux/386/${exe} $gocmd
+
+echo ""
+echo "RPi 4 (64-bit) ARMv8"
+GOOS=linux GOARCH=arm64 go build -mod=vendor -o ${distpre}/dist/linux/armv8/${exe} $gocmd
+echo "RPi 3 B+ ARMv7"
+GOOS=linux GOARCH=arm GOARM=7 go build -mod=vendor -o ${distpre}/dist/linux/armv7/${exe} $gocmd
+echo "ARMv6"
+GOOS=linux GOARCH=arm GOARM=6 go build -mod=vendor -o ${distpre}/dist/linux/armv6/${exe} $gocmd
+echo "RPi Zero ARMv5"
+GOOS=linux GOARCH=arm GOARM=5 go build -mod=vendor -o ${distpre}/dist/linux/armv5/${exe} $gocmd
+
+echo ""
+popd
+rsync -av ./dist/ ubuntu@rootprojects.org:/srv/www/rootprojects.org/$exe/dist/
+# https://rootprojects.org/serviceman/dist/windows/amd64/serviceman.exe

cmd/watchdog/version.go

@@ -0,0 +1,8 @@
+package main
+
+// Fallback to recent version if not in a git repository
+func init() {
+	GitRev = "d5c026948cf134997c7260e78d4bd5864ac5b9b3"
+	GitVersion = "v1.1.3"
+	GitTimestamp = "2019-06-21T01:03:19-06:00"
+}

cmd/watchdog/watchdog.go

@@ -0,0 +1,116 @@
+//go:generate go run -mod=vendor git.rootprojects.org/root/go-gitver
+
+// Watchdog Binary
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	watchdog "git.rootprojects.org/root/go-watchdog"
+)
+
+var GitRev, GitVersion, GitTimestamp string
+
+func usage() {
+	fmt.Println("Usage: watchdog -c config.json")
+}
+
+func main() {
+	for i := range os.Args {
+		switch {
+		case strings.HasSuffix(os.Args[i], "version"):
+			fmt.Println(GitTimestamp)
+			fmt.Println(GitVersion)
+			fmt.Println(GitRev)
+			os.Exit(0)
+		case strings.HasSuffix(os.Args[i], "help"):
+			usage()
+			os.Exit(0)
+		}
+	}
+
+	if 3 != len(os.Args) {
+		usage()
+		os.Exit(1)
+		return
+	}
+	if "-c" != os.Args[1] {
+		usage()
+		os.Exit(1)
+		return
+	}
+
+	filename := os.Args[2]
+	f, err := os.Open(filename)
+	if nil != err {
+		log.Fatal(err)
+		return
+	}
+
+	configFile, err := ioutil.ReadAll(f)
+	if nil != err {
+		log.Fatal(err)
+		return
+	}
+
+	config := &watchdog.Config{}
+	err = json.Unmarshal(configFile, config)
+	if nil != err {
+		log.Fatal(err)
+		return
+	}
+
+	//fmt.Printf("%#v\n", config)
+
+	done := make(chan struct{}, 1)
+
+	allWebhooks := make(map[string]watchdog.Webhook)
+
+	for i := range config.Webhooks {
+		h := config.Webhooks[i]
+		allWebhooks[h.Name] = h
+	}
+
+	logQueue := make(chan string, 10)
+	go logger(logQueue)
+	for i := range config.Watches {
+		c := config.Watches[i]
+		logQueue <- fmt.Sprintf("Watching '%s'", c.Name)
+		go func(c watchdog.ConfigWatch) {
+			d := watchdog.New(&watchdog.Dog{
+				Watchdog:      config.Watchdog,
+				Name:          c.Name,
+				CheckURL:      c.URL,
+				Keywords:      c.Keywords,
+				Badwords:      c.Badwords,
+				Localizations: config.Localizations,
+				Recover:       c.RecoverScript,
+				Webhooks:      c.Webhooks,
+				AllWebhooks:   allWebhooks,
+				Logger:        logQueue,
+			})
+			d.Watch()
+		}(config.Watches[i])
+	}
+
+	if 0 == len(config.Watches) {
+		log.Fatal("Nothing to watch")
+		return
+	}
+
+	<-done
+}
+
+// This is so that the log messages don't trample
+// over each other when they happen simultaneously.
+func logger(msgs chan string) {
+	for {
+		msg := <-msgs
+		log.Println(msg)
+	}
+}

doc.go

@@ -0,0 +1,9 @@
+// LIBARRY NOT DOCUMENTED.
+// LIBRARY NOT VERSIONED.
+// DO NOT USE YET.
+// The watchdog package is meant to be used as a binary only.
+// The git tag version describes the state of the binary,
+// not the state of the library. The API is not yet stable.
+//
+// See https://git.rootproject.org/root/go-watchdog for pre-built binaries.
+package watchdog

go.mod

@@ -1,3 +1,5 @@
-module git.coolaj86.com/coolaj86/watchdog.go
+module git.rootprojects.org/root/go-watchdog
 
 go 1.12
+
+require git.rootprojects.org/root/go-gitver v1.1.1

go.sum

@@ -0,0 +1,2 @@
+git.rootprojects.org/root/go-gitver v1.1.1 h1:5b0lxnTYnft5hqpln0XCrJaGPH0SKzhPaazVAvAlZ8I=
+git.rootprojects.org/root/go-gitver v1.1.1/go.mod h1:Rj1v3TBhvdaSphFEqMynUYwAz/4f+wY/+syBTvRrmlI=

tools/tools.go

@@ -0,0 +1,7 @@
+// +build tools
+
+package tools
+
+import (
+	_ "git.rootprojects.org/root/go-gitver"
+)

vendor/git.rootprojects.org/root/go-gitver/.gitignore

@@ -0,0 +1,17 @@
+xversion.go
+zversion.go
+
+# ---> Go
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+

vendor/git.rootprojects.org/root/go-gitver/LICENSE

@@ -0,0 +1,312 @@
+Mozilla Public License Version 2.0
+
+   1. Definitions
+
+1.1. "Contributor" means each individual or legal entity that creates, contributes
+to the creation of, or owns Covered Software.
+
+1.2. "Contributor Version" means the combination of the Contributions of others
+(if any) used by a Contributor and that particular Contributor's Contribution.
+
+      1.3. "Contribution" means Covered Software of a particular Contributor.
+
+1.4. "Covered Software" means Source Code Form to which the initial Contributor
+has attached the notice in Exhibit A, the Executable Form of such Source Code
+Form, and Modifications of such Source Code Form, in each case including portions
+thereof.
+
+      1.5. "Incompatible With Secondary Licenses" means
+
+(a) that the initial Contributor has attached the notice described in Exhibit
+B to the Covered Software; or
+
+(b) that the Covered Software was made available under the terms of version
+1.1 or earlier of the License, but not also under the terms of a Secondary
+License.
+
+1.6. "Executable Form" means any form of the work other than Source Code Form.
+
+1.7. "Larger Work" means a work that combines Covered Software with other
+material, in a separate file or files, that is not Covered Software.
+
+      1.8. "License" means this document.
+
+1.9. "Licensable" means having the right to grant, to the maximum extent possible,
+whether at the time of the initial grant or subsequently, any and all of the
+rights conveyed by this License.
+
+      1.10. "Modifications" means any of the following:
+
+(a) any file in Source Code Form that results from an addition to, deletion
+from, or modification of the contents of Covered Software; or
+
+(b) any new file in Source Code Form that contains any Covered Software.
+
+1.11. "Patent Claims" of a Contributor means any patent claim(s), including
+without limitation, method, process, and apparatus claims, in any patent Licensable
+by such Contributor that would be infringed, but for the grant of the License,
+by the making, using, selling, offering for sale, having made, import, or
+transfer of either its Contributions or its Contributor Version.
+
+1.12. "Secondary License" means either the GNU General Public License, Version
+2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General
+Public License, Version 3.0, or any later versions of those licenses.
+
+1.13. "Source Code Form" means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your") means an individual or a legal entity exercising rights
+under this License. For legal entities, "You" includes any entity that controls,
+is controlled by, or is under common control with You. For purposes of this
+definition, "control" means (a) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or otherwise,
+or (b) ownership of more than fifty percent (50%) of the outstanding shares
+or beneficial ownership of such entity.
+
+   2. License Grants and Conditions
+
+      2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive
+license:
+
+(a) under intellectual property rights (other than patent or trademark) Licensable
+by such Contributor to use, reproduce, make available, modify, display, perform,
+distribute, and otherwise exploit its Contributions, either on an unmodified
+basis, with Modifications, or as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer for
+sale, have made, import, and otherwise transfer either its Contributions or
+its Contributor Version.
+
+      2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution become
+effective for each Contribution on the date the Contributor first distributes
+such Contribution.
+
+      2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under this
+License. No additional rights or licenses will be implied from the distribution
+or licensing of Covered Software under this License. Notwithstanding Section
+2.1(b) above, no patent license is granted by a Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software; or
+
+(b) for infringements caused by: (i) Your and any other third party's modifications
+of Covered Software, or (ii) the combination of its Contributions with other
+software (except as part of its Contributor Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of its
+Contributions.
+
+This License does not grant any rights in the trademarks, service marks, or
+logos of any Contributor (except as may be necessary to comply with the notice
+requirements in Section 3.4).
+
+      2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to distribute
+the Covered Software under a subsequent version of this License (see Section
+10.2) or under the terms of a Secondary License (if permitted under the terms
+of Section 3.3).
+
+      2.5. Representation
+
+Each Contributor represents that the Contributor believes its Contributions
+are its original creation(s) or it has sufficient rights to grant the rights
+to its Contributions conveyed by this License.
+
+      2.6. Fair Use
+
+This License is not intended to limit any rights You have under applicable
+copyright doctrines of fair use, fair dealing, or other equivalents.
+
+      2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
+Section 2.1.
+
+   3. Responsibilities
+
+      3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any Modifications
+that You create or to which You contribute, must be under the terms of this
+License. You must inform recipients that the Source Code Form of the Covered
+Software is governed by the terms of this License, and how they can obtain
+a copy of this License. You may not attempt to alter or restrict the recipients'
+rights in the Source Code Form.
+
+      3.2. Distribution of Executable Form
+
+      If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code Form,
+as described in Section 3.1, and You must inform recipients of the Executable
+Form how they can obtain a copy of such Source Code Form by reasonable means
+in a timely manner, at a charge no more than the cost of distribution to the
+recipient; and
+
+(b) You may distribute such Executable Form under the terms of this License,
+or sublicense it under different terms, provided that the license for the
+Executable Form does not attempt to limit or alter the recipients' rights
+in the Source Code Form under this License.
+
+      3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice, provided
+that You also comply with the requirements of this License for the Covered
+Software. If the Larger Work is a combination of Covered Software with a work
+governed by one or more Secondary Licenses, and the Covered Software is not
+Incompatible With Secondary Licenses, this License permits You to additionally
+distribute such Covered Software under the terms of such Secondary License(s),
+so that the recipient of the Larger Work may, at their option, further distribute
+the Covered Software under the terms of either this License or such Secondary
+License(s).
+
+      3.4. Notices
+
+You may not remove or alter the substance of any license notices (including
+copyright notices, patent notices, disclaimers of warranty, or limitations
+of liability) contained within the Source Code Form of the Covered Software,
+except that You may alter any license notices to the extent required to remedy
+known factual inaccuracies.
+
+      3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support, indemnity
+or liability obligations to one or more recipients of Covered Software. However,
+You may do so only on Your own behalf, and not on behalf of any Contributor.
+You must make it absolutely clear that any such warranty, support, indemnity,
+or liability obligation is offered by You alone, and You hereby agree to indemnify
+every Contributor for any liability incurred by such Contributor as a result
+of warranty, support, indemnity or liability terms You offer. You may include
+additional disclaimers of warranty and limitations of liability specific to
+any jurisdiction.
+
+   4. Inability to Comply Due to Statute or Regulation
+
+If it is impossible for You to comply with any of the terms of this License
+with respect to some or all of the Covered Software due to statute, judicial
+order, or regulation then You must: (a) comply with the terms of this License
+to the maximum extent possible; and (b) describe the limitations and the code
+they affect. Such description must be placed in a text file included with
+all distributions of the Covered Software under this License. Except to the
+extent prohibited by statute or regulation, such description must be sufficiently
+detailed for a recipient of ordinary skill to be able to understand it.
+
+   5. Termination
+
+5.1. The rights granted under this License will terminate automatically if
+You fail to comply with any of its terms. However, if You become compliant,
+then the rights granted under this License from a particular Contributor are
+reinstated (a) provisionally, unless and until such Contributor explicitly
+and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor
+fails to notify You of the non-compliance by some reasonable means prior to
+60 days after You have come back into compliance. Moreover, Your grants from
+a particular Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the first
+time You have received notice of non-compliance with this License from such
+Contributor, and You become compliant prior to 30 days after Your receipt
+of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent infringement
+claim (excluding declaratory judgment actions, counter-claims, and cross-claims)
+alleging that a Contributor Version directly or indirectly infringes any patent,
+then the rights granted to You by any and all Contributors for the Covered
+Software under Section 2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end
+user license agreements (excluding distributors and resellers) which have
+been validly granted by You or Your distributors under this License prior
+to termination shall survive termination.
+
+   6. Disclaimer of Warranty
+
+Covered Software is provided under this License on an "as is" basis, without
+warranty of any kind, either expressed, implied, or statutory, including,
+without limitation, warranties that the Covered Software is free of defects,
+merchantable, fit for a particular purpose or non-infringing. The entire risk
+as to the quality and performance of the Covered Software is with You. Should
+any Covered Software prove defective in any respect, You (not any Contributor)
+assume the cost of any necessary servicing, repair, or correction. This disclaimer
+of warranty constitutes an essential part of this License. No use of any Covered
+Software is authorized under this License except under this disclaimer.
+
+   7. Limitation of Liability
+
+Under no circumstances and under no legal theory, whether tort (including
+negligence), contract, or otherwise, shall any Contributor, or anyone who
+distributes Covered Software as permitted above, be liable to You for any
+direct, indirect, special, incidental, or consequential damages of any character
+including, without limitation, damages for lost profits, loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all other commercial
+damages or losses, even if such party shall have been informed of the possibility
+of such damages. This limitation of liability shall not apply to liability
+for death or personal injury resulting from such party's negligence to the
+extent applicable law prohibits such limitation. Some jurisdictions do not
+allow the exclusion or limitation of incidental or consequential damages,
+so this exclusion and limitation may not apply to You.
+
+   8. Litigation
+
+Any litigation relating to this License may be brought only in the courts
+of a jurisdiction where the defendant maintains its principal place of business
+and such litigation shall be governed by laws of that jurisdiction, without
+reference to its conflict-of-law provisions. Nothing in this Section shall
+prevent a party's ability to bring cross-claims or counter-claims.
+
+   9. Miscellaneous
+
+This License represents the complete agreement concerning the subject matter
+hereof. If any provision of this License is held to be unenforceable, such
+provision shall be reformed only to the extent necessary to make it enforceable.
+Any law or regulation which provides that the language of a contract shall
+be construed against the drafter shall not be used to construe this License
+against a Contributor.
+
+   10. Versions of the License
+
+      10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section 10.3,
+no one other than the license steward has the right to modify or publish new
+versions of this License. Each version will be given a distinguishing version
+number.
+
+      10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version of
+the License under which You originally received the Covered Software, or under
+the terms of any subsequent version published by the license steward.
+
+      10.3. Modified Versions
+
+If you create software not governed by this License, and you want to create
+a new license for such software, you may create and use a modified version
+of this License if you rename the license and remove any references to the
+name of the license steward (except to note that such modified license differs
+from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With Secondary
+Licenses under the terms of this version of the License, the notice described
+in Exhibit B of this License must be attached. Exhibit A - Source Code Form
+License Notice
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular file,
+then You may include the notice in a location (such as a LICENSE file in a
+relevant directory) where a recipient would be likely to look for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+
+This Source Code Form is "Incompatible With Secondary Licenses", as defined
+by the Mozilla Public License, v. 2.0.

vendor/git.rootprojects.org/root/go-gitver/README.md

@@ -0,0 +1,178 @@
+# git-version.go
+
+Use git tags to add semver to your go package.
+
+```txt
+Goal: Either use an exact version like v1.0.0
+      or translate the git version like v1.0.0-4-g0000000
+      to a semver like v1.0.1-pre4+g0000000
+
+      Fail gracefully when git repo isn't available.
+```
+
+# Demo
+
+Generate an `xversion.go` file:
+
+```bash
+go run git.rootprojects.org/root/go-gitver
+cat xversion.go
+```
+
+<small>**Note**: The file is named `xversion.go` by default so that the
+generated file's `init()` will come later, and thus take priority, over
+most other files.</small>
+
+See `go-gitver`s self-generated version:
+
+```bash
+go run git.rootprojects.org/root/go-gitver version
+```
+
+# QuickStart
+
+Add this to the top of your main file:
+
+```go
+//go:generate go run -mod=vendor git.rootprojects.org/root/go-gitver
+
+```
+
+Add a file that imports go-gitver (for versioning)
+
+```go
+// +build tools
+
+package example
+
+import _ "git.rootprojects.org/root/go-gitver"
+```
+
+Change you build instructions to be something like this:
+
+```bash
+go mod vendor
+go generate -mod=vendor ./...
+go build -mod=vendor -o example cmd/example/*.go
+```
+
+You don't have to use `mod vendor`, but I highly recommend it.
+
+# Options
+
+```txt
+version           print version and exit
+--fail            exit with non-zero status code on failure
+--package <name>  will set the package name
+--outfile <name>  will replace `xversion.go` with the given file path
+```
+
+ENVs
+
+```bash
+# Alias for --fail
+GITVER_FAIL=true
+```
+
+For example:
+
+```go
+//go:generate go run -mod=vendor git.rootprojects.org/root/go-gitver --fail
+
+```
+
+```bash
+go run -mod=vendor git.rootprojects.org/root/go-gitver version
+```
+
+# Usage
+
+See `examples/basic`
+
+1. Create a `tools` package in your project
+2. Guard it against regular builds with `// +build tools`
+3. Include `_ "git.rootprojects.org/root/go-gitver"` in the imports
+4. Declare `var GitRev, GitVersion, GitTimestamp string` in your `package main`
+5. Include `//go:generate go run -mod=vendor git.rootprojects.org/root/go-gitver` as well
+
+`tools/tools.go`:
+
+```go
+// +build tools
+
+// This is a dummy package for build tooling
+package tools
+
+import (
+	_ "git.rootprojects.org/root/go-gitver"
+)
+```
+
+`main.go`:
+
+```go
+//go:generate go run git.rootprojects.org/root/go-gitver --fail
+
+package main
+
+import "fmt"
+
+var (
+	GitRev       = "0000000"
+	GitVersion   = "v0.0.0-pre0+0000000"
+	GitTimestamp = "0000-00-00T00:00:00+0000"
+)
+
+func main() {
+  fmt.Println(GitRev)
+  fmt.Println(GitVersion)
+  fmt.Println(GitTimestamp)
+}
+```
+
+If you're using `go mod vendor` (which I highly recommend that you do),
+you'd modify the `go:generate` ever so slightly:
+
+```go
+//go:generate go run -mod=vendor git.rootprojects.org/root/go-gitver --fail
+```
+
+The only reason I didn't do that in the example is that I'd be included
+the repository in itself and that would be... weird.
+
+# Why a tools package?
+
+> import "git.rootprojects.org/root/go-gitver" is a program, not an importable package
+
+Having a tools package with a build tag that you don't use is a nice way to add exact
+versions of a command package used for tooling to your `go.mod` with `go mod tidy`,
+without getting the error above.
+
+# git: behind the curtain
+
+These are the commands that are used under the hood to produce the versions.
+
+Shows the git tag + description. Assumes that you're using the semver format `v1.0.0` for your base tags.
+
+```bash
+git describe --tags --dirty --always
+# v1.0.0
+# v1.0.0-1-g0000000
+# v1.0.0-dirty
+```
+
+Show the commit date (when the commit made it into the current tree).
+Internally we use the current date when the working tree is dirty.
+
+```bash
+git show v1.0.0-1-g0000000 --format=%cd --date=format:%Y-%m-%dT%H:%M:%SZ%z --no-patch
+# 2010-01-01T20:30:00Z-0600
+# fatal: ambiguous argument 'v1.0.0-1-g0000000-dirty': unknown revision or path not in the working tree.
+```
+
+Shows the most recent commit.
+
+```bash
+git rev-parse HEAD
+# 0000000000000000000000000000000000000000
+```

vendor/git.rootprojects.org/root/go-gitver/gitver.go

@@ -0,0 +1,217 @@
+//go:generate go run -mod=vendor git.rootprojects.org/root/go-gitver
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/format"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+)
+
+var exitCode int
+var exactVer *regexp.Regexp
+var gitVer *regexp.Regexp
+var verFile = "xversion.go"
+
+var (
+	GitRev       = "0000000"
+	GitVersion   = "v0.0.0-pre0+g0000000"
+	GitTimestamp = "0000-00-00T00:00:00+0000"
+)
+
+func init() {
+	// exactly vX.Y.Z (go-compatible semver)
+	exactVer = regexp.MustCompile(`^v\d+\.\d+\.\d+$`)
+
+	// vX.Y.Z-n-g0000000 git post-release, semver prerelease
+	// vX.Y.Z-dirty git post-release, semver prerelease
+	gitVer = regexp.MustCompile(`^(v\d+\.\d+)\.(\d+)(-(\d+))?(-(g[0-9a-f]+))?(-(dirty))?`)
+}
+
+func main() {
+	pkg := "main"
+
+	args := os.Args[1:]
+	for i := range args {
+		arg := args[i]
+		if "-f" == arg || "--fail" == arg {
+			exitCode = 1
+		} else if ("--outfile" == arg || "-o" == arg) && len(args) > i+1 {
+			verFile = args[i+1]
+			args[i+1] = ""
+		} else if "--package" == arg && len(args) > i+1 {
+			pkg = args[i+1]
+			args[i+1] = ""
+		} else if "-V" == arg || "version" == arg || "-version" == arg || "--version" == arg {
+			fmt.Println(GitRev)
+			fmt.Println(GitVersion)
+			fmt.Println(GitTimestamp)
+			os.Exit(0)
+		}
+	}
+	if "" != os.Getenv("GITVER_FAIL") && "false" != os.Getenv("GITVER_FAIL") {
+		exitCode = 1
+	}
+
+	desc, err := gitDesc()
+	if nil != err {
+		log.Fatalf("Failed to get git version: %s\n", err)
+		os.Exit(exitCode)
+	}
+	rev := gitRev()
+	ver := semVer(desc)
+	ts, err := gitTimestamp(desc)
+	if nil != err {
+		ts = time.Now()
+	}
+
+	v := struct {
+		Package   string
+		Timestamp string
+		Version   string
+		GitRev    string
+	}{
+		Package:   pkg,
+		Timestamp: ts.Format(time.RFC3339),
+		Version:   ver,
+		GitRev:    rev,
+	}
+
+	// Create or overwrite the go file from template
+	var buf bytes.Buffer
+	if err := versionTpl.Execute(&buf, v); nil != err {
+		panic(err)
+	}
+
+	// Format
+	src, err := format.Source(buf.Bytes())
+	if nil != err {
+		panic(err)
+	}
+
+	// Write to disk (in the Current Working Directory)
+	f, err := os.Create(verFile)
+	if nil != err {
+		panic(err)
+	}
+	if _, err := f.Write(src); nil != err {
+		panic(err)
+	}
+	if err := f.Close(); nil != err {
+		panic(err)
+	}
+}
+
+func gitDesc() (string, error) {
+	args := strings.Split("git describe --tags --dirty --always", " ")
+	cmd := exec.Command(args[0], args[1:]...)
+	out, err := cmd.CombinedOutput()
+	if nil != err {
+		// Don't panic, just carry on
+		//out = []byte("v0.0.0-0-g0000000")
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func gitRev() string {
+	args := strings.Split("git rev-parse HEAD", " ")
+	cmd := exec.Command(args[0], args[1:]...)
+	out, err := cmd.CombinedOutput()
+	if nil != err {
+		fmt.Fprintf(os.Stderr,
+			"\nUnexpected Error\n\n"+
+				"Please open an issue at https://git.rootprojects.org/root/go-gitver/issues/new \n"+
+				"Please include the following:\n\n"+
+				"Command: %s\n"+
+				"Output: %s\n"+
+				"Error: %s\n"+
+				"\nPlease and Thank You.\n\n", strings.Join(args, " "), out, err)
+		os.Exit(exitCode)
+	}
+	return strings.TrimSpace(string(out))
+}
+
+func semVer(desc string) string {
+	if exactVer.MatchString(desc) {
+		// v1.0.0
+		return desc
+	}
+
+	if !gitVer.MatchString(desc) {
+		return ""
+	}
+
+	// (v1.0).(0)(-(1))(-(g0000000))(-(dirty))
+	vers := gitVer.FindStringSubmatch(desc)
+	patch, err := strconv.Atoi(vers[2])
+	if nil != err {
+		fmt.Fprintf(os.Stderr,
+			"\nUnexpected Error\n\n"+
+				"Please open an issue at https://git.rootprojects.org/root/go-gitver/issues/new \n"+
+				"Please include the following:\n\n"+
+				"git description: %s\n"+
+				"RegExp: %#v\n"+
+				"Error: %s\n"+
+				"\nPlease and Thank You.\n\n", desc, gitVer, err)
+		os.Exit(exitCode)
+	}
+
+	// v1.0.1-pre1
+	// v1.0.1-pre1+g0000000
+	// v1.0.1-pre0+dirty
+	// v1.0.1-pre0+g0000000-dirty
+	if "" == vers[4] {
+		vers[4] = "0"
+	}
+	ver := fmt.Sprintf("%s.%d-pre%s", vers[1], patch+1, vers[4])
+	if "" != vers[6] || "dirty" == vers[8] {
+		ver += "+"
+		if "" != vers[6] {
+			ver += vers[6]
+			if "" != vers[8] {
+				ver += "-"
+			}
+		}
+		ver += vers[8]
+	}
+
+	return ver
+}
+
+func gitTimestamp(desc string) (time.Time, error) {
+	args := []string{
+		"git",
+		"show", desc,
+		"--format=%cd",
+		"--date=format:%Y-%m-%dT%H:%M:%SZ%z",
+		"--no-patch",
+	}
+	cmd := exec.Command(args[0], args[1:]...)
+	out, err := cmd.CombinedOutput()
+	if nil != err {
+		// a dirty desc was probably used
+		return time.Time{}, err
+	}
+	return time.Parse(time.RFC3339, strings.TrimSpace(string(out)))
+}
+
+var versionTpl = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
+package {{ .Package }}
+
+func init() {
+	GitRev = "{{ .GitRev }}"
+  {{- if .Version }}
+	GitVersion = "{{ .Version }}"
+	{{ end -}}
+	GitTimestamp = "{{ .Timestamp }}"
+}
+`))

vendor/git.rootprojects.org/root/go-gitver/go.mod

@@ -0,0 +1,3 @@
+module git.rootprojects.org/root/go-gitver
+
+go 1.12

vendor/git.rootprojects.org/root/go-gitver/version.go

@@ -0,0 +1,9 @@
+package main
+
+// use recently generated version info as a fallback
+// for when git isn't present (i.e. go run <url>)
+func init() {
+	GitRev = "9f05e2304ccd40ac8a6b6bdba176942b475e272f"
+	GitVersion = "v1.1.0"
+	GitTimestamp = "2019-06-21T00:01:09-06:00"
+}

vendor/modules.txt

@@ -0,0 +1,2 @@
+# git.rootprojects.org/root/go-gitver v1.1.1
+git.rootprojects.org/root/go-gitver

watchdog.go

@@ -1,4 +1,4 @@
-package main
+package watchdog
 
 import (
 	"bytes"
@@ -6,108 +6,64 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"os/exec"
 	"strings"
 	"time"
 )
 
-func usage() {
-	fmt.Println("Usage: go run watchdog.go -c dog.json")
+type Status int
+
+const (
+	StatusDown Status = iota
+	StatusUp
+)
+
+func (s Status) String() string {
+	// ... just wishing Go had enums like Rust...
+	switch s {
+	case StatusUp:
+		return "up"
+	case StatusDown:
+		return "down"
+	default:
+		return "[[internal error]]"
+	}
 }
 
-func main() {
-	if 3 != len(os.Args) {
-		usage()
-		os.Exit(1)
-		return
-	}
-	if "-c" != os.Args[1] {
-		usage()
-		os.Exit(1)
-		return
-	}
-
-	filename := os.Args[2]
-	f, err := os.Open(filename)
-	if nil != err {
-		log.Fatal(err)
-		return
-	}
-
-	configFile, err := ioutil.ReadAll(f)
-	if nil != err {
-		log.Fatal(err)
-		return
-	}
-
-	config := &Config{}
-	err = json.Unmarshal(configFile, config)
-	if nil != err {
-		log.Fatal(err)
-		return
-	}
-
-	//fmt.Printf("%#v\n", config)
-
-	done := make(chan struct{}, 1)
-
-	allWebhooks := make(map[string]ConfigWebhook)
-
-	for i := range config.Webhooks {
-		h := config.Webhooks[i]
-		allWebhooks[h.Name] = h
-	}
-
-	logQueue := make(chan string, 10)
-	go logger(logQueue)
-	for i := range config.Watches {
-		c := config.Watches[i]
-		logQueue <- fmt.Sprintf("Watching '%s'", c.Name)
-		go func(c ConfigWatch) {
-			d := New(&Dog{
-				Name:        c.Name,
-				CheckURL:    c.URL,
-				Keywords:    c.Keywords,
-				Recover:     c.RecoverScript,
-				Webhooks:    c.Webhooks,
-				AllWebhooks: allWebhooks,
-				logger:      logQueue,
-			})
-			d.Watch()
-		}(config.Watches[i])
-	}
-
-	if 0 == len(config.Watches) {
-		log.Fatal("Nothing to watch")
-		return
-	}
-
-	<-done
-}
+const (
+	MessageDown   = "went down"
+	MessageUp     = "came back up"
+	MessageHiccup = "hiccupped"
+)
 
 type Dog struct {
-	Name         string
-	CheckURL     string
-	Keywords     string
-	Recover      string
-	Webhooks     []string
-	AllWebhooks  map[string]ConfigWebhook
-	logger       chan string
-	error        error
-	failures     int
-	passes       int
-	lastFailed   time.Time
-	lastPassed   time.Time
-	lastNotified time.Time
+	Watchdog      string
+	Name          string
+	CheckURL      string
+	Keywords      string
+	Badwords      string
+	Localizations map[string]string
+	Recover       string
+	Webhooks      []string
+	AllWebhooks   map[string]Webhook
+	Logger        chan string
+	status        Status
+	changed       bool
+	error         error
+	//failures      int
+	//passes        int
+	//lastFailed    time.Time
+	//lastPassed    time.Time
+	//lastNotified time.Time
 }
 
 func New(d *Dog) *Dog {
-	d.lastPassed = time.Now().Add(-5 * time.Minute)
+	//d.lastPassed = time.Now().Add(-5 * time.Minute)
+	d.status = StatusUp
+	d.changed = false
 	return d
 }
 
@@ -120,51 +76,87 @@ func (d *Dog) Watch() {
 	}
 }
 
+// Now that I've added the ability to notify when a server is back up
+// this definitely needs some refactoring. It's bad now.
 func (d *Dog) watch() {
-	d.logger <- fmt.Sprintf("Check: '%s'", d.Name)
+	d.Logger <- fmt.Sprintf("Check: '%s'", d.Name)
 
-	err := d.check()
+	// This may be up or down
+	err := d.hardcheck()
 	if nil == err {
+		d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
+		// if it's down, coming up, notify
+		if d.changed {
+			d.notify(MessageUp)
+		}
 		return
 	}
 
-	failure := false
+	// If being down is a change, check to see if it's just a hiccup
+	if d.changed {
+		time.Sleep(time.Duration(5) * time.Second)
+		err2 := d.softcheck()
+		if nil != err2 {
+			// it's really down
+			d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
+		} else {
+			// it's not really down, so reset the change info
+			d.changed = false
+			d.status = StatusUp
+			// and notify of the hiccup
+			d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
+			d.notify(MessageHiccup)
+			return
+		}
+	}
+
+	// TODO what if the server is flip-flopping rapidly?
+	// how to rate limit?
+	// "{{ .Server }} is on cooldown for 30 minutes"
+
+	// * We've had success since the last notification
+	// * It's been at least 5 minutes since the last notification
+	//fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
+	//if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
+	//}
+
 	t := 10
 	for {
+		// try to recover, then backoff exponentially
 		d.recover()
 		time.Sleep(time.Duration(t) * time.Second)
-		// backoff
 		t *= 2
-		err := d.check()
+		if t > 120 {
+			t = 120
+		}
+
+		err := d.softcheck()
 		if nil != err {
-			failure = true
-		}
-		// We should notify if
-		// * We've had success since the last notification
-		// * It's been at least 5 minutes since the last notification
-		fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
-		if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
-			d.notify(failure)
-		}
-		if d.failures >= 5 {
-			// go back to the main 5-minute loop
+			// this is down, and we know it's down
+			d.status = StatusDown
+			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
+			if d.changed {
+				d.changed = false
+				d.notify(MessageDown)
+			}
+		} else {
+			// it came back up
+			d.status = StatusUp
+			d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
+			if d.changed {
+				// and the downtime was short - just a recovery
+				d.notify(MessageHiccup)
+			} else {
+				// and the downtime was some time
+				d.notify(MessageUp)
+			}
+			d.changed = false
 			break
 		}
 	}
 }
 
-func (d *Dog) check() error {
-	var err error
-	defer func() {
-		if nil != err {
-			d.failures += 1
-			d.lastFailed = time.Now()
-		} else {
-			d.lastPassed = time.Now()
-			d.passes += 1
-		}
-	}()
-
+func (d *Dog) softcheck() error {
 	client := NewHTTPClient()
 	response, err := client.Get(d.CheckURL)
 	if nil != err {
@@ -178,18 +170,52 @@ func (d *Dog) check() error {
 		return err
 	}
 
+	// Note: empty matches empty as true, so this works for checking redirects
 	if !bytes.Contains(b, []byte(d.Keywords)) {
 		err = fmt.Errorf("Down: '%s' Not Found for '%s'", d.Keywords, d.Name)
-		d.logger <- fmt.Sprintf("%s", err)
+		d.Logger <- fmt.Sprintf("%s", err)
 		d.error = err
 		return err
-	} else {
-		d.logger <- fmt.Sprintf("Up: '%s'", d.Name)
+	}
+
+	if "" != d.Badwords {
+		if bytes.Contains(b, []byte(d.Badwords)) {
+			err = fmt.Errorf("Down: '%s' Found for '%s'", d.Badwords, d.Name)
+			d.Logger <- fmt.Sprintf("%s", err)
+			d.error = err
+			return err
+		}
 	}
 
 	return nil
 }
 
+func (d *Dog) hardcheck() error {
+	previousStatus := d.status
+
+	err := d.softcheck()
+
+	// Are we up, or down?
+	if nil != err {
+		d.status = StatusDown
+		//d.failures += 1
+		//d.lastFailed = time.Now()
+	} else {
+		d.status = StatusUp
+		//d.lastPassed = time.Now()
+		//d.passes += 1
+	}
+
+	// Has that changed?
+	if previousStatus != d.status {
+		d.changed = true
+	} else {
+		d.changed = false
+	}
+
+	return err
+}
+
 func (d *Dog) recover() {
 	if "" == d.Recover {
 		return
@@ -200,26 +226,26 @@ func (d *Dog) recover() {
 	pipe, err := cmd.StdinPipe()
 	pipe.Write([]byte(d.Recover))
 	if nil != err {
-		d.logger <- fmt.Sprintf("[Recover] Could not write to bash '%s': %s", d.Recover, err)
+		d.Logger <- fmt.Sprintf("[Recover] Could not write to bash '%s': %s", d.Recover, err)
 	}
 	err = cmd.Start()
 	if nil != err {
-		d.logger <- fmt.Sprintf("[Recover] Could not start '%s': %s", d.Recover, err)
+		d.Logger <- fmt.Sprintf("[Recover] Could not start '%s': %s", d.Recover, err)
 	}
 	err = pipe.Close()
 	if nil != err {
-		d.logger <- fmt.Sprintf("[Recover] Could not close '%s': %s", d.Recover, err)
+		d.Logger <- fmt.Sprintf("[Recover] Could not close '%s': %s", d.Recover, err)
 	}
 	err = cmd.Wait()
 	cancel()
 	if nil != err {
-		d.logger <- fmt.Sprintf("[Recover] '%s' failed: %s", d.Recover, err)
+		d.Logger <- fmt.Sprintf("[Recover] '%s' failed for '%s': %s", d.Recover, d.Name, err)
 	}
 }
 
-func (d *Dog) notify(hardFail bool) {
-	d.logger <- fmt.Sprintf("Notifying the authorities of %s's failure", d.Name)
-	d.lastNotified = time.Now()
+func (d *Dog) notify(msg string) {
+	d.Logger <- fmt.Sprintf("Notifying the authorities of %s's status change", d.Name)
+	//d.lastNotified = time.Now()
 
 	for i := range d.Webhooks {
 		name := d.Webhooks[i]
@@ -231,101 +257,139 @@ func (d *Dog) notify(hardFail bool) {
 		if !ok {
 			// TODO check in main when config is read
 			d.Webhooks[i] = ""
-			d.logger <- fmt.Sprintf("[Warning] Could not find webhook '%s' for '%s'", name, h.Name)
+			d.Logger <- fmt.Sprintf("[Warning] Could not find webhook '%s' for '%s'", name, h.Name)
 			continue
 		}
 
-		// TODO do this in main on config init
-		if "" == h.Method {
-			h.Method = "POST"
-		}
-
-		var body *strings.Reader
-		if 0 != len(h.Form) {
-			form := url.Values{}
-			for k := range h.Form {
-				v := h.Form[k]
-				// TODO real templates
-				v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
-				form.Set(k, v)
-			}
-			body = strings.NewReader(form.Encode())
-		}
-
-		client := NewHTTPClient()
-		req, err := http.NewRequest(h.Method, h.URL, body)
-		if nil != err {
-			log.Println("[Notify] HTTP Client Network Error:", err)
-			continue
-		}
-
-		if 0 != len(h.Form) {
-			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-		}
-
-		if 0 != len(h.Auth) {
-			user := h.Auth["user"]
-			if "" == user {
-				user = h.Auth["username"]
-			}
-			pass := h.Auth["pass"]
-			if "" == user {
-				pass = h.Auth["password"]
-			}
-			req.SetBasicAuth(user, pass)
-		}
-
-		req.Header.Set("User-Agent", "Watchdog/1.0")
-		for k := range h.Headers {
-			req.Header.Set(k, h.Headers[k])
-		}
-
-		resp, err := client.Do(req)
-		if nil != err {
-			d.logger <- fmt.Sprintf("[Notify] HTTP Client Error: %s", err)
-			continue
-		}
-
-		if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
-			d.logger <- fmt.Sprintf("[Notify] Response Error: %s", resp.Status)
-			continue
-		}
-
-		// TODO json vs xml vs txt
-		var data map[string]interface{}
-		req.Header.Add("Accept", "application/json")
-		decoder := json.NewDecoder(resp.Body)
-		err = decoder.Decode(&data)
-		if err != nil {
-			d.logger <- fmt.Sprintf("[Notify] Response Body Error: %s", resp.Status)
-			continue
-		}
-
-		// TODO some sort of way to determine if data is successful (keywords)
-		d.logger <- fmt.Sprintf("[Notify] Success? %#v", data)
+		d.notifyOne(h, msg)
 	}
 }
 
+func (d *Dog) notifyOne(h Webhook, msg string) {
+	// TODO do this in main on config init
+	if "" == h.Method {
+		h.Method = "POST"
+	}
+
+	var body *strings.Reader
+	var err error
+	// TODO real templates
+	if 0 != len(h.Form) {
+		form := url.Values{}
+		for k := range h.Form {
+			v := h.Form[k]
+			// because `{{` gets urlencoded
+			//k = strings.Replace(k, "{{ .Name }}", d.Name, -1)
+			v = strings.Replace(v, "{{ .Watchdog }}", d.Watchdog, -1)
+			v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
+			v = strings.Replace(v, "{{ .Status }}", d.localize(d.status.String()), -1)
+			v = strings.Replace(v, "{{ .Message }}", d.localize(msg), -1)
+			d.Logger <- fmt.Sprintf("[HEADER] %s: %s", k, v)
+			form.Set(k, v)
+		}
+		body = strings.NewReader(form.Encode())
+	} else if 0 != len(h.JSON) {
+		bodyBuf, err := json.Marshal(h.JSON)
+		if nil != err {
+			d.Logger <- fmt.Sprintf("[Notify] JSON Marshal Error for '%s': %s", h.Name, err)
+			return
+		}
+		// `{{` should be left alone
+		v := string(bodyBuf)
+		v = strings.Replace(v, "{{ .Watchdog }}", d.Watchdog, -1)
+		v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
+		v = strings.Replace(v, "{{ .Status }}", d.localize(d.status.String()), -1)
+		v = strings.Replace(v, "{{ .Message }}", d.localize(msg), -1)
+		body = strings.NewReader(v)
+	}
+
+	client := NewHTTPClient()
+	req, err := http.NewRequest(h.Method, h.URL, body)
+	if nil != err {
+		d.Logger <- fmt.Sprintf("[Notify] HTTP Client Network Error for '%s': %s", h.Name, err)
+		return
+	}
+
+	if 0 != len(h.Form) {
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	} else if 0 != len(h.JSON) {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	if 0 != len(h.Auth) {
+		user := h.Auth["user"]
+		if "" == user {
+			user = h.Auth["username"]
+		}
+		pass := h.Auth["pass"]
+		if "" == user {
+			pass = h.Auth["password"]
+		}
+		req.SetBasicAuth(user, pass)
+	}
+
+	req.Header.Set("User-Agent", "Watchdog/1.0")
+	for k := range h.Headers {
+		req.Header.Set(k, h.Headers[k])
+	}
+
+	resp, err := client.Do(req)
+	if nil != err {
+		d.Logger <- fmt.Sprintf("[Notify] HTTP Client Error for '%s': %s", h.Name, err)
+		return
+	}
+
+	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
+		d.Logger <- fmt.Sprintf("[Notify] Response Error for '%s': %s", h.Name, resp.Status)
+		return
+	}
+
+	// TODO json vs xml vs txt
+	var data map[string]interface{}
+	req.Header.Add("Accept", "application/json")
+	decoder := json.NewDecoder(resp.Body)
+	err = decoder.Decode(&data)
+	if err != nil {
+		d.Logger <- fmt.Sprintf("[Notify] Response Body Error for '%s': %s", h.Name, resp.Status)
+		return
+	}
+
+	// TODO some sort of way to determine if data is successful (keywords)
+	d.Logger <- fmt.Sprintf("[Notify] Success? %#v", data)
+}
+func (d *Dog) localize(msg string) string {
+	for k := range d.Localizations {
+		if k == msg {
+			return d.Localizations[k]
+		}
+	}
+	return msg
+}
+
 type Config struct {
-	Watches  []ConfigWatch   `json:"watches"`
-	Webhooks []ConfigWebhook `json:"webhooks"`
+	Watchdog      string            `json:"watchdog"`
+	Watches       []ConfigWatch     `json:"watches"`
+	Webhooks      []Webhook         `json:"webhooks"`
+	Localizations map[string]string `json:"localizations"`
 }
 
 type ConfigWatch struct {
 	Name          string   `json:"name"`
 	URL           string   `json:"url"`
 	Keywords      string   `json:"keywords"`
+	Badwords      string   `json:"badwords"`
 	Webhooks      []string `json:"webhooks"`
 	RecoverScript string   `json:"recover_script"`
 }
 
-type ConfigWebhook struct {
+type Webhook struct {
 	Name    string              `json:"name"`
 	Method  string              `json:"method"`
 	URL     string              `json:"url"`
 	Auth    map[string]string   `json:"auth"`
 	Headers map[string]string   `json:"headers"`
 	Form    map[string]string   `json:"form"`
+	JSON    map[string]string   `json:"json"`
 	Config  map[string]string   `json:"config"`
 	Configs []map[string]string `json:"configs"`
 }
@@ -344,12 +408,3 @@ func NewHTTPClient() *http.Client {
 	}
 	return client
 }
-
-// This is so that the log messages don't trample
-// over each other when they happen simultaneously.
-func logger(msgs chan string) {
-	for {
-		msg := <-msgs
-		log.Println(msg)
-	}
-}