From 70ac2f1e8cf91d12e9b20ea25c20b3938c8f9c65 Mon Sep 17 00:00:00 2001
From: Kegsay <kegsay@gmail.com>
Date: Fri, 4 Nov 2016 16:04:40 +0000
Subject: [PATCH 01/51] Update README.md

---
 README.md | 45 ++++++++++++++++++++++-----------------------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 160ce49..ff3c047 100644
--- a/README.md
+++ b/README.md
@@ -8,17 +8,10 @@ Go-NEB is a [Matrix](https://matrix.org) bot written in Go. It is the successor
  * [Installing](#installing)
  * [Running](#running)
     * [Configuration file](#configuration-file)
+ * [API](#api)
     * [Configuring clients](#configuring-clients)
     * [Configuring services](#configuring-services)
-        * [Echo Service](#echo-service)
-        * [Github Service](#github-service)
-        * [Github Webhook Service](#github-webhook-service)
-        * [JIRA Service](#jira-service)
-        * [Giphy Service](#giphy-service)
     * [Configuring realms](#configuring-realms)
-        * [Github Realm](#github-realm)
-           * [Github Authentication](#github-authentication)
-        * [JIRA Realm](#jira-realm)
  * [Developing](#developing)
     * [Architecture](#architecture)
     * [API Docs](#viewing-the-api-docs)
@@ -75,6 +68,8 @@ Invite the bot user into a Matrix room and type `!echo hello world`. It will rep
 
 ### Giphy
  - Ability to query Giphy's "text-to-gif" engine.
+ 
+### Guggy
 
 
 # Installing
@@ -107,6 +102,14 @@ Go-NEB needs to be "configured" with clients and services before it will do anyt
 ## Configuration file
 If you run Go-NEB with a `CONFIG_FILE` environment variable, it will load that file and use it for services, clients, etc. There is a [sample configuration file](config.sample.yaml) which explains all the options. In most cases, these are *direct mappings* to the corresponding HTTP API.
 
+# API
+The API is documented in sections using godoc. The sections consists of:
+ - An HTTP API (the path and method to use)
+ - A "JSON request body" (the JSON that is inside the HTTP request body)
+ - "Configuration" information (any additional information that is specific to what you're creating)
+ 
+To form the complete API, you need to combine the HTTP API with the JSON request body, and the "Configuration" information (which is always under a JSON key called `Config`). In addition, most APIs have a `Type` which determines which piece of code to load. To find out what the right type is for the thing you're creating, check the constants defined in godoc.
+
 ## Configuring Clients
 Go-NEB needs to connect as a matrix user to receive messages. Go-NEB can listen for messages as multiple matrix users. The users are configured using an HTTP API and the config is stored in the database.
 
@@ -138,21 +141,17 @@ Realms are how Go-NEB authenticates users on third-party websites.
  - [HTTP API Docs](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/api/handlers/index.html#ConfigureAuthRealm.OnIncomingRequest)
  - [JSON Request Body Docs](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/api/index.html#ConfigureAuthRealmRequest)
 
-### Github
- - [Realm configuration](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/github/index.html#Realm)
-
-#### Authentication of Matrix users
-
- * [Configuration for config file](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/github/index.html#Session)
- * [Configuration for HTTP](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/github/index.html#Realm.RequestAuthSession)
-
-### JIRA
- - [Realm configuration](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/jira/index.html#Realm)
-
-#### Authentication of Matrix users
- * [Configuration for config file](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/jira/index.html#Session)
- * [Configuration for HTTP](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/jira/index.html#Realm.RequestAuthSession)
-
+List of Realms:
+ - [Github](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/github/index.html#Realm)
+ - [JIRA](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/jira/index.html#Realm)
+ 
+Authentication via HTTP:
+ - [Github](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/github/index.html#Realm.RequestAuthSession)
+ - [JIRA](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/jira/index.html#Realm.RequestAuthSession)
+
+Authentication via the config file:
+ - [Github](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/github/index.html#Session)
+ - [JIRA](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/realms/jira/index.html#Session)
 
 # Developing
 There's a bunch more tools this project uses when developing in order to do

From b93077c935ac0675e804313603e357309814acf4 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 9 Nov 2016 14:06:01 +0000
Subject: [PATCH 02/51] Ignore robots.txt

---
 gendoc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gendoc.sh b/gendoc.sh
index d088e12..679308a 100755
--- a/gendoc.sh
+++ b/gendoc.sh
@@ -20,7 +20,7 @@ done
 
 # Scrape the pkg directory for the API docs. Scrap lib for the CSS/JS. Ignore everything else.
 # The output is dumped to the directory "localhost:6060".
-wget -r -m -k -E -p --include-directories="/pkg,/lib" --exclude-directories="*" http://localhost:6060/pkg/github.com/matrix-org/go-neb/
+wget -r -m -k -E -p -erobots=off --include-directories="/pkg,/lib" --exclude-directories="*" http://localhost:6060/pkg/github.com/matrix-org/go-neb/
 
 # Stop the godoc server
 kill -9 $DOC_PID

From 0b26e6330a1db16c1b93bc751a9e24aa397c044d Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 9 Nov 2016 15:42:53 +0000
Subject: [PATCH 03/51] Add Guggy tests

---
 .../matrix-org/go-neb/matrix/matrix.go        |   2 +-
 .../matrix-org/go-neb/services/guggy/guggy.go |   6 +-
 .../go-neb/services/guggy/guggy_test.go       | 110 ++++++++++++++++++
 3 files changed, 114 insertions(+), 4 deletions(-)
 create mode 100644 src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go

diff --git a/src/github.com/matrix-org/go-neb/matrix/matrix.go b/src/github.com/matrix-org/go-neb/matrix/matrix.go
index 1834ca2..e03c9eb 100644
--- a/src/github.com/matrix-org/go-neb/matrix/matrix.go
+++ b/src/github.com/matrix-org/go-neb/matrix/matrix.go
@@ -156,7 +156,7 @@ func (cli *Client) SendText(roomID, text string) (string, error) {
 
 // UploadLink uploads an HTTP URL and then returns an MXC URI.
 func (cli *Client) UploadLink(link string) (string, error) {
-	res, err := http.Get(link)
+	res, err := cli.httpClient.Get(link)
 	if res != nil {
 		defer res.Body.Close()
 	}
diff --git a/src/github.com/matrix-org/go-neb/services/guggy/guggy.go b/src/github.com/matrix-org/go-neb/services/guggy/guggy.go
index e661fa3..940a416 100644
--- a/src/github.com/matrix-org/go-neb/services/guggy/guggy.go
+++ b/src/github.com/matrix-org/go-neb/services/guggy/guggy.go
@@ -18,6 +18,8 @@ import (
 // ServiceType of the Guggy service
 const ServiceType = "guggy"
 
+var httpClient = &http.Client{}
+
 type guggyQuery struct {
 	// "mp4" or "gif"
 	Format string `json:"format"`
@@ -93,8 +95,6 @@ func (s *Service) cmdGuggy(client *matrix.Client, roomID, userID string, args []
 func (s *Service) text2gifGuggy(querySentence string) (*guggyGifResult, error) {
 	log.Info("Transforming to GIF query ", querySentence)
 
-	client := &http.Client{}
-
 	var query guggyQuery
 	query.Format = "gif"
 	query.Sentence = querySentence
@@ -114,7 +114,7 @@ func (s *Service) text2gifGuggy(querySentence string) (*guggyGifResult, error) {
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("apiKey", s.APIKey)
 
-	res, err := client.Do(req)
+	res, err := httpClient.Do(req)
 	if res != nil {
 		defer res.Body.Close()
 	}
diff --git a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
new file mode 100644
index 0000000..b7d5a05
--- /dev/null
+++ b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
@@ -0,0 +1,110 @@
+package guggy
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/types"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+type MockTransport struct {
+	roundTrip func(*http.Request) (*http.Response, error)
+}
+
+func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.roundTrip(req)
+}
+
+// TODO: It would be nice to tabularise this test so we can try failing different combinations of responses to make
+//       sure all cases are handled, rather than just the general case as is here.
+func TestCommand(t *testing.T) {
+	database.SetServiceDB(&database.NopStorage{})
+	apiKey := "secret"
+	guggyImageURL := "https://guggy.com/gifs/23ryf872fg"
+
+	// Mock the response from Guggy
+	guggyTrans := struct{ MockTransport }{}
+	guggyTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+		guggyURL := "https://text2gif.guggy.com/guggify"
+		if req.URL.String() != guggyURL {
+			t.Fatalf("Bad URL: got %s want %s", req.URL.String(), guggyURL)
+		}
+		if req.Method != "POST" {
+			t.Fatalf("Bad method: got %s want POST", req.Method)
+		}
+		if req.Header.Get("apiKey") != apiKey {
+			t.Fatalf("Bad apiKey: got %s want %s", req.Header.Get("apiKey"), apiKey)
+		}
+		// check the query is in the request body
+		var reqBody guggyQuery
+		if err := json.NewDecoder(req.Body).Decode(&reqBody); err != nil {
+			t.Fatalf("Failed to read request body: %s", err)
+		}
+		if reqBody.Sentence != "hey listen!" {
+			t.Fatalf("Bad query: got '%s' want '%s'", reqBody.Sentence, "hey listen!")
+		}
+
+		res := guggyGifResult{
+			Width:  64,
+			Height: 64,
+			ReqID:  "12345",
+			GIF:    guggyImageURL,
+		}
+		b, err := json.Marshal(res)
+		if err != nil {
+			t.Fatalf("Failed to marshal guggy response", err)
+		}
+		return &http.Response{
+			StatusCode: 200,
+			Body:       ioutil.NopCloser(bytes.NewBuffer(b)),
+		}, nil
+	}
+	// clobber the guggy service http client instance
+	httpClient = &http.Client{Transport: guggyTrans}
+
+	// Create the Guggy service
+	srv, err := types.CreateService("id", ServiceType, "@guggybot:hyrule", []byte(
+		`{"api_key":"`+apiKey+`"}`,
+	))
+	if err != nil {
+		t.Fatal("Failed to create Guggy service: ", err)
+	}
+	guggy := srv.(*Service)
+
+	// Mock the response from Matrix
+	matrixTrans := struct{ MockTransport }{}
+	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+		if req.URL.String() == guggyImageURL { // getting the guggy image
+			return &http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString("some image data")),
+			}, nil
+		} else if strings.Contains(req.URL.String(), "_matrix/media/r0/upload") { // uploading the image to matrix
+			return &http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"content_uri":"mxc://foo/bar"}`)),
+			}, nil
+		}
+		return nil, fmt.Errorf("Unknown URL: %s", req.URL.String())
+	}
+	u, _ := url.Parse("https://hyrule")
+	matrixCli := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@guggybot:hyrule")
+
+	// Execute the matrix !command
+	cmds := guggy.Commands(matrixCli)
+	if len(cmds) != 1 {
+		t.Fatalf("Unexpected number of commands: %d", len(cmds))
+	}
+	cmd := cmds[0]
+	_, err = cmd.Command("!someroom:hyrule", "@navi:hyrule", []string{"hey", "listen!"})
+	if err != nil {
+		t.Fatalf("Failed to process command: ", err.Error())
+	}
+}

From ff8b4c41d078809412ef0425dcb5b21e525d05ff Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 15 Nov 2016 09:55:30 +0000
Subject: [PATCH 04/51] Add stub travis-ci service

---
 .../go-neb/services/travisci/travisci.go      | 99 +++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 src/github.com/matrix-org/go-neb/services/travisci/travisci.go

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
new file mode 100644
index 0000000..87d8bc6
--- /dev/null
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -0,0 +1,99 @@
+package travisci
+
+import (
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/services/github/client"
+	"github.com/matrix-org/go-neb/services/github/webhook"
+	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/go-neb/util"
+)
+
+// ServiceType of the Travis-CI service.
+const ServiceType = "travis-ci"
+
+// Service contains the Config fields for the Travis-CI service.
+//
+// This service will send notifications into a Matrix room when Travis-CI sends
+// webhook events to it. It requires a public domain which Travis-CI can reach.
+// Notices will be sent as the service user ID.
+//
+// Example JSON request:
+//   {
+//       rooms: {
+//           "!ewfug483gsfe:localhost": {
+//               repos: {
+//                   "matrix-org/go-neb": {
+//                       template: "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}\nBuild details : %{build_url}"
+//                   }
+//               }
+//           }
+//       }
+//   }
+type Service struct {
+	types.DefaultService
+	webhookEndpointURL string
+	// A map from Matrix room ID to Github-style owner/repo repositories.
+	Rooms map[string]struct {
+		// A map of "owner/repo" to configuration information
+		Repos map[string]struct {
+			// The template string to use when creating notifications.
+			//
+			// This is identical to the format of Slack Notifications for Travis-CI:
+			// https://docs.travis-ci.com/user/notifications#Customizing-slack-notifications
+			//
+			// The following variables are available:
+			//   repository_slug: your GitHub repo identifier (like svenfuchs/minimal)
+			//   repository_name: the slug without the username
+			//   build_number: build number
+			//   build_id: build id
+			//   branch: branch build name
+			//   commit: shortened commit SHA
+			//   author: commit author name
+			//   commit_message: commit message of build
+			//   commit_subject: first line of the commit message
+			//   result: result of build
+			//   message: Travis CI message to the build
+			//   duration: total duration of all builds in the matrix
+			//   elapsed_time: time between build start and finish
+			//   compare_url: commit change view URL
+			//   build_url: URL of the build detail
+			Template string `json:"template"`
+		} `json:"repos"`
+	} `json:"rooms"`
+}
+
+// OnReceiveWebhook receives requests from Travis-CI and possibly sends requests to Matrix as a result.
+//
+// If the "repository.url" matches a known Github repository, a notification will be formed from the
+// template for that repository and a notice will be sent to Matrix.
+//
+// Go-NEB cannot register with Travis-CI for webhooks automatically. The user must manually add the
+// webhook endpoint URL to their .travis.yml file:
+//    notifications:
+//        webhooks: http://your-domain.com/notifications
+//
+// See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
+func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
+	return
+}
+
+// Register makes sure the Config information supplied is valid.
+func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+	return nil
+}
+
+func init() {
+	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
+		return &Service{
+			DefaultService:     types.NewDefaultService(serviceID, serviceUserID, ServiceType),
+			webhookEndpointURL: webhookEndpointURL,
+		}
+	})
+}

From 2b3cbba2e63992de676a2a6fe004b51caa09e5f6 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 15 Nov 2016 10:36:47 +0000
Subject: [PATCH 05/51] Begin to flesh out the Travis-CI service

---
 .../go-neb/services/travisci/travisci.go      | 84 +++++++++++++++++--
 1 file changed, 79 insertions(+), 5 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 87d8bc6..87658fd 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -3,21 +3,25 @@ package travisci
 import (
 	"fmt"
 	"net/http"
-	"sort"
-	"strings"
+	"regexp"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/database"
 	"github.com/matrix-org/go-neb/matrix"
-	"github.com/matrix-org/go-neb/services/github/client"
-	"github.com/matrix-org/go-neb/services/github/webhook"
 	"github.com/matrix-org/go-neb/types"
-	"github.com/matrix-org/go-neb/util"
 )
 
 // ServiceType of the Travis-CI service.
 const ServiceType = "travis-ci"
 
+// DefaultTemplate contains the template that will be used if none is supplied.
+// This matches the default mentioned at: https://docs.travis-ci.com/user/notifications#Customizing-slack-notifications
+const DefaultTemplate = (`%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}
+	Change view : %{compare_url}
+	Build details : %{build_url}`)
+
+var ownerRepoRegex = regexp.MustCompile(`^([A-z0-9-_.]+)/([A-z0-9-_.]+)$`)
+
 // Service contains the Config fields for the Travis-CI service.
 //
 // This service will send notifications into a Matrix room when Travis-CI sends
@@ -69,6 +73,50 @@ type Service struct {
 	} `json:"rooms"`
 }
 
+// The payload from Travis-CI
+type webhookNotification struct {
+	Number         string `json:"number"`
+	Status         string `json:"status"`
+	StartedAt      string `json:"started_at"`
+	FinishedAt     string `json:"finished_at"`
+	StatusMessage  string `json:"status_message"`
+	Commit         string `json:"commit"`
+	Branch         string `json:"branch"`
+	Message        string `json:"message"`
+	CompareURL     string `json:"compare_url"`
+	CommittedAt    string `json:"committed_at"`
+	CommitterName  string `json:"committer_name"`
+	CommitterEmail string `json:"committer_email"`
+	AuthorName     string `json:"author_name"`
+	AuthorEmail    string `json:"author_email"`
+	Type           string `json:"type"`
+	BuildURL       string `json:"build_url"`
+	Repository     struct {
+		Name      string `json:"name"`
+		OwnerName string `json:"owner_name"`
+		URL       string `json:"url"`
+	} `json:"repository"`
+}
+
+// The template variables a user can use in their messages
+type notificationTemplate struct {
+	RepositorySlug string
+	RepositoryName string
+	BuildNumber    string
+	BuildID        string
+	Branch         string
+	Commit         string
+	Author         string
+	CommitMessage  string
+	CommitSubject  string
+	Result         string
+	Message        string
+	Duration       string
+	ElapsedTime    string
+	CompareURL     string
+	BuildURL       string
+}
+
 // OnReceiveWebhook receives requests from Travis-CI and possibly sends requests to Matrix as a result.
 //
 // If the "repository.url" matches a known Github repository, a notification will be formed from the
@@ -86,9 +134,35 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 
 // Register makes sure the Config information supplied is valid.
 func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+	for _, roomData := range s.Rooms {
+		for repo := range roomData.Repos {
+			match := ownerRepoRegex.FindStringSubmatch(repo)
+			if len(match) == 0 {
+				return fmt.Errorf("Repository '%s' is not a valid repository name.", repo)
+			}
+		}
+	}
 	return nil
 }
 
+// PostRegister deletes this service if there are no registered repos.
+func (s *Service) PostRegister(oldService types.Service) {
+	for _, roomData := range s.Rooms {
+		for _ = range roomData.Repos {
+			return // at least 1 repo exists
+		}
+	}
+	// Delete this service since no repos are configured
+	logger := log.WithFields(log.Fields{
+		"service_type": s.ServiceType(),
+		"service_id":   s.ServiceID(),
+	})
+	logger.Info("Removing service as no repositories are registered.")
+	if err := database.GetServiceDB().DeleteService(s.ServiceID()); err != nil {
+		logger.WithError(err).Error("Failed to delete service")
+	}
+}
+
 func init() {
 	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
 		return &Service{

From 6d297f63ad004161151ced4a488ba8e451df2acd Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 15 Nov 2016 11:47:15 +0000
Subject: [PATCH 06/51] Process Travis webhook events

---
 .../go-neb/services/travisci/travisci.go      | 116 +++++++++++++++---
 1 file changed, 99 insertions(+), 17 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 87658fd..3660940 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -1,9 +1,13 @@
 package travisci
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
+	"strconv"
+	"strings"
+	"time"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/database"
@@ -75,22 +79,23 @@ type Service struct {
 
 // The payload from Travis-CI
 type webhookNotification struct {
-	Number         string `json:"number"`
-	Status         string `json:"status"`
-	StartedAt      string `json:"started_at"`
-	FinishedAt     string `json:"finished_at"`
-	StatusMessage  string `json:"status_message"`
-	Commit         string `json:"commit"`
-	Branch         string `json:"branch"`
-	Message        string `json:"message"`
-	CompareURL     string `json:"compare_url"`
-	CommittedAt    string `json:"committed_at"`
-	CommitterName  string `json:"committer_name"`
-	CommitterEmail string `json:"committer_email"`
-	AuthorName     string `json:"author_name"`
-	AuthorEmail    string `json:"author_email"`
-	Type           string `json:"type"`
-	BuildURL       string `json:"build_url"`
+	ID             int     `json:"id"`
+	Number         string  `json:"number"`
+	Status         *string `json:"status"`
+	StartedAt      *string `json:"started_at"`
+	FinishedAt     *string `json:"finished_at"`
+	StatusMessage  string  `json:"status_message"`
+	Commit         string  `json:"commit"`
+	Branch         string  `json:"branch"`
+	Message        string  `json:"message"`
+	CompareURL     string  `json:"compare_url"`
+	CommittedAt    string  `json:"committed_at"`
+	CommitterName  string  `json:"committer_name"`
+	CommitterEmail string  `json:"committer_email"`
+	AuthorName     string  `json:"author_name"`
+	AuthorEmail    string  `json:"author_email"`
+	Type           string  `json:"type"`
+	BuildURL       string  `json:"build_url"`
 	Repository     struct {
 		Name      string `json:"name"`
 		OwnerName string `json:"owner_name"`
@@ -117,6 +122,50 @@ type notificationTemplate struct {
 	BuildURL       string
 }
 
+func notifToTemplate(n webhookNotification) (t notificationTemplate) {
+	t.RepositorySlug = n.Repository.OwnerName + "/" + n.Repository.Name
+	t.RepositoryName = n.Repository.Name
+	t.BuildNumber = n.Number
+	t.BuildID = strconv.Itoa(n.ID)
+	t.Branch = n.Branch
+	t.Commit = n.Commit
+	t.Author = n.CommitterName // author: commit author name
+	// commit_message: commit message of build
+	// commit_subject: first line of the commit message
+	t.CommitMessage = n.Message
+	subjAndMsg := strings.SplitN(n.Message, "\n", 2)
+	t.CommitSubject = subjAndMsg[0]
+	if n.Status != nil {
+		t.Result = *n.Status
+	}
+	t.Message = n.StatusMessage
+
+	if n.StartedAt != nil && n.FinishedAt != nil {
+		// duration: total duration of all builds in the matrix -- TODO
+		// elapsed_time: time between build start and finish
+		// Example from docs: "2011-11-11T11: 11: 11Z"
+		start, err := time.Parse("2006-01-02T15: 04: 05Z", *n.StartedAt)
+		finish, err2 := time.Parse("2006-01-02T15: 04: 05Z", *n.FinishedAt)
+		if err != nil || err2 != nil {
+			log.WithFields(log.Fields{
+				"started_at":  *n.StartedAt,
+				"finished_at": *n.FinishedAt,
+			}).Warn("Failed to parse Travis-CI start/finish times.")
+		} else {
+			t.Duration = finish.Sub(start).String()
+			t.ElapsedTime = t.Duration
+		}
+	}
+
+	t.CompareURL = n.CompareURL
+	t.BuildURL = n.BuildURL
+	return
+}
+
+func travisToGoTemplate(travisTmpl string) (goTmpl string) {
+	return
+}
+
 // OnReceiveWebhook receives requests from Travis-CI and possibly sends requests to Matrix as a result.
 //
 // If the "repository.url" matches a known Github repository, a notification will be formed from the
@@ -129,7 +178,40 @@ type notificationTemplate struct {
 //
 // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
 func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
-	return
+	var notif webhookNotification
+	if err := json.NewDecoder(req.Body).Decode(&notif); err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	if notif.Repository.URL == "" || notif.Repository.OwnerName == "" || notif.Repository.Name == "" {
+		w.WriteHeader(400)
+		return
+	}
+	whForRepo := notif.Repository.OwnerName + "/" + notif.Repository.Name
+	// TODO tmplData := notifToTemplate(notif)
+
+	logger := log.WithFields(log.Fields{
+		"repo": whForRepo,
+	})
+
+	for roomID, roomData := range s.Rooms {
+		for ownerRepo, repoData := range roomData.Repos {
+			if ownerRepo != whForRepo {
+				continue
+			}
+			tmpl := travisToGoTemplate(repoData.Template)
+			msg := tmpl // TODO
+
+			logger.WithFields(log.Fields{
+				"msg":     msg,
+				"room_id": roomID,
+			}).Print("Sending Travis-CI notification to room")
+			if _, e := cli.SendMessageEvent(roomID, "m.room.message", msg); e != nil {
+				logger.WithError(e).WithField("room_id", roomID).Print(
+					"Failed to send Travis-CI notification to room.")
+			}
+		}
+	}
 }
 
 // Register makes sure the Config information supplied is valid.

From 5bf126473e38764b355627f4efa555420977a83d Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 15 Nov 2016 14:37:59 +0000
Subject: [PATCH 07/51] Verify incoming Travis-CI webhook requests

---
 .../go-neb/services/travisci/travisci.go      | 23 ++++-
 .../go-neb/services/travisci/verify.go        | 95 +++++++++++++++++++
 2 files changed, 116 insertions(+), 2 deletions(-)
 create mode 100644 src/github.com/matrix-org/go-neb/services/travisci/verify.go

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 3660940..652a458 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -3,6 +3,7 @@ package travisci
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -24,8 +25,11 @@ const DefaultTemplate = (`%{repository}#%{build_number} (%{branch} - %{commit} :
 	Change view : %{compare_url}
 	Build details : %{build_url}`)
 
+// Matches 'owner/repo'
 var ownerRepoRegex = regexp.MustCompile(`^([A-z0-9-_.]+)/([A-z0-9-_.]+)$`)
 
+var httpClient = &http.Client{}
+
 // Service contains the Config fields for the Travis-CI service.
 //
 // This service will send notifications into a Matrix room when Travis-CI sends
@@ -81,7 +85,7 @@ type Service struct {
 type webhookNotification struct {
 	ID             int     `json:"id"`
 	Number         string  `json:"number"`
-	Status         *string `json:"status"`
+	Status         *string `json:"status"` // 0 (success) or 1 (incomplete/fail).
 	StartedAt      *string `json:"started_at"`
 	FinishedAt     *string `json:"finished_at"`
 	StatusMessage  string  `json:"status_message"`
@@ -178,8 +182,23 @@ func travisToGoTemplate(travisTmpl string) (goTmpl string) {
 //
 // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
 func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
+	payload, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		log.WithError(err).Error("Failed to read incoming Travis-CI webhook")
+		w.WriteHeader(500)
+		return
+	}
+	if err := verifyOrigin(req.Host, payload, req.Header.Get("Signature")); err != nil {
+		log.WithFields(log.Fields{
+			"Signature":  req.Header.Get("Signature"),
+			log.ErrorKey: err,
+		}).Warn("Received unauthorised Travis-CI webhook request.")
+		w.WriteHeader(403)
+		return
+	}
+
 	var notif webhookNotification
-	if err := json.NewDecoder(req.Body).Decode(&notif); err != nil {
+	if err := json.Unmarshal(payload, &notif); err != nil {
 		w.WriteHeader(400)
 		return
 	}
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/verify.go b/src/github.com/matrix-org/go-neb/services/travisci/verify.go
new file mode 100644
index 0000000..9454c2b
--- /dev/null
+++ b/src/github.com/matrix-org/go-neb/services/travisci/verify.go
@@ -0,0 +1,95 @@
+package travisci
+
+import (
+	"crypto"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+// Host => Public Key.
+// Travis has a .com and .org with different public keys.
+var travisPublicKeyMap = map[string]*rsa.PublicKey{
+	"api.travis-ci.org": nil,
+	"api.travis-ci.com": nil,
+}
+
+func verifyOrigin(host string, payload []byte, sigHeader string) error {
+	/*
+		From: https://docs.travis-ci.com/user/notifications#Verifying-Webhook-requests
+			 1. Pick up the payload data from the HTTP request’s body.
+			 2. Obtain the Signature header value, and base64-decode it.
+			 3. Obtain the public key corresponding to the private key that signed the payload.
+				This is available at the /config endpoint’s config.notifications.webhook.public_key on
+				the relevant API server. (e.g., https://api.travis-ci.org/config)
+			4. Verify the signature using the public key and SHA1 digest.
+	*/
+	// 2. Get signature bytes
+	sig, err := base64.StdEncoding.DecodeString(sigHeader)
+	if err != nil {
+		return err
+	}
+
+	// 3. Get public key
+	pubKey, validURL := travisPublicKeyMap[host]
+	if !validURL {
+		return fmt.Errorf("Invalid host: %s", host)
+	}
+	if pubKey == nil {
+		pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config")
+		if err != nil {
+			return err
+		}
+		block, _ := pem.Decode([]byte(pemPubKey))
+		if block == nil {
+			return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host)
+		}
+
+		k, err := x509.ParsePKIXPublicKey(block.Bytes)
+		if err != nil {
+			return err
+		}
+		pubKey = k.(*rsa.PublicKey)
+		travisPublicKeyMap[host] = pubKey
+	}
+
+	// 4. Verify with SHA1
+	h := sha1.New()
+	h.Write(payload)
+	digest := h.Sum(nil)
+	return rsa.VerifyPKCS1v15(pubKey, crypto.SHA1, digest, sig)
+}
+
+func fetchPEMPublicKey(travisURL string) (key string, err error) {
+	var res *http.Response
+	res, err = httpClient.Get(travisURL)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return
+	}
+	configStruct := struct {
+		Config struct {
+			Notifications struct {
+				Webhook struct {
+					PublicKey string `json:"public_key"`
+				} `json:"webhook"`
+			} `json:"notifications"`
+		} `json:"config"`
+	}{}
+	if err = json.NewDecoder(res.Body).Decode(&configStruct); err != nil {
+		return
+	}
+	key = configStruct.Config.Notifications.Webhook.PublicKey
+	if key == "" || strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") {
+		err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key.")
+	}
+	return
+}

From 4825c5c4ef982cfdec97cf9ccfb9584f48118be1 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 15 Nov 2016 16:46:02 +0000
Subject: [PATCH 08/51] Begin to add Travis-CI Service tests

Currently this verifies the signature logic, but not the templating
logic which isn't implemented yet.
---
 .../go-neb/services/travisci/travisci.go      |  29 ++-
 .../go-neb/services/travisci/travisci_test.go | 178 ++++++++++++++++++
 .../go-neb/services/travisci/verify.go        |  70 ++++---
 3 files changed, 239 insertions(+), 38 deletions(-)
 create mode 100644 src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 652a458..fa82a5a 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -3,7 +3,6 @@ package travisci
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -85,7 +84,7 @@ type Service struct {
 type webhookNotification struct {
 	ID             int     `json:"id"`
 	Number         string  `json:"number"`
-	Status         *string `json:"status"` // 0 (success) or 1 (incomplete/fail).
+	Status         *int    `json:"status"` // 0 (success) or 1 (incomplete/fail).
 	StartedAt      *string `json:"started_at"`
 	FinishedAt     *string `json:"finished_at"`
 	StatusMessage  string  `json:"status_message"`
@@ -110,6 +109,7 @@ type webhookNotification struct {
 // The template variables a user can use in their messages
 type notificationTemplate struct {
 	RepositorySlug string
+	Repository     string // Deprecated: alias for RepositorySlug
 	RepositoryName string
 	BuildNumber    string
 	BuildID        string
@@ -128,6 +128,7 @@ type notificationTemplate struct {
 
 func notifToTemplate(n webhookNotification) (t notificationTemplate) {
 	t.RepositorySlug = n.Repository.OwnerName + "/" + n.Repository.Name
+	t.Repository = t.RepositorySlug
 	t.RepositoryName = n.Repository.Name
 	t.BuildNumber = n.Number
 	t.BuildID = strconv.Itoa(n.ID)
@@ -140,7 +141,7 @@ func notifToTemplate(n webhookNotification) (t notificationTemplate) {
 	subjAndMsg := strings.SplitN(n.Message, "\n", 2)
 	t.CommitSubject = subjAndMsg[0]
 	if n.Status != nil {
-		t.Result = *n.Status
+		t.Result = strconv.Itoa(*n.Status)
 	}
 	t.Message = n.StatusMessage
 
@@ -182,13 +183,18 @@ func travisToGoTemplate(travisTmpl string) (goTmpl string) {
 //
 // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
 func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
-	payload, err := ioutil.ReadAll(req.Body)
-	if err != nil {
-		log.WithError(err).Error("Failed to read incoming Travis-CI webhook")
-		w.WriteHeader(500)
+	if err := req.ParseForm(); err != nil {
+		log.WithError(err).Error("Failed to read incoming Travis-CI webhook form")
+		w.WriteHeader(400)
+		return
+	}
+	payload := req.PostFormValue("payload")
+	if payload == "" {
+		log.Error("Travis-CI webhook is missing payload= form value")
+		w.WriteHeader(400)
 		return
 	}
-	if err := verifyOrigin(req.Host, payload, req.Header.Get("Signature")); err != nil {
+	if err := verifyOrigin([]byte(payload), req.Header.Get("Signature")); err != nil {
 		log.WithFields(log.Fields{
 			"Signature":  req.Header.Get("Signature"),
 			log.ErrorKey: err,
@@ -198,11 +204,13 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 	}
 
 	var notif webhookNotification
-	if err := json.Unmarshal(payload, &notif); err != nil {
+	if err := json.Unmarshal([]byte(payload), &notif); err != nil {
+		log.WithError(err).Error("Travis-CI webhook received an invalid JSON payload=")
 		w.WriteHeader(400)
 		return
 	}
-	if notif.Repository.URL == "" || notif.Repository.OwnerName == "" || notif.Repository.Name == "" {
+	if notif.Repository.OwnerName == "" || notif.Repository.Name == "" {
+		log.WithField("repo", notif.Repository).Error("Travis-CI webhook missing repository fields")
 		w.WriteHeader(400)
 		return
 	}
@@ -231,6 +239,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 			}
 		}
 	}
+	w.WriteHeader(200)
 }
 
 // Register makes sure the Config information supplied is valid.
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
new file mode 100644
index 0000000..1848a81
--- /dev/null
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -0,0 +1,178 @@
+package travisci
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/types"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+const travisOrgPEMPublicKey = (`-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvtjdLkS+FP+0fPC09j25
+y/PiuYDDivIT86COVedvlElk99BBYTrqNaJybxjXbIZ1Q6xFNhOY+iTcBr4E1zJu
+tizF3Xi0V9tOuP/M8Wn4Y/1lCWbQKlWrNQuqNBmhovF4K3mDCYswVbpgTmp+JQYu
+Bm9QMdieZMNry5s6aiMA9aSjDlNyedvSENYo18F+NYg1J0C0JiPYTxheCb4optr1
+5xNzFKhAkuGs4XTOA5C7Q06GCKtDNf44s/CVE30KODUxBi0MCKaxiXw/yy55zxX2
+/YdGphIyQiA5iO1986ZmZCLLW8udz9uhW5jUr3Jlp9LbmphAC61bVSf4ou2YsJaN
+0QIDAQAB
+-----END PUBLIC KEY-----`)
+
+const travisComPEMPublicKey = (`-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQU2j9lnRtyuW36arNOc
+dzCzyKVirLUi3/aLh6UfnTVXzTnx8eHUnBn1ZeQl7Eh3J3qqdbIKl6npS27ONzCy
+3PIcfjpLPaVyGagIL8c8XgDEvB45AesC0osVP5gkXQkPUM3B2rrUmp1AZzG+Fuo0
+SAeNnS71gN63U3brL9fN/MTCXJJ6TvMt3GrcJUq5uq56qNeJTsiowK6eiiWFUSfh
+e1qapOdMFmcEs9J/R1XQ/scxbAnLcWfl8lqH/MjMdCMe0j3X2ZYMTqOHsb3cQGSS
+dMPwZGeLWV+OaxjJ7TrJ+riqMANOgqCBGpvWUnUfo046ACOx7p6u4fFc3aRiuqYK
+VQIDAQAB
+-----END PUBLIC KEY-----`)
+
+const exampleSignature = ("pW0CDpmcAeWw3qf2Ufx8UvzfrZRUpYx30HBl9nJcDkh2v9FrF1GjJVsrcqx7ly0FPjb7dkfMJ/d0Q3JpDb1EL4p509cN4Vy8+HpfINw35Wg6JqzOQqTa" +
+	"TidwoDLXo0NgL78zfiL3dra7ZwOGTA+LmnLSuNp38ROxn70u26uqJzWprGSdVNbRu1LkF1QKLa61NZegfxK7RZn1PlIsznWIyTS0qj81mg2sXMDLH1J4" +
+	"pHxjEpzydjSb5b8tCjrN+vFLDdAtP5RjU8NwvQM4LRRGbLDIlRsO77HDwfXrPgUE3DjPIqVpHhMcCusygp0ClH2b1J1O3LkhxSS9ol5w99Hkpg==")
+const exampleBody = ("payload=%7B%22id%22%3A176075135%2C%22repository%22%3A%7B%22id%22%3A6461770%2C%22name%22%3A%22flow-jsdoc%22%2C%22owner_" +
+	"name%22%3A%22Kegsay%22%2C%22url%22%3Anull%7D%2C%22number%22%3A%2218%22%2C%22config%22%3A%7B%22notifications%22%3A%7B%22web" +
+	"hooks%22%3A%5B%22http%3A%2F%2F7abbe705.ngrok.io%22%5D%7D%2C%22language%22%3A%22node_js%22%2C%22node_js%22%3A%5B%224.1%22%5D%2C%22.resu" +
+	"lt%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22precise%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22status_" +
+	"message%22%3A%22Passed%22%2C%22result_message%22%3A%22Passed%22%2C%22started_at%22%3A%222016-11-15T15%3A10%3A22Z%22%2C%22finished_" +
+	"at%22%3A%222016-11-15T15%3A10%3A54Z%22%2C%22duration%22%3A32%2C%22build_url%22%3A%22https%3A%2F%2Ftravis-ci.org%2FKegsay%2Fflow-js" +
+	"doc%2Fbuilds%2F176075135%22%2C%22commit_id%22%3A50222535%2C%22commit%22%3A%223a092c3a6032ebb50384c99b445f947e9ce86e2a%22%2C%22base_com" +
+	"mit%22%3Anull%2C%22head_commit%22%3Anull%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22Test+Travis+webhook+support%22%2C%22compare_" +
+	"url%22%3A%22https%3A%2F%2Fgithub.com%2FKegsay%2Fflow-jsdoc%2Fcompare%2F9f9d459ba082...3a092c3a6032%22%2C%22committed_at%22%3A%222016-1" +
+	"1-15T15%3A08%3A16Z%22%2C%22author_name%22%3A%22Kegan+Dougal%22%2C%22author_email%22%3A%22kegan%40matrix.org%22%2C%22committer_" +
+	"name%22%3A%22Kegan+Dougal%22%2C%22committer_email%22%3A%22kegan%40matrix.org%22%2C%22matrix%22%3A%5B%7B%22id%22%3A176075137%2C%22reposit" +
+	"ory_id%22%3A6461770%2C%22parent_id%22%3A176075135%2C%22number%22%3A%2218.1%22%2C%22state%22%3A%22finished%22%2C%22config%22%3A%7B%22notifi" +
+	"cations%22%3A%7B%22webhooks%22%3A%5B%22http%3A%2F%2F7abbe705.ngrok.io%22%5D%7D%2C%22language%22%3A%22node_js%22%2C%22node_" +
+	"js%22%3A%224.1%22%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22precise%22%2C%22os%22%3A%22li" +
+	"nux%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22commit%22%3A%223a092c3a6032ebb50384c99b445f947e9ce86e2a%22%2C%22branch%22%3A%22mas" +
+	"ter%22%2C%22message%22%3A%22Test+Travis+webhook+support%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2FKegsay%2Fflow-jsdoc%2Fcomp" +
+	"are%2F9f9d459ba082...3a092c3a6032%22%2C%22started_at%22%3A%222016-11-15T15%3A10%3A22Z%22%2C%22finished_at%22%3A%222016-11-" +
+	"15T15%3A10%3A54Z%22%2C%22committed_at%22%3A%222016-11-15T15%3A08%3A16Z%22%2C%22author_name%22%3A%22Kegan+Dougal%22%2C%22author_ema" +
+	"il%22%3A%22kegan%40matrix.org%22%2C%22committer_name%22%3A%22Kegan+Dougal%22%2C%22committer_email%22%3A%22kegan%40matrix.org%22%2C%22allow_f" +
+	"ailure%22%3Afalse%7D%5D%2C%22type%22%3A%22push%22%2C%22state%22%3A%22passed%22%2C%22pull_request%22%3Afalse%2C%22pull_request_number%22%3Anu" +
+	"ll%2C%22pull_request_title%22%3Anull%2C%22tag%22%3Anull%7D")
+
+var travisTests = []struct {
+	Signature      string
+	ValidSignature bool
+	Body           string
+	Template       string
+	ExpectedOutput string
+}{
+	{
+		exampleSignature, true, exampleBody,
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"Kegsay/flow-jsdoc#22 master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal: Test Travis webhook support",
+	},
+}
+
+type MockTransport struct {
+	roundTrip func(*http.Request) (*http.Response, error)
+}
+
+func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.roundTrip(req)
+}
+
+func TestTravisCI(t *testing.T) {
+	database.SetServiceDB(&database.NopStorage{})
+
+	// When the service tries to get Travis' public key, return the constant
+	urlToKey := make(map[string]string)
+	urlToKey["https://api.travis-ci.org/config"] = travisOrgPEMPublicKey
+	urlToKey["https://api.travis-ci.com/config"] = travisComPEMPublicKey
+	travisTransport := struct{ MockTransport }{}
+	travisTransport.roundTrip = func(req *http.Request) (*http.Response, error) {
+		if key := urlToKey[req.URL.String()]; key != "" {
+			escKey, _ := json.Marshal(key)
+			return &http.Response{
+				StatusCode: 200,
+				Body: ioutil.NopCloser(bytes.NewBufferString(
+					`{"config":{"notifications":{"webhook":{"public_key":` + string(escKey) + `}}}}`,
+				)),
+			}, nil
+		}
+		return nil, fmt.Errorf("Unhandled URL %s", req.URL.String())
+	}
+	// clobber the http client that the service uses to talk to Travis
+	httpClient = &http.Client{Transport: travisTransport}
+
+	// Intercept message sending to Matrix and mock responses
+	msgs := []matrix.HTMLMessage{}
+	matrixTrans := struct{ MockTransport }{}
+	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+		if !strings.Contains(req.URL.String(), "/send/m.room.message") {
+			return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String())
+		}
+		var msg matrix.HTMLMessage
+		if err := json.NewDecoder(req.Body).Decode(&msg); err != nil {
+			return nil, err
+		}
+		msgs = append(msgs, msg)
+		return &http.Response{
+			StatusCode: 200,
+			Body:       ioutil.NopCloser(bytes.NewBufferString(`{"event_id":"$yup:event"}`)),
+		}, nil
+	}
+	u, _ := url.Parse("https://hyrule")
+	matrixCli := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@travisci:hyrule")
+
+	// BEGIN running the Travis-CI table tests
+	// ---------------------------------------
+	for _, test := range travisTests {
+		msgs = []matrix.HTMLMessage{} // reset sent messages
+		mockWriter := httptest.NewRecorder()
+		travis := makeService(t, test.Template)
+		if travis == nil {
+			t.Error("TestTravisCI Failed to create service")
+			continue
+		}
+		if err := travis.Register(nil, matrixCli); err != nil {
+			t.Errorf("TestTravisCI Failed to Register(): %s", err)
+			continue
+		}
+		req, err := http.NewRequest(
+			"POST", "https://neb.endpoint/travis-ci-service", bytes.NewBufferString(test.Body),
+		)
+		if err != nil {
+			t.Errorf("TestTravisCI Failed to create webhook request: %s", err)
+			continue
+		}
+		req.Header.Set("Signature", test.Signature)
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		travis.OnReceiveWebhook(mockWriter, req, matrixCli)
+
+		if mockWriter.Code != 200 {
+			t.Errorf("TestTravisCI OnReceiveWebhook want code %d, got %d", 200, mockWriter.Code)
+		}
+	}
+}
+
+func makeService(t *testing.T, template string) *Service {
+	srv, err := types.CreateService("id", ServiceType, "@travisci:hyrule", []byte(
+		`{
+			"rooms":{
+				"!ewfug483gsfe:localhost": {
+					"repos": {
+						"Kegsay/flow-jsdoc": {
+							"template": "`+template+`"
+						}
+					}
+				}
+			}
+		}`,
+	))
+	if err != nil {
+		t.Error("Failed to create Travis-CI service: ", err)
+		return nil
+	}
+	return srv.(*Service)
+}
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/verify.go b/src/github.com/matrix-org/go-neb/services/travisci/verify.go
index 9454c2b..dfde078 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/verify.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/verify.go
@@ -15,12 +15,13 @@ import (
 
 // Host => Public Key.
 // Travis has a .com and .org with different public keys.
+// .org is the public one and is one we will try first, then .com
 var travisPublicKeyMap = map[string]*rsa.PublicKey{
 	"api.travis-ci.org": nil,
 	"api.travis-ci.com": nil,
 }
 
-func verifyOrigin(host string, payload []byte, sigHeader string) error {
+func verifyOrigin(payload []byte, sigHeader string) error {
 	/*
 		From: https://docs.travis-ci.com/user/notifications#Verifying-Webhook-requests
 			 1. Pick up the payload data from the HTTP request’s body.
@@ -30,40 +31,53 @@ func verifyOrigin(host string, payload []byte, sigHeader string) error {
 				the relevant API server. (e.g., https://api.travis-ci.org/config)
 			4. Verify the signature using the public key and SHA1 digest.
 	*/
-	// 2. Get signature bytes
 	sig, err := base64.StdEncoding.DecodeString(sigHeader)
 	if err != nil {
-		return err
+		return fmt.Errorf("verifyOrigin: Failed to decode signature as base64: %s", err)
 	}
 
-	// 3. Get public key
-	pubKey, validURL := travisPublicKeyMap[host]
-	if !validURL {
-		return fmt.Errorf("Invalid host: %s", host)
+	if err := loadPublicKeys(); err != nil {
+		return fmt.Errorf("verifyOrigin: Failed to cache Travis public keys: %s", err)
 	}
-	if pubKey == nil {
-		pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config")
-		if err != nil {
-			return err
-		}
-		block, _ := pem.Decode([]byte(pemPubKey))
-		if block == nil {
-			return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host)
-		}
 
-		k, err := x509.ParsePKIXPublicKey(block.Bytes)
-		if err != nil {
-			return err
+	// 4. Verify with SHA1
+	// NB: We don't know who sent this request (no Referer header or anything) so we need to try
+	//     both public keys at both endpoints. We use the .org one first since it's more popular.
+	var verifyErr error
+	for _, host := range []string{"api.travis-ci.org", "api.travis-ci.com"} {
+		h := sha1.New()
+		h.Write(payload)
+		digest := h.Sum(nil)
+		verifyErr = rsa.VerifyPKCS1v15(travisPublicKeyMap[host], crypto.SHA1, digest, sig)
+		if verifyErr == nil {
+			return nil // Valid for this key
 		}
-		pubKey = k.(*rsa.PublicKey)
-		travisPublicKeyMap[host] = pubKey
 	}
+	return fmt.Errorf("verifyOrigin: Signature verification failed: %s", verifyErr)
+}
 
-	// 4. Verify with SHA1
-	h := sha1.New()
-	h.Write(payload)
-	digest := h.Sum(nil)
-	return rsa.VerifyPKCS1v15(pubKey, crypto.SHA1, digest, sig)
+func loadPublicKeys() error {
+	for _, host := range []string{"api.travis-ci.com", "api.travis-ci.org"} {
+		pubKey := travisPublicKeyMap[host]
+		if pubKey == nil {
+			pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config")
+			if err != nil {
+				return err
+			}
+			block, _ := pem.Decode([]byte(pemPubKey))
+			if block == nil {
+				return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host)
+			}
+
+			k, err := x509.ParsePKIXPublicKey(block.Bytes)
+			if err != nil {
+				return err
+			}
+			pubKey = k.(*rsa.PublicKey)
+			travisPublicKeyMap[host] = pubKey
+		}
+	}
+	return nil
 }
 
 func fetchPEMPublicKey(travisURL string) (key string, err error) {
@@ -88,8 +102,8 @@ func fetchPEMPublicKey(travisURL string) (key string, err error) {
 		return
 	}
 	key = configStruct.Config.Notifications.Webhook.PublicKey
-	if key == "" || strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") {
-		err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key.")
+	if key == "" || !strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") {
+		err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key: %s", key)
 	}
 	return
 }

From 3a8b4a6aca4de0361e5bc301cc4afd6300d616f1 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 16 Nov 2016 11:51:58 +0000
Subject: [PATCH 09/51] Add working test and implement template parsing

---
 .../go-neb/services/travisci/travisci.go      | 74 ++++++++-----------
 .../go-neb/services/travisci/travisci_test.go | 24 ++++--
 2 files changed, 47 insertions(+), 51 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index fa82a5a..7391268 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -106,44 +106,26 @@ type webhookNotification struct {
 	} `json:"repository"`
 }
 
-// The template variables a user can use in their messages
-type notificationTemplate struct {
-	RepositorySlug string
-	Repository     string // Deprecated: alias for RepositorySlug
-	RepositoryName string
-	BuildNumber    string
-	BuildID        string
-	Branch         string
-	Commit         string
-	Author         string
-	CommitMessage  string
-	CommitSubject  string
-	Result         string
-	Message        string
-	Duration       string
-	ElapsedTime    string
-	CompareURL     string
-	BuildURL       string
-}
-
-func notifToTemplate(n webhookNotification) (t notificationTemplate) {
-	t.RepositorySlug = n.Repository.OwnerName + "/" + n.Repository.Name
-	t.Repository = t.RepositorySlug
-	t.RepositoryName = n.Repository.Name
-	t.BuildNumber = n.Number
-	t.BuildID = strconv.Itoa(n.ID)
-	t.Branch = n.Branch
-	t.Commit = n.Commit
-	t.Author = n.CommitterName // author: commit author name
+// Converts a webhook notification into a map of template var name to value
+func notifToTemplate(n webhookNotification) map[string]string {
+	t := make(map[string]string)
+	t["repository_slug"] = n.Repository.OwnerName + "/" + n.Repository.Name
+	t["repository"] = t["repository_slug"] // Deprecated form but still used everywhere in people's templates
+	t["repository_name"] = n.Repository.Name
+	t["build_number"] = n.Number
+	t["build_id"] = strconv.Itoa(n.ID)
+	t["branch"] = n.Branch
+	t["commit"] = n.Commit
+	t["author"] = n.CommitterName // author: commit author name
 	// commit_message: commit message of build
 	// commit_subject: first line of the commit message
-	t.CommitMessage = n.Message
+	t["commit_message"] = n.Message
 	subjAndMsg := strings.SplitN(n.Message, "\n", 2)
-	t.CommitSubject = subjAndMsg[0]
+	t["commit_subject"] = subjAndMsg[0]
 	if n.Status != nil {
-		t.Result = strconv.Itoa(*n.Status)
+		t["result"] = strconv.Itoa(*n.Status)
 	}
-	t.Message = n.StatusMessage
+	t["message"] = n.StatusMessage
 
 	if n.StartedAt != nil && n.FinishedAt != nil {
 		// duration: total duration of all builds in the matrix -- TODO
@@ -157,18 +139,22 @@ func notifToTemplate(n webhookNotification) (t notificationTemplate) {
 				"finished_at": *n.FinishedAt,
 			}).Warn("Failed to parse Travis-CI start/finish times.")
 		} else {
-			t.Duration = finish.Sub(start).String()
-			t.ElapsedTime = t.Duration
+			t["duration"] = finish.Sub(start).String()
+			t["elapsed_time"] = t["duration"]
 		}
 	}
 
-	t.CompareURL = n.CompareURL
-	t.BuildURL = n.BuildURL
-	return
+	t["compare_url"] = n.CompareURL
+	t["build_url"] = n.BuildURL
+	return t
 }
 
-func travisToGoTemplate(travisTmpl string) (goTmpl string) {
-	return
+func outputForTemplate(travisTmpl string, tmpl map[string]string) (out string) {
+	out = travisTmpl
+	for tmplVar, tmplValue := range tmpl {
+		out = strings.Replace(out, "%{"+tmplVar+"}", tmplValue, -1)
+	}
+	return out
 }
 
 // OnReceiveWebhook receives requests from Travis-CI and possibly sends requests to Matrix as a result.
@@ -215,7 +201,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 		return
 	}
 	whForRepo := notif.Repository.OwnerName + "/" + notif.Repository.Name
-	// TODO tmplData := notifToTemplate(notif)
+	tmplData := notifToTemplate(notif)
 
 	logger := log.WithFields(log.Fields{
 		"repo": whForRepo,
@@ -226,8 +212,10 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 			if ownerRepo != whForRepo {
 				continue
 			}
-			tmpl := travisToGoTemplate(repoData.Template)
-			msg := tmpl // TODO
+			msg := matrix.TextMessage{
+				Body:    outputForTemplate(repoData.Template, tmplData),
+				MsgType: "m.notice",
+			}
 
 			logger.WithFields(log.Fields{
 				"msg":     msg,
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index 1848a81..0affa2f 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -4,15 +4,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
-	"github.com/matrix-org/go-neb/types"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strings"
 	"testing"
+
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/types"
 )
 
 const travisOrgPEMPublicKey = (`-----BEGIN PUBLIC KEY-----
@@ -70,7 +71,7 @@ var travisTests = []struct {
 	{
 		exampleSignature, true, exampleBody,
 		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
-		"Kegsay/flow-jsdoc#22 master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal: Test Travis webhook support",
+		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
 	},
 }
 
@@ -106,15 +107,15 @@ func TestTravisCI(t *testing.T) {
 	httpClient = &http.Client{Transport: travisTransport}
 
 	// Intercept message sending to Matrix and mock responses
-	msgs := []matrix.HTMLMessage{}
+	msgs := []matrix.TextMessage{}
 	matrixTrans := struct{ MockTransport }{}
 	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
 		if !strings.Contains(req.URL.String(), "/send/m.room.message") {
 			return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String())
 		}
-		var msg matrix.HTMLMessage
+		var msg matrix.TextMessage
 		if err := json.NewDecoder(req.Body).Decode(&msg); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("Failed to decode request JSON: %s", err)
 		}
 		msgs = append(msgs, msg)
 		return &http.Response{
@@ -128,7 +129,7 @@ func TestTravisCI(t *testing.T) {
 	// BEGIN running the Travis-CI table tests
 	// ---------------------------------------
 	for _, test := range travisTests {
-		msgs = []matrix.HTMLMessage{} // reset sent messages
+		msgs = []matrix.TextMessage{} // reset sent messages
 		mockWriter := httptest.NewRecorder()
 		travis := makeService(t, test.Template)
 		if travis == nil {
@@ -153,6 +154,13 @@ func TestTravisCI(t *testing.T) {
 		if mockWriter.Code != 200 {
 			t.Errorf("TestTravisCI OnReceiveWebhook want code %d, got %d", 200, mockWriter.Code)
 		}
+		if len(msgs) != 1 {
+			t.Errorf("TestTravisCI want sent messages %d, got %d ", 1, len(msgs))
+			continue
+		}
+		if msgs[0].Body != test.ExpectedOutput {
+			t.Errorf("TestTravisCI want matrix body '%s', got '%s'", test.ExpectedOutput, msgs[0].Body)
+		}
 	}
 }
 

From 7f8c36ba89534bdcf517076a4da0e9afd0c131b3 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 16 Nov 2016 13:09:17 +0000
Subject: [PATCH 10/51] Add a few more tests

---
 .../go-neb/services/travisci/travisci.go      |  2 +-
 .../go-neb/services/travisci/travisci_test.go | 42 +++++++++++++++----
 2 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 7391268..9fc4fed 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -125,7 +125,7 @@ func notifToTemplate(n webhookNotification) map[string]string {
 	if n.Status != nil {
 		t["result"] = strconv.Itoa(*n.Status)
 	}
-	t["message"] = n.StatusMessage
+	t["message"] = n.StatusMessage // message: Travis CI message to the build
 
 	if n.StartedAt != nil && n.FinishedAt != nil {
 		// duration: total duration of all builds in the matrix -- TODO
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index 0affa2f..81d17b7 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -73,6 +73,17 @@ var travisTests = []struct {
 		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
 		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
 	},
+	{
+		"obviously_invalid_signature", false, exampleBody,
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
+	},
+	{
+		// Payload is valid but doesn't match signature now
+		exampleSignature, false, strings.TrimSuffix(exampleBody, "%7D") + "%2C%22EXTRA_KEY%22%3Anull%7D",
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
+	},
 }
 
 type MockTransport struct {
@@ -151,19 +162,32 @@ func TestTravisCI(t *testing.T) {
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 		travis.OnReceiveWebhook(mockWriter, req, matrixCli)
 
-		if mockWriter.Code != 200 {
-			t.Errorf("TestTravisCI OnReceiveWebhook want code %d, got %d", 200, mockWriter.Code)
-		}
-		if len(msgs) != 1 {
-			t.Errorf("TestTravisCI want sent messages %d, got %d ", 1, len(msgs))
-			continue
-		}
-		if msgs[0].Body != test.ExpectedOutput {
-			t.Errorf("TestTravisCI want matrix body '%s', got '%s'", test.ExpectedOutput, msgs[0].Body)
+		if test.ValidSignature {
+			if !assertResponse(t, mockWriter, msgs, 200, 1) {
+				continue
+			}
+
+			if msgs[0].Body != test.ExpectedOutput {
+				t.Errorf("TestTravisCI want matrix body '%s', got '%s'", test.ExpectedOutput, msgs[0].Body)
+			}
+		} else {
+			assertResponse(t, mockWriter, msgs, 403, 0)
 		}
 	}
 }
 
+func assertResponse(t *testing.T, w *httptest.ResponseRecorder, msgs []matrix.TextMessage, expectCode int, expectMsgLength int) bool {
+	if w.Code != expectCode {
+		t.Errorf("TestTravisCI OnReceiveWebhook want HTTP code %d, got %d", expectCode, w.Code)
+		return false
+	}
+	if len(msgs) != expectMsgLength {
+		t.Errorf("TestTravisCI want %d sent messages, got %d ", expectMsgLength, len(msgs))
+		return false
+	}
+	return true
+}
+
 func makeService(t *testing.T, template string) *Service {
 	srv, err := types.CreateService("id", ServiceType, "@travisci:hyrule", []byte(
 		`{

From be52bdedf11f181d6532f2ee6c8023612bdbe31b Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 16 Nov 2016 13:21:39 +0000
Subject: [PATCH 11/51] Clarify docs

---
 .../matrix-org/go-neb/services/travisci/travisci.go    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 9fc4fed..ba78c59 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -1,3 +1,4 @@
+// Package travisci implements a Service capable of processing webhooks from Travis-CI.
 package travisci
 
 import (
@@ -49,7 +50,8 @@ var httpClient = &http.Client{}
 //   }
 type Service struct {
 	types.DefaultService
-	webhookEndpointURL string
+	// The URL which should be added to .travis.yml - Populated by Go-NEB after Service registration.
+	WebhookEndpointURL string
 	// A map from Matrix room ID to Github-style owner/repo repositories.
 	Rooms map[string]struct {
 		// A map of "owner/repo" to configuration information
@@ -159,13 +161,13 @@ func outputForTemplate(travisTmpl string, tmpl map[string]string) (out string) {
 
 // OnReceiveWebhook receives requests from Travis-CI and possibly sends requests to Matrix as a result.
 //
-// If the "repository.url" matches a known Github repository, a notification will be formed from the
+// If the repository matches a known Github repository, a notification will be formed from the
 // template for that repository and a notice will be sent to Matrix.
 //
 // Go-NEB cannot register with Travis-CI for webhooks automatically. The user must manually add the
 // webhook endpoint URL to their .travis.yml file:
 //    notifications:
-//        webhooks: http://your-domain.com/notifications
+//        webhooks: http://go-neb-endpoint.com/notifications
 //
 // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
 func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
@@ -265,7 +267,7 @@ func init() {
 	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
 		return &Service{
 			DefaultService:     types.NewDefaultService(serviceID, serviceUserID, ServiceType),
-			webhookEndpointURL: webhookEndpointURL,
+			WebhookEndpointURL: webhookEndpointURL,
 		}
 	})
 }

From 1d23f6bd94e637e36f6992ae2fe7eb8b3aeae214 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 16 Nov 2016 13:42:20 +0000
Subject: [PATCH 12/51] Actually use the default and enable travisci service

---
 src/github.com/matrix-org/go-neb/goneb.go                      | 1 +
 src/github.com/matrix-org/go-neb/services/travisci/travisci.go | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/src/github.com/matrix-org/go-neb/goneb.go b/src/github.com/matrix-org/go-neb/goneb.go
index d881aca..79ac3e4 100644
--- a/src/github.com/matrix-org/go-neb/goneb.go
+++ b/src/github.com/matrix-org/go-neb/goneb.go
@@ -26,6 +26,7 @@ import (
 	_ "github.com/matrix-org/go-neb/services/guggy"
 	_ "github.com/matrix-org/go-neb/services/jira"
 	_ "github.com/matrix-org/go-neb/services/rssbot"
+	_ "github.com/matrix-org/go-neb/services/travisci"
 	"github.com/matrix-org/go-neb/types"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/prometheus/client_golang/prometheus"
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index ba78c59..8185794 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -152,6 +152,9 @@ func notifToTemplate(n webhookNotification) map[string]string {
 }
 
 func outputForTemplate(travisTmpl string, tmpl map[string]string) (out string) {
+	if travisTmpl == "" {
+		travisTmpl = DefaultTemplate
+	}
 	out = travisTmpl
 	for tmplVar, tmplValue := range tmpl {
 		out = strings.Replace(out, "%{"+tmplVar+"}", tmplValue, -1)

From 9b55ba9963cce0458b63e6494621157c062bd9e4 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 11:18:27 +0000
Subject: [PATCH 13/51] Travis: Join rooms on Register()

---
 .../matrix-org/go-neb/services/travisci/travisci.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 8185794..543b9d6 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -245,6 +245,7 @@ func (s *Service) Register(oldService types.Service, client *matrix.Client) erro
 			}
 		}
 	}
+	s.joinRooms(client)
 	return nil
 }
 
@@ -266,6 +267,18 @@ func (s *Service) PostRegister(oldService types.Service) {
 	}
 }
 
+func (s *Service) joinRooms(client *matrix.Client) {
+	for roomID := range s.Rooms {
+		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
+			log.WithFields(log.Fields{
+				log.ErrorKey: err,
+				"room_id":    roomID,
+				"user_id":    client.UserID,
+			}).Error("Failed to join room")
+		}
+	}
+}
+
 func init() {
 	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
 		return &Service{

From 6ee73d5c496832315de4fc93a9d76fa7327dc118 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 13:51:38 +0000
Subject: [PATCH 14/51] lower_case_and_underscores webhook_url

---
 src/github.com/matrix-org/go-neb/services/travisci/travisci.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 543b9d6..47c661b 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -51,7 +51,7 @@ var httpClient = &http.Client{}
 type Service struct {
 	types.DefaultService
 	// The URL which should be added to .travis.yml - Populated by Go-NEB after Service registration.
-	WebhookEndpointURL string
+	WebhookEndpointURL string `json:"webhook_url"`
 	// A map from Matrix room ID to Github-style owner/repo repositories.
 	Rooms map[string]struct {
 		// A map of "owner/repo" to configuration information

From db5683d6f5f5ac13589d24d7243fd1270b209d15 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 14:23:43 +0000
Subject: [PATCH 15/51] HACK: Have private/public fields for the webhook URL

We can't just use a public field because the caller may clobber it. If they
do clobber it, we've then lost the ability to set it back to what it was because
we don't store another copy anywhere. This patch fixes it by keeping a private
ref and always clobbering on Register().

This is horrible. We need to separate internal-storage and external-fields
better.
---
 .../matrix-org/go-neb/services/travisci/travisci.go         | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 47c661b..5f2392d 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -50,8 +50,9 @@ var httpClient = &http.Client{}
 //   }
 type Service struct {
 	types.DefaultService
+	webhookEndpointURL string
 	// The URL which should be added to .travis.yml - Populated by Go-NEB after Service registration.
-	WebhookEndpointURL string `json:"webhook_url"`
+	WebhookURL string `json:"webhook_url"`
 	// A map from Matrix room ID to Github-style owner/repo repositories.
 	Rooms map[string]struct {
 		// A map of "owner/repo" to configuration information
@@ -237,6 +238,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 
 // Register makes sure the Config information supplied is valid.
 func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+	s.WebhookURL = s.webhookEndpointURL
 	for _, roomData := range s.Rooms {
 		for repo := range roomData.Repos {
 			match := ownerRepoRegex.FindStringSubmatch(repo)
@@ -283,7 +285,7 @@ func init() {
 	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
 		return &Service{
 			DefaultService:     types.NewDefaultService(serviceID, serviceUserID, ServiceType),
-			WebhookEndpointURL: webhookEndpointURL,
+			webhookEndpointURL: webhookEndpointURL,
 		}
 	})
 }

From 61c3110cfe12597c3e1206a6ac34bfd8e00cdc59 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 16:28:08 +0000
Subject: [PATCH 16/51] Fix travis-ci duration parsing

The example from the docs was very clearly with spaces after hours and minutes:
```
  "committed_at": "2011-11-11T11: 11: 11Z",
```
But in fact those spaces do not exist in the wild. Great.
---
 .../matrix-org/go-neb/services/travisci/travisci.go         | 6 +++---
 .../matrix-org/go-neb/services/travisci/travisci_test.go    | 5 +++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 5f2392d..716bcbd 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -133,9 +133,9 @@ func notifToTemplate(n webhookNotification) map[string]string {
 	if n.StartedAt != nil && n.FinishedAt != nil {
 		// duration: total duration of all builds in the matrix -- TODO
 		// elapsed_time: time between build start and finish
-		// Example from docs: "2011-11-11T11: 11: 11Z"
-		start, err := time.Parse("2006-01-02T15: 04: 05Z", *n.StartedAt)
-		finish, err2 := time.Parse("2006-01-02T15: 04: 05Z", *n.FinishedAt)
+		// Example from docs: "2011-11-11T11:11:11Z"
+		start, err := time.Parse("2006-01-02T15:04:05Z", *n.StartedAt)
+		finish, err2 := time.Parse("2006-01-02T15:04:05Z", *n.FinishedAt)
 		if err != nil || err2 != nil {
 			log.WithFields(log.Fields{
 				"started_at":  *n.StartedAt,
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index 81d17b7..f2527c3 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -84,6 +84,11 @@ var travisTests = []struct {
 		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
 		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
 	},
+	{
+		exampleSignature, true, exampleBody,
+		"%{repository}#%{build_number} %{duration}",
+		"Kegsay/flow-jsdoc#18 32s",
+	},
 }
 
 type MockTransport struct {

From 9376c73838b60a52d47d5f4e40041b2121540e6d Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 16:57:01 +0000
Subject: [PATCH 17/51] Test travis config

---
 .travis.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..4d4ab05
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,17 @@
+language: go
+go:
+ - 1.6
+install:
+ - go get github.com/constabulary/gb/...
+ - go get github.com/golang/lint/golint
+ - go get github.com/fzipp/gocyclo
+script: gb build github.com/matrix-org/go-neb && ./hooks/pre-commit
+
+notifications:
+    webhooks:
+        urls:
+            - "https://scalar.vector.im/api/neb/services/hooks/dHJhdmlzLWNpLyU0MGtlZ2FuJTNBbWF0cml4Lm9yZy8lMjFhWmthbkFuV0VkeGNSSVFrV24lM0FtYXRyaXgub3Jn"
+        on_success: always  # always|never|change
+        on_failure: always
+        on_start: never
+

From e8b2454b566ab32837902d1f885b0f7cd352fc60 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 16:58:35 +0000
Subject: [PATCH 18/51] Kick travis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 4d4ab05..354284f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ install:
  - go get github.com/constabulary/gb/...
  - go get github.com/golang/lint/golint
  - go get github.com/fzipp/gocyclo
+
 script: gb build github.com/matrix-org/go-neb && ./hooks/pre-commit
 
 notifications:

From 45d167736dd70195532cc8111964914618559fe5 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 17:06:32 +0000
Subject: [PATCH 19/51] Purposefully break build

---
 .../matrix-org/go-neb/services/travisci/travisci_test.go        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index f2527c3..e0bd20c 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -86,7 +86,7 @@ var travisTests = []struct {
 	},
 	{
 		exampleSignature, true, exampleBody,
-		"%{repository}#%{build_number} %{duration}",
+		"%{repository}#%{build_number} %{duration}a",
 		"Kegsay/flow-jsdoc#18 32s",
 	},
 }

From 2d61f5f18682e9dbeba69d86c5a1e2517946766e Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 17:12:53 +0000
Subject: [PATCH 20/51] Fix build

---
 .../matrix-org/go-neb/services/travisci/travisci_test.go        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index e0bd20c..f2527c3 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -86,7 +86,7 @@ var travisTests = []struct {
 	},
 	{
 		exampleSignature, true, exampleBody,
-		"%{repository}#%{build_number} %{duration}a",
+		"%{repository}#%{build_number} %{duration}",
 		"Kegsay/flow-jsdoc#18 32s",
 	},
 }

From 07574c1f0d716f7efb5c3eea4f5c0e8a6a62a445 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 17 Nov 2016 17:23:36 +0000
Subject: [PATCH 21/51] Actually shorten the commit SHA

---
 .../matrix-org/go-neb/services/travisci/travisci.go       | 8 ++++++--
 .../matrix-org/go-neb/services/travisci/travisci_test.go  | 6 +++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 716bcbd..89ad304 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -118,8 +118,12 @@ func notifToTemplate(n webhookNotification) map[string]string {
 	t["build_number"] = n.Number
 	t["build_id"] = strconv.Itoa(n.ID)
 	t["branch"] = n.Branch
-	t["commit"] = n.Commit
-	t["author"] = n.CommitterName // author: commit author name
+	shaLength := len(n.Commit)
+	if shaLength > 10 {
+		shaLength = 10
+	}
+	t["commit"] = n.Commit[:shaLength] // shortened commit SHA
+	t["author"] = n.CommitterName      // author: commit author name
 	// commit_message: commit message of build
 	// commit_subject: first line of the commit message
 	t["commit_message"] = n.Message
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index f2527c3..5a8e2e5 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -71,18 +71,18 @@ var travisTests = []struct {
 	{
 		exampleSignature, true, exampleBody,
 		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
-		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
+		"Kegsay/flow-jsdoc#18 (master - 3a092c3a60 : Kegan Dougal): Passed",
 	},
 	{
 		"obviously_invalid_signature", false, exampleBody,
 		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
-		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
+		"Kegsay/flow-jsdoc#18 (master - 3a092c3a60 : Kegan Dougal): Passed",
 	},
 	{
 		// Payload is valid but doesn't match signature now
 		exampleSignature, false, strings.TrimSuffix(exampleBody, "%7D") + "%2C%22EXTRA_KEY%22%3Anull%7D",
 		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
-		"Kegsay/flow-jsdoc#18 (master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal): Passed",
+		"Kegsay/flow-jsdoc#18 (master - 3a092c3a60 : Kegan Dougal): Passed",
 	},
 	{
 		exampleSignature, true, exampleBody,

From 182f0bbb6b548aa9138f43920a9e916659606746 Mon Sep 17 00:00:00 2001
From: Kegsay <kegsay@gmail.com>
Date: Thu, 17 Nov 2016 17:26:59 +0000
Subject: [PATCH 22/51] Add build badge

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index ff3c047..87e565f 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 # Go-NEB
+[![Build Status](https://travis-ci.org/matrix-org/go-neb.svg?branch=master)](https://travis-ci.org/matrix-org/go-neb)
 
 Go-NEB is a [Matrix](https://matrix.org) bot written in Go. It is the successor to [Matrix-NEB](https://github.com/matrix-org/Matrix-NEB), the original Matrix bot written in Python.
 

From ca24fb1955991832f28da60e193df1c6123734ae Mon Sep 17 00:00:00 2001
From: Kegsay <kegsay@gmail.com>
Date: Fri, 18 Nov 2016 14:55:50 +0000
Subject: [PATCH 23/51] Update README.md

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 87e565f..c1e37c1 100644
--- a/README.md
+++ b/README.md
@@ -71,6 +71,14 @@ Invite the bot user into a Matrix room and type `!echo hello world`. It will rep
  - Ability to query Giphy's "text-to-gif" engine.
  
 ### Guggy
+ - Ability to query Guggy's gif engine.
+ 
+### RSS Bot
+ - Ability to read Atom/RSS feeds.
+ 
+### Travis CI
+ - Ability to receive incoming build notifications.
+ - Ability to adjust the message which is sent into the room.
 
 
 # Installing

From 6bfcbfee181e7d8c96e0da6f2663da410c44d4de Mon Sep 17 00:00:00 2001
From: Kegsay <kegsay@gmail.com>
Date: Fri, 18 Nov 2016 15:18:37 +0000
Subject: [PATCH 24/51] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index c1e37c1..87c7d2f 100644
--- a/README.md
+++ b/README.md
@@ -142,6 +142,7 @@ List of Services:
  - [Guggy](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/services/guggy/) - A GIF bot
  - [JIRA](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/services/jira/) - Integration with JIRA
  - [RSS Bot](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/services/rssbot/) - An Atom/RSS feed reader
+ - [Travis CI](https://matrix-org.github.io/go-neb/pkg/github.com/matrix-org/go-neb/services/travisci/) - Receive build notifications from Travis CI
 
 
 ## Configuring Realms

From 58df215ad63c726085429d304ac893fe900e605b Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 18 Nov 2016 15:07:16 +0000
Subject: [PATCH 25/51] Only emit travis results on success change

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 354284f..2971d15 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ notifications:
     webhooks:
         urls:
             - "https://scalar.vector.im/api/neb/services/hooks/dHJhdmlzLWNpLyU0MGtlZ2FuJTNBbWF0cml4Lm9yZy8lMjFhWmthbkFuV0VkeGNSSVFrV24lM0FtYXRyaXgub3Jn"
-        on_success: always  # always|never|change
+        on_success: change  # always|never|change
         on_failure: always
         on_start: never
 

From 14f49f2fde1acb3fe8061529a5075a7383ae3ece Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 18 Nov 2016 16:31:59 +0000
Subject: [PATCH 26/51] Use a text formatter which logs as key=val instead of
 JSON

We exclusively view logs using `less` and `tail`. These do not read JSON well.
Logging as JSON makes it a PITA to read logs and debug problems. We do not
appear to make use of JSON logging, and have no good terminal-based
structured log viewer either.

As a result, I've now removed JSON logging from this project and replaced it
with the standard `TextFormatter` (colors off). This still logs in a structured
way:

```
time="2016-11-18 16:25:46.787373" level=info msg="Got filter ID" filter=717 syncing=1 user_id="@goneb:localhost"
time="2016-11-18 16:25:46.787525" level=info msg="Starting sync" next_batch="s26928_287972_2_1029_26_1_2" syncing=1 user_id="@goneb:localhost"
```

So we can still analyse logs sanely at a later date should we need to. This
feels like the best compromise here between pragmatism and ideals.
---
 src/github.com/matrix-org/go-neb/goneb.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/goneb.go b/src/github.com/matrix-org/go-neb/goneb.go
index 79ac3e4..168d45f 100644
--- a/src/github.com/matrix-org/go-neb/goneb.go
+++ b/src/github.com/matrix-org/go-neb/goneb.go
@@ -227,7 +227,12 @@ func main() {
 			filepath.Join(e.LogDir, "info.log"),
 			filepath.Join(e.LogDir, "warn.log"),
 			filepath.Join(e.LogDir, "error.log"),
-			nil, &dugong.DailyRotationSchedule{GZip: true},
+			&log.TextFormatter{
+				TimestampFormat:  "2006-01-02 15:04:05.000000",
+				DisableColors:    true,
+				DisableTimestamp: false,
+				DisableSorting:   false,
+			}, &dugong.DailyRotationSchedule{GZip: true},
 		))
 	}
 

From 4a3431efbb51247a79e34b3838d6ba2a9628af7b Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 18 Nov 2016 16:55:01 +0000
Subject: [PATCH 27/51] Fix #59 : Make pling commands case-insensitive

---
 src/github.com/matrix-org/go-neb/types/actions.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/types/actions.go b/src/github.com/matrix-org/go-neb/types/actions.go
index 707fa04..dbe9e7d 100644
--- a/src/github.com/matrix-org/go-neb/types/actions.go
+++ b/src/github.com/matrix-org/go-neb/types/actions.go
@@ -1,6 +1,9 @@
 package types
 
-import "regexp"
+import (
+	"regexp"
+	"strings"
+)
 
 // A Command is something that a user invokes by sending a message starting with '!'
 // followed by a list of strings that name the command, followed by a list of argument
@@ -28,7 +31,7 @@ func (command *Command) Matches(arguments []string) bool {
 		return false
 	}
 	for i, segment := range command.Path {
-		if segment != arguments[i] {
+		if strings.ToLower(segment) != strings.ToLower(arguments[i]) {
 			return false
 		}
 	}

From 085d3ecfd0e07d5deeb1fd2ad48019dadcac8fb4 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 18 Nov 2016 17:03:39 +0000
Subject: [PATCH 28/51] Allow '.' in owner/repo strings

---
 src/github.com/matrix-org/go-neb/services/github/github.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/github/github.go b/src/github.com/matrix-org/go-neb/services/github/github.go
index 40bdfb4..048c50c 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github.go
@@ -25,8 +25,8 @@ const ServiceType = "github"
 
 // Matches alphanumeric then a /, then more alphanumeric then a #, then a number.
 // E.g. owner/repo#11 (issue/PR numbers) - Captured groups for owner/repo/number
-var ownerRepoIssueRegex = regexp.MustCompile(`(([A-z0-9-_]+)/([A-z0-9-_]+))?#([0-9]+)`)
-var ownerRepoRegex = regexp.MustCompile(`^([A-z0-9-_]+)/([A-z0-9-_]+)$`)
+var ownerRepoIssueRegex = regexp.MustCompile(`(([A-z0-9-_.]+)/([A-z0-9-_.]+))?#([0-9]+)`)
+var ownerRepoRegex = regexp.MustCompile(`^([A-z0-9-_.]+)/([A-z0-9-_.]+)$`)
 
 // Service contains the Config fields for the Github service.
 //

From 314db5f6c7380ab269c1f55f2b388387736bfbeb Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 18 Nov 2016 17:36:28 +0000
Subject: [PATCH 29/51] Fix #105 : Mention which label was added/removed

---
 .../go-neb/services/github/webhook/webhook.go     | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
index b70614e..8d1b625 100644
--- a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
@@ -6,14 +6,15 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	log "github.com/Sirupsen/logrus"
-	"github.com/google/go-github/github"
-	"github.com/matrix-org/go-neb/errors"
-	"github.com/matrix-org/go-neb/matrix"
 	"html"
 	"io/ioutil"
 	"net/http"
 	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/google/go-github/github"
+	"github.com/matrix-org/go-neb/errors"
+	"github.com/matrix-org/go-neb/matrix"
 )
 
 // OnReceiveRequest processes incoming github webhook requests and returns a
@@ -148,11 +149,15 @@ func issueHTMLMessage(p github.IssuesEvent) string {
 	if p.Issue.Assignee != nil && p.Issue.Assignee.Login != nil {
 		actionTarget = fmt.Sprintf(" to %s", *p.Issue.Assignee.Login)
 	}
+	action := html.EscapeString(*p.Action)
+	if p.Label != nil && (*p.Action == "labeled" || *p.Action == "unlabeled") {
+		action = *p.Action + " [" + html.EscapeString(*p.Label.Name) + "] to"
+	}
 	return fmt.Sprintf(
 		"[<u>%s</u>] %s %s <b>issue #%d</b>: %s [%s]%s - %s",
 		html.EscapeString(*p.Repo.FullName),
 		html.EscapeString(*p.Sender.Login),
-		html.EscapeString(*p.Action),
+		action,
 		*p.Issue.Number,
 		html.EscapeString(*p.Issue.Title),
 		html.EscapeString(*p.Issue.State),

From 8ec37a4cf9f69b7d7091df63f851aa91d25f572d Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 24 Nov 2016 10:45:36 +0000
Subject: [PATCH 30/51] Add a dedicated testutils package

In the spirit of "if you have to do something 3 times then factor it
out", make a testutils package to put all the `RoundTrip` boilerplate.

I don't overly like having test packages, especially mixed in with
code, but I don't see a nicer way of doing this without ending up
with a sprawling mess of copypasta'd test boilerplate which will
be an absolute nightmare to maintain.

I think this is the lesser of two evils.
---
 .../go-neb/services/guggy/guggy_test.go         | 17 +++++------------
 .../go-neb/services/rssbot/rssbot_test.go       | 17 +++++------------
 .../go-neb/services/travisci/travisci_test.go   | 17 +++++------------
 .../matrix-org/go-neb/testutils/testutils.go    | 15 +++++++++++++++
 4 files changed, 30 insertions(+), 36 deletions(-)
 create mode 100644 src/github.com/matrix-org/go-neb/testutils/testutils.go

diff --git a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
index b7d5a05..f79bf74 100644
--- a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
+++ b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/matrix-org/go-neb/database"
 	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/testutils"
 	"github.com/matrix-org/go-neb/types"
 	"io/ioutil"
 	"net/http"
@@ -14,14 +15,6 @@ import (
 	"testing"
 )
 
-type MockTransport struct {
-	roundTrip func(*http.Request) (*http.Response, error)
-}
-
-func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	return t.roundTrip(req)
-}
-
 // TODO: It would be nice to tabularise this test so we can try failing different combinations of responses to make
 //       sure all cases are handled, rather than just the general case as is here.
 func TestCommand(t *testing.T) {
@@ -30,8 +23,8 @@ func TestCommand(t *testing.T) {
 	guggyImageURL := "https://guggy.com/gifs/23ryf872fg"
 
 	// Mock the response from Guggy
-	guggyTrans := struct{ MockTransport }{}
-	guggyTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+	guggyTrans := struct{ testutils.MockTransport }{}
+	guggyTrans.RT = func(req *http.Request) (*http.Response, error) {
 		guggyURL := "https://text2gif.guggy.com/guggify"
 		if req.URL.String() != guggyURL {
 			t.Fatalf("Bad URL: got %s want %s", req.URL.String(), guggyURL)
@@ -79,8 +72,8 @@ func TestCommand(t *testing.T) {
 	guggy := srv.(*Service)
 
 	// Mock the response from Matrix
-	matrixTrans := struct{ MockTransport }{}
-	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+	matrixTrans := struct{ testutils.MockTransport }{}
+	matrixTrans.RT = func(req *http.Request) (*http.Response, error) {
 		if req.URL.String() == guggyImageURL { // getting the guggy image
 			return &http.Response{
 				StatusCode: 200,
diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
index cc6b260..040b2ea 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/matrix-org/go-neb/database"
 	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/testutils"
 	"github.com/matrix-org/go-neb/types"
 )
 
@@ -36,20 +37,12 @@ const rssFeedXML = `
 </channel>
 </rss>`
 
-type MockTransport struct {
-	roundTrip func(*http.Request) (*http.Response, error)
-}
-
-func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	return t.roundTrip(req)
-}
-
 func TestHTMLEntities(t *testing.T) {
 	database.SetServiceDB(&database.NopStorage{})
 	feedURL := "https://thehappymaskshop.hyrule"
 	// Replace the cachingClient with a mock so we can intercept RSS requests
-	rssTrans := struct{ MockTransport }{}
-	rssTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+	rssTrans := struct{ testutils.MockTransport }{}
+	rssTrans.RT = func(req *http.Request) (*http.Response, error) {
 		if req.URL.String() != feedURL {
 			return nil, errors.New("Unknown test URL")
 		}
@@ -79,8 +72,8 @@ func TestHTMLEntities(t *testing.T) {
 	// Create the Matrix client which will send the notification
 	wg := sync.WaitGroup{}
 	wg.Add(1)
-	matrixTrans := struct{ MockTransport }{}
-	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+	matrixTrans := struct{ testutils.MockTransport }{}
+	matrixTrans.RT = func(req *http.Request) (*http.Response, error) {
 		if strings.HasPrefix(req.URL.Path, "/_matrix/client/r0/rooms/!linksroom:hyrule/send/m.room.message") {
 			// Check content body to make sure it is decoded
 			var msg matrix.HTMLMessage
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index 5a8e2e5..77b14e7 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/matrix-org/go-neb/database"
 	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/testutils"
 	"github.com/matrix-org/go-neb/types"
 )
 
@@ -91,14 +92,6 @@ var travisTests = []struct {
 	},
 }
 
-type MockTransport struct {
-	roundTrip func(*http.Request) (*http.Response, error)
-}
-
-func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	return t.roundTrip(req)
-}
-
 func TestTravisCI(t *testing.T) {
 	database.SetServiceDB(&database.NopStorage{})
 
@@ -106,8 +99,8 @@ func TestTravisCI(t *testing.T) {
 	urlToKey := make(map[string]string)
 	urlToKey["https://api.travis-ci.org/config"] = travisOrgPEMPublicKey
 	urlToKey["https://api.travis-ci.com/config"] = travisComPEMPublicKey
-	travisTransport := struct{ MockTransport }{}
-	travisTransport.roundTrip = func(req *http.Request) (*http.Response, error) {
+	travisTransport := struct{ testutils.MockTransport }{}
+	travisTransport.RT = func(req *http.Request) (*http.Response, error) {
 		if key := urlToKey[req.URL.String()]; key != "" {
 			escKey, _ := json.Marshal(key)
 			return &http.Response{
@@ -124,8 +117,8 @@ func TestTravisCI(t *testing.T) {
 
 	// Intercept message sending to Matrix and mock responses
 	msgs := []matrix.TextMessage{}
-	matrixTrans := struct{ MockTransport }{}
-	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+	matrixTrans := struct{ testutils.MockTransport }{}
+	matrixTrans.RT = func(req *http.Request) (*http.Response, error) {
 		if !strings.Contains(req.URL.String(), "/send/m.room.message") {
 			return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String())
 		}
diff --git a/src/github.com/matrix-org/go-neb/testutils/testutils.go b/src/github.com/matrix-org/go-neb/testutils/testutils.go
new file mode 100644
index 0000000..ba8508e
--- /dev/null
+++ b/src/github.com/matrix-org/go-neb/testutils/testutils.go
@@ -0,0 +1,15 @@
+package testutils
+
+import (
+	"net/http"
+)
+
+// MockTransport implements RoundTripper
+type MockTransport struct {
+	RT func(*http.Request) (*http.Response, error)
+}
+
+// RoundTrip is a RoundTripper
+func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.RT(req)
+}

From 7a382c8120a76de5b366226e729c2fbd96b409dd Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 24 Nov 2016 14:14:37 +0000
Subject: [PATCH 31/51] Add failing command parse tests

---
 .../matrix-org/go-neb/clients/clients.go      |  6 +-
 .../matrix-org/go-neb/clients/clients_test.go | 95 +++++++++++++++++++
 2 files changed, 98 insertions(+), 3 deletions(-)
 create mode 100644 src/github.com/matrix-org/go-neb/clients/clients_test.go

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 0b4287a..fec651c 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -16,7 +16,7 @@ import (
 )
 
 type nextBatchStore struct {
-	db *database.ServiceDB
+	db database.Storer
 }
 
 func (s nextBatchStore) Save(userID, nextBatch string) {
@@ -42,7 +42,7 @@ func (s nextBatchStore) Load(userID string) string {
 
 // A Clients is a collection of clients used for bot services.
 type Clients struct {
-	db         *database.ServiceDB
+	db         database.Storer
 	httpClient *http.Client
 	dbMutex    sync.Mutex
 	mapMutex   sync.Mutex
@@ -50,7 +50,7 @@ type Clients struct {
 }
 
 // New makes a new collection of matrix clients
-func New(db *database.ServiceDB, cli *http.Client) *Clients {
+func New(db database.Storer, cli *http.Client) *Clients {
 	clients := &Clients{
 		db:         db,
 		httpClient: cli,
diff --git a/src/github.com/matrix-org/go-neb/clients/clients_test.go b/src/github.com/matrix-org/go-neb/clients/clients_test.go
new file mode 100644
index 0000000..bc6f70e
--- /dev/null
+++ b/src/github.com/matrix-org/go-neb/clients/clients_test.go
@@ -0,0 +1,95 @@
+package clients
+
+import (
+	"fmt"
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/types"
+	"net/http"
+	"net/url"
+	"reflect"
+	"testing"
+)
+
+var commandParseTests = []struct {
+	body       string
+	expectArgs []string
+}{
+	{"!test word", []string{"word"}},
+	{"!test two words", []string{"two", "words"}},
+	{`!test "words with double quotes"`, []string{"words with double quotes"}},
+	{"!test 'words with single quotes'", []string{"words with single quotes"}},
+	{`!test 'single quotes' "double quotes"`, []string{"single quotes", "double quotes"}},
+	{`!test ‘smart single quotes’ “smart double quotes”`, []string{"smart single quotes", "smart double quotes"}},
+}
+
+type MockService struct {
+	types.DefaultService
+	commands []types.Command
+}
+
+func (s *MockService) Commands(cli *matrix.Client) []types.Command {
+	return s.commands
+}
+
+type MockStore struct {
+	database.NopStorage
+	service types.Service
+}
+
+func (d *MockStore) LoadServicesForUser(userID string) ([]types.Service, error) {
+	return []types.Service{d.service}, nil
+}
+
+type MockTransport struct {
+	roundTrip func(*http.Request) (*http.Response, error)
+}
+
+func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.roundTrip(req)
+}
+
+func TestCommandParsing(t *testing.T) {
+	var executedCmdArgs []string
+	cmds := []types.Command{
+		types.Command{
+			Path: []string{"test"},
+			Command: func(roomID, userID string, args []string) (interface{}, error) {
+				executedCmdArgs = args
+				return nil, nil
+			},
+		},
+	}
+	s := MockService{commands: cmds}
+	store := MockStore{service: &s}
+	database.SetServiceDB(&store)
+
+	trans := struct{ MockTransport }{}
+	trans.roundTrip = func(*http.Request) (*http.Response, error) {
+		return nil, fmt.Errorf("unhandled test path")
+	}
+	cli := &http.Client{
+		Transport: trans,
+	}
+	clients := New(&store, cli)
+	hsURL, _ := url.Parse("https://someplace.somewhere")
+	mxCli := matrix.NewClient(cli, hsURL, "token", "@service:user")
+
+	for _, input := range commandParseTests {
+		executedCmdArgs = []string{}
+		event := matrix.Event{
+			Type:   "m.room.message",
+			Sender: "@someone:somewhere",
+			RoomID: "!foo:bar",
+			Content: map[string]interface{}{
+				"body":    input.body,
+				"msgtype": "m.text",
+			},
+		}
+		clients.onMessageEvent(mxCli, &event)
+		if !reflect.DeepEqual(executedCmdArgs, input.expectArgs) {
+			t.Errorf("TestCommandParsing want %s, got %s", input.expectArgs, executedCmdArgs)
+		}
+	}
+
+}

From a9899557c57f853851e391613854b51c2f581988 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 24 Nov 2016 14:18:47 +0000
Subject: [PATCH 32/51] Fix #72: Support smart quotes

---
 src/github.com/matrix-org/go-neb/clients/clients.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index fec651c..4352766 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -192,6 +192,12 @@ func (c *Clients) onMessageEvent(client *matrix.Client, event *matrix.Event) {
 		return
 	}
 
+	// replace all smart quotes with their normal counterparts so shellwords can parse it
+	body = strings.Replace(body, `‘`, `'`, -1)
+	body = strings.Replace(body, `’`, `'`, -1)
+	body = strings.Replace(body, `“`, `"`, -1)
+	body = strings.Replace(body, `”`, `"`, -1)
+
 	var responses []interface{}
 
 	for _, service := range services {

From 683d36b5c505658f442c1cf713ece1ae1413c5f3 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 24 Nov 2016 16:10:12 +0000
Subject: [PATCH 33/51] Review comments

---
 .../matrix-org/go-neb/services/guggy/guggy_test.go  |  5 ++---
 .../go-neb/services/rssbot/rssbot_test.go           |  5 ++---
 .../go-neb/services/travisci/travisci_test.go       |  5 ++---
 .../matrix-org/go-neb/testutils/testutils.go        | 13 +++++++++++++
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
index f79bf74..480b8b8 100644
--- a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
+++ b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
@@ -23,8 +23,7 @@ func TestCommand(t *testing.T) {
 	guggyImageURL := "https://guggy.com/gifs/23ryf872fg"
 
 	// Mock the response from Guggy
-	guggyTrans := struct{ testutils.MockTransport }{}
-	guggyTrans.RT = func(req *http.Request) (*http.Response, error) {
+	guggyTrans := testutils.NewRoundTripper(func(req *http.Request) (*http.Response, error) {
 		guggyURL := "https://text2gif.guggy.com/guggify"
 		if req.URL.String() != guggyURL {
 			t.Fatalf("Bad URL: got %s want %s", req.URL.String(), guggyURL)
@@ -58,7 +57,7 @@ func TestCommand(t *testing.T) {
 			StatusCode: 200,
 			Body:       ioutil.NopCloser(bytes.NewBuffer(b)),
 		}, nil
-	}
+	})
 	// clobber the guggy service http client instance
 	httpClient = &http.Client{Transport: guggyTrans}
 
diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
index 040b2ea..4912d8c 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
@@ -41,8 +41,7 @@ func TestHTMLEntities(t *testing.T) {
 	database.SetServiceDB(&database.NopStorage{})
 	feedURL := "https://thehappymaskshop.hyrule"
 	// Replace the cachingClient with a mock so we can intercept RSS requests
-	rssTrans := struct{ testutils.MockTransport }{}
-	rssTrans.RT = func(req *http.Request) (*http.Response, error) {
+	rssTrans := testutils.NewRoundTripper(func(req *http.Request) (*http.Response, error) {
 		if req.URL.String() != feedURL {
 			return nil, errors.New("Unknown test URL")
 		}
@@ -50,7 +49,7 @@ func TestHTMLEntities(t *testing.T) {
 			StatusCode: 200,
 			Body:       ioutil.NopCloser(bytes.NewBufferString(rssFeedXML)),
 		}, nil
-	}
+	})
 	cachingClient = &http.Client{Transport: rssTrans}
 
 	// Create the RSS service
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index 77b14e7..3c1d695 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -99,8 +99,7 @@ func TestTravisCI(t *testing.T) {
 	urlToKey := make(map[string]string)
 	urlToKey["https://api.travis-ci.org/config"] = travisOrgPEMPublicKey
 	urlToKey["https://api.travis-ci.com/config"] = travisComPEMPublicKey
-	travisTransport := struct{ testutils.MockTransport }{}
-	travisTransport.RT = func(req *http.Request) (*http.Response, error) {
+	travisTransport := testutils.NewRoundTripper(func(req *http.Request) (*http.Response, error) {
 		if key := urlToKey[req.URL.String()]; key != "" {
 			escKey, _ := json.Marshal(key)
 			return &http.Response{
@@ -111,7 +110,7 @@ func TestTravisCI(t *testing.T) {
 			}, nil
 		}
 		return nil, fmt.Errorf("Unhandled URL %s", req.URL.String())
-	}
+	})
 	// clobber the http client that the service uses to talk to Travis
 	httpClient = &http.Client{Transport: travisTransport}
 
diff --git a/src/github.com/matrix-org/go-neb/testutils/testutils.go b/src/github.com/matrix-org/go-neb/testutils/testutils.go
index ba8508e..ee678fa 100644
--- a/src/github.com/matrix-org/go-neb/testutils/testutils.go
+++ b/src/github.com/matrix-org/go-neb/testutils/testutils.go
@@ -6,6 +6,12 @@ import (
 
 // MockTransport implements RoundTripper
 type MockTransport struct {
+	// RT is the RoundTrip function. Replace this function with your test function.
+	// For example:
+	//   t := MockTransport{}
+	//   t.RT = func(req *http.Request) (*http.Response, error) {
+	//       // assert req args, return res or error
+	//   }
 	RT func(*http.Request) (*http.Response, error)
 }
 
@@ -13,3 +19,10 @@ type MockTransport struct {
 func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	return t.RT(req)
 }
+
+// NewRoundTripper returns a new RoundTripper which will call the provided function.
+func NewRoundTripper(roundTrip func(*http.Request) (*http.Response, error)) http.RoundTripper {
+	rt := MockTransport{}
+	rt.RT = roundTrip
+	return rt
+}

From 4d918673c0ce7553e9d5e06904dc7d895ba0e24d Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 25 Nov 2016 14:29:01 +0000
Subject: [PATCH 34/51] Break apart 'issues' and 'pull_request' events

Now broken down into:
 - `labels` : Labelling/Unlabelling issues/PRs
 - `milestones` : Milestoning/Demilestoning issues/PRs
 - `assign` : Assigning/Unassigning issues/PRs

This is broken down in the guts of parsing the webhook event such that it
appears to be a unique `X-GitHub-Event` type.
---
 .../go-neb/services/github/github_webhook.go  |  9 ++--
 .../go-neb/services/github/webhook/webhook.go | 48 +++++++++++++------
 .../services/github/webhook/webhook_test.go   | 22 +++++----
 3 files changed, 52 insertions(+), 27 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
index ef7e196..9d4417b 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
@@ -36,7 +36,7 @@ const WebhookServiceType = "github-webhook"
 //           "!qmElAGdFYCHoCJuaNt:localhost": {
 //               Repos: {
 //                   "matrix-org/go-neb": {
-//                       Events: ["push", "issues", "pull_request"]
+//                       Events: ["push", "issues", "pull_request", "labels"]
 //                   }
 //               }
 //           }
@@ -57,10 +57,13 @@ type WebhookService struct {
 			// The webhook events to listen for. Currently supported:
 			//    push : When users push to this repository.
 			//    pull_request : When a pull request is made to this repository.
-			//    issues : When an issue is opened/closed.
+			//    issues : When an issue is opened/edited/closed/reopened.
 			//    issue_comment : When an issue or pull request is commented on.
 			//    pull_request_review_comment : When a line comment is made on a pull request.
-			// Full list: https://developer.github.com/webhooks/#events
+			//    labels : When any issue or pull request is labeled/unlabeled. Unique to Go-NEB.
+			//    milestones : When any issue or pull request is milestoned/demilestoned. Unique to Go-NEB.
+			//    assign : When any issue or pull request is assigned/unassigned. Unique to Go-NEB.
+			// Most of these events are directly from: https://developer.github.com/webhooks/#events
 			Events []string
 		}
 	}
diff --git a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
index 8d1b625..d610edb 100644
--- a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
@@ -61,14 +61,15 @@ func OnReceiveRequest(r *http.Request, secretToken string) (string, *github.Repo
 		return "", nil, nil, &errors.HTTPError{nil, "pong", 200}
 	}
 
-	htmlStr, repo, err := parseGithubEvent(eventType, content)
+	htmlStr, repo, refinedType, err := parseGithubEvent(eventType, content)
 	if err != nil {
 		log.WithError(err).Print("Failed to parse github event")
 		return "", nil, nil, &errors.HTTPError{nil, "Failed to parse github event", 500}
 	}
 
 	msg := matrix.GetHTMLMessage("m.notice", htmlStr)
-	return eventType, repo, &msg, nil
+
+	return refinedType, repo, &msg, nil
 }
 
 // checkMAC reports whether messageMAC is a valid HMAC tag for message.
@@ -80,24 +81,26 @@ func checkMAC(message, messageMAC, key []byte) bool {
 }
 
 // parseGithubEvent parses a github event type and JSON data and returns an explanatory
-// HTML string and the github repository this event affects, or an error.
-func parseGithubEvent(eventType string, data []byte) (string, *github.Repository, error) {
+// HTML string, the github repository and the refined event type, or an error.
+func parseGithubEvent(eventType string, data []byte) (string, *github.Repository, string, error) {
 	if eventType == "pull_request" {
 		var ev github.PullRequestEvent
 		if err := json.Unmarshal(data, &ev); err != nil {
-			return "", nil, err
+			return "", nil, eventType, err
 		}
-		return pullRequestHTMLMessage(ev), ev.Repo, nil
+		refinedEventType := refineEventType(eventType, ev.Action)
+		return pullRequestHTMLMessage(ev), ev.Repo, refinedEventType, nil
 	} else if eventType == "issues" {
 		var ev github.IssuesEvent
 		if err := json.Unmarshal(data, &ev); err != nil {
-			return "", nil, err
+			return "", nil, eventType, err
 		}
-		return issueHTMLMessage(ev), ev.Repo, nil
+		refinedEventType := refineEventType(eventType, ev.Action)
+		return issueHTMLMessage(ev), ev.Repo, refinedEventType, nil
 	} else if eventType == "push" {
 		var ev github.PushEvent
 		if err := json.Unmarshal(data, &ev); err != nil {
-			return "", nil, err
+			return "", nil, eventType, err
 		}
 
 		// The 'push' event repository format is subtly different from normal, so munge the bits we need.
@@ -109,21 +112,36 @@ func parseGithubEvent(eventType string, data []byte) (string, *github.Repository
 			Name:     ev.Repo.Name,
 			FullName: &fullName,
 		}
-		return pushHTMLMessage(ev), &repo, nil
+		return pushHTMLMessage(ev), &repo, eventType, nil
 	} else if eventType == "issue_comment" {
 		var ev github.IssueCommentEvent
 		if err := json.Unmarshal(data, &ev); err != nil {
-			return "", nil, err
+			return "", nil, eventType, err
 		}
-		return issueCommentHTMLMessage(ev), ev.Repo, nil
+		return issueCommentHTMLMessage(ev), ev.Repo, eventType, nil
 	} else if eventType == "pull_request_review_comment" {
 		var ev github.PullRequestReviewCommentEvent
 		if err := json.Unmarshal(data, &ev); err != nil {
-			return "", nil, err
+			return "", nil, eventType, err
 		}
-		return prReviewCommentHTMLMessage(ev), ev.Repo, nil
+		return prReviewCommentHTMLMessage(ev), ev.Repo, eventType, nil
+	}
+	return "", nil, eventType, fmt.Errorf("Unrecognized event type")
+}
+
+func refineEventType(eventType string, action *string) string {
+	if action == nil {
+		return eventType
+	}
+	a := *action
+	if a == "assigned" || a == "unassigned" {
+		return "assign"
+	} else if a == "milestoned" || a == "demilestoned" {
+		return "milestones"
+	} else if a == "labeled" || a == "unlabeled" {
+		return "labels"
 	}
-	return "", nil, fmt.Errorf("Unrecognized event type")
+	return eventType
 }
 
 func pullRequestHTMLMessage(p github.PullRequestEvent) string {
diff --git a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
index 3a4b810..4994de6 100644
--- a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
+++ b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
@@ -10,6 +10,7 @@ var ghtests = []struct {
 	jsonBody    string
 	outHTML     string
 	outFullRepo string
+	outType     string
 }{
 	{"issues",
 		`{
@@ -165,7 +166,7 @@ var ghtests = []struct {
 		  }
 		}`,
 		`[<u>DummyAccount/reponame</u>] DummyAccount closed <b>issue #15</b>: aaaaaa [closed] - https://github.com/DummyAccount/reponame/issues/15`,
-		"DummyAccount/reponame"},
+		"DummyAccount/reponame", "issues"},
 	// ==================================================================
 	{
 		"issue_comment",
@@ -350,7 +351,7 @@ var ghtests = []struct {
 		  }
 		}`,
 		"[<u>DummyAccount/arepo</u>] DummyAccount commented on DummyAccount's <b>issue #15</b>: aaaaaa - https://github.com/DummyAccount/arepo/issues/15",
-		"DummyAccount/arepo",
+		"DummyAccount/arepo", "issue_comment",
 	},
 	// ==================================================================
 	{
@@ -561,7 +562,7 @@ var ghtests = []struct {
 		  }
 		}`,
 		"[<u>matrix-org/sytest</u>] NegativeMjark pushed 2 commits to <b>develop</b>: https://github.com/matrix-org/sytest/commit/4a05c601f6b806110e63160cf7cf41b37787461f<br>NegativeMjark: Fix arguments to postgres connector to work with go<br>NegativeMjark: Add necessary info to the second postgres db",
-		"matrix-org/sytest",
+		"matrix-org/sytest", "push",
 	},
 	// ==================================================================
 	{
@@ -1032,7 +1033,7 @@ var ghtests = []struct {
 		  }
 		}`,
 		"[<u>matrix-org/matrix-react-sdk</u>] richvdh assigned <b>pull request #303</b>: Factor out common parts of room creation [open] to dbkr - https://github.com/matrix-org/matrix-react-sdk/pull/303",
-		"matrix-org/matrix-react-sdk",
+		"matrix-org/matrix-react-sdk", "assign",
 	},
 	// ==================================================================
 	{
@@ -1500,25 +1501,28 @@ var ghtests = []struct {
 		  }
 		}`,
 		"[<u>matrix-org/synapse</u>] erikjohnston made a line comment on negzi's <b>pull request #860</b> (assignee: None): Fix a bug caused by a change in auth_handler function - https://github.com/matrix-org/synapse/pull/860#discussion_r66413356",
-		"matrix-org/synapse",
+		"matrix-org/synapse", "pull_request_review_comment",
 	},
 }
 
 func TestParseGithubEvent(t *testing.T) {
 	for _, gh := range ghtests {
-		outHTML, outRepo, outErr := parseGithubEvent(gh.eventType, []byte(gh.jsonBody))
+		outHTML, outRepo, outType, outErr := parseGithubEvent(gh.eventType, []byte(gh.jsonBody))
 		if outErr != nil {
 			t.Fatal(outErr)
 		}
 		if strings.TrimSpace(outHTML) != strings.TrimSpace(gh.outHTML) {
-			t.Fatalf("ParseGithubEvent(%s) => HTML output does not match. Got:\n%s\n\nExpected:\n%s", gh.eventType,
+			t.Errorf("ParseGithubEvent(%s) => HTML output does not match. Got:\n%s\n\nExpected:\n%s", gh.eventType,
 				strings.TrimSpace(outHTML), strings.TrimSpace(gh.outHTML))
 		}
 		if outRepo == nil {
-			t.Fatalf("ParseGithubEvent(%s) => Repo is nil", gh.eventType)
+			t.Errorf("ParseGithubEvent(%s) => Repo is nil", gh.eventType)
 		}
 		if *outRepo.FullName != gh.outFullRepo {
-			t.Fatalf("ParseGithubEvent(%s) => Repo: Want %s got %s", gh.eventType, gh.outFullRepo, *outRepo.FullName)
+			t.Errorf("ParseGithubEvent(%s) => Repo: Want %s got %s", gh.eventType, gh.outFullRepo, *outRepo.FullName)
+		}
+		if outType != gh.outType {
+			t.Errorf("ParseGithubEvent(%s) => Event type: Want %s got %s", gh.eventType, gh.outType, outType)
 		}
 	}
 }

From aec58233d977a4dee91e01c5ff59da879ff37fb3 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 25 Nov 2016 14:38:45 +0000
Subject: [PATCH 35/51] s/assign/assignments/

---
 .../matrix-org/go-neb/services/github/github_webhook.go         | 2 +-
 .../matrix-org/go-neb/services/github/webhook/webhook.go        | 2 +-
 .../matrix-org/go-neb/services/github/webhook/webhook_test.go   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
index 9d4417b..c5b05b9 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
@@ -62,7 +62,7 @@ type WebhookService struct {
 			//    pull_request_review_comment : When a line comment is made on a pull request.
 			//    labels : When any issue or pull request is labeled/unlabeled. Unique to Go-NEB.
 			//    milestones : When any issue or pull request is milestoned/demilestoned. Unique to Go-NEB.
-			//    assign : When any issue or pull request is assigned/unassigned. Unique to Go-NEB.
+			//    assignments : When any issue or pull request is assigned/unassigned. Unique to Go-NEB.
 			// Most of these events are directly from: https://developer.github.com/webhooks/#events
 			Events []string
 		}
diff --git a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
index d610edb..87a8385 100644
--- a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
@@ -135,7 +135,7 @@ func refineEventType(eventType string, action *string) string {
 	}
 	a := *action
 	if a == "assigned" || a == "unassigned" {
-		return "assign"
+		return "assignments"
 	} else if a == "milestoned" || a == "demilestoned" {
 		return "milestones"
 	} else if a == "labeled" || a == "unlabeled" {
diff --git a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
index 4994de6..e8e6616 100644
--- a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
+++ b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
@@ -1033,7 +1033,7 @@ var ghtests = []struct {
 		  }
 		}`,
 		"[<u>matrix-org/matrix-react-sdk</u>] richvdh assigned <b>pull request #303</b>: Factor out common parts of room creation [open] to dbkr - https://github.com/matrix-org/matrix-react-sdk/pull/303",
-		"matrix-org/matrix-react-sdk", "assign",
+		"matrix-org/matrix-react-sdk", "assignments",
 	},
 	// ==================================================================
 	{

From 729d004ad075058eb846dc81c6469f0164a94036 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 25 Nov 2016 16:29:09 +0000
Subject: [PATCH 36/51] Use Giphy's translate API for better accuracy

---
 .../matrix-org/go-neb/services/giphy/giphy.go | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/giphy/giphy.go b/src/github.com/matrix-org/go-neb/services/giphy/giphy.go
index 13b64ed..a38f741 100644
--- a/src/github.com/matrix-org/go-neb/services/giphy/giphy.go
+++ b/src/github.com/matrix-org/go-neb/services/giphy/giphy.go
@@ -3,7 +3,7 @@ package giphy
 
 import (
 	"encoding/json"
-	"errors"
+	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -31,7 +31,7 @@ type result struct {
 }
 
 type giphySearch struct {
-	Data []result
+	Data result `json:"data"`
 }
 
 // Service contains the Config fields for the Giphy Service.
@@ -68,6 +68,9 @@ func (s *Service) cmdGiphy(client *matrix.Client, roomID, userID string, args []
 	if err != nil {
 		return nil, err
 	}
+	if gifResult.Images.Original.URL == "" {
+		return nil, fmt.Errorf("No results")
+	}
 	mxc, err := client.UploadLink(gifResult.Images.Original.URL)
 	if err != nil {
 		return nil, err
@@ -89,12 +92,12 @@ func (s *Service) cmdGiphy(client *matrix.Client, roomID, userID string, args []
 // searchGiphy returns info about a gif
 func (s *Service) searchGiphy(query string) (*result, error) {
 	log.Info("Searching giphy for ", query)
-	u, err := url.Parse("http://api.giphy.com/v1/gifs/search")
+	u, err := url.Parse("http://api.giphy.com/v1/gifs/translate")
 	if err != nil {
 		return nil, err
 	}
 	q := u.Query()
-	q.Set("q", query)
+	q.Set("s", query)
 	q.Set("api_key", s.APIKey)
 	u.RawQuery = q.Encode()
 	res, err := http.Get(u.String())
@@ -106,12 +109,11 @@ func (s *Service) searchGiphy(query string) (*result, error) {
 	}
 	var search giphySearch
 	if err := json.NewDecoder(res.Body).Decode(&search); err != nil {
-		return nil, err
-	}
-	if len(search.Data) == 0 {
-		return nil, errors.New("No results")
+		// Giphy returns a JSON object which has { data: [] } if there are 0 results.
+		// This fails to be deserialised by Go.
+		return nil, fmt.Errorf("No results")
 	}
-	return &search.Data[0], nil
+	return &search.Data, nil
 }
 
 func asInt(strInt string) uint {

From 66fb3253f74e5c1e9f0859ca6c4c66f7473909c4 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Mon, 28 Nov 2016 10:38:58 +0000
Subject: [PATCH 37/51] Treat RSS feeds with 0 items as errors

In the wild it looks like some RSS feeds will occasionally return 0 items
to requests *but not return an error*. This previously meant we would clobber
our knowledge of recent GUIDs with the empty set. This meant that the next
successful poll would resend the **entire** RSS feed.
---
 src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index 21c9b1d..0d21ab3 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -241,7 +241,9 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 	fp := gofeed.NewParser()
 	fp.Client = cachingClient
 	feed, err := fp.ParseURL(feedURL)
-	if err != nil {
+	// check for no items in addition to any returned errors as it appears some RSS feeds
+	// do not consistently return items.
+	if err != nil || len(feed.Items) == 0 {
 		f := s.Feeds[feedURL]
 		f.IsFailing = true
 		s.Feeds[feedURL] = f

From 677b5563768ca1bda1dd9123eca8a25913b03705 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Mon, 28 Nov 2016 10:43:48 +0000
Subject: [PATCH 38/51] Return an error so it is logged

---
 src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index 0d21ab3..5ddbe54 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -243,7 +243,11 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 	feed, err := fp.ParseURL(feedURL)
 	// check for no items in addition to any returned errors as it appears some RSS feeds
 	// do not consistently return items.
-	if err != nil || len(feed.Items) == 0 {
+	if err == nil && len(feed.Items) == 0 {
+		err = errors.New("feed has 0 items")
+	}
+
+	if err != nil {
 		f := s.Feeds[feedURL]
 		f.IsFailing = true
 		s.Feeds[feedURL] = f

From d128ed2626a29fc81ad0beb27cd49aba0487199f Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Mon, 28 Nov 2016 16:26:42 +0000
Subject: [PATCH 39/51] Add more logging for RSS feeds

---
 .../matrix-org/go-neb/polling/polling.go         |  8 ++++----
 .../matrix-org/go-neb/services/rssbot/rssbot.go  | 16 +++++++++++++++-
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/polling/polling.go b/src/github.com/matrix-org/go-neb/polling/polling.go
index 0a84a92..a131eb6 100644
--- a/src/github.com/matrix-org/go-neb/polling/polling.go
+++ b/src/github.com/matrix-org/go-neb/polling/polling.go
@@ -1,13 +1,14 @@
 package polling
 
 import (
+	"runtime/debug"
+	"sync"
+	"time"
+
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/clients"
 	"github.com/matrix-org/go-neb/database"
 	"github.com/matrix-org/go-neb/types"
-	"runtime/debug"
-	"sync"
-	"time"
 )
 
 // Remember when we first started polling on this service ID. Polling routines will
@@ -107,7 +108,6 @@ func pollLoop(service types.Service, ts int64) {
 			break
 		}
 		now := time.Now()
-		logger.Info("Sleeping for ", nextTime.Sub(now))
 		time.Sleep(nextTime.Sub(now))
 
 		if pollTimeChanged(service, ts) {
diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index 5ddbe54..b9c32d4 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -175,6 +175,11 @@ func (s *Service) OnPoll(cli *matrix.Client) time.Time {
 			continue
 		}
 		incrementMetrics(u, nil)
+		logger.WithFields(log.Fields{
+			"feed_url":   u,
+			"feed_items": len(feed.Items),
+			"new_items":  len(items),
+		}).Info("Sending new items")
 		// Loop backwards since [0] is the most recent and we want to send in chronological order
 		for i := len(items) - 1; i >= 0; i-- {
 			item := items[i]
@@ -290,8 +295,17 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 		guids = append(guids, itm.GUID)
 	}
 
-	// Update the service config to persist the new times
 	f := s.Feeds[feedURL]
+
+	if len(guids) != len(f.RecentGUIDs) {
+		log.WithFields(log.Fields{
+			"new_guids": guids,
+			"old_guids": f.RecentGUIDs,
+			"feed_url":  feedURL,
+		}).Warn("GUID length mismatch")
+	}
+
+	// Update the service config to persist the new times
 	f.NextPollTimestampSecs = nextPollTsSec
 	f.FeedUpdatedTimestampSecs = now
 	f.RecentGUIDs = guids

From 332b451d7725d7c40b35e0a5852739b9eacac87e Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 29 Nov 2016 13:45:55 +0000
Subject: [PATCH 40/51] Redo how recent GUIDs are calculated

This is an attempt to fix #133.

Previously, we just clobbered the recent GUIDs with the lastest response
every single time, assuming that Atom/RSS feeds would consistently return the
same items. This appears to not be the case. In the wild, the number of items
returned on a single request can vary (sometimes even being 1 or 2 when usually
it is 50!).

This patch alters *how many* and *which* GUIDs we keep between requests, in an
attempt to prevent sending old news for buggy RSS feeds.
---
 .../go-neb/services/rssbot/rssbot.go          | 97 ++++++++++++++-----
 1 file changed, 71 insertions(+), 26 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index b9c32d4..7844748 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -260,17 +260,7 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 	}
 
 	// Patch up the item list: make sure each item has a GUID.
-	for idx := 0; idx < len(feed.Items); idx++ {
-		itm := feed.Items[idx]
-		if itm.GUID == "" {
-			if itm.Link != "" {
-				itm.GUID = itm.Link
-			} else if itm.Title != "" {
-				itm.GUID = itm.Title
-			}
-			feed.Items[idx] = itm
-		}
-	}
+	ensureItemsHaveGUIDs(feed)
 
 	// Work out which items are new, if any (based on the last updated TS we have)
 	// If the TS is 0 then this is the first ever poll, so let's not send 10s of events
@@ -289,26 +279,33 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 	// TODO: Handle the 'sy' Syndication extension to control update interval.
 	// See http://www.feedforall.com/syndication.htm and http://web.resource.org/rss/1.0/modules/syndication/
 
-	// map items to guid strings
-	var guids []string
-	for _, itm := range feed.Items {
-		guids = append(guids, itm.GUID)
-	}
-
+	// Work out which GUIDs to remember. We don't want to remember every GUID ever as that leads to completely
+	// unbounded growth of data.
 	f := s.Feeds[feedURL]
+	// Some RSS feeds can return a very small number of items then bounce
+	// back to their "normal" size, so we cannot just clobber the recent GUID list per request or else we'll
+	// forget what we sent and resend it. Instead, we'll keep 2x the max number of items that we've ever
+	// seen from this feed, up to a max of 1000.
+	maxGuids := 2 * len(feed.Items)
+	if len(f.RecentGUIDs) > maxGuids {
+		maxGuids = len(f.RecentGUIDs) // already 2x'd.
+	}
+	if maxGuids > 1000 {
+		maxGuids = 1000
+	}
 
-	if len(guids) != len(f.RecentGUIDs) {
-		log.WithFields(log.Fields{
-			"new_guids": guids,
-			"old_guids": f.RecentGUIDs,
-			"feed_url":  feedURL,
-		}).Warn("GUID length mismatch")
+	lastSet := uniqueStrings(f.RecentGUIDs) // e.g. [4,5,6]
+	thisSet := uniqueGuids(feed.Items)      // e.g. [1,2,3]
+	guids := append(thisSet, lastSet...)    // e.g. [1,2,3,4,5,6]
+	if len(guids) > maxGuids {
+		// Critically this favours the NEWEST elements, which are the ones we're most likely to see again.
+		guids = guids[0:maxGuids]
 	}
 
 	// Update the service config to persist the new times
 	f.NextPollTimestampSecs = nextPollTsSec
 	f.FeedUpdatedTimestampSecs = now
-	f.RecentGUIDs = guids
+	f.RecentGUIDs = uniqueStrings(guids)
 	f.IsFailing = false
 	s.Feeds[feedURL] = f
 
@@ -347,8 +344,12 @@ func (s *Service) newItems(feedURL string, allItems []*gofeed.Item) (items []gof
 }
 
 func (s *Service) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error {
-	logger := log.WithField("feed_url", feedURL).WithField("title", item.Title)
-	logger.Info("New feed item")
+	logger := log.WithFields(log.Fields{
+		"feed_url": feedURL,
+		"title":    item.Title,
+		"guid":     item.GUID,
+	})
+	logger.Info("Sending new feed item")
 	for _, roomID := range s.Feeds[feedURL].Rooms {
 		if _, err := cli.SendMessageEvent(roomID, "m.room.message", itemToHTML(feed, item)); err != nil {
 			logger.WithError(err).WithField("room_id", roomID).Error("Failed to send to room")
@@ -365,6 +366,50 @@ func itemToHTML(feed *gofeed.Feed, item gofeed.Item) matrix.HTMLMessage {
 	))
 }
 
+func ensureItemsHaveGUIDs(feed *gofeed.Feed) {
+	for idx := 0; idx < len(feed.Items); idx++ {
+		itm := feed.Items[idx]
+		if itm.GUID == "" {
+			if itm.Link != "" {
+				itm.GUID = itm.Link
+			} else if itm.Title != "" {
+				itm.GUID = itm.Title
+			}
+			feed.Items[idx] = itm
+		}
+	}
+}
+
+// uniqueStrings returns a new slice of strings with duplicate elements removed.
+// Order is otherwise preserved.
+func uniqueStrings(a []string) []string {
+	ret := []string{}
+	seen := make(map[string]bool)
+	for _, str := range a {
+		if seen[str] {
+			continue
+		}
+		seen[str] = true
+		ret = append(ret, str)
+	}
+	return ret
+}
+
+// uniqueGuids returns a new slice of GUID strings with duplicate elements removed.
+// Order is otherwise preserved.
+func uniqueGuids(a []*gofeed.Item) []string {
+	ret := []string{}
+	seen := make(map[string]bool)
+	for _, item := range a {
+		if seen[item.GUID] {
+			continue
+		}
+		seen[item.GUID] = true
+		ret = append(ret, item.GUID)
+	}
+	return ret
+}
+
 type userAgentRoundTripper struct {
 	Transport http.RoundTripper
 }

From 17228713450829f80204268ffa3b62036f3c0a6c Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 29 Nov 2016 14:17:19 +0000
Subject: [PATCH 41/51] Don't use gofeed.ParseURL

It leaks the response because it doesn't close the `resp.Body` on non-2xx.
---
 .../go-neb/services/rssbot/rssbot.go          | 27 ++++++++++++++-----
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index b9c32d4..531c687 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -88,9 +88,7 @@ func (s *Service) Register(oldService types.Service, client *matrix.Client) erro
 	}
 	// Make sure we can parse the feed
 	for feedURL, feedInfo := range s.Feeds {
-		fp := gofeed.NewParser()
-		fp.Client = cachingClient
-		if _, err := fp.ParseURL(feedURL); err != nil {
+		if _, err := readFeed(feedURL); err != nil {
 			return fmt.Errorf("Failed to read URL %s: %s", feedURL, err.Error())
 		}
 		if len(feedInfo.Rooms) == 0 {
@@ -243,9 +241,7 @@ func (s *Service) nextTimestamp() time.Time {
 func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error) {
 	log.WithField("feed_url", feedURL).Info("Querying feed")
 	var items []gofeed.Item
-	fp := gofeed.NewParser()
-	fp.Client = cachingClient
-	feed, err := fp.ParseURL(feedURL)
+	feed, err := readFeed(feedURL)
 	// check for no items in addition to any returned errors as it appears some RSS feeds
 	// do not consistently return items.
 	if err == nil && len(feed.Items) == 0 {
@@ -374,6 +370,25 @@ func (rt userAgentRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
 	return rt.Transport.RoundTrip(req)
 }
 
+func readFeed(feedURL string) (*gofeed.Feed, error) {
+	fp := gofeed.NewParser()
+	resp, err := cachingClient.Get(feedURL)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, gofeed.HTTPError{
+			StatusCode: resp.StatusCode,
+			Status:     resp.Status,
+		}
+	}
+	return fp.Parse(resp.Body)
+}
+
 func init() {
 	lruCache := lrucache.New(1024*1024*20, 0) // 20 MB cache, no max-age
 	cachingClient = &http.Client{

From c4e98238d8b72183a3d8f8b54be9cb811660f527 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 29 Nov 2016 14:41:14 +0000
Subject: [PATCH 42/51] uniq prior to length checks for accuracy

---
 src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index 7844748..ae4fb3e 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -297,6 +297,7 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 	lastSet := uniqueStrings(f.RecentGUIDs) // e.g. [4,5,6]
 	thisSet := uniqueGuids(feed.Items)      // e.g. [1,2,3]
 	guids := append(thisSet, lastSet...)    // e.g. [1,2,3,4,5,6]
+	guids = uniqueStrings(guids)
 	if len(guids) > maxGuids {
 		// Critically this favours the NEWEST elements, which are the ones we're most likely to see again.
 		guids = guids[0:maxGuids]
@@ -305,7 +306,7 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error)
 	// Update the service config to persist the new times
 	f.NextPollTimestampSecs = nextPollTsSec
 	f.FeedUpdatedTimestampSecs = now
-	f.RecentGUIDs = uniqueStrings(guids)
+	f.RecentGUIDs = guids
 	f.IsFailing = false
 	s.Feeds[feedURL] = f
 

From 7522ddf1a4e93b58b01db19258a6e88d35163c72 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 29 Nov 2016 14:44:30 +0000
Subject: [PATCH 43/51] Explain why no ParseURL

---
 src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index 531c687..7dc3312 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -371,6 +371,7 @@ func (rt userAgentRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
 }
 
 func readFeed(feedURL string) (*gofeed.Feed, error) {
+	// Don't use fp.ParseURL because it leaks on non-2xx responses as of 2016/11/29 (cac19c6c27)
 	fp := gofeed.NewParser()
 	resp, err := cachingClient.Get(feedURL)
 	if resp != nil {

From b98e878227987d5469669c274e94f48842a6844e Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 1 Dec 2016 14:34:17 +0000
Subject: [PATCH 44/51] Use gomatrix

---
 .../matrix-org/go-neb/api/handlers/service.go |   6 +-
 .../matrix-org/go-neb/clients/clients.go      |  65 +--
 .../matrix-org/go-neb/clients/clients_test.go |  16 +-
 .../matrix-org/go-neb/matrix/matrix.go        | 460 ++----------------
 .../matrix-org/go-neb/matrix/responses.go     |  40 --
 .../matrix-org/go-neb/matrix/worker.go        |  81 ---
 .../matrix-org/go-neb/services/echo/echo.go   |   6 +-
 .../matrix-org/go-neb/services/giphy/giphy.go |  14 +-
 .../go-neb/services/github/github.go          |  17 +-
 .../go-neb/services/github/github_webhook.go  |   8 +-
 .../go-neb/services/github/webhook/webhook.go |   6 +-
 .../matrix-org/go-neb/services/guggy/guggy.go |  16 +-
 .../go-neb/services/guggy/guggy_test.go       |  14 +-
 .../matrix-org/go-neb/services/jira/jira.go   |  15 +-
 .../go-neb/services/rssbot/rssbot.go          |  14 +-
 .../go-neb/services/rssbot/rssbot_test.go     |   9 +-
 .../go-neb/services/travisci/travisci.go      |  10 +-
 .../go-neb/services/travisci/travisci_test.go |  15 +-
 .../matrix-org/go-neb/types/service.go        |  20 +-
 vendor/manifest                               |   6 +
 .../github.com/matrix-org/gomatrix/LICENSE    | 201 ++++++++
 .../github.com/matrix-org/gomatrix/README.md  |   4 +
 .../github.com/matrix-org/gomatrix/client.go  | 381 +++++++++++++++
 .../matrix-org/gomatrix/client_test.go        |  28 ++
 .../github.com/matrix-org/gomatrix/events.go  |  79 +--
 .../matrix-org/gomatrix/hooks/install.sh      |   5 +
 .../matrix-org/gomatrix/hooks/pre-commit      |   9 +
 .../matrix-org/gomatrix/responses.go          |  61 +++
 .../github.com/matrix-org/gomatrix/room.go    |  50 ++
 .../github.com/matrix-org/gomatrix/store.go   |  64 +++
 .../github.com/matrix-org/gomatrix/sync.go    | 154 ++++++
 31 files changed, 1123 insertions(+), 751 deletions(-)
 delete mode 100644 src/github.com/matrix-org/go-neb/matrix/responses.go
 delete mode 100644 src/github.com/matrix-org/go-neb/matrix/worker.go
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/LICENSE
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/README.md
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/client.go
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/client_test.go
 rename src/github.com/matrix-org/go-neb/matrix/types.go => vendor/src/github.com/matrix-org/gomatrix/events.go (55%)
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/hooks/install.sh
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/hooks/pre-commit
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/responses.go
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/room.go
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/store.go
 create mode 100644 vendor/src/github.com/matrix-org/gomatrix/sync.go

diff --git a/src/github.com/matrix-org/go-neb/api/handlers/service.go b/src/github.com/matrix-org/go-neb/api/handlers/service.go
index f33bd97..6acb777 100644
--- a/src/github.com/matrix-org/go-neb/api/handlers/service.go
+++ b/src/github.com/matrix-org/go-neb/api/handlers/service.go
@@ -16,6 +16,7 @@ import (
 	"github.com/matrix-org/go-neb/metrics"
 	"github.com/matrix-org/go-neb/polling"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ConfigureService represents an HTTP handler which can process /admin/configureService requests.
@@ -211,13 +212,14 @@ func (h *GetService) OnIncomingRequest(req *http.Request) (interface{}, *errors.
 	}{srv.ServiceID(), srv.ServiceType(), srv}, nil
 }
 
-func checkClientForService(service types.Service, client *matrix.Client) error {
+func checkClientForService(service types.Service, client *gomatrix.Client) error {
 	// If there are any commands or expansions for this Service then the service user ID
 	// MUST be a syncing client or else the Service will never get the incoming command/expansion!
 	cmds := service.Commands(client)
 	expans := service.Expansions(client)
 	if len(cmds) > 0 || len(expans) > 0 {
-		if !client.ClientConfig.Sync {
+		nebStore := client.Store.(*matrix.NEBStore)
+		if !nebStore.ClientConfig.Sync {
 			return fmt.Errorf(
 				"Service type '%s' requires a syncing client", service.ServiceType(),
 			)
diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 4352766..cce5add 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -2,7 +2,6 @@ package clients
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 	"sync"
 
@@ -12,34 +11,10 @@ import (
 	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/metrics"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 	shellwords "github.com/mattn/go-shellwords"
 )
 
-type nextBatchStore struct {
-	db database.Storer
-}
-
-func (s nextBatchStore) Save(userID, nextBatch string) {
-	if err := s.db.UpdateNextBatch(userID, nextBatch); err != nil {
-		log.WithFields(log.Fields{
-			log.ErrorKey: err,
-			"user_id":    userID,
-			"next_batch": nextBatch,
-		}).Error("Failed to persist next_batch token")
-	}
-}
-func (s nextBatchStore) Load(userID string) string {
-	token, err := s.db.LoadNextBatch(userID)
-	if err != nil {
-		log.WithFields(log.Fields{
-			log.ErrorKey: err,
-			"user_id":    userID,
-		}).Error("Failed to load next_batch token")
-		return ""
-	}
-	return token
-}
-
 // A Clients is a collection of clients used for bot services.
 type Clients struct {
 	db         database.Storer
@@ -60,7 +35,7 @@ func New(db database.Storer, cli *http.Client) *Clients {
 }
 
 // Client gets a client for the userID
-func (c *Clients) Client(userID string) (*matrix.Client, error) {
+func (c *Clients) Client(userID string) (*gomatrix.Client, error) {
 	entry := c.getClient(userID)
 	if entry.client != nil {
 		return entry.client, nil
@@ -93,7 +68,7 @@ func (c *Clients) Start() error {
 
 type clientEntry struct {
 	config api.ClientConfig
-	client *matrix.Client
+	client *gomatrix.Client
 }
 
 func (c *Clients) getClient(userID string) clientEntry {
@@ -172,7 +147,7 @@ func (c *Clients) updateClientInDB(newConfig api.ClientConfig) (new clientEntry,
 	return
 }
 
-func (c *Clients) onMessageEvent(client *matrix.Client, event *matrix.Event) {
+func (c *Clients) onMessageEvent(client *gomatrix.Client, event *gomatrix.Event) {
 	services, err := c.db.LoadServicesForUser(client.UserID)
 	if err != nil {
 		log.WithFields(log.Fields{
@@ -232,7 +207,7 @@ func (c *Clients) onMessageEvent(client *matrix.Client, event *matrix.Event) {
 // the matching command with the longest path. Returns the JSON encodable
 // content of a single matrix message event to use as a response or nil if no
 // response is appropriate.
-func runCommandForService(cmds []types.Command, event *matrix.Event, arguments []string) interface{} {
+func runCommandForService(cmds []types.Command, event *gomatrix.Event, arguments []string) interface{} {
 	var bestMatch *types.Command
 	for _, command := range cmds {
 		matches := command.Matches(arguments)
@@ -264,7 +239,7 @@ func runCommandForService(cmds []types.Command, event *matrix.Event, arguments [
 			}).Warn("Command returned both error and content.")
 		}
 		metrics.IncrementCommand(bestMatch.Path[0], metrics.StatusFailure)
-		content = matrix.TextMessage{"m.notice", err.Error()}
+		content = gomatrix.TextMessage{"m.notice", err.Error()}
 	} else {
 		metrics.IncrementCommand(bestMatch.Path[0], metrics.StatusSuccess)
 	}
@@ -273,7 +248,7 @@ func runCommandForService(cmds []types.Command, event *matrix.Event, arguments [
 }
 
 // run the expansions for a matrix event.
-func runExpansionsForService(expans []types.Expansion, event *matrix.Event, body string) []interface{} {
+func runExpansionsForService(expans []types.Expansion, event *gomatrix.Event, body string) []interface{} {
 	var responses []interface{}
 
 	for _, expansion := range expans {
@@ -294,7 +269,7 @@ func runExpansionsForService(expans []types.Expansion, event *matrix.Event, body
 	return responses
 }
 
-func (c *Clients) onBotOptionsEvent(client *matrix.Client, event *matrix.Event) {
+func (c *Clients) onBotOptionsEvent(client *gomatrix.Client, event *gomatrix.Event) {
 	// see if these options are for us. The state key is the user ID with a leading _
 	// to get around restrictions in the HS about having user IDs as state keys.
 	targetUserID := strings.TrimPrefix(event.StateKey, "_")
@@ -318,7 +293,7 @@ func (c *Clients) onBotOptionsEvent(client *matrix.Client, event *matrix.Event)
 	}
 }
 
-func (c *Clients) onRoomMemberEvent(client *matrix.Client, event *matrix.Event) {
+func (c *Clients) onRoomMemberEvent(client *gomatrix.Client, event *gomatrix.Event) {
 	if event.StateKey != client.UserID {
 		return // not our member event
 	}
@@ -343,29 +318,33 @@ func (c *Clients) onRoomMemberEvent(client *matrix.Client, event *matrix.Event)
 	}
 }
 
-func (c *Clients) newClient(config api.ClientConfig) (*matrix.Client, error) {
-	homeserverURL, err := url.Parse(config.HomeserverURL)
+func (c *Clients) newClient(config api.ClientConfig) (*gomatrix.Client, error) {
+	client, err := gomatrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken)
 	if err != nil {
 		return nil, err
 	}
-
-	client := matrix.NewClient(c.httpClient, homeserverURL, config.AccessToken, config.UserID)
-	client.NextBatchStorer = nextBatchStore{c.db}
-	client.ClientConfig = config
+	client.Client = c.httpClient
+	syncer := client.Syncer.(*gomatrix.DefaultSyncer)
+	client.Store = &matrix.NEBStore{
+		InMemoryStore: *gomatrix.NewInMemoryStore(),
+		Database:      c.db,
+		ClientConfig:  config,
+	}
+	syncer.Store = client.Store
 
 	// TODO: Check that the access token is valid for the userID by peforming
 	// a request against the server.
 
-	client.Worker.OnEventType("m.room.message", func(event *matrix.Event) {
+	syncer.OnEventType("m.room.message", func(event *gomatrix.Event) {
 		c.onMessageEvent(client, event)
 	})
 
-	client.Worker.OnEventType("m.room.bot.options", func(event *matrix.Event) {
+	syncer.OnEventType("m.room.bot.options", func(event *gomatrix.Event) {
 		c.onBotOptionsEvent(client, event)
 	})
 
 	if config.AutoJoinRooms {
-		client.Worker.OnEventType("m.room.member", func(event *matrix.Event) {
+		syncer.OnEventType("m.room.member", func(event *gomatrix.Event) {
 			c.onRoomMemberEvent(client, event)
 		})
 	}
diff --git a/src/github.com/matrix-org/go-neb/clients/clients_test.go b/src/github.com/matrix-org/go-neb/clients/clients_test.go
index bc6f70e..2fcc613 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients_test.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients_test.go
@@ -2,13 +2,13 @@ package clients
 
 import (
 	"fmt"
-	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
-	"github.com/matrix-org/go-neb/types"
 	"net/http"
-	"net/url"
 	"reflect"
 	"testing"
+
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 var commandParseTests = []struct {
@@ -28,7 +28,7 @@ type MockService struct {
 	commands []types.Command
 }
 
-func (s *MockService) Commands(cli *matrix.Client) []types.Command {
+func (s *MockService) Commands(cli *gomatrix.Client) []types.Command {
 	return s.commands
 }
 
@@ -72,12 +72,12 @@ func TestCommandParsing(t *testing.T) {
 		Transport: trans,
 	}
 	clients := New(&store, cli)
-	hsURL, _ := url.Parse("https://someplace.somewhere")
-	mxCli := matrix.NewClient(cli, hsURL, "token", "@service:user")
+	mxCli, _ := gomatrix.NewClient("https://someplace.somewhere", "@service:user", "token")
+	mxCli.Client = cli
 
 	for _, input := range commandParseTests {
 		executedCmdArgs = []string{}
-		event := matrix.Event{
+		event := gomatrix.Event{
 			Type:   "m.room.message",
 			Sender: "@someone:somewhere",
 			RoomID: "!foo:bar",
diff --git a/src/github.com/matrix-org/go-neb/matrix/matrix.go b/src/github.com/matrix-org/go-neb/matrix/matrix.go
index e03c9eb..d863949 100644
--- a/src/github.com/matrix-org/go-neb/matrix/matrix.go
+++ b/src/github.com/matrix-org/go-neb/matrix/matrix.go
@@ -1,443 +1,69 @@
-// Package matrix provides an HTTP client that can interact with a Homeserver via r0 APIs (/sync).
-//
-// It is NOT safe to access the field (or any sub-fields of) 'Rooms' concurrently. In essence, this
-// structure MUST be treated as read-only. The matrix client will update this structure as new events
-// arrive from the homeserver.
-//
-// Internally, the client has 1 goroutine for polling the server, and 1 goroutine for processing data
-// returned. The polling goroutine communicates to the processing goroutine by a buffered channel
-// which feedback loops if processing takes a while as it will delay more data from being pulled down
-// if the buffer gets full. Modification of the 'Rooms' field of the client is done EXCLUSIVELY on the
-// processing goroutine.
 package matrix
 
 import (
-	"bytes"
 	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"path"
-	"strconv"
-	"sync"
-	"time"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/api"
-	"github.com/matrix-org/go-neb/errors"
-)
-
-var (
-	filterJSON = json.RawMessage(`{"room":{"timeline":{"limit":50}}}`)
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/gomatrix"
 )
 
-// NextBatchStorer controls loading/saving of next_batch tokens for users
-type NextBatchStorer interface {
-	// Save a next_batch token for a given user. Best effort.
-	Save(userID, nextBatch string)
-	// Load a next_batch token for a given user. Return an empty string if no token exists.
-	Load(userID string) string
-}
-
-// noopNextBatchStore does not load or save next_batch tokens.
-type noopNextBatchStore struct{}
-
-func (s noopNextBatchStore) Save(userID, nextBatch string) {}
-func (s noopNextBatchStore) Load(userID string) string     { return "" }
-
-// Client represents a Matrix client.
-type Client struct {
-	HomeserverURL   *url.URL
-	Prefix          string
-	UserID          string
-	AccessToken     string
-	Rooms           map[string]*Room
-	Worker          *Worker
-	syncingMutex    sync.Mutex
-	syncingID       uint32 // Identifies the current Sync. Only one Sync can be active at any given time.
-	httpClient      *http.Client
-	filterID        string
-	NextBatchStorer NextBatchStorer
-	ClientConfig    api.ClientConfig
-}
-
-func (cli *Client) buildURL(urlPath ...string) string {
-	ps := []string{cli.Prefix}
-	for _, p := range urlPath {
-		ps = append(ps, p)
-	}
-	return cli.buildBaseURL(ps...)
-}
-
-func (cli *Client) buildBaseURL(urlPath ...string) string {
-	// copy the URL. Purposefully ignore error as the input is from a valid URL already
-	hsURL, _ := url.Parse(cli.HomeserverURL.String())
-	parts := []string{hsURL.Path}
-	parts = append(parts, urlPath...)
-	hsURL.Path = path.Join(parts...)
-	query := hsURL.Query()
-	query.Set("access_token", cli.AccessToken)
-	hsURL.RawQuery = query.Encode()
-	return hsURL.String()
-}
-
-func (cli *Client) buildURLWithQuery(urlPath []string, urlQuery map[string]string) string {
-	u, _ := url.Parse(cli.buildURL(urlPath...))
-	q := u.Query()
-	for k, v := range urlQuery {
-		q.Set(k, v)
-	}
-	u.RawQuery = q.Encode()
-	return u.String()
-}
-
-// JoinRoom joins the client to a room ID or alias. If serverName is specified, this will be added as a query param
-// to instruct the homeserver to join via that server. If invitingUserID is specified, the inviting user ID will be
-// inserted into the content of the join request. Returns a room ID.
-func (cli *Client) JoinRoom(roomIDorAlias, serverName, invitingUserID string) (string, error) {
-	var urlPath string
-	if serverName != "" {
-		urlPath = cli.buildURLWithQuery([]string{"join", roomIDorAlias}, map[string]string{
-			"server_name": serverName,
-		})
-	} else {
-		urlPath = cli.buildURL("join", roomIDorAlias)
-	}
-
-	content := struct {
-		Inviter string `json:"inviter,omitempty"`
-	}{}
-	content.Inviter = invitingUserID
-
-	resBytes, err := cli.sendJSON("POST", urlPath, content)
-	if err != nil {
-		return "", err
-	}
-	var joinRoomResponse joinRoomHTTPResponse
-	if err = json.Unmarshal(resBytes, &joinRoomResponse); err != nil {
-		return "", err
-	}
-	return joinRoomResponse.RoomID, nil
-}
-
-// SetDisplayName sets the user's profile display name
-func (cli *Client) SetDisplayName(displayName string) error {
-	urlPath := cli.buildURL("profile", cli.UserID, "displayname")
-	s := struct {
-		DisplayName string `json:"displayname"`
-	}{displayName}
-	_, err := cli.sendJSON("PUT", urlPath, &s)
-	return err
+// NEBStore implements the gomatrix.Storer interface.
+//
+// It persists the next batch token in the database, and includes a ClientConfig for the client.
+type NEBStore struct {
+	gomatrix.InMemoryStore
+	Database     database.Storer
+	ClientConfig api.ClientConfig
 }
 
-// SendMessageEvent sends a message event into a room, returning the event_id on success.
-// contentJSON should be a pointer to something that can be encoded as JSON using json.Marshal.
-func (cli *Client) SendMessageEvent(roomID string, eventType string, contentJSON interface{}) (string, error) {
-	txnID := "go" + strconv.FormatInt(time.Now().UnixNano(), 10)
-	urlPath := cli.buildURL("rooms", roomID, "send", eventType, txnID)
-	resBytes, err := cli.sendJSON("PUT", urlPath, contentJSON)
-	if err != nil {
-		return "", err
-	}
-	var sendEventResponse sendEventHTTPResponse
-	if err = json.Unmarshal(resBytes, &sendEventResponse); err != nil {
-		return "", err
+// SaveNextBatch saves to the database.
+func (s *NEBStore) SaveNextBatch(userID, nextBatch string) {
+	if err := s.Database.UpdateNextBatch(userID, nextBatch); err != nil {
+		log.WithFields(log.Fields{
+			log.ErrorKey: err,
+			"user_id":    userID,
+			"next_batch": nextBatch,
+		}).Error("Failed to persist next_batch token")
 	}
-	return sendEventResponse.EventID, nil
 }
 
-// SendText sends an m.room.message event into the given room with a msgtype of m.text
-func (cli *Client) SendText(roomID, text string) (string, error) {
-	return cli.SendMessageEvent(roomID, "m.room.message",
-		TextMessage{"m.text", text})
-}
-
-// UploadLink uploads an HTTP URL and then returns an MXC URI.
-func (cli *Client) UploadLink(link string) (string, error) {
-	res, err := cli.httpClient.Get(link)
-	if res != nil {
-		defer res.Body.Close()
-	}
+// LoadNextBatch loads from the database.
+func (s *NEBStore) LoadNextBatch(userID string) string {
+	token, err := s.Database.LoadNextBatch(userID)
 	if err != nil {
-		return "", err
+		log.WithFields(log.Fields{
+			log.ErrorKey: err,
+			"user_id":    userID,
+		}).Error("Failed to load next_batch token")
+		return ""
 	}
-	return cli.UploadToContentRepo(res.Body, res.Header.Get("Content-Type"), res.ContentLength)
+	return token
 }
 
-// UploadToContentRepo uploads the given bytes to the content repository and returns an MXC URI.
-func (cli *Client) UploadToContentRepo(content io.Reader, contentType string, contentLength int64) (string, error) {
-	req, err := http.NewRequest("POST", cli.buildBaseURL("_matrix/media/r0/upload"), content)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("Content-Type", contentType)
-	req.ContentLength = contentLength
-	log.WithFields(log.Fields{
-		"content_type":   contentType,
-		"content_length": contentLength,
-	}).Print("Uploading to content repo")
-	res, err := cli.httpClient.Do(req)
-	if res != nil {
-		defer res.Body.Close()
-	}
-	if err != nil {
-		return "", err
-	}
-	if res.StatusCode != 200 {
-		return "", fmt.Errorf("Upload request returned HTTP %d", res.StatusCode)
-	}
-	m := struct {
-		ContentURI string `json:"content_uri"`
-	}{}
-	if err := json.NewDecoder(res.Body).Decode(&m); err != nil {
-		return "", err
-	}
-	return m.ContentURI, nil
+// StarterLinkMessage represents a message with a starter_link custom data.
+type StarterLinkMessage struct {
+	Body string
+	Link string
 }
 
-// Sync starts syncing with the provided Homeserver. This function will be invoked continually.
-// If Sync is called twice then the first sync will be stopped.
-func (cli *Client) Sync() {
-	// Mark the client as syncing.
-	// We will keep syncing until the syncing state changes. Either because
-	// Sync is called or StopSync is called.
-	syncingID := cli.incrementSyncingID()
-	logger := log.WithFields(log.Fields{
-		"syncing": syncingID,
-		"user_id": cli.UserID,
-	})
-
-	// TODO: Store the filter ID in the database
-	filterID, err := cli.createFilter()
-	if err != nil {
-		logger.WithError(err).Fatal("Failed to create filter")
-		// TODO: Maybe do some sort of error handling here?
-	}
-	cli.filterID = filterID
-	logger.WithField("filter", filterID).Print("Got filter ID")
-	nextToken := cli.NextBatchStorer.Load(cli.UserID)
-
-	logger.WithField("next_batch", nextToken).Print("Starting sync")
-
-	channel := make(chan syncHTTPResponse, 5)
-
-	go func() {
-		for response := range channel {
-			cli.Worker.onSyncHTTPResponse(response)
-		}
-	}()
-	defer close(channel)
-
-	for {
-		// Do a /sync
-		syncBytes, err := cli.doSync(30000, nextToken)
-		if err != nil {
-			logger.WithError(err).Warn("doSync failed")
-			time.Sleep(5 * time.Second)
-			continue
-		}
-
-		// Decode sync response into syncHTTPResponse
-		var syncResponse syncHTTPResponse
-		if err = json.Unmarshal(syncBytes, &syncResponse); err != nil {
-			logger.WithError(err).Warn("Failed to decode sync data")
-			time.Sleep(5 * time.Second)
-			continue
-		}
+// MarshalJSON converts this message into actual event content JSON.
+func (m StarterLinkMessage) MarshalJSON() ([]byte, error) {
+	var data map[string]string
 
-		//  Check that the syncing state hasn't changed
-		// Either because we've stopped syncing or another sync has been started.
-		// We discard the response from our sync.
-		if cli.getSyncingID() != syncingID {
-			logger.Print("Stopping sync")
-			return
-		}
-
-		processResponse := cli.shouldProcessResponse(nextToken, &syncResponse)
-		nextToken = syncResponse.NextBatch
-
-		// Save the token now *before* passing it through to the worker. This means it's possible
-		// to not process some events, but it means that we won't get constantly stuck processing
-		// a malformed/buggy event which keeps making us panic.
-		cli.NextBatchStorer.Save(cli.UserID, nextToken)
-
-		if processResponse {
-			// Update client state
-			channel <- syncResponse
+	if m.Link != "" {
+		data = map[string]string{
+			"org.matrix.neb.starter_link": m.Link,
 		}
 	}
-}
 
-// shouldProcessResponse returns true if the response should be processed. May modify the response to remove
-// stuff that shouldn't be processed.
-func (cli *Client) shouldProcessResponse(tokenOnSync string, syncResponse *syncHTTPResponse) bool {
-	if tokenOnSync == "" {
-		return false
-	}
-	// This is a horrible hack because /sync will return the most recent messages for a room
-	// as soon as you /join it. We do NOT want to process those events in that particular room
-	// because they may have already been processed (if you toggle the bot in/out of the room).
-	//
-	// Work around this by inspecting each room's timeline and seeing if an m.room.member event for us
-	// exists and is "join" and then discard processing that room entirely if so.
-	// TODO: We probably want to process the !commands from after the last join event in the timeline.
-	for roomID, roomData := range syncResponse.Rooms.Join {
-		for i := len(roomData.Timeline.Events) - 1; i >= 0; i-- {
-			e := roomData.Timeline.Events[i]
-			if e.Type == "m.room.member" && e.StateKey == cli.UserID {
-				m := e.Content["membership"]
-				mship, ok := m.(string)
-				if !ok {
-					continue
-				}
-				if mship == "join" {
-					log.WithFields(log.Fields{
-						"room_id":     roomID,
-						"user_id":     cli.UserID,
-						"start_token": tokenOnSync,
-					}).Info("Discarding /sync events in room: just joined it.")
-					_, ok := syncResponse.Rooms.Join[roomID]
-					if !ok {
-						panic("room " + roomID + " does not exist in Join?!")
-					}
-					delete(syncResponse.Rooms.Join, roomID)   // don't re-process !commands
-					delete(syncResponse.Rooms.Invite, roomID) // don't re-process invites
-					break
-				}
-			}
-		}
-	}
-	return true
-}
-
-func (cli *Client) incrementSyncingID() uint32 {
-	cli.syncingMutex.Lock()
-	defer cli.syncingMutex.Unlock()
-	cli.syncingID++
-	return cli.syncingID
-}
-
-func (cli *Client) getSyncingID() uint32 {
-	cli.syncingMutex.Lock()
-	defer cli.syncingMutex.Unlock()
-	return cli.syncingID
-}
-
-// StopSync stops the ongoing sync started by Sync.
-func (cli *Client) StopSync() {
-	// Advance the syncing state so that any running Syncs will terminate.
-	cli.incrementSyncingID()
-}
-
-// This should only be called by the worker goroutine
-func (cli *Client) getOrCreateRoom(roomID string) *Room {
-	room := cli.Rooms[roomID]
-	if room == nil { // create a new Room
-		room = NewRoom(roomID)
-		cli.Rooms[roomID] = room
-	}
-	return room
-}
-
-func (cli *Client) sendJSON(method string, httpURL string, contentJSON interface{}) ([]byte, error) {
-	jsonStr, err := json.Marshal(contentJSON)
-	if err != nil {
-		return nil, err
-	}
-	req, err := http.NewRequest(method, httpURL, bytes.NewBuffer(jsonStr))
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	logger := log.WithFields(log.Fields{
-		"method": method,
-		"url":    httpURL,
-		"json":   string(jsonStr),
-	})
-	logger.Print("Sending JSON request")
-	res, err := cli.httpClient.Do(req)
-	if res != nil {
-		defer res.Body.Close()
-	}
-	if err != nil {
-		logger.WithError(err).Warn("Failed to send JSON request")
-		return nil, err
+	msg := struct {
+		MsgType string            `json:"msgtype"`
+		Body    string            `json:"body"`
+		Data    map[string]string `json:"data,omitempty"`
+	}{
+		"m.notice", m.Body, data,
 	}
-	contents, err := ioutil.ReadAll(res.Body)
-	if res.StatusCode >= 300 {
-		logger.WithFields(log.Fields{
-			"code": res.StatusCode,
-			"body": string(contents),
-		}).Warn("Failed to send JSON request")
-		return nil, errors.HTTPError{
-			Code:    res.StatusCode,
-			Message: "Failed to " + method + " JSON: HTTP " + strconv.Itoa(res.StatusCode),
-		}
-	}
-	if err != nil {
-		logger.WithError(err).Warn("Failed to read response")
-		return nil, err
-	}
-	return contents, nil
-}
-
-func (cli *Client) createFilter() (string, error) {
-	urlPath := cli.buildURL("user", cli.UserID, "filter")
-	resBytes, err := cli.sendJSON("POST", urlPath, &filterJSON)
-	if err != nil {
-		return "", err
-	}
-	var filterResponse filterHTTPResponse
-	if err = json.Unmarshal(resBytes, &filterResponse); err != nil {
-		return "", err
-	}
-	return filterResponse.FilterID, nil
-}
-
-func (cli *Client) doSync(timeout int, since string) ([]byte, error) {
-	query := map[string]string{
-		"timeout": strconv.Itoa(timeout),
-	}
-	if since != "" {
-		query["since"] = since
-	}
-	if cli.filterID != "" {
-		query["filter"] = cli.filterID
-	}
-	urlPath := cli.buildURLWithQuery([]string{"sync"}, query)
-	req, err := http.NewRequest("GET", urlPath, nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := cli.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-	contents, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	return contents, nil
-}
-
-// NewClient creates a new Matrix Client ready for syncing
-func NewClient(httpClient *http.Client, homeserverURL *url.URL, accessToken, userID string) *Client {
-	cli := Client{
-		AccessToken:   accessToken,
-		HomeserverURL: homeserverURL,
-		UserID:        userID,
-		Prefix:        "/_matrix/client/r0",
-	}
-	cli.Worker = newWorker(&cli)
-	// By default, use a no-op next_batch storer which will never save tokens and always
-	// "load" the empty string as a token. The client will work with this storer: it just won't
-	// remember the token across restarts. In practice, a database backend should be used.
-	cli.NextBatchStorer = noopNextBatchStore{}
-	cli.Rooms = make(map[string]*Room)
-	cli.httpClient = httpClient
-
-	return &cli
+	return json.Marshal(msg)
 }
diff --git a/src/github.com/matrix-org/go-neb/matrix/responses.go b/src/github.com/matrix-org/go-neb/matrix/responses.go
deleted file mode 100644
index 284554e..0000000
--- a/src/github.com/matrix-org/go-neb/matrix/responses.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package matrix
-
-type filterHTTPResponse struct {
-	FilterID string `json:"filter_id"`
-}
-
-type joinRoomHTTPResponse struct {
-	RoomID string `json:"room_id"`
-}
-
-type sendEventHTTPResponse struct {
-	EventID string `json:"event_id"`
-}
-
-type syncHTTPResponse struct {
-	NextBatch   string `json:"next_batch"`
-	AccountData struct {
-		Events []Event `json:"events"`
-	} `json:"account_data"`
-	Presence struct {
-		Events []Event `json:"events"`
-	} `json:"presence"`
-	Rooms struct {
-		Join map[string]struct {
-			State struct {
-				Events []Event `json:"events"`
-			} `json:"state"`
-			Timeline struct {
-				Events    []Event `json:"events"`
-				Limited   bool    `json:"limited"`
-				PrevBatch string  `json:"prev_batch"`
-			} `json:"timeline"`
-		} `json:"join"`
-		Invite map[string]struct {
-			State struct {
-				Events []Event
-			} `json:"invite_state"`
-		} `json:"invite"`
-	} `json:"rooms"`
-}
diff --git a/src/github.com/matrix-org/go-neb/matrix/worker.go b/src/github.com/matrix-org/go-neb/matrix/worker.go
deleted file mode 100644
index 3674601..0000000
--- a/src/github.com/matrix-org/go-neb/matrix/worker.go
+++ /dev/null
@@ -1,81 +0,0 @@
-package matrix
-
-import (
-	log "github.com/Sirupsen/logrus"
-	"runtime/debug"
-)
-
-// Worker processes incoming events and updates the Matrix client's data structures. It also informs
-// any attached listeners of the new events.
-type Worker struct {
-	client    *Client
-	listeners map[string][]OnEventListener // event type to listeners array
-}
-
-// OnEventListener can be used with Worker.OnEventType to be informed of incoming events.
-type OnEventListener func(*Event)
-
-func newWorker(client *Client) *Worker {
-	return &Worker{
-		client,
-		make(map[string][]OnEventListener),
-	}
-}
-
-// OnEventType allows callers to be notified when there are new events for the given event type.
-// There are no duplicate checks.
-func (worker *Worker) OnEventType(eventType string, callback OnEventListener) {
-	_, exists := worker.listeners[eventType]
-	if !exists {
-		worker.listeners[eventType] = []OnEventListener{}
-	}
-	worker.listeners[eventType] = append(worker.listeners[eventType], callback)
-}
-
-func (worker *Worker) notifyListeners(event *Event) {
-	listeners, exists := worker.listeners[event.Type]
-	if !exists {
-		return
-	}
-	for _, fn := range listeners {
-		fn(event)
-	}
-}
-
-func (worker *Worker) onSyncHTTPResponse(res syncHTTPResponse) {
-	defer func() {
-		if r := recover(); r != nil {
-			userID := ""
-			if worker.client != nil {
-				userID = worker.client.UserID
-			}
-			log.WithFields(log.Fields{
-				"panic":   r,
-				"user_id": userID,
-			}).Errorf(
-				"onSyncHTTPResponse panicked!\n%s", debug.Stack(),
-			)
-		}
-	}()
-
-	for roomID, roomData := range res.Rooms.Join {
-		room := worker.client.getOrCreateRoom(roomID)
-		for _, event := range roomData.State.Events {
-			event.RoomID = roomID
-			room.UpdateState(&event)
-			worker.notifyListeners(&event)
-		}
-		for _, event := range roomData.Timeline.Events {
-			event.RoomID = roomID
-			worker.notifyListeners(&event)
-		}
-	}
-	for roomID, roomData := range res.Rooms.Invite {
-		room := worker.client.getOrCreateRoom(roomID)
-		for _, event := range roomData.State.Events {
-			event.RoomID = roomID
-			room.UpdateState(&event)
-			worker.notifyListeners(&event)
-		}
-	}
-}
diff --git a/src/github.com/matrix-org/go-neb/services/echo/echo.go b/src/github.com/matrix-org/go-neb/services/echo/echo.go
index 6630bef..32ae181 100644
--- a/src/github.com/matrix-org/go-neb/services/echo/echo.go
+++ b/src/github.com/matrix-org/go-neb/services/echo/echo.go
@@ -4,8 +4,8 @@ package echo
 import (
 	"strings"
 
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ServiceType of the Echo service
@@ -19,12 +19,12 @@ type Service struct {
 // Commands supported:
 //    !echo some message
 // Responds with a notice of "some message".
-func (e *Service) Commands(cli *matrix.Client) []types.Command {
+func (e *Service) Commands(cli *gomatrix.Client) []types.Command {
 	return []types.Command{
 		types.Command{
 			Path: []string{"echo"},
 			Command: func(roomID, userID string, args []string) (interface{}, error) {
-				return &matrix.TextMessage{"m.notice", strings.Join(args, " ")}, nil
+				return &gomatrix.TextMessage{"m.notice", strings.Join(args, " ")}, nil
 			},
 		},
 	}
diff --git a/src/github.com/matrix-org/go-neb/services/giphy/giphy.go b/src/github.com/matrix-org/go-neb/services/giphy/giphy.go
index a38f741..6bc51be 100644
--- a/src/github.com/matrix-org/go-neb/services/giphy/giphy.go
+++ b/src/github.com/matrix-org/go-neb/services/giphy/giphy.go
@@ -10,8 +10,8 @@ import (
 	"strings"
 
 	log "github.com/Sirupsen/logrus"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ServiceType of the Giphy service.
@@ -50,7 +50,7 @@ type Service struct {
 // Commands supported:
 //   !giphy some search query without quotes
 // Responds with a suitable GIF into the same room as the command.
-func (s *Service) Commands(client *matrix.Client) []types.Command {
+func (s *Service) Commands(client *gomatrix.Client) []types.Command {
 	return []types.Command{
 		types.Command{
 			Path: []string{"giphy"},
@@ -61,7 +61,7 @@ func (s *Service) Commands(client *matrix.Client) []types.Command {
 	}
 }
 
-func (s *Service) cmdGiphy(client *matrix.Client, roomID, userID string, args []string) (interface{}, error) {
+func (s *Service) cmdGiphy(client *gomatrix.Client, roomID, userID string, args []string) (interface{}, error) {
 	// only 1 arg which is the text to search for.
 	query := strings.Join(args, " ")
 	gifResult, err := s.searchGiphy(query)
@@ -71,16 +71,16 @@ func (s *Service) cmdGiphy(client *matrix.Client, roomID, userID string, args []
 	if gifResult.Images.Original.URL == "" {
 		return nil, fmt.Errorf("No results")
 	}
-	mxc, err := client.UploadLink(gifResult.Images.Original.URL)
+	resUpload, err := client.UploadLink(gifResult.Images.Original.URL)
 	if err != nil {
 		return nil, err
 	}
 
-	return matrix.ImageMessage{
+	return gomatrix.ImageMessage{
 		MsgType: "m.image",
 		Body:    gifResult.Slug,
-		URL:     mxc,
-		Info: matrix.ImageInfo{
+		URL:     resUpload.ContentURI,
+		Info: gomatrix.ImageInfo{
 			Height:   asInt(gifResult.Images.Original.Height),
 			Width:    asInt(gifResult.Images.Original.Width),
 			Mimetype: "image/gif",
diff --git a/src/github.com/matrix-org/go-neb/services/github/github.go b/src/github.com/matrix-org/go-neb/services/github/github.go
index 048c50c..5de367b 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github.go
@@ -18,6 +18,7 @@ import (
 	"github.com/matrix-org/go-neb/realms/github"
 	"github.com/matrix-org/go-neb/services/github/client"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ServiceType of the Github service
@@ -71,7 +72,7 @@ func (s *Service) cmdGithubCreate(roomID, userID string, args []string) (interfa
 		}, nil
 	}
 	if len(args) == 0 {
-		return &matrix.TextMessage{"m.notice",
+		return &gomatrix.TextMessage{"m.notice",
 			`Usage: !github create owner/repo "issue title" "description"`}, nil
 	}
 
@@ -85,13 +86,13 @@ func (s *Service) cmdGithubCreate(roomID, userID string, args []string) (interfa
 		// look for a default repo
 		defaultRepo := s.defaultRepo(roomID)
 		if defaultRepo == "" {
-			return &matrix.TextMessage{"m.notice",
+			return &gomatrix.TextMessage{"m.notice",
 				`Usage: !github create owner/repo "issue title" "description"`}, nil
 		}
 		// default repo should pass the regexp
 		ownerRepoGroups = ownerRepoRegex.FindStringSubmatch(defaultRepo)
 		if len(ownerRepoGroups) == 0 {
-			return &matrix.TextMessage{"m.notice",
+			return &gomatrix.TextMessage{"m.notice",
 				`Malformed default repo. Usage: !github create owner/repo "issue title" "description"`}, nil
 		}
 
@@ -127,7 +128,7 @@ func (s *Service) cmdGithubCreate(roomID, userID string, args []string) (interfa
 		return nil, fmt.Errorf("Failed to create issue. HTTP %d", res.StatusCode)
 	}
 
-	return matrix.TextMessage{"m.notice", fmt.Sprintf("Created issue: %s", *issue.HTMLURL)}, nil
+	return gomatrix.TextMessage{"m.notice", fmt.Sprintf("Created issue: %s", *issue.HTMLURL)}, nil
 }
 
 func (s *Service) expandIssue(roomID, userID, owner, repo string, issueNum int) interface{} {
@@ -143,7 +144,7 @@ func (s *Service) expandIssue(roomID, userID, owner, repo string, issueNum int)
 		return nil
 	}
 
-	return &matrix.TextMessage{
+	return &gomatrix.TextMessage{
 		"m.notice",
 		fmt.Sprintf("%s : %s", *i.HTMLURL, *i.Title),
 	}
@@ -154,7 +155,7 @@ func (s *Service) expandIssue(roomID, userID, owner, repo string, issueNum int)
 // Responds with the outcome of the issue creation request. This command requires
 // a Github account to be linked to the Matrix user ID issuing the command. If there
 // is no link, it will return a Starter Link instead.
-func (s *Service) Commands(cli *matrix.Client) []types.Command {
+func (s *Service) Commands(cli *gomatrix.Client) []types.Command {
 	return []types.Command{
 		types.Command{
 			Path: []string{"github", "create"},
@@ -171,7 +172,7 @@ func (s *Service) Commands(cli *matrix.Client) []types.Command {
 // it will also expand strings of the form:
 //   #12
 // using the default repository.
-func (s *Service) Expansions(cli *matrix.Client) []types.Expansion {
+func (s *Service) Expansions(cli *gomatrix.Client) []types.Expansion {
 	return []types.Expansion{
 		types.Expansion{
 			Regexp: ownerRepoIssueRegex,
@@ -220,7 +221,7 @@ func (s *Service) Expansions(cli *matrix.Client) []types.Expansion {
 }
 
 // Register makes sure that the given realm ID maps to a github realm.
-func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+func (s *Service) Register(oldService types.Service, client *gomatrix.Client) error {
 	if s.RealmID == "" {
 		return fmt.Errorf("RealmID is required")
 	}
diff --git a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
index c5b05b9..7a2a0cd 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
@@ -9,11 +9,11 @@ import (
 	log "github.com/Sirupsen/logrus"
 	gogithub "github.com/google/go-github/github"
 	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/services/github/client"
 	"github.com/matrix-org/go-neb/services/github/webhook"
 	"github.com/matrix-org/go-neb/types"
 	"github.com/matrix-org/go-neb/util"
+	"github.com/matrix-org/gomatrix"
 )
 
 // WebhookServiceType of the Github Webhook service.
@@ -80,7 +80,7 @@ type WebhookService struct {
 //
 // If the "owner/repo" string doesn't exist in this Service config, then the webhook will be deleted from
 // Github.
-func (s *WebhookService) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
+func (s *WebhookService) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client) {
 	evType, repo, msg, err := webhook.OnReceiveRequest(req, s.SecretToken)
 	if err != nil {
 		w.WriteHeader(err.Code)
@@ -145,7 +145,7 @@ func (s *WebhookService) OnReceiveWebhook(w http.ResponseWriter, req *http.Reque
 //
 // Hooks can get out of sync if a user manually deletes a hook in the Github UI. In this case, toggling the repo configuration will
 // force NEB to recreate the hook.
-func (s *WebhookService) Register(oldService types.Service, client *matrix.Client) error {
+func (s *WebhookService) Register(oldService types.Service, client *gomatrix.Client) error {
 	if s.RealmID == "" || s.ClientUserID == "" {
 		return fmt.Errorf("RealmID and ClientUserID is required")
 	}
@@ -251,7 +251,7 @@ func (s *WebhookService) PostRegister(oldService types.Service) {
 	}
 }
 
-func (s *WebhookService) joinWebhookRooms(client *matrix.Client) error {
+func (s *WebhookService) joinWebhookRooms(client *gomatrix.Client) error {
 	for roomID := range s.Rooms {
 		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
 			// TODO: Leave the rooms we successfully joined?
diff --git a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
index 87a8385..467d033 100644
--- a/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
@@ -14,14 +14,14 @@ import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/google/go-github/github"
 	"github.com/matrix-org/go-neb/errors"
-	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/gomatrix"
 )
 
 // OnReceiveRequest processes incoming github webhook requests and returns a
 // matrix message to send, along with parsed repo information.
 // The secretToken, if supplied, will be used to verify the request is from
 // Github. If it isn't, an error is returned.
-func OnReceiveRequest(r *http.Request, secretToken string) (string, *github.Repository, *matrix.HTMLMessage, *errors.HTTPError) {
+func OnReceiveRequest(r *http.Request, secretToken string) (string, *github.Repository, *gomatrix.HTMLMessage, *errors.HTTPError) {
 	// Verify the HMAC signature if NEB was configured with a secret token
 	eventType := r.Header.Get("X-GitHub-Event")
 	signatureSHA1 := r.Header.Get("X-Hub-Signature")
@@ -67,7 +67,7 @@ func OnReceiveRequest(r *http.Request, secretToken string) (string, *github.Repo
 		return "", nil, nil, &errors.HTTPError{nil, "Failed to parse github event", 500}
 	}
 
-	msg := matrix.GetHTMLMessage("m.notice", htmlStr)
+	msg := gomatrix.GetHTMLMessage("m.notice", htmlStr)
 
 	return refinedType, repo, &msg, nil
 }
diff --git a/src/github.com/matrix-org/go-neb/services/guggy/guggy.go b/src/github.com/matrix-org/go-neb/services/guggy/guggy.go
index 940a416..c027544 100644
--- a/src/github.com/matrix-org/go-neb/services/guggy/guggy.go
+++ b/src/github.com/matrix-org/go-neb/services/guggy/guggy.go
@@ -11,8 +11,8 @@ import (
 	"strings"
 
 	log "github.com/Sirupsen/logrus"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ServiceType of the Guggy service
@@ -49,7 +49,7 @@ type Service struct {
 // Commands supported:
 //    !guggy some search query without quotes
 // Responds with a suitable GIF into the same room as the command.
-func (s *Service) Commands(client *matrix.Client) []types.Command {
+func (s *Service) Commands(client *gomatrix.Client) []types.Command {
 	return []types.Command{
 		types.Command{
 			Path: []string{"guggy"},
@@ -59,7 +59,7 @@ func (s *Service) Commands(client *matrix.Client) []types.Command {
 		},
 	}
 }
-func (s *Service) cmdGuggy(client *matrix.Client, roomID, userID string, args []string) (interface{}, error) {
+func (s *Service) cmdGuggy(client *gomatrix.Client, roomID, userID string, args []string) (interface{}, error) {
 	// only 1 arg which is the text to search for.
 	querySentence := strings.Join(args, " ")
 	gifResult, err := s.text2gifGuggy(querySentence)
@@ -68,22 +68,22 @@ func (s *Service) cmdGuggy(client *matrix.Client, roomID, userID string, args []
 	}
 
 	if gifResult.GIF == "" {
-		return matrix.TextMessage{
+		return gomatrix.TextMessage{
 			MsgType: "m.text.notice",
 			Body:    "No GIF found!",
 		}, nil
 	}
 
-	mxc, err := client.UploadLink(gifResult.GIF)
+	resUpload, err := client.UploadLink(gifResult.GIF)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to upload Guggy image to matrix: %s", err.Error())
 	}
 
-	return matrix.ImageMessage{
+	return gomatrix.ImageMessage{
 		MsgType: "m.image",
 		Body:    querySentence,
-		URL:     mxc,
-		Info: matrix.ImageInfo{
+		URL:     resUpload.ContentURI,
+		Info: gomatrix.ImageInfo{
 			Height:   uint(math.Floor(gifResult.Height)),
 			Width:    uint(math.Floor(gifResult.Width)),
 			Mimetype: "image/gif",
diff --git a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
index 480b8b8..688649c 100644
--- a/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
+++ b/src/github.com/matrix-org/go-neb/services/guggy/guggy_test.go
@@ -4,15 +4,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
-	"github.com/matrix-org/go-neb/testutils"
-	"github.com/matrix-org/go-neb/types"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 	"testing"
+
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/testutils"
+	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // TODO: It would be nice to tabularise this test so we can try failing different combinations of responses to make
@@ -86,8 +86,8 @@ func TestCommand(t *testing.T) {
 		}
 		return nil, fmt.Errorf("Unknown URL: %s", req.URL.String())
 	}
-	u, _ := url.Parse("https://hyrule")
-	matrixCli := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@guggybot:hyrule")
+	matrixCli, _ := gomatrix.NewClient("https://hyrule", "@guggybot:hyrule", "its_a_secret")
+	matrixCli.Client = &http.Client{Transport: matrixTrans}
 
 	// Execute the matrix !command
 	cmds := guggy.Commands(matrixCli)
diff --git a/src/github.com/matrix-org/go-neb/services/jira/jira.go b/src/github.com/matrix-org/go-neb/services/jira/jira.go
index dcadd26..d4e8735 100644
--- a/src/github.com/matrix-org/go-neb/services/jira/jira.go
+++ b/src/github.com/matrix-org/go-neb/services/jira/jira.go
@@ -20,6 +20,7 @@ import (
 	"github.com/matrix-org/go-neb/realms/jira/urls"
 	"github.com/matrix-org/go-neb/services/jira/webhook"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ServiceType of the JIRA Service
@@ -72,7 +73,7 @@ type Service struct {
 
 // Register ensures that the given realm IDs are valid JIRA realms and registers webhooks
 // with those JIRA endpoints.
-func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+func (s *Service) Register(oldService types.Service, client *gomatrix.Client) error {
 	// We only ever make 1 JIRA webhook which listens for all projects and then filter
 	// on receive. So we simply need to know if we need to make a webhook or not. We
 	// need to do this for each unique realm.
@@ -163,7 +164,7 @@ func (s *Service) cmdJiraCreate(roomID, userID string, args []string) (interface
 		return nil, fmt.Errorf("Failed to create issue: JIRA returned %d", res.StatusCode)
 	}
 
-	return &matrix.TextMessage{
+	return &gomatrix.TextMessage{
 		"m.notice",
 		fmt.Sprintf("Created issue: %sbrowse/%s", r.JIRAEndpoint, i.Key),
 	}, nil
@@ -220,7 +221,7 @@ func (s *Service) expandIssue(roomID, userID string, issueKeyGroups []string) in
 		logger.WithError(err).Print("Failed to GET issue")
 		return err
 	}
-	return matrix.GetHTMLMessage(
+	return gomatrix.GetHTMLMessage(
 		"m.notice",
 		fmt.Sprintf(
 			"%sbrowse/%s : %s",
@@ -238,7 +239,7 @@ func (s *Service) expandIssue(roomID, userID string, issueKeyGroups []string) in
 // same project key, which project is chosen is undefined. If there
 // is no JIRA account linked to the Matrix user ID, it will return a Starter Link
 // if there is a known public project with that project key.
-func (s *Service) Commands(cli *matrix.Client) []types.Command {
+func (s *Service) Commands(cli *gomatrix.Client) []types.Command {
 	return []types.Command{
 		types.Command{
 			Path: []string{"jira", "create"},
@@ -255,7 +256,7 @@ func (s *Service) Commands(cli *matrix.Client) []types.Command {
 // to map the project key to a realm, and subsequently the JIRA endpoint to hit.
 // If there are multiple projects with the same project key in the Service Config, one will
 // be chosen arbitrarily.
-func (s *Service) Expansions(cli *matrix.Client) []types.Expansion {
+func (s *Service) Expansions(cli *gomatrix.Client) []types.Expansion {
 	return []types.Expansion{
 		types.Expansion{
 			Regexp: issueKeyRegex,
@@ -267,7 +268,7 @@ func (s *Service) Expansions(cli *matrix.Client) []types.Expansion {
 }
 
 // OnReceiveWebhook receives requests from JIRA and possibly sends requests to Matrix as a result.
-func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
+func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client) {
 	eventProjectKey, event, httpErr := webhook.OnReceiveRequest(req)
 	if httpErr != nil {
 		log.WithError(httpErr).Print("Failed to handle JIRA webhook")
@@ -296,7 +297,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 					continue
 				}
 				_, msgErr := cli.SendMessageEvent(
-					roomID, "m.room.message", matrix.GetHTMLMessage("m.notice", htmlText),
+					roomID, "m.room.message", gomatrix.GetHTMLMessage("m.notice", htmlText),
 				)
 				if msgErr != nil {
 					log.WithFields(log.Fields{
diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index ccc9885..9708ab7 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -14,9 +14,9 @@ import (
 	"github.com/die-net/lrucache"
 	"github.com/gregjones/httpcache"
 	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/polling"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 	"github.com/mmcdole/gofeed"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -71,7 +71,7 @@ type Service struct {
 }
 
 // Register will check the liveness of each RSS feed given. If all feeds check out okay, no error is returned.
-func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+func (s *Service) Register(oldService types.Service, client *gomatrix.Client) error {
 	if len(s.Feeds) == 0 {
 		// this is an error UNLESS the old service had some feeds in which case they are deleting us :(
 		var numOldFeeds int
@@ -100,7 +100,7 @@ func (s *Service) Register(oldService types.Service, client *matrix.Client) erro
 	return nil
 }
 
-func (s *Service) joinRooms(client *matrix.Client) {
+func (s *Service) joinRooms(client *gomatrix.Client) {
 	roomSet := make(map[string]bool)
 	for _, feedInfo := range s.Feeds {
 		for _, roomID := range feedInfo.Rooms {
@@ -144,7 +144,7 @@ func (s *Service) PostRegister(oldService types.Service) {
 //   - Else if there is a Title field, use it as the GUID.
 //
 // Returns a timestamp representing when this Service should have OnPoll called again.
-func (s *Service) OnPoll(cli *matrix.Client) time.Time {
+func (s *Service) OnPoll(cli *gomatrix.Client) time.Time {
 	logger := log.WithFields(log.Fields{
 		"service_id":   s.ServiceID(),
 		"service_type": s.ServiceType(),
@@ -340,7 +340,7 @@ func (s *Service) newItems(feedURL string, allItems []*gofeed.Item) (items []gof
 	return
 }
 
-func (s *Service) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error {
+func (s *Service) sendToRooms(cli *gomatrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error {
 	logger := log.WithFields(log.Fields{
 		"feed_url": feedURL,
 		"title":    item.Title,
@@ -356,8 +356,8 @@ func (s *Service) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.F
 }
 
 // SomeOne posted a new article: Title Of The Entry ( https://someurl.com/blag )
-func itemToHTML(feed *gofeed.Feed, item gofeed.Item) matrix.HTMLMessage {
-	return matrix.GetHTMLMessage("m.notice", fmt.Sprintf(
+func itemToHTML(feed *gofeed.Feed, item gofeed.Item) gomatrix.HTMLMessage {
+	return gomatrix.GetHTMLMessage("m.notice", fmt.Sprintf(
 		"<i>%s</i> posted a new article: %s ( %s )",
 		html.EscapeString(feed.Title), html.EscapeString(item.Title), html.EscapeString(item.Link),
 	))
diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
index 4912d8c..5db87e6 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go
@@ -6,16 +6,15 @@ import (
 	"errors"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/testutils"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 const rssFeedXML = `
@@ -75,7 +74,7 @@ func TestHTMLEntities(t *testing.T) {
 	matrixTrans.RT = func(req *http.Request) (*http.Response, error) {
 		if strings.HasPrefix(req.URL.Path, "/_matrix/client/r0/rooms/!linksroom:hyrule/send/m.room.message") {
 			// Check content body to make sure it is decoded
-			var msg matrix.HTMLMessage
+			var msg gomatrix.HTMLMessage
 			if err := json.NewDecoder(req.Body).Decode(&msg); err != nil {
 				t.Fatal("Failed to decode request JSON: ", err)
 				return nil, errors.New("Error handling matrix client test request")
@@ -95,8 +94,8 @@ func TestHTMLEntities(t *testing.T) {
 		}
 		return nil, errors.New("Unhandled matrix client test request")
 	}
-	u, _ := url.Parse("https://hyrule")
-	matrixClient := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@happy_mask_salesman:hyrule")
+	matrixClient, _ := gomatrix.NewClient("https://hyrule", "@happy_mask_salesman:hyrule", "its_a_secret")
+	matrixClient.Client = &http.Client{Transport: matrixTrans}
 
 	// Invoke OnPoll to trigger the RSS feed update
 	_ = rssbot.OnPoll(matrixClient)
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 89ad304..9248d72 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -12,8 +12,8 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 // ServiceType of the Travis-CI service.
@@ -178,7 +178,7 @@ func outputForTemplate(travisTmpl string, tmpl map[string]string) (out string) {
 //        webhooks: http://go-neb-endpoint.com/notifications
 //
 // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
-func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
+func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client) {
 	if err := req.ParseForm(); err != nil {
 		log.WithError(err).Error("Failed to read incoming Travis-CI webhook form")
 		w.WriteHeader(400)
@@ -222,7 +222,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 			if ownerRepo != whForRepo {
 				continue
 			}
-			msg := matrix.TextMessage{
+			msg := gomatrix.TextMessage{
 				Body:    outputForTemplate(repoData.Template, tmplData),
 				MsgType: "m.notice",
 			}
@@ -241,7 +241,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 }
 
 // Register makes sure the Config information supplied is valid.
-func (s *Service) Register(oldService types.Service, client *matrix.Client) error {
+func (s *Service) Register(oldService types.Service, client *gomatrix.Client) error {
 	s.WebhookURL = s.webhookEndpointURL
 	for _, roomData := range s.Rooms {
 		for repo := range roomData.Repos {
@@ -273,7 +273,7 @@ func (s *Service) PostRegister(oldService types.Service) {
 	}
 }
 
-func (s *Service) joinRooms(client *matrix.Client) {
+func (s *Service) joinRooms(client *gomatrix.Client) {
 	for roomID := range s.Rooms {
 		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
 			log.WithFields(log.Fields{
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
index 3c1d695..7bcbf61 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go
@@ -7,14 +7,13 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"strings"
 	"testing"
 
 	"github.com/matrix-org/go-neb/database"
-	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/testutils"
 	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
 )
 
 const travisOrgPEMPublicKey = (`-----BEGIN PUBLIC KEY-----
@@ -115,13 +114,13 @@ func TestTravisCI(t *testing.T) {
 	httpClient = &http.Client{Transport: travisTransport}
 
 	// Intercept message sending to Matrix and mock responses
-	msgs := []matrix.TextMessage{}
+	msgs := []gomatrix.TextMessage{}
 	matrixTrans := struct{ testutils.MockTransport }{}
 	matrixTrans.RT = func(req *http.Request) (*http.Response, error) {
 		if !strings.Contains(req.URL.String(), "/send/m.room.message") {
 			return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String())
 		}
-		var msg matrix.TextMessage
+		var msg gomatrix.TextMessage
 		if err := json.NewDecoder(req.Body).Decode(&msg); err != nil {
 			return nil, fmt.Errorf("Failed to decode request JSON: %s", err)
 		}
@@ -131,13 +130,13 @@ func TestTravisCI(t *testing.T) {
 			Body:       ioutil.NopCloser(bytes.NewBufferString(`{"event_id":"$yup:event"}`)),
 		}, nil
 	}
-	u, _ := url.Parse("https://hyrule")
-	matrixCli := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@travisci:hyrule")
+	matrixCli, _ := gomatrix.NewClient("https://hyrule", "@travisci:hyrule", "its_a_secret")
+	matrixCli.Client = &http.Client{Transport: matrixTrans}
 
 	// BEGIN running the Travis-CI table tests
 	// ---------------------------------------
 	for _, test := range travisTests {
-		msgs = []matrix.TextMessage{} // reset sent messages
+		msgs = []gomatrix.TextMessage{} // reset sent messages
 		mockWriter := httptest.NewRecorder()
 		travis := makeService(t, test.Template)
 		if travis == nil {
@@ -173,7 +172,7 @@ func TestTravisCI(t *testing.T) {
 	}
 }
 
-func assertResponse(t *testing.T, w *httptest.ResponseRecorder, msgs []matrix.TextMessage, expectCode int, expectMsgLength int) bool {
+func assertResponse(t *testing.T, w *httptest.ResponseRecorder, msgs []gomatrix.TextMessage, expectCode int, expectMsgLength int) bool {
 	if w.Code != expectCode {
 		t.Errorf("TestTravisCI OnReceiveWebhook want HTTP code %d, got %d", expectCode, w.Code)
 		return false
diff --git a/src/github.com/matrix-org/go-neb/types/service.go b/src/github.com/matrix-org/go-neb/types/service.go
index e776cc8..832f7d6 100644
--- a/src/github.com/matrix-org/go-neb/types/service.go
+++ b/src/github.com/matrix-org/go-neb/types/service.go
@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/gomatrix"
 )
 
 // BotOptions for a given bot user in a given room
@@ -23,7 +23,7 @@ type BotOptions struct {
 type Poller interface {
 	// OnPoll is called when the poller should poll. Return the timestamp when you want to be polled again.
 	// Return 0 to never be polled again.
-	OnPoll(client *matrix.Client) time.Time
+	OnPoll(client *gomatrix.Client) time.Time
 }
 
 // A Service is the configuration for a bot service.
@@ -34,14 +34,14 @@ type Service interface {
 	ServiceID() string
 	// Return the type of service. This string MUST NOT change.
 	ServiceType() string
-	Commands(cli *matrix.Client) []Command
-	Expansions(cli *matrix.Client) []Expansion
-	OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client)
+	Commands(cli *gomatrix.Client) []Command
+	Expansions(cli *gomatrix.Client) []Expansion
+	OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client)
 	// A lifecycle function which is invoked when the service is being registered. The old service, if one exists, is provided,
 	// along with a Client instance for ServiceUserID(). If this function returns an error, the service will not be registered
 	// or persisted to the database, and the user's request will fail. This can be useful if you depend on external factors
 	// such as registering webhooks.
-	Register(oldService Service, client *matrix.Client) error
+	Register(oldService Service, client *gomatrix.Client) error
 	// A lifecycle function which is invoked after the service has been successfully registered and persisted to the database.
 	// This function is invoked within the critical section for configuring services, guaranteeing that there will not be
 	// concurrent modifications to this service whilst this function executes. This lifecycle hook should be used to clean
@@ -82,23 +82,23 @@ func (s *DefaultService) ServiceType() string {
 }
 
 // Commands returns no commands.
-func (s *DefaultService) Commands(cli *matrix.Client) []Command {
+func (s *DefaultService) Commands(cli *gomatrix.Client) []Command {
 	return []Command{}
 }
 
 // Expansions returns no expansions.
-func (s *DefaultService) Expansions(cli *matrix.Client) []Expansion {
+func (s *DefaultService) Expansions(cli *gomatrix.Client) []Expansion {
 	return []Expansion{}
 }
 
 // Register does nothing and returns no error.
-func (s *DefaultService) Register(oldService Service, client *matrix.Client) error { return nil }
+func (s *DefaultService) Register(oldService Service, client *gomatrix.Client) error { return nil }
 
 // PostRegister does nothing.
 func (s *DefaultService) PostRegister(oldService Service) {}
 
 // OnReceiveWebhook does nothing but 200 OK the request.
-func (s *DefaultService) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
+func (s *DefaultService) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client) {
 	w.WriteHeader(200) // Do nothing
 }
 
diff --git a/vendor/manifest b/vendor/manifest
index 7ef95ff..dd0a845 100644
--- a/vendor/manifest
+++ b/vendor/manifest
@@ -135,6 +135,12 @@
 			"revision": "193b8f88e381d12f2d53023fba25e43fc81dc5ac",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/matrix-org/gomatrix",
+			"repository": "https://github.com/matrix-org/gomatrix",
+			"revision": "16b47cb061d2bc08761824259f4e453fe5dbd023",
+			"branch": "master"
+		},
 		{
 			"importpath": "github.com/mattn/go-shellwords",
 			"repository": "https://github.com/mattn/go-shellwords",
diff --git a/vendor/src/github.com/matrix-org/gomatrix/LICENSE b/vendor/src/github.com/matrix-org/gomatrix/LICENSE
new file mode 100644
index 0000000..8dada3e
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/vendor/src/github.com/matrix-org/gomatrix/README.md b/vendor/src/github.com/matrix-org/gomatrix/README.md
new file mode 100644
index 0000000..4f6df02
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/README.md
@@ -0,0 +1,4 @@
+# gomatrix
+[![GoDoc](https://godoc.org/github.com/matrix-org/gomatrix?status.svg)](https://godoc.org/github.com/matrix-org/gomatrix)
+
+A Golang Matrix client
diff --git a/vendor/src/github.com/matrix-org/gomatrix/client.go b/vendor/src/github.com/matrix-org/gomatrix/client.go
new file mode 100644
index 0000000..d061e3c
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/client.go
@@ -0,0 +1,381 @@
+// Package gomatrix implements the Matrix Client-Server API.
+//
+// Specification can be found at http://matrix.org/docs/spec/client_server/r0.2.0.html
+//
+// Example usage of this library: (blocking version)
+// 	cli, _ := gomatrix.NewClient("https://matrix.org", "@example:matrix.org", "MDAefhiuwehfuiwe")
+//	syncer := cli.Syncer.(*gomatrix.DefaultSyncer)
+//	syncer.OnEventType("m.room.message", func(ev *gomatrix.Event) {
+//		fmt.Println("Message: ", ev)
+//	})
+//	if err := cli.Sync(); err != nil {
+//		fmt.Println("Sync() returned ", err)
+//	}
+//
+// To make the example non-blocking, call Sync() in a goroutine.
+package gomatrix
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// Client represents a Matrix client.
+type Client struct {
+	HomeserverURL *url.URL     // The base homeserver URL
+	Prefix        string       // The API prefix eg '/_matrix/client/r0'
+	UserID        string       // The user ID of the client. Used for forming HTTP paths which use the client's user ID.
+	AccessToken   string       // The access_token for the client.
+	syncingMutex  sync.Mutex   // protects syncingID
+	syncingID     uint32       // Identifies the current Sync. Only one Sync can be active at any given time.
+	Client        *http.Client // The underlying HTTP client which will be used to make HTTP requests.
+	Syncer        Syncer       // The thing which can process /sync responses
+	Store         Storer       // The thing which can store rooms/tokens/ids
+}
+
+// HTTPError An HTTP Error response, which may wrap an underlying native Go Error.
+type HTTPError struct {
+	WrappedError error
+	Message      string
+	Code         int
+}
+
+func (e HTTPError) Error() string {
+	var wrappedErrMsg string
+	if e.WrappedError != nil {
+		wrappedErrMsg = e.WrappedError.Error()
+	}
+	return fmt.Sprintf("msg=%s code=%d wrapped=%s", e.Message, e.Code, wrappedErrMsg)
+}
+
+// BuildURL builds a URL with the Client's homserver/prefix/access_token set already.
+func (cli *Client) BuildURL(urlPath ...string) string {
+	ps := []string{cli.Prefix}
+	for _, p := range urlPath {
+		ps = append(ps, p)
+	}
+	return cli.BuildBaseURL(ps...)
+}
+
+// BuildBaseURL builds a URL with the Client's homeserver/access_token set already. You must
+// supply the prefix in the path.
+func (cli *Client) BuildBaseURL(urlPath ...string) string {
+	// copy the URL. Purposefully ignore error as the input is from a valid URL already
+	hsURL, _ := url.Parse(cli.HomeserverURL.String())
+	parts := []string{hsURL.Path}
+	parts = append(parts, urlPath...)
+	hsURL.Path = path.Join(parts...)
+	query := hsURL.Query()
+	query.Set("access_token", cli.AccessToken)
+	hsURL.RawQuery = query.Encode()
+	return hsURL.String()
+}
+
+// BuildURLWithQuery builds a URL with query paramters in addition to the Client's homeserver/prefix/access_token set already.
+func (cli *Client) BuildURLWithQuery(urlPath []string, urlQuery map[string]string) string {
+	u, _ := url.Parse(cli.BuildURL(urlPath...))
+	q := u.Query()
+	for k, v := range urlQuery {
+		q.Set(k, v)
+	}
+	u.RawQuery = q.Encode()
+	return u.String()
+}
+
+// Sync starts syncing with the provided Homeserver. This function will block until a fatal /sync error occurs, so should
+// almost always be started as a new goroutine. If Sync() is called twice then the first sync will be stopped.
+func (cli *Client) Sync() error {
+	// Mark the client as syncing.
+	// We will keep syncing until the syncing state changes. Either because
+	// Sync is called or StopSync is called.
+	syncingID := cli.incrementSyncingID()
+	nextBatch := cli.Store.LoadNextBatch(cli.UserID)
+	filterID := cli.Store.LoadFilterID(cli.UserID)
+	if filterID == "" {
+		filterJSON := cli.Syncer.GetFilterJSON(cli.UserID)
+		resFilter, err := cli.CreateFilter(filterJSON)
+		if err != nil {
+			return err
+		}
+		filterID = resFilter.FilterID
+		cli.Store.SaveFilterID(cli.UserID, filterID)
+	}
+
+	for {
+		resSync, err := cli.SyncRequest(30000, nextBatch, filterID, false, "")
+		if err != nil {
+			duration, err2 := cli.Syncer.OnFailedSync(resSync, err)
+			if err2 != nil {
+				return err2
+			}
+			time.Sleep(duration)
+			continue
+		}
+
+		// Check that the syncing state hasn't changed
+		// Either because we've stopped syncing or another sync has been started.
+		// We discard the response from our sync.
+		if cli.getSyncingID() != syncingID {
+			return nil
+		}
+
+		// Save the token now *before* processing it. This means it's possible
+		// to not process some events, but it means that we won't get constantly stuck processing
+		// a malformed/buggy event which keeps making us panic.
+		cli.Store.SaveNextBatch(cli.UserID, resSync.NextBatch)
+		if err = cli.Syncer.ProcessResponse(resSync, nextBatch); err != nil {
+			return err
+		}
+
+		nextBatch = resSync.NextBatch
+	}
+}
+
+func (cli *Client) incrementSyncingID() uint32 {
+	cli.syncingMutex.Lock()
+	defer cli.syncingMutex.Unlock()
+	cli.syncingID++
+	return cli.syncingID
+}
+
+func (cli *Client) getSyncingID() uint32 {
+	cli.syncingMutex.Lock()
+	defer cli.syncingMutex.Unlock()
+	return cli.syncingID
+}
+
+// StopSync stops the ongoing sync started by Sync.
+func (cli *Client) StopSync() {
+	// Advance the syncing state so that any running Syncs will terminate.
+	cli.incrementSyncingID()
+}
+
+// SendJSON sends JSON to the given URL.
+//
+// Returns the HTTP body as bytes on 2xx. Returns an error if the response is not 2xx. This error
+// is an HTTPError which includes the returned HTTP status code and possibly a RespError as the
+// WrappedError, if the HTTP body could be decoded as a RespError.
+func (cli *Client) SendJSON(method string, httpURL string, contentJSON interface{}) ([]byte, error) {
+	jsonStr, err := json.Marshal(contentJSON)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest(method, httpURL, bytes.NewBuffer(jsonStr))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	res, err := cli.Client.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return nil, err
+	}
+	contents, err := ioutil.ReadAll(res.Body)
+	if res.StatusCode >= 300 || res.StatusCode < 200 {
+		var wrap error
+		var respErr RespError
+		if _ = json.Unmarshal(contents, respErr); respErr.ErrCode != "" {
+			wrap = respErr
+		}
+
+		// If we failed to decode as RespError, don't just drop the HTTP body, include it in the
+		// HTTP error instead (e.g proxy errors which return HTML).
+		msg := "Failed to " + method + " JSON"
+		if wrap == nil {
+			msg = msg + ": " + string(contents)
+		}
+
+		return nil, HTTPError{
+			Code:         res.StatusCode,
+			Message:      msg,
+			WrappedError: wrap,
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	return contents, nil
+}
+
+// CreateFilter makes an HTTP request according to http://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-user-userid-filter
+func (cli *Client) CreateFilter(filter json.RawMessage) (*RespCreateFilter, error) {
+	urlPath := cli.BuildURL("user", cli.UserID, "filter")
+	resBytes, err := cli.SendJSON("POST", urlPath, &filter)
+	if err != nil {
+		return nil, err
+	}
+	var filterResponse RespCreateFilter
+	if err = json.Unmarshal(resBytes, &filterResponse); err != nil {
+		return nil, err
+	}
+	return &filterResponse, nil
+}
+
+// SyncRequest makes an HTTP request according to http://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync
+func (cli *Client) SyncRequest(timeout int, since, filterID string, fullState bool, setPresence string) (*RespSync, error) {
+	query := map[string]string{
+		"timeout": strconv.Itoa(timeout),
+	}
+	if since != "" {
+		query["since"] = since
+	}
+	if filterID != "" {
+		query["filter"] = filterID
+	}
+	if setPresence != "" {
+		query["set_presence"] = setPresence
+	}
+	if fullState {
+		query["full_state"] = "true"
+	}
+	urlPath := cli.BuildURLWithQuery([]string{"sync"}, query)
+	req, err := http.NewRequest("GET", urlPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := cli.Client.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var syncResponse RespSync
+	err = json.NewDecoder(res.Body).Decode(&syncResponse)
+	return &syncResponse, err
+}
+
+// JoinRoom joins the client to a room ID or alias. See http://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-join-roomidoralias
+//
+// If serverName is specified, this will be added as a query param to instruct the homeserver to join via that server. If content is specified, it will
+// be JSON encoded and used as the request body.
+func (cli *Client) JoinRoom(roomIDorAlias, serverName string, content interface{}) (*RespJoinRoom, error) {
+	var urlPath string
+	if serverName != "" {
+		urlPath = cli.BuildURLWithQuery([]string{"join", roomIDorAlias}, map[string]string{
+			"server_name": serverName,
+		})
+	} else {
+		urlPath = cli.BuildURL("join", roomIDorAlias)
+	}
+
+	resBytes, err := cli.SendJSON("POST", urlPath, content)
+	if err != nil {
+		return nil, err
+	}
+	var joinRoomResponse RespJoinRoom
+	if err = json.Unmarshal(resBytes, &joinRoomResponse); err != nil {
+		return nil, err
+	}
+	return &joinRoomResponse, nil
+}
+
+// SetDisplayName sets the user's profile display name. See http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-profile-userid-displayname
+func (cli *Client) SetDisplayName(displayName string) error {
+	urlPath := cli.BuildURL("profile", cli.UserID, "displayname")
+	s := struct {
+		DisplayName string `json:"displayname"`
+	}{displayName}
+	_, err := cli.SendJSON("PUT", urlPath, &s)
+	return err
+}
+
+// SendMessageEvent sends a message event into a room. See http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
+// contentJSON should be a pointer to something that can be encoded as JSON using json.Marshal.
+func (cli *Client) SendMessageEvent(roomID string, eventType string, contentJSON interface{}) (*RespSendEvent, error) {
+	txnID := "go" + strconv.FormatInt(time.Now().UnixNano(), 10)
+	urlPath := cli.BuildURL("rooms", roomID, "send", eventType, txnID)
+	resBytes, err := cli.SendJSON("PUT", urlPath, contentJSON)
+	if err != nil {
+		return nil, err
+	}
+	var sendEventResponse RespSendEvent
+	if err = json.Unmarshal(resBytes, &sendEventResponse); err != nil {
+		return nil, err
+	}
+	return &sendEventResponse, nil
+}
+
+// SendText sends an m.room.message event into the given room with a msgtype of m.text
+// See http://matrix.org/docs/spec/client_server/r0.2.0.html#m-text
+func (cli *Client) SendText(roomID, text string) (*RespSendEvent, error) {
+	return cli.SendMessageEvent(roomID, "m.room.message",
+		TextMessage{"m.text", text})
+}
+
+// UploadLink uploads an HTTP URL and then returns an MXC URI.
+func (cli *Client) UploadLink(link string) (*RespMediaUpload, error) {
+	res, err := cli.Client.Get(link)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return nil, err
+	}
+	return cli.UploadToContentRepo(res.Body, res.Header.Get("Content-Type"), res.ContentLength)
+}
+
+// UploadToContentRepo uploads the given bytes to the content repository and returns an MXC URI.
+// See http://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-media-r0-upload
+func (cli *Client) UploadToContentRepo(content io.Reader, contentType string, contentLength int64) (*RespMediaUpload, error) {
+	req, err := http.NewRequest("POST", cli.BuildBaseURL("_matrix/media/r0/upload"), content)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", contentType)
+	req.ContentLength = contentLength
+	res, err := cli.Client.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, HTTPError{
+			Message: "Upload request failed",
+			Code:    res.StatusCode,
+		}
+	}
+	var m RespMediaUpload
+	if err := json.NewDecoder(res.Body).Decode(&m); err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+// NewClient creates a new Matrix Client ready for syncing
+func NewClient(homeserverURL, userID, accessToken string) (*Client, error) {
+	hsURL, err := url.Parse(homeserverURL)
+	if err != nil {
+		return nil, err
+	}
+	// By default, use an in-memory store which will never save filter ids / next batch tokens to disk.
+	// The client will work with this storer: it just won't remember across restarts.
+	// In practice, a database backend should be used.
+	store := NewInMemoryStore()
+	cli := Client{
+		AccessToken:   accessToken,
+		HomeserverURL: hsURL,
+		UserID:        userID,
+		Prefix:        "/_matrix/client/r0",
+		Syncer:        NewDefaultSyncer(userID, store),
+		Store:         store,
+	}
+	// By default, use the default HTTP client.
+	cli.Client = http.DefaultClient
+
+	return &cli, nil
+}
diff --git a/vendor/src/github.com/matrix-org/gomatrix/client_test.go b/vendor/src/github.com/matrix-org/gomatrix/client_test.go
new file mode 100644
index 0000000..6a12a9a
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/client_test.go
@@ -0,0 +1,28 @@
+package gomatrix
+
+import "fmt"
+
+func ExampleClient_BuildURLWithQuery() {
+	cli, _ := NewClient("https://matrix.org", "@example:matrix.org", "abcdef123456")
+	out := cli.BuildURLWithQuery([]string{"sync"}, map[string]string{
+		"filter_id": "5",
+	})
+	fmt.Println(out)
+	// Output: https://matrix.org/_matrix/client/r0/sync?access_token=abcdef123456&filter_id=5
+}
+
+func ExampleClient_BuildURL() {
+	userID := "@example:matrix.org"
+	cli, _ := NewClient("https://matrix.org", userID, "abcdef123456")
+	out := cli.BuildURL("user", userID, "filter")
+	fmt.Println(out)
+	// Output: https://matrix.org/_matrix/client/r0/user/@example:matrix.org/filter?access_token=abcdef123456
+}
+
+func ExampleClient_BuildBaseURL() {
+	userID := "@example:matrix.org"
+	cli, _ := NewClient("https://matrix.org", userID, "abcdef123456")
+	out := cli.BuildBaseURL("_matrix", "client", "r0", "directory", "room", "#matrix:matrix.org")
+	fmt.Println(out)
+	// Output: https://matrix.org/_matrix/client/r0/directory/room/%23matrix:matrix.org?access_token=abcdef123456
+}
diff --git a/src/github.com/matrix-org/go-neb/matrix/types.go b/vendor/src/github.com/matrix-org/gomatrix/events.go
similarity index 55%
rename from src/github.com/matrix-org/go-neb/matrix/types.go
rename to vendor/src/github.com/matrix-org/gomatrix/events.go
index 08dc1ba..6ea259e 100644
--- a/src/github.com/matrix-org/go-neb/matrix/types.go
+++ b/vendor/src/github.com/matrix-org/gomatrix/events.go
@@ -1,61 +1,10 @@
-package matrix
+package gomatrix
 
 import (
-	"encoding/json"
 	"html"
 	"regexp"
 )
 
-// Room represents a single Matrix room.
-type Room struct {
-	ID       string
-	State    map[string]map[string]*Event
-	Timeline []Event
-}
-
-// UpdateState updates the room's current state with the given Event. This will clobber events based
-// on the type/state_key combination.
-func (room Room) UpdateState(event *Event) {
-	_, exists := room.State[event.Type]
-	if !exists {
-		room.State[event.Type] = make(map[string]*Event)
-	}
-	room.State[event.Type][event.StateKey] = event
-}
-
-// GetStateEvent returns the state event for the given type/state_key combo, or nil.
-func (room Room) GetStateEvent(eventType string, stateKey string) *Event {
-	stateEventMap, _ := room.State[eventType]
-	event, _ := stateEventMap[stateKey]
-	return event
-}
-
-// GetMembershipState returns the membership state of the given user ID in this room. If there is
-// no entry for this member, 'leave' is returned for consistency with left users.
-func (room Room) GetMembershipState(userID string) string {
-	state := "leave"
-	event := room.GetStateEvent("m.room.member", userID)
-	if event != nil {
-		membershipState, found := event.Content["membership"]
-		if found {
-			mState, isString := membershipState.(string)
-			if isString {
-				state = mState
-			}
-		}
-	}
-	return state
-}
-
-// NewRoom creates a new Room with the given ID
-func NewRoom(roomID string) *Room {
-	// Init the State map and return a pointer to the Room
-	return &Room{
-		ID:    roomID,
-		State: make(map[string]map[string]*Event),
-	}
-}
-
 // Event represents a single Matrix event.
 type Event struct {
 	StateKey  string                 `json:"state_key"`        // The state key for the event. Only present on State Events.
@@ -131,29 +80,3 @@ func GetHTMLMessage(msgtype, htmlText string) HTMLMessage {
 		FormattedBody: htmlText,
 	}
 }
-
-// StarterLinkMessage represents a message with a starter_link custom data.
-type StarterLinkMessage struct {
-	Body string
-	Link string
-}
-
-// MarshalJSON converts this message into actual event content JSON.
-func (m StarterLinkMessage) MarshalJSON() ([]byte, error) {
-	var data map[string]string
-
-	if m.Link != "" {
-		data = map[string]string{
-			"org.matrix.neb.starter_link": m.Link,
-		}
-	}
-
-	msg := struct {
-		MsgType string            `json:"msgtype"`
-		Body    string            `json:"body"`
-		Data    map[string]string `json:"data,omitempty"`
-	}{
-		"m.notice", m.Body, data,
-	}
-	return json.Marshal(msg)
-}
diff --git a/vendor/src/github.com/matrix-org/gomatrix/hooks/install.sh b/vendor/src/github.com/matrix-org/gomatrix/hooks/install.sh
new file mode 100644
index 0000000..f8aa331
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/hooks/install.sh
@@ -0,0 +1,5 @@
+#! /bin/bash
+
+DOT_GIT="$(dirname $0)/../.git"
+
+ln -s "../../hooks/pre-commit" "$DOT_GIT/hooks/pre-commit"
\ No newline at end of file
diff --git a/vendor/src/github.com/matrix-org/gomatrix/hooks/pre-commit b/vendor/src/github.com/matrix-org/gomatrix/hooks/pre-commit
new file mode 100644
index 0000000..6a14ccf
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/hooks/pre-commit
@@ -0,0 +1,9 @@
+#! /bin/bash
+
+set -eu
+
+golint
+go fmt
+go tool vet --shadow .
+gocyclo -over 12 .
+go test -timeout 5s -test.v
diff --git a/vendor/src/github.com/matrix-org/gomatrix/responses.go b/vendor/src/github.com/matrix-org/gomatrix/responses.go
new file mode 100644
index 0000000..76bfbe2
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/responses.go
@@ -0,0 +1,61 @@
+package gomatrix
+
+// RespError is the standard JSON error response from Homeservers. It also implements the Golang "error" interface.
+// See http://matrix.org/docs/spec/client_server/r0.2.0.html#api-standards
+type RespError struct {
+	ErrCode string `json:"errcode"`
+	Err     string `json:"error"`
+}
+
+// Error returns the errcode and error message.
+func (e RespError) Error() string {
+	return e.ErrCode + ": " + e.Err
+}
+
+// RespCreateFilter is the JSON response for http://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-user-userid-filter
+type RespCreateFilter struct {
+	FilterID string `json:"filter_id"`
+}
+
+// RespJoinRoom is the JSON response for http://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-rooms-roomid-join
+type RespJoinRoom struct {
+	RoomID string `json:"room_id"`
+}
+
+// RespSendEvent is the JSON response for http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
+type RespSendEvent struct {
+	EventID string `json:"event_id"`
+}
+
+// RespMediaUpload is the JSON response for http://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-media-r0-upload
+type RespMediaUpload struct {
+	ContentURI string `json:"content_uri"`
+}
+
+// RespSync is the JSON response for http://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync
+type RespSync struct {
+	NextBatch   string `json:"next_batch"`
+	AccountData struct {
+		Events []Event `json:"events"`
+	} `json:"account_data"`
+	Presence struct {
+		Events []Event `json:"events"`
+	} `json:"presence"`
+	Rooms struct {
+		Join map[string]struct {
+			State struct {
+				Events []Event `json:"events"`
+			} `json:"state"`
+			Timeline struct {
+				Events    []Event `json:"events"`
+				Limited   bool    `json:"limited"`
+				PrevBatch string  `json:"prev_batch"`
+			} `json:"timeline"`
+		} `json:"join"`
+		Invite map[string]struct {
+			State struct {
+				Events []Event
+			} `json:"invite_state"`
+		} `json:"invite"`
+	} `json:"rooms"`
+}
diff --git a/vendor/src/github.com/matrix-org/gomatrix/room.go b/vendor/src/github.com/matrix-org/gomatrix/room.go
new file mode 100644
index 0000000..0533b3e
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/room.go
@@ -0,0 +1,50 @@
+package gomatrix
+
+// Room represents a single Matrix room.
+type Room struct {
+	ID    string
+	State map[string]map[string]*Event
+}
+
+// UpdateState updates the room's current state with the given Event. This will clobber events based
+// on the type/state_key combination.
+func (room Room) UpdateState(event *Event) {
+	_, exists := room.State[event.Type]
+	if !exists {
+		room.State[event.Type] = make(map[string]*Event)
+	}
+	room.State[event.Type][event.StateKey] = event
+}
+
+// GetStateEvent returns the state event for the given type/state_key combo, or nil.
+func (room Room) GetStateEvent(eventType string, stateKey string) *Event {
+	stateEventMap, _ := room.State[eventType]
+	event, _ := stateEventMap[stateKey]
+	return event
+}
+
+// GetMembershipState returns the membership state of the given user ID in this room. If there is
+// no entry for this member, 'leave' is returned for consistency with left users.
+func (room Room) GetMembershipState(userID string) string {
+	state := "leave"
+	event := room.GetStateEvent("m.room.member", userID)
+	if event != nil {
+		membershipState, found := event.Content["membership"]
+		if found {
+			mState, isString := membershipState.(string)
+			if isString {
+				state = mState
+			}
+		}
+	}
+	return state
+}
+
+// NewRoom creates a new Room with the given ID
+func NewRoom(roomID string) *Room {
+	// Init the State map and return a pointer to the Room
+	return &Room{
+		ID:    roomID,
+		State: make(map[string]map[string]*Event),
+	}
+}
diff --git a/vendor/src/github.com/matrix-org/gomatrix/store.go b/vendor/src/github.com/matrix-org/gomatrix/store.go
new file mode 100644
index 0000000..20d40b8
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/store.go
@@ -0,0 +1,64 @@
+package gomatrix
+
+// Storer is an interface which must be satisfied to store client data.
+//
+// You can either write a struct which persists this data to disk, or you can use the
+// provided "InMemoryStore" which just keeps data around in-memory which is lost on
+// restarts.
+type Storer interface {
+	SaveFilterID(userID, filterID string)
+	LoadFilterID(userID string) string
+	SaveNextBatch(userID, nextBatchToken string)
+	LoadNextBatch(userID string) string
+	SaveRoom(room *Room)
+	LoadRoom(roomID string) *Room
+}
+
+// InMemoryStore implements the Storer interface.
+//
+// Everything is persisted in-memory as maps. It is not safe to load/save filter IDs
+// or next batch tokens on any goroutine other than the syncing goroutine: the one
+// which called Client.Sync().
+type InMemoryStore struct {
+	Filters   map[string]string
+	NextBatch map[string]string
+	Rooms     map[string]*Room
+}
+
+// SaveFilterID to memory.
+func (s *InMemoryStore) SaveFilterID(userID, filterID string) {
+	s.Filters[userID] = filterID
+}
+
+// LoadFilterID from memory.
+func (s *InMemoryStore) LoadFilterID(userID string) string {
+	return s.Filters[userID]
+}
+
+// SaveNextBatch to memory.
+func (s *InMemoryStore) SaveNextBatch(userID, nextBatchToken string) {
+	s.NextBatch[userID] = nextBatchToken
+}
+
+// LoadNextBatch from memory.
+func (s *InMemoryStore) LoadNextBatch(userID string) string {
+	return s.NextBatch[userID]
+}
+
+// SaveRoom to memory.
+func (s *InMemoryStore) SaveRoom(room *Room) {
+	s.Rooms[room.ID] = room
+}
+
+// LoadRoom from memory.
+func (s *InMemoryStore) LoadRoom(roomID string) *Room {
+	return s.Rooms[roomID]
+}
+
+// NewInMemoryStore constructs a new InMemoryStore.
+func NewInMemoryStore() *InMemoryStore {
+	return &InMemoryStore{
+		Filters:   make(map[string]string),
+		NextBatch: make(map[string]string),
+	}
+}
diff --git a/vendor/src/github.com/matrix-org/gomatrix/sync.go b/vendor/src/github.com/matrix-org/gomatrix/sync.go
new file mode 100644
index 0000000..347e5dc
--- /dev/null
+++ b/vendor/src/github.com/matrix-org/gomatrix/sync.go
@@ -0,0 +1,154 @@
+package gomatrix
+
+import (
+	"encoding/json"
+	"fmt"
+	"runtime/debug"
+	"time"
+)
+
+// Syncer represents an interface that must be satisfied in order to do /sync requests on a client.
+type Syncer interface {
+	// Process the /sync response. The since parameter is the since= value that was used to produce the response.
+	// This is useful for detecting the very first sync (since=""). If an error is return, Syncing will be stopped
+	// permanently.
+	ProcessResponse(resp *RespSync, since string) error
+	// OnFailedSync returns either the time to wait before retrying or an error to stop syncing permanently.
+	OnFailedSync(res *RespSync, err error) (time.Duration, error)
+	// GetFilterJSON for the given user ID. NOT the filter ID.
+	GetFilterJSON(userID string) json.RawMessage
+}
+
+// DefaultSyncer is the default syncing implementation. You can either write your own syncer, or selectively
+// replace parts of this default syncer (e.g. the ProcessResponse method). The default syncer uses the observer
+// pattern to notify callers about incoming events. See DefaultSyncer.OnEventType for more information.
+type DefaultSyncer struct {
+	UserID    string
+	Store     Storer
+	listeners map[string][]OnEventListener // event type to listeners array
+}
+
+// OnEventListener can be used with DefaultSyncer.OnEventType to be informed of incoming events.
+type OnEventListener func(*Event)
+
+// NewDefaultSyncer returns an instantiated DefaultSyncer
+func NewDefaultSyncer(userID string, store Storer) *DefaultSyncer {
+	return &DefaultSyncer{
+		UserID:    userID,
+		Store:     store,
+		listeners: make(map[string][]OnEventListener),
+	}
+}
+
+// ProcessResponse processes the /sync response in a way suitable for bots. "Suitable for bots" means a stream of
+// unrepeating events. Returns a fatal error if a listener panics.
+func (s *DefaultSyncer) ProcessResponse(res *RespSync, since string) (err error) {
+	if !s.shouldProcessResponse(res, since) {
+		return
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("ProcessResponse panicked! userID=%s since=%s panic=%s\n%s", s.UserID, since, r, debug.Stack())
+		}
+	}()
+
+	for roomID, roomData := range res.Rooms.Join {
+		room := s.getOrCreateRoom(roomID)
+		for _, event := range roomData.State.Events {
+			event.RoomID = roomID
+			room.UpdateState(&event)
+			s.notifyListeners(&event)
+		}
+		for _, event := range roomData.Timeline.Events {
+			event.RoomID = roomID
+			s.notifyListeners(&event)
+		}
+	}
+	for roomID, roomData := range res.Rooms.Invite {
+		room := s.getOrCreateRoom(roomID)
+		for _, event := range roomData.State.Events {
+			event.RoomID = roomID
+			room.UpdateState(&event)
+			s.notifyListeners(&event)
+		}
+	}
+	return
+}
+
+// OnEventType allows callers to be notified when there are new events for the given event type.
+// There are no duplicate checks.
+func (s *DefaultSyncer) OnEventType(eventType string, callback OnEventListener) {
+	_, exists := s.listeners[eventType]
+	if !exists {
+		s.listeners[eventType] = []OnEventListener{}
+	}
+	s.listeners[eventType] = append(s.listeners[eventType], callback)
+}
+
+// shouldProcessResponse returns true if the response should be processed. May modify the response to remove
+// stuff that shouldn't be processed.
+func (s *DefaultSyncer) shouldProcessResponse(resp *RespSync, since string) bool {
+	if since == "" {
+		return false
+	}
+	// This is a horrible hack because /sync will return the most recent messages for a room
+	// as soon as you /join it. We do NOT want to process those events in that particular room
+	// because they may have already been processed (if you toggle the bot in/out of the room).
+	//
+	// Work around this by inspecting each room's timeline and seeing if an m.room.member event for us
+	// exists and is "join" and then discard processing that room entirely if so.
+	// TODO: We probably want to process messages from after the last join event in the timeline.
+	for roomID, roomData := range resp.Rooms.Join {
+		for i := len(roomData.Timeline.Events) - 1; i >= 0; i-- {
+			e := roomData.Timeline.Events[i]
+			if e.Type == "m.room.member" && e.StateKey == s.UserID {
+				m := e.Content["membership"]
+				mship, ok := m.(string)
+				if !ok {
+					continue
+				}
+				if mship == "join" {
+					_, ok := resp.Rooms.Join[roomID]
+					if !ok {
+						continue
+					}
+					delete(resp.Rooms.Join, roomID)   // don't re-process messages
+					delete(resp.Rooms.Invite, roomID) // don't re-process invites
+					break
+				}
+			}
+		}
+	}
+	return true
+}
+
+// getOrCreateRoom must only be called by the Sync() goroutine which calls ProcessResponse()
+func (s *DefaultSyncer) getOrCreateRoom(roomID string) *Room {
+	room := s.Store.LoadRoom(roomID)
+	if room == nil { // create a new Room
+		room = NewRoom(roomID)
+		s.Store.SaveRoom(room)
+	}
+	return room
+}
+
+func (s *DefaultSyncer) notifyListeners(event *Event) {
+	listeners, exists := s.listeners[event.Type]
+	if !exists {
+		return
+	}
+	for _, fn := range listeners {
+		fn(event)
+	}
+}
+
+// OnFailedSync always returns a 10 second wait period between failed /syncs, never a fatal error.
+func (s *DefaultSyncer) OnFailedSync(res *RespSync, err error) (time.Duration, error) {
+	return 10 * time.Second, nil
+}
+
+// GetFilterJSON returns a filter with a timeline limit of 50.
+func (s *DefaultSyncer) GetFilterJSON(userID string) json.RawMessage {
+	return json.RawMessage(`{"room":{"timeline":{"limit":50}}}`)
+}

From 60429f32b3fec4b8e190baa6825ecb4eca66187c Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 1 Dec 2016 14:56:40 +0000
Subject: [PATCH 45/51] Bug fixes

---
 src/github.com/matrix-org/go-neb/clients/clients.go | 13 ++++++++++++-
 vendor/manifest                                     |  2 +-
 vendor/src/github.com/matrix-org/gomatrix/store.go  |  1 +
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index cce5add..098cba9 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/api"
@@ -350,7 +351,17 @@ func (c *Clients) newClient(config api.ClientConfig) (*gomatrix.Client, error) {
 	}
 
 	if config.Sync {
-		go client.Sync()
+		go func() {
+			for {
+				if e := client.Sync(); e != nil {
+					log.WithFields(log.Fields{
+						log.ErrorKey: e,
+						"user_id":    config.UserID,
+					}).Error("Fatal Sync() error")
+					time.Sleep(10 * time.Second)
+				}
+			}
+		}()
 	}
 
 	return client, nil
diff --git a/vendor/manifest b/vendor/manifest
index dd0a845..1027008 100644
--- a/vendor/manifest
+++ b/vendor/manifest
@@ -138,7 +138,7 @@
 		{
 			"importpath": "github.com/matrix-org/gomatrix",
 			"repository": "https://github.com/matrix-org/gomatrix",
-			"revision": "16b47cb061d2bc08761824259f4e453fe5dbd023",
+			"revision": "e66d1ef529b7851262b49dc42a26ff1f1d1d9e4d",
 			"branch": "master"
 		},
 		{
diff --git a/vendor/src/github.com/matrix-org/gomatrix/store.go b/vendor/src/github.com/matrix-org/gomatrix/store.go
index 20d40b8..6dc687e 100644
--- a/vendor/src/github.com/matrix-org/gomatrix/store.go
+++ b/vendor/src/github.com/matrix-org/gomatrix/store.go
@@ -60,5 +60,6 @@ func NewInMemoryStore() *InMemoryStore {
 	return &InMemoryStore{
 		Filters:   make(map[string]string),
 		NextBatch: make(map[string]string),
+		Rooms:     make(map[string]*Room),
 	}
 }

From 9708e6351bf40065844d77dae7bb5cbe6330b0a0 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 1 Dec 2016 15:14:39 +0000
Subject: [PATCH 46/51] Stop syncing on nil errors

---
 src/github.com/matrix-org/go-neb/clients/clients.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 098cba9..657abea 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -359,6 +359,8 @@ func (c *Clients) newClient(config api.ClientConfig) (*gomatrix.Client, error) {
 						"user_id":    config.UserID,
 					}).Error("Fatal Sync() error")
 					time.Sleep(10 * time.Second)
+				} else {
+					return
 				}
 			}
 		}()

From e94ab0d18ed574acce909114b2c8eadfe58499c9 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 1 Dec 2016 15:15:35 +0000
Subject: [PATCH 47/51] And log when we stop

---
 src/github.com/matrix-org/go-neb/clients/clients.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 657abea..6989bde 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -360,6 +360,7 @@ func (c *Clients) newClient(config api.ClientConfig) (*gomatrix.Client, error) {
 					}).Error("Fatal Sync() error")
 					time.Sleep(10 * time.Second)
 				} else {
+					log.WithField("user_id", config.UserID).Info("Stopping Sync()")
 					return
 				}
 			}

From efe04d791701bc0973ea724e46a765b8711fac2b Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 1 Dec 2016 15:23:34 +0000
Subject: [PATCH 48/51] Keep doing the Inviter hack when joining rooms so
 non-ops can kick the bot they invited

---
 src/github.com/matrix-org/go-neb/clients/clients.go         | 6 +++++-
 .../matrix-org/go-neb/services/github/github_webhook.go     | 2 +-
 src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go  | 2 +-
 .../matrix-org/go-neb/services/travisci/travisci.go         | 2 +-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 6989bde..3a7d816 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -311,7 +311,11 @@ func (c *Clients) onRoomMemberEvent(client *gomatrix.Client, event *gomatrix.Eve
 		})
 		logger.Print("Accepting invite from user")
 
-		if _, err := client.JoinRoom(event.RoomID, "", event.Sender); err != nil {
+		content := struct {
+			Inviter string `json:"inviter"`
+		}{event.Sender}
+
+		if _, err := client.JoinRoom(event.RoomID, "", content); err != nil {
 			logger.WithError(err).Print("Failed to join room")
 		} else {
 			logger.Print("Joined room")
diff --git a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
index 7a2a0cd..6951874 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github_webhook.go
@@ -253,7 +253,7 @@ func (s *WebhookService) PostRegister(oldService types.Service) {
 
 func (s *WebhookService) joinWebhookRooms(client *gomatrix.Client) error {
 	for roomID := range s.Rooms {
-		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
+		if _, err := client.JoinRoom(roomID, "", nil); err != nil {
 			// TODO: Leave the rooms we successfully joined?
 			return err
 		}
diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
index 9708ab7..a0112af 100644
--- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
+++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go
@@ -109,7 +109,7 @@ func (s *Service) joinRooms(client *gomatrix.Client) {
 	}
 
 	for roomID := range roomSet {
-		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
+		if _, err := client.JoinRoom(roomID, "", nil); err != nil {
 			log.WithFields(log.Fields{
 				log.ErrorKey: err,
 				"room_id":    roomID,
diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
index 9248d72..62a0565 100644
--- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
+++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go
@@ -275,7 +275,7 @@ func (s *Service) PostRegister(oldService types.Service) {
 
 func (s *Service) joinRooms(client *gomatrix.Client) {
 	for roomID := range s.Rooms {
-		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
+		if _, err := client.JoinRoom(roomID, "", nil); err != nil {
 			log.WithFields(log.Fields{
 				log.ErrorKey: err,
 				"room_id":    roomID,

From 8d75494c19c428504768b0251ba3d18847eb5ef3 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 1 Dec 2016 15:30:11 +0000
Subject: [PATCH 49/51] Also log when clients are created

---
 src/github.com/matrix-org/go-neb/clients/clients.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 3a7d816..842e7f2 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -330,12 +330,13 @@ func (c *Clients) newClient(config api.ClientConfig) (*gomatrix.Client, error) {
 	}
 	client.Client = c.httpClient
 	syncer := client.Syncer.(*gomatrix.DefaultSyncer)
-	client.Store = &matrix.NEBStore{
+	nebStore := &matrix.NEBStore{
 		InMemoryStore: *gomatrix.NewInMemoryStore(),
 		Database:      c.db,
 		ClientConfig:  config,
 	}
-	syncer.Store = client.Store
+	client.Store = nebStore
+	syncer.Store = nebStore
 
 	// TODO: Check that the access token is valid for the userID by peforming
 	// a request against the server.
@@ -354,6 +355,13 @@ func (c *Clients) newClient(config api.ClientConfig) (*gomatrix.Client, error) {
 		})
 	}
 
+	log.WithFields(log.Fields{
+		"user_id":         config.UserID,
+		"sync":            config.Sync,
+		"auto_join_rooms": config.AutoJoinRooms,
+		"since":           nebStore.LoadNextBatch(config.UserID),
+	}).Info("Created new client")
+
 	if config.Sync {
 		go func() {
 			for {

From 34ea2d06ce61e5af5652a75f3e2a7bb3f64e37c9 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 9 Dec 2016 13:56:43 +0000
Subject: [PATCH 50/51] Give a more helpful error message if a service is
 created with an unknown user ID

---
 src/github.com/matrix-org/go-neb/clients/clients.go    | 5 +++++
 src/github.com/matrix-org/go-neb/database/schema.go    | 3 ++-
 src/github.com/matrix-org/go-neb/server/server_test.go | 3 +--
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 842e7f2..1927f64 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -1,6 +1,8 @@
 package clients
 
 import (
+	"database/sql"
+	"fmt"
 	"net/http"
 	"strings"
 	"sync"
@@ -94,6 +96,9 @@ func (c *Clients) loadClientFromDB(userID string) (entry clientEntry, err error)
 	}
 
 	if entry.config, err = c.db.LoadMatrixClientConfig(userID); err != nil {
+		if err == sql.ErrNoRows {
+			err = fmt.Errorf("client with user ID %s does not exist", userID)
+		}
 		return
 	}
 
diff --git a/src/github.com/matrix-org/go-neb/database/schema.go b/src/github.com/matrix-org/go-neb/database/schema.go
index 7e3a30f..9a9fc51 100644
--- a/src/github.com/matrix-org/go-neb/database/schema.go
+++ b/src/github.com/matrix-org/go-neb/database/schema.go
@@ -4,9 +4,10 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"time"
+
 	"github.com/matrix-org/go-neb/api"
 	"github.com/matrix-org/go-neb/types"
-	"time"
 )
 
 const schemaSQL = `
diff --git a/src/github.com/matrix-org/go-neb/server/server_test.go b/src/github.com/matrix-org/go-neb/server/server_test.go
index 920d0ae..b9afb12 100644
--- a/src/github.com/matrix-org/go-neb/server/server_test.go
+++ b/src/github.com/matrix-org/go-neb/server/server_test.go
@@ -10,8 +10,7 @@ func TestProtect(t *testing.T) {
 	mockWriter := httptest.NewRecorder()
 	mockReq, _ := http.NewRequest("GET", "http://example.com/foo", nil)
 	h := Protect(func(w http.ResponseWriter, req *http.Request) {
-		var array []string
-		w.Write([]byte(array[5])) // NPE
+		panic("oh noes!")
 	})
 
 	h(mockWriter, mockReq)

From 96109afd41c4ebfcb20686c6075fd21a20ebf911 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Fri, 23 Dec 2016 15:13:25 +0000
Subject: [PATCH 51/51] Add "!github help" and fix command selection bug

The last command would always be chosen due to pointer fail.
---
 src/github.com/matrix-org/go-neb/clients/clients.go      | 4 ++--
 .../matrix-org/go-neb/services/github/github.go          | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/github.com/matrix-org/go-neb/clients/clients.go b/src/github.com/matrix-org/go-neb/clients/clients.go
index 1927f64..bd850a4 100644
--- a/src/github.com/matrix-org/go-neb/clients/clients.go
+++ b/src/github.com/matrix-org/go-neb/clients/clients.go
@@ -215,11 +215,11 @@ func (c *Clients) onMessageEvent(client *gomatrix.Client, event *gomatrix.Event)
 // response is appropriate.
 func runCommandForService(cmds []types.Command, event *gomatrix.Event, arguments []string) interface{} {
 	var bestMatch *types.Command
-	for _, command := range cmds {
+	for i, command := range cmds {
 		matches := command.Matches(arguments)
 		betterMatch := bestMatch == nil || len(bestMatch.Path) < len(command.Path)
 		if matches && betterMatch {
-			bestMatch = &command
+			bestMatch = &cmds[i]
 		}
 	}
 
diff --git a/src/github.com/matrix-org/go-neb/services/github/github.go b/src/github.com/matrix-org/go-neb/services/github/github.go
index 5de367b..e9897a4 100644
--- a/src/github.com/matrix-org/go-neb/services/github/github.go
+++ b/src/github.com/matrix-org/go-neb/services/github/github.go
@@ -163,6 +163,15 @@ func (s *Service) Commands(cli *gomatrix.Client) []types.Command {
 				return s.cmdGithubCreate(roomID, userID, args)
 			},
 		},
+		types.Command{
+			Path: []string{"github", "help"},
+			Command: func(roomID, userID string, args []string) (interface{}, error) {
+				return &gomatrix.TextMessage{
+					"m.notice",
+					fmt.Sprintf(`!github create owner/repo "title text" "description text"`),
+				}, nil
+			},
+		},
 	}
 }