Begin to add Travis-CI Service tests

Currently this verifies the signature logic, but not the templating
logic which isn't implemented yet.

src/github.com/matrix-org/go-neb/services/travisci/travisci.go

@@ -3,7 +3,6 @@ package travisci
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -85,7 +84,7 @@ type Service struct {
 type webhookNotification struct {
 	ID             int     `json:"id"`
 	Number         string  `json:"number"`
-	Status         *string `json:"status"` // 0 (success) or 1 (incomplete/fail).
+	Status         *int    `json:"status"` // 0 (success) or 1 (incomplete/fail).
 	StartedAt      *string `json:"started_at"`
 	FinishedAt     *string `json:"finished_at"`
 	StatusMessage  string  `json:"status_message"`
@@ -110,6 +109,7 @@ type webhookNotification struct {
 // The template variables a user can use in their messages
 type notificationTemplate struct {
 	RepositorySlug string
+	Repository     string // Deprecated: alias for RepositorySlug
 	RepositoryName string
 	BuildNumber    string
 	BuildID        string
@@ -128,6 +128,7 @@ type notificationTemplate struct {
 
 func notifToTemplate(n webhookNotification) (t notificationTemplate) {
 	t.RepositorySlug = n.Repository.OwnerName + "/" + n.Repository.Name
+	t.Repository = t.RepositorySlug
 	t.RepositoryName = n.Repository.Name
 	t.BuildNumber = n.Number
 	t.BuildID = strconv.Itoa(n.ID)
@@ -140,7 +141,7 @@ func notifToTemplate(n webhookNotification) (t notificationTemplate) {
 	subjAndMsg := strings.SplitN(n.Message, "\n", 2)
 	t.CommitSubject = subjAndMsg[0]
 	if n.Status != nil {
-		t.Result = *n.Status
+		t.Result = strconv.Itoa(*n.Status)
 	}
 	t.Message = n.StatusMessage
 
@@ -182,13 +183,18 @@ func travisToGoTemplate(travisTmpl string) (goTmpl string) {
 //
 // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information.
 func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) {
-	payload, err := ioutil.ReadAll(req.Body)
-	if err != nil {
-		log.WithError(err).Error("Failed to read incoming Travis-CI webhook")
-		w.WriteHeader(500)
+	if err := req.ParseForm(); err != nil {
+		log.WithError(err).Error("Failed to read incoming Travis-CI webhook form")
+		w.WriteHeader(400)
+		return
+	}
+	payload := req.PostFormValue("payload")
+	if payload == "" {
+		log.Error("Travis-CI webhook is missing payload= form value")
+		w.WriteHeader(400)
 		return
 	}
-	if err := verifyOrigin(req.Host, payload, req.Header.Get("Signature")); err != nil {
+	if err := verifyOrigin([]byte(payload), req.Header.Get("Signature")); err != nil {
 		log.WithFields(log.Fields{
 			"Signature":  req.Header.Get("Signature"),
 			log.ErrorKey: err,
@@ -198,11 +204,13 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 	}
 
 	var notif webhookNotification
-	if err := json.Unmarshal(payload, &notif); err != nil {
+	if err := json.Unmarshal([]byte(payload), &notif); err != nil {
+		log.WithError(err).Error("Travis-CI webhook received an invalid JSON payload=")
 		w.WriteHeader(400)
 		return
 	}
-	if notif.Repository.URL == "" || notif.Repository.OwnerName == "" || notif.Repository.Name == "" {
+	if notif.Repository.OwnerName == "" || notif.Repository.Name == "" {
+		log.WithField("repo", notif.Repository).Error("Travis-CI webhook missing repository fields")
 		w.WriteHeader(400)
 		return
 	}
@@ -231,6 +239,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli
 			}
 		}
 	}
+	w.WriteHeader(200)
 }
 
 // Register makes sure the Config information supplied is valid.

src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go

@@ -0,0 +1,178 @@
+package travisci
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/types"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+const travisOrgPEMPublicKey = (`-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvtjdLkS+FP+0fPC09j25
+y/PiuYDDivIT86COVedvlElk99BBYTrqNaJybxjXbIZ1Q6xFNhOY+iTcBr4E1zJu
+tizF3Xi0V9tOuP/M8Wn4Y/1lCWbQKlWrNQuqNBmhovF4K3mDCYswVbpgTmp+JQYu
+Bm9QMdieZMNry5s6aiMA9aSjDlNyedvSENYo18F+NYg1J0C0JiPYTxheCb4optr1
+5xNzFKhAkuGs4XTOA5C7Q06GCKtDNf44s/CVE30KODUxBi0MCKaxiXw/yy55zxX2
+/YdGphIyQiA5iO1986ZmZCLLW8udz9uhW5jUr3Jlp9LbmphAC61bVSf4ou2YsJaN
+0QIDAQAB
+-----END PUBLIC KEY-----`)
+
+const travisComPEMPublicKey = (`-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQU2j9lnRtyuW36arNOc
+dzCzyKVirLUi3/aLh6UfnTVXzTnx8eHUnBn1ZeQl7Eh3J3qqdbIKl6npS27ONzCy
+3PIcfjpLPaVyGagIL8c8XgDEvB45AesC0osVP5gkXQkPUM3B2rrUmp1AZzG+Fuo0
+SAeNnS71gN63U3brL9fN/MTCXJJ6TvMt3GrcJUq5uq56qNeJTsiowK6eiiWFUSfh
+e1qapOdMFmcEs9J/R1XQ/scxbAnLcWfl8lqH/MjMdCMe0j3X2ZYMTqOHsb3cQGSS
+dMPwZGeLWV+OaxjJ7TrJ+riqMANOgqCBGpvWUnUfo046ACOx7p6u4fFc3aRiuqYK
+VQIDAQAB
+-----END PUBLIC KEY-----`)
+
+const exampleSignature = ("pW0CDpmcAeWw3qf2Ufx8UvzfrZRUpYx30HBl9nJcDkh2v9FrF1GjJVsrcqx7ly0FPjb7dkfMJ/d0Q3JpDb1EL4p509cN4Vy8+HpfINw35Wg6JqzOQqTa" +
+	"TidwoDLXo0NgL78zfiL3dra7ZwOGTA+LmnLSuNp38ROxn70u26uqJzWprGSdVNbRu1LkF1QKLa61NZegfxK7RZn1PlIsznWIyTS0qj81mg2sXMDLH1J4" +
+	"pHxjEpzydjSb5b8tCjrN+vFLDdAtP5RjU8NwvQM4LRRGbLDIlRsO77HDwfXrPgUE3DjPIqVpHhMcCusygp0ClH2b1J1O3LkhxSS9ol5w99Hkpg==")
+const exampleBody = ("payload=%7B%22id%22%3A176075135%2C%22repository%22%3A%7B%22id%22%3A6461770%2C%22name%22%3A%22flow-jsdoc%22%2C%22owner_" +
+	"name%22%3A%22Kegsay%22%2C%22url%22%3Anull%7D%2C%22number%22%3A%2218%22%2C%22config%22%3A%7B%22notifications%22%3A%7B%22web" +
+	"hooks%22%3A%5B%22http%3A%2F%2F7abbe705.ngrok.io%22%5D%7D%2C%22language%22%3A%22node_js%22%2C%22node_js%22%3A%5B%224.1%22%5D%2C%22.resu" +
+	"lt%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22precise%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22status_" +
+	"message%22%3A%22Passed%22%2C%22result_message%22%3A%22Passed%22%2C%22started_at%22%3A%222016-11-15T15%3A10%3A22Z%22%2C%22finished_" +
+	"at%22%3A%222016-11-15T15%3A10%3A54Z%22%2C%22duration%22%3A32%2C%22build_url%22%3A%22https%3A%2F%2Ftravis-ci.org%2FKegsay%2Fflow-js" +
+	"doc%2Fbuilds%2F176075135%22%2C%22commit_id%22%3A50222535%2C%22commit%22%3A%223a092c3a6032ebb50384c99b445f947e9ce86e2a%22%2C%22base_com" +
+	"mit%22%3Anull%2C%22head_commit%22%3Anull%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22Test+Travis+webhook+support%22%2C%22compare_" +
+	"url%22%3A%22https%3A%2F%2Fgithub.com%2FKegsay%2Fflow-jsdoc%2Fcompare%2F9f9d459ba082...3a092c3a6032%22%2C%22committed_at%22%3A%222016-1" +
+	"1-15T15%3A08%3A16Z%22%2C%22author_name%22%3A%22Kegan+Dougal%22%2C%22author_email%22%3A%22kegan%40matrix.org%22%2C%22committer_" +
+	"name%22%3A%22Kegan+Dougal%22%2C%22committer_email%22%3A%22kegan%40matrix.org%22%2C%22matrix%22%3A%5B%7B%22id%22%3A176075137%2C%22reposit" +
+	"ory_id%22%3A6461770%2C%22parent_id%22%3A176075135%2C%22number%22%3A%2218.1%22%2C%22state%22%3A%22finished%22%2C%22config%22%3A%7B%22notifi" +
+	"cations%22%3A%7B%22webhooks%22%3A%5B%22http%3A%2F%2F7abbe705.ngrok.io%22%5D%7D%2C%22language%22%3A%22node_js%22%2C%22node_" +
+	"js%22%3A%224.1%22%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22precise%22%2C%22os%22%3A%22li" +
+	"nux%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22commit%22%3A%223a092c3a6032ebb50384c99b445f947e9ce86e2a%22%2C%22branch%22%3A%22mas" +
+	"ter%22%2C%22message%22%3A%22Test+Travis+webhook+support%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2FKegsay%2Fflow-jsdoc%2Fcomp" +
+	"are%2F9f9d459ba082...3a092c3a6032%22%2C%22started_at%22%3A%222016-11-15T15%3A10%3A22Z%22%2C%22finished_at%22%3A%222016-11-" +
+	"15T15%3A10%3A54Z%22%2C%22committed_at%22%3A%222016-11-15T15%3A08%3A16Z%22%2C%22author_name%22%3A%22Kegan+Dougal%22%2C%22author_ema" +
+	"il%22%3A%22kegan%40matrix.org%22%2C%22committer_name%22%3A%22Kegan+Dougal%22%2C%22committer_email%22%3A%22kegan%40matrix.org%22%2C%22allow_f" +
+	"ailure%22%3Afalse%7D%5D%2C%22type%22%3A%22push%22%2C%22state%22%3A%22passed%22%2C%22pull_request%22%3Afalse%2C%22pull_request_number%22%3Anu" +
+	"ll%2C%22pull_request_title%22%3Anull%2C%22tag%22%3Anull%7D")
+
+var travisTests = []struct {
+	Signature      string
+	ValidSignature bool
+	Body           string
+	Template       string
+	ExpectedOutput string
+}{
+	{
+		exampleSignature, true, exampleBody,
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"Kegsay/flow-jsdoc#22 master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal: Test Travis webhook support",
+	},
+}
+
+type MockTransport struct {
+	roundTrip func(*http.Request) (*http.Response, error)
+}
+
+func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.roundTrip(req)
+}
+
+func TestTravisCI(t *testing.T) {
+	database.SetServiceDB(&database.NopStorage{})
+
+	// When the service tries to get Travis' public key, return the constant
+	urlToKey := make(map[string]string)
+	urlToKey["https://api.travis-ci.org/config"] = travisOrgPEMPublicKey
+	urlToKey["https://api.travis-ci.com/config"] = travisComPEMPublicKey
+	travisTransport := struct{ MockTransport }{}
+	travisTransport.roundTrip = func(req *http.Request) (*http.Response, error) {
+		if key := urlToKey[req.URL.String()]; key != "" {
+			escKey, _ := json.Marshal(key)
+			return &http.Response{
+				StatusCode: 200,
+				Body: ioutil.NopCloser(bytes.NewBufferString(
+					`{"config":{"notifications":{"webhook":{"public_key":` + string(escKey) + `}}}}`,
+				)),
+			}, nil
+		}
+		return nil, fmt.Errorf("Unhandled URL %s", req.URL.String())
+	}
+	// clobber the http client that the service uses to talk to Travis
+	httpClient = &http.Client{Transport: travisTransport}
+
+	// Intercept message sending to Matrix and mock responses
+	msgs := []matrix.HTMLMessage{}
+	matrixTrans := struct{ MockTransport }{}
+	matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) {
+		if !strings.Contains(req.URL.String(), "/send/m.room.message") {
+			return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String())
+		}
+		var msg matrix.HTMLMessage
+		if err := json.NewDecoder(req.Body).Decode(&msg); err != nil {
+			return nil, err
+		}
+		msgs = append(msgs, msg)
+		return &http.Response{
+			StatusCode: 200,
+			Body:       ioutil.NopCloser(bytes.NewBufferString(`{"event_id":"$yup:event"}`)),
+		}, nil
+	}
+	u, _ := url.Parse("https://hyrule")
+	matrixCli := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@travisci:hyrule")
+
+	// BEGIN running the Travis-CI table tests
+	// ---------------------------------------
+	for _, test := range travisTests {
+		msgs = []matrix.HTMLMessage{} // reset sent messages
+		mockWriter := httptest.NewRecorder()
+		travis := makeService(t, test.Template)
+		if travis == nil {
+			t.Error("TestTravisCI Failed to create service")
+			continue
+		}
+		if err := travis.Register(nil, matrixCli); err != nil {
+			t.Errorf("TestTravisCI Failed to Register(): %s", err)
+			continue
+		}
+		req, err := http.NewRequest(
+			"POST", "https://neb.endpoint/travis-ci-service", bytes.NewBufferString(test.Body),
+		)
+		if err != nil {
+			t.Errorf("TestTravisCI Failed to create webhook request: %s", err)
+			continue
+		}
+		req.Header.Set("Signature", test.Signature)
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		travis.OnReceiveWebhook(mockWriter, req, matrixCli)
+
+		if mockWriter.Code != 200 {
+			t.Errorf("TestTravisCI OnReceiveWebhook want code %d, got %d", 200, mockWriter.Code)
+		}
+	}
+}
+
+func makeService(t *testing.T, template string) *Service {
+	srv, err := types.CreateService("id", ServiceType, "@travisci:hyrule", []byte(
+		`{
+			"rooms":{
+				"!ewfug483gsfe:localhost": {
+					"repos": {
+						"Kegsay/flow-jsdoc": {
+							"template": "`+template+`"
+						}
+					}
+				}
+			}
+		}`,
+	))
+	if err != nil {
+		t.Error("Failed to create Travis-CI service: ", err)
+		return nil
+	}
+	return srv.(*Service)
+}

src/github.com/matrix-org/go-neb/services/travisci/verify.go

@@ -15,12 +15,13 @@ import (
 
 // Host => Public Key.
 // Travis has a .com and .org with different public keys.
+// .org is the public one and is one we will try first, then .com
 var travisPublicKeyMap = map[string]*rsa.PublicKey{
 	"api.travis-ci.org": nil,
 	"api.travis-ci.com": nil,
 }
 
-func verifyOrigin(host string, payload []byte, sigHeader string) error {
+func verifyOrigin(payload []byte, sigHeader string) error {
 	/*
 		From: https://docs.travis-ci.com/user/notifications#Verifying-Webhook-requests
 			 1. Pick up the payload data from the HTTP request’s body.
@@ -30,40 +31,53 @@ func verifyOrigin(host string, payload []byte, sigHeader string) error {
 				the relevant API server. (e.g., https://api.travis-ci.org/config)
 			4. Verify the signature using the public key and SHA1 digest.
 	*/
-	// 2. Get signature bytes
 	sig, err := base64.StdEncoding.DecodeString(sigHeader)
 	if err != nil {
-		return err
+		return fmt.Errorf("verifyOrigin: Failed to decode signature as base64: %s", err)
 	}
 
-	// 3. Get public key
-	pubKey, validURL := travisPublicKeyMap[host]
-	if !validURL {
-		return fmt.Errorf("Invalid host: %s", host)
+	if err := loadPublicKeys(); err != nil {
+		return fmt.Errorf("verifyOrigin: Failed to cache Travis public keys: %s", err)
 	}
-	if pubKey == nil {
-		pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config")
-		if err != nil {
-			return err
-		}
-		block, _ := pem.Decode([]byte(pemPubKey))
-		if block == nil {
-			return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host)
-		}
 
-		k, err := x509.ParsePKIXPublicKey(block.Bytes)
-		if err != nil {
-			return err
+	// 4. Verify with SHA1
+	// NB: We don't know who sent this request (no Referer header or anything) so we need to try
+	//     both public keys at both endpoints. We use the .org one first since it's more popular.
+	var verifyErr error
+	for _, host := range []string{"api.travis-ci.org", "api.travis-ci.com"} {
+		h := sha1.New()
+		h.Write(payload)
+		digest := h.Sum(nil)
+		verifyErr = rsa.VerifyPKCS1v15(travisPublicKeyMap[host], crypto.SHA1, digest, sig)
+		if verifyErr == nil {
+			return nil // Valid for this key
 		}
-		pubKey = k.(*rsa.PublicKey)
-		travisPublicKeyMap[host] = pubKey
 	}
+	return fmt.Errorf("verifyOrigin: Signature verification failed: %s", verifyErr)
+}
 
-	// 4. Verify with SHA1
-	h := sha1.New()
-	h.Write(payload)
-	digest := h.Sum(nil)
-	return rsa.VerifyPKCS1v15(pubKey, crypto.SHA1, digest, sig)
+func loadPublicKeys() error {
+	for _, host := range []string{"api.travis-ci.com", "api.travis-ci.org"} {
+		pubKey := travisPublicKeyMap[host]
+		if pubKey == nil {
+			pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config")
+			if err != nil {
+				return err
+			}
+			block, _ := pem.Decode([]byte(pemPubKey))
+			if block == nil {
+				return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host)
+			}
+
+			k, err := x509.ParsePKIXPublicKey(block.Bytes)
+			if err != nil {
+				return err
+			}
+			pubKey = k.(*rsa.PublicKey)
+			travisPublicKeyMap[host] = pubKey
+		}
+	}
+	return nil
 }
 
 func fetchPEMPublicKey(travisURL string) (key string, err error) {
@@ -88,8 +102,8 @@ func fetchPEMPublicKey(travisURL string) (key string, err error) {
 		return
 	}
 	key = configStruct.Config.Notifications.Webhook.PublicKey
-	if key == "" || strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") {
-		err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key.")
+	if key == "" || !strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") {
+		err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key: %s", key)
 	}
 	return
 }