From 4825c5c4ef982cfdec97cf9ccfb9584f48118be1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 15 Nov 2016 16:46:02 +0000 Subject: [PATCH] Begin to add Travis-CI Service tests Currently this verifies the signature logic, but not the templating logic which isn't implemented yet. --- .../go-neb/services/travisci/travisci.go | 29 ++- .../go-neb/services/travisci/travisci_test.go | 178 ++++++++++++++++++ .../go-neb/services/travisci/verify.go | 70 ++++--- 3 files changed, 239 insertions(+), 38 deletions(-) create mode 100644 src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go index 652a458..fa82a5a 100644 --- a/src/github.com/matrix-org/go-neb/services/travisci/travisci.go +++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci.go @@ -3,7 +3,6 @@ package travisci import ( "encoding/json" "fmt" - "io/ioutil" "net/http" "regexp" "strconv" @@ -85,7 +84,7 @@ type Service struct { type webhookNotification struct { ID int `json:"id"` Number string `json:"number"` - Status *string `json:"status"` // 0 (success) or 1 (incomplete/fail). + Status *int `json:"status"` // 0 (success) or 1 (incomplete/fail). StartedAt *string `json:"started_at"` FinishedAt *string `json:"finished_at"` StatusMessage string `json:"status_message"` @@ -110,6 +109,7 @@ type webhookNotification struct { // The template variables a user can use in their messages type notificationTemplate struct { RepositorySlug string + Repository string // Deprecated: alias for RepositorySlug RepositoryName string BuildNumber string BuildID string @@ -128,6 +128,7 @@ type notificationTemplate struct { func notifToTemplate(n webhookNotification) (t notificationTemplate) { t.RepositorySlug = n.Repository.OwnerName + "/" + n.Repository.Name + t.Repository = t.RepositorySlug t.RepositoryName = n.Repository.Name t.BuildNumber = n.Number t.BuildID = strconv.Itoa(n.ID) @@ -140,7 +141,7 @@ func notifToTemplate(n webhookNotification) (t notificationTemplate) { subjAndMsg := strings.SplitN(n.Message, "\n", 2) t.CommitSubject = subjAndMsg[0] if n.Status != nil { - t.Result = *n.Status + t.Result = strconv.Itoa(*n.Status) } t.Message = n.StatusMessage @@ -182,13 +183,18 @@ func travisToGoTemplate(travisTmpl string) (goTmpl string) { // // See https://docs.travis-ci.com/user/notifications#Webhook-notifications for more information. func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *matrix.Client) { - payload, err := ioutil.ReadAll(req.Body) - if err != nil { - log.WithError(err).Error("Failed to read incoming Travis-CI webhook") - w.WriteHeader(500) + if err := req.ParseForm(); err != nil { + log.WithError(err).Error("Failed to read incoming Travis-CI webhook form") + w.WriteHeader(400) + return + } + payload := req.PostFormValue("payload") + if payload == "" { + log.Error("Travis-CI webhook is missing payload= form value") + w.WriteHeader(400) return } - if err := verifyOrigin(req.Host, payload, req.Header.Get("Signature")); err != nil { + if err := verifyOrigin([]byte(payload), req.Header.Get("Signature")); err != nil { log.WithFields(log.Fields{ "Signature": req.Header.Get("Signature"), log.ErrorKey: err, @@ -198,11 +204,13 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli } var notif webhookNotification - if err := json.Unmarshal(payload, ¬if); err != nil { + if err := json.Unmarshal([]byte(payload), ¬if); err != nil { + log.WithError(err).Error("Travis-CI webhook received an invalid JSON payload=") w.WriteHeader(400) return } - if notif.Repository.URL == "" || notif.Repository.OwnerName == "" || notif.Repository.Name == "" { + if notif.Repository.OwnerName == "" || notif.Repository.Name == "" { + log.WithField("repo", notif.Repository).Error("Travis-CI webhook missing repository fields") w.WriteHeader(400) return } @@ -231,6 +239,7 @@ func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli } } } + w.WriteHeader(200) } // Register makes sure the Config information supplied is valid. diff --git a/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go new file mode 100644 index 0000000..1848a81 --- /dev/null +++ b/src/github.com/matrix-org/go-neb/services/travisci/travisci_test.go @@ -0,0 +1,178 @@ +package travisci + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/matrix-org/go-neb/database" + "github.com/matrix-org/go-neb/matrix" + "github.com/matrix-org/go-neb/types" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +const travisOrgPEMPublicKey = (`-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvtjdLkS+FP+0fPC09j25 +y/PiuYDDivIT86COVedvlElk99BBYTrqNaJybxjXbIZ1Q6xFNhOY+iTcBr4E1zJu +tizF3Xi0V9tOuP/M8Wn4Y/1lCWbQKlWrNQuqNBmhovF4K3mDCYswVbpgTmp+JQYu +Bm9QMdieZMNry5s6aiMA9aSjDlNyedvSENYo18F+NYg1J0C0JiPYTxheCb4optr1 +5xNzFKhAkuGs4XTOA5C7Q06GCKtDNf44s/CVE30KODUxBi0MCKaxiXw/yy55zxX2 +/YdGphIyQiA5iO1986ZmZCLLW8udz9uhW5jUr3Jlp9LbmphAC61bVSf4ou2YsJaN +0QIDAQAB +-----END PUBLIC KEY-----`) + +const travisComPEMPublicKey = (`-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQU2j9lnRtyuW36arNOc +dzCzyKVirLUi3/aLh6UfnTVXzTnx8eHUnBn1ZeQl7Eh3J3qqdbIKl6npS27ONzCy +3PIcfjpLPaVyGagIL8c8XgDEvB45AesC0osVP5gkXQkPUM3B2rrUmp1AZzG+Fuo0 +SAeNnS71gN63U3brL9fN/MTCXJJ6TvMt3GrcJUq5uq56qNeJTsiowK6eiiWFUSfh +e1qapOdMFmcEs9J/R1XQ/scxbAnLcWfl8lqH/MjMdCMe0j3X2ZYMTqOHsb3cQGSS +dMPwZGeLWV+OaxjJ7TrJ+riqMANOgqCBGpvWUnUfo046ACOx7p6u4fFc3aRiuqYK +VQIDAQAB +-----END PUBLIC KEY-----`) + +const exampleSignature = ("pW0CDpmcAeWw3qf2Ufx8UvzfrZRUpYx30HBl9nJcDkh2v9FrF1GjJVsrcqx7ly0FPjb7dkfMJ/d0Q3JpDb1EL4p509cN4Vy8+HpfINw35Wg6JqzOQqTa" + + "TidwoDLXo0NgL78zfiL3dra7ZwOGTA+LmnLSuNp38ROxn70u26uqJzWprGSdVNbRu1LkF1QKLa61NZegfxK7RZn1PlIsznWIyTS0qj81mg2sXMDLH1J4" + + "pHxjEpzydjSb5b8tCjrN+vFLDdAtP5RjU8NwvQM4LRRGbLDIlRsO77HDwfXrPgUE3DjPIqVpHhMcCusygp0ClH2b1J1O3LkhxSS9ol5w99Hkpg==") +const exampleBody = ("payload=%7B%22id%22%3A176075135%2C%22repository%22%3A%7B%22id%22%3A6461770%2C%22name%22%3A%22flow-jsdoc%22%2C%22owner_" + + "name%22%3A%22Kegsay%22%2C%22url%22%3Anull%7D%2C%22number%22%3A%2218%22%2C%22config%22%3A%7B%22notifications%22%3A%7B%22web" + + "hooks%22%3A%5B%22http%3A%2F%2F7abbe705.ngrok.io%22%5D%7D%2C%22language%22%3A%22node_js%22%2C%22node_js%22%3A%5B%224.1%22%5D%2C%22.resu" + + "lt%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22precise%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22status_" + + "message%22%3A%22Passed%22%2C%22result_message%22%3A%22Passed%22%2C%22started_at%22%3A%222016-11-15T15%3A10%3A22Z%22%2C%22finished_" + + "at%22%3A%222016-11-15T15%3A10%3A54Z%22%2C%22duration%22%3A32%2C%22build_url%22%3A%22https%3A%2F%2Ftravis-ci.org%2FKegsay%2Fflow-js" + + "doc%2Fbuilds%2F176075135%22%2C%22commit_id%22%3A50222535%2C%22commit%22%3A%223a092c3a6032ebb50384c99b445f947e9ce86e2a%22%2C%22base_com" + + "mit%22%3Anull%2C%22head_commit%22%3Anull%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22Test+Travis+webhook+support%22%2C%22compare_" + + "url%22%3A%22https%3A%2F%2Fgithub.com%2FKegsay%2Fflow-jsdoc%2Fcompare%2F9f9d459ba082...3a092c3a6032%22%2C%22committed_at%22%3A%222016-1" + + "1-15T15%3A08%3A16Z%22%2C%22author_name%22%3A%22Kegan+Dougal%22%2C%22author_email%22%3A%22kegan%40matrix.org%22%2C%22committer_" + + "name%22%3A%22Kegan+Dougal%22%2C%22committer_email%22%3A%22kegan%40matrix.org%22%2C%22matrix%22%3A%5B%7B%22id%22%3A176075137%2C%22reposit" + + "ory_id%22%3A6461770%2C%22parent_id%22%3A176075135%2C%22number%22%3A%2218.1%22%2C%22state%22%3A%22finished%22%2C%22config%22%3A%7B%22notifi" + + "cations%22%3A%7B%22webhooks%22%3A%5B%22http%3A%2F%2F7abbe705.ngrok.io%22%5D%7D%2C%22language%22%3A%22node_js%22%2C%22node_" + + "js%22%3A%224.1%22%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22precise%22%2C%22os%22%3A%22li" + + "nux%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22commit%22%3A%223a092c3a6032ebb50384c99b445f947e9ce86e2a%22%2C%22branch%22%3A%22mas" + + "ter%22%2C%22message%22%3A%22Test+Travis+webhook+support%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2FKegsay%2Fflow-jsdoc%2Fcomp" + + "are%2F9f9d459ba082...3a092c3a6032%22%2C%22started_at%22%3A%222016-11-15T15%3A10%3A22Z%22%2C%22finished_at%22%3A%222016-11-" + + "15T15%3A10%3A54Z%22%2C%22committed_at%22%3A%222016-11-15T15%3A08%3A16Z%22%2C%22author_name%22%3A%22Kegan+Dougal%22%2C%22author_ema" + + "il%22%3A%22kegan%40matrix.org%22%2C%22committer_name%22%3A%22Kegan+Dougal%22%2C%22committer_email%22%3A%22kegan%40matrix.org%22%2C%22allow_f" + + "ailure%22%3Afalse%7D%5D%2C%22type%22%3A%22push%22%2C%22state%22%3A%22passed%22%2C%22pull_request%22%3Afalse%2C%22pull_request_number%22%3Anu" + + "ll%2C%22pull_request_title%22%3Anull%2C%22tag%22%3Anull%7D") + +var travisTests = []struct { + Signature string + ValidSignature bool + Body string + Template string + ExpectedOutput string +}{ + { + exampleSignature, true, exampleBody, + "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}", + "Kegsay/flow-jsdoc#22 master - 3a092c3a6032ebb50384c99b445f947e9ce86e2a : Kegan Dougal: Test Travis webhook support", + }, +} + +type MockTransport struct { + roundTrip func(*http.Request) (*http.Response, error) +} + +func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return t.roundTrip(req) +} + +func TestTravisCI(t *testing.T) { + database.SetServiceDB(&database.NopStorage{}) + + // When the service tries to get Travis' public key, return the constant + urlToKey := make(map[string]string) + urlToKey["https://api.travis-ci.org/config"] = travisOrgPEMPublicKey + urlToKey["https://api.travis-ci.com/config"] = travisComPEMPublicKey + travisTransport := struct{ MockTransport }{} + travisTransport.roundTrip = func(req *http.Request) (*http.Response, error) { + if key := urlToKey[req.URL.String()]; key != "" { + escKey, _ := json.Marshal(key) + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString( + `{"config":{"notifications":{"webhook":{"public_key":` + string(escKey) + `}}}}`, + )), + }, nil + } + return nil, fmt.Errorf("Unhandled URL %s", req.URL.String()) + } + // clobber the http client that the service uses to talk to Travis + httpClient = &http.Client{Transport: travisTransport} + + // Intercept message sending to Matrix and mock responses + msgs := []matrix.HTMLMessage{} + matrixTrans := struct{ MockTransport }{} + matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) { + if !strings.Contains(req.URL.String(), "/send/m.room.message") { + return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String()) + } + var msg matrix.HTMLMessage + if err := json.NewDecoder(req.Body).Decode(&msg); err != nil { + return nil, err + } + msgs = append(msgs, msg) + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(`{"event_id":"$yup:event"}`)), + }, nil + } + u, _ := url.Parse("https://hyrule") + matrixCli := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@travisci:hyrule") + + // BEGIN running the Travis-CI table tests + // --------------------------------------- + for _, test := range travisTests { + msgs = []matrix.HTMLMessage{} // reset sent messages + mockWriter := httptest.NewRecorder() + travis := makeService(t, test.Template) + if travis == nil { + t.Error("TestTravisCI Failed to create service") + continue + } + if err := travis.Register(nil, matrixCli); err != nil { + t.Errorf("TestTravisCI Failed to Register(): %s", err) + continue + } + req, err := http.NewRequest( + "POST", "https://neb.endpoint/travis-ci-service", bytes.NewBufferString(test.Body), + ) + if err != nil { + t.Errorf("TestTravisCI Failed to create webhook request: %s", err) + continue + } + req.Header.Set("Signature", test.Signature) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + travis.OnReceiveWebhook(mockWriter, req, matrixCli) + + if mockWriter.Code != 200 { + t.Errorf("TestTravisCI OnReceiveWebhook want code %d, got %d", 200, mockWriter.Code) + } + } +} + +func makeService(t *testing.T, template string) *Service { + srv, err := types.CreateService("id", ServiceType, "@travisci:hyrule", []byte( + `{ + "rooms":{ + "!ewfug483gsfe:localhost": { + "repos": { + "Kegsay/flow-jsdoc": { + "template": "`+template+`" + } + } + } + } + }`, + )) + if err != nil { + t.Error("Failed to create Travis-CI service: ", err) + return nil + } + return srv.(*Service) +} diff --git a/src/github.com/matrix-org/go-neb/services/travisci/verify.go b/src/github.com/matrix-org/go-neb/services/travisci/verify.go index 9454c2b..dfde078 100644 --- a/src/github.com/matrix-org/go-neb/services/travisci/verify.go +++ b/src/github.com/matrix-org/go-neb/services/travisci/verify.go @@ -15,12 +15,13 @@ import ( // Host => Public Key. // Travis has a .com and .org with different public keys. +// .org is the public one and is one we will try first, then .com var travisPublicKeyMap = map[string]*rsa.PublicKey{ "api.travis-ci.org": nil, "api.travis-ci.com": nil, } -func verifyOrigin(host string, payload []byte, sigHeader string) error { +func verifyOrigin(payload []byte, sigHeader string) error { /* From: https://docs.travis-ci.com/user/notifications#Verifying-Webhook-requests 1. Pick up the payload data from the HTTP request’s body. @@ -30,40 +31,53 @@ func verifyOrigin(host string, payload []byte, sigHeader string) error { the relevant API server. (e.g., https://api.travis-ci.org/config) 4. Verify the signature using the public key and SHA1 digest. */ - // 2. Get signature bytes sig, err := base64.StdEncoding.DecodeString(sigHeader) if err != nil { - return err + return fmt.Errorf("verifyOrigin: Failed to decode signature as base64: %s", err) } - // 3. Get public key - pubKey, validURL := travisPublicKeyMap[host] - if !validURL { - return fmt.Errorf("Invalid host: %s", host) + if err := loadPublicKeys(); err != nil { + return fmt.Errorf("verifyOrigin: Failed to cache Travis public keys: %s", err) } - if pubKey == nil { - pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config") - if err != nil { - return err - } - block, _ := pem.Decode([]byte(pemPubKey)) - if block == nil { - return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host) - } - k, err := x509.ParsePKIXPublicKey(block.Bytes) - if err != nil { - return err + // 4. Verify with SHA1 + // NB: We don't know who sent this request (no Referer header or anything) so we need to try + // both public keys at both endpoints. We use the .org one first since it's more popular. + var verifyErr error + for _, host := range []string{"api.travis-ci.org", "api.travis-ci.com"} { + h := sha1.New() + h.Write(payload) + digest := h.Sum(nil) + verifyErr = rsa.VerifyPKCS1v15(travisPublicKeyMap[host], crypto.SHA1, digest, sig) + if verifyErr == nil { + return nil // Valid for this key } - pubKey = k.(*rsa.PublicKey) - travisPublicKeyMap[host] = pubKey } + return fmt.Errorf("verifyOrigin: Signature verification failed: %s", verifyErr) +} - // 4. Verify with SHA1 - h := sha1.New() - h.Write(payload) - digest := h.Sum(nil) - return rsa.VerifyPKCS1v15(pubKey, crypto.SHA1, digest, sig) +func loadPublicKeys() error { + for _, host := range []string{"api.travis-ci.com", "api.travis-ci.org"} { + pubKey := travisPublicKeyMap[host] + if pubKey == nil { + pemPubKey, err := fetchPEMPublicKey("https://" + host + "/config") + if err != nil { + return err + } + block, _ := pem.Decode([]byte(pemPubKey)) + if block == nil { + return fmt.Errorf("public_key at %s doesn't have a valid PEM block", host) + } + + k, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + pubKey = k.(*rsa.PublicKey) + travisPublicKeyMap[host] = pubKey + } + } + return nil } func fetchPEMPublicKey(travisURL string) (key string, err error) { @@ -88,8 +102,8 @@ func fetchPEMPublicKey(travisURL string) (key string, err error) { return } key = configStruct.Config.Notifications.Webhook.PublicKey - if key == "" || strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") { - err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key.") + if key == "" || !strings.HasPrefix(key, "-----BEGIN PUBLIC KEY-----") { + err = fmt.Errorf("Couldn't fetch Travis-CI public key. Missing or malformed key: %s", key) } return }