From a2ffc20350617689d42c0093e6b71a453eb686b2 Mon Sep 17 00:00:00 2001 From: MTRNord Date: Tue, 12 Dec 2017 15:57:25 +0100 Subject: [PATCH] Implement CircleCI Webhook Service --- .../go-neb/services/circleci/circleci.go | 248 ++++++++++++++++++ .../go-neb/services/circleci/circleci_test.go | 141 ++++++++++ .../go-neb/services/circleci/types.go | 44 ++++ 3 files changed, 433 insertions(+) create mode 100644 src/github.com/matrix-org/go-neb/services/circleci/circleci.go create mode 100644 src/github.com/matrix-org/go-neb/services/circleci/circleci_test.go create mode 100644 src/github.com/matrix-org/go-neb/services/circleci/types.go diff --git a/src/github.com/matrix-org/go-neb/services/circleci/circleci.go b/src/github.com/matrix-org/go-neb/services/circleci/circleci.go new file mode 100644 index 0000000..a0f3c7d --- /dev/null +++ b/src/github.com/matrix-org/go-neb/services/circleci/circleci.go @@ -0,0 +1,248 @@ +// Package circleci implements a Service capable of processing webhooks from CircleCI. +package circleci + +import ( + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/matrix-org/go-neb/database" + "github.com/matrix-org/go-neb/types" + "github.com/matrix-org/gomatrix" + "io/ioutil" +) + +// ServiceType of the Travis-CI service. +const ServiceType = "circleci" + +// DefaultTemplate contains the template that will be used if none is supplied. +// This matches the default mentioned at: https://docs.travis-ci.com/user/notifications#Customizing-slack-notifications +const DefaultTemplate = (`%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} + Build details : %{build_url}`) + +// Matches 'owner/repo' +var ownerRepoRegex = regexp.MustCompile(`^([A-z0-9-_.]+)/([A-z0-9-_.]+)$`) + +var httpClient = &http.Client{} + +// Service contains the Config fields for the Travis-CI service. +// +// This service will send notifications into a Matrix room when CircleCI sends +// webhook events to it. It requires a public domain which CircleCI can reach. +// Notices will be sent as the service user ID. +// +// Example JSON request: +// { +// rooms: { +// "!ewfug483gsfe:localhost": { +// repos: { +// "matrix-org/go-neb": { +// template: "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}\nBuild details : %{build_url}" +// } +// } +// } +// } +// } +type Service struct { + types.DefaultService + webhookEndpointURL string + // The URL which should be added to .circleci/config.yml - Populated by Go-NEB after Service registration. + WebhookURL string `json:"webhook_url"` + // A map from Matrix room ID to Github-style owner/repo repositories. + Rooms map[string]struct { + // A map of "owner/repo" to configuration information + Repos map[string]struct { + // The template string to use when creating notifications. + // + // This is identical to the format of Slack Notifications for Travis-CI: + // https://docs.travis-ci.com/user/notifications#Customizing-slack-notifications + // + // The following variables are available: + // repository_slug: your Git* repo identifier (like svenfuchs/minimal) + // repository_name: the slug without the username + // build_number: build number + // build_id: build id + // branch: branch build name + // commit: shortened commit SHA + // author: commit author name + // commit_message: commit message of build + // commit_subject: first line of the commit message + // result: result of build + // message: CircleCI message to the build + // duration: total duration of all builds in the matrix + // elapsed_time: time between build start and finish + // compare_url: commit change view URL + // build_url: URL of the build detail + Template string `json:"template"` + } `json:"repos"` + } `json:"rooms"` +} + +// Converts a webhook notification into a map of template var name to value +func notifToTemplate(n WebhookNotification) map[string]string { + t := make(map[string]string) + //Get Payload to variable + p := n.Payload + t["repository_slug"] = p.Username + "/" + p.Reponame + t["repository"] = t["repository_slug"] // Deprecated form but still used everywhere in people's templates + t["repository_name"] = p.Reponame + t["build_number"] = string(p.BuildNum) + t["build_id"] = t["build_number"] // CircleCI doesn't have a difference between number and ID but to be consistent with TravisCI + t["branch"] = p.Branch + shaLength := len(p.VcsRevision) + if shaLength > 10 { + shaLength = 10 + } + t["commit"] = p.VcsRevision[:shaLength] // shortened commit SHA + t["author"] = p.CommitterName // author: commit author name + // commit_message: commit message of build + // commit_subject: first line of the commit message + t["commit_message"] = p.Body + subjAndMsg := strings.SplitN(p.Body, "\n", 2) + t["commit_subject"] = subjAndMsg[0] + if p.Status != "" { + t["result"] = p.Status + } + t["message"] = p.Outcome // message: Travis CI message to the build + + if !p.StartTime.IsZero() && !p.StopTime.IsZero() { + t["duration"] = p.StopTime.Sub(p.StartTime).String() + t["elapsed_time"] = t["duration"] + } + + t["build_url"] = p.BuildURL + return t +} + +func outputForTemplate(circleciTmpl string, tmpl map[string]string) (out string) { + if circleciTmpl == "" { + circleciTmpl = DefaultTemplate + } + out = circleciTmpl + for tmplVar, tmplValue := range tmpl { + out = strings.Replace(out, "%{"+tmplVar+"}", tmplValue, -1) + } + return out +} + +// OnReceiveWebhook receives requests from CircleCI and possibly sends requests to Matrix as a result. +// +// If the repository matches a known Git* repository, a notification will be formed from the +// template for that repository and a notice will be sent to Matrix. +// +// Go-NEB cannot register with CircleCI for webhooks automatically. The user must manually add the +// webhook endpoint URL to their .circleci/config.yml file: +// notify: +// webhooks: +// - url: https://example.com/hooks/circle +// +// See https://circleci.com/docs/1.0/configuration/#notify for more information. +func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client) { + body, err := ioutil.ReadAll(req.Body) + if err != nil { + log.WithFields(log.Fields{ + "Body": req.Body, + log.ErrorKey: err, + }).Warn("Failed to Read Body") + w.WriteHeader(403) + return + } + //fmt.Printf("%s\n", body) + + var notif WebhookNotification + if err := json.Unmarshal(body, ¬if); err != nil { + log.WithError(err).Error("CircleCI webhook received an invalid JSON body") + w.WriteHeader(400) + return + } + if notif.Payload.Username == "" || notif.Payload.Reponame == "" { + log.WithField("repo", notif.Payload).Error("CircleCI webhook missing repository fields") + w.WriteHeader(400) + return + } + whForRepo := notif.Payload.Username + "/" + notif.Payload.Reponame + tmplData := notifToTemplate(notif) + + logger := log.WithFields(log.Fields{ + "repo": whForRepo, + }) + + for roomID, roomData := range s.Rooms { + for ownerRepo, repoData := range roomData.Repos { + if ownerRepo != whForRepo { + continue + } + msg := gomatrix.TextMessage{ + Body: outputForTemplate(repoData.Template, tmplData), + MsgType: "m.notice", + } + + logger.WithFields(log.Fields{ + "message": msg, + "room_id": roomID, + }).Print("Sending CircleCI notification to room") + if _, e := cli.SendMessageEvent(roomID, "m.room.message", msg); e != nil { + logger.WithError(e).WithField("room_id", roomID).Print( + "Failed to send CircleCI notification to room.") + } + } + } + w.WriteHeader(200) +} + +// Register makes sure the Config information supplied is valid. +func (s *Service) Register(oldService types.Service, client *gomatrix.Client) error { + s.WebhookURL = s.webhookEndpointURL + for _, roomData := range s.Rooms { + for repo := range roomData.Repos { + match := ownerRepoRegex.FindStringSubmatch(repo) + if len(match) == 0 { + return fmt.Errorf("Repository '%s' is not a valid repository name.", repo) + } + } + } + s.joinRooms(client) + return nil +} + +// PostRegister deletes this service if there are no registered repos. +func (s *Service) PostRegister(oldService types.Service) { + for _, roomData := range s.Rooms { + for _ = range roomData.Repos { + return // at least 1 repo exists + } + } + // Delete this service since no repos are configured + logger := log.WithFields(log.Fields{ + "service_type": s.ServiceType(), + "service_id": s.ServiceID(), + }) + logger.Info("Removing service as no repositories are registered.") + if err := database.GetServiceDB().DeleteService(s.ServiceID()); err != nil { + logger.WithError(err).Error("Failed to delete service") + } +} + +func (s *Service) joinRooms(client *gomatrix.Client) { + for roomID := range s.Rooms { + if _, err := client.JoinRoom(roomID, "", nil); err != nil { + log.WithFields(log.Fields{ + log.ErrorKey: err, + "room_id": roomID, + "user_id": client.UserID, + }).Error("Failed to join room") + } + } +} + +func init() { + types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service { + return &Service{ + DefaultService: types.NewDefaultService(serviceID, serviceUserID, ServiceType), + webhookEndpointURL: webhookEndpointURL, + } + }) +} diff --git a/src/github.com/matrix-org/go-neb/services/circleci/circleci_test.go b/src/github.com/matrix-org/go-neb/services/circleci/circleci_test.go new file mode 100644 index 0000000..453f49e --- /dev/null +++ b/src/github.com/matrix-org/go-neb/services/circleci/circleci_test.go @@ -0,0 +1,141 @@ +package circleci + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/matrix-org/go-neb/database" + "github.com/matrix-org/go-neb/testutils" + "github.com/matrix-org/go-neb/types" + "github.com/matrix-org/gomatrix" + "net/url" +) + +const exampleBody = ("%7B%22payload%22%3A%7B%22vcs_url%22%3A%22https%3A%2F%2Fgithub.com%2Fcircleci%2Fmongofinil%22%2C%22build_url%22%3A%22https%3A%2F%2Fcircleci.com%2Fgh%2Fcircleci%2Fmongofinil%2F22%22%2C%22build_num%22%3A22%2C%22branch%22%3A%22master%22%2C%22vcs_revision%22%3A%221d231626ba1d2838e599c5c598d28e2306ad4e48%22%2C%22committer_name%22%3A%22Allen%20Rohner%22%2C%22committer_email%22%3A%22arohner%40gmail.com%22%2C%22subject%22%3A%22Don%27t%20explode%20when%20the%20system%20clock%20shifts%20backwards%22%2C%22body%22%3A%22%22%2C%22why%22%3A%22github%22%2C%22dont_build%22%3Anull%2C%22queued_at%22%3A%222013-02-12T21%3A33%3A30Z%22%2C%22start_time%22%3A%222013-02-12T21%3A33%3A38Z%22%2C%22stop_time%22%3A%222013-02-12T21%3A34%3A01Z%22%2C%22build_time_millis%22%3A23505%2C%22username%22%3A%22circleci%22%2C%22reponame%22%3A%22mongofinil%22%2C%22lifecycle%22%3A%22finished%22%2C%22outcome%22%3A%22success%22%2C%22status%22%3A%22success%22%2C%22retry_of%22%3Anull%2C%22steps%22%3A%5B%7B%22name%22%3A%22configure%20the%20build%22%2C%22actions%22%3A%5B%7B%22bash_command%22%3Anull%2C%22run_time_millis%22%3A1646%2C%22start_time%22%3A%222013-02-12T21%3A33%3A38Z%22%2C%22end_time%22%3A%222013-02-12T21%3A33%3A39Z%22%2C%22name%22%3A%22configure%20the%20build%22%2C%22exit_code%22%3Anull%2C%22type%22%3A%22infrastructure%22%2C%22index%22%3A0%2C%22status%22%3A%22success%22%7D%5D%7D%2C%7B%22name%22%3A%22lein2%20deps%22%2C%22actions%22%3A%5B%7B%22bash_command%22%3A%22lein2%20deps%22%2C%22run_time_millis%22%3A7555%2C%22start_time%22%3A%222013-02-12T21%3A33%3A47Z%22%2C%22messages%22%3A%5B%5D%2C%22step%22%3A1%2C%22exit_code%22%3A0%2C%22end_time%22%3A%222013-02-12T21%3A33%3A54Z%22%2C%22index%22%3A0%2C%22status%22%3A%22success%22%2C%22type%22%3A%22dependencies%22%2C%22source%22%3A%22inference%22%2C%22failed%22%3Anull%7D%5D%7D%2C%7B%22name%22%3A%22lein2%20trampoline%20midje%22%2C%22actions%22%3A%5B%7B%22bash_command%22%3A%22lein2%20trampoline%20midje%22%2C%22run_time_millis%22%3A2310%2C%22continue%22%3Anull%2C%22parallel%22%3Atrue%2C%22start_time%22%3A%222013-02-12T21%3A33%3A59Z%22%2C%22name%22%3A%22lein2%20trampoline%20midje%22%2C%22messages%22%3A%5B%5D%2C%22step%22%3A6%2C%22exit_code%22%3A1%2C%22end_time%22%3A%222013-02-12T21%3A34%3A01Z%22%2C%22index%22%3A0%2C%22status%22%3A%22failed%22%2C%22timedout%22%3Anull%2C%22infrastructure_fail%22%3Anull%2C%22type%22%3A%22test%22%2C%22source%22%3A%22inference%22%2C%22failed%22%3Atrue%7D%5D%7D%5D%7D%7D") +var circleciTests = []struct { + Body string + Template string + ExpectedOutput string +}{ + { + exampleBody, + "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}", + "circleci/mongofinil#22 (master - 1d231626ba : Allen Rohner): success", + }, + { + exampleBody, + "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}", + "circleci/mongofinil#22 (master - 1d231626ba : Allen Rohner): success", + }, + { + strings.TrimSuffix(exampleBody, "%7D") + "%2C%22EXTRA_KEY%22%3Anull%7D", + "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}", + "circleci/mongofinil#22 (master - 1d231626ba : Allen Rohner): success", + }, + { + exampleBody, + "%{repository}#%{build_number} %{duration}", + "circleci/mongofinil#22 23s", + }, +} + +func TestCircleCI(t *testing.T) { + database.SetServiceDB(&database.NopStorage{}) + + // Intercept message sending to Matrix and mock responses + msgs := []gomatrix.TextMessage{} + matrixTrans := struct{ testutils.MockTransport }{} + matrixTrans.RT = func(req *http.Request) (*http.Response, error) { + if !strings.Contains(req.URL.String(), "/send/m.room.message") { + return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String()) + } + var msg gomatrix.TextMessage + if err := json.NewDecoder(req.Body).Decode(&msg); err != nil { + return nil, fmt.Errorf("Failed to decode request JSON: %s", err) + } + msgs = append(msgs, msg) + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(`{"event_id":"$yup:event"}`)), + }, nil + } + matrixCli, _ := gomatrix.NewClient("https://hyrule", "@travisci:hyrule", "its_a_secret") + matrixCli.Client = &http.Client{Transport: matrixTrans} + + // BEGIN running the CircleCI table tests + // --------------------------------------- + for _, test := range circleciTests { + msgs = []gomatrix.TextMessage{} // reset sent messages + mockWriter := httptest.NewRecorder() + circleci := makeService(t, test.Template) + if circleci == nil { + t.Error("TestCircleCI Failed to create service") + continue + } + if err := circleci.Register(nil, matrixCli); err != nil { + t.Errorf("TestCircleCI Failed to Register(): %s", err) + continue + } + correctBody, convErr := url.QueryUnescape(test.Body) + if convErr != nil { + t.Errorf("TestCircleCI Failed to UnUrlEncode Test Body: %s", convErr) + continue + } + req, err := http.NewRequest( + "POST", "https://neb.endpoint/circleci-service", bytes.NewBufferString(correctBody), + ) + if err != nil { + t.Errorf("TestCircleCI Failed to create webhook request: %s", err) + continue + } + req.Header.Set("Content-Type", "application/json") + circleci.OnReceiveWebhook(mockWriter, req, matrixCli) + + if !assertResponse(t, mockWriter, msgs, 200, 1) { + continue + } + + if msgs[0].Body != test.ExpectedOutput { + t.Errorf("TestCircleCI want matrix body '%s', got '%s'", test.ExpectedOutput, msgs[0].Body) + } + } +} + +func assertResponse(t *testing.T, w *httptest.ResponseRecorder, msgs []gomatrix.TextMessage, expectCode int, expectMsgLength int) bool { + if w.Code != expectCode { + t.Errorf("TestCircleCI OnReceiveWebhook want HTTP code %d, got %d", expectCode, w.Code) + return false + } + if len(msgs) != expectMsgLength { + t.Errorf("TestCircleCI want %d sent messages, got %d ", expectMsgLength, len(msgs)) + return false + } + return true +} + +func makeService(t *testing.T, template string) *Service { + srv, err := types.CreateService("id", ServiceType, "@travisci:hyrule", []byte( + `{ + "rooms":{ + "!ewfug483gsfe:localhost": { + "repos": { + "circleci/mongofinil": { + "template": "`+template+`" + } + } + } + } + }`, + )) + if err != nil { + t.Error("Failed to create CircleCI service: ", err) + return nil + } + return srv.(*Service) +} diff --git a/src/github.com/matrix-org/go-neb/services/circleci/types.go b/src/github.com/matrix-org/go-neb/services/circleci/types.go new file mode 100644 index 0000000..c58a124 --- /dev/null +++ b/src/github.com/matrix-org/go-neb/services/circleci/types.go @@ -0,0 +1,44 @@ +package circleci + +import "time" + +// WebhookNotification is the response of a webhook notification by circleCI +type WebhookNotification struct { + Payload struct { + VcsURL string `json:"vcs_url,omitempty"` + BuildURL string `json:"build_url,omitempty"` + BuildNum int `json:"build_num,omitempty"` + Branch string `json:"branch,omitempty"` + VcsRevision string `json:"vcs_revision,omitempty"` + CommitterName string `json:"committer_name,omitempty"` + CommitterEmail string `json:"committer_email,omitempty"` + Subject string `json:"subject,omitempty"` + Body string `json:"body,omitempty"` + Why string `json:"why,omitempty"` + DontBuild interface{} `json:"dont_build,omitempty"` + QueuedAt time.Time `json:"queued_at,omitempty"` + StartTime time.Time `json:"start_time,omitempty"` + StopTime time.Time `json:"stop_time,omitempty"` + BuildTimeMillis int `json:"build_time_millis,omitempty"` + Username string `json:"username,omitempty"` + Reponame string `json:"reponame,omitempty"` + Lifecycle string `json:"lifecycle,omitempty"` + Outcome string `json:"outcome,omitempty"` + Status string `json:"status,omitempty"` + RetryOf interface{} `json:"retry_of,omitempty"` + Steps []struct { + Name string `json:"name,omitempty"` + Actions []struct { + BashCommand interface{} `json:"bash_command,omitempty"` + RunTimeMillis int `json:"run_time_millis,omitempty"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + Name string `json:"name,omitempty"` + ExitCode interface{} `json:"exit_code,omitempty"` + Type string `json:"type,omitempty"` + Index int `json:"index,omitempty"` + Status string `json:"status,omitempty"` + } `json:"actions,omitempty"` + } `json:"steps,omitempty"` + } `json:"payload"` +} \ No newline at end of file