Implement CircleCI Webhook Service

src/github.com/matrix-org/go-neb/services/circleci/circleci.go

@@ -0,0 +1,248 @@
+// Package circleci implements a Service capable of processing webhooks from CircleCI.
+package circleci
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
+	"io/ioutil"
+)
+
+// ServiceType of the Travis-CI service.
+const ServiceType = "circleci"
+
+// DefaultTemplate contains the template that will be used if none is supplied.
+// This matches the default mentioned at: https://docs.travis-ci.com/user/notifications#Customizing-slack-notifications
+const DefaultTemplate = (`%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}
+	Build details : %{build_url}`)
+
+// Matches 'owner/repo'
+var ownerRepoRegex = regexp.MustCompile(`^([A-z0-9-_.]+)/([A-z0-9-_.]+)$`)
+
+var httpClient = &http.Client{}
+
+// Service contains the Config fields for the Travis-CI service.
+//
+// This service will send notifications into a Matrix room when CircleCI sends
+// webhook events to it. It requires a public domain which CircleCI can reach.
+// Notices will be sent as the service user ID.
+//
+// Example JSON request:
+//   {
+//       rooms: {
+//           "!ewfug483gsfe:localhost": {
+//               repos: {
+//                   "matrix-org/go-neb": {
+//                       template: "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}\nBuild details : %{build_url}"
+//                   }
+//               }
+//           }
+//       }
+//   }
+type Service struct {
+	types.DefaultService
+	webhookEndpointURL string
+	// The URL which should be added to .circleci/config.yml - Populated by Go-NEB after Service registration.
+	WebhookURL string `json:"webhook_url"`
+	// A map from Matrix room ID to Github-style owner/repo repositories.
+	Rooms map[string]struct {
+		// A map of "owner/repo" to configuration information
+		Repos map[string]struct {
+			// The template string to use when creating notifications.
+			//
+			// This is identical to the format of Slack Notifications for Travis-CI:
+			// https://docs.travis-ci.com/user/notifications#Customizing-slack-notifications
+			//
+			// The following variables are available:
+			//   repository_slug: your Git* repo identifier (like svenfuchs/minimal)
+			//   repository_name: the slug without the username
+			//   build_number: build number
+			//   build_id: build id
+			//   branch: branch build name
+			//   commit: shortened commit SHA
+			//   author: commit author name
+			//   commit_message: commit message of build
+			//   commit_subject: first line of the commit message
+			//   result: result of build
+			//   message: CircleCI message to the build
+			//   duration: total duration of all builds in the matrix
+			//   elapsed_time: time between build start and finish
+			//   compare_url: commit change view URL
+			//   build_url: URL of the build detail
+			Template string `json:"template"`
+		} `json:"repos"`
+	} `json:"rooms"`
+}
+
+// Converts a webhook notification into a map of template var name to value
+func notifToTemplate(n WebhookNotification) map[string]string {
+	t := make(map[string]string)
+	//Get Payload to variable
+	p := n.Payload
+	t["repository_slug"] = p.Username + "/" + p.Reponame
+	t["repository"] = t["repository_slug"] // Deprecated form but still used everywhere in people's templates
+	t["repository_name"] = p.Reponame
+	t["build_number"] = string(p.BuildNum)
+	t["build_id"] = t["build_number"] // CircleCI doesn't have a difference between number and ID but to be consistent with TravisCI
+	t["branch"] = p.Branch
+	shaLength := len(p.VcsRevision)
+	if shaLength > 10 {
+		shaLength = 10
+	}
+	t["commit"] = p.VcsRevision[:shaLength] // shortened commit SHA
+	t["author"] = p.CommitterName      // author: commit author name
+	// commit_message: commit message of build
+	// commit_subject: first line of the commit message
+	t["commit_message"] = p.Body
+	subjAndMsg := strings.SplitN(p.Body, "\n", 2)
+	t["commit_subject"] = subjAndMsg[0]
+	if p.Status != "" {
+		t["result"] = p.Status
+	}
+	t["message"] = p.Outcome // message: Travis CI message to the build
+
+	if !p.StartTime.IsZero()  && !p.StopTime.IsZero() {
+			t["duration"] = p.StopTime.Sub(p.StartTime).String()
+			t["elapsed_time"] = t["duration"]
+	}
+
+	t["build_url"] = p.BuildURL
+	return t
+}
+
+func outputForTemplate(circleciTmpl string, tmpl map[string]string) (out string) {
+	if circleciTmpl == "" {
+		circleciTmpl = DefaultTemplate
+	}
+	out = circleciTmpl
+	for tmplVar, tmplValue := range tmpl {
+		out = strings.Replace(out, "%{"+tmplVar+"}", tmplValue, -1)
+	}
+	return out
+}
+
+// OnReceiveWebhook receives requests from CircleCI and possibly sends requests to Matrix as a result.
+//
+// If the repository matches a known Git* repository, a notification will be formed from the
+// template for that repository and a notice will be sent to Matrix.
+//
+// Go-NEB cannot register with CircleCI for webhooks automatically. The user must manually add the
+// webhook endpoint URL to their .circleci/config.yml file:
+//    notify:
+//		webhooks:
+//			- url: https://example.com/hooks/circle
+//
+// See https://circleci.com/docs/1.0/configuration/#notify for more information.
+func (s *Service) OnReceiveWebhook(w http.ResponseWriter, req *http.Request, cli *gomatrix.Client) {
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"Body":  req.Body,
+			log.ErrorKey: err,
+		}).Warn("Failed to Read Body")
+		w.WriteHeader(403)
+		return
+	}
+	//fmt.Printf("%s\n", body)
+
+	var notif WebhookNotification
+	if err := json.Unmarshal(body, &notif); err != nil {
+		log.WithError(err).Error("CircleCI webhook received an invalid JSON body")
+		w.WriteHeader(400)
+		return
+	}
+	if notif.Payload.Username == "" || notif.Payload.Reponame == "" {
+		log.WithField("repo", notif.Payload).Error("CircleCI webhook missing repository fields")
+		w.WriteHeader(400)
+		return
+	}
+	whForRepo := notif.Payload.Username + "/" + notif.Payload.Reponame
+	tmplData := notifToTemplate(notif)
+
+	logger := log.WithFields(log.Fields{
+		"repo": whForRepo,
+	})
+
+	for roomID, roomData := range s.Rooms {
+		for ownerRepo, repoData := range roomData.Repos {
+			if ownerRepo != whForRepo {
+				continue
+			}
+			msg := gomatrix.TextMessage{
+				Body:    outputForTemplate(repoData.Template, tmplData),
+				MsgType: "m.notice",
+			}
+
+			logger.WithFields(log.Fields{
+				"message": msg,
+				"room_id": roomID,
+			}).Print("Sending CircleCI notification to room")
+			if _, e := cli.SendMessageEvent(roomID, "m.room.message", msg); e != nil {
+				logger.WithError(e).WithField("room_id", roomID).Print(
+					"Failed to send CircleCI notification to room.")
+			}
+		}
+	}
+	w.WriteHeader(200)
+}
+
+// Register makes sure the Config information supplied is valid.
+func (s *Service) Register(oldService types.Service, client *gomatrix.Client) error {
+	s.WebhookURL = s.webhookEndpointURL
+	for _, roomData := range s.Rooms {
+		for repo := range roomData.Repos {
+			match := ownerRepoRegex.FindStringSubmatch(repo)
+			if len(match) == 0 {
+				return fmt.Errorf("Repository '%s' is not a valid repository name.", repo)
+			}
+		}
+	}
+	s.joinRooms(client)
+	return nil
+}
+
+// PostRegister deletes this service if there are no registered repos.
+func (s *Service) PostRegister(oldService types.Service) {
+	for _, roomData := range s.Rooms {
+		for _ = range roomData.Repos {
+			return // at least 1 repo exists
+		}
+	}
+	// Delete this service since no repos are configured
+	logger := log.WithFields(log.Fields{
+		"service_type": s.ServiceType(),
+		"service_id":   s.ServiceID(),
+	})
+	logger.Info("Removing service as no repositories are registered.")
+	if err := database.GetServiceDB().DeleteService(s.ServiceID()); err != nil {
+		logger.WithError(err).Error("Failed to delete service")
+	}
+}
+
+func (s *Service) joinRooms(client *gomatrix.Client) {
+	for roomID := range s.Rooms {
+		if _, err := client.JoinRoom(roomID, "", nil); err != nil {
+			log.WithFields(log.Fields{
+				log.ErrorKey: err,
+				"room_id":    roomID,
+				"user_id":    client.UserID,
+			}).Error("Failed to join room")
+		}
+	}
+}
+
+func init() {
+	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
+		return &Service{
+			DefaultService:     types.NewDefaultService(serviceID, serviceUserID, ServiceType),
+			webhookEndpointURL: webhookEndpointURL,
+		}
+	})
+}

src/github.com/matrix-org/go-neb/services/circleci/circleci_test.go

@@ -0,0 +1,141 @@
+package circleci
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/go-neb/database"
+	"github.com/matrix-org/go-neb/testutils"
+	"github.com/matrix-org/go-neb/types"
+	"github.com/matrix-org/gomatrix"
+	"net/url"
+)
+
+const exampleBody = ("%7B%22payload%22%3A%7B%22vcs_url%22%3A%22https%3A%2F%2Fgithub.com%2Fcircleci%2Fmongofinil%22%2C%22build_url%22%3A%22https%3A%2F%2Fcircleci.com%2Fgh%2Fcircleci%2Fmongofinil%2F22%22%2C%22build_num%22%3A22%2C%22branch%22%3A%22master%22%2C%22vcs_revision%22%3A%221d231626ba1d2838e599c5c598d28e2306ad4e48%22%2C%22committer_name%22%3A%22Allen%20Rohner%22%2C%22committer_email%22%3A%22arohner%40gmail.com%22%2C%22subject%22%3A%22Don%27t%20explode%20when%20the%20system%20clock%20shifts%20backwards%22%2C%22body%22%3A%22%22%2C%22why%22%3A%22github%22%2C%22dont_build%22%3Anull%2C%22queued_at%22%3A%222013-02-12T21%3A33%3A30Z%22%2C%22start_time%22%3A%222013-02-12T21%3A33%3A38Z%22%2C%22stop_time%22%3A%222013-02-12T21%3A34%3A01Z%22%2C%22build_time_millis%22%3A23505%2C%22username%22%3A%22circleci%22%2C%22reponame%22%3A%22mongofinil%22%2C%22lifecycle%22%3A%22finished%22%2C%22outcome%22%3A%22success%22%2C%22status%22%3A%22success%22%2C%22retry_of%22%3Anull%2C%22steps%22%3A%5B%7B%22name%22%3A%22configure%20the%20build%22%2C%22actions%22%3A%5B%7B%22bash_command%22%3Anull%2C%22run_time_millis%22%3A1646%2C%22start_time%22%3A%222013-02-12T21%3A33%3A38Z%22%2C%22end_time%22%3A%222013-02-12T21%3A33%3A39Z%22%2C%22name%22%3A%22configure%20the%20build%22%2C%22exit_code%22%3Anull%2C%22type%22%3A%22infrastructure%22%2C%22index%22%3A0%2C%22status%22%3A%22success%22%7D%5D%7D%2C%7B%22name%22%3A%22lein2%20deps%22%2C%22actions%22%3A%5B%7B%22bash_command%22%3A%22lein2%20deps%22%2C%22run_time_millis%22%3A7555%2C%22start_time%22%3A%222013-02-12T21%3A33%3A47Z%22%2C%22messages%22%3A%5B%5D%2C%22step%22%3A1%2C%22exit_code%22%3A0%2C%22end_time%22%3A%222013-02-12T21%3A33%3A54Z%22%2C%22index%22%3A0%2C%22status%22%3A%22success%22%2C%22type%22%3A%22dependencies%22%2C%22source%22%3A%22inference%22%2C%22failed%22%3Anull%7D%5D%7D%2C%7B%22name%22%3A%22lein2%20trampoline%20midje%22%2C%22actions%22%3A%5B%7B%22bash_command%22%3A%22lein2%20trampoline%20midje%22%2C%22run_time_millis%22%3A2310%2C%22continue%22%3Anull%2C%22parallel%22%3Atrue%2C%22start_time%22%3A%222013-02-12T21%3A33%3A59Z%22%2C%22name%22%3A%22lein2%20trampoline%20midje%22%2C%22messages%22%3A%5B%5D%2C%22step%22%3A6%2C%22exit_code%22%3A1%2C%22end_time%22%3A%222013-02-12T21%3A34%3A01Z%22%2C%22index%22%3A0%2C%22status%22%3A%22failed%22%2C%22timedout%22%3Anull%2C%22infrastructure_fail%22%3Anull%2C%22type%22%3A%22test%22%2C%22source%22%3A%22inference%22%2C%22failed%22%3Atrue%7D%5D%7D%5D%7D%7D")
+var circleciTests = []struct {
+	Body           string
+	Template       string
+	ExpectedOutput string
+}{
+	{
+		exampleBody,
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"circleci/mongofinil#22 (master - 1d231626ba : Allen Rohner): success",
+	},
+	{
+		exampleBody,
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"circleci/mongofinil#22 (master - 1d231626ba : Allen Rohner): success",
+	},
+	{
+		strings.TrimSuffix(exampleBody, "%7D") + "%2C%22EXTRA_KEY%22%3Anull%7D",
+		"%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}",
+		"circleci/mongofinil#22 (master - 1d231626ba : Allen Rohner): success",
+	},
+	{
+		exampleBody,
+		"%{repository}#%{build_number} %{duration}",
+		"circleci/mongofinil#22 23s",
+	},
+}
+
+func TestCircleCI(t *testing.T) {
+	database.SetServiceDB(&database.NopStorage{})
+
+	// Intercept message sending to Matrix and mock responses
+	msgs := []gomatrix.TextMessage{}
+	matrixTrans := struct{ testutils.MockTransport }{}
+	matrixTrans.RT = func(req *http.Request) (*http.Response, error) {
+		if !strings.Contains(req.URL.String(), "/send/m.room.message") {
+			return nil, fmt.Errorf("Unhandled URL: %s", req.URL.String())
+		}
+		var msg gomatrix.TextMessage
+		if err := json.NewDecoder(req.Body).Decode(&msg); err != nil {
+			return nil, fmt.Errorf("Failed to decode request JSON: %s", err)
+		}
+		msgs = append(msgs, msg)
+		return &http.Response{
+			StatusCode: 200,
+			Body:       ioutil.NopCloser(bytes.NewBufferString(`{"event_id":"$yup:event"}`)),
+		}, nil
+	}
+	matrixCli, _ := gomatrix.NewClient("https://hyrule", "@travisci:hyrule", "its_a_secret")
+	matrixCli.Client = &http.Client{Transport: matrixTrans}
+
+	// BEGIN running the CircleCI table tests
+	// ---------------------------------------
+	for _, test := range circleciTests {
+		msgs = []gomatrix.TextMessage{} // reset sent messages
+		mockWriter := httptest.NewRecorder()
+		circleci := makeService(t, test.Template)
+		if circleci == nil {
+			t.Error("TestCircleCI Failed to create service")
+			continue
+		}
+		if err := circleci.Register(nil, matrixCli); err != nil {
+			t.Errorf("TestCircleCI Failed to Register(): %s", err)
+			continue
+		}
+		correctBody, convErr := url.QueryUnescape(test.Body)
+		if convErr != nil {
+			t.Errorf("TestCircleCI Failed to UnUrlEncode Test Body: %s", convErr)
+			continue
+		}
+		req, err := http.NewRequest(
+			"POST", "https://neb.endpoint/circleci-service", bytes.NewBufferString(correctBody),
+		)
+		if err != nil {
+			t.Errorf("TestCircleCI Failed to create webhook request: %s", err)
+			continue
+		}
+		req.Header.Set("Content-Type", "application/json")
+		circleci.OnReceiveWebhook(mockWriter, req, matrixCli)
+
+		if !assertResponse(t, mockWriter, msgs, 200, 1) {
+			continue
+		}
+
+		if msgs[0].Body != test.ExpectedOutput {
+			t.Errorf("TestCircleCI want matrix body '%s', got '%s'", test.ExpectedOutput, msgs[0].Body)
+		}
+	}
+}
+
+func assertResponse(t *testing.T, w *httptest.ResponseRecorder, msgs []gomatrix.TextMessage, expectCode int, expectMsgLength int) bool {
+	if w.Code != expectCode {
+		t.Errorf("TestCircleCI OnReceiveWebhook want HTTP code %d, got %d", expectCode, w.Code)
+		return false
+	}
+	if len(msgs) != expectMsgLength {
+		t.Errorf("TestCircleCI want %d sent messages, got %d ", expectMsgLength, len(msgs))
+		return false
+	}
+	return true
+}
+
+func makeService(t *testing.T, template string) *Service {
+	srv, err := types.CreateService("id", ServiceType, "@travisci:hyrule", []byte(
+		`{
+			"rooms":{
+				"!ewfug483gsfe:localhost": {
+					"repos": {
+						"circleci/mongofinil": {
+							"template": "`+template+`"
+						}
+					}
+				}
+			}
+		}`,
+	))
+	if err != nil {
+		t.Error("Failed to create CircleCI service: ", err)
+		return nil
+	}
+	return srv.(*Service)
+}

src/github.com/matrix-org/go-neb/services/circleci/types.go

@@ -0,0 +1,44 @@
+package circleci
+
+import "time"
+
+// WebhookNotification is the response of a webhook notification by circleCI
+type WebhookNotification struct {
+	Payload struct {
+		VcsURL          string      `json:"vcs_url,omitempty"`
+		BuildURL        string      `json:"build_url,omitempty"`
+		BuildNum        int         `json:"build_num,omitempty"`
+		Branch          string      `json:"branch,omitempty"`
+		VcsRevision     string      `json:"vcs_revision,omitempty"`
+		CommitterName   string      `json:"committer_name,omitempty"`
+		CommitterEmail  string      `json:"committer_email,omitempty"`
+		Subject         string      `json:"subject,omitempty"`
+		Body            string      `json:"body,omitempty"`
+		Why             string      `json:"why,omitempty"`
+		DontBuild       interface{} `json:"dont_build,omitempty"`
+		QueuedAt        time.Time   `json:"queued_at,omitempty"`
+		StartTime       time.Time   `json:"start_time,omitempty"`
+		StopTime        time.Time   `json:"stop_time,omitempty"`
+		BuildTimeMillis int         `json:"build_time_millis,omitempty"`
+		Username        string      `json:"username,omitempty"`
+		Reponame        string      `json:"reponame,omitempty"`
+		Lifecycle       string      `json:"lifecycle,omitempty"`
+		Outcome         string      `json:"outcome,omitempty"`
+		Status          string      `json:"status,omitempty"`
+		RetryOf         interface{} `json:"retry_of,omitempty"`
+		Steps           []struct {
+			Name    string `json:"name,omitempty"`
+			Actions []struct {
+				BashCommand   interface{} `json:"bash_command,omitempty"`
+				RunTimeMillis int         `json:"run_time_millis,omitempty"`
+				StartTime     time.Time   `json:"start_time,omitempty"`
+				EndTime       time.Time   `json:"end_time,omitempty"`
+				Name          string      `json:"name,omitempty"`
+				ExitCode      interface{} `json:"exit_code,omitempty"`
+				Type          string      `json:"type,omitempty"`
+				Index         int         `json:"index,omitempty"`
+				Status        string      `json:"status,omitempty"`
+			} `json:"actions,omitempty"`
+		} `json:"steps,omitempty"`
+	} `json:"payload"`
+}