Browse Source

Merge pull request #2 from matrix-org/kegan/webhooks

Add a webhook handler and parse incoming Github webhook events
pull/4/head
Kegsay 9 years ago
committed by GitHub
parent
commit
e6813169b8
  1. 26
      src/github.com/matrix-org/go-neb/api.go
  2. 9
      src/github.com/matrix-org/go-neb/clients/clients.go
  3. 11
      src/github.com/matrix-org/go-neb/database/db.go
  4. 15
      src/github.com/matrix-org/go-neb/database/schema.go
  5. 1
      src/github.com/matrix-org/go-neb/goneb.go
  6. 8
      src/github.com/matrix-org/go-neb/services/echo/echo.go
  7. 10
      src/github.com/matrix-org/go-neb/services/github/github.go
  8. 259
      src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go
  9. 1524
      src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
  10. 4
      src/github.com/matrix-org/go-neb/types/types.go

26
src/github.com/matrix-org/go-neb/api.go

@ -5,7 +5,9 @@ import (
"github.com/matrix-org/go-neb/clients"
"github.com/matrix-org/go-neb/database"
"github.com/matrix-org/go-neb/errors"
"github.com/matrix-org/go-neb/types"
"net/http"
"strings"
)
type heartbeatHandler struct{}
@ -14,6 +16,18 @@ func (*heartbeatHandler) OnIncomingRequest(req *http.Request) (interface{}, *err
return &struct{}{}, nil
}
func handleWebhook(w http.ResponseWriter, req *http.Request) {
segments := strings.Split(req.URL.Path, "/")
// last path segment is the service type which we will pass the incoming request to
srvType := segments[len(segments)-1]
service := types.CreateService("", srvType)
if service == nil {
w.WriteHeader(404)
return
}
service.OnReceiveWebhook(w, req)
}
type configureClientHandler struct {
db *database.ServiceDB
clients *clients.Clients
@ -24,7 +38,7 @@ func (s *configureClientHandler) OnIncomingRequest(req *http.Request) (interface
return nil, &errors.HTTPError{nil, "Unsupported Method", 405}
}
var body database.ClientConfig
var body types.ClientConfig
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
return nil, &errors.HTTPError{err, "Error parsing request JSON", 400}
}
@ -39,8 +53,8 @@ func (s *configureClientHandler) OnIncomingRequest(req *http.Request) (interface
}
return &struct {
OldClient database.ClientConfig
NewClient database.ClientConfig
OldClient types.ClientConfig
NewClient types.ClientConfig
}{oldClient, body}, nil
}
@ -67,7 +81,7 @@ func (s *configureServiceHandler) OnIncomingRequest(req *http.Request) (interfac
return nil, &errors.HTTPError{nil, `Must supply a "ID", a "Type" and a "Config"`, 400}
}
service := database.CreateService(body.ID, body.Type)
service := types.CreateService(body.ID, body.Type)
if service == nil {
return nil, &errors.HTTPError{nil, "Unknown service type", 400}
}
@ -89,7 +103,7 @@ func (s *configureServiceHandler) OnIncomingRequest(req *http.Request) (interfac
return &struct {
ID string
Type string
OldConfig database.Service
NewConfig database.Service
OldConfig types.Service
NewConfig types.Service
}{body.ID, body.Type, oldService, service}, nil
}

9
src/github.com/matrix-org/go-neb/clients/clients.go

@ -5,6 +5,7 @@ import (
"github.com/matrix-org/go-neb/database"
"github.com/matrix-org/go-neb/matrix"
"github.com/matrix-org/go-neb/plugin"
"github.com/matrix-org/go-neb/types"
"net/url"
"sync"
)
@ -37,7 +38,7 @@ func (c *Clients) Client(userID string) (*matrix.Client, error) {
}
// Update updates the config for a matrix client
func (c *Clients) Update(config database.ClientConfig) (database.ClientConfig, error) {
func (c *Clients) Update(config types.ClientConfig) (types.ClientConfig, error) {
_, old, err := c.updateClientInDB(config)
return old.config, err
}
@ -68,7 +69,7 @@ func (c *Clients) Start() error {
}
type clientEntry struct {
config database.ClientConfig
config types.ClientConfig
client *matrix.Client
}
@ -105,7 +106,7 @@ func (c *Clients) loadClientFromDB(userID string) (entry clientEntry, err error)
return
}
func (c *Clients) updateClientInDB(newConfig database.ClientConfig) (new clientEntry, old clientEntry, err error) {
func (c *Clients) updateClientInDB(newConfig types.ClientConfig) (new clientEntry, old clientEntry, err error) {
c.dbMutex.Lock()
defer c.dbMutex.Unlock()
@ -136,7 +137,7 @@ func (c *Clients) updateClientInDB(newConfig database.ClientConfig) (new clientE
return
}
func (c *Clients) newClient(config database.ClientConfig) (*matrix.Client, error) {
func (c *Clients) newClient(config types.ClientConfig) (*matrix.Client, error) {
homeserverURL, err := url.Parse(config.HomeserverURL)
if err != nil {

11
src/github.com/matrix-org/go-neb/database/db.go

@ -3,6 +3,7 @@ package database
import (
"database/sql"
"github.com/matrix-org/go-neb/matrix"
"github.com/matrix-org/go-neb/types"
"sort"
"time"
)
@ -29,7 +30,7 @@ func Open(databaseType, databaseURL string) (serviceDB *ServiceDB, err error) {
// StoreMatrixClientConfig stores the Matrix client config for a bot service.
// If a config already exists then it will be updated, otherwise a new config
// will be inserted. The previous config is returned.
func (d *ServiceDB) StoreMatrixClientConfig(config ClientConfig) (oldConfig ClientConfig, err error) {
func (d *ServiceDB) StoreMatrixClientConfig(config types.ClientConfig) (oldConfig types.ClientConfig, err error) {
err = runTransaction(d.db, func(txn *sql.Tx) error {
oldConfig, err = selectMatrixClientConfigTxn(txn, config.UserID)
now := time.Now()
@ -56,7 +57,7 @@ func (d *ServiceDB) LoadServiceUserIds() (userIDsToRooms map[string][]string, er
// LoadMatrixClientConfig loads a Matrix client config from the database.
// Returns sql.ErrNoRows if the client isn't in the database.
func (d *ServiceDB) LoadMatrixClientConfig(userID string) (config ClientConfig, err error) {
func (d *ServiceDB) LoadMatrixClientConfig(userID string) (config types.ClientConfig, err error) {
err = runTransaction(d.db, func(txn *sql.Tx) error {
config, err = selectMatrixClientConfigTxn(txn, userID)
return err
@ -66,7 +67,7 @@ func (d *ServiceDB) LoadMatrixClientConfig(userID string) (config ClientConfig,
// LoadService loads a service from the database.
// Returns sql.ErrNoRows if the service isn't in the database.
func (d *ServiceDB) LoadService(serviceID string) (service Service, err error) {
func (d *ServiceDB) LoadService(serviceID string) (service types.Service, err error) {
err = runTransaction(d.db, func(txn *sql.Tx) error {
service, err = selectServiceTxn(txn, serviceID)
return err
@ -76,7 +77,7 @@ func (d *ServiceDB) LoadService(serviceID string) (service Service, err error) {
// LoadServicesInRoom loads all the bot services configured for a room.
// Returns the empty list if there aren't any services configured.
func (d *ServiceDB) LoadServicesInRoom(serviceUserID, roomID string) (services []Service, err error) {
func (d *ServiceDB) LoadServicesInRoom(serviceUserID, roomID string) (services []types.Service, err error) {
err = runTransaction(d.db, func(txn *sql.Tx) error {
serviceIDs, err := selectRoomServicesTxn(txn, serviceUserID, roomID)
if err != nil {
@ -97,7 +98,7 @@ func (d *ServiceDB) LoadServicesInRoom(serviceUserID, roomID string) (services [
// StoreService stores a service into the database either by inserting a new
// service or updating an existing service. Returns the old service if there
// was one.
func (d *ServiceDB) StoreService(service Service, client *matrix.Client) (oldService Service, err error) {
func (d *ServiceDB) StoreService(service types.Service, client *matrix.Client) (oldService types.Service, err error) {
err = runTransaction(d.db, func(txn *sql.Tx) error {
oldService, err = selectServiceTxn(txn, service.ServiceID())
if err != nil && err != sql.ErrNoRows {

15
src/github.com/matrix-org/go-neb/database/schema.go

@ -4,6 +4,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"github.com/matrix-org/go-neb/types"
"time"
)
@ -61,7 +62,7 @@ const selectMatrixClientConfigSQL = `
SELECT client_json FROM matrix_clients WHERE user_id = $1
`
func selectMatrixClientConfigTxn(txn *sql.Tx, userID string) (config ClientConfig, err error) {
func selectMatrixClientConfigTxn(txn *sql.Tx, userID string) (config types.ClientConfig, err error) {
var configJSON []byte
err = txn.QueryRow(selectMatrixClientConfigSQL, userID).Scan(&configJSON)
if err != nil {
@ -77,7 +78,7 @@ INSERT INTO matrix_clients(
) VALUES ($1, $2, '', $3, $4)
`
func insertMatrixClientConfigTxn(txn *sql.Tx, now time.Time, config ClientConfig) error {
func insertMatrixClientConfigTxn(txn *sql.Tx, now time.Time, config types.ClientConfig) error {
t := now.UnixNano() / 1000000
configJSON, err := json.Marshal(&config)
if err != nil {
@ -92,7 +93,7 @@ UPDATE matrix_clients SET client_json = $1, time_updated_ms = $2
WHERE user_id = $3
`
func updateMatrixClientConfigTxn(txn *sql.Tx, now time.Time, config ClientConfig) error {
func updateMatrixClientConfigTxn(txn *sql.Tx, now time.Time, config types.ClientConfig) error {
t := now.UnixNano() / 1000000
configJSON, err := json.Marshal(&config)
if err != nil {
@ -107,14 +108,14 @@ SELECT service_type, service_json FROM services
WHERE service_id = $1
`
func selectServiceTxn(txn *sql.Tx, serviceID string) (Service, error) {
func selectServiceTxn(txn *sql.Tx, serviceID string) (types.Service, error) {
var serviceType string
var serviceJSON []byte
row := txn.QueryRow(selectServiceSQL, serviceID)
if err := row.Scan(&serviceType, &serviceJSON); err != nil {
return nil, err
}
service := CreateService(serviceID, serviceType)
service := types.CreateService(serviceID, serviceType)
if service == nil {
return nil, fmt.Errorf("Cannot create services of type %s", serviceType)
}
@ -129,7 +130,7 @@ UPDATE services SET service_type=$1, service_json=$2, time_updated_ms=$3
WHERE service_id=$4
`
func updateServiceTxn(txn *sql.Tx, now time.Time, service Service) error {
func updateServiceTxn(txn *sql.Tx, now time.Time, service types.Service) error {
serviceJSON, err := json.Marshal(service)
if err != nil {
return err
@ -148,7 +149,7 @@ INSERT INTO services(
) VALUES ($1, $2, $3, $4, $5)
`
func insertServiceTxn(txn *sql.Tx, now time.Time, service Service) error {
func insertServiceTxn(txn *sql.Tx, now time.Time, service types.Service) error {
serviceJSON, err := json.Marshal(service)
if err != nil {
return err

1
src/github.com/matrix-org/go-neb/goneb.go

@ -31,6 +31,7 @@ func main() {
http.Handle("/test", server.MakeJSONAPI(&heartbeatHandler{}))
http.Handle("/admin/configureClient", server.MakeJSONAPI(&configureClientHandler{db: db, clients: clients}))
http.Handle("/admin/configureService", server.MakeJSONAPI(&configureServiceHandler{db: db, clients: clients}))
http.HandleFunc("/services/hooks/", handleWebhook)
http.ListenAndServe(bindAddress, nil)
}

8
src/github.com/matrix-org/go-neb/services/echo/echo.go

@ -1,9 +1,10 @@
package services
import (
"github.com/matrix-org/go-neb/database"
"github.com/matrix-org/go-neb/matrix"
"github.com/matrix-org/go-neb/plugin"
"github.com/matrix-org/go-neb/types"
"net/http"
"strings"
)
@ -29,9 +30,12 @@ func (e *echoService) Plugin(roomID string) plugin.Plugin {
},
}
}
func (e *echoService) OnReceiveWebhook(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(200) // Do nothing
}
func init() {
database.RegisterService(func(serviceID string) database.Service {
types.RegisterService(func(serviceID string) types.Service {
return &echoService{id: serviceID}
})
}

10
src/github.com/matrix-org/go-neb/services/github/github.go

@ -4,10 +4,12 @@ import (
"fmt"
log "github.com/Sirupsen/logrus"
"github.com/google/go-github/github"
"github.com/matrix-org/go-neb/database"
"github.com/matrix-org/go-neb/matrix"
"github.com/matrix-org/go-neb/plugin"
"github.com/matrix-org/go-neb/services/github/webhook"
"github.com/matrix-org/go-neb/types"
"golang.org/x/oauth2"
"net/http"
"regexp"
"strconv"
)
@ -60,6 +62,10 @@ func (s *githubService) Plugin(roomID string) plugin.Plugin {
},
}
}
func (s *githubService) OnReceiveWebhook(w http.ResponseWriter, req *http.Request) {
// defer entirely to the webhook package
webhook.OnReceiveRequest(w, req, "")
}
// githubClient returns a github Client which can perform Github API operations.
// If `token` is empty, a non-authenticated client will be created. This should be
@ -91,7 +97,7 @@ func ownerRepoNumberFromText(ownerRepoNumberText string) (string, string, int, e
}
func init() {
database.RegisterService(func(serviceID string) database.Service {
types.RegisterService(func(serviceID string) types.Service {
return &githubService{id: serviceID}
})
}

259
src/github.com/matrix-org/go-neb/services/github/webhook/webhook.go

@ -0,0 +1,259 @@
package webhook
import (
"crypto/hmac"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
log "github.com/Sirupsen/logrus"
"github.com/google/go-github/github"
"html"
"io/ioutil"
"net/http"
"strings"
)
// OnReceiveRequest processes incoming github webhook requests. The secretToken
// parameter is optional.
func OnReceiveRequest(w http.ResponseWriter, r *http.Request, secretToken string) {
// Verify the HMAC signature if NEB was configured with a secret token
eventType := r.Header.Get("X-GitHub-Event")
signatureSHA1 := r.Header.Get("X-Hub-Signature")
content, err := ioutil.ReadAll(r.Body)
if err != nil {
log.WithError(err).Print("Failed to read Github webhook body")
w.WriteHeader(400)
return
}
// Verify request if a secret token has been supplied.
if secretToken != "" {
sigHex := strings.Split(signatureSHA1, "=")[1]
var sigBytes []byte
sigBytes, err = hex.DecodeString(sigHex)
if err != nil {
log.WithError(err).WithField("X-Hub-Signature", sigHex).Print(
"Failed to decode signature as hex.")
w.WriteHeader(400)
return
}
if !checkMAC([]byte(content), sigBytes, []byte(secretToken)) {
log.WithFields(log.Fields{
"X-Hub-Signature": signatureSHA1,
}).Print("Received Github event which failed MAC check.")
w.WriteHeader(403)
return
}
}
log.WithFields(log.Fields{
"event_type": eventType,
"signature": signatureSHA1,
}).Print("Received Github event")
htmlStr, repo, err := parseGithubEvent(eventType, content)
if err != nil {
log.WithError(err).Print("Failed to parse github event")
w.WriteHeader(500)
return
}
if err := handleWebhookEvent(eventType, htmlStr, repo); err != nil {
log.WithError(err).Print("Failed to handle Github webhook event")
w.WriteHeader(500)
return
}
w.WriteHeader(200)
}
// checkMAC reports whether messageMAC is a valid HMAC tag for message.
func checkMAC(message, messageMAC, key []byte) bool {
mac := hmac.New(sha1.New, key)
mac.Write(message)
expectedMAC := mac.Sum(nil)
return hmac.Equal(messageMAC, expectedMAC)
}
// parseGithubEvent parses a github event type and JSON data and returns an explanatory
// HTML string and the github repository this event affects, or an error.
func parseGithubEvent(eventType string, data []byte) (string, *github.Repository, error) {
if eventType == "pull_request" {
var ev github.PullRequestEvent
if err := json.Unmarshal(data, &ev); err != nil {
return "", nil, err
}
return pullRequestHTMLMessage(ev), ev.Repo, nil
} else if eventType == "issues" {
var ev github.IssuesEvent
if err := json.Unmarshal(data, &ev); err != nil {
return "", nil, err
}
return issueHTMLMessage(ev), ev.Repo, nil
} else if eventType == "push" {
var ev github.PushEvent
if err := json.Unmarshal(data, &ev); err != nil {
return "", nil, err
}
// The 'push' event repository format is subtly different from normal, so munge the bits we need.
fullName := *ev.Repo.Owner.Name + "/" + *ev.Repo.Name
repo := github.Repository{
Owner: &github.User{
Login: ev.Repo.Owner.Name,
},
Name: ev.Repo.Name,
FullName: &fullName,
}
return pushHTMLMessage(ev), &repo, nil
} else if eventType == "issue_comment" {
var ev github.IssueCommentEvent
if err := json.Unmarshal(data, &ev); err != nil {
return "", nil, err
}
return issueCommentHTMLMessage(ev), ev.Repo, nil
} else if eventType == "pull_request_review_comment" {
var ev github.PullRequestReviewCommentEvent
if err := json.Unmarshal(data, &ev); err != nil {
return "", nil, err
}
return prReviewCommentHTMLMessage(ev), ev.Repo, nil
}
return "", nil, fmt.Errorf("Unrecognized event type")
}
func handleWebhookEvent(eventType string, htmlStr string, repo *github.Repository) error {
return nil
}
func pullRequestHTMLMessage(p github.PullRequestEvent) string {
var actionTarget string
if p.PullRequest.Assignee != nil && p.PullRequest.Assignee.Login != nil {
actionTarget = fmt.Sprintf(" to %s", *p.PullRequest.Assignee.Login)
}
return fmt.Sprintf(
"[<u>%s</u>] %s %s <b>pull request #%d</b>: %s [%s]%s - %s",
html.EscapeString(*p.Repo.FullName),
html.EscapeString(*p.Sender.Login),
html.EscapeString(*p.Action),
*p.Number,
html.EscapeString(*p.PullRequest.Title),
html.EscapeString(*p.PullRequest.State),
html.EscapeString(actionTarget),
html.EscapeString(*p.PullRequest.HTMLURL),
)
}
func issueHTMLMessage(p github.IssuesEvent) string {
var actionTarget string
if p.Issue.Assignee != nil && p.Issue.Assignee.Login != nil {
actionTarget = fmt.Sprintf(" to %s", *p.Issue.Assignee.Login)
}
return fmt.Sprintf(
"[<u>%s</u>] %s %s <b>issue #%d</b>: %s [%s]%s - %s",
html.EscapeString(*p.Repo.FullName),
html.EscapeString(*p.Sender.Login),
html.EscapeString(*p.Action),
*p.Issue.Number,
html.EscapeString(*p.Issue.Title),
html.EscapeString(*p.Issue.State),
html.EscapeString(actionTarget),
html.EscapeString(*p.Issue.HTMLURL),
)
}
func issueCommentHTMLMessage(p github.IssueCommentEvent) string {
var kind string
if p.Issue.PullRequestLinks == nil {
kind = "issue"
} else {
kind = "pull request"
}
return fmt.Sprintf(
"[<u>%s</u>] %s commented on %s's <b>%s #%d</b>: %s - %s",
html.EscapeString(*p.Repo.FullName),
html.EscapeString(*p.Comment.User.Login),
html.EscapeString(*p.Issue.User.Login),
kind,
*p.Issue.Number,
html.EscapeString(*p.Issue.Title),
html.EscapeString(*p.Issue.HTMLURL),
)
}
func prReviewCommentHTMLMessage(p github.PullRequestReviewCommentEvent) string {
assignee := "None"
if p.PullRequest.Assignee != nil {
assignee = html.EscapeString(*p.PullRequest.Assignee.Login)
}
return fmt.Sprintf(
"[<u>%s</u>] %s made a line comment on %s's <b>pull request #%d</b> (assignee: %s): %s - %s",
html.EscapeString(*p.Repo.FullName),
html.EscapeString(*p.Sender.Login),
html.EscapeString(*p.PullRequest.User.Login),
*p.PullRequest.Number,
assignee,
html.EscapeString(*p.PullRequest.Title),
html.EscapeString(*p.Comment.HTMLURL),
)
}
func pushHTMLMessage(p github.PushEvent) string {
// /refs/heads/alice/branch-name => alice/branch-name
branch := strings.Replace(*p.Ref, "refs/heads/", "", -1)
// this branch was deleted, no HeadCommit object and deleted=true
if p.HeadCommit == nil && p.Deleted != nil && *p.Deleted {
return fmt.Sprintf(
`[<u>%s</u>] %s <font color="red"><b>deleted</font> %s</b>`,
html.EscapeString(*p.Repo.FullName),
html.EscapeString(*p.Pusher.Name),
html.EscapeString(branch),
)
}
if p.Commits != nil && len(p.Commits) > 1 {
// multi-commit message
// [<repo>] <username> pushed <num> commits to <branch>: <git.io link>
// <up to 3 commits>
var cList []string
for _, c := range p.Commits {
cList = append(cList, fmt.Sprintf(
`%s: %s`,
html.EscapeString(nameForAuthor(c.Author)),
html.EscapeString(*c.Message),
))
}
return fmt.Sprintf(
`[<u>%s</u>] %s pushed %d commits to <b>%s</b>: %s<br>%s`,
html.EscapeString(*p.Repo.FullName),
html.EscapeString(nameForAuthor(p.HeadCommit.Committer)),
len(p.Commits),
html.EscapeString(branch),
html.EscapeString(*p.HeadCommit.URL),
strings.Join(cList, "<br>"),
)
}
// single commit message
// [<repo>] <username> pushed to <branch>: <msg> - <git.io link>
return fmt.Sprintf(
`[<u>%s</u>] %s pushed to <b>%s</b>: %s - %s`,
html.EscapeString(*p.Repo.FullName),
html.EscapeString(nameForAuthor(p.HeadCommit.Committer)),
html.EscapeString(branch),
html.EscapeString(*p.HeadCommit.Message),
html.EscapeString(*p.HeadCommit.URL),
)
}
func nameForAuthor(a *github.CommitAuthor) string {
if a == nil {
return ""
}
if a.Login != nil { // prefer to use their GH username than the name they commited as
return *a.Login
}
return *a.Name
}

1524
src/github.com/matrix-org/go-neb/services/github/webhook/webhook_test.go
File diff suppressed because it is too large
View File

4
src/github.com/matrix-org/go-neb/database/types.go → src/github.com/matrix-org/go-neb/types/types.go

@ -1,8 +1,9 @@
package database
package types
import (
"errors"
"github.com/matrix-org/go-neb/plugin"
"net/http"
"net/url"
)
@ -31,6 +32,7 @@ type Service interface {
ServiceType() string
RoomIDs() []string
Plugin(roomID string) plugin.Plugin
OnReceiveWebhook(w http.ResponseWriter, req *http.Request)
}
var servicesByType = map[string]func(string) Service{}
Loading…
Cancel
Save