Merge pull request #91 from matrix-org/kegan/neb-metrics

Add metrics to NEB

src/github.com/matrix-org/go-neb/api.go

@@ -8,6 +8,7 @@ import (
 	"github.com/matrix-org/go-neb/clients"
 	"github.com/matrix-org/go-neb/database"
 	"github.com/matrix-org/go-neb/errors"
+	"github.com/matrix-org/go-neb/metrics"
 	"github.com/matrix-org/go-neb/polling"
 	"github.com/matrix-org/go-neb/types"
 	"net/http"
@@ -180,7 +181,6 @@ func (wh *webhookHandler) handle(w http.ResponseWriter, req *http.Request) {
 	// but we've base64d it.
 	base64srvID := segments[len(segments)-1]
 	bytesSrvID, err := base64.RawURLEncoding.DecodeString(base64srvID)
-	srvID := string(bytesSrvID)
 	if err != nil {
 		log.WithError(err).WithField("base64_service_id", base64srvID).Print(
 			"Not a b64 encoded string",
@@ -188,6 +188,7 @@ func (wh *webhookHandler) handle(w http.ResponseWriter, req *http.Request) {
 		w.WriteHeader(400)
 		return
 	}
+	srvID := string(bytesSrvID)
 
 	service, err := wh.db.LoadService(srvID)
 	if err != nil {
@@ -204,8 +205,9 @@ func (wh *webhookHandler) handle(w http.ResponseWriter, req *http.Request) {
 	}
 	log.WithFields(log.Fields{
 		"service_id":   service.ServiceID(),
-		"service_typ": service.ServiceType(),
+		"service_type": service.ServiceType(),
 	}).Print("Incoming webhook for service")
+	metrics.IncrementWebhook(service.ServiceType())
 	service.OnReceiveWebhook(w, req, cli)
 }
 
@@ -319,6 +321,7 @@ func (s *configureServiceHandler) OnIncomingRequest(req *http.Request) (interfac
 	}
 
 	service.PostRegister(old)
+	metrics.IncrementConfigureService(service.ServiceType())
 
 	return &struct {
 		ID        string

src/github.com/matrix-org/go-neb/goneb.go

@@ -5,6 +5,7 @@ import (
 	"github.com/matrix-org/dugong"
 	"github.com/matrix-org/go-neb/clients"
 	"github.com/matrix-org/go-neb/database"
+	_ "github.com/matrix-org/go-neb/metrics"
 	"github.com/matrix-org/go-neb/polling"
 	_ "github.com/matrix-org/go-neb/realms/github"
 	_ "github.com/matrix-org/go-neb/realms/jira"

src/github.com/matrix-org/go-neb/metrics/metrics.go

@@ -0,0 +1,50 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Status is the status of a measurable metric (incoming commands, outgoing polls, etc)
+type Status string
+
+// Common status values
+const (
+	StatusSuccess = "success"
+	StatusFailure = "failure"
+)
+
+var (
+	cmdCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "goneb_pling_cmd_total",
+		Help: "The number of incoming commands from matrix clients",
+	}, []string{"cmd", "status"})
+	configureServicesCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "goneb_configure_services_total",
+		Help: "The total number of configured services requests",
+	}, []string{"service_type"})
+	webhookCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "goneb_webhook_total",
+		Help: "The total number of recognised incoming webhook requests",
+	}, []string{"service_type"})
+)
+
+// IncrementCommand increments the pling command counter
+func IncrementCommand(cmdName string, st Status) {
+	cmdCounter.With(prometheus.Labels{"cmd": cmdName, "status": string(st)}).Inc()
+}
+
+// IncrementConfigureService increments the /configureService counter
+func IncrementConfigureService(serviceType string) {
+	configureServicesCounter.With(prometheus.Labels{"service_type": serviceType}).Inc()
+}
+
+// IncrementWebhook increments the incoming webhook request counter
+func IncrementWebhook(serviceType string) {
+	webhookCounter.With(prometheus.Labels{"service_type": serviceType}).Inc()
+}
+
+func init() {
+	prometheus.MustRegister(cmdCounter)
+	prometheus.MustRegister(configureServicesCounter)
+	prometheus.MustRegister(webhookCounter)
+}

src/github.com/matrix-org/go-neb/plugin/plugin.go

@@ -3,6 +3,7 @@ package plugin
 import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/matrix"
+	"github.com/matrix-org/go-neb/metrics"
 	"github.com/mattn/go-shellwords"
 	"regexp"
 	"strings"
@@ -82,7 +83,10 @@ func runCommandForPlugin(plugin Plugin, event *matrix.Event, arguments []string)
 				"args":       cmdArgs,
 			}).Warn("Command returned both error and content.")
 		}
+		metrics.IncrementCommand(bestMatch.Path[0], metrics.StatusFailure)
 		content = matrix.TextMessage{"m.notice", err.Error()}
+	} else {
+		metrics.IncrementCommand(bestMatch.Path[0], metrics.StatusSuccess)
 	}
 
 	return content

src/github.com/matrix-org/go-neb/services/guggy/guggy.go

@@ -3,10 +3,12 @@ package services
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	log "github.com/Sirupsen/logrus"
 	"github.com/matrix-org/go-neb/matrix"
 	"github.com/matrix-org/go-neb/plugin"
 	"github.com/matrix-org/go-neb/types"
+	"io/ioutil"
 	"math"
 	"net/http"
 	"strings"
@@ -54,7 +56,7 @@ func (s *guggyService) cmdGuggy(client *matrix.Client, roomID, userID string, ar
 	querySentence := strings.Join(args, " ")
 	gifResult, err := s.text2gifGuggy(querySentence)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Failed to query Guggy: %s", err.Error())
 	}
 
 	if gifResult.GIF == "" {
@@ -66,7 +68,7 @@ func (s *guggyService) cmdGuggy(client *matrix.Client, roomID, userID string, ar
 
 	mxc, err := client.UploadLink(gifResult.GIF)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Failed to upload Guggy image to matrix: %s", err.Error())
 	}
 
 	return matrix.ImageMessage{
@@ -114,9 +116,20 @@ func (s *guggyService) text2gifGuggy(querySentence string) (*guggyGifResult, err
 		log.Error(err)
 		return nil, err
 	}
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		resBytes, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			log.WithError(err).Error("Failed to decode Guggy response body")
+		}
+		log.WithFields(log.Fields{
+			"code": res.StatusCode,
+			"body": string(resBytes),
+		}).Error("Failed to query Guggy")
+		return nil, fmt.Errorf("Failed to decode response (HTTP %d)", res.StatusCode)
+	}
 	var result guggyGifResult
 	if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Failed to decode response (HTTP %d): %s", res.StatusCode, err.Error())
 	}
 
 	return &result, nil

src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go

@@ -11,13 +11,22 @@ import (
 	"github.com/matrix-org/go-neb/polling"
 	"github.com/matrix-org/go-neb/types"
 	"github.com/mmcdole/gofeed"
+	"github.com/prometheus/client_golang/prometheus"
 	"html"
 	"net/http"
+	"net/url"
 	"time"
 )
 
 var cachingClient *http.Client
 
+var (
+	pollCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "goneb_rss_polls_total",
+		Help: "The number of feed polls from RSS services",
+	}, []string{"url", "http_status"})
+)
+
 const minPollingIntervalSeconds = 60 * 5 // 5 min (News feeds can be genuinely spammy)
 
 type rssBotService struct {
@@ -127,8 +136,10 @@ func (s *rssBotService) OnPoll(cli *matrix.Client) time.Time {
 		feed, items, err := s.queryFeed(u)
 		if err != nil {
 			logger.WithField("feed_url", u).WithError(err).Error("Failed to query feed")
+			incrementMetrics(u, err)
 			continue
 		}
+		incrementMetrics(u, nil)
 		// Loop backwards since [0] is the most recent and we want to send in chronological order
 		for i := len(items) - 1; i >= 0; i-- {
 			item := items[i]
@@ -150,6 +161,25 @@ func (s *rssBotService) OnPoll(cli *matrix.Client) time.Time {
 	return s.nextTimestamp()
 }
 
+func incrementMetrics(urlStr string, err error) {
+	// extract domain part of RSS feed URL to get coarser (more useful) statistics
+	domain := urlStr
+	u, urlErr := url.Parse(urlStr)
+	if urlErr == nil {
+		domain = u.Host
+	}
+	if err != nil {
+		herr, ok := err.(gofeed.HTTPError)
+		statusCode := 0 // e.g. network timeout
+		if ok {
+			statusCode = herr.StatusCode
+		}
+		pollCounter.With(prometheus.Labels{"url": domain, "http_status": string(statusCode)}).Inc()
+	} else {
+		pollCounter.With(prometheus.Labels{"url": domain, "http_status": "200"}).Inc() // technically 2xx but gofeed doesn't tell us which
+	}
+}
+
 func (s *rssBotService) nextTimestamp() time.Time {
 	// return the earliest next poll ts
 	var earliestNextTs int64
@@ -285,4 +315,5 @@ func init() {
 		}
 		return r
 	})
+	prometheus.MustRegister(pollCounter)
 }