Browse Source

Merge pull request #100 from matrix-org/kegan/service-isolation

Prevent panicking code from taking down the entire process
kegan/tests
Kegsay 8 years ago
committed by GitHub
parent
commit
2b0e244d3f
  1. 4
      src/github.com/matrix-org/go-neb/goneb.go
  2. 20
      src/github.com/matrix-org/go-neb/matrix/worker.go
  3. 12
      src/github.com/matrix-org/go-neb/polling/polling.go
  4. 31
      src/github.com/matrix-org/go-neb/server/server.go
  5. 29
      src/github.com/matrix-org/go-neb/server/server_test.go

4
src/github.com/matrix-org/go-neb/goneb.go

@ -193,9 +193,9 @@ func main() {
http.Handle("/metrics", prometheus.Handler()) http.Handle("/metrics", prometheus.Handler())
http.Handle("/test", prometheus.InstrumentHandler("test", server.MakeJSONAPI(&heartbeatHandler{}))) http.Handle("/test", prometheus.InstrumentHandler("test", server.MakeJSONAPI(&heartbeatHandler{})))
wh := &webhookHandler{db: db, clients: clients} wh := &webhookHandler{db: db, clients: clients}
http.HandleFunc("/services/hooks/", prometheus.InstrumentHandlerFunc("webhookHandler", wh.handle))
http.HandleFunc("/services/hooks/", prometheus.InstrumentHandlerFunc("webhookHandler", server.Protect(wh.handle)))
rh := &realmRedirectHandler{db: db} rh := &realmRedirectHandler{db: db}
http.HandleFunc("/realms/redirects/", prometheus.InstrumentHandlerFunc("realmRedirectHandler", rh.handle))
http.HandleFunc("/realms/redirects/", prometheus.InstrumentHandlerFunc("realmRedirectHandler", server.Protect(rh.handle)))
// Read exclusively from the config file if one was supplied. // Read exclusively from the config file if one was supplied.
// Otherwise, add HTTP listeners for new Services/Sessions/Clients/etc. // Otherwise, add HTTP listeners for new Services/Sessions/Clients/etc.

20
src/github.com/matrix-org/go-neb/matrix/worker.go

@ -1,5 +1,10 @@
package matrix package matrix
import (
log "github.com/Sirupsen/logrus"
"runtime/debug"
)
// Worker processes incoming events and updates the Matrix client's data structures. It also informs // Worker processes incoming events and updates the Matrix client's data structures. It also informs
// any attached listeners of the new events. // any attached listeners of the new events.
type Worker struct { type Worker struct {
@ -38,6 +43,21 @@ func (worker *Worker) notifyListeners(event *Event) {
} }
func (worker *Worker) onSyncHTTPResponse(res syncHTTPResponse) { func (worker *Worker) onSyncHTTPResponse(res syncHTTPResponse) {
defer func() {
if r := recover(); r != nil {
userID := ""
if worker.client != nil {
userID = worker.client.UserID
}
log.WithFields(log.Fields{
"panic": r,
"user_id": userID,
}).Errorf(
"onSyncHTTPResponse panicked!\n%s", debug.Stack(),
)
}
}()
for roomID, roomData := range res.Rooms.Join { for roomID, roomData := range res.Rooms.Join {
room := worker.client.getOrCreateRoom(roomID) room := worker.client.getOrCreateRoom(roomID)
for _, event := range roomData.State.Events { for _, event := range roomData.State.Events {

12
src/github.com/matrix-org/go-neb/polling/polling.go

@ -5,6 +5,7 @@ import (
"github.com/matrix-org/go-neb/clients" "github.com/matrix-org/go-neb/clients"
"github.com/matrix-org/go-neb/database" "github.com/matrix-org/go-neb/database"
"github.com/matrix-org/go-neb/types" "github.com/matrix-org/go-neb/types"
"runtime/debug"
"sync" "sync"
"time" "time"
) )
@ -71,6 +72,17 @@ func pollLoop(service types.Service, ts int64) {
"service_id": service.ServiceID(), "service_id": service.ServiceID(),
"service_type": service.ServiceType(), "service_type": service.ServiceType(),
}) })
defer func() {
// Kill the poll loop entirely as it is likely that whatever made us panic will
// make us panic again. We can whine bitterly about it though.
if r := recover(); r != nil {
logger.WithField("panic", r).Errorf(
"pollLoop panicked!\n%s", debug.Stack(),
)
}
}()
poller, ok := service.(types.Poller) poller, ok := service.(types.Poller)
if !ok { if !ok {
logger.Error("Service is not a Poller.") logger.Error("Service is not a Poller.")

31
src/github.com/matrix-org/go-neb/server/server.go

@ -6,6 +6,7 @@ import (
log "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus"
"github.com/matrix-org/go-neb/errors" "github.com/matrix-org/go-neb/errors"
"net/http" "net/http"
"runtime/debug"
) )
// JSONRequestHandler represents an interface that must be satisfied in order to respond to incoming // JSONRequestHandler represents an interface that must be satisfied in order to respond to incoming
@ -34,14 +35,38 @@ func WithCORSOptions(handler http.HandlerFunc) http.HandlerFunc {
} }
} }
// Protect panicking HTTP requests from taking down the entire process, and log them using
// the correct logger, returning a 500 with a JSON response rather than abruptly closing the
// connection.
func Protect(handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
defer func() {
if r := recover(); r != nil {
log.WithFields(log.Fields{
"panic": r,
"method": req.Method,
"url": req.URL,
}).Errorf(
"Request panicked!\n%s", debug.Stack(),
)
jsonErrorResponse(
w, req, &errors.HTTPError{nil, "Internal Server Error", 500},
)
}
}()
handler(w, req)
}
}
// MakeJSONAPI creates an HTTP handler which always responds to incoming requests with JSON responses. // MakeJSONAPI creates an HTTP handler which always responds to incoming requests with JSON responses.
func MakeJSONAPI(handler JSONRequestHandler) http.HandlerFunc { func MakeJSONAPI(handler JSONRequestHandler) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
return Protect(func(w http.ResponseWriter, req *http.Request) {
logger := log.WithFields(log.Fields{ logger := log.WithFields(log.Fields{
"method": req.Method, "method": req.Method,
"url": req.URL, "url": req.URL,
}) })
logger.Print(">>> Incoming request")
logger.Print("Incoming request")
res, httpErr := handler.OnIncomingRequest(req) res, httpErr := handler.OnIncomingRequest(req)
// Set common headers returned regardless of the outcome of the request // Set common headers returned regardless of the outcome of the request
@ -67,7 +92,7 @@ func MakeJSONAPI(handler JSONRequestHandler) http.HandlerFunc {
resBytes = r resBytes = r
} }
w.Write(resBytes) w.Write(resBytes)
}
})
} }
func jsonErrorResponse(w http.ResponseWriter, req *http.Request, httpErr *errors.HTTPError) { func jsonErrorResponse(w http.ResponseWriter, req *http.Request, httpErr *errors.HTTPError) {

29
src/github.com/matrix-org/go-neb/server/server_test.go

@ -0,0 +1,29 @@
package server
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestProtect(t *testing.T) {
mockWriter := httptest.NewRecorder()
mockReq, _ := http.NewRequest("GET", "http://example.com/foo", nil)
h := Protect(func(w http.ResponseWriter, req *http.Request) {
var array []string
w.Write([]byte(array[5])) // NPE
})
h(mockWriter, mockReq)
expectCode := 500
if mockWriter.Code != expectCode {
t.Errorf("TestProtect wanted HTTP status %d, got %d", expectCode, mockWriter.Code)
}
expectBody := `{"message":"Internal Server Error"}`
actualBody := mockWriter.Body.String()
if actualBody != expectBody {
t.Errorf("TestProtect wanted body %s, got %s", expectBody, actualBody)
}
}
Loading…
Cancel
Save