mirror of https://github.com/matrix-org/go-neb.git
Browse Source
Merge pull request #76 from matrix-org/kegan/rss
Merge pull request #76 from matrix-org/kegan/rss
Implement polling for Servicespull/68/merge
Kegsay
8 years ago
committed by
GitHub
GitHub
12 changed files with 330 additions and 29 deletions
-
12src/github.com/matrix-org/go-neb/api.go
-
13src/github.com/matrix-org/go-neb/database/db.go
-
27src/github.com/matrix-org/go-neb/database/schema.go
-
5src/github.com/matrix-org/go-neb/goneb.go
-
103src/github.com/matrix-org/go-neb/polling/polling.go
-
7src/github.com/matrix-org/go-neb/services/echo/echo.go
-
5src/github.com/matrix-org/go-neb/services/giphy/giphy.go
-
7src/github.com/matrix-org/go-neb/services/github/github.go
-
7src/github.com/matrix-org/go-neb/services/github/github_webhook.go
-
2src/github.com/matrix-org/go-neb/services/jira/jira.go
-
111src/github.com/matrix-org/go-neb/services/rss/rss.go
-
48src/github.com/matrix-org/go-neb/types/types.go
@ -0,0 +1,103 @@ |
|||
package polling |
|||
|
|||
import ( |
|||
"fmt" |
|||
log "github.com/Sirupsen/logrus" |
|||
"github.com/matrix-org/go-neb/database" |
|||
"github.com/matrix-org/go-neb/types" |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
// Remember when we first started polling on this service ID. Polling routines will
|
|||
// continually check this time. If the service gets updated, this will change, prompting
|
|||
// older instances to die away. If this service gets removed, the time will be 0.
|
|||
var ( |
|||
pollMutex sync.Mutex |
|||
startPollTime = make(map[string]int64) // ServiceID => unix timestamp
|
|||
) |
|||
|
|||
// Start polling already existing services
|
|||
func Start() error { |
|||
// Work out which service types require polling
|
|||
for serviceType, poller := range types.PollersByType() { |
|||
if poller == nil { |
|||
continue |
|||
} |
|||
// Query for all services with said service type
|
|||
srvs, err := database.GetServiceDB().LoadServicesByType(serviceType) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
for _, s := range srvs { |
|||
if err := StartPolling(s); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// StartPolling begins a polling loop for this service.
|
|||
// If one already exists for this service, it will be instructed to die. The new poll will not wait for this to happen,
|
|||
// so there may be a brief period of overlap. It is safe to immediately call `StopPolling(service)` to immediately terminate
|
|||
// this poll.
|
|||
func StartPolling(service types.Service) error { |
|||
p := types.PollersByType()[service.ServiceType()] |
|||
if p == nil { |
|||
return fmt.Errorf("Service %s (type=%s) doesn't have a Poller", service.ServiceID(), service.ServiceType()) |
|||
} |
|||
// Set the poll time BEFORE spinning off the goroutine in case the caller immediately stops us. If we don't do this here,
|
|||
// we risk them setting the ts to 0 BEFORE we've set the start time, resulting in a poll when one was not intended.
|
|||
ts := time.Now().UnixNano() |
|||
setPollStartTime(service, ts) |
|||
go pollLoop(service, p, ts) |
|||
return nil |
|||
} |
|||
|
|||
// StopPolling stops all pollers for this service.
|
|||
func StopPolling(service types.Service) { |
|||
log.WithFields(log.Fields{ |
|||
"service_id": service.ServiceID(), |
|||
"service_type": service.ServiceType(), |
|||
}).Info("StopPolling") |
|||
setPollStartTime(service, 0) |
|||
} |
|||
|
|||
// pollLoop begins the polling loop for this service. Does not return, so call this
|
|||
// as a goroutine!
|
|||
func pollLoop(service types.Service, poller types.Poller, ts int64) { |
|||
logger := log.WithFields(log.Fields{ |
|||
"timestamp": ts, |
|||
"service_id": service.ServiceID(), |
|||
"service_type": service.ServiceType(), |
|||
"interval_secs": poller.IntervalSecs(), |
|||
}) |
|||
logger.Info("Starting polling loop") |
|||
for { |
|||
poller.OnPoll(service) |
|||
if pollTimeChanged(service, ts) { |
|||
logger.Info("Terminating poll.") |
|||
break |
|||
} |
|||
time.Sleep(time.Duration(poller.IntervalSecs()) * time.Second) |
|||
if pollTimeChanged(service, ts) { |
|||
logger.Info("Terminating poll.") |
|||
break |
|||
} |
|||
} |
|||
} |
|||
|
|||
// setPollStartTime clobbers the current poll time
|
|||
func setPollStartTime(service types.Service, startTs int64) { |
|||
pollMutex.Lock() |
|||
defer pollMutex.Unlock() |
|||
startPollTime[service.ServiceID()] = startTs |
|||
} |
|||
|
|||
// pollTimeChanged returns true if the poll start time for this service ID is different to the one supplied.
|
|||
func pollTimeChanged(service types.Service, ts int64) bool { |
|||
pollMutex.Lock() |
|||
defer pollMutex.Unlock() |
|||
return startPollTime[service.ServiceID()] != ts |
|||
} |
|||
@ -0,0 +1,111 @@ |
|||
package services |
|||
|
|||
import ( |
|||
"errors" |
|||
log "github.com/Sirupsen/logrus" |
|||
"github.com/matrix-org/go-neb/database" |
|||
"github.com/matrix-org/go-neb/matrix" |
|||
"github.com/matrix-org/go-neb/polling" |
|||
"github.com/matrix-org/go-neb/types" |
|||
"time" |
|||
) |
|||
|
|||
type rssPoller struct{} |
|||
|
|||
func (p *rssPoller) IntervalSecs() int64 { return 10 } |
|||
func (p *rssPoller) OnPoll(s types.Service) { |
|||
rsss, ok := s.(*rssService) |
|||
if !ok { |
|||
log.WithField("service_id", s.ServiceID()).Error("RSS: OnPoll called without an RSS Service") |
|||
return |
|||
} |
|||
now := time.Now().Unix() // Second resolution
|
|||
// URL => [ RoomID ]
|
|||
urlsToRooms := make(map[string][]string) |
|||
|
|||
for roomID, roomInfo := range rsss.Rooms { |
|||
for u, feedInfo := range roomInfo.Feeds { |
|||
if feedInfo.LastPollTimestampSecs == 0 || (feedInfo.LastPollTimestampSecs+(int64(feedInfo.PollIntervalMins)*60)) > now { |
|||
// re-query this feed
|
|||
urlsToRooms[u] = append(urlsToRooms[u], roomID) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// TODO: Some polling
|
|||
} |
|||
|
|||
type rssService struct { |
|||
types.DefaultService |
|||
id string |
|||
serviceUserID string |
|||
Rooms map[string]struct { // room_id => {}
|
|||
Feeds map[string]struct { // URL => { }
|
|||
PollIntervalMins int `json:"poll_interval_mins"` |
|||
LastPollTimestampSecs int64 |
|||
} |
|||
} |
|||
} |
|||
|
|||
func (s *rssService) ServiceUserID() string { return s.serviceUserID } |
|||
func (s *rssService) ServiceID() string { return s.id } |
|||
func (s *rssService) ServiceType() string { return "rss" } |
|||
func (s *rssService) Poller() types.Poller { return &rssPoller{} } |
|||
|
|||
// Register will check the liveness of each RSS feed given. If all feeds check out okay, no error is returned.
|
|||
func (s *rssService) Register(oldService types.Service, client *matrix.Client) error { |
|||
feeds := feedUrls(s) |
|||
if len(feeds) == 0 { |
|||
// this is an error UNLESS the old service had some feeds in which case they are deleting us :(
|
|||
oldFeeds := feedUrls(oldService) |
|||
if len(oldFeeds) == 0 { |
|||
return errors.New("An RSS feed must be specified.") |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (s *rssService) PostRegister(oldService types.Service) { |
|||
if len(feedUrls(s)) == 0 { // bye-bye :(
|
|||
logger := log.WithFields(log.Fields{ |
|||
"service_id": s.ServiceID(), |
|||
"service_type": s.ServiceType(), |
|||
}) |
|||
logger.Info("Deleting service (0 feeds)") |
|||
polling.StopPolling(s) |
|||
if err := database.GetServiceDB().DeleteService(s.ServiceID()); err != nil { |
|||
logger.WithError(err).Error("Failed to delete service") |
|||
} |
|||
} |
|||
} |
|||
|
|||
// feedUrls returns a list of feed urls for this service
|
|||
func feedUrls(srv types.Service) []string { |
|||
var feeds []string |
|||
s, ok := srv.(*rssService) |
|||
if !ok { |
|||
return feeds |
|||
} |
|||
|
|||
urlSet := make(map[string]bool) |
|||
for _, roomInfo := range s.Rooms { |
|||
for u := range roomInfo.Feeds { |
|||
urlSet[u] = true |
|||
} |
|||
} |
|||
|
|||
for u := range urlSet { |
|||
feeds = append(feeds, u) |
|||
} |
|||
return feeds |
|||
} |
|||
|
|||
func init() { |
|||
types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service { |
|||
r := &rssService{ |
|||
id: serviceID, |
|||
serviceUserID: serviceUserID, |
|||
} |
|||
return r |
|||
}) |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue