Merge branch 'master' into kegan/config-yaml

src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go

@@ -37,8 +37,9 @@ type rssBotService struct {
 	Feeds         map[string]struct { // feed_url => { }
 		PollIntervalMins         int      `json:"poll_interval_mins"`
 		Rooms                    []string `json:"rooms"`
+		IsFailing                bool     `json:"is_failing"`           // True if rss bot is unable to poll this feed
+		FeedUpdatedTimestampSecs int64    `json:"last_updated_ts_secs"` // The time of the last successful poll
 		NextPollTimestampSecs    int64    // Internal: When we should poll again
-		FeedUpdatedTimestampSecs int64    // Internal: The last time the feed was updated
 		RecentGUIDs              []string // Internal: The most recently seen GUIDs. Sized to the number of items in the feed.
 	} `json:"feeds"`
 }
@@ -190,11 +191,11 @@ func (s *rssBotService) nextTimestamp() time.Time {
 		}
 	}
 
-	// Don't allow times in the past. Set a min re-poll threshold of 20s to avoid
+	// Don't allow times in the past. Set a min re-poll threshold of 60s to avoid
 	// tight-looping on feeds which 500.
 	now := time.Now().Unix()
 	if earliestNextTs <= now {
-		earliestNextTs = now + 20
+		earliestNextTs = now + 60
 	}
 
 	return time.Unix(earliestNextTs, 0)
@@ -208,6 +209,9 @@ func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item,
 	fp.Client = cachingClient
 	feed, err := fp.ParseURL(feedURL)
 	if err != nil {
+		f := s.Feeds[feedURL]
+		f.IsFailing = true
+		s.Feeds[feedURL] = f
 		return nil, items, err
 	}
 
@@ -227,25 +231,12 @@ func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item,
 	// Work out which items are new, if any (based on the last updated TS we have)
 	// If the TS is 0 then this is the first ever poll, so let's not send 10s of events
 	// into the room and just do new ones from this point onwards.
-	if s.Feeds[feedURL].FeedUpdatedTimestampSecs != 0 {
+	if s.Feeds[feedURL].NextPollTimestampSecs != 0 {
 		items = s.newItems(feedURL, feed.Items)
 	}
 
 	now := time.Now().Unix() // Second resolution
 
-	// Work out when this feed was last updated
-	var feedLastUpdatedTs int64
-	if feed.UpdatedParsed != nil {
-		feedLastUpdatedTs = feed.UpdatedParsed.Unix()
-	} else if len(feed.Items) > 0 {
-		i := feed.Items[0]
-		if i != nil && i.PublishedParsed != nil {
-			feedLastUpdatedTs = i.PublishedParsed.Unix()
-		} else {
-			feedLastUpdatedTs = time.Now().Unix()
-		}
-	}
-
 	// Work out when to next poll this feed
 	nextPollTsSec := now + minPollingIntervalSeconds
 	if s.Feeds[feedURL].PollIntervalMins > int(minPollingIntervalSeconds/60) {
@@ -254,7 +245,20 @@ func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item,
 	// TODO: Handle the 'sy' Syndication extension to control update interval.
 	// See http://www.feedforall.com/syndication.htm and http://web.resource.org/rss/1.0/modules/syndication/
 
-	s.updateFeedInfo(feedURL, feed.Items, nextPollTsSec, feedLastUpdatedTs)
+	// map items to guid strings
+	var guids []string
+	for _, itm := range feed.Items {
+		guids = append(guids, itm.GUID)
+	}
+
+	// Update the service config to persist the new times
+	f := s.Feeds[feedURL]
+	f.NextPollTimestampSecs = nextPollTsSec
+	f.FeedUpdatedTimestampSecs = now
+	f.RecentGUIDs = guids
+	f.IsFailing = false
+	s.Feeds[feedURL] = f
+
 	return feed, items, nil
 }
 
@@ -280,25 +284,6 @@ func (s *rssBotService) newItems(feedURL string, allItems []*gofeed.Item) (items
 	return
 }
 
-func (s *rssBotService) updateFeedInfo(feedURL string, allFeedItems []*gofeed.Item, nextPollTs, feedUpdatedTs int64) {
-	// map items to guid strings
-	var guids []string
-	for _, i := range allFeedItems {
-		guids = append(guids, i.GUID)
-	}
-
-	for u := range s.Feeds {
-		if u != feedURL {
-			continue
-		}
-		f := s.Feeds[u]
-		f.NextPollTimestampSecs = nextPollTs
-		f.FeedUpdatedTimestampSecs = feedUpdatedTs
-		f.RecentGUIDs = guids
-		s.Feeds[u] = f
-	}
-}
-
 func (s *rssBotService) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error {
 	logger := log.WithField("feed_url", feedURL).WithField("title", item.Title)
 	logger.Info("New feed item")