|
@ -37,8 +37,9 @@ type rssBotService struct { |
|
|
Feeds map[string]struct { // feed_url => { }
|
|
|
Feeds map[string]struct { // feed_url => { }
|
|
|
PollIntervalMins int `json:"poll_interval_mins"` |
|
|
PollIntervalMins int `json:"poll_interval_mins"` |
|
|
Rooms []string `json:"rooms"` |
|
|
Rooms []string `json:"rooms"` |
|
|
|
|
|
IsFailing bool `json:"is_failing"` // True if rss bot is unable to poll this feed
|
|
|
|
|
|
FeedUpdatedTimestampSecs int64 `json:"last_updated_ts_secs"` // The time of the last successful poll
|
|
|
NextPollTimestampSecs int64 // Internal: When we should poll again
|
|
|
NextPollTimestampSecs int64 // Internal: When we should poll again
|
|
|
FeedUpdatedTimestampSecs int64 // Internal: The last time the feed was updated
|
|
|
|
|
|
RecentGUIDs []string // Internal: The most recently seen GUIDs. Sized to the number of items in the feed.
|
|
|
RecentGUIDs []string // Internal: The most recently seen GUIDs. Sized to the number of items in the feed.
|
|
|
} `json:"feeds"` |
|
|
} `json:"feeds"` |
|
|
} |
|
|
} |
|
@ -190,11 +191,11 @@ func (s *rssBotService) nextTimestamp() time.Time { |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
// Don't allow times in the past. Set a min re-poll threshold of 20s to avoid
|
|
|
|
|
|
|
|
|
// Don't allow times in the past. Set a min re-poll threshold of 60s to avoid
|
|
|
// tight-looping on feeds which 500.
|
|
|
// tight-looping on feeds which 500.
|
|
|
now := time.Now().Unix() |
|
|
now := time.Now().Unix() |
|
|
if earliestNextTs <= now { |
|
|
if earliestNextTs <= now { |
|
|
earliestNextTs = now + 20 |
|
|
|
|
|
|
|
|
earliestNextTs = now + 60 |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
return time.Unix(earliestNextTs, 0) |
|
|
return time.Unix(earliestNextTs, 0) |
|
@ -208,6 +209,9 @@ func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, |
|
|
fp.Client = cachingClient |
|
|
fp.Client = cachingClient |
|
|
feed, err := fp.ParseURL(feedURL) |
|
|
feed, err := fp.ParseURL(feedURL) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
|
|
|
f := s.Feeds[feedURL] |
|
|
|
|
|
f.IsFailing = true |
|
|
|
|
|
s.Feeds[feedURL] = f |
|
|
return nil, items, err |
|
|
return nil, items, err |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
@ -227,25 +231,12 @@ func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, |
|
|
// Work out which items are new, if any (based on the last updated TS we have)
|
|
|
// Work out which items are new, if any (based on the last updated TS we have)
|
|
|
// If the TS is 0 then this is the first ever poll, so let's not send 10s of events
|
|
|
// If the TS is 0 then this is the first ever poll, so let's not send 10s of events
|
|
|
// into the room and just do new ones from this point onwards.
|
|
|
// into the room and just do new ones from this point onwards.
|
|
|
if s.Feeds[feedURL].FeedUpdatedTimestampSecs != 0 { |
|
|
|
|
|
|
|
|
if s.Feeds[feedURL].NextPollTimestampSecs != 0 { |
|
|
items = s.newItems(feedURL, feed.Items) |
|
|
items = s.newItems(feedURL, feed.Items) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
now := time.Now().Unix() // Second resolution
|
|
|
now := time.Now().Unix() // Second resolution
|
|
|
|
|
|
|
|
|
// Work out when this feed was last updated
|
|
|
|
|
|
var feedLastUpdatedTs int64 |
|
|
|
|
|
if feed.UpdatedParsed != nil { |
|
|
|
|
|
feedLastUpdatedTs = feed.UpdatedParsed.Unix() |
|
|
|
|
|
} else if len(feed.Items) > 0 { |
|
|
|
|
|
i := feed.Items[0] |
|
|
|
|
|
if i != nil && i.PublishedParsed != nil { |
|
|
|
|
|
feedLastUpdatedTs = i.PublishedParsed.Unix() |
|
|
|
|
|
} else { |
|
|
|
|
|
feedLastUpdatedTs = time.Now().Unix() |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Work out when to next poll this feed
|
|
|
// Work out when to next poll this feed
|
|
|
nextPollTsSec := now + minPollingIntervalSeconds |
|
|
nextPollTsSec := now + minPollingIntervalSeconds |
|
|
if s.Feeds[feedURL].PollIntervalMins > int(minPollingIntervalSeconds/60) { |
|
|
if s.Feeds[feedURL].PollIntervalMins > int(minPollingIntervalSeconds/60) { |
|
@ -254,7 +245,20 @@ func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, |
|
|
// TODO: Handle the 'sy' Syndication extension to control update interval.
|
|
|
// TODO: Handle the 'sy' Syndication extension to control update interval.
|
|
|
// See http://www.feedforall.com/syndication.htm and http://web.resource.org/rss/1.0/modules/syndication/
|
|
|
// See http://www.feedforall.com/syndication.htm and http://web.resource.org/rss/1.0/modules/syndication/
|
|
|
|
|
|
|
|
|
s.updateFeedInfo(feedURL, feed.Items, nextPollTsSec, feedLastUpdatedTs) |
|
|
|
|
|
|
|
|
// map items to guid strings
|
|
|
|
|
|
var guids []string |
|
|
|
|
|
for _, itm := range feed.Items { |
|
|
|
|
|
guids = append(guids, itm.GUID) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Update the service config to persist the new times
|
|
|
|
|
|
f := s.Feeds[feedURL] |
|
|
|
|
|
f.NextPollTimestampSecs = nextPollTsSec |
|
|
|
|
|
f.FeedUpdatedTimestampSecs = now |
|
|
|
|
|
f.RecentGUIDs = guids |
|
|
|
|
|
f.IsFailing = false |
|
|
|
|
|
s.Feeds[feedURL] = f |
|
|
|
|
|
|
|
|
return feed, items, nil |
|
|
return feed, items, nil |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
@ -280,25 +284,6 @@ func (s *rssBotService) newItems(feedURL string, allItems []*gofeed.Item) (items |
|
|
return |
|
|
return |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func (s *rssBotService) updateFeedInfo(feedURL string, allFeedItems []*gofeed.Item, nextPollTs, feedUpdatedTs int64) { |
|
|
|
|
|
// map items to guid strings
|
|
|
|
|
|
var guids []string |
|
|
|
|
|
for _, i := range allFeedItems { |
|
|
|
|
|
guids = append(guids, i.GUID) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
for u := range s.Feeds { |
|
|
|
|
|
if u != feedURL { |
|
|
|
|
|
continue |
|
|
|
|
|
} |
|
|
|
|
|
f := s.Feeds[u] |
|
|
|
|
|
f.NextPollTimestampSecs = nextPollTs |
|
|
|
|
|
f.FeedUpdatedTimestampSecs = feedUpdatedTs |
|
|
|
|
|
f.RecentGUIDs = guids |
|
|
|
|
|
s.Feeds[u] = f |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func (s *rssBotService) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error { |
|
|
func (s *rssBotService) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error { |
|
|
logger := log.WithField("feed_url", feedURL).WithField("title", item.Title) |
|
|
logger := log.WithField("feed_url", feedURL).WithField("title", item.Title) |
|
|
logger.Info("New feed item") |
|
|
logger.Info("New feed item") |
|
|