From 38ef6c591d0065488b75931dee0d44d17f6862c2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 4 May 2017 16:30:31 +0100 Subject: [PATCH] Increase the max number of GUIDs we keep per RSS feed Some feeds have >1k items such as: - https://dashboard.downdetector.com/events/qfu1JvvwVwYf1OvgvORuD4HQMt5xMF2Y/217:405-5025.xml If Go-NEB tracks this feed, it will continually send N-1000 items into the matrix room because it doesn't remember all the GUIDs. This is only a hack to fix this problem for now, to avoid DoSing Synapse. --- src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go index a0112af..7d65271 100644 --- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go +++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go @@ -281,13 +281,13 @@ func (s *Service) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error) // Some RSS feeds can return a very small number of items then bounce // back to their "normal" size, so we cannot just clobber the recent GUID list per request or else we'll // forget what we sent and resend it. Instead, we'll keep 2x the max number of items that we've ever - // seen from this feed, up to a max of 1000. + // seen from this feed, up to a max of 10,000. maxGuids := 2 * len(feed.Items) if len(f.RecentGUIDs) > maxGuids { maxGuids = len(f.RecentGUIDs) // already 2x'd. } - if maxGuids > 1000 { - maxGuids = 1000 + if maxGuids > 10000 { + maxGuids = 10000 } lastSet := uniqueStrings(f.RecentGUIDs) // e.g. [4,5,6]