From 07e93c5ba2443f2d975d6eb7f9811b279aee2234 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 27 Oct 2016 14:47:39 +0100 Subject: [PATCH 1/3] Add failing TestHTMLEntities test --- .../go-neb/services/rssbot/rssbot_test.go | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go new file mode 100644 index 0000000..19bbac5 --- /dev/null +++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go @@ -0,0 +1,120 @@ +package services + +import ( + "bytes" + "encoding/json" + "errors" + "github.com/matrix-org/go-neb/database" + "github.com/matrix-org/go-neb/matrix" + "github.com/matrix-org/go-neb/types" + _ "github.com/mattn/go-sqlite3" + "io/ioutil" + "net/http" + "net/url" + "strings" + "sync" + "testing" + "time" +) + +type MockTransport struct { + roundTrip func(*http.Request) (*http.Response, error) +} + +func (t MockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return t.roundTrip(req) +} + +func TestHTMLEntities(t *testing.T) { + // FIXME: Make ServiceDB an interface so we don't need to do this and import sqlite3! + // We are NOT interested in db operations, but need them because OnPoll will + // call StoreService. + db, err := database.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal("Failed to create in-memory db: ", err) + return + } + database.SetServiceDB(db) + + feedURL := "https://thehappymaskshop.hyrule" + // Replace the cachingClient with a mock so we can intercept RSS requests + rssTrans := struct{ MockTransport }{} + rssTrans.roundTrip = func(req *http.Request) (*http.Response, error) { + if req.URL.String() != feedURL { + return nil, errors.New("Unknown test URL") + } + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(` + + + + + Mask Shop + + New Item: Majora’s Mask + http://go.neb/rss/majoras-mask + + + `)), + }, nil + } + cachingClient = &http.Client{Transport: rssTrans} + + // Create the RSS service + srv, err := types.CreateService("id", "rssbot", "@happy_mask_salesman:hyrule", []byte( + `{"feeds": {"`+feedURL+`":{}}}`, // no config yet + )) + if err != nil { + t.Fatal("Failed to create RSS bot: ", err) + } + rssbot := srv.(*rssBotService) + f := rssbot.Feeds[feedURL] + f.Rooms = []string{"!linksroom:hyrule"} + f.FeedUpdatedTimestampSecs = 12345 + f.NextPollTimestampSecs = time.Now().Unix() + rssbot.Feeds[feedURL] = f + + // Create the Matrix client which will send the notification + wg := sync.WaitGroup{} + wg.Add(1) + matrixTrans := struct{ MockTransport }{} + matrixTrans.roundTrip = func(req *http.Request) (*http.Response, error) { + if strings.HasPrefix(req.URL.Path, "/_matrix/client/r0/rooms/!linksroom:hyrule/send/m.room.message") { + // Check content body to make sure it is decoded + var msg matrix.HTMLMessage + if err := json.NewDecoder(req.Body).Decode(&msg); err != nil { + t.Fatal("Failed to decode request JSON: ", err) + return nil, errors.New("Error handling matrix client test request") + } + want := "New Item: Majora's Mask" + if !strings.Contains(msg.Body, want) { + t.Errorf("TestHTMLEntities: want '%s' in body, got '%s'", want, msg.Body) + } + + wg.Done() + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(` + {"event_id":"$123456:hyrule"} + `)), + }, nil + } + return nil, errors.New("Unhandled matrix client test request") + } + u, _ := url.Parse("https://hyrule") + matrixClient := matrix.NewClient(&http.Client{Transport: matrixTrans}, u, "its_a_secret", "@happy_mask_salesman:hyrule") + + // Invoke OnPoll to trigger the RSS feed update + _ = rssbot.OnPoll(matrixClient) + + // Check that the Matrix client sent a message + wg.Wait() +} From fcd3befb09b023422e9b993ca7161b4f840c2426 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 27 Oct 2016 15:09:19 +0100 Subject: [PATCH 2/3] HTML decode the RSS title/description fields --- .../matrix-org/go-neb/services/rssbot/rssbot.go | 9 +++++++++ .../matrix-org/go-neb/services/rssbot/rssbot_test.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go index 1533e86..613e65e 100644 --- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go +++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot.go @@ -279,6 +279,15 @@ func (s *rssBotService) newItems(feedURL string, allItems []*gofeed.Item) (items continue } + // Decode HTML for and <description>: + // The RSS 2.0 Spec http://cyber.harvard.edu/rss/rss.html#hrelementsOfLtitemgt supports a bunch + // of weird ways to put HTML into <title> and <description> tags. Not all RSS feed producers run + // these fields through entity encoders (some have ' unencoded, others have it as ’). We'll + // assume that all RSS fields are sending HTML for these fields and run them through a standard decoder. + // This will inevitably break for some people, but that group of people are probably smaller, so *shrug*. + i.Title = html.UnescapeString(i.Title) + i.Description = html.UnescapeString(i.Description) + items = append(items, *i) } return diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go index 19bbac5..d75d096 100644 --- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go +++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go @@ -94,7 +94,7 @@ func TestHTMLEntities(t *testing.T) { t.Fatal("Failed to decode request JSON: ", err) return nil, errors.New("Error handling matrix client test request") } - want := "New Item: Majora's Mask" + want := "New Item: Majora’s Mask" if !strings.Contains(msg.Body, want) { t.Errorf("TestHTMLEntities: want '%s' in body, got '%s'", want, msg.Body) } From 6fd0f20c819ca687df3b99b2930e8c681363f4c6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal <kegan@matrix.org> Date: Thu, 27 Oct 2016 15:37:31 +0100 Subject: [PATCH 3/3] Review comments --- .../go-neb/services/rssbot/rssbot_test.go | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go index d75d096..0e64751 100644 --- a/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go +++ b/src/github.com/matrix-org/go-neb/services/rssbot/rssbot_test.go @@ -17,6 +17,25 @@ import ( "time" ) +const rssFeedXML = ` +<?xml version="1.0" encoding="UTF-8"?> +<rss version="2.0" + xmlns:content="http://purl.org/rss/1.0/modules/content/" + xmlns:wfw="http://wellformedweb.org/CommentAPI/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:atom="http://www.w3.org/2005/Atom" + xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" + xmlns:slash="http://purl.org/rss/1.0/modules/slash/" + > +<channel> + <title>Mask Shop + + New Item: Majora’s Mask + http://go.neb/rss/majoras-mask + + +` + type MockTransport struct { roundTrip func(*http.Request) (*http.Response, error) } @@ -45,25 +64,7 @@ func TestHTMLEntities(t *testing.T) { } return &http.Response{ StatusCode: 200, - Body: ioutil.NopCloser(bytes.NewBufferString(` - - - - - Mask Shop - - New Item: Majora’s Mask - http://go.neb/rss/majoras-mask - - - `)), + Body: ioutil.NopCloser(bytes.NewBufferString(rssFeedXML)), }, nil } cachingClient = &http.Client{Transport: rssTrans} @@ -76,9 +77,11 @@ func TestHTMLEntities(t *testing.T) { t.Fatal("Failed to create RSS bot: ", err) } rssbot := srv.(*rssBotService) + + // Configure the service to force OnPoll to query the RSS feed and attempt to send results + // to the right room. f := rssbot.Feeds[feedURL] f.Rooms = []string{"!linksroom:hyrule"} - f.FeedUpdatedTimestampSecs = 12345 f.NextPollTimestampSecs = time.Now().Unix() rssbot.Feeds[feedURL] = f @@ -94,7 +97,7 @@ func TestHTMLEntities(t *testing.T) { t.Fatal("Failed to decode request JSON: ", err) return nil, errors.New("Error handling matrix client test request") } - want := "New Item: Majora’s Mask" + want := "New Item: Majora\u2019s Mask" // 0x2019 = 8217 if !strings.Contains(msg.Body, want) { t.Errorf("TestHTMLEntities: want '%s' in body, got '%s'", want, msg.Body) }