southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Handle RSS entries with Atom links

Browse source
This commit is contained in:
Frédéric Guillot 2017-11-20 15:48:26 -08:00
parent cf8af56a99
commit 557cf9c21d
3 changed files with 114 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
reader/feed/rss/parser.go
View file

@ -14,7 +14,7 @@ import (
"golang.org/x/net/html/charset"
)
// Parse returns a normalized feed struct.
// Parse returns a normalized feed struct from a RSS feed.
func Parse(data io.Reader) (*model.Feed, error) {
feed := new(rssFeed)
decoder := xml.NewDecoder(data)

53
reader/feed/rss/parser_test.go
View file

@ -161,6 +161,51 @@ func TestParseEntryWithoutLink(t *testing.T) {
}
}
func TestParseEntryWithAtomLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<link>https://example.org/</link>
<item>
<title>Test</title>
<atom:link href="https://example.org/item" />
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].URL != "https://example.org/item" {
t.Errorf("Incorrect entry link, got: %s", feed.Entries[0].URL)
}
}
func TestParseEntryWithMultipleAtomLinks(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<link>https://example.org/</link>
<item>
<title>Test</title>
<atom:link rel="payment" href="https://example.org/a" />
<atom:link rel="http://foobar.tld" href="https://example.org/b" />
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].URL != "https://example.org/b" {
t.Errorf("Incorrect entry link, got: %s", feed.Entries[0].URL)
}
}
func TestParseFeedURLWithAtomLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
@ -489,3 +534,11 @@ func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
}
func TestParseInvalidXml(t *testing.T) {
data := `garbage`
_, err := Parse(bytes.NewBufferString(data))
if err == nil {
t.Error("Parse should returns an error")
}
}

97
reader/feed/rss/rss.go
View file

@ -35,12 +35,13 @@ type rssLink struct {
XMLName xml.Name
Data string `xml:",chardata"`
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
}
type rssItem struct {
GUID string `xml:"guid"`
Title string `xml:"title"`
Link string `xml:"link"`
Links []rssLink `xml:"link"`
OriginalLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"`
Description string `xml:"description"`
Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
@ -65,9 +66,9 @@ type rssEnclosure struct {
}
func (r *rssFeed) GetSiteURL() string {
for _, elem := range r.Links {
if elem.XMLName.Space == "" {
return elem.Data
for _, element := range r.Links {
if element.XMLName.Space == "" {
return element.Data
}
}
@ -75,9 +76,9 @@ func (r *rssFeed) GetSiteURL() string {
}
func (r *rssFeed) GetFeedURL() string {
for _, elem := range r.Links {
if elem.XMLName.Space == "http://www.w3.org/2005/Atom" {
return elem.Href
for _, element := range r.Links {
if element.XMLName.Space == "http://www.w3.org/2005/Atom" {
return element.Href
}
}
@ -111,10 +112,10 @@ func (r *rssFeed) Transform() *model.Feed {
return feed
}
func (i *rssItem) GetDate() time.Time {
value := i.PubDate
if i.Date != "" {
value = i.Date
func (r *rssItem) GetDate() time.Time {
value := r.PubDate
if r.Date != "" {
value = r.Date
}
if value != "" {
@ -130,8 +131,8 @@ func (i *rssItem) GetDate() time.Time {
return time.Now()
}
func (i *rssItem) GetAuthor() string {
for _, element := range i.Authors {
func (r *rssItem) GetAuthor() string {
for _, element := range r.Authors {
if element.Name != "" {
return element.Name
}
@ -141,11 +142,11 @@ func (i *rssItem) GetAuthor() string {
}
}
return i.Creator
return r.Creator
}
func (i *rssItem) GetHash() string {
for _, value := range []string{i.GUID, i.Link} {
func (r *rssItem) GetHash() string {
for _, value := range []string{r.GUID, r.GetURL()} {
if value != "" {
return helper.Hash(value)
}
@ -154,33 +155,43 @@ func (i *rssItem) GetHash() string {
return ""
}
func (i *rssItem) GetContent() string {
if i.Content != "" {
return i.Content
func (r *rssItem) GetContent() string {
if r.Content != "" {
return r.Content
}
return i.Description
return r.Description
}
func (i *rssItem) GetURL() string {
if i.OriginalLink != "" {
return i.OriginalLink
func (r *rssItem) GetURL() string {
if r.OriginalLink != "" {
return r.OriginalLink
}
return i.Link
for _, link := range r.Links {
if link.XMLName.Space == "http://www.w3.org/2005/Atom" && link.Href != "" && isValidLinkRelation(link.Rel) {
return link.Href
}
if link.Data != "" {
return link.Data
}
}
return ""
}
func (i *rssItem) GetEnclosures() model.EnclosureList {
func (r *rssItem) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
for _, enclosure := range i.Enclosures {
for _, enclosure := range r.Enclosures {
length, _ := strconv.Atoi(enclosure.Length)
enclosureURL := enclosure.URL
if i.OrigEnclosureLink != "" {
filename := path.Base(i.OrigEnclosureLink)
if r.OrigEnclosureLink != "" {
filename := path.Base(r.OrigEnclosureLink)
if strings.Contains(enclosureURL, filename) {
enclosureURL = i.OrigEnclosureLink
enclosureURL = r.OrigEnclosureLink
}
}
@ -194,15 +205,15 @@ func (i *rssItem) GetEnclosures() model.EnclosureList {
return enclosures
}
func (i *rssItem) Transform() *model.Entry {
func (r *rssItem) Transform() *model.Entry {
entry := new(model.Entry)
entry.URL = i.GetURL()
entry.Date = i.GetDate()
entry.Author = i.GetAuthor()
entry.Hash = i.GetHash()
entry.Content = processor.ItemContentProcessor(entry.URL, i.GetContent())
entry.Title = sanitizer.StripTags(strings.Trim(i.Title, " \n\t"))
entry.Enclosures = i.GetEnclosures()
entry.URL = r.GetURL()
entry.Date = r.GetDate()
entry.Author = r.GetAuthor()
entry.Hash = r.GetHash()
entry.Content = processor.ItemContentProcessor(entry.URL, r.GetContent())
entry.Title = sanitizer.StripTags(strings.Trim(r.Title, " \n\t"))
entry.Enclosures = r.GetEnclosures()
if entry.Title == "" {
entry.Title = entry.URL
@ -210,3 +221,15 @@ func (i *rssItem) Transform() *model.Entry {
return entry
}
func isValidLinkRelation(rel string) bool {
switch rel {
case "", "alternate", "enclosure", "related", "self", "via":
return true
default:
if strings.HasPrefix(rel, "http") {
return true
}
return false
}
}