southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Handle RSS feeds without entry links

Browse source
This commit is contained in:
Frédéric Guillot 2017-11-20 15:15:10 -08:00
parent a76c2a8c22
commit cf8af56a99
11 changed files with 83 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
locale/translations.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.283078515 -0800 PST m=+0.039164593
// 2017-11-20 15:11:09.541312146 -0800 PST m=+0.018445746
package locale
@ -59,7 +59,7 @@ var Translations = map[string]string{
"%d days ago": "il y a %d jours",
"%d weeks ago": "il y a %d semaines",
"%d months ago": "il y a %d mois",
"%d years ago": "il y a %d années",
"%d years ago": "il y a %d ans",
"Date": "Date",
"IP Address": "Adresse IP",
"User Agent": "Navigateur Web",
@ -133,5 +133,5 @@ var Translations = map[string]string{
var TranslationsChecksums = map[string]string{
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
"fr_FR": "8b38f8e33f70c463560b49e7f408509304e4d8f61ae78ca26e33bbba28a9ec76",
"fr_FR": "ec81374ce72a1cf3a06bda027faa338745a77035f8bd3504ad4d452958260525",
}

2
locale/translations/fr_FR.json
View file

@ -43,7 +43,7 @@
"%d days ago": "il y a %d jours",
"%d weeks ago": "il y a %d semaines",
"%d months ago": "il y a %d mois",
"%d years ago": "il y a %d années",
"%d years ago": "il y a %d ans",
"Date": "Date",
"IP Address": "Adresse IP",
"User Agent": "Navigateur Web",

11
reader/feed/rss/parser.go
View file

@ -7,22 +7,23 @@ package rss
import (
"encoding/xml"
"fmt"
"github.com/miniflux/miniflux2/model"
"io"
"github.com/miniflux/miniflux2/model"
"golang.org/x/net/html/charset"
)
// Parse returns a normalized feed struct.
func Parse(data io.Reader) (*model.Feed, error) {
rssFeed := new(RssFeed)
feed := new(rssFeed)
decoder := xml.NewDecoder(data)
decoder.CharsetReader = charset.NewReaderLabel
err := decoder.Decode(rssFeed)
err := decoder.Decode(feed)
if err != nil {
return nil, fmt.Errorf("Unable to parse RSS feed: %v", err)
return nil, fmt.Errorf("unable to parse RSS feed: %v", err)
}
return rssFeed.Transform(), nil
return feed.Transform(), nil
}

25
reader/feed/rss/parser_test.go
View file

@ -136,6 +136,31 @@ func TestParseEntryWithoutTitle(t *testing.T) {
}
}
func TestParseEntryWithoutLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<link>https://example.org/</link>
<item>
<guid isPermaLink="false">1234</guid>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].URL != "https://example.org/" {
t.Errorf("Incorrect entry link, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Hash != "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
}
func TestParseFeedURLWithAtomLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">

79
reader/feed/rss/rss.go
View file

@ -6,38 +6,39 @@ package rss
import (
"encoding/xml"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
"log"
"path"
"strconv"
"strings"
"time"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
)
type RssLink struct {
type rssFeed struct {
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Title string `xml:"channel>title"`
Links []rssLink `xml:"channel>link"`
Language string `xml:"channel>language"`
Description string `xml:"channel>description"`
PubDate string `xml:"channel>pubDate"`
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"`
Items []rssItem `xml:"channel>item"`
}
type rssLink struct {
XMLName xml.Name
Data string `xml:",chardata"`
Href string `xml:"href,attr"`
}
type RssFeed struct {
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Title string `xml:"channel>title"`
Links []RssLink `xml:"channel>link"`
Language string `xml:"channel>language"`
Description string `xml:"channel>description"`
PubDate string `xml:"channel>pubDate"`
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"`
Items []RssItem `xml:"channel>item"`
}
type RssItem struct {
Guid string `xml:"guid"`
type rssItem struct {
GUID string `xml:"guid"`
Title string `xml:"title"`
Link string `xml:"link"`
OriginalLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"`
@ -45,25 +46,25 @@ type RssItem struct {
Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"http://purl.org/dc/elements/1.1/ date"`
Authors []RssAuthor `xml:"author"`
Authors []rssAuthor `xml:"author"`
Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
Enclosures []RssEnclosure `xml:"enclosure"`
Enclosures []rssEnclosure `xml:"enclosure"`
OrigEnclosureLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origEnclosureLink"`
}
type RssAuthor struct {
type rssAuthor struct {
XMLName xml.Name
Data string `xml:",chardata"`
Name string `xml:"name"`
}
type RssEnclosure struct {
Url string `xml:"url,attr"`
type rssEnclosure struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Length string `xml:"length,attr"`
}
func (r *RssFeed) GetSiteURL() string {
func (r *rssFeed) GetSiteURL() string {
for _, elem := range r.Links {
if elem.XMLName.Space == "" {
return elem.Data
@ -73,7 +74,7 @@ func (r *RssFeed) GetSiteURL() string {
return ""
}
func (r *RssFeed) GetFeedURL() string {
func (r *rssFeed) GetFeedURL() string {
for _, elem := range r.Links {
if elem.XMLName.Space == "http://www.w3.org/2005/Atom" {
return elem.Href
@ -83,7 +84,7 @@ func (r *RssFeed) GetFeedURL() string {
return ""
}
func (r *RssFeed) Transform() *model.Feed {
func (r *rssFeed) Transform() *model.Feed {
feed := new(model.Feed)
feed.SiteURL = r.GetSiteURL()
feed.FeedURL = r.GetFeedURL()
@ -101,12 +102,16 @@ func (r *RssFeed) Transform() *model.Feed {
}
entry.Author = sanitizer.StripTags(entry.Author)
if entry.URL == "" {
entry.URL = feed.SiteURL
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}
func (i *RssItem) GetDate() time.Time {
func (i *rssItem) GetDate() time.Time {
value := i.PubDate
if i.Date != "" {
value = i.Date
@ -125,7 +130,7 @@ func (i *RssItem) GetDate() time.Time {
return time.Now()
}
func (i *RssItem) GetAuthor() string {
func (i *rssItem) GetAuthor() string {
for _, element := range i.Authors {
if element.Name != "" {
return element.Name
@ -139,8 +144,8 @@ func (i *RssItem) GetAuthor() string {
return i.Creator
}
func (i *RssItem) GetHash() string {
for _, value := range []string{i.Guid, i.Link} {
func (i *rssItem) GetHash() string {
for _, value := range []string{i.GUID, i.Link} {
if value != "" {
return helper.Hash(value)
}
@ -149,7 +154,7 @@ func (i *RssItem) GetHash() string {
return ""
}
func (i *RssItem) GetContent() string {
func (i *rssItem) GetContent() string {
if i.Content != "" {
return i.Content
}
@ -157,7 +162,7 @@ func (i *RssItem) GetContent() string {
return i.Description
}
func (i *RssItem) GetURL() string {
func (i *rssItem) GetURL() string {
if i.OriginalLink != "" {
return i.OriginalLink
}
@ -165,12 +170,12 @@ func (i *RssItem) GetURL() string {
return i.Link
}
func (i *RssItem) GetEnclosures() model.EnclosureList {
func (i *rssItem) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
for _, enclosure := range i.Enclosures {
length, _ := strconv.Atoi(enclosure.Length)
enclosureURL := enclosure.Url
enclosureURL := enclosure.URL
if i.OrigEnclosureLink != "" {
filename := path.Base(i.OrigEnclosureLink)
@ -189,7 +194,7 @@ func (i *RssItem) GetEnclosures() model.EnclosureList {
return enclosures
}
func (i *RssItem) Transform() *model.Entry {
func (i *rssItem) Transform() *model.Entry {
entry := new(model.Entry)
entry.URL = i.GetURL()
entry.Date = i.GetDate()

2
server/static/bin.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.249018092 -0800 PST m=+0.005104170
// 2017-11-20 15:11:09.5282708 -0800 PST m=+0.005404400
package static

2
server/static/css.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.25118969 -0800 PST m=+0.007275768
// 2017-11-20 15:11:09.529138622 -0800 PST m=+0.006272222
package static

2
server/static/js.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.255571671 -0800 PST m=+0.011657749
// 2017-11-20 15:11:09.530927342 -0800 PST m=+0.008060942
package static

2
server/template/common.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.28176883 -0800 PST m=+0.037854908
// 2017-11-20 15:11:09.54007082 -0800 PST m=+0.017204420
package template

2
server/template/views.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.257809595 -0800 PST m=+0.013895673
// 2017-11-20 15:11:09.531826622 -0800 PST m=+0.008960222
package template

2
sql/sql.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.247054842 -0800 PST m=+0.003140920
// 2017-11-20 15:11:09.527012847 -0800 PST m=+0.004146447
package sql