Refactor JSON Feed parser to use an adapter
This commit is contained in:
parent
6bc4b35e38
commit
8429c6b0ab
4 changed files with 487 additions and 253 deletions
173
internal/reader/json/adapter.go
Normal file
173
internal/reader/json/adapter.go
Normal file
|
@ -0,0 +1,173 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package json // import "miniflux.app/v2/internal/reader/json"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/crypto"
|
||||||
|
"miniflux.app/v2/internal/model"
|
||||||
|
"miniflux.app/v2/internal/reader/date"
|
||||||
|
"miniflux.app/v2/internal/reader/sanitizer"
|
||||||
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
)
|
||||||
|
|
||||||
|
type JSONAdapter struct {
|
||||||
|
jsonFeed *JSONFeed
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewJSONAdapter(jsonFeed *JSONFeed) *JSONAdapter {
|
||||||
|
return &JSONAdapter{jsonFeed}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *JSONAdapter) BuildFeed(feedURL string) *model.Feed {
|
||||||
|
feed := &model.Feed{
|
||||||
|
Title: strings.TrimSpace(j.jsonFeed.Title),
|
||||||
|
FeedURL: j.jsonFeed.FeedURL,
|
||||||
|
SiteURL: j.jsonFeed.HomePageURL,
|
||||||
|
}
|
||||||
|
|
||||||
|
if feed.FeedURL == "" {
|
||||||
|
feed.FeedURL = feedURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to the feed URL if the site URL is empty.
|
||||||
|
if feed.SiteURL == "" {
|
||||||
|
feed.SiteURL = feed.FeedURL
|
||||||
|
}
|
||||||
|
|
||||||
|
if feedURL, err := urllib.AbsoluteURL(feedURL, j.jsonFeed.FeedURL); err == nil {
|
||||||
|
feed.FeedURL = feedURL
|
||||||
|
}
|
||||||
|
|
||||||
|
if siteURL, err := urllib.AbsoluteURL(feedURL, j.jsonFeed.HomePageURL); err == nil {
|
||||||
|
feed.SiteURL = siteURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to the feed URL if the title is empty.
|
||||||
|
if feed.Title == "" {
|
||||||
|
feed.Title = feed.SiteURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Populate the icon URL if present.
|
||||||
|
for _, iconURL := range []string{j.jsonFeed.FaviconURL, j.jsonFeed.IconURL} {
|
||||||
|
iconURL = strings.TrimSpace(iconURL)
|
||||||
|
if iconURL != "" {
|
||||||
|
if absoluteIconURL, err := urllib.AbsoluteURL(feed.SiteURL, iconURL); err == nil {
|
||||||
|
feed.IconURL = absoluteIconURL
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range j.jsonFeed.Items {
|
||||||
|
entry := model.NewEntry()
|
||||||
|
entry.Title = strings.TrimSpace(item.Title)
|
||||||
|
entry.URL = strings.TrimSpace(item.URL)
|
||||||
|
|
||||||
|
// Make sure the entry URL is absolute.
|
||||||
|
if entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL); err == nil {
|
||||||
|
entry.URL = entryURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// The entry title is optional, so we need to find a fallback.
|
||||||
|
if entry.Title == "" {
|
||||||
|
for _, value := range []string{item.Summary, item.ContentText, item.ContentHTML} {
|
||||||
|
if value != "" {
|
||||||
|
entry.Title = sanitizer.TruncateHTML(value, 100)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to the entry URL if the title is empty.
|
||||||
|
if entry.Title == "" {
|
||||||
|
entry.Title = entry.URL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Populate the entry content.
|
||||||
|
for _, value := range []string{item.ContentHTML, item.ContentText, item.Summary} {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value != "" {
|
||||||
|
entry.Content = value
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Populate the entry date.
|
||||||
|
entry.Date = time.Now()
|
||||||
|
for _, value := range []string{item.DatePublished, item.DateModified} {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value != "" {
|
||||||
|
if date, err := date.Parse(value); err != nil {
|
||||||
|
slog.Debug("Unable to parse date from JSON feed",
|
||||||
|
slog.String("date", value),
|
||||||
|
slog.String("url", entry.URL),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
entry.Date = date
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Populate the entry author.
|
||||||
|
itemAuthors := append(item.Authors, j.jsonFeed.Authors...)
|
||||||
|
itemAuthors = append(itemAuthors, item.Author, j.jsonFeed.Author)
|
||||||
|
|
||||||
|
authorNamesMap := make(map[string]bool)
|
||||||
|
for _, author := range itemAuthors {
|
||||||
|
authorName := strings.TrimSpace(author.Name)
|
||||||
|
if authorName != "" {
|
||||||
|
authorNamesMap[authorName] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var authors []string
|
||||||
|
for authorName := range authorNamesMap {
|
||||||
|
authors = append(authors, authorName)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Strings(authors)
|
||||||
|
entry.Author = strings.Join(authors, ", ")
|
||||||
|
|
||||||
|
// Populate the entry enclosures.
|
||||||
|
for _, attachment := range item.Attachments {
|
||||||
|
attachmentURL := strings.TrimSpace(attachment.URL)
|
||||||
|
if attachmentURL != "" {
|
||||||
|
if absoluteAttachmentURL, err := urllib.AbsoluteURL(feed.SiteURL, attachmentURL); err == nil {
|
||||||
|
entry.Enclosures = append(entry.Enclosures, &model.Enclosure{
|
||||||
|
URL: absoluteAttachmentURL,
|
||||||
|
MimeType: attachment.MimeType,
|
||||||
|
Size: attachment.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Populate the entry tags.
|
||||||
|
for _, tag := range item.Tags {
|
||||||
|
tag = strings.TrimSpace(tag)
|
||||||
|
if tag != "" {
|
||||||
|
entry.Tags = append(entry.Tags, tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate a hash for the entry.
|
||||||
|
for _, value := range []string{item.ID, item.URL, item.ContentText + item.ContentHTML + item.Summary} {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value != "" {
|
||||||
|
entry.Hash = crypto.Hash(value)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
feed.Entries = append(feed.Entries, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
return feed
|
||||||
|
}
|
|
@ -3,207 +3,141 @@
|
||||||
|
|
||||||
package json // import "miniflux.app/v2/internal/reader/json"
|
package json // import "miniflux.app/v2/internal/reader/json"
|
||||||
|
|
||||||
import (
|
// JSON Feed specs:
|
||||||
"log/slog"
|
// https://www.jsonfeed.org/version/1.1/
|
||||||
"strings"
|
// https://www.jsonfeed.org/version/1/
|
||||||
"time"
|
type JSONFeed struct {
|
||||||
|
// Version is the URL of the version of the format the feed uses.
|
||||||
|
// This should appear at the very top, though we recognize that not all JSON generators allow for ordering.
|
||||||
|
Version string `json:"version"`
|
||||||
|
|
||||||
"miniflux.app/v2/internal/crypto"
|
// Title is the name of the feed, which will often correspond to the name of the website.
|
||||||
"miniflux.app/v2/internal/model"
|
Title string `json:"title"`
|
||||||
"miniflux.app/v2/internal/reader/date"
|
|
||||||
"miniflux.app/v2/internal/reader/sanitizer"
|
|
||||||
"miniflux.app/v2/internal/urllib"
|
|
||||||
)
|
|
||||||
|
|
||||||
type jsonFeed struct {
|
// HomePageURL is the URL of the resource that the feed describes.
|
||||||
Version string `json:"version"`
|
// This resource may or may not actually be a “home” page, but it should be an HTML page.
|
||||||
Title string `json:"title"`
|
HomePageURL string `json:"home_page_url"`
|
||||||
SiteURL string `json:"home_page_url"`
|
|
||||||
IconURL string `json:"icon"`
|
// FeedURL is the URL of the feed, and serves as the unique identifier for the feed.
|
||||||
FaviconURL string `json:"favicon"`
|
FeedURL string `json:"feed_url"`
|
||||||
FeedURL string `json:"feed_url"`
|
|
||||||
Authors []jsonAuthor `json:"authors"`
|
// Description provides more detail, beyond the title, on what the feed is about.
|
||||||
Author jsonAuthor `json:"author"`
|
Description string `json:"description"`
|
||||||
Items []jsonItem `json:"items"`
|
|
||||||
|
// IconURL is the URL of an image for the feed suitable to be used in a timeline, much the way an avatar might be used.
|
||||||
|
IconURL string `json:"icon"`
|
||||||
|
|
||||||
|
// FaviconURL is the URL of an image for the feed suitable to be used in a source list. It should be square and relatively small.
|
||||||
|
FaviconURL string `json:"favicon"`
|
||||||
|
|
||||||
|
// Authors specifies one or more feed authors. The author object has several members.
|
||||||
|
Authors []JSONAuthor `json:"authors"` // JSON Feed v1.1
|
||||||
|
|
||||||
|
// Author specifies the feed author. The author object has several members.
|
||||||
|
// JSON Feed v1 (deprecated)
|
||||||
|
Author JSONAuthor `json:"author"`
|
||||||
|
|
||||||
|
// Language is the primary language for the feed in the format specified in RFC 5646.
|
||||||
|
// The value is usually a 2-letter language tag from ISO 639-1, optionally followed by a region tag. (Examples: en or en-US.)
|
||||||
|
Language string `json:"language"`
|
||||||
|
|
||||||
|
// Expired is a boolean value that specifies whether or not the feed is finished.
|
||||||
|
Expired bool `json:"expired"`
|
||||||
|
|
||||||
|
// Items is an array, each representing an individual item in the feed.
|
||||||
|
Items []JSONItem `json:"items"`
|
||||||
|
|
||||||
|
// Hubs describes endpoints that can be used to subscribe to real-time notifications from the publisher of this feed.
|
||||||
|
Hubs []JSONHub `json:"hubs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type jsonAuthor struct {
|
type JSONAuthor struct {
|
||||||
|
// Author's name.
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
URL string `json:"url"`
|
|
||||||
|
// Author's website URL (Blog or micro-blog).
|
||||||
|
WebsiteURL string `json:"url"`
|
||||||
|
|
||||||
|
// Author's avatar URL.
|
||||||
|
AvatarURL string `json:"avatar"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type jsonItem struct {
|
type JSONHub struct {
|
||||||
ID string `json:"id"`
|
// Type defines the protocol used to talk with the hub: "rssCloud" or "WebSub".
|
||||||
URL string `json:"url"`
|
Type string `json:"type"`
|
||||||
Title string `json:"title"`
|
|
||||||
Summary string `json:"summary"`
|
// URL is the location of the hub.
|
||||||
Text string `json:"content_text"`
|
URL string `json:"url"`
|
||||||
HTML string `json:"content_html"`
|
|
||||||
DatePublished string `json:"date_published"`
|
|
||||||
DateModified string `json:"date_modified"`
|
|
||||||
Authors []jsonAuthor `json:"authors"`
|
|
||||||
Author jsonAuthor `json:"author"`
|
|
||||||
Attachments []jsonAttachment `json:"attachments"`
|
|
||||||
Tags []string `json:"tags"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type jsonAttachment struct {
|
type JSONItem struct {
|
||||||
URL string `json:"url"`
|
// Unique identifier for the item.
|
||||||
|
// Ideally, the id is the full URL of the resource described by the item, since URLs make great unique identifiers.
|
||||||
|
ID string `json:"id"`
|
||||||
|
|
||||||
|
// URL of the resource described by the item.
|
||||||
|
URL string `json:"url"`
|
||||||
|
|
||||||
|
// ExternalURL is the URL of a page elsewhere.
|
||||||
|
// This is especially useful for linkblogs.
|
||||||
|
// If url links to where you’re talking about a thing, then external_url links to the thing you’re talking about.
|
||||||
|
ExternalURL string `json:"external_url"`
|
||||||
|
|
||||||
|
// Title of the item (optional).
|
||||||
|
// Microblog items in particular may omit titles.
|
||||||
|
Title string `json:"title"`
|
||||||
|
|
||||||
|
// ContentHTML is the HTML body of the item.
|
||||||
|
ContentHTML string `json:"content_html"`
|
||||||
|
|
||||||
|
// ContentText is the text body of the item.
|
||||||
|
ContentText string `json:"content_text"`
|
||||||
|
|
||||||
|
// Summary is a plain text sentence or two describing the item.
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
|
||||||
|
// ImageURL is the URL of the main image for the item.
|
||||||
|
ImageURL string `json:"image"`
|
||||||
|
|
||||||
|
// BannerImageURL is the URL of an image to use as a banner.
|
||||||
|
BannerImageURL string `json:"banner_image"`
|
||||||
|
|
||||||
|
// DatePublished is the date the item was published.
|
||||||
|
DatePublished string `json:"date_published"`
|
||||||
|
|
||||||
|
// DateModified is the date the item was modified.
|
||||||
|
DateModified string `json:"date_modified"`
|
||||||
|
|
||||||
|
// Language is the language of the item.
|
||||||
|
Language string `json:"language"`
|
||||||
|
|
||||||
|
// Authors is an array of JSONAuthor.
|
||||||
|
Authors []JSONAuthor `json:"authors"`
|
||||||
|
|
||||||
|
// Author is a JSONAuthor.
|
||||||
|
// JSON Feed v1 (deprecated)
|
||||||
|
Author JSONAuthor `json:"author"`
|
||||||
|
|
||||||
|
// Tags is an array of strings.
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
|
||||||
|
// Attachments is an array of JSONAttachment.
|
||||||
|
Attachments []JSONAttachment `json:"attachments"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type JSONAttachment struct {
|
||||||
|
// URL of the attachment.
|
||||||
|
URL string `json:"url"`
|
||||||
|
|
||||||
|
// MIME type of the attachment.
|
||||||
MimeType string `json:"mime_type"`
|
MimeType string `json:"mime_type"`
|
||||||
Title string `json:"title"`
|
|
||||||
Size int64 `json:"size_in_bytes"`
|
// Title of the attachment.
|
||||||
Duration int `json:"duration_in_seconds"`
|
Title string `json:"title"`
|
||||||
}
|
|
||||||
|
// Size of the attachment in bytes.
|
||||||
func (j *jsonFeed) GetAuthor() string {
|
Size int64 `json:"size_in_bytes"`
|
||||||
if len(j.Authors) > 0 {
|
|
||||||
return (getAuthor(j.Authors[0]))
|
// Duration of the attachment in seconds.
|
||||||
}
|
Duration int `json:"duration_in_seconds"`
|
||||||
return getAuthor(j.Author)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonFeed) Transform(baseURL string) *model.Feed {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
feed := new(model.Feed)
|
|
||||||
|
|
||||||
feed.FeedURL, err = urllib.AbsoluteURL(baseURL, j.FeedURL)
|
|
||||||
if err != nil {
|
|
||||||
feed.FeedURL = j.FeedURL
|
|
||||||
}
|
|
||||||
|
|
||||||
feed.SiteURL, err = urllib.AbsoluteURL(baseURL, j.SiteURL)
|
|
||||||
if err != nil {
|
|
||||||
feed.SiteURL = j.SiteURL
|
|
||||||
}
|
|
||||||
|
|
||||||
feed.IconURL = strings.TrimSpace(j.IconURL)
|
|
||||||
|
|
||||||
if feed.IconURL == "" {
|
|
||||||
feed.IconURL = strings.TrimSpace(j.FaviconURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
feed.Title = strings.TrimSpace(j.Title)
|
|
||||||
if feed.Title == "" {
|
|
||||||
feed.Title = feed.SiteURL
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, item := range j.Items {
|
|
||||||
entry := item.Transform()
|
|
||||||
entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL)
|
|
||||||
if err == nil {
|
|
||||||
entry.URL = entryURL
|
|
||||||
}
|
|
||||||
|
|
||||||
if entry.Author == "" {
|
|
||||||
entry.Author = j.GetAuthor()
|
|
||||||
}
|
|
||||||
|
|
||||||
feed.Entries = append(feed.Entries, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
return feed
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) GetDate() time.Time {
|
|
||||||
for _, value := range []string{j.DatePublished, j.DateModified} {
|
|
||||||
if value != "" {
|
|
||||||
d, err := date.Parse(value)
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Unable to parse date from JSON feed",
|
|
||||||
slog.String("date", value),
|
|
||||||
slog.String("url", j.URL),
|
|
||||||
slog.Any("error", err),
|
|
||||||
)
|
|
||||||
return time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) GetAuthor() string {
|
|
||||||
if len(j.Authors) > 0 {
|
|
||||||
return getAuthor(j.Authors[0])
|
|
||||||
}
|
|
||||||
return getAuthor(j.Author)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) GetHash() string {
|
|
||||||
for _, value := range []string{j.ID, j.URL, j.Text + j.HTML + j.Summary} {
|
|
||||||
if value != "" {
|
|
||||||
return crypto.Hash(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) GetTitle() string {
|
|
||||||
if j.Title != "" {
|
|
||||||
return j.Title
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, value := range []string{j.Summary, j.Text, j.HTML} {
|
|
||||||
if value != "" {
|
|
||||||
return sanitizer.TruncateHTML(value, 100)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return j.URL
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) GetContent() string {
|
|
||||||
for _, value := range []string{j.HTML, j.Text, j.Summary} {
|
|
||||||
if value != "" {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) GetEnclosures() model.EnclosureList {
|
|
||||||
enclosures := make(model.EnclosureList, 0)
|
|
||||||
|
|
||||||
for _, attachment := range j.Attachments {
|
|
||||||
if attachment.URL == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
enclosures = append(enclosures, &model.Enclosure{
|
|
||||||
URL: attachment.URL,
|
|
||||||
MimeType: attachment.MimeType,
|
|
||||||
Size: attachment.Size,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return enclosures
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *jsonItem) Transform() *model.Entry {
|
|
||||||
entry := model.NewEntry()
|
|
||||||
entry.URL = j.URL
|
|
||||||
entry.Date = j.GetDate()
|
|
||||||
entry.Author = j.GetAuthor()
|
|
||||||
entry.Hash = j.GetHash()
|
|
||||||
entry.Content = j.GetContent()
|
|
||||||
entry.Title = strings.TrimSpace(j.GetTitle())
|
|
||||||
entry.Enclosures = j.GetEnclosures()
|
|
||||||
if len(j.Tags) > 0 {
|
|
||||||
entry.Tags = j.Tags
|
|
||||||
}
|
|
||||||
|
|
||||||
return entry
|
|
||||||
}
|
|
||||||
|
|
||||||
func getAuthor(author jsonAuthor) string {
|
|
||||||
if author.Name != "" {
|
|
||||||
return strings.TrimSpace(author.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
return ""
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,10 +13,10 @@ import (
|
||||||
|
|
||||||
// Parse returns a normalized feed struct from a JSON feed.
|
// Parse returns a normalized feed struct from a JSON feed.
|
||||||
func Parse(baseURL string, data io.Reader) (*model.Feed, error) {
|
func Parse(baseURL string, data io.Reader) (*model.Feed, error) {
|
||||||
feed := new(jsonFeed)
|
jsonFeed := new(JSONFeed)
|
||||||
if err := json.NewDecoder(data).Decode(&feed); err != nil {
|
if err := json.NewDecoder(data).Decode(&jsonFeed); err != nil {
|
||||||
return nil, fmt.Errorf("json: unable to parse feed: %w", err)
|
return nil, fmt.Errorf("json: unable to parse feed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return feed.Transform(baseURL), nil
|
return NewJSONAdapter(jsonFeed).BuildFeed(baseURL), nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseJsonFeed(t *testing.T) {
|
func TestParseJsonFeedVersion1(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -49,7 +49,7 @@ func TestParseJsonFeed(t *testing.T) {
|
||||||
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
|
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
if feed.IconURL != "https://micro.blog/jsonfeed/avatar.jpg" {
|
if feed.IconURL != "https://micro.blog/jsonfeed/favicon.png" {
|
||||||
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
|
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -177,7 +177,81 @@ func TestParsePodcast(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEntryWithoutAttachmentURL(t *testing.T) {
|
func TestParseFeedWithoutTitle(t *testing.T) {
|
||||||
|
data := `{
|
||||||
|
"version": "https://jsonfeed.org/version/1",
|
||||||
|
"home_page_url": "https://example.org/",
|
||||||
|
"feed_url": "https://example.org/feed.json",
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": "2347259",
|
||||||
|
"url": "https://example.org/2347259",
|
||||||
|
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
||||||
|
"date_published": "2016-02-09T14:22:00-07:00"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if feed.Title != "https://example.org/" {
|
||||||
|
t.Errorf("Incorrect title, got: %s", feed.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFeedWithoutHomePage(t *testing.T) {
|
||||||
|
data := `{
|
||||||
|
"version": "https://jsonfeed.org/version/1",
|
||||||
|
"feed_url": "https://example.org/feed.json",
|
||||||
|
"title": "Some test",
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": "2347259",
|
||||||
|
"url": "https://example.org/2347259",
|
||||||
|
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
||||||
|
"date_published": "2016-02-09T14:22:00-07:00"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if feed.SiteURL != "https://example.org/feed.json" {
|
||||||
|
t.Errorf("Incorrect title, got: %s", feed.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFeedWithoutFeedURL(t *testing.T) {
|
||||||
|
data := `{
|
||||||
|
"version": "https://jsonfeed.org/version/1",
|
||||||
|
"title": "Some test",
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": "2347259",
|
||||||
|
"url": "https://example.org/2347259",
|
||||||
|
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
||||||
|
"date_published": "2016-02-09T14:22:00-07:00"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if feed.SiteURL != "https://example.org/feed.json" {
|
||||||
|
t.Errorf("Incorrect title, got: %s", feed.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseItemWithoutAttachmentURL(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"user_comment": "This is a podcast feed. You can add this feed to your podcast client using the following URL: http://therecord.co/feed.json",
|
"user_comment": "This is a podcast feed. You can add this feed to your podcast client using the following URL: http://therecord.co/feed.json",
|
||||||
|
@ -216,7 +290,7 @@ func TestParseEntryWithoutAttachmentURL(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedWithRelativeURL(t *testing.T) {
|
func TestParseItemWithRelativeURL(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "Example",
|
"title": "Example",
|
||||||
|
@ -241,7 +315,7 @@ func TestParseFeedWithRelativeURL(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseAuthor(t *testing.T) {
|
func TestParseItemWithLegacyAuthorField(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
|
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
|
||||||
|
@ -277,7 +351,7 @@ func TestParseAuthor(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseAuthors(t *testing.T) {
|
func TestParseItemWithMultipleAuthorFields(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1.1",
|
"version": "https://jsonfeed.org/version/1.1",
|
||||||
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
|
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
|
||||||
|
@ -285,11 +359,11 @@ func TestParseAuthors(t *testing.T) {
|
||||||
"home_page_url": "https://example.org/",
|
"home_page_url": "https://example.org/",
|
||||||
"feed_url": "https://example.org/feed.json",
|
"feed_url": "https://example.org/feed.json",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "This field is deprecated, use authors",
|
"name": "Deprecated Author Field",
|
||||||
"url": "http://example.org/",
|
"url": "http://example.org/",
|
||||||
"avatar": "https://example.org/avatar.png"
|
"avatar": "https://example.org/avatar.png"
|
||||||
},
|
},
|
||||||
"authors": [
|
"authors": [
|
||||||
{
|
{
|
||||||
"name": "Brent Simmons",
|
"name": "Brent Simmons",
|
||||||
"url": "http://example.org/",
|
"url": "http://example.org/",
|
||||||
|
@ -315,14 +389,15 @@ func TestParseAuthors(t *testing.T) {
|
||||||
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
||||||
}
|
}
|
||||||
|
|
||||||
if feed.Entries[0].Author != "Brent Simmons" {
|
if feed.Entries[0].Author != "Brent Simmons, Deprecated Author Field" {
|
||||||
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
|
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedWithoutTitle(t *testing.T) {
|
func TestParseItemWithMultipleDuplicateAuthors(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1.1",
|
||||||
|
"title": "Example",
|
||||||
"home_page_url": "https://example.org/",
|
"home_page_url": "https://example.org/",
|
||||||
"feed_url": "https://example.org/feed.json",
|
"feed_url": "https://example.org/feed.json",
|
||||||
"items": [
|
"items": [
|
||||||
|
@ -330,7 +405,24 @@ func TestParseFeedWithoutTitle(t *testing.T) {
|
||||||
"id": "2347259",
|
"id": "2347259",
|
||||||
"url": "https://example.org/2347259",
|
"url": "https://example.org/2347259",
|
||||||
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
||||||
"date_published": "2016-02-09T14:22:00-07:00"
|
"date_published": "2016-02-09T14:22:00-07:00",
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Author B",
|
||||||
|
"url": "http://example.org/",
|
||||||
|
"avatar": "https://example.org/avatar.png"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Author A",
|
||||||
|
"url": "http://example.org/",
|
||||||
|
"avatar": "https://example.org/avatar.png"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Author B",
|
||||||
|
"url": "http://example.org/",
|
||||||
|
"avatar": "https://example.org/avatar.png"
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}`
|
}`
|
||||||
|
@ -340,12 +432,16 @@ func TestParseFeedWithoutTitle(t *testing.T) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if feed.Title != "https://example.org/" {
|
if len(feed.Entries) != 1 {
|
||||||
t.Errorf("Incorrect title, got: %s", feed.Title)
|
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
||||||
|
}
|
||||||
|
|
||||||
|
if feed.Entries[0].Author != "Author A, Author B" {
|
||||||
|
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedItemWithInvalidDate(t *testing.T) {
|
func TestParseItemWithInvalidDate(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -376,34 +472,7 @@ func TestParseFeedItemWithInvalidDate(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedItemWithoutID(t *testing.T) {
|
func TestParseItemWithoutTitleButWithURL(t *testing.T) {
|
||||||
data := `{
|
|
||||||
"version": "https://jsonfeed.org/version/1",
|
|
||||||
"title": "My Example Feed",
|
|
||||||
"home_page_url": "https://example.org/",
|
|
||||||
"feed_url": "https://example.org/feed.json",
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"content_text": "Some text."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}`
|
|
||||||
|
|
||||||
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(feed.Entries) != 1 {
|
|
||||||
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
|
||||||
}
|
|
||||||
|
|
||||||
if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" {
|
|
||||||
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseFeedItemWithoutTitleButWithURL(t *testing.T) {
|
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -430,7 +499,7 @@ func TestParseFeedItemWithoutTitleButWithURL(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedItemWithoutTitleButWithSummary(t *testing.T) {
|
func TestParseItemWithoutTitleButWithSummary(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -457,7 +526,7 @@ func TestParseFeedItemWithoutTitleButWithSummary(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedItemWithoutTitleButWithHTMLContent(t *testing.T) {
|
func TestParseItemWithoutTitleButWithHTMLContent(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -484,7 +553,7 @@ func TestParseFeedItemWithoutTitleButWithHTMLContent(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFeedItemWithoutTitleButWithTextContent(t *testing.T) {
|
func TestParseItemWithoutTitleButWithTextContent(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -515,7 +584,7 @@ func TestParseFeedItemWithoutTitleButWithTextContent(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseTruncateItemTitleUnicode(t *testing.T) {
|
func TestParseItemWithTooLongUnicodeTitle(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
|
@ -573,15 +642,34 @@ func TestParseItemTitleWithXMLTags(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseInvalidJSON(t *testing.T) {
|
func TestParseItemWithoutID(t *testing.T) {
|
||||||
data := `garbage`
|
data := `{
|
||||||
_, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
"version": "https://jsonfeed.org/version/1",
|
||||||
if err == nil {
|
"title": "My Example Feed",
|
||||||
t.Error("Parse should returns an error")
|
"home_page_url": "https://example.org/",
|
||||||
|
"feed_url": "https://example.org/feed.json",
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"content_text": "Some text."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(feed.Entries) != 1 {
|
||||||
|
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
|
||||||
|
}
|
||||||
|
|
||||||
|
if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" {
|
||||||
|
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseTags(t *testing.T) {
|
func TestParseItemTags(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
|
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
|
||||||
|
@ -600,7 +688,8 @@ func TestParseTags(t *testing.T) {
|
||||||
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
|
||||||
"date_published": "2016-02-09T14:22:00-07:00",
|
"date_published": "2016-02-09T14:22:00-07:00",
|
||||||
"tags": [
|
"tags": [
|
||||||
"tag 1",
|
" tag 1",
|
||||||
|
" ",
|
||||||
"tag 2"
|
"tag 2"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
@ -623,11 +712,11 @@ func TestParseTags(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseFavicon(t *testing.T) {
|
func TestParseFeedFavicon(t *testing.T) {
|
||||||
data := `{
|
data := `{
|
||||||
"version": "https://jsonfeed.org/version/1",
|
"version": "https://jsonfeed.org/version/1",
|
||||||
"title": "My Example Feed",
|
"title": "My Example Feed",
|
||||||
"favicon": "https://micro.blog/jsonfeed/favicon.png",
|
"favicon": "https://example.org/jsonfeed/favicon.png",
|
||||||
"home_page_url": "https://example.org/",
|
"home_page_url": "https://example.org/",
|
||||||
"feed_url": "https://example.org/feed.json",
|
"feed_url": "https://example.org/feed.json",
|
||||||
"items": [
|
"items": [
|
||||||
|
@ -648,7 +737,45 @@ func TestParseFavicon(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if feed.IconURL != "https://micro.blog/jsonfeed/favicon.png" {
|
if feed.IconURL != "https://example.org/jsonfeed/favicon.png" {
|
||||||
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
|
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseFeedIcon(t *testing.T) {
|
||||||
|
data := `{
|
||||||
|
"version": "https://jsonfeed.org/version/1",
|
||||||
|
"title": "My Example Feed",
|
||||||
|
"icon": "https://example.org/jsonfeed/icon.png",
|
||||||
|
"home_page_url": "https://example.org/",
|
||||||
|
"feed_url": "https://example.org/feed.json",
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": "2",
|
||||||
|
"content_text": "This is a second item.",
|
||||||
|
"url": "https://example.org/second-item"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "1",
|
||||||
|
"content_html": "<p>Hello, world!</p>",
|
||||||
|
"url": "https://example.org/initial-post"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if feed.IconURL != "https://example.org/jsonfeed/icon.png" {
|
||||||
|
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseInvalidJSON(t *testing.T) {
|
||||||
|
data := `garbage`
|
||||||
|
_, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("Parse should returns an error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue