From 87ccad5c7f1edf8bce37af547ca1659326398fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Sun, 10 Dec 2017 20:51:04 -0800 Subject: [PATCH] Add scraper rules --- locale/translations.go | 7 ++-- locale/translations/fr_FR.json | 3 +- model/feed.go | 1 + reader/scraper/rules.go | 16 +++++++++ reader/scraper/scraper.go | 56 +++++++++++++++++++++++++++-- reader/scraper/scraper_test.go | 21 +++++++++++ server/template/html/edit_feed.html | 3 ++ server/template/views.go | 7 ++-- server/ui/controller/entry.go | 10 ++---- server/ui/controller/feed.go | 9 ++--- server/ui/form/feed.go | 19 +++++----- sql/schema_version_6.sql | 1 + sql/sql.go | 5 ++- storage/entry_query_builder.go | 3 +- storage/feed.go | 11 +++--- storage/migration.go | 2 +- 16 files changed, 140 insertions(+), 34 deletions(-) create mode 100644 reader/scraper/rules.go create mode 100644 reader/scraper/scraper_test.go create mode 100644 sql/schema_version_6.sql diff --git a/locale/translations.go b/locale/translations.go index 217541ee..7124f1b6 100644 --- a/locale/translations.go +++ b/locale/translations.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-12-10 18:56:24.387844114 -0800 PST m=+0.029823201 +// 2017-12-10 20:08:14.447304303 -0800 PST m=+0.040286758 package locale @@ -167,12 +167,13 @@ var translations = map[string]string{ "Activate Fever API": "Activer l'API de Fever", "Fever Username": "Nom d'utilisateur pour l'API de Fever", "Fever Password": "Mot de passe pour l'API de Fever", - "Fetch original content": "Récupérer le contenu original" + "Fetch original content": "Récupérer le contenu original", + "Scraper Rules": "Règles pour récupérer le contenu original" } `, } var translationsChecksums = map[string]string{ "en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897", - "fr_FR": "fd629b171aefa50dd0a6100acaac8fbecbdf1a1d53e3fce984234565ec5bb5d5", + "fr_FR": "4426cea875ee2c9acb1a2b0619cb82f3a32f71aabe5d07657eaf2f6b7387c5f9", } diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json index cc82efed..0a51ec3c 100644 --- a/locale/translations/fr_FR.json +++ b/locale/translations/fr_FR.json @@ -151,5 +151,6 @@ "Activate Fever API": "Activer l'API de Fever", "Fever Username": "Nom d'utilisateur pour l'API de Fever", "Fever Password": "Mot de passe pour l'API de Fever", - "Fetch original content": "Récupérer le contenu original" + "Fetch original content": "Récupérer le contenu original", + "Scraper Rules": "Règles pour récupérer le contenu original" } diff --git a/model/feed.go b/model/feed.go index dbdb9d6a..fb2819da 100644 --- a/model/feed.go +++ b/model/feed.go @@ -22,6 +22,7 @@ type Feed struct { LastModifiedHeader string `json:"last_modified_header,omitempty"` ParsingErrorMsg string `json:"parsing_error_message,omitempty"` ParsingErrorCount int `json:"parsing_error_count,omitempty"` + ScraperRules string `json:"scraper_rules"` Category *Category `json:"category,omitempty"` Entries Entries `json:"entries,omitempty"` Icon *FeedIcon `json:"icon,omitempty"` diff --git a/reader/scraper/rules.go b/reader/scraper/rules.go new file mode 100644 index 00000000..ae6c4a57 --- /dev/null +++ b/reader/scraper/rules.go @@ -0,0 +1,16 @@ +// Copyright 2017 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package scraper + +// List of predefined scraper rules (alphabetically sorted) +// domain => CSS selectors +var predefinedRules = map[string]string{ + "lemonde.fr": "div#articleBody", + "lesjoiesducode.fr": ".blog-post-content img", + "linux.com": "div.content, div[property]", + "opensource.com": "div[property]", + "phoronix.com": "div.content", + "techcrunch.com": "div.article-entry", +} diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go index 6c518621..b79a088a 100644 --- a/reader/scraper/scraper.go +++ b/reader/scraper/scraper.go @@ -6,14 +6,19 @@ package scraper import ( "errors" + "io" + "log" + "strings" + "github.com/PuerkitoBio/goquery" "github.com/miniflux/miniflux2/http" "github.com/miniflux/miniflux2/reader/readability" "github.com/miniflux/miniflux2/reader/sanitizer" + "github.com/miniflux/miniflux2/url" ) // Fetch download a web page a returns relevant contents. -func Fetch(websiteURL string) (string, error) { +func Fetch(websiteURL, rules string) (string, error) { client := http.NewClient(websiteURL) response, err := client.Get() if err != nil { @@ -29,10 +34,57 @@ func Fetch(websiteURL string) (string, error) { return "", err } - content, err := readability.ExtractContent(page) + var content string + if rules == "" { + rules = getPredefinedScraperRules(websiteURL) + } + + if rules != "" { + log.Printf(`[Scraper] Using rules "%s" for "%s"`, rules, websiteURL) + content, err = scrapContent(page, rules) + } else { + log.Printf(`[Scraper] Using readability for "%s"`, websiteURL) + content, err = readability.ExtractContent(page) + } + if err != nil { return "", err } return sanitizer.Sanitize(websiteURL, content), nil } + +func scrapContent(page io.Reader, rules string) (string, error) { + document, err := goquery.NewDocumentFromReader(page) + if err != nil { + return "", err + } + + contents := "" + document.Find(rules).Each(func(i int, s *goquery.Selection) { + var content string + + // For some inline elements, we get the parent. + if s.Is("img") { + content, _ = s.Parent().Html() + } else { + content, _ = s.Html() + } + + contents += content + }) + + return contents, nil +} + +func getPredefinedScraperRules(websiteURL string) string { + urlDomain := url.Domain(websiteURL) + + for domain, rules := range predefinedRules { + if strings.Contains(urlDomain, domain) { + return rules + } + } + + return "" +} diff --git a/reader/scraper/scraper_test.go b/reader/scraper/scraper_test.go new file mode 100644 index 00000000..b493e25c --- /dev/null +++ b/reader/scraper/scraper_test.go @@ -0,0 +1,21 @@ +// Copyright 2017 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package scraper + +import "testing" + +func TestGetPredefinedRules(t *testing.T) { + if getPredefinedScraperRules("http://www.phoronix.com/") == "" { + t.Error("Unable to find rule for phoronix.com") + } + + if getPredefinedScraperRules("https://www.linux.com/") == "" { + t.Error("Unable to find rule for linux.com") + } + + if getPredefinedScraperRules("https://example.org/") != "" { + t.Error("A rule not defined should not return anything") + } +} diff --git a/server/template/html/edit_feed.html b/server/template/html/edit_feed.html index fac2a9b7..04950926 100644 --- a/server/template/html/edit_feed.html +++ b/server/template/html/edit_feed.html @@ -45,6 +45,9 @@ + + + + + +