Add feed filters (Keeplist and Blocklist)
This commit is contained in:
Manuel Garrido
GitHub
parent
3afdf25012
commit
84b83fc3c8
34 changed files with 359 additions and 89 deletions
|
|
@ -51,6 +51,8 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
|
|||
feedInfo.Password,
|
||||
feedInfo.ScraperRules,
|
||||
feedInfo.RewriteRules,
|
||||
feedInfo.BlocklistRules,
|
||||
feedInfo.KeeplistRules,
|
||||
feedInfo.FetchViaProxy,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -24,15 +24,17 @@ type entriesResponse struct {
|
|||
}
|
||||
|
||||
type feedCreation struct {
|
||||
FeedURL string `json:"feed_url"`
|
||||
CategoryID int64 `json:"category_id"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
Crawler bool `json:"crawler"`
|
||||
FetchViaProxy bool `json:"fetch_via_proxy"`
|
||||
ScraperRules string `json:"scraper_rules"`
|
||||
RewriteRules string `json:"rewrite_rules"`
|
||||
FeedURL string `json:"feed_url"`
|
||||
CategoryID int64 `json:"category_id"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
Crawler bool `json:"crawler"`
|
||||
FetchViaProxy bool `json:"fetch_via_proxy"`
|
||||
ScraperRules string `json:"scraper_rules"`
|
||||
RewriteRules string `json:"rewrite_rules"`
|
||||
BlocklistRules string `json:"blocklist_rules"`
|
||||
KeeplistRules string `json:"keeplist_rules"`
|
||||
}
|
||||
|
||||
type subscriptionDiscovery struct {
|
||||
|
|
@ -44,17 +46,19 @@ type subscriptionDiscovery struct {
|
|||
}
|
||||
|
||||
type feedModification struct {
|
||||
FeedURL *string `json:"feed_url"`
|
||||
SiteURL *string `json:"site_url"`
|
||||
Title *string `json:"title"`
|
||||
ScraperRules *string `json:"scraper_rules"`
|
||||
RewriteRules *string `json:"rewrite_rules"`
|
||||
Crawler *bool `json:"crawler"`
|
||||
UserAgent *string `json:"user_agent"`
|
||||
Username *string `json:"username"`
|
||||
Password *string `json:"password"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
Disabled *bool `json:"disabled"`
|
||||
FeedURL *string `json:"feed_url"`
|
||||
SiteURL *string `json:"site_url"`
|
||||
Title *string `json:"title"`
|
||||
ScraperRules *string `json:"scraper_rules"`
|
||||
RewriteRules *string `json:"rewrite_rules"`
|
||||
BlocklistRules *string `json:"blocklist_rules"`
|
||||
KeeplistRules *string `json:"keeplist_rules"`
|
||||
Crawler *bool `json:"crawler"`
|
||||
UserAgent *string `json:"user_agent"`
|
||||
Username *string `json:"username"`
|
||||
Password *string `json:"password"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
Disabled *bool `json:"disabled"`
|
||||
}
|
||||
|
||||
func (f *feedModification) Update(feed *model.Feed) {
|
||||
|
|
@ -78,6 +82,14 @@ func (f *feedModification) Update(feed *model.Feed) {
|
|||
feed.RewriteRules = *f.RewriteRules
|
||||
}
|
||||
|
||||
if f.KeeplistRules != nil {
|
||||
feed.KeeplistRules = *f.KeeplistRules
|
||||
}
|
||||
|
||||
if f.BlocklistRules != nil {
|
||||
feed.BlocklistRules = *f.BlocklistRules
|
||||
}
|
||||
|
||||
if f.Crawler != nil {
|
||||
feed.Crawler = *f.Crawler
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ type Feed struct {
|
|||
ParsingErrorCount int `json:"parsing_error_count,omitempty"`
|
||||
ScraperRules string `json:"scraper_rules"`
|
||||
RewriteRules string `json:"rewrite_rules"`
|
||||
BlocklistRules string `json:"blocklist_rules"`
|
||||
KeeplistRules string `json:"keeplist_rules"`
|
||||
Crawler bool `json:"crawler"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Username string `json:"username"`
|
||||
|
|
@ -101,16 +103,18 @@ type Feed struct {
|
|||
|
||||
// FeedModification represents changes for a feed.
|
||||
type FeedModification struct {
|
||||
FeedURL *string `json:"feed_url"`
|
||||
SiteURL *string `json:"site_url"`
|
||||
Title *string `json:"title"`
|
||||
ScraperRules *string `json:"scraper_rules"`
|
||||
RewriteRules *string `json:"rewrite_rules"`
|
||||
Crawler *bool `json:"crawler"`
|
||||
UserAgent *string `json:"user_agent"`
|
||||
Username *string `json:"username"`
|
||||
Password *string `json:"password"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
FeedURL *string `json:"feed_url"`
|
||||
SiteURL *string `json:"site_url"`
|
||||
Title *string `json:"title"`
|
||||
ScraperRules *string `json:"scraper_rules"`
|
||||
RewriteRules *string `json:"rewrite_rules"`
|
||||
BlocklistRules *string `json:"blocklist_rules"`
|
||||
KeeplistRules *string `json:"keeplist_rules"`
|
||||
Crawler *bool `json:"crawler"`
|
||||
UserAgent *string `json:"user_agent"`
|
||||
Username *string `json:"username"`
|
||||
Password *string `json:"password"`
|
||||
CategoryID *int64 `json:"category_id"`
|
||||
}
|
||||
|
||||
// FeedIcon represents the feed icon.
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
"miniflux.app/logger"
|
||||
)
|
||||
|
||||
const schemaVersion = 39
|
||||
const schemaVersion = 40
|
||||
|
||||
// Migrate executes database migrations.
|
||||
func Migrate(db *sql.DB) {
|
||||
|
|
|
|||
|
|
@ -197,6 +197,11 @@ create index entries_user_feed_idx on entries (user_id, feed_id);
|
|||
`,
|
||||
"schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc');
|
||||
alter table users add column entry_direction entry_sorting_direction default 'asc';
|
||||
`,
|
||||
"schema_version_40": `alter table feeds
|
||||
add column blocklist_rules text not null default '',
|
||||
add column keeplist_rules text not null default ''
|
||||
;
|
||||
`,
|
||||
"schema_version_5": `create table integrations (
|
||||
user_id int not null,
|
||||
|
|
@ -258,6 +263,7 @@ var SqlMapChecksums = map[string]string{
|
|||
"schema_version_38": "e91d2f4075ceb7b8a16a25f350f36dee12cfd1ad86b8b6414c4cf2e9a003358c",
|
||||
"schema_version_39": "b0f90b97502921d4681a07c64d180a91a0b4ccac7d3c1dbe30519ad6f1bf1737",
|
||||
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
|
||||
"schema_version_40": "6a8fec92399f853ed6817aff4cfa43255dce4c19afad796e41519d09de62105e",
|
||||
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",
|
||||
"schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4",
|
||||
"schema_version_7": "33f298c9aa30d6de3ca28e1270df51c2884d7596f1283a75716e2aeb634cd05c",
|
||||
|
|
|
|||
4
database/sql/schema_version_40.sql
Normal file
4
database/sql/schema_version_40.sql
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
alter table feeds
|
||||
add column blocklist_rules text not null default '',
|
||||
add column keeplist_rules text not null default ''
|
||||
;
|
||||
11
go.sum
11
go.sum
|
|
@ -348,7 +348,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BG
|
|||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -362,7 +361,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -371,14 +369,12 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c h1:38q6VNPWR010vN82/SB121GujZNIfAUb4YttE2rhGuc=
|
||||
golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
|
|
@ -399,14 +395,11 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
|
@ -428,16 +421,13 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
|
|||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
|
|
@ -457,7 +447,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -253,6 +253,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
|
||||
"form.feed.label.scraper_rules": "Extraktionsregeln",
|
||||
"form.feed.label.rewrite_rules": "Umschreiberegeln",
|
||||
"form.feed.label.blocklist_rules": "Regeln blockieren",
|
||||
"form.feed.label.keeplist_rules": "Regeln einhalten",
|
||||
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
|
||||
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
|
||||
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
|
||||
|
|
@ -600,6 +602,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Override Default User Agent",
|
||||
"form.feed.label.scraper_rules": "Scraper Rules",
|
||||
"form.feed.label.rewrite_rules": "Rewrite Rules",
|
||||
"form.feed.label.blocklist_rules": "Block Rules",
|
||||
"form.feed.label.keeplist_rules": "Keep Rules",
|
||||
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
|
||||
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
|
||||
"form.feed.label.disabled": "Do not refresh this feed",
|
||||
|
|
@ -927,6 +931,9 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
|
||||
"form.feed.label.scraper_rules": "Reglas de raspador",
|
||||
"form.feed.label.rewrite_rules": "Reglas de reescribir",
|
||||
"form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",
|
||||
"form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)",
|
||||
"form.feed.label.blocklist_rules": "Reglas de Blacklist",
|
||||
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
|
||||
"form.feed.label.disabled": "No actualice este feed",
|
||||
|
|
@ -1254,6 +1261,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
|
||||
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
|
||||
"form.feed.label.rewrite_rules": "Règles de réécriture",
|
||||
"form.feed.label.blocklist_rules": "Règles de blocage",
|
||||
"form.feed.label.keeplist_rules": "Règles d'autorisation",
|
||||
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
|
||||
"form.feed.label.disabled": "Ne pas actualiser ce flux",
|
||||
|
|
@ -1601,6 +1610,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Usa user agent personalizzato",
|
||||
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
|
||||
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
|
||||
"form.feed.label.blocklist_rules": "Regole di blocco",
|
||||
"form.feed.label.keeplist_rules": "Regole di autorizzazione",
|
||||
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
|
||||
"form.feed.label.disabled": "Non aggiornare questo feed",
|
||||
|
|
@ -1928,6 +1939,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "ディフォルトの User Agent を上書きする",
|
||||
"form.feed.label.scraper_rules": "スクラップルール",
|
||||
"form.feed.label.rewrite_rules": "Rewrite ルール",
|
||||
"form.feed.label.blocklist_rules": "ブロックルール",
|
||||
"form.feed.label.keeplist_rules": "許可規則",
|
||||
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
|
||||
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
|
||||
"form.feed.label.disabled": "このフィードを更新しない",
|
||||
|
|
@ -2255,6 +2268,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Standaard User Agent overschrijven",
|
||||
"form.feed.label.scraper_rules": "Scraper regels",
|
||||
"form.feed.label.rewrite_rules": "Rewrite regels",
|
||||
"form.feed.label.blocklist_rules": "Blokkeer regels",
|
||||
"form.feed.label.keeplist_rules": "toestemmingsregels",
|
||||
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
|
||||
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
|
||||
"form.feed.label.disabled": "Vernieuw deze feed niet",
|
||||
|
|
@ -2602,6 +2617,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
|
||||
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
|
||||
"form.feed.label.rewrite_rules": "Reguły zapisu",
|
||||
"form.feed.label.blocklist_rules": "Zasady blokowania",
|
||||
"form.feed.label.keeplist_rules": "Zasady zezwoleń",
|
||||
"form.feed.label.ignore_http_cache": "Zignoruj pamięć podręczną HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
|
||||
"form.feed.label.disabled": "Не обновлять этот канал",
|
||||
|
|
@ -2953,6 +2970,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
|
||||
"form.feed.label.scraper_rules": "Regras do scraper",
|
||||
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
|
||||
"form.feed.label.blocklist_rules": "Regras de bloqueio",
|
||||
"form.feed.label.keeplist_rules": "Regras de permissão",
|
||||
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
|
||||
"form.feed.label.disabled": "Não atualizar esta fonte",
|
||||
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
|
||||
|
|
@ -3282,6 +3301,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
|
||||
"form.feed.label.scraper_rules": "Правила Scraper",
|
||||
"form.feed.label.rewrite_rules": "Правила Rewrite",
|
||||
"form.feed.label.blocklist_rules": "Правила блокировки",
|
||||
"form.feed.label.keeplist_rules": "правила разрешений",
|
||||
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
|
||||
"form.feed.label.fetch_via_proxy": "Получить через прокси",
|
||||
"form.feed.label.disabled": "Не обновлять этот канал",
|
||||
|
|
@ -3613,6 +3634,8 @@ var translations = map[string]string{
|
|||
"form.feed.label.user_agent": "覆盖默认 User-Agent",
|
||||
"form.feed.label.scraper_rules": "Scraper 规则",
|
||||
"form.feed.label.rewrite_rules": "重写规则",
|
||||
"form.feed.label.blocklist_rules": "封锁规则",
|
||||
"form.feed.label.keeplist_rules": "许可规则",
|
||||
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
|
||||
"form.feed.label.fetch_via_proxy": "通过代理获取",
|
||||
"form.feed.label.disabled": "请勿刷新此Feed",
|
||||
|
|
@ -3706,15 +3729,15 @@ var translations = map[string]string{
|
|||
}
|
||||
|
||||
var translationsChecksums = map[string]string{
|
||||
"de_DE": "dbd9141c3b2f436d16ff660735904897c235406094f0e7d70cbceb509c6063b6",
|
||||
"en_US": "61ef12ca29c271905a594ba1fe600871c930e7cfa25f0721bf076cd056cf60d4",
|
||||
"es_ES": "74e1b08adc78c9bd2fe05df69d19232fca86cb5fc94379ee42cef34878e4b8e5",
|
||||
"fr_FR": "3b61ad39a8d5227aac7b49897a6f78828c7e4a2040fe9636af932515c89d9785",
|
||||
"it_IT": "048498833a7c9bb519cde0a61f955e15cbcf117b2006955305fb23cc652ed9f4",
|
||||
"ja_JP": "cd8cac53f606066d60b3bc5586c0a484ee97466e3cca4816228ce74faf5916fe",
|
||||
"nl_NL": "b76c996efdbc814688ad3db50a906397d45717b19f49a4702b4e65f50df23600",
|
||||
"pl_PL": "a27aecbb6cd96d0714b2c934f5d96e01ab229e031e9e5ea0de96cec8c5cae0bf",
|
||||
"pt_BR": "8e3cdaa0e39485a6eb5cf3942ec23b8f6a16e8efc551f52a35f74e444ebd70e3",
|
||||
"ru_RU": "92d224b6563777efe64e1972884547f27e7374074b9f4f1fc2eeb331f6c5597c",
|
||||
"zh_CN": "de59177700df00c27352d785c62f66ee49955b8b294f0eaf944ad7c4025838c7",
|
||||
"de_DE": "8acdf65175293ab3684f945582f9f4adb442b5008afba3312a100bb75c7910d8",
|
||||
"en_US": "0fdf8969fa460ffb6c21ad8afd624eef130dcd1bb18c489e3a7fd38e6ed2d563",
|
||||
"es_ES": "86e8bf5fab817c536aa73bdf2aa0f5c09ddad6f7457b662a5e091304815aac05",
|
||||
"fr_FR": "020deb5822f8bec9d94c9b9a387651efaef21f21239e7c203d76c71260bbbd54",
|
||||
"it_IT": "77c5963a80c173e2257909b47e46cf7f1ecbbae773c61454fcc987d8b26e4a2a",
|
||||
"ja_JP": "f2b6b61ac847f17031ab34284a3f3b6355c03fd1534164024e33d1659126e227",
|
||||
"nl_NL": "4dba877744bb92911fc488c3cfa47aad605501d1ba351f7136b92cb4872fbd34",
|
||||
"pl_PL": "10acf265cea096636f640d0e275b136fd06cf992d8a306eb418d6dcd21cb8b24",
|
||||
"pt_BR": "dc853281eb52ae1225a36a9bd2c6bb3c62304834b3b1c092edcdec86823468b4",
|
||||
"ru_RU": "46d039f6d50e48d841d09285d28810c609cd62dd821da4c68798490d57d1d090",
|
||||
"zh_CN": "80e8a170246b0b90dffda393d897760ecb4e94b264cacc124db2be6845ac9361",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
|
||||
"form.feed.label.scraper_rules": "Extraktionsregeln",
|
||||
"form.feed.label.rewrite_rules": "Umschreiberegeln",
|
||||
"form.feed.label.blocklist_rules": "Regeln blockieren",
|
||||
"form.feed.label.keeplist_rules": "Regeln einhalten",
|
||||
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
|
||||
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
|
||||
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "Override Default User Agent",
|
||||
"form.feed.label.scraper_rules": "Scraper Rules",
|
||||
"form.feed.label.rewrite_rules": "Rewrite Rules",
|
||||
"form.feed.label.blocklist_rules": "Block Rules",
|
||||
"form.feed.label.keeplist_rules": "Keep Rules",
|
||||
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
|
||||
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
|
||||
"form.feed.label.disabled": "Do not refresh this feed",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,9 @@
|
|||
"form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
|
||||
"form.feed.label.scraper_rules": "Reglas de raspador",
|
||||
"form.feed.label.rewrite_rules": "Reglas de reescribir",
|
||||
"form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",
|
||||
"form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)",
|
||||
"form.feed.label.blocklist_rules": "Reglas de Blacklist",
|
||||
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
|
||||
"form.feed.label.disabled": "No actualice este feed",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
|
||||
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
|
||||
"form.feed.label.rewrite_rules": "Règles de réécriture",
|
||||
"form.feed.label.blocklist_rules": "Règles de blocage",
|
||||
"form.feed.label.keeplist_rules": "Règles d'autorisation",
|
||||
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
|
||||
"form.feed.label.disabled": "Ne pas actualiser ce flux",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "Usa user agent personalizzato",
|
||||
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
|
||||
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
|
||||
"form.feed.label.blocklist_rules": "Regole di blocco",
|
||||
"form.feed.label.keeplist_rules": "Regole di autorizzazione",
|
||||
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
|
||||
"form.feed.label.disabled": "Non aggiornare questo feed",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "ディフォルトの User Agent を上書きする",
|
||||
"form.feed.label.scraper_rules": "スクラップルール",
|
||||
"form.feed.label.rewrite_rules": "Rewrite ルール",
|
||||
"form.feed.label.blocklist_rules": "ブロックルール",
|
||||
"form.feed.label.keeplist_rules": "許可規則",
|
||||
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
|
||||
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
|
||||
"form.feed.label.disabled": "このフィードを更新しない",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "Standaard User Agent overschrijven",
|
||||
"form.feed.label.scraper_rules": "Scraper regels",
|
||||
"form.feed.label.rewrite_rules": "Rewrite regels",
|
||||
"form.feed.label.blocklist_rules": "Blokkeer regels",
|
||||
"form.feed.label.keeplist_rules": "toestemmingsregels",
|
||||
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
|
||||
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
|
||||
"form.feed.label.disabled": "Vernieuw deze feed niet",
|
||||
|
|
|
|||
|
|
@ -250,6 +250,8 @@
|
|||
"form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
|
||||
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
|
||||
"form.feed.label.rewrite_rules": "Reguły zapisu",
|
||||
"form.feed.label.blocklist_rules": "Zasady blokowania",
|
||||
"form.feed.label.keeplist_rules": "Zasady zezwoleń",
|
||||
"form.feed.label.ignore_http_cache": "Zignoruj pamięć podręczną HTTP",
|
||||
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
|
||||
"form.feed.label.disabled": "Не обновлять этот канал",
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@
|
|||
"form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
|
||||
"form.feed.label.scraper_rules": "Regras do scraper",
|
||||
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
|
||||
"form.feed.label.blocklist_rules": "Regras de bloqueio",
|
||||
"form.feed.label.keeplist_rules": "Regras de permissão",
|
||||
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
|
||||
"form.feed.label.disabled": "Não atualizar esta fonte",
|
||||
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
|
||||
|
|
|
|||
|
|
@ -250,6 +250,8 @@
|
|||
"form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
|
||||
"form.feed.label.scraper_rules": "Правила Scraper",
|
||||
"form.feed.label.rewrite_rules": "Правила Rewrite",
|
||||
"form.feed.label.blocklist_rules": "Правила блокировки",
|
||||
"form.feed.label.keeplist_rules": "правила разрешений",
|
||||
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
|
||||
"form.feed.label.fetch_via_proxy": "Получить через прокси",
|
||||
"form.feed.label.disabled": "Не обновлять этот канал",
|
||||
|
|
|
|||
|
|
@ -246,6 +246,8 @@
|
|||
"form.feed.label.user_agent": "覆盖默认 User-Agent",
|
||||
"form.feed.label.scraper_rules": "Scraper 规则",
|
||||
"form.feed.label.rewrite_rules": "重写规则",
|
||||
"form.feed.label.blocklist_rules": "封锁规则",
|
||||
"form.feed.label.keeplist_rules": "许可规则",
|
||||
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
|
||||
"form.feed.label.fetch_via_proxy": "通过代理获取",
|
||||
"form.feed.label.disabled": "请勿刷新此Feed",
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ type Feed struct {
|
|||
ScraperRules string `json:"scraper_rules"`
|
||||
RewriteRules string `json:"rewrite_rules"`
|
||||
Crawler bool `json:"crawler"`
|
||||
BlocklistRules string `json:"blocklist_rules"`
|
||||
KeeplistRules string `json:"keeplist_rules"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
|
|
@ -72,7 +74,7 @@ func (f *Feed) WithCategoryID(categoryID int64) {
|
|||
}
|
||||
|
||||
// WithBrowsingParameters defines browsing parameters.
|
||||
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) {
|
||||
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules, blacklistRules, keeplistRules string, fetchViaProxy bool) {
|
||||
f.Crawler = crawler
|
||||
f.UserAgent = userAgent
|
||||
f.Username = username
|
||||
|
|
@ -80,6 +82,8 @@ func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, passwor
|
|||
f.ScraperRules = scraperRules
|
||||
f.RewriteRules = rewriteRules
|
||||
f.FetchViaProxy = fetchViaProxy
|
||||
f.BlocklistRules = blacklistRules
|
||||
f.KeeplistRules = keeplistRules
|
||||
}
|
||||
|
||||
// WithError adds a new error message and increment the error counter.
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) {
|
|||
|
||||
func TestFeedBrowsingParams(t *testing.T) {
|
||||
feed := &Feed{}
|
||||
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false)
|
||||
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", "Look a Rule", "Oh wow another Rule", false)
|
||||
|
||||
if !feed.Crawler {
|
||||
t.Error(`The crawler must be activated`)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ type Handler struct {
|
|||
}
|
||||
|
||||
// CreateFeed fetch, parse and store a new feed.
|
||||
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) {
|
||||
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules, blocklistRules, keeplistRules string, fetchViaProxy bool) (*model.Feed, error) {
|
||||
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
|
||||
|
||||
if !h.store.CategoryExists(userID, categoryID) {
|
||||
|
|
@ -65,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
|
|||
|
||||
subscription.UserID = userID
|
||||
subscription.WithCategoryID(categoryID)
|
||||
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy)
|
||||
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, blocklistRules, keeplistRules, fetchViaProxy)
|
||||
subscription.WithClientResponse(response)
|
||||
subscription.CheckedNow()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
package processor
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"miniflux.app/config"
|
||||
|
|
@ -19,9 +20,11 @@ import (
|
|||
|
||||
// ProcessFeedEntries downloads original web page for entries and apply filters.
|
||||
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
|
||||
|
||||
filterFeedEntries(feed)
|
||||
|
||||
for _, entry := range feed.Entries {
|
||||
logger.Debug("[Feed #%d] Processing entry %s", feed.ID, entry.URL)
|
||||
|
||||
if feed.Crawler {
|
||||
if !store.EntryURLExists(feed.ID, entry.URL) {
|
||||
startTime := time.Now()
|
||||
|
|
@ -51,6 +54,37 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Filters feed entries based on regex rules
|
||||
First we filter based on our keep list, then we remove those entries that match the block list
|
||||
*/
|
||||
func filterFeedEntries(feed *model.Feed) {
|
||||
var filteredEntries []*model.Entry
|
||||
|
||||
if len(feed.KeeplistRules) > 0 {
|
||||
for _, entry := range feed.Entries {
|
||||
match, _ := regexp.MatchString(feed.KeeplistRules, entry.Title)
|
||||
if match == true {
|
||||
filteredEntries = append(filteredEntries, entry)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
filteredEntries = feed.Entries
|
||||
}
|
||||
if len(feed.BlocklistRules) > 0 {
|
||||
k := 0
|
||||
for _, entry := range filteredEntries {
|
||||
match, _ := regexp.MatchString(feed.BlocklistRules, entry.Title)
|
||||
if match != true {
|
||||
filteredEntries[k] = entry
|
||||
k++
|
||||
}
|
||||
}
|
||||
filteredEntries = filteredEntries[:k]
|
||||
}
|
||||
feed.Entries = filteredEntries
|
||||
}
|
||||
|
||||
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
|
||||
func ProcessEntryWebPage(entry *model.Entry) error {
|
||||
startTime := time.Now()
|
||||
|
|
|
|||
88
reader/processor/processor_test.go
Normal file
88
reader/processor/processor_test.go
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package processor // import "miniflux.app/reader/processor"
|
||||
|
||||
import (
|
||||
"miniflux.app/reader/parser"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKeeplistRules(t *testing.T) {
|
||||
data := `<?xml version="1.0"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>SomeGood News</title>
|
||||
<link>http://foo.bar/</link>
|
||||
<item>
|
||||
<title>Kitten News</title>
|
||||
<link>http://kitties.today/daily-kitten</link>
|
||||
<description>Kitten picture of the day.</description>
|
||||
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
|
||||
<guid>http://kitties.today</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Daily Covid DoomScrolling News</title>
|
||||
<link>http://covid.doom/daily-panic-dose</link>
|
||||
<description>Did you know that you can get COVID IN YOUR DREAMS?.</description>
|
||||
<pubDate>Tue, 03 Jun 2020 09:39:21 GMT</pubDate>
|
||||
<guid>http://covid.doom</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
feed, err := parser.ParseFeed(data)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(feed.Entries) != 2 {
|
||||
t.Errorf("Error parsing feed")
|
||||
}
|
||||
|
||||
//case insensitive
|
||||
feed.KeeplistRules = "(?i)kitten"
|
||||
filterFeedEntries(feed)
|
||||
if len(feed.Entries) != 1 {
|
||||
t.Errorf("Keeplist filter rule did not properly filter the feed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocklistRules(t *testing.T) {
|
||||
data := `<?xml version="1.0"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>SomeGood News</title>
|
||||
<link>http://foo.bar/</link>
|
||||
<item>
|
||||
<title>Kitten News</title>
|
||||
<link>http://kitties.today/daily-kitten</link>
|
||||
<description>Kitten picture of the day.</description>
|
||||
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
|
||||
<guid>http://kitties.today</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Daily Covid DoomScrolling News</title>
|
||||
<link>http://covid.doom/daily-panic-dose</link>
|
||||
<description>Did you know that you can get COVID IN YOUR DREAMS?.</description>
|
||||
<pubDate>Tue, 03 Jun 2020 09:39:21 GMT</pubDate>
|
||||
<guid>http://covid.doom</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
feed, err := parser.ParseFeed(data)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(feed.Entries) != 2 {
|
||||
t.Errorf("Error parsing feed")
|
||||
}
|
||||
|
||||
//case insensitive
|
||||
feed.BlocklistRules = "(?i)covid"
|
||||
filterFeedEntries(feed)
|
||||
if len(feed.Entries) != 1 {
|
||||
t.Errorf("Keeplist filter rule did not properly filter the feed")
|
||||
}
|
||||
}
|
||||
|
|
@ -27,6 +27,8 @@ var feedListQuery = `
|
|||
f.parsing_error_msg,
|
||||
f.scraper_rules,
|
||||
f.rewrite_rules,
|
||||
f.blocklist_rules,
|
||||
f.keeplist_rules,
|
||||
f.crawler,
|
||||
f.user_agent,
|
||||
f.username,
|
||||
|
|
@ -180,6 +182,8 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F
|
|||
f.parsing_error_msg,
|
||||
f.scraper_rules,
|
||||
f.rewrite_rules,
|
||||
f.blocklist_rules,
|
||||
f.keeplist_rules,
|
||||
f.crawler,
|
||||
f.user_agent,
|
||||
f.username,
|
||||
|
|
@ -290,6 +294,8 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
|
|||
&feed.ParsingErrorMsg,
|
||||
&feed.ScraperRules,
|
||||
&feed.RewriteRules,
|
||||
&feed.BlocklistRules,
|
||||
&feed.KeeplistRules,
|
||||
&feed.Crawler,
|
||||
&feed.UserAgent,
|
||||
&feed.Username,
|
||||
|
|
@ -375,6 +381,8 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
|
|||
f.parsing_error_msg,
|
||||
f.scraper_rules,
|
||||
f.rewrite_rules,
|
||||
f.blocklist_rules,
|
||||
f.keeplist_rules,
|
||||
f.crawler,
|
||||
f.user_agent,
|
||||
f.username,
|
||||
|
|
@ -407,6 +415,8 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
|
|||
&feed.ParsingErrorMsg,
|
||||
&feed.ScraperRules,
|
||||
&feed.RewriteRules,
|
||||
&feed.BlocklistRules,
|
||||
&feed.KeeplistRules,
|
||||
&feed.Crawler,
|
||||
&feed.UserAgent,
|
||||
&feed.Username,
|
||||
|
|
@ -453,10 +463,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
|
|||
disabled,
|
||||
scraper_rules,
|
||||
rewrite_rules,
|
||||
blocklist_rules,
|
||||
keeplist_rules,
|
||||
fetch_via_proxy
|
||||
)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
|
||||
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
||||
RETURNING
|
||||
id
|
||||
`
|
||||
|
|
@ -476,6 +488,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
|
|||
feed.Disabled,
|
||||
feed.ScraperRules,
|
||||
feed.RewriteRules,
|
||||
feed.BlocklistRules,
|
||||
feed.KeeplistRules,
|
||||
feed.FetchViaProxy,
|
||||
).Scan(&feed.ID)
|
||||
if err != nil {
|
||||
|
|
@ -523,16 +537,18 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
|
|||
parsing_error_count=$9,
|
||||
scraper_rules=$10,
|
||||
rewrite_rules=$11,
|
||||
crawler=$12,
|
||||
user_agent=$13,
|
||||
username=$14,
|
||||
password=$15,
|
||||
disabled=$16,
|
||||
next_check_at=$17,
|
||||
ignore_http_cache=$18,
|
||||
fetch_via_proxy=$19
|
||||
blocklist_rules=$12,
|
||||
keeplist_rules=$13,
|
||||
crawler=$14,
|
||||
user_agent=$15,
|
||||
username=$16,
|
||||
password=$17,
|
||||
disabled=$18,
|
||||
next_check_at=$19,
|
||||
ignore_http_cache=$20,
|
||||
fetch_via_proxy=$21
|
||||
WHERE
|
||||
id=$20 AND user_id=$21
|
||||
id=$22 AND user_id=$23
|
||||
`
|
||||
_, err = s.db.Exec(query,
|
||||
feed.FeedURL,
|
||||
|
|
@ -546,6 +562,8 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
|
|||
feed.ParsingErrorCount,
|
||||
feed.ScraperRules,
|
||||
feed.RewriteRules,
|
||||
feed.BlocklistRules,
|
||||
feed.KeeplistRules,
|
||||
feed.Crawler,
|
||||
feed.UserAgent,
|
||||
feed.Username,
|
||||
|
|
|
|||
|
|
@ -55,6 +55,12 @@
|
|||
|
||||
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
|
||||
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
|
||||
|
||||
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
|
||||
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
|
||||
|
||||
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
|
||||
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
|
||||
</div>
|
||||
</details>
|
||||
|
||||
|
|
|
|||
|
|
@ -64,6 +64,12 @@
|
|||
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
|
||||
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
|
||||
|
||||
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
|
||||
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
|
||||
|
||||
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
|
||||
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
|
||||
|
||||
<label for="form-category">{{ t "form.feed.label.category" }}</label>
|
||||
<select id="form-category" name="category_id">
|
||||
{{ range .categories }}
|
||||
|
|
|
|||
|
|
@ -86,6 +86,12 @@ var templateViewsMap = map[string]string{
|
|||
|
||||
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
|
||||
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
|
||||
|
||||
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
|
||||
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
|
||||
|
||||
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
|
||||
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
|
||||
</div>
|
||||
</details>
|
||||
|
||||
|
|
@ -589,6 +595,12 @@ var templateViewsMap = map[string]string{
|
|||
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
|
||||
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
|
||||
|
||||
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
|
||||
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
|
||||
|
||||
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
|
||||
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
|
||||
|
||||
<label for="form-category">{{ t "form.feed.label.category" }}</label>
|
||||
<select id="form-category" name="category_id">
|
||||
{{ range .categories }}
|
||||
|
|
@ -1582,7 +1594,7 @@ var templateViewsMap = map[string]string{
|
|||
|
||||
var templateViewsMapChecksums = map[string]string{
|
||||
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
|
||||
"add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032",
|
||||
"add_subscription": "82bf0dfadd64d3b6eda323a8174dd1446665b0804d27ca8718a828488627b287",
|
||||
"api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284",
|
||||
"bookmark_entries": "eacbbdce7fa85ec66c4c12f02879daab562a17ff79f1aac1805617e83e3a3a42",
|
||||
"categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9",
|
||||
|
|
@ -1593,7 +1605,7 @@ var templateViewsMapChecksums = map[string]string{
|
|||
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
|
||||
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
|
||||
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",
|
||||
"edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b",
|
||||
"edit_feed": "5de7626448c48de384a0388227ab0c3b75b1ec19b5de440c91039180852cc5dc",
|
||||
"edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a",
|
||||
"entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c",
|
||||
"feed_entries": "89977ea86b8d43305d587b70e6d9c45c2c88249b3966f2d31051dc7a5f1c48b6",
|
||||
|
|
|
|||
|
|
@ -195,6 +195,31 @@ func TestUpdateFeedRewriteRules(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestUpdateFeedKeeplistRules(t *testing.T) {
|
||||
client := createClient(t)
|
||||
feed, _ := createFeed(t, client)
|
||||
|
||||
keeplistRules := "test"
|
||||
updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModification{KeeplistRules: &keeplistRules})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if updatedFeed.KeeplistRules != keeplistRules {
|
||||
t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules)
|
||||
}
|
||||
|
||||
keeplistRules = ""
|
||||
updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModification{KeeplistRules: &keeplistRules})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if updatedFeed.KeeplistRules != keeplistRules {
|
||||
t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFeedUserAgent(t *testing.T) {
|
||||
client := createClient(t)
|
||||
feed, _ := createFeed(t, client)
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
|
|||
Title: feed.Title,
|
||||
ScraperRules: feed.ScraperRules,
|
||||
RewriteRules: feed.RewriteRules,
|
||||
BlocklistRules: feed.BlocklistRules,
|
||||
KeeplistRules: feed.KeeplistRules,
|
||||
Crawler: feed.Crawler,
|
||||
UserAgent: feed.UserAgent,
|
||||
CategoryID: feed.Category.ID,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ type FeedForm struct {
|
|||
Title string
|
||||
ScraperRules string
|
||||
RewriteRules string
|
||||
BlocklistRules string
|
||||
KeeplistRules string
|
||||
Crawler bool
|
||||
UserAgent string
|
||||
CategoryID int64
|
||||
|
|
@ -45,6 +47,8 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
|
|||
feed.FeedURL = f.FeedURL
|
||||
feed.ScraperRules = f.ScraperRules
|
||||
feed.RewriteRules = f.RewriteRules
|
||||
feed.BlocklistRules = f.BlocklistRules
|
||||
feed.KeeplistRules = f.KeeplistRules
|
||||
feed.Crawler = f.Crawler
|
||||
feed.UserAgent = f.UserAgent
|
||||
feed.ParsingErrorCount = 0
|
||||
|
|
@ -63,7 +67,6 @@ func NewFeedForm(r *http.Request) *FeedForm {
|
|||
if err != nil {
|
||||
categoryID = 0
|
||||
}
|
||||
|
||||
return &FeedForm{
|
||||
FeedURL: r.FormValue("feed_url"),
|
||||
SiteURL: r.FormValue("site_url"),
|
||||
|
|
@ -71,6 +74,8 @@ func NewFeedForm(r *http.Request) *FeedForm {
|
|||
ScraperRules: r.FormValue("scraper_rules"),
|
||||
UserAgent: r.FormValue("user_agent"),
|
||||
RewriteRules: r.FormValue("rewrite_rules"),
|
||||
BlocklistRules: r.FormValue("blocklist_rules"),
|
||||
KeeplistRules: r.FormValue("keeplist_rules"),
|
||||
Crawler: r.FormValue("crawler") == "1",
|
||||
CategoryID: int64(categoryID),
|
||||
Username: r.FormValue("feed_username"),
|
||||
|
|
|
|||
|
|
@ -13,15 +13,17 @@ import (
|
|||
|
||||
// SubscriptionForm represents the subscription form.
|
||||
type SubscriptionForm struct {
|
||||
URL string
|
||||
CategoryID int64
|
||||
Crawler bool
|
||||
FetchViaProxy bool
|
||||
UserAgent string
|
||||
Username string
|
||||
Password string
|
||||
ScraperRules string
|
||||
RewriteRules string
|
||||
URL string
|
||||
CategoryID int64
|
||||
Crawler bool
|
||||
FetchViaProxy bool
|
||||
UserAgent string
|
||||
Username string
|
||||
Password string
|
||||
ScraperRules string
|
||||
RewriteRules string
|
||||
BlocklistRules string
|
||||
KeeplistRules string
|
||||
}
|
||||
|
||||
// Validate makes sure the form values are valid.
|
||||
|
|
@ -41,14 +43,15 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
|
|||
}
|
||||
|
||||
return &SubscriptionForm{
|
||||
URL: r.FormValue("url"),
|
||||
Crawler: r.FormValue("crawler") == "1",
|
||||
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
|
||||
CategoryID: int64(categoryID),
|
||||
UserAgent: r.FormValue("user_agent"),
|
||||
Username: r.FormValue("feed_username"),
|
||||
Password: r.FormValue("feed_password"),
|
||||
ScraperRules: r.FormValue("scraper_rules"),
|
||||
RewriteRules: r.FormValue("rewrite_rules"),
|
||||
URL: r.FormValue("url"),
|
||||
Crawler: r.FormValue("crawler") == "1",
|
||||
CategoryID: int64(categoryID),
|
||||
UserAgent: r.FormValue("user_agent"),
|
||||
Username: r.FormValue("feed_username"),
|
||||
Password: r.FormValue("feed_password"),
|
||||
ScraperRules: r.FormValue("scraper_rules"),
|
||||
RewriteRules: r.FormValue("rewrite_rules"),
|
||||
BlocklistRules: r.FormValue("blocklist_rules"),
|
||||
KeeplistRules: r.FormValue("keeplist_rules"),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
|
|||
subscriptionForm.Password,
|
||||
subscriptionForm.ScraperRules,
|
||||
subscriptionForm.RewriteRules,
|
||||
subscriptionForm.BlocklistRules,
|
||||
subscriptionForm.KeeplistRules,
|
||||
subscriptionForm.FetchViaProxy,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
|
|||
subscriptionForm.Password,
|
||||
subscriptionForm.ScraperRules,
|
||||
subscriptionForm.RewriteRules,
|
||||
subscriptionForm.BlocklistRules,
|
||||
subscriptionForm.KeeplistRules,
|
||||
subscriptionForm.FetchViaProxy,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue