southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add the possibility to override default user agent for each feed

Browse source
This commit is contained in:
Patrick 2018-09-20 03:19:24 +02:00 committed by Frédéric Guillot
parent 1d335390c2
commit 2538eea177
29 changed files with 129 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/feed.go
View file

@ -47,6 +47,7 @@ func (c *Controller) CreateFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.CategoryID,
feedInfo.FeedURL,
feedInfo.Crawler,
feedInfo.UserAgent,
feedInfo.Username,
feedInfo.Password,
)

13
api/payload.go
View file

@ -26,15 +26,17 @@ type entriesResponse struct {
type feedCreation struct {
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
}
type subscriptionDiscovery struct {
URL string `json:"url"`
Username string `json:"username"`
Password string `json:"password"`
URL string `json:"url"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
}
type feedModification struct {
@ -44,6 +46,7 @@ type feedModification struct {
ScraperRules *string `json:"scraper_rules"`
RewriteRules *string `json:"rewrite_rules"`
Crawler *bool `json:"crawler"`
UserAgent *string `json:"user_agent"`
Username *string `json:"username"`
Password *string `json:"password"`
CategoryID *int64 `json:"category_id"`
@ -74,6 +77,10 @@ func (f *feedModification) Update(feed *model.Feed) {
feed.Crawler = *f.Crawler
}
if f.UserAgent != nil {
feed.UserAgent = *f.UserAgent
}
if f.Username != nil {
feed.Username = *f.Username
}

1
api/subscription.go
View file

@ -22,6 +22,7 @@ func (c *Controller) GetSubscriptions(w http.ResponseWriter, r *http.Request) {
subscriptions, err := subscription.FindSubscriptions(
subscriptionInfo.URL,
subscriptionInfo.UserAgent,
subscriptionInfo.Username,
subscriptionInfo.Password,
)

2
client/core.go
View file

@ -91,6 +91,7 @@ type Feed struct {
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
Crawler bool `json:"crawler"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Category *Category `json:"category,omitempty"`
@ -105,6 +106,7 @@ type FeedModification struct {
ScraperRules *string `json:"scraper_rules"`
RewriteRules *string `json:"rewrite_rules"`
Crawler *bool `json:"crawler"`
UserAgent *string `json:"user_agent"`
Username *string `json:"username"`
Password *string `json:"password"`
CategoryID *int64 `json:"category_id"`

2
database/migration.go
View file

@ -12,7 +12,7 @@ import (
"miniflux.app/logger"
)
const schemaVersion = 20
const schemaVersion = 21
// Migrate executes database migrations.
func Migrate(db *sql.DB) {

2
database/sql.go
View file

@ -144,6 +144,7 @@ create index users_extra_idx on users using gin(extra);
"schema_version_20": `alter table entries add column document_vectors tsvector;
update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, ''));
create index document_vectors_idx on entries using gin(document_vectors);`,
"schema_version_21": `alter table feeds add column user_agent text default '';`,
"schema_version_3": `create table tokens (
id text not null,
value text not null,
@ -192,6 +193,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_19": "a83f77b41cc213d282805a5b518f15abbf96331599119f0ef4aca4be037add7b",
"schema_version_2": "e8e9ff32478df04fcddad10a34cba2e8bb1e67e7977b5bd6cdc4c31ec94282b4",
"schema_version_20": "6c4e9b2c5bccdc3243c239c390fb1caa5e15624e669b2c07e14c126f6d2e2cd6",
"schema_version_21": "77da01ee38918ff4fe33985fbb20ed3276a717a7584c2ca9ebcf4d4ab6cb6910",
"schema_version_3": "a54745dbc1c51c000f74d4e5068f1e2f43e83309f023415b1749a47d5c1e0f12",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",

1
database/sql/schema_version_21.sql Normal file
View file

@ -0,0 +1 @@
alter table feeds add column user_agent text default '';

16
http/client/client.go
View file

@ -33,6 +33,9 @@ const (
)
var (
// DefaultUserAgent sets the User-Agent header used for any requests by miniflux.
DefaultUserAgent = "Mozilla/5.0 (compatible; Miniflux/" + version.Version + "; +https://miniflux.app)"
errInvalidCertificate = "Invalid SSL certificate (original error: %q)"
errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)"
errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)"
@ -47,6 +50,7 @@ type Client struct {
authorizationHeader string
username string
password string
userAgent string
Insecure bool
}
@ -72,6 +76,14 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
return c
}
// WithUserAgent defines the User-Agent header to use for outgoing requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
if userAgent != "" {
c.userAgent = userAgent
}
return c
}
// Get execute a GET HTTP request.
func (c *Client) Get() (*Response, error) {
request, err := c.buildRequest(http.MethodGet, nil)
@ -212,7 +224,7 @@ func (c *Client) buildClient() http.Client {
func (c *Client) buildHeaders() http.Header {
headers := make(http.Header)
headers.Add("User-Agent", "Mozilla/5.0 (compatible; Miniflux/"+version.Version+"; +https://miniflux.app)")
headers.Add("User-Agent", c.userAgent)
headers.Add("Accept", "*/*")
if c.etagHeader != "" {
@ -233,5 +245,5 @@ func (c *Client) buildHeaders() http.Header {
// New returns a new HTTP client.
func New(url string) *Client {
return &Client{url: url, Insecure: false}
return &Client{url: url, userAgent: DefaultUserAgent, Insecure: false}
}

1
model/feed.go
View file

@ -24,6 +24,7 @@ type Feed struct {
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
Crawler bool `json:"crawler"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Category *Category `json:"category,omitempty"`

6
reader/feed/handler.go
View file

@ -37,7 +37,7 @@ type Handler struct {
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, username, password string) (*model.Feed, error) {
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@ -46,6 +46,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
clt := client.New(url)
clt.WithCredentials(username, password)
clt.WithUserAgent(userAgent)
response, err := clt.Get()
if err != nil {
if _, ok := err.(*errors.LocalizedError); ok {
@ -87,6 +88,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.FeedURL = response.EffectiveURL
subscription.UserID = userID
subscription.Crawler = crawler
subscription.UserAgent = userAgent
subscription.Username = username
subscription.Password = password
@ -136,6 +138,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
clt := client.New(originalFeed.FeedURL)
clt.WithCredentials(originalFeed.Username, originalFeed.Password)
clt.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
clt.WithUserAgent(originalFeed.UserAgent)
response, err := clt.Get()
if err != nil {
var customErr errors.LocalizedError
@ -196,6 +199,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
feedProcessor := processor.NewFeedProcessor(userID, h.store, subscription)
feedProcessor.WithScraperRules(originalFeed.ScraperRules)
feedProcessor.WithUserAgent(originalFeed.UserAgent)
feedProcessor.WithRewriteRules(originalFeed.RewriteRules)
feedProcessor.WithCrawler(originalFeed.Crawler)
feedProcessor.Process()

8
reader/processor/processor.go
View file

@ -21,6 +21,7 @@ type FeedProcessor struct {
scraperRules string
rewriteRules string
crawler bool
userAgent string
}
// WithCrawler enables the crawler.
@ -33,6 +34,11 @@ func (f *FeedProcessor) WithScraperRules(rules string) {
f.scraperRules = rules
}
// WithUserAgent sets the User-Agent header for fetching article content.
func (f *FeedProcessor) WithUserAgent(userAgent string) {
f.userAgent = userAgent
}
// WithRewriteRules adds rewrite rules to the processing.
func (f *FeedProcessor) WithRewriteRules(rules string) {
f.rewriteRules = rules
@ -45,7 +51,7 @@ func (f *FeedProcessor) Process() {
if f.store.EntryURLExists(f.userID, entry.URL) {
logger.Debug(`[FeedProcessor] Do not crawl existing entry URL: "%s"`, entry.URL)
} else {
content, err := scraper.Fetch(entry.URL, f.scraperRules)
content, err := scraper.Fetch(entry.URL, f.scraperRules, f.userAgent)
if err != nil {
logger.Error("[FeedProcessor] %v", err)
} else {

6
reader/scraper/scraper.go
View file

@ -19,8 +19,12 @@ import (
)
// Fetch downloads a web page a returns relevant contents.
func Fetch(websiteURL, rules string) (string, error) {
func Fetch(websiteURL, rules, userAgent string) (string, error) {
clt := client.New(websiteURL)
if userAgent != "" {
clt.WithUserAgent(userAgent)
}
response, err := clt.Get()
if err != nil {
return "", err

3
reader/subscription/finder.go
View file

@ -29,11 +29,12 @@ var (
)
// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL, username, password string) (Subscriptions, error) {
func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[FindSubscriptions] url=%s", websiteURL))
clt := client.New(websiteURL)
clt.WithCredentials(username, password)
clt.WithUserAgent(userAgent)
response, err := clt.Get()
if err != nil {
if _, ok := err.(errors.LocalizedError); ok {

3
storage/entry_query_builder.go
View file

@ -192,7 +192,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
e.id, e.user_id, e.feed_id, e.hash, e.published_at at time zone u.timezone, e.title,
e.url, e.comments_url, e.author, e.content, e.status, e.starred,
f.title as feed_title, f.feed_url, f.site_url, f.checked_at,
f.category_id, c.title as category_title, f.scraper_rules, f.rewrite_rules, f.crawler,
f.category_id, c.title as category_title, f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent,
fi.icon_id,
u.timezone
FROM entries e
@ -247,6 +247,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
&entry.Feed.ScraperRules,
&entry.Feed.RewriteRules,
&entry.Feed.Crawler,
&entry.Feed.UserAgent,
&iconID,
&tz,
)

18
storage/feed.go
View file

@ -66,7 +66,7 @@ func (s *Storage) Feeds(userID int64) (model.Feeds, error) {
f.id, f.feed_url, f.site_url, f.title, f.etag_header, f.last_modified_header,
f.user_id, f.checked_at at time zone u.timezone,
f.parsing_error_count, f.parsing_error_msg,
f.scraper_rules, f.rewrite_rules, f.crawler,
f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent,
f.username, f.password,
f.category_id, c.title as category_title,
fi.icon_id,
@ -104,6 +104,7 @@ func (s *Storage) Feeds(userID int64) (model.Feeds, error) {
&feed.ScraperRules,
&feed.RewriteRules,
&feed.Crawler,
&feed.UserAgent,
&feed.Username,
&feed.Password,
&feed.Category.ID,
@ -141,7 +142,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
f.id, f.feed_url, f.site_url, f.title, f.etag_header, f.last_modified_header,
f.user_id, f.checked_at at time zone u.timezone,
f.parsing_error_count, f.parsing_error_msg,
f.scraper_rules, f.rewrite_rules, f.crawler,
f.scraper_rules, f.rewrite_rules, f.crawler, f.user_agent,
f.username, f.password,
f.category_id, c.title as category_title,
fi.icon_id,
@ -166,6 +167,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
&feed.ScraperRules,
&feed.RewriteRules,
&feed.Crawler,
&feed.UserAgent,
&feed.Username,
&feed.Password,
&feed.Category.ID,
@ -194,8 +196,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:CreateFeed] feedURL=%s", feed.FeedURL))
sql := `
INSERT INTO feeds
(feed_url, site_url, title, category_id, user_id, etag_header, last_modified_header, crawler, username, password)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
(feed_url, site_url, title, category_id, user_id, etag_header, last_modified_header, crawler, user_agent, username, password)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
RETURNING id
`
@ -209,6 +211,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.EtagHeader,
feed.LastModifiedHeader,
feed.Crawler,
feed.UserAgent,
feed.Username,
feed.Password,
).Scan(&feed.ID)
@ -234,9 +237,9 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
query := `UPDATE feeds SET
feed_url=$1, site_url=$2, title=$3, category_id=$4, etag_header=$5, last_modified_header=$6, checked_at=$7,
parsing_error_msg=$8, parsing_error_count=$9, scraper_rules=$10, rewrite_rules=$11, crawler=$12,
username=$13, password=$14
WHERE id=$15 AND user_id=$16`
parsing_error_msg=$8, parsing_error_count=$9, scraper_rules=$10, rewrite_rules=$11, crawler=$12, user_agent=$13,
username=$14, password=$15
WHERE id=$16 AND user_id=$17`
_, err = s.db.Exec(query,
feed.FeedURL,
@ -251,6 +254,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.ScraperRules,
feed.RewriteRules,
feed.Crawler,
feed.UserAgent,
feed.Username,
feed.Password,
feed.ID,

3
template/html/add_subscription.html
View file

@ -41,6 +41,9 @@
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "Fetch original content" }}</label>
<label for="form-user-agent">{{ t "User-Agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
<label for="form-feed-username">{{ t "Feed Username" }}</label>
<input type="text" name="feed_username" id="form-feed-username" value="{{ .form.Username }}">

1
template/html/choose_subscription.html
View file

@ -19,6 +19,7 @@
<form action="{{ route "chooseSubscription" }}" method="POST">
<input type="hidden" name="csrf" value="{{ .csrf }}">
<input type="hidden" name="category_id" value="{{ .form.CategoryID }}">
<input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
<input type="hidden" name="feed_username" value="{{ .form.Username }}">
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
{{ if .form.Crawler }}

3
template/html/edit_feed.html
View file

@ -58,6 +58,9 @@
-->
<input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
<label for="form-user-agent">{{ t "User-Agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
<label for="form-scraper-rules">{{ t "Scraper Rules" }}</label>
<input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">

13
template/views.go
View file

@ -87,6 +87,9 @@ var templateViewsMap = map[string]string{
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "Fetch original content" }}</label>
<label for="form-user-agent">{{ t "User-Agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
<label for="form-feed-username">{{ t "Feed Username" }}</label>
<input type="text" name="feed_username" id="form-feed-username" value="{{ .form.Username }}">
@ -265,6 +268,7 @@ var templateViewsMap = map[string]string{
<form action="{{ route "chooseSubscription" }}" method="POST">
<input type="hidden" name="csrf" value="{{ .csrf }}">
<input type="hidden" name="category_id" value="{{ .form.CategoryID }}">
<input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
<input type="hidden" name="feed_username" value="{{ .form.Username }}">
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
{{ if .form.Crawler }}
@ -453,6 +457,9 @@ var templateViewsMap = map[string]string{
-->
<input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
<label for="form-user-agent">{{ t "User-Agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}">
<label for="form-scraper-rules">{{ t "Scraper Rules" }}</label>
<input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
@ -1359,15 +1366,15 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "ad2fb778fc73c39b733b3f81b13e5c7d689b041fadd24ee2d4577f545aa788ad",
"add_subscription": "3fbcffefc94fb0fccfcf870d602f5ba78ce3ab7ebaeacd04198a6e529143cb29",
"add_subscription": "dc6593913e6ff14a6f957349c2ce43f0e3e095e1985f9d739929a88f5f6550bd",
"bookmark_entries": "49423f84c05d77368e20c8e14c53ad237308cdaf4143413487d1b0e11c18d148",
"categories": "ca1280cd157bb527d4fc907da67b05a8347378f6dce965b9389d4bcdf3600a11",
"category_entries": "d219d4bd5376c526c00a3da49b511fb73e812be5d1e12acadeceee8dfa4bbfe2",
"choose_subscription": "7266b269ddbe145e757a24a57f3fbc7611e34a20383fbd887988204cebce2681",
"choose_subscription": "57280e5b6ba3118c14ef1238e8003f5e92401a671dd6e6e198c864ebbd0b8030",
"create_category": "2b82af5d2dcd67898dc5daa57a6461e6ff8121a6089b2a2a1be909f35e4a2275",
"create_user": "1ef0a1f9bf119d44929c81f13073a257d69650cf5064960cf06a63fe51923e86",
"edit_category": "cee720faadcec58289b707ad30af623d2ee66c1ce23a732965463250d7ff41c5",
"edit_feed": "1a8e342e4fac80e8b9c73537c7fe8aaf7f9e3e7af22f411927010897dd37e9c3",
"edit_feed": "6ff765208828e7533f865009de26edd87d8109b547bdff08132d862aea0e223b",
"edit_user": "7373e09f805e6c017167001519b9feb04226be6c81c2875cbacd5ce94f2c24bf",
"entry": "82a0a4e715da94b12370b380072f1175c9f0e07b37e7f54a9adca4ed1fe015c0",
"feed_entries": "bebc42317ca9e908fcdb98cc1c4a2dc3f4bb7ef6d4c288d3d3fba8f8339403b6",

25
tests/feed_test.go
View file

@ -195,6 +195,31 @@ func TestUpdateFeedRewriteRules(t *testing.T) {
}
}
func TestUpdateFeedUserAgent(t *testing.T) {
client := createClient(t)
feed, _ := createFeed(t, client)
userAgent := "test"
updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModification{UserAgent: &userAgent})
if err != nil {
t.Fatal(err)
}
if updatedFeed.UserAgent != userAgent {
t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent)
}
userAgent = ""
updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModification{UserAgent: &userAgent})
if err != nil {
t.Fatal(err)
}
if updatedFeed.UserAgent != userAgent {
t.Fatalf(`Wrong UserAgent value, got "%v" instead of "%v"`, updatedFeed.UserAgent, userAgent)
}
}
func TestUpdateFeedUsername(t *testing.T) {
client := createClient(t)
feed, _ := createFeed(t, client)

2
ui/entry_scraper.go
View file

@ -38,7 +38,7 @@ func (c *Controller) FetchContent(w http.ResponseWriter, r *http.Request) {
return
}
content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules)
content, err := scraper.Fetch(entry.URL, entry.Feed.ScraperRules, entry.Feed.UserAgent)
if err != nil {
json.ServerError(w, err)
return

3
ui/feed_edit.go
View file

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/ui/form"
@ -52,6 +53,7 @@ func (c *Controller) EditFeed(w http.ResponseWriter, r *http.Request) {
ScraperRules: feed.ScraperRules,
RewriteRules: feed.RewriteRules,
Crawler: feed.Crawler,
UserAgent: feed.UserAgent,
CategoryID: feed.Category.ID,
Username: feed.Username,
Password: feed.Password,
@ -66,6 +68,7 @@ func (c *Controller) EditFeed(w http.ResponseWriter, r *http.Request) {
view.Set("user", user)
view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
html.OK(w, r, view.Render("edit_feed"))
}

2
ui/feed_update.go
View file

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response"
"miniflux.app/http/response/html"
@ -59,6 +60,7 @@ func (c *Controller) UpdateFeed(w http.ResponseWriter, r *http.Request) {
view.Set("user", user)
view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
if err := feedForm.ValidateModification(); err != nil {
view.Set("errorMessage", err.Error())

3
ui/form/feed.go
View file

@ -20,6 +20,7 @@ type FeedForm struct {
ScraperRules string
RewriteRules string
Crawler bool
UserAgent string
CategoryID int64
Username string
Password string
@ -42,6 +43,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.ScraperRules = f.ScraperRules
feed.RewriteRules = f.RewriteRules
feed.Crawler = f.Crawler
feed.UserAgent = f.UserAgent
feed.ParsingErrorCount = 0
feed.ParsingErrorMsg = ""
feed.Username = f.Username
@ -61,6 +63,7 @@ func NewFeedForm(r *http.Request) *FeedForm {
SiteURL: r.FormValue("site_url"),
Title: r.FormValue("title"),
ScraperRules: r.FormValue("scraper_rules"),
UserAgent: r.FormValue("user_agent"),
RewriteRules: r.FormValue("rewrite_rules"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),

2
ui/form/subscription.go
View file

@ -16,6 +16,7 @@ type SubscriptionForm struct {
URL string
CategoryID int64
Crawler bool
UserAgent string
Username string
Password string
}
@ -40,6 +41,7 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
}

2
ui/subscription_add.go
View file

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
"miniflux.app/ui/session"
@ -35,6 +36,7 @@ func (c *Controller) AddSubscription(w http.ResponseWriter, r *http.Request) {
view.Set("user", user)
view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
html.OK(w, r, view.Render("add_subscription"))
}

2
ui/subscription_bookmarklet.go
View file

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/ui/form"
@ -39,6 +40,7 @@ func (c *Controller) Bookmarklet(w http.ResponseWriter, r *http.Request) {
view.Set("user", user)
view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
html.OK(w, r, view.Render("add_subscription"))
}

3
ui/subscription_choose.go
View file

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/response"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
@ -38,6 +39,7 @@ func (c *Controller) ChooseSubscription(w http.ResponseWriter, r *http.Request)
view.Set("user", user)
view.Set("countUnread", c.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
subscriptionForm := form.NewSubscriptionForm(r)
if err := subscriptionForm.Validate(); err != nil {
@ -52,6 +54,7 @@ func (c *Controller) ChooseSubscription(w http.ResponseWriter, r *http.Request)
subscriptionForm.CategoryID,
subscriptionForm.URL,
subscriptionForm.Crawler,
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
)

4
ui/subscription_submit.go
View file

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/response"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
@ -40,6 +41,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request)
v.Set("user", user)
v.Set("countUnread", c.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", c.store.CountErrorFeeds(user.ID))
v.Set("defaultUserAgent", client.DefaultUserAgent)
subscriptionForm := form.NewSubscriptionForm(r)
if err := subscriptionForm.Validate(); err != nil {
@ -51,6 +53,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request)
subscriptions, err := subscription.FindSubscriptions(
subscriptionForm.URL,
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
)
@ -76,6 +79,7 @@ func (c *Controller) SubmitSubscription(w http.ResponseWriter, r *http.Request)
subscriptionForm.CategoryID,
subscriptions[0].URL,
subscriptionForm.Crawler,
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
)