diff --git a/api/feed.go b/api/feed.go index ffd3928c..fc7ae6a2 100644 --- a/api/feed.go +++ b/api/feed.go @@ -51,6 +51,7 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) { feedInfo.Password, feedInfo.ScraperRules, feedInfo.RewriteRules, + feedInfo.FetchViaProxy, ) if err != nil { json.ServerError(w, r, err) diff --git a/api/payload.go b/api/payload.go index 0a2b57fc..9b209017 100644 --- a/api/payload.go +++ b/api/payload.go @@ -24,21 +24,23 @@ type entriesResponse struct { } type feedCreation struct { - FeedURL string `json:"feed_url"` - CategoryID int64 `json:"category_id"` - UserAgent string `json:"user_agent"` - Username string `json:"username"` - Password string `json:"password"` - Crawler bool `json:"crawler"` - ScraperRules string `json:"scraper_rules"` - RewriteRules string `json:"rewrite_rules"` + FeedURL string `json:"feed_url"` + CategoryID int64 `json:"category_id"` + UserAgent string `json:"user_agent"` + Username string `json:"username"` + Password string `json:"password"` + Crawler bool `json:"crawler"` + FetchViaProxy bool `json:"fetch_via_proxy"` + ScraperRules string `json:"scraper_rules"` + RewriteRules string `json:"rewrite_rules"` } type subscriptionDiscovery struct { - URL string `json:"url"` - UserAgent string `json:"user_agent"` - Username string `json:"username"` - Password string `json:"password"` + URL string `json:"url"` + UserAgent string `json:"user_agent"` + Username string `json:"username"` + Password string `json:"password"` + FetchViaProxy bool `json:"fetch_via_proxy"` } type feedModification struct { diff --git a/api/subscription.go b/api/subscription.go index 983a3ca5..43b3131a 100644 --- a/api/subscription.go +++ b/api/subscription.go @@ -23,6 +23,7 @@ func (h *handler) getSubscriptions(w http.ResponseWriter, r *http.Request) { subscriptionInfo.UserAgent, subscriptionInfo.Username, subscriptionInfo.Password, + subscriptionInfo.FetchViaProxy, ) if finderErr != nil { json.ServerError(w, r, finderErr) diff --git a/config/options.go b/config/options.go index 5fc6c36c..6cae62a1 100644 --- a/config/options.go +++ b/config/options.go @@ -50,6 +50,7 @@ const ( defaultPocketConsumerKey = "" defaultHTTPClientTimeout = 20 defaultHTTPClientMaxBodySize = 15 + defaultHTTPClientProxy = "" defaultAuthProxyHeader = "" defaultAuthProxyUserCreation = false ) @@ -96,6 +97,7 @@ type Options struct { pocketConsumerKey string httpClientTimeout int httpClientMaxBodySize int64 + httpClientProxy string authProxyHeader string authProxyUserCreation bool } @@ -141,6 +143,7 @@ func NewOptions() *Options { pocketConsumerKey: defaultPocketConsumerKey, httpClientTimeout: defaultHTTPClientTimeout, httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024, + httpClientProxy: defaultHTTPClientProxy, authProxyHeader: defaultAuthProxyHeader, authProxyUserCreation: defaultAuthProxyUserCreation, } @@ -349,6 +352,16 @@ func (o *Options) HTTPClientMaxBodySize() int64 { return o.httpClientMaxBodySize } +// HTTPClientProxy returns the proxy URL for HTTP client. +func (o *Options) HTTPClientProxy() string { + return o.httpClientProxy +} + +// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured. +func (o *Options) HasHTTPClientProxyConfigured() bool { + return o.httpClientProxy != "" +} + // AuthProxyHeader returns an HTTP header name that contains username for // authentication using auth proxy. func (o *Options) AuthProxyHeader() string { @@ -403,6 +416,7 @@ func (o *Options) String() string { builder.WriteString(fmt.Sprintf("OAUTH2_PROVIDER: %v\n", o.oauth2Provider)) builder.WriteString(fmt.Sprintf("HTTP_CLIENT_TIMEOUT: %v\n", o.httpClientTimeout)) builder.WriteString(fmt.Sprintf("HTTP_CLIENT_MAX_BODY_SIZE: %v\n", o.httpClientMaxBodySize)) + builder.WriteString(fmt.Sprintf("HTTP_CLIENT_PROXY: %v\n", o.httpClientProxy)) builder.WriteString(fmt.Sprintf("AUTH_PROXY_HEADER: %v\n", o.authProxyHeader)) builder.WriteString(fmt.Sprintf("AUTH_PROXY_USER_CREATION: %v\n", o.authProxyUserCreation)) return builder.String() diff --git a/config/parser.go b/config/parser.go index 77b74357..e338c0b6 100644 --- a/config/parser.go +++ b/config/parser.go @@ -184,6 +184,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.httpClientTimeout = parseInt(value, defaultHTTPClientTimeout) case "HTTP_CLIENT_MAX_BODY_SIZE": p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024) + case "HTTP_CLIENT_PROXY": + p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy) case "AUTH_PROXY_HEADER": p.opts.authProxyHeader = parseString(value, defaultAuthProxyHeader) case "AUTH_PROXY_USER_CREATION": diff --git a/database/migration.go b/database/migration.go index f662f5e6..88d6519d 100644 --- a/database/migration.go +++ b/database/migration.go @@ -12,7 +12,7 @@ import ( "miniflux.app/logger" ) -const schemaVersion = 34 +const schemaVersion = 35 // Migrate executes database migrations. func Migrate(db *sql.DB) { diff --git a/database/sql.go b/database/sql.go index 440c85c2..bc8a4c39 100644 --- a/database/sql.go +++ b/database/sql.go @@ -187,6 +187,8 @@ create index entries_user_feed_idx on entries (user_id, feed_id); `, "schema_version_33": `alter table users add column show_reading_time boolean default 't';`, "schema_version_34": `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status);`, + "schema_version_35": `alter table feeds add column fetch_via_proxy bool default false; +`, "schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc'); alter table users add column entry_direction entry_sorting_direction default 'asc'; `, @@ -244,6 +246,7 @@ var SqlMapChecksums = map[string]string{ "schema_version_32": "5b4de8dd2d7e3c6ae4150e0e3931df2ee989f2c667145bd67294e5a5f3fae456", "schema_version_33": "bf38514efeb6c12511f41b1cc484f92722240b0a6ae874c32a958dfea3433d02", "schema_version_34": "1a3e036f652fc98b7564a27013f04e1eb36dd0d68893c723168f134dc1065822", + "schema_version_35": "162a55df78eed4b9c9c141878132d5f1d97944b96f35a79e38f55716cdd6b3d2", "schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9", "schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c", "schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4", diff --git a/database/sql/schema_version_35.sql b/database/sql/schema_version_35.sql new file mode 100644 index 00000000..50533751 --- /dev/null +++ b/database/sql/schema_version_35.sql @@ -0,0 +1 @@ +alter table feeds add column fetch_via_proxy bool default false; diff --git a/http/client/client.go b/http/client/client.go index 0288b721..177f1f44 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -47,6 +47,7 @@ type Client struct { password string userAgent string Insecure bool + fetchViaProxy bool } func (c *Client) String() string { @@ -93,6 +94,12 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client return c } +// WithProxy enable proxy for current HTTP client request. +func (c *Client) WithProxy() *Client { + c.fetchViaProxy = true + return c +} + // WithUserAgent defines the User-Agent header to use for outgoing requests. func (c *Client) WithUserAgent(userAgent string) *Client { if userAgent != "" { @@ -230,12 +237,23 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err func (c *Client) buildClient() http.Client { client := http.Client{Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second} + transport := &http.Transport{} if c.Insecure { - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + if c.fetchViaProxy && config.Opts.HasHTTPClientProxyConfigured() { + proxyURL, err := url.Parse(config.Opts.HTTPClientProxy()) + if err != nil { + logger.Error("[HttpClient] Proxy URL error: %v", err) + } else { + logger.Debug("[HttpClient] Use proxy: %s", proxyURL) + transport.Proxy = http.ProxyURL(proxyURL) } } + client.Transport = transport + return client } diff --git a/locale/translations.go b/locale/translations.go index 37a3638e..c0ab3260 100644 --- a/locale/translations.go +++ b/locale/translations.go @@ -254,6 +254,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Extraktionsregeln", "form.feed.label.rewrite_rules": "Umschreiberegeln", "form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache", + "form.feed.label.fetch_via_proxy": "Über Proxy abrufen", "form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren", "form.category.label.title": "Titel", "form.user.label.username": "Benutzername", @@ -599,6 +600,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Scraper Rules", "form.feed.label.rewrite_rules": "Rewrite Rules", "form.feed.label.ignore_http_cache": "Ignore HTTP cache", + "form.feed.label.fetch_via_proxy": "Fetch via proxy", "form.feed.label.disabled": "Do not refresh this feed", "form.category.label.title": "Title", "form.user.label.username": "Username", @@ -924,6 +926,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Reglas de raspador", "form.feed.label.rewrite_rules": "Reglas de reescribir", "form.feed.label.ignore_http_cache": "Ignorar caché HTTP", + "form.feed.label.fetch_via_proxy": "Buscar a través de proxy", "form.feed.label.disabled": "No actualice este feed", "form.category.label.title": "Título", "form.user.label.username": "Nombre de usuario", @@ -1249,6 +1252,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Règles pour récupérer le contenu original", "form.feed.label.rewrite_rules": "Règles de réécriture", "form.feed.label.ignore_http_cache": "Ignore cache HTTP", + "form.feed.label.fetch_via_proxy": "Récupérer via proxy", "form.feed.label.disabled": "Ne pas actualiser ce flux", "form.category.label.title": "Titre", "form.user.label.username": "Nom d'utilisateur", @@ -1594,6 +1598,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Regole di estrazione del contenuto", "form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto", "form.feed.label.ignore_http_cache": "Ignora cache HTTP", + "form.feed.label.fetch_via_proxy": "Recuperare tramite proxy", "form.feed.label.disabled": "Non aggiornare questo feed", "form.category.label.title": "Titolo", "form.user.label.username": "Nome utente", @@ -1919,6 +1924,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "スクラップルール", "form.feed.label.rewrite_rules": "Rewrite ルール", "form.feed.label.ignore_http_cache": "HTTPキャッシュを無視", + "form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ", "form.feed.label.disabled": "このフィードを更新しない", "form.category.label.title": "タイトル", "form.user.label.username": "ユーザー名", @@ -2244,6 +2250,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Scraper regels", "form.feed.label.rewrite_rules": "Rewrite regels", "form.feed.label.ignore_http_cache": "Negeer HTTP-cache", + "form.feed.label.fetch_via_proxy": "Ophalen via proxy", "form.feed.label.disabled": "Vernieuw deze feed niet", "form.category.label.title": "Naam", "form.user.label.username": "Gebruikersnaam", @@ -2589,6 +2596,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Zasady ekstrakcji", "form.feed.label.rewrite_rules": "Reguły zapisu", "form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP", + "form.feed.label.fetch_via_proxy": "Pobierz przez proxy", "form.feed.label.disabled": "Не обновлять этот канал", "form.category.label.title": "Tytuł", "form.user.label.username": "Nazwa użytkownika", @@ -2939,6 +2947,7 @@ var translations = map[string]string{ "form.feed.label.rewrite_rules": "Regras para o Rewrite", "form.feed.label.ignore_http_cache": "Ignorar cache HTTP", "form.feed.label.disabled": "Não atualizar esta fonte", + "form.feed.label.fetch_via_proxy": "Buscar via proxy", "form.category.label.title": "Título", "form.user.label.username": "Nome de usuário", "form.user.label.password": "Senha", @@ -3265,6 +3274,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Правила Scraper", "form.feed.label.rewrite_rules": "Правила Rewrite", "form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш", + "form.feed.label.fetch_via_proxy": "Получить через прокси", "form.feed.label.disabled": "Не обновлять этот канал", "form.category.label.title": "Название", "form.user.label.username": "Имя пользователя", @@ -3594,6 +3604,7 @@ var translations = map[string]string{ "form.feed.label.scraper_rules": "Scraper 规则", "form.feed.label.rewrite_rules": "重写规则", "form.feed.label.ignore_http_cache": "忽略HTTP缓存", + "form.feed.label.fetch_via_proxy": "通过代理获取", "form.feed.label.disabled": "请勿刷新此Feed", "form.category.label.title": "标题", "form.user.label.username": "用户名", @@ -3684,15 +3695,15 @@ var translations = map[string]string{ } var translationsChecksums = map[string]string{ - "de_DE": "21e1bfb0f43d71efe38812b4337ddf6980c11ed18f4d06446ff7eda9dfa6b1f1", - "en_US": "30cbcb2170782f1e66f69066947bf053f68065d7b270eea879f2c573819dd52b", - "es_ES": "50dc7c8c2db7368bae133f5b455721470d314321153d41e4f27436a0f3f176e6", - "fr_FR": "373fd2db868961758bd1483c34f117b03aadea17080f268bc8bbd0acdfbc5eed", - "it_IT": "8d8f0bd75b4e7dec9370647c888dd9438b691130d9c41f839cdfff8cbc606cb5", - "ja_JP": "ec3a21c547e4625ad359624e43ba31b556fb8d8b8ff7fc7a20df089317db99b3", - "nl_NL": "20e180be2375f07ec02eb05f372a9102c13037a79e5651ce9bd41507fd2180d2", - "pl_PL": "b1526955641823708b4c1ca753b61e1e0561d0a3d33da3f62170540903031b0d", - "pt_BR": "cf8e131d39daac82d3157c6538c0643392a06358b7bc98be8579412ebd63f60e", - "ru_RU": "4056e4e94861835d44064273371adbbded7190e2b719769886eb99e6c9feaf82", - "zh_CN": "044abb0a34eee3d8d5597811d40166762311b8e4cd08b891796113790cc775f0", + "de_DE": "8f96cb46f5a7e8f64ee8f10176dc3a2f3d53953d250317da83a79d0700b47c82", + "en_US": "d33324caed406ecf6ce03920b15e235d46b258457a8bd48cd1ade685b9a3ad6b", + "es_ES": "2ff9333218dba2b86cb84f377dad66b9dc73848aee6bb09889cbdc10e58ca077", + "fr_FR": "07dc2cfdbc14cdf16312423158656f5526d3c3c7be490abf5503109a408e5056", + "it_IT": "d4f68a507e1deb9fab3aa38fb78d9e9e4040386d6f36611ec5f105adfb4b0d03", + "ja_JP": "5c4c063ebaee14bed941b020e0d19de5ef5e8d3bf11c1967b1f321d57d5af6a9", + "nl_NL": "f862027e192be7a09730470acc2639971c4abf01b256d5bb81246960cc54adcf", + "pl_PL": "35147e55f1800964d268dc04b9cc25a9c8fa98077f759c5d3a5bd339f0eee53e", + "pt_BR": "2461105ebc2a2d57b3a63a29ee21f74e3d1eba54c049abcfd077dd30acc8d0a2", + "ru_RU": "402f15d3c68e008a1ffa3dddb4002126a29c1cf93359a64bca944a8da541da72", + "zh_CN": "bbea61a08bec37518d8c8c7735c7b8001d011a1d43ddb15ab639fee11b45ca87", } diff --git a/locale/translations/de_DE.json b/locale/translations/de_DE.json index 628d3a68..b2d9b5ad 100644 --- a/locale/translations/de_DE.json +++ b/locale/translations/de_DE.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "Extraktionsregeln", "form.feed.label.rewrite_rules": "Umschreiberegeln", "form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache", + "form.feed.label.fetch_via_proxy": "Über Proxy abrufen", "form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren", "form.category.label.title": "Titel", "form.user.label.username": "Benutzername", diff --git a/locale/translations/en_US.json b/locale/translations/en_US.json index 860ef317..4805c33b 100644 --- a/locale/translations/en_US.json +++ b/locale/translations/en_US.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "Scraper Rules", "form.feed.label.rewrite_rules": "Rewrite Rules", "form.feed.label.ignore_http_cache": "Ignore HTTP cache", + "form.feed.label.fetch_via_proxy": "Fetch via proxy", "form.feed.label.disabled": "Do not refresh this feed", "form.category.label.title": "Title", "form.user.label.username": "Username", diff --git a/locale/translations/es_ES.json b/locale/translations/es_ES.json index 4a890d8e..0134800b 100644 --- a/locale/translations/es_ES.json +++ b/locale/translations/es_ES.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "Reglas de raspador", "form.feed.label.rewrite_rules": "Reglas de reescribir", "form.feed.label.ignore_http_cache": "Ignorar caché HTTP", + "form.feed.label.fetch_via_proxy": "Buscar a través de proxy", "form.feed.label.disabled": "No actualice este feed", "form.category.label.title": "Título", "form.user.label.username": "Nombre de usuario", diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json index c20d8d32..543e35c1 100644 --- a/locale/translations/fr_FR.json +++ b/locale/translations/fr_FR.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "Règles pour récupérer le contenu original", "form.feed.label.rewrite_rules": "Règles de réécriture", "form.feed.label.ignore_http_cache": "Ignore cache HTTP", + "form.feed.label.fetch_via_proxy": "Récupérer via proxy", "form.feed.label.disabled": "Ne pas actualiser ce flux", "form.category.label.title": "Titre", "form.user.label.username": "Nom d'utilisateur", diff --git a/locale/translations/it_IT.json b/locale/translations/it_IT.json index 77f03d34..3b32e388 100644 --- a/locale/translations/it_IT.json +++ b/locale/translations/it_IT.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "Regole di estrazione del contenuto", "form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto", "form.feed.label.ignore_http_cache": "Ignora cache HTTP", + "form.feed.label.fetch_via_proxy": "Recuperare tramite proxy", "form.feed.label.disabled": "Non aggiornare questo feed", "form.category.label.title": "Titolo", "form.user.label.username": "Nome utente", diff --git a/locale/translations/ja_JP.json b/locale/translations/ja_JP.json index 3e3e97f2..01833d31 100644 --- a/locale/translations/ja_JP.json +++ b/locale/translations/ja_JP.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "スクラップルール", "form.feed.label.rewrite_rules": "Rewrite ルール", "form.feed.label.ignore_http_cache": "HTTPキャッシュを無視", + "form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ", "form.feed.label.disabled": "このフィードを更新しない", "form.category.label.title": "タイトル", "form.user.label.username": "ユーザー名", diff --git a/locale/translations/nl_NL.json b/locale/translations/nl_NL.json index c50ca705..cf95433b 100644 --- a/locale/translations/nl_NL.json +++ b/locale/translations/nl_NL.json @@ -249,6 +249,7 @@ "form.feed.label.scraper_rules": "Scraper regels", "form.feed.label.rewrite_rules": "Rewrite regels", "form.feed.label.ignore_http_cache": "Negeer HTTP-cache", + "form.feed.label.fetch_via_proxy": "Ophalen via proxy", "form.feed.label.disabled": "Vernieuw deze feed niet", "form.category.label.title": "Naam", "form.user.label.username": "Gebruikersnaam", diff --git a/locale/translations/pl_PL.json b/locale/translations/pl_PL.json index 8a3a7f92..4b4e0081 100644 --- a/locale/translations/pl_PL.json +++ b/locale/translations/pl_PL.json @@ -251,6 +251,7 @@ "form.feed.label.scraper_rules": "Zasady ekstrakcji", "form.feed.label.rewrite_rules": "Reguły zapisu", "form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP", + "form.feed.label.fetch_via_proxy": "Pobierz przez proxy", "form.feed.label.disabled": "Не обновлять этот канал", "form.category.label.title": "Tytuł", "form.user.label.username": "Nazwa użytkownika", diff --git a/locale/translations/pt_BR.json b/locale/translations/pt_BR.json index e425190f..4d41d490 100644 --- a/locale/translations/pt_BR.json +++ b/locale/translations/pt_BR.json @@ -250,6 +250,7 @@ "form.feed.label.rewrite_rules": "Regras para o Rewrite", "form.feed.label.ignore_http_cache": "Ignorar cache HTTP", "form.feed.label.disabled": "Não atualizar esta fonte", + "form.feed.label.fetch_via_proxy": "Buscar via proxy", "form.category.label.title": "Título", "form.user.label.username": "Nome de usuário", "form.user.label.password": "Senha", diff --git a/locale/translations/ru_RU.json b/locale/translations/ru_RU.json index 197e1f72..20522a9f 100644 --- a/locale/translations/ru_RU.json +++ b/locale/translations/ru_RU.json @@ -251,6 +251,7 @@ "form.feed.label.scraper_rules": "Правила Scraper", "form.feed.label.rewrite_rules": "Правила Rewrite", "form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш", + "form.feed.label.fetch_via_proxy": "Получить через прокси", "form.feed.label.disabled": "Не обновлять этот канал", "form.category.label.title": "Название", "form.user.label.username": "Имя пользователя", diff --git a/locale/translations/zh_CN.json b/locale/translations/zh_CN.json index 57313fe6..1562fc60 100644 --- a/locale/translations/zh_CN.json +++ b/locale/translations/zh_CN.json @@ -247,6 +247,7 @@ "form.feed.label.scraper_rules": "Scraper 规则", "form.feed.label.rewrite_rules": "重写规则", "form.feed.label.ignore_http_cache": "忽略HTTP缓存", + "form.feed.label.fetch_via_proxy": "通过代理获取", "form.feed.label.disabled": "请勿刷新此Feed", "form.category.label.title": "标题", "form.user.label.username": "用户名", diff --git a/miniflux.1 b/miniflux.1 index 46a398b7..01a1b5f3 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -250,6 +250,11 @@ Maximum body size for HTTP requests in Mebibyte (MiB)\&. .br Default is 15 MiB\&. .TP +.B HTTP_CLIENT_PROXY +Proxy URL for HTTP client\&. +.br +Default is empty\&. +.TP .B AUTH_PROXY_HEADER Proxy authentication HTTP header\&. .TP diff --git a/model/feed.go b/model/feed.go index c90e0db0..f13f0be9 100644 --- a/model/feed.go +++ b/model/feed.go @@ -34,6 +34,7 @@ type Feed struct { Password string `json:"password"` Disabled bool `json:"disabled"` IgnoreHTTPCache bool `json:"ignore_http_cache"` + FetchViaProxy bool `json:"fetch_via_proxy"` Category *Category `json:"category,omitempty"` Entries Entries `json:"entries,omitempty"` Icon *FeedIcon `json:"icon"` @@ -71,13 +72,14 @@ func (f *Feed) WithCategoryID(categoryID int64) { } // WithBrowsingParameters defines browsing parameters. -func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) { +func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) { f.Crawler = crawler f.UserAgent = userAgent f.Username = username f.Password = password f.ScraperRules = scraperRules f.RewriteRules = rewriteRules + f.FetchViaProxy = fetchViaProxy } // WithError adds a new error message and increment the error counter. diff --git a/model/feed_test.go b/model/feed_test.go index 8843bb49..8dc9fa0f 100644 --- a/model/feed_test.go +++ b/model/feed_test.go @@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) { func TestFeedBrowsingParams(t *testing.T) { feed := &Feed{} - feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule") + feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false) if !feed.Crawler { t.Error(`The crawler must be activated`) diff --git a/reader/feed/handler.go b/reader/feed/handler.go index 5b27b267..5feac58d 100644 --- a/reader/feed/handler.go +++ b/reader/feed/handler.go @@ -34,7 +34,7 @@ type Handler struct { } // CreateFeed fetch, parse and store a new feed. -func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) { +func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url)) if !h.store.CategoryExists(userID, categoryID) { @@ -44,6 +44,11 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, request := client.New(url) request.WithCredentials(username, password) request.WithUserAgent(userAgent) + + if fetchViaProxy { + request.WithProxy() + } + response, requestErr := browser.Exec(request) if requestErr != nil { return nil, requestErr @@ -60,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, subscription.UserID = userID subscription.WithCategoryID(categoryID) - subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules) + subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy) subscription.WithClientResponse(response) subscription.CheckedNow() @@ -72,7 +77,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, logger.Debug("[Handler:CreateFeed] Feed saved with ID: %d", subscription.ID) - checkFeedIcon(h.store, subscription.ID, subscription.SiteURL) + checkFeedIcon(h.store, subscription.ID, subscription.SiteURL, fetchViaProxy) return subscription, nil } @@ -111,6 +116,10 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error { request.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) } + if originalFeed.FetchViaProxy { + request.WithProxy() + } + response, requestErr := browser.Exec(request) if requestErr != nil { originalFeed.WithError(requestErr.Localize(printer)) @@ -141,7 +150,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error { // We update caching headers only if the feed has been modified, // because some websites don't return the same headers when replying with a 304. originalFeed.WithClientResponse(response) - checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL) + checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL, originalFeed.FetchViaProxy) } else { logger.Debug("[Handler:RefreshFeed] Feed #%d not modified", feedID) } @@ -162,9 +171,9 @@ func NewFeedHandler(store *storage.Storage) *Handler { return &Handler{store} } -func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) { +func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string, fetchViaProxy bool) { if !store.HasIcon(feedID) { - icon, err := icon.FindIcon(websiteURL) + icon, err := icon.FindIcon(websiteURL, fetchViaProxy) if err != nil { logger.Debug("CheckFeedIcon: %v (feedID=%d websiteURL=%s)", err, feedID, websiteURL) } else if icon == nil { diff --git a/reader/icon/finder.go b/reader/icon/finder.go index c9da1bc0..5dde2235 100644 --- a/reader/icon/finder.go +++ b/reader/icon/finder.go @@ -21,9 +21,12 @@ import ( ) // FindIcon try to find the website's icon. -func FindIcon(websiteURL string) (*model.Icon, error) { +func FindIcon(websiteURL string, fetchViaProxy bool) (*model.Icon, error) { rootURL := url.RootURL(websiteURL) clt := client.New(rootURL) + if fetchViaProxy { + clt.WithProxy() + } response, err := clt.Get() if err != nil { return nil, fmt.Errorf("unable to download website index page: %v", err) @@ -43,7 +46,7 @@ func FindIcon(websiteURL string) (*model.Icon, error) { } logger.Debug("[FindIcon] Fetching icon => %s", iconURL) - icon, err := downloadIcon(iconURL) + icon, err := downloadIcon(iconURL, fetchViaProxy) if err != nil { return nil, err } @@ -86,8 +89,11 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) { return iconURL, nil } -func downloadIcon(iconURL string) (*model.Icon, error) { +func downloadIcon(iconURL string, fetchViaProxy bool) (*model.Icon, error) { clt := client.New(iconURL) + if fetchViaProxy { + clt.WithProxy() + } response, err := clt.Get() if err != nil { return nil, fmt.Errorf("unable to download iconURL: %v", err) diff --git a/reader/subscription/finder.go b/reader/subscription/finder.go index 62db5a33..e4f2a812 100644 --- a/reader/subscription/finder.go +++ b/reader/subscription/finder.go @@ -26,13 +26,18 @@ var ( ) // FindSubscriptions downloads and try to find one or more subscriptions from an URL. -func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, *errors.LocalizedError) { +func FindSubscriptions(websiteURL, userAgent, username, password string, fetchViaProxy bool) (Subscriptions, *errors.LocalizedError) { websiteURL = findYoutubeChannelFeed(websiteURL) websiteURL = parseYoutubeVideoPage(websiteURL) request := client.New(websiteURL) request.WithCredentials(username, password) request.WithUserAgent(userAgent) + + if fetchViaProxy { + request.WithProxy() + } + response, err := browser.Exec(request) if err != nil { return nil, err diff --git a/storage/feed.go b/storage/feed.go index 7f73e4c0..49f7187f 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -32,6 +32,7 @@ var feedListQuery = ` f.username, f.password, f.ignore_http_cache, + f.fetch_via_proxy, f.disabled, f.category_id, c.title as category_title, @@ -133,6 +134,7 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F f.username, f.password, f.ignore_http_cache, + f.fetch_via_proxy, f.disabled, f.category_id, c.title as category_title, @@ -242,6 +244,7 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{} &feed.Username, &feed.Password, &feed.IgnoreHTTPCache, + &feed.FetchViaProxy, &feed.Disabled, &feed.Category.ID, &feed.Category.Title, @@ -326,6 +329,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) { f.username, f.password, f.ignore_http_cache, + f.fetch_via_proxy, f.disabled, f.category_id, c.title as category_title, @@ -357,6 +361,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) { &feed.Username, &feed.Password, &feed.IgnoreHTTPCache, + &feed.FetchViaProxy, &feed.Disabled, &feed.Category.ID, &feed.Category.Title, @@ -396,10 +401,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { password, disabled, scraper_rules, - rewrite_rules + rewrite_rules, + fetch_via_proxy ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id ` @@ -419,6 +425,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.Disabled, feed.ScraperRules, feed.RewriteRules, + feed.FetchViaProxy, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) @@ -462,9 +469,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { password=$15, disabled=$16, next_check_at=$17, - ignore_http_cache=$18 + ignore_http_cache=$18, + fetch_via_proxy=$19 WHERE - id=$19 AND user_id=$20 + id=$20 AND user_id=$21 ` _, err = s.db.Exec(query, feed.FeedURL, @@ -485,6 +493,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.Disabled, feed.NextCheckAt, feed.IgnoreHTTPCache, + feed.FetchViaProxy, feed.ID, feed.UserID, ) diff --git a/template/common.go b/template/common.go index 32acb24a..4c5cbc84 100644 --- a/template/common.go +++ b/template/common.go @@ -519,7 +519,7 @@ SOFTWARE. var templateCommonMapChecksums = map[string]string{ "entry_pagination": "cdca9cf12586e41e5355190b06d9168f57f77b85924d1e63b13524bc15abcbf6", - "feed_list": "30acc9ecc413811e73a1dad120b5d44e29564de3ba794fb07ee886b30addfb19", + "feed_list": "931e43d328a116318c510de5658c688cd940b934c86b6ec82a472e1f81e020ae", "feed_menu": "318d8662dda5ca9dfc75b909c8461e79c86fb5082df1428f67aaf856f19f4b50", "icons": "3dbe754a98f524a227111191d76b8c6944711b13613cc548ee9e9808fe0bffb4", "item_meta": "8306adf3ef9966de3e3dc74ca1042e51d778b027ab8cf0a60a2e94a0115982dc", diff --git a/template/html/add_subscription.html b/template/html/add_subscription.html index e435f32d..74a6963f 100644 --- a/template/html/add_subscription.html +++ b/template/html/add_subscription.html @@ -30,6 +30,9 @@ {{ t "page.add_feed.legend.advanced_options" }}
+ {{ if .hasProxyConfigured }} + + {{ end }} diff --git a/template/html/choose_subscription.html b/template/html/choose_subscription.html index cf7cf5a6..6da42eaf 100644 --- a/template/html/choose_subscription.html +++ b/template/html/choose_subscription.html @@ -14,6 +14,9 @@ + {{ if .form.FetchViaProxy }} + + {{ end }} {{ if .form.Crawler }} {{ end }} diff --git a/template/html/edit_feed.html b/template/html/edit_feed.html index 267b6b92..8242da9e 100644 --- a/template/html/edit_feed.html +++ b/template/html/edit_feed.html @@ -73,6 +73,9 @@ + {{ if .hasProxyConfigured }} + + {{ end }}
diff --git a/template/views.go b/template/views.go index 94fd2e54..a657a783 100644 --- a/template/views.go +++ b/template/views.go @@ -61,6 +61,9 @@ var templateViewsMap = map[string]string{ {{ t "page.add_feed.legend.advanced_options" }}
+ {{ if .hasProxyConfigured }} + + {{ end }} @@ -380,6 +383,9 @@ var templateViewsMap = map[string]string{ + {{ if .form.FetchViaProxy }} + + {{ end }} {{ if .form.Crawler }} {{ end }} @@ -592,6 +598,9 @@ var templateViewsMap = map[string]string{ + {{ if .hasProxyConfigured }} + + {{ end }}
@@ -1548,18 +1557,18 @@ var templateViewsMap = map[string]string{ var templateViewsMapChecksums = map[string]string{ "about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc", - "add_subscription": "0dbea93b6fc07423fa066122ad960c69616b829533371a2dbadec1e22d4f1ae0", + "add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032", "api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284", "bookmark_entries": "892fe6cbf5a3301416dfb76e62935b495ca194275cfe113105a85b40ce7c200f", "categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9", "category_entries": "8fa0e0b8f85e2572c40dee855b6d636207c3561086b234c93100673774c06746", "category_feeds": "07154127087f9b127f7290abad6020c35ad9ceb2490b869120b7628bc4413808", - "choose_subscription": "84c9730cadd78e6ee5a6b4c499aab33acddb4324ac01924d33387543eec4d702", + "choose_subscription": "22109d760ea8079c491561d0106f773c885efbf66f87d81fcf8700218260d2a0", "create_api_key": "5f74d4e92a6684927f5305096378c8be278159a5cd88ce652c7be3280a7d1685", "create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d", "create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a", "edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36", - "edit_feed": "ff90b1883e2934e0236d530d8b778affe7665a6b08cdf9ae612c7e56469818ef", + "edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b", "edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a", "entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c", "feed_entries": "ea5b88e3ad6b166d83b70e021d7b420d025f80decb6e24c79d13f8ce7c910b04", diff --git a/ui/feed_edit.go b/ui/feed_edit.go index f2a2d692..3dddb1b7 100644 --- a/ui/feed_edit.go +++ b/ui/feed_edit.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/config" "miniflux.app/http/client" "miniflux.app/http/request" "miniflux.app/http/response/html" @@ -52,6 +53,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) { Username: feed.Username, Password: feed.Password, IgnoreHTTPCache: feed.IgnoreHTTPCache, + FetchViaProxy: feed.FetchViaProxy, Disabled: feed.Disabled, } @@ -65,6 +67,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) { view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID)) view.Set("defaultUserAgent", client.DefaultUserAgent) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) html.OK(w, r, view.Render("edit_feed")) } diff --git a/ui/form/feed.go b/ui/form/feed.go index 6ad8f613..eeeeb7b0 100644 --- a/ui/form/feed.go +++ b/ui/form/feed.go @@ -25,6 +25,7 @@ type FeedForm struct { Username string Password string IgnoreHTTPCache bool + FetchViaProxy bool Disabled bool } @@ -51,6 +52,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed { feed.Username = f.Username feed.Password = f.Password feed.IgnoreHTTPCache = f.IgnoreHTTPCache + feed.FetchViaProxy = f.FetchViaProxy feed.Disabled = f.Disabled return feed } @@ -74,6 +76,7 @@ func NewFeedForm(r *http.Request) *FeedForm { Username: r.FormValue("feed_username"), Password: r.FormValue("feed_password"), IgnoreHTTPCache: r.FormValue("ignore_http_cache") == "1", + FetchViaProxy: r.FormValue("fetch_via_proxy") == "1", Disabled: r.FormValue("disabled") == "1", } } diff --git a/ui/form/subscription.go b/ui/form/subscription.go index f6348e22..74be3c0c 100644 --- a/ui/form/subscription.go +++ b/ui/form/subscription.go @@ -13,14 +13,15 @@ import ( // SubscriptionForm represents the subscription form. type SubscriptionForm struct { - URL string - CategoryID int64 - Crawler bool - UserAgent string - Username string - Password string - ScraperRules string - RewriteRules string + URL string + CategoryID int64 + Crawler bool + FetchViaProxy bool + UserAgent string + Username string + Password string + ScraperRules string + RewriteRules string } // Validate makes sure the form values are valid. @@ -40,13 +41,14 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm { } return &SubscriptionForm{ - URL: r.FormValue("url"), - Crawler: r.FormValue("crawler") == "1", - CategoryID: int64(categoryID), - UserAgent: r.FormValue("user_agent"), - Username: r.FormValue("feed_username"), - Password: r.FormValue("feed_password"), - ScraperRules: r.FormValue("scraper_rules"), - RewriteRules: r.FormValue("rewrite_rules"), + URL: r.FormValue("url"), + Crawler: r.FormValue("crawler") == "1", + FetchViaProxy: r.FormValue("fetch_via_proxy") == "1", + CategoryID: int64(categoryID), + UserAgent: r.FormValue("user_agent"), + Username: r.FormValue("feed_username"), + Password: r.FormValue("feed_password"), + ScraperRules: r.FormValue("scraper_rules"), + RewriteRules: r.FormValue("rewrite_rules"), } } diff --git a/ui/subscription_add.go b/ui/subscription_add.go index 7cb201a9..e67f2510 100644 --- a/ui/subscription_add.go +++ b/ui/subscription_add.go @@ -2,14 +2,15 @@ // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. -package ui // import "miniflux.app/ui" +package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/config" "miniflux.app/http/client" - "miniflux.app/http/response/html" "miniflux.app/http/request" + "miniflux.app/http/response/html" "miniflux.app/ui/form" "miniflux.app/ui/session" "miniflux.app/ui/view" @@ -38,6 +39,7 @@ func (h *handler) showAddSubscriptionPage(w http.ResponseWriter, r *http.Request view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID)) view.Set("defaultUserAgent", client.DefaultUserAgent) view.Set("form", &form.SubscriptionForm{CategoryID: 0}) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) html.OK(w, r, view.Render("add_subscription")) } diff --git a/ui/subscription_bookmarklet.go b/ui/subscription_bookmarklet.go index 060aa486..bf1674ea 100644 --- a/ui/subscription_bookmarklet.go +++ b/ui/subscription_bookmarklet.go @@ -2,11 +2,12 @@ // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. -package ui // import "miniflux.app/ui" +package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/config" "miniflux.app/http/client" "miniflux.app/http/request" "miniflux.app/http/response/html" @@ -40,6 +41,7 @@ func (h *handler) bookmarklet(w http.ResponseWriter, r *http.Request) { view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID)) view.Set("defaultUserAgent", client.DefaultUserAgent) + view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) html.OK(w, r, view.Render("add_subscription")) } diff --git a/ui/subscription_choose.go b/ui/subscription_choose.go index 37c74617..3df6e13a 100644 --- a/ui/subscription_choose.go +++ b/ui/subscription_choose.go @@ -57,6 +57,7 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ subscriptionForm.Password, subscriptionForm.ScraperRules, subscriptionForm.RewriteRules, + subscriptionForm.FetchViaProxy, ) if err != nil { view.Set("form", subscriptionForm) diff --git a/ui/subscription_submit.go b/ui/subscription_submit.go index 0a3c1af0..44ca6f19 100644 --- a/ui/subscription_submit.go +++ b/ui/subscription_submit.go @@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui" import ( "net/http" + "miniflux.app/config" "miniflux.app/http/client" "miniflux.app/http/request" "miniflux.app/http/response/html" @@ -40,6 +41,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { v.Set("countUnread", h.store.CountUnreadEntries(user.ID)) v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID)) v.Set("defaultUserAgent", client.DefaultUserAgent) + v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) subscriptionForm := form.NewSubscriptionForm(r) if err := subscriptionForm.Validate(); err != nil { @@ -54,6 +56,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { subscriptionForm.UserAgent, subscriptionForm.Username, subscriptionForm.Password, + subscriptionForm.FetchViaProxy, ) if findErr != nil { logger.Error("[UI:SubmitSubscription] %s", findErr) @@ -82,6 +85,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { subscriptionForm.Password, subscriptionForm.ScraperRules, subscriptionForm.RewriteRules, + subscriptionForm.FetchViaProxy, ) if err != nil { v.Set("form", subscriptionForm) @@ -99,6 +103,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { v.Set("user", user) v.Set("countUnread", h.store.CountUnreadEntries(user.ID)) v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID)) + v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured()) html.OK(w, r, v.Render("choose_subscription")) }