Do not follow redirects when trying known feed URLs
Some websites redirects unknown URLs to the home page. As result, the list of known URLs is returned to the subscription list. We don't want the user to choose between invalid feed URLs.
This commit is contained in:
parent
40e983664c
commit
246a48359c
2 changed files with 26 additions and 3 deletions
|
@ -52,7 +52,8 @@ type Client struct {
|
||||||
requestPassword string
|
requestPassword string
|
||||||
requestUserAgent string
|
requestUserAgent string
|
||||||
|
|
||||||
useProxy bool
|
useProxy bool
|
||||||
|
doNotFollowRedirects bool
|
||||||
|
|
||||||
ClientTimeout int
|
ClientTimeout int
|
||||||
ClientMaxBodySize int64
|
ClientMaxBodySize int64
|
||||||
|
@ -124,12 +125,18 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithProxy enable proxy for the current HTTP request.
|
// WithProxy enables proxy for the current HTTP request.
|
||||||
func (c *Client) WithProxy() *Client {
|
func (c *Client) WithProxy() *Client {
|
||||||
c.useProxy = true
|
c.useProxy = true
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithoutRedirects disables HTTP redirects.
|
||||||
|
func (c *Client) WithoutRedirects() *Client {
|
||||||
|
c.doNotFollowRedirects = true
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// WithUserAgent defines the User-Agent header to use for HTTP requests.
|
// WithUserAgent defines the User-Agent header to use for HTTP requests.
|
||||||
func (c *Client) WithUserAgent(userAgent string) *Client {
|
func (c *Client) WithUserAgent(userAgent string) *Client {
|
||||||
if userAgent != "" {
|
if userAgent != "" {
|
||||||
|
@ -266,7 +273,10 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) buildClient() http.Client {
|
func (c *Client) buildClient() http.Client {
|
||||||
client := http.Client{Timeout: time.Duration(c.ClientTimeout) * time.Second}
|
client := http.Client{
|
||||||
|
Timeout: time.Duration(c.ClientTimeout) * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
transport := &http.Transport{
|
transport := &http.Transport{
|
||||||
Proxy: http.ProxyFromEnvironment,
|
Proxy: http.ProxyFromEnvironment,
|
||||||
DialContext: (&net.Dialer{
|
DialContext: (&net.Dialer{
|
||||||
|
@ -284,6 +294,12 @@ func (c *Client) buildClient() http.Client {
|
||||||
IdleConnTimeout: 10 * time.Second,
|
IdleConnTimeout: 10 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if c.doNotFollowRedirects {
|
||||||
|
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||||
|
return http.ErrUseLastResponse
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if c.useProxy && c.ClientProxyURL != "" {
|
if c.useProxy && c.ClientProxyURL != "" {
|
||||||
proxyURL, err := url.Parse(c.ClientProxyURL)
|
proxyURL, err := url.Parse(c.ClientProxyURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -156,9 +156,16 @@ func tryWellKnownUrls(websiteURL, userAgent, username, password string) (Subscri
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
clt := client.NewClientWithConfig(fullURL, config.Opts)
|
clt := client.NewClientWithConfig(fullURL, config.Opts)
|
||||||
clt.WithCredentials(username, password)
|
clt.WithCredentials(username, password)
|
||||||
clt.WithUserAgent(userAgent)
|
clt.WithUserAgent(userAgent)
|
||||||
|
|
||||||
|
// Some websites redirects unknown URLs to the home page.
|
||||||
|
// As result, the list of known URLs is returned to the subscription list.
|
||||||
|
// We don't want the user to choose between invalid feed URLs.
|
||||||
|
clt.WithoutRedirects()
|
||||||
|
|
||||||
response, err := clt.Get()
|
response, err := clt.Get()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
|
|
Loading…
Reference in a new issue