miniflux/http/client/client.go

336 lines
8.5 KiB
Go
Raw Normal View History

2018-04-28 19:51:07 +02:00
// Copyright 2018 Frédéric Guillot. All rights reserved.
2017-11-20 06:10:04 +01:00
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
2018-08-25 06:51:50 +02:00
package client // import "miniflux.app/http/client"
2017-11-20 06:10:04 +01:00
import (
2017-12-19 05:52:46 +01:00
"bytes"
2018-02-09 03:16:54 +01:00
"crypto/x509"
2017-12-19 05:52:46 +01:00
"encoding/json"
2017-11-20 06:10:04 +01:00
"fmt"
2017-12-19 05:52:46 +01:00
"io"
"io/ioutil"
2018-02-09 03:16:54 +01:00
"net"
2017-11-20 06:10:04 +01:00
"net/http"
"net/url"
2017-12-19 05:52:46 +01:00
"strings"
2017-11-20 06:10:04 +01:00
"time"
"miniflux.app/config"
2018-08-25 06:51:50 +02:00
"miniflux.app/errors"
"miniflux.app/logger"
"miniflux.app/timer"
url_helper "miniflux.app/url"
2017-11-20 06:10:04 +01:00
)
const (
defaultHTTPClientTimeout = 20
defaultHTTPClientMaxBodySize = 15 * 1024 * 1024
)
2018-02-09 03:16:54 +01:00
var (
errInvalidCertificate = "Invalid SSL certificate (original error: %q)"
errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)"
errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)"
errRequestTimeout = "Website unreachable, the request timed out after %d seconds"
)
2017-11-20 06:10:04 +01:00
// Client builds and executes HTTP requests.
type Client struct {
inputURL string
requestURL string
requestEtagHeader string
requestLastModifiedHeader string
requestAuthorizationHeader string
requestUsername string
requestPassword string
requestUserAgent string
useProxy bool
doNotFollowRedirects bool
ClientTimeout int
ClientMaxBodySize int64
ClientProxyURL string
}
// New initializes a new HTTP client.
func New(url string) *Client {
return &Client{
inputURL: url,
ClientTimeout: defaultHTTPClientTimeout,
ClientMaxBodySize: defaultHTTPClientMaxBodySize,
}
}
// NewClientWithConfig initializes a new HTTP client with application config options.
func NewClientWithConfig(url string, opts *config.Options) *Client {
return &Client{
inputURL: url,
requestUserAgent: opts.HTTPClientUserAgent(),
ClientTimeout: opts.HTTPClientTimeout(),
ClientMaxBodySize: opts.HTTPClientMaxBodySize(),
ClientProxyURL: opts.HTTPClientProxy(),
}
2017-11-20 06:10:04 +01:00
}
func (c *Client) String() string {
etagHeader := c.requestEtagHeader
if c.requestEtagHeader == "" {
2020-06-06 06:50:59 +02:00
etagHeader = "None"
}
lastModifiedHeader := c.requestLastModifiedHeader
if c.requestLastModifiedHeader == "" {
2020-06-06 06:50:59 +02:00
lastModifiedHeader = "None"
}
return fmt.Sprintf(
`InputURL=%q RequestURL=%q ETag=%s LastModified=%s Auth=%v UserAgent=%q`,
c.inputURL,
c.requestURL,
2020-06-06 06:50:59 +02:00
etagHeader,
lastModifiedHeader,
c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != ""),
c.requestUserAgent,
)
}
2018-04-28 19:51:07 +02:00
// WithCredentials defines the username/password for HTTP Basic authentication.
func (c *Client) WithCredentials(username, password string) *Client {
if username != "" && password != "" {
c.requestUsername = username
c.requestPassword = password
}
2018-04-28 19:51:07 +02:00
return c
}
// WithAuthorization defines the authorization HTTP header value.
2018-04-28 19:51:07 +02:00
func (c *Client) WithAuthorization(authorization string) *Client {
c.requestAuthorizationHeader = authorization
2018-04-28 19:51:07 +02:00
return c
}
// WithCacheHeaders defines caching headers.
func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client {
c.requestEtagHeader = etagHeader
c.requestLastModifiedHeader = lastModifiedHeader
2018-04-28 19:51:07 +02:00
return c
}
// WithProxy enables proxy for the current HTTP request.
func (c *Client) WithProxy() *Client {
c.useProxy = true
return c
}
// WithoutRedirects disables HTTP redirects.
func (c *Client) WithoutRedirects() *Client {
c.doNotFollowRedirects = true
return c
}
// WithUserAgent defines the User-Agent header to use for HTTP requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
if userAgent != "" {
c.requestUserAgent = userAgent
}
return c
}
// Get performs a GET HTTP request.
2017-12-03 04:32:14 +01:00
func (c *Client) Get() (*Response, error) {
2017-12-19 05:52:46 +01:00
request, err := c.buildRequest(http.MethodGet, nil)
if err != nil {
return nil, err
}
return c.executeRequest(request)
}
// PostForm performs a POST HTTP request with form encoded values.
2017-12-19 05:52:46 +01:00
func (c *Client) PostForm(values url.Values) (*Response, error) {
request, err := c.buildRequest(http.MethodPost, strings.NewReader(values.Encode()))
if err != nil {
return nil, err
}
request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
return c.executeRequest(request)
}
// PostJSON performs a POST HTTP request with a JSON payload.
2017-12-19 05:52:46 +01:00
func (c *Client) PostJSON(data interface{}) (*Response, error) {
b, err := json.Marshal(data)
if err != nil {
return nil, err
}
request, err := c.buildRequest(http.MethodPost, bytes.NewReader(b))
if err != nil {
return nil, err
}
request.Header.Add("Content-Type", "application/json")
return c.executeRequest(request)
}
func (c *Client) executeRequest(request *http.Request) (*Response, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient] inputURL=%s", c.inputURL))
logger.Debug("[HttpClient:Before] Method=%s %s",
request.Method,
c.String(),
)
2017-12-19 05:52:46 +01:00
2017-12-03 04:32:14 +01:00
client := c.buildClient()
2017-12-19 05:52:46 +01:00
resp, err := client.Do(request)
if resp != nil {
defer resp.Body.Close()
}
2017-11-20 06:10:04 +01:00
if err != nil {
2018-02-09 03:16:54 +01:00
if uerr, ok := err.(*url.Error); ok {
switch uerr.Err.(type) {
case x509.CertificateInvalidError, x509.HostnameError:
err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err)
case *net.OpError:
if uerr.Err.(*net.OpError).Temporary() {
err = errors.NewLocalizedError(errTemporaryNetworkOperation, uerr.Err)
} else {
err = errors.NewLocalizedError(errPermanentNetworkOperation, uerr.Err)
}
case net.Error:
nerr := uerr.Err.(net.Error)
if nerr.Timeout() {
err = errors.NewLocalizedError(errRequestTimeout, c.ClientTimeout)
2018-02-09 03:16:54 +01:00
} else if nerr.Temporary() {
err = errors.NewLocalizedError(errTemporaryNetworkOperation, nerr)
}
}
}
2017-11-20 06:10:04 +01:00
return nil, err
}
if resp.ContentLength > c.ClientMaxBodySize {
2018-01-03 03:30:26 +01:00
return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength)
}
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("client: error while reading body %v", err)
}
response := &Response{
Body: bytes.NewReader(buf),
StatusCode: resp.StatusCode,
EffectiveURL: resp.Request.URL.String(),
LastModified: resp.Header.Get("Last-Modified"),
ETag: resp.Header.Get("ETag"),
Expires: resp.Header.Get("Expires"),
ContentType: resp.Header.Get("Content-Type"),
ContentLength: resp.ContentLength,
2017-11-20 06:10:04 +01:00
}
logger.Debug("[HttpClient:After] Method=%s %s; Response => %s",
2017-12-19 05:52:46 +01:00
request.Method,
c.String(),
response,
2017-11-20 06:10:04 +01:00
)
// Ignore caching headers for feeds that do not want any cache.
if resp.Header.Get("Expires") == "0" {
logger.Debug("[HttpClient] Ignore caching headers for %q", response.EffectiveURL)
response.ETag = ""
response.LastModified = ""
}
2017-11-20 06:10:04 +01:00
return response, err
}
2017-12-19 05:52:46 +01:00
func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) {
c.requestURL = url_helper.RequestURI(c.inputURL)
request, err := http.NewRequest(method, c.requestURL, body)
2017-12-19 05:52:46 +01:00
if err != nil {
return nil, err
2017-12-03 04:32:14 +01:00
}
2018-02-25 20:49:08 +01:00
request.Header = c.buildHeaders()
if c.requestUsername != "" && c.requestPassword != "" {
request.SetBasicAuth(c.requestUsername, c.requestPassword)
2017-12-03 04:32:14 +01:00
}
2017-12-19 05:52:46 +01:00
return request, nil
2017-12-03 04:32:14 +01:00
}
func (c *Client) buildClient() http.Client {
client := http.Client{
Timeout: time.Duration(c.ClientTimeout) * time.Second,
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
// Default is 30s.
Timeout: 10 * time.Second,
// Default is 30s.
KeepAlive: 15 * time.Second,
}).DialContext,
// Default is 100.
MaxIdleConns: 50,
// Default is 90s.
IdleConnTimeout: 10 * time.Second,
}
if c.doNotFollowRedirects {
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}
if c.useProxy && c.ClientProxyURL != "" {
proxyURL, err := url.Parse(c.ClientProxyURL)
if err != nil {
logger.Error("[HttpClient] Proxy URL error: %v", err)
} else {
logger.Debug("[HttpClient] Use proxy: %s", proxyURL)
transport.Proxy = http.ProxyURL(proxyURL)
2017-11-20 06:10:04 +01:00
}
}
client.Transport = transport
2017-11-21 04:44:28 +01:00
return client
2017-11-20 06:10:04 +01:00
}
2017-12-03 04:32:14 +01:00
func (c *Client) buildHeaders() http.Header {
2017-11-20 06:10:04 +01:00
headers := make(http.Header)
headers.Add("Accept", "*/*")
2017-11-20 06:10:04 +01:00
if c.requestUserAgent != "" {
headers.Add("User-Agent", c.requestUserAgent)
}
if c.requestEtagHeader != "" {
headers.Add("If-None-Match", c.requestEtagHeader)
2017-11-20 06:10:04 +01:00
}
if c.requestLastModifiedHeader != "" {
headers.Add("If-Modified-Since", c.requestLastModifiedHeader)
2017-11-20 06:10:04 +01:00
}
if c.requestAuthorizationHeader != "" {
headers.Add("Authorization", c.requestAuthorizationHeader)
2017-12-19 05:52:46 +01:00
}
2018-06-20 05:21:24 +02:00
headers.Add("Connection", "close")
2017-11-20 06:10:04 +01:00
return headers
}