// Copyright 2018 Frédéric Guillot. All rights reserved. // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. package client // import "miniflux.app/http/client" import ( "bytes" "crypto/x509" "encoding/json" "fmt" "io" "io/ioutil" "net" "net/http" "net/url" "strings" "time" "miniflux.app/config" "miniflux.app/errors" "miniflux.app/logger" "miniflux.app/timer" url_helper "miniflux.app/url" "miniflux.app/version" ) const ( defaultHTTPClientTimeout = 20 defaultHTTPClientMaxBodySize = 15 * 1024 * 1024 ) var ( // DefaultUserAgent sets the User-Agent header used for any requests by miniflux. DefaultUserAgent = "Mozilla/5.0 (compatible; Miniflux/" + version.Version + "; +https://miniflux.app)" errInvalidCertificate = "Invalid SSL certificate (original error: %q)" errTemporaryNetworkOperation = "This website is temporarily unreachable (original error: %q)" errPermanentNetworkOperation = "This website is permanently unreachable (original error: %q)" errRequestTimeout = "Website unreachable, the request timed out after %d seconds" ) // Client builds and executes HTTP requests. type Client struct { inputURL string requestURL string requestEtagHeader string requestLastModifiedHeader string requestAuthorizationHeader string requestUsername string requestPassword string requestUserAgent string useProxy bool doNotFollowRedirects bool ClientTimeout int ClientMaxBodySize int64 ClientProxyURL string } // New initializes a new HTTP client. func New(url string) *Client { return &Client{ inputURL: url, requestUserAgent: DefaultUserAgent, ClientTimeout: defaultHTTPClientTimeout, ClientMaxBodySize: defaultHTTPClientMaxBodySize, } } // NewClientWithConfig initializes a new HTTP client with application config options. func NewClientWithConfig(url string, opts *config.Options) *Client { userAgent := opts.HTTPClientUserAgent() if userAgent == "" { userAgent = DefaultUserAgent } return &Client{ inputURL: url, requestUserAgent: userAgent, ClientTimeout: opts.HTTPClientTimeout(), ClientMaxBodySize: opts.HTTPClientMaxBodySize(), ClientProxyURL: opts.HTTPClientProxy(), } } func (c *Client) String() string { etagHeader := c.requestEtagHeader if c.requestEtagHeader == "" { etagHeader = "None" } lastModifiedHeader := c.requestLastModifiedHeader if c.requestLastModifiedHeader == "" { lastModifiedHeader = "None" } return fmt.Sprintf( `InputURL=%q RequestURL=%q ETag=%s LastModified=%s Auth=%v UserAgent=%q`, c.inputURL, c.requestURL, etagHeader, lastModifiedHeader, c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != ""), c.requestUserAgent, ) } // WithCredentials defines the username/password for HTTP Basic authentication. func (c *Client) WithCredentials(username, password string) *Client { if username != "" && password != "" { c.requestUsername = username c.requestPassword = password } return c } // WithAuthorization defines the authorization HTTP header value. func (c *Client) WithAuthorization(authorization string) *Client { c.requestAuthorizationHeader = authorization return c } // WithCacheHeaders defines caching headers. func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client { c.requestLastModifiedHeader = etagHeader c.requestLastModifiedHeader = lastModifiedHeader return c } // WithProxy enables proxy for the current HTTP request. func (c *Client) WithProxy() *Client { c.useProxy = true return c } // WithoutRedirects disables HTTP redirects. func (c *Client) WithoutRedirects() *Client { c.doNotFollowRedirects = true return c } // WithUserAgent defines the User-Agent header to use for HTTP requests. func (c *Client) WithUserAgent(userAgent string) *Client { if userAgent != "" { c.requestUserAgent = userAgent } return c } // Get performs a GET HTTP request. func (c *Client) Get() (*Response, error) { request, err := c.buildRequest(http.MethodGet, nil) if err != nil { return nil, err } return c.executeRequest(request) } // PostForm performs a POST HTTP request with form encoded values. func (c *Client) PostForm(values url.Values) (*Response, error) { request, err := c.buildRequest(http.MethodPost, strings.NewReader(values.Encode())) if err != nil { return nil, err } request.Header.Add("Content-Type", "application/x-www-form-urlencoded") return c.executeRequest(request) } // PostJSON performs a POST HTTP request with a JSON payload. func (c *Client) PostJSON(data interface{}) (*Response, error) { b, err := json.Marshal(data) if err != nil { return nil, err } request, err := c.buildRequest(http.MethodPost, bytes.NewReader(b)) if err != nil { return nil, err } request.Header.Add("Content-Type", "application/json") return c.executeRequest(request) } func (c *Client) executeRequest(request *http.Request) (*Response, error) { defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient] inputURL=%s", c.inputURL)) logger.Debug("[HttpClient:Before] Method=%s %s", request.Method, c.String(), ) client := c.buildClient() resp, err := client.Do(request) if resp != nil { defer resp.Body.Close() } if err != nil { if uerr, ok := err.(*url.Error); ok { switch uerr.Err.(type) { case x509.CertificateInvalidError, x509.HostnameError: err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err) case *net.OpError: if uerr.Err.(*net.OpError).Temporary() { err = errors.NewLocalizedError(errTemporaryNetworkOperation, uerr.Err) } else { err = errors.NewLocalizedError(errPermanentNetworkOperation, uerr.Err) } case net.Error: nerr := uerr.Err.(net.Error) if nerr.Timeout() { err = errors.NewLocalizedError(errRequestTimeout, c.ClientTimeout) } else if nerr.Temporary() { err = errors.NewLocalizedError(errTemporaryNetworkOperation, nerr) } } } return nil, err } if resp.ContentLength > c.ClientMaxBodySize { return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength) } buf, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("client: error while reading body %v", err) } response := &Response{ Body: bytes.NewReader(buf), StatusCode: resp.StatusCode, EffectiveURL: resp.Request.URL.String(), LastModified: resp.Header.Get("Last-Modified"), ETag: resp.Header.Get("ETag"), Expires: resp.Header.Get("Expires"), ContentType: resp.Header.Get("Content-Type"), ContentLength: resp.ContentLength, } logger.Debug("[HttpClient:After] Method=%s %s; Response => %s", request.Method, c.String(), response, ) // Ignore caching headers for feeds that do not want any cache. if resp.Header.Get("Expires") == "0" { logger.Debug("[HttpClient] Ignore caching headers for %q", response.EffectiveURL) response.ETag = "" response.LastModified = "" } return response, err } func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) { c.requestURL = url_helper.RequestURI(c.inputURL) request, err := http.NewRequest(method, c.requestURL, body) if err != nil { return nil, err } request.Header = c.buildHeaders() if c.requestUsername != "" && c.requestPassword != "" { request.SetBasicAuth(c.requestUsername, c.requestPassword) } return request, nil } func (c *Client) buildClient() http.Client { client := http.Client{ Timeout: time.Duration(c.ClientTimeout) * time.Second, } transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ // Default is 30s. Timeout: 10 * time.Second, // Default is 30s. KeepAlive: 15 * time.Second, }).DialContext, // Default is 100. MaxIdleConns: 50, // Default is 90s. IdleConnTimeout: 10 * time.Second, } if c.doNotFollowRedirects { client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } } if c.useProxy && c.ClientProxyURL != "" { proxyURL, err := url.Parse(c.ClientProxyURL) if err != nil { logger.Error("[HttpClient] Proxy URL error: %v", err) } else { logger.Debug("[HttpClient] Use proxy: %s", proxyURL) transport.Proxy = http.ProxyURL(proxyURL) } } client.Transport = transport return client } func (c *Client) buildHeaders() http.Header { headers := make(http.Header) headers.Add("User-Agent", c.requestUserAgent) headers.Add("Accept", "*/*") if c.requestEtagHeader != "" { headers.Add("If-None-Match", c.requestEtagHeader) } if c.requestLastModifiedHeader != "" { headers.Add("If-Modified-Since", c.requestLastModifiedHeader) } if c.requestAuthorizationHeader != "" { headers.Add("Authorization", c.requestAuthorizationHeader) } headers.Add("Connection", "close") return headers }