322 lines
8.6 KiB
Go
322 lines
8.6 KiB
Go
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package client // import "miniflux.app/v2/internal/http/client"
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
|
|
"miniflux.app/v2/internal/config"
|
|
"miniflux.app/v2/internal/errors"
|
|
)
|
|
|
|
const (
|
|
defaultHTTPClientTimeout = 20
|
|
defaultHTTPClientMaxBodySize = 15 * 1024 * 1024
|
|
)
|
|
|
|
var (
|
|
errInvalidCertificate = "Invalid SSL certificate (original error: %q)"
|
|
errNetworkOperation = "This website is unreachable (original error: %q)"
|
|
errRequestTimeout = "Website unreachable, the request timed out after %d seconds"
|
|
)
|
|
|
|
// Client builds and executes HTTP requests.
|
|
type Client struct {
|
|
inputURL string
|
|
|
|
requestEtagHeader string
|
|
requestLastModifiedHeader string
|
|
requestAuthorizationHeader string
|
|
requestUsername string
|
|
requestPassword string
|
|
requestUserAgent string
|
|
requestCookie string
|
|
customHeaders map[string]string
|
|
useProxy bool
|
|
doNotFollowRedirects bool
|
|
|
|
ClientTimeout int
|
|
ClientMaxBodySize int64
|
|
ClientProxyURL string
|
|
AllowSelfSignedCertificates bool
|
|
}
|
|
|
|
// New initializes a new HTTP client.
|
|
func New(url string) *Client {
|
|
return &Client{
|
|
inputURL: url,
|
|
ClientTimeout: defaultHTTPClientTimeout,
|
|
ClientMaxBodySize: defaultHTTPClientMaxBodySize,
|
|
}
|
|
}
|
|
|
|
// NewClientWithConfig initializes a new HTTP client with application config options.
|
|
func NewClientWithConfig(url string, opts *config.Options) *Client {
|
|
return &Client{
|
|
inputURL: url,
|
|
requestUserAgent: opts.HTTPClientUserAgent(),
|
|
ClientTimeout: opts.HTTPClientTimeout(),
|
|
ClientMaxBodySize: opts.HTTPClientMaxBodySize(),
|
|
ClientProxyURL: opts.HTTPClientProxy(),
|
|
}
|
|
}
|
|
|
|
// WithCredentials defines the username/password for HTTP Basic authentication.
|
|
func (c *Client) WithCredentials(username, password string) *Client {
|
|
if username != "" && password != "" {
|
|
c.requestUsername = username
|
|
c.requestPassword = password
|
|
}
|
|
return c
|
|
}
|
|
|
|
// WithCustomHeaders defines custom HTTP headers.
|
|
func (c *Client) WithCustomHeaders(customHeaders map[string]string) *Client {
|
|
c.customHeaders = customHeaders
|
|
return c
|
|
}
|
|
|
|
// WithCacheHeaders defines caching headers.
|
|
func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client {
|
|
c.requestEtagHeader = etagHeader
|
|
c.requestLastModifiedHeader = lastModifiedHeader
|
|
return c
|
|
}
|
|
|
|
// WithProxy enables proxy for the current HTTP request.
|
|
func (c *Client) WithProxy() *Client {
|
|
c.useProxy = true
|
|
return c
|
|
}
|
|
|
|
// WithoutRedirects disables HTTP redirects.
|
|
func (c *Client) WithoutRedirects() *Client {
|
|
c.doNotFollowRedirects = true
|
|
return c
|
|
}
|
|
|
|
// WithUserAgent defines the User-Agent header to use for HTTP requests.
|
|
func (c *Client) WithUserAgent(userAgent string) *Client {
|
|
if userAgent != "" {
|
|
c.requestUserAgent = userAgent
|
|
}
|
|
return c
|
|
}
|
|
|
|
// WithCookie defines the Cookies to use for HTTP requests.
|
|
func (c *Client) WithCookie(cookie string) *Client {
|
|
if cookie != "" {
|
|
c.requestCookie = cookie
|
|
}
|
|
return c
|
|
}
|
|
|
|
// Get performs a GET HTTP request.
|
|
func (c *Client) Get() (*Response, error) {
|
|
request, err := c.buildRequest(http.MethodGet, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return c.executeRequest(request)
|
|
}
|
|
|
|
func (c *Client) executeRequest(request *http.Request) (*Response, error) {
|
|
startTime := time.Now()
|
|
|
|
slog.Debug("Executing outgoing HTTP request",
|
|
slog.Group("request",
|
|
slog.String("method", request.Method),
|
|
slog.String("url", request.URL.String()),
|
|
slog.String("user_agent", request.UserAgent()),
|
|
slog.Bool("is_authenticated", c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != "")),
|
|
slog.Bool("has_cookie", c.requestCookie != ""),
|
|
slog.Bool("with_redirects", !c.doNotFollowRedirects),
|
|
slog.Bool("with_proxy", c.useProxy),
|
|
slog.String("proxy_url", c.ClientProxyURL),
|
|
slog.Bool("with_caching_headers", c.requestEtagHeader != "" || c.requestLastModifiedHeader != ""),
|
|
),
|
|
)
|
|
|
|
client := c.buildClient()
|
|
resp, err := client.Do(request)
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
}
|
|
|
|
if err != nil {
|
|
if uerr, ok := err.(*url.Error); ok {
|
|
switch uerr.Err.(type) {
|
|
case x509.CertificateInvalidError, x509.HostnameError:
|
|
err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err)
|
|
case *net.OpError:
|
|
err = errors.NewLocalizedError(errNetworkOperation, uerr.Err)
|
|
case net.Error:
|
|
nerr := uerr.Err.(net.Error)
|
|
if nerr.Timeout() {
|
|
err = errors.NewLocalizedError(errRequestTimeout, c.ClientTimeout)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil, err
|
|
}
|
|
|
|
if resp.ContentLength > c.ClientMaxBodySize {
|
|
return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength)
|
|
}
|
|
|
|
buf, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("client: error while reading body %v", err)
|
|
}
|
|
|
|
response := &Response{
|
|
Body: bytes.NewReader(buf),
|
|
StatusCode: resp.StatusCode,
|
|
EffectiveURL: resp.Request.URL.String(),
|
|
LastModified: resp.Header.Get("Last-Modified"),
|
|
ETag: resp.Header.Get("ETag"),
|
|
Expires: resp.Header.Get("Expires"),
|
|
ContentType: resp.Header.Get("Content-Type"),
|
|
ContentLength: resp.ContentLength,
|
|
}
|
|
|
|
slog.Debug("Completed outgoing HTTP request",
|
|
slog.Duration("duration", time.Since(startTime)),
|
|
slog.Group("request",
|
|
slog.String("method", request.Method),
|
|
slog.String("url", request.URL.String()),
|
|
slog.String("user_agent", request.UserAgent()),
|
|
slog.Bool("is_authenticated", c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != "")),
|
|
slog.Bool("has_cookie", c.requestCookie != ""),
|
|
slog.Bool("with_redirects", !c.doNotFollowRedirects),
|
|
slog.Bool("with_proxy", c.useProxy),
|
|
slog.String("proxy_url", c.ClientProxyURL),
|
|
slog.Bool("with_caching_headers", c.requestEtagHeader != "" || c.requestLastModifiedHeader != ""),
|
|
),
|
|
slog.Group("response",
|
|
slog.Int("status_code", response.StatusCode),
|
|
slog.String("effective_url", response.EffectiveURL),
|
|
slog.String("content_type", response.ContentType),
|
|
slog.Int64("content_length", response.ContentLength),
|
|
slog.String("last_modified", response.LastModified),
|
|
slog.String("etag", response.ETag),
|
|
slog.String("expires", response.Expires),
|
|
),
|
|
)
|
|
|
|
// Ignore caching headers for feeds that do not want any cache.
|
|
if resp.Header.Get("Expires") == "0" {
|
|
response.ETag = ""
|
|
response.LastModified = ""
|
|
}
|
|
|
|
return response, err
|
|
}
|
|
|
|
func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) {
|
|
request, err := http.NewRequest(method, c.inputURL, body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
request.Header = c.buildHeaders()
|
|
|
|
if c.requestUsername != "" && c.requestPassword != "" {
|
|
request.SetBasicAuth(c.requestUsername, c.requestPassword)
|
|
}
|
|
|
|
return request, nil
|
|
}
|
|
|
|
func (c *Client) buildClient() http.Client {
|
|
client := http.Client{
|
|
Timeout: time.Duration(c.ClientTimeout) * time.Second,
|
|
}
|
|
|
|
transport := &http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
DialContext: (&net.Dialer{
|
|
// Default is 30s.
|
|
Timeout: 10 * time.Second,
|
|
|
|
// Default is 30s.
|
|
KeepAlive: 15 * time.Second,
|
|
}).DialContext,
|
|
|
|
// Default is 100.
|
|
MaxIdleConns: 50,
|
|
|
|
// Default is 90s.
|
|
IdleConnTimeout: 10 * time.Second,
|
|
}
|
|
|
|
if c.AllowSelfSignedCertificates {
|
|
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
|
}
|
|
|
|
if c.doNotFollowRedirects {
|
|
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
|
return http.ErrUseLastResponse
|
|
}
|
|
}
|
|
|
|
if c.useProxy && c.ClientProxyURL != "" {
|
|
proxyURL, err := url.Parse(c.ClientProxyURL)
|
|
if err != nil {
|
|
slog.Error("Unable to parse proxy URL",
|
|
slog.String("proxy_url", c.ClientProxyURL),
|
|
slog.Any("error", err),
|
|
)
|
|
} else {
|
|
transport.Proxy = http.ProxyURL(proxyURL)
|
|
}
|
|
}
|
|
|
|
client.Transport = transport
|
|
|
|
return client
|
|
}
|
|
|
|
func (c *Client) buildHeaders() http.Header {
|
|
headers := make(http.Header)
|
|
headers.Add("Accept", "*/*")
|
|
|
|
if c.requestUserAgent != "" {
|
|
headers.Add("User-Agent", c.requestUserAgent)
|
|
}
|
|
|
|
if c.requestEtagHeader != "" {
|
|
headers.Add("If-None-Match", c.requestEtagHeader)
|
|
}
|
|
|
|
if c.requestLastModifiedHeader != "" {
|
|
headers.Add("If-Modified-Since", c.requestLastModifiedHeader)
|
|
}
|
|
|
|
if c.requestAuthorizationHeader != "" {
|
|
headers.Add("Authorization", c.requestAuthorizationHeader)
|
|
}
|
|
|
|
if c.requestCookie != "" {
|
|
headers.Add("Cookie", c.requestCookie)
|
|
}
|
|
|
|
for key, value := range c.customHeaders {
|
|
headers.Add(key, value)
|
|
}
|
|
|
|
headers.Add("Connection", "close")
|
|
return headers
|
|
}
|