southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Proxy support for several media types

Browse source 
closes #615
closes #635
This commit is contained in:
Romain de Laage 2023-02-25 09:36:19 +01:00 committed by Frédéric Guillot
parent 8f9ccc6540
commit 2c2700a31d
20 changed files with 534 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
api/entry.go
View file

@ -35,12 +35,17 @@ func (h *handler) getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b
return
}
entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content)
proxyImage := config.Opts.ProxyImages()
entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content)
proxyOption := config.Opts.ProxyOption()
for i := range entry.Enclosures {
if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) {
if proxyOption == "all" || proxyOption != "none" && !url.IsHTTPS(entry.Enclosures[i].URL) {
for _, mediaType := range config.Opts.ProxyMediaTypes() {
if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") {
entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL)
break
}
}
}
}
@ -158,7 +163,7 @@ func (h *handler) findEntries(w http.ResponseWriter, r *http.Request, feedID int
}
for i := range entries {
entries[i].Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entries[i].Content)
entries[i].Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entries[i].Content)
}
json.OK(w, r, &entriesResponse{Total: count, Entries: entries})

141
config/config_test.go
View file

@ -1163,9 +1163,9 @@ func TestPocketConsumerKeyFromUserPrefs(t *testing.T) {
}
}
func TestProxyImages(t *testing.T) {
func TestProxyOption(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -1174,14 +1174,14 @@ func TestProxyImages(t *testing.T) {
}
expected := "all"
result := opts.ProxyImages()
result := opts.ProxyOption()
if result != expected {
t.Fatalf(`Unexpected PROXY_IMAGES value, got %q instead of %q`, result, expected)
t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected)
}
}
func TestDefaultProxyImagesValue(t *testing.T) {
func TestDefaultProxyOptionValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
@ -1190,11 +1190,101 @@ func TestDefaultProxyImagesValue(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultProxyImages
result := opts.ProxyImages()
expected := defaultProxyOption
result := opts.ProxyOption()
if result != expected {
t.Fatalf(`Unexpected PROXY_IMAGES value, got %q instead of %q`, result, expected)
t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected)
}
}
func TestProxyMediaTypes(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_MEDIA_TYPES", "image,audio")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{"audio", "image"}
if len(expected) != len(opts.ProxyMediaTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.ProxyMediaTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
}
}
}
func TestDefaultProxyMediaTypes(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{"image"}
if len(expected) != len(opts.ProxyMediaTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.ProxyMediaTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
}
}
}
func TestProxyHTTPClientTimeout(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_HTTP_CLIENT_TIMEOUT", "24")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 24
result := opts.ProxyHTTPClientTimeout()
if result != expected {
t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
func TestDefaultProxyHTTPClientTimeoutValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultProxyHTTPClientTimeout
result := opts.ProxyHTTPClientTimeout()
if result != expected {
t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
@ -1297,6 +1387,41 @@ func TestDefaultHTTPClientMaxBodySizeValue(t *testing.T) {
}
}
func TestHTTPServerTimeout(t *testing.T) {
os.Clearenv()
os.Setenv("HTTP_SERVER_TIMEOUT", "342")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 342
result := opts.HTTPServerTimeout()
if result != expected {
t.Fatalf(`Unexpected HTTP_SERVER_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
func TestDefaultHTTPServerTimeoutValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultHTTPServerTimeout
result := opts.HTTPServerTimeout()
if result != expected {
t.Fatalf(`Unexpected HTTP_SERVER_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
func TestParseConfigFile(t *testing.T) {
content := []byte(`
# This is a comment

55
config/options.go
View file

@ -46,8 +46,10 @@ const (
defaultCleanupArchiveUnreadDays = 180
defaultCleanupArchiveBatchSize = 10000
defaultCleanupRemoveSessionsDays = 30
defaultProxyImages = "http-only"
defaultProxyImageUrl = ""
defaultProxyHTTPClientTimeout = 120
defaultProxyOption = "http-only"
defaultProxyMediaTypes = "image"
defaultProxyUrl = ""
defaultFetchYouTubeWatchTime = false
defaultCreateAdmin = false
defaultAdminUsername = ""
@ -62,6 +64,7 @@ const (
defaultHTTPClientTimeout = 20
defaultHTTPClientMaxBodySize = 15
defaultHTTPClientProxy = ""
defaultHTTPServerTimeout = 300
defaultAuthProxyHeader = ""
defaultAuthProxyUserCreation = false
defaultMaintenanceMode = false
@ -117,8 +120,10 @@ type Options struct {
createAdmin bool
adminUsername string
adminPassword string
proxyImages string
proxyImageUrl string
proxyHTTPClientTimeout int
proxyOption string
proxyMediaTypes []string
proxyUrl string
fetchYouTubeWatchTime bool
oauth2UserCreationAllowed bool
oauth2ClientID string
@ -131,6 +136,7 @@ type Options struct {
httpClientMaxBodySize int64
httpClientProxy string
httpClientUserAgent string
httpServerTimeout int
authProxyHeader string
authProxyUserCreation bool
maintenanceMode bool
@ -181,8 +187,10 @@ func NewOptions() *Options {
pollingParsingErrorLimit: defaultPollingParsingErrorLimit,
workerPoolSize: defaultWorkerPoolSize,
createAdmin: defaultCreateAdmin,
proxyImages: defaultProxyImages,
proxyImageUrl: defaultProxyImageUrl,
proxyHTTPClientTimeout: defaultProxyHTTPClientTimeout,
proxyOption: defaultProxyOption,
proxyMediaTypes: []string{defaultProxyMediaTypes},
proxyUrl: defaultProxyUrl,
fetchYouTubeWatchTime: defaultFetchYouTubeWatchTime,
oauth2UserCreationAllowed: defaultOAuth2UserCreation,
oauth2ClientID: defaultOAuth2ClientID,
@ -195,6 +203,7 @@ func NewOptions() *Options {
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
httpClientProxy: defaultHTTPClientProxy,
httpClientUserAgent: defaultHTTPClientUserAgent,
httpServerTimeout: defaultHTTPServerTimeout,
authProxyHeader: defaultAuthProxyHeader,
authProxyUserCreation: defaultAuthProxyUserCreation,
maintenanceMode: defaultMaintenanceMode,
@ -414,14 +423,24 @@ func (o *Options) FetchYouTubeWatchTime() bool {
return o.fetchYouTubeWatchTime
}
// ProxyImages returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy.
func (o *Options) ProxyImages() string {
return o.proxyImages
// ProxyOption returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy.
func (o *Options) ProxyOption() string {
return o.proxyOption
}
// ProxyImageUrl returns a string of a URL to use to proxy image requests
func (o *Options) ProxyImageUrl() string {
return o.proxyImageUrl
// ProxyMediaTypes returns a slice of media types to proxy.
func (o *Options) ProxyMediaTypes() []string {
return o.proxyMediaTypes
}
// ProxyUrl returns a string of a URL to use to proxy image requests
func (o *Options) ProxyUrl() string {
return o.proxyUrl
}
// ProxyHTTPClientTimeout returns the time limit in seconds before the proxy HTTP client cancel the request.
func (o *Options) ProxyHTTPClientTimeout() int {
return o.proxyHTTPClientTimeout
}
// HasHTTPService returns true if the HTTP service is enabled.
@ -457,6 +476,11 @@ func (o *Options) HTTPClientProxy() string {
return o.httpClientProxy
}
// HTTPServerTimeout returns the time limit in seconds before the HTTP server cancel the request.
func (o *Options) HTTPServerTimeout() int {
return o.httpServerTimeout
}
// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured.
func (o *Options) HasHTTPClientProxyConfigured() bool {
return o.httpClientProxy != ""
@ -541,6 +565,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"HTTP_CLIENT_PROXY": o.httpClientProxy,
"HTTP_CLIENT_TIMEOUT": o.httpClientTimeout,
"HTTP_CLIENT_USER_AGENT": o.httpClientUserAgent,
"HTTP_SERVER_TIMEOUT": o.httpServerTimeout,
"HTTP_SERVICE": o.httpService,
"KEY_FILE": o.certKeyFile,
"INVIDIOUS_INSTANCE": o.invidiousInstance,
@ -561,9 +586,11 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"POLLING_FREQUENCY": o.pollingFrequency,
"POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit,
"POLLING_SCHEDULER": o.pollingScheduler,
"PROXY_IMAGES": o.proxyImages,
"PROXY_IMAGE_URL": o.proxyImageUrl,
"PROXY_HTTP_CLIENT_TIMEOUT": o.proxyHTTPClientTimeout,
"PROXY_PRIVATE_KEY": redactSecretValue(string(o.proxyPrivateKey), redactSecret),
"PROXY_MEDIA_TYPES": o.proxyMediaTypes,
"PROXY_OPTION": o.proxyOption,
"PROXY_URL": o.proxyUrl,
"ROOT_URL": o.rootURL,
"RUN_MIGRATIONS": o.runMigrations,
"SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval,

17
config/parser.go
View file

@ -138,10 +138,21 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
case "POLLING_PARSING_ERROR_LIMIT":
p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
// kept for compatibility purpose
case "PROXY_IMAGES":
p.opts.proxyImages = parseString(value, defaultProxyImages)
p.opts.proxyOption = parseString(value, defaultProxyOption)
p.opts.proxyMediaTypes = append(p.opts.proxyMediaTypes, "image")
case "PROXY_HTTP_CLIENT_TIMEOUT":
p.opts.proxyHTTPClientTimeout = parseInt(value, defaultProxyHTTPClientTimeout)
case "PROXY_OPTION":
p.opts.proxyOption = parseString(value, defaultProxyOption)
case "PROXY_MEDIA_TYPES":
p.opts.proxyMediaTypes = parseStringList(value, []string{defaultProxyMediaTypes})
// kept for compatibility purpose
case "PROXY_IMAGE_URL":
p.opts.proxyImageUrl = parseString(value, defaultProxyImageUrl)
p.opts.proxyUrl = parseString(value, defaultProxyUrl)
case "PROXY_URL":
p.opts.proxyUrl = parseString(value, defaultProxyUrl)
case "CREATE_ADMIN":
p.opts.createAdmin = parseBool(value, defaultCreateAdmin)
case "ADMIN_USERNAME":
@ -180,6 +191,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy)
case "HTTP_CLIENT_USER_AGENT":
p.opts.httpClientUserAgent = parseString(value, defaultHTTPClientUserAgent)
case "HTTP_SERVER_TIMEOUT":
p.opts.httpServerTimeout = parseInt(value, defaultHTTPServerTimeout)
case "AUTH_PROXY_HEADER":
p.opts.authProxyHeader = parseString(value, defaultAuthProxyHeader)
case "AUTH_PROXY_USER_CREATION":

2
fever/handler.go
View file

@ -310,7 +310,7 @@ func (h *handler) handleItems(w http.ResponseWriter, r *http.Request) {
FeedID: entry.FeedID,
Title: entry.Title,
Author: entry.Author,
HTML: proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content),
HTML: proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content),
URL: entry.URL,
IsSaved: isSaved,
IsRead: isRead,

11
googlereader/handler.go
View file

@ -841,12 +841,17 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) {
categories = append(categories, userStarred)
}
entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content)
proxyImage := config.Opts.ProxyImages()
entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content)
proxyOption := config.Opts.ProxyOption()
for i := range entry.Enclosures {
if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) {
if proxyOption == "all" || proxyOption != "none" && !url.IsHTTPS(entry.Enclosures[i].URL) {
for _, mediaType := range config.Opts.ProxyMediaTypes() {
if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") {
entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL)
break
}
}
}
}

7
http/response/builder.go
View file

@ -12,6 +12,8 @@ import (
"net/http"
"strings"
"time"
"miniflux.app/logger"
)
const compressionThreshold = 1024
@ -88,7 +90,10 @@ func (b *Builder) Write() {
case io.Reader:
// Compression not implemented in this case
b.writeHeaders()
io.Copy(b.w, v)
_, err := io.Copy(b.w, v)
if err != nil {
logger.Error("%v", err)
}
}
}

13
http/response/html/html.go
View file

@ -72,3 +72,16 @@ func NotFound(w http.ResponseWriter, r *http.Request) {
func Redirect(w http.ResponseWriter, r *http.Request, uri string) {
http.Redirect(w, r, uri, http.StatusFound)
}
// RequestedRangeNotSatisfiable sends a range not satisfiable error to the client.
func RequestedRangeNotSatisfiable(w http.ResponseWriter, r *http.Request, contentRange string) {
logger.Error("[HTTP:Range Not Satisfiable] %s", r.URL)
builder := response.New(w, r)
builder.WithStatus(http.StatusRequestedRangeNotSatisfiable)
builder.WithHeader("Content-Type", "text/html; charset=utf-8")
builder.WithHeader("Cache-Control", "no-cache, max-age=0, must-revalidate, no-store")
builder.WithHeader("Content-Range", contentRange)
builder.WithBody("Range Not Satisfiable")
builder.Write()
}

29
http/response/html/html_test.go
View file

@ -210,3 +210,32 @@ func TestRedirectResponse(t *testing.T) {
t.Fatalf(`Unexpected redirect location, got %q instead of %q`, actualResult, expectedResult)
}
}
func TestRequestedRangeNotSatisfiable(t *testing.T) {
r, err := http.NewRequest("GET", "/", nil)
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
RequestedRangeNotSatisfiable(w, r, "bytes */12777")
})
handler.ServeHTTP(w, r)
resp := w.Result()
defer resp.Body.Close()
expectedStatusCode := http.StatusRequestedRangeNotSatisfiable
if resp.StatusCode != expectedStatusCode {
t.Fatalf(`Unexpected status code, got %d instead of %d`, resp.StatusCode, expectedStatusCode)
}
expectedContentRangeHeader := "bytes */12777"
actualContentRangeHeader := resp.Header.Get("Content-Range")
if actualContentRangeHeader != expectedContentRangeHeader {
t.Fatalf(`Unexpected content range header, got %q instead of %q`, actualContentRangeHeader, expectedContentRangeHeader)
}
}

23
miniflux.1
View file

@ -365,13 +365,23 @@ Path to a secret key exposed as a file, it should contain $POCKET_CONSUMER_KEY v
.br
Default is empty\&.
.TP
.B PROXY_IMAGES
Avoids mixed content warnings for external images: http-only, all, or none\&.
.B PROXY_OPTION
Avoids mixed content warnings for external media: http-only, all, or none\&.
.br
Default is http-only\&.
.TP
.B PROXY_IMAGE_URL
Sets a server to proxy images through\&.
.B PROXY_MEDIA_TYPES
A list of media types to proxify (comma-separated values): image, audio, video\&.
.br
Default is image only\&.
.TP
.B PROXY_HTTP_CLIENT_TIMEOUT
Time limit in seconds before the proxy HTTP client cancel the request\&.
.br
Default is 120 seconds\&.
.TP
.B PROXY_URL
Sets a server to proxy media through\&.
.br
Default is empty, miniflux does the proxying\&.
.TP
@ -397,6 +407,11 @@ When empty, Miniflux uses a default User-Agent that includes the Miniflux versio
.br
Default is empty.
.TP
.B HTTP_SERVER_TIMEOUT
Time limit in seconds before the HTTP client cancel the request\&.
.br
Default is 300 seconds\&.
.TP
.B AUTH_PROXY_HEADER
Proxy authentication HTTP header\&.
.br

84
proxy/image_proxy.go
View file

@ -1,84 +0,0 @@
// Copyright 2020 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package proxy // import "miniflux.app/proxy"
import (
"strings"
"miniflux.app/config"
"miniflux.app/reader/sanitizer"
"miniflux.app/url"
"github.com/PuerkitoBio/goquery"
"github.com/gorilla/mux"
)
type urlProxyRewriter func(router *mux.Router, url string) string
// ImageProxyRewriter replaces image URLs with internal proxy URLs.
func ImageProxyRewriter(router *mux.Router, data string) string {
return genericImageProxyRewriter(router, ProxifyURL, data)
}
// AbsoluteImageProxyRewriter do the same as ImageProxyRewriter except it uses absolute URLs.
func AbsoluteImageProxyRewriter(router *mux.Router, host, data string) string {
proxifyFunction := func(router *mux.Router, url string) string {
return AbsoluteProxifyURL(router, host, url)
}
return genericImageProxyRewriter(router, proxifyFunction, data)
}
func genericImageProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string {
proxyImages := config.Opts.ProxyImages()
if proxyImages == "none" {
return data
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
if err != nil {
return data
}
doc.Find("img").Each(func(i int, img *goquery.Selection) {
if srcAttrValue, ok := img.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyImages == "all" || !url.IsHTTPS(srcAttrValue)) {
img.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
if srcsetAttrValue, ok := img.Attr("srcset"); ok {
proxifySourceSet(img, router, proxifyFunction, proxyImages, srcsetAttrValue)
}
})
doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) {
if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok {
proxifySourceSet(sourceElement, router, proxifyFunction, proxyImages, srcsetAttrValue)
}
})
output, err := doc.Find("body").First().Html()
if err != nil {
return data
}
return output
}
func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyImages, srcsetAttrValue string) {
imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue)
for _, imageCandidate := range imageCandidates {
if !isDataURL(imageCandidate.ImageURL) && (proxyImages == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) {
imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL)
}
}
element.SetAttr("srcset", imageCandidates.String())
}
func isDataURL(s string) bool {
return strings.HasPrefix(s, "data:")
}

123
proxy/media_proxy.go Normal file
View file

@ -0,0 +1,123 @@
// Copyright 2020 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package proxy // import "miniflux.app/proxy"
import (
"strings"
"miniflux.app/config"
"miniflux.app/reader/sanitizer"
"miniflux.app/url"
"github.com/PuerkitoBio/goquery"
"github.com/gorilla/mux"
)
type urlProxyRewriter func(router *mux.Router, url string) string
// ProxyRewriter replaces media URLs with internal proxy URLs.
func ProxyRewriter(router *mux.Router, data string) string {
return genericProxyRewriter(router, ProxifyURL, data)
}
// AbsoluteProxyRewriter do the same as ProxyRewriter except it uses absolute URLs.
func AbsoluteProxyRewriter(router *mux.Router, host, data string) string {
proxifyFunction := func(router *mux.Router, url string) string {
return AbsoluteProxifyURL(router, host, url)
}
return genericProxyRewriter(router, proxifyFunction, data)
}
func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string {
proxyOption := config.Opts.ProxyOption()
if proxyOption == "none" {
return data
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
if err != nil {
return data
}
for _, mediaType := range config.Opts.ProxyMediaTypes() {
switch mediaType {
case "image":
doc.Find("img").Each(func(i int, img *goquery.Selection) {
if srcAttrValue, ok := img.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
img.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
if srcsetAttrValue, ok := img.Attr("srcset"); ok {
proxifySourceSet(img, router, proxifyFunction, proxyOption, srcsetAttrValue)
}
})
doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) {
if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok {
proxifySourceSet(sourceElement, router, proxifyFunction, proxyOption, srcsetAttrValue)
}
})
case "audio":
doc.Find("audio").Each(func(i int, audio *goquery.Selection) {
if srcAttrValue, ok := audio.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
audio.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
doc.Find("audio source").Each(func(i int, sourceElement *goquery.Selection) {
if srcAttrValue, ok := sourceElement.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
sourceElement.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
case "video":
doc.Find("video").Each(func(i int, video *goquery.Selection) {
if srcAttrValue, ok := video.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
video.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
doc.Find("video source").Each(func(i int, sourceElement *goquery.Selection) {
if srcAttrValue, ok := sourceElement.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) {
sourceElement.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
})
}
}
output, err := doc.Find("body").First().Html()
if err != nil {
return data
}
return output
}
func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyOption, srcsetAttrValue string) {
imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue)
for _, imageCandidate := range imageCandidates {
if !isDataURL(imageCandidate.ImageURL) && (proxyOption == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) {
imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL)
}
}
element.SetAttr("srcset", imageCandidates.String())
}
func isDataURL(s string) bool {
return strings.HasPrefix(s, "data:")
}

132
proxy/image_proxy_test.go → proxy/media_proxy_test.go
View file

@ -15,7 +15,9 @@ import (
func TestProxyFilterWithHttpDefault(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "http-only")
os.Setenv("PROXY_OPTION", "http-only")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -25,11 +27,11 @@ func TestProxyFilterWithHttpDefault(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
expected := `<p><img src="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
@ -38,7 +40,8 @@ func TestProxyFilterWithHttpDefault(t *testing.T) {
func TestProxyFilterWithHttpsDefault(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "http-only")
os.Setenv("PROXY_OPTION", "http-only")
os.Setenv("PROXY_MEDIA_TYPES", "image")
var err error
parser := config.NewParser()
@ -48,10 +51,10 @@ func TestProxyFilterWithHttpsDefault(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
@ -61,7 +64,7 @@ func TestProxyFilterWithHttpsDefault(t *testing.T) {
func TestProxyFilterWithHttpNever(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "none")
os.Setenv("PROXY_OPTION", "none")
var err error
parser := config.NewParser()
@ -71,10 +74,10 @@ func TestProxyFilterWithHttpNever(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
expected := input
if expected != output {
@ -84,7 +87,7 @@ func TestProxyFilterWithHttpNever(t *testing.T) {
func TestProxyFilterWithHttpsNever(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "none")
os.Setenv("PROXY_OPTION", "none")
var err error
parser := config.NewParser()
@ -94,10 +97,10 @@ func TestProxyFilterWithHttpsNever(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
expected := input
if expected != output {
@ -107,7 +110,9 @@ func TestProxyFilterWithHttpsNever(t *testing.T) {
func TestProxyFilterWithHttpAlways(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -117,11 +122,11 @@ func TestProxyFilterWithHttpAlways(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
expected := `<p><img src="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
@ -130,7 +135,9 @@ func TestProxyFilterWithHttpAlways(t *testing.T) {
func TestProxyFilterWithHttpsAlways(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -140,11 +147,11 @@ func TestProxyFilterWithHttpsAlways(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
expected := `<p><img src="/proxy/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/LdPNR1GBDigeeNp2ArUQRyZsVqT_PWLfHGjYFrrWWIY=/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
@ -153,8 +160,9 @@ func TestProxyFilterWithHttpsAlways(t *testing.T) {
func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_IMAGE_URL", "https://proxy-example/proxy")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_URL", "https://proxy-example/proxy")
var err error
parser := config.NewParser()
@ -164,10 +172,10 @@ func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
expected := `<p><img src="https://proxy-example/proxy/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
@ -177,7 +185,8 @@ func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) {
func TestProxyFilterWithHttpInvalid(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "invalid")
os.Setenv("PROXY_OPTION", "invalid")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -187,11 +196,11 @@ func TestProxyFilterWithHttpInvalid(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
expected := `<p><img src="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
output := ProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
@ -200,7 +209,8 @@ func TestProxyFilterWithHttpInvalid(t *testing.T) {
func TestProxyFilterWithHttpsInvalid(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "invalid")
os.Setenv("PROXY_OPTION", "invalid")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -210,10 +220,10 @@ func TestProxyFilterWithHttpsInvalid(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
@ -223,7 +233,9 @@ func TestProxyFilterWithHttpsInvalid(t *testing.T) {
func TestProxyFilterWithSrcset(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -233,11 +245,11 @@ func TestProxyFilterWithSrcset(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" srcset="http://website/folder/image2.png 656w, http://website/folder/image3.png 360w" alt="test"></p>`
expected := `<p><img src="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w" alt="test"/></p>`
output := ImageProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/QgAmrJWiAud_nNAsz3F8OTxaIofwAiO36EDzH_YfMzo=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w" alt="test"/></p>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -246,7 +258,9 @@ func TestProxyFilterWithSrcset(t *testing.T) {
func TestProxyFilterWithEmptySrcset(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -256,11 +270,11 @@ func TestProxyFilterWithEmptySrcset(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" srcset="" alt="test"></p>`
expected := `<p><img src="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="" alt="test"/></p>`
output := ImageProxyRewriter(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="" alt="test"/></p>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -269,7 +283,9 @@ func TestProxyFilterWithEmptySrcset(t *testing.T) {
func TestProxyFilterWithPictureSource(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -279,11 +295,11 @@ func TestProxyFilterWithPictureSource(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<picture><source srcset="http://website/folder/image2.png 656w, http://website/folder/image3.png 360w, https://website/some,image.png 2x"></picture>`
expected := `<picture><source srcset="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w, /proxy/aHR0cHM6Ly93ZWJzaXRlL3NvbWUsaW1hZ2UucG5n 2x"/></picture>`
output := ImageProxyRewriter(r, input)
expected := `<picture><source srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/QgAmrJWiAud_nNAsz3F8OTxaIofwAiO36EDzH_YfMzo=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w, /proxy/ZIw0hv8WhSTls5aSqhnFaCXlUrKIqTnBRaY0-NaLnds=/aHR0cHM6Ly93ZWJzaXRlL3NvbWUsaW1hZ2UucG5n 2x"/></picture>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -292,7 +308,9 @@ func TestProxyFilterWithPictureSource(t *testing.T) {
func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "https")
os.Setenv("PROXY_OPTION", "https")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
@ -302,20 +320,21 @@ func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<picture><source srcset="http://website/folder/image2.png 656w, https://website/some,image.png 2x"></picture>`
expected := `<picture><source srcset="/proxy/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, https://website/some,image.png 2x"/></picture>`
output := ImageProxyRewriter(r, input)
expected := `<picture><source srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, https://website/some,image.png 2x"/></picture>`
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestImageProxyWithImageDataURL(t *testing.T) {
func TestProxyWithImageDataURL(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
var err error
parser := config.NewParser()
@ -325,20 +344,21 @@ func TestImageProxyWithImageDataURL(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<img src="data:image/gif;base64,test">`
expected := `<img src="data:image/gif;base64,test"/>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
}
}
func TestImageProxyWithImageSourceDataURL(t *testing.T) {
func TestProxyWithImageSourceDataURL(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
var err error
parser := config.NewParser()
@ -348,11 +368,11 @@ func TestImageProxyWithImageSourceDataURL(t *testing.T) {
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<picture><source srcset="data:image/gif;base64,test"/></picture>`
expected := `<picture><source srcset="data:image/gif;base64,test"/></picture>`
output := ImageProxyRewriter(r, input)
output := ProxyRewriter(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)

4
proxy/proxy.go
View file

@ -21,7 +21,7 @@ import (
// ProxifyURL generates a relative URL for a proxified resource.
func ProxifyURL(router *mux.Router, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyImageUrl()
proxyImageUrl := config.Opts.ProxyUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
@ -44,7 +44,7 @@ func ProxifyURL(router *mux.Router, link string) string {
// AbsoluteProxifyURL generates an absolute URL for a proxified resource.
func AbsoluteProxifyURL(router *mux.Router, host, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyImageUrl()
proxyImageUrl := config.Opts.ProxyUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())

6
service/httpd/httpd.go
View file

@ -37,9 +37,9 @@ func Serve(store *storage.Storage, pool *worker.Pool) *http.Server {
certDomain := config.Opts.CertDomain()
listenAddr := config.Opts.ListenAddr()
server := &http.Server{
ReadTimeout: 300 * time.Second,
WriteTimeout: 300 * time.Second,
IdleTimeout: 300 * time.Second,
ReadTimeout: time.Duration(config.Opts.HTTPServerTimeout()) * time.Second,
WriteTimeout: time.Duration(config.Opts.HTTPServerTimeout()) * time.Second,
IdleTimeout: time.Duration(config.Opts.HTTPServerTimeout()) * time.Second,
Handler: setupHandler(store, pool),
}

14
template/functions.go
View file

@ -61,17 +61,25 @@ func (f *funcMap) Map() template.FuncMap {
return template.HTML(str)
},
"proxyFilter": func(data string) string {
return proxy.ImageProxyRewriter(f.router, data)
return proxy.ProxyRewriter(f.router, data)
},
"proxyURL": func(link string) string {
proxyImages := config.Opts.ProxyImages()
proxyOption := config.Opts.ProxyOption()
if proxyImages == "all" || (proxyImages != "none" && !url.IsHTTPS(link)) {
if proxyOption == "all" || (proxyOption != "none" && !url.IsHTTPS(link)) {
return proxy.ProxifyURL(f.router, link)
}
return link
},
"mustBeProxyfied": func(mediaType string) bool {
for _, t := range config.Opts.ProxyMediaTypes() {
if t == mediaType {
return true
}
}
return false
},
"domain": func(websiteURL string) string {
return url.Domain(websiteURL)
},

10
template/templates/views/entry.html
View file

@ -159,18 +159,26 @@
{{ if hasPrefix .MimeType "audio/" }}
<div class="enclosure-audio">
<audio controls preload="metadata">
{{ if (and $.user (mustBeProxyfied "audio")) }}
<source src="{{ proxyURL .URL }}" type="{{ .MimeType }}">
{{ else }}
<source src="{{ .URL | safeURL }}" type="{{ .MimeType }}">
{{ end }}
</audio>
</div>
{{ else if hasPrefix .MimeType "video/" }}
<div class="enclosure-video">
<video controls preload="metadata">
{{ if (and $.user (mustBeProxyfied "video")) }}
<source src="{{ proxyURL .URL }}" type="{{ .MimeType }}">
{{ else }}
<source src="{{ .URL | safeURL }}" type="{{ .MimeType }}">
{{ end }}
</video>
</div>
{{ else if hasPrefix .MimeType "image/" }}
<div class="enclosure-image">
{{ if $.user }}
{{ if (and $.user (mustBeProxyfied "image")) }}
<img src="{{ proxyURL .URL }}" title="{{ .URL }} ({{ .MimeType }})" loading="lazy" alt="{{ .URL }} ({{ .MimeType }})">
{{ else }}
<img src="{{ .URL | safeURL }}" title="{{ .URL }} ({{ .MimeType }})" loading="lazy" alt="{{ .URL }} ({{ .MimeType }})">

2
ui/entry_scraper.go
View file

@ -67,5 +67,5 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) {
readingTime := locale.NewPrinter(user.Language).Plural("entry.estimated_reading_time", entry.ReadingTime, entry.ReadingTime)
json.OK(w, r, map[string]string{"content": proxy.ImageProxyRewriter(h.router, entry.Content), "reading_time": readingTime})
json.OK(w, r, map[string]string{"content": proxy.ProxyRewriter(h.router, entry.Content), "reading_time": readingTime})
}

38
ui/proxy.go
View file

@ -20,8 +20,8 @@ import (
"miniflux.app/logger"
)
func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
// If we receive a "If-None-Match" header, we assume the image is already stored in browser cache.
func (h *handler) mediaProxy(w http.ResponseWriter, r *http.Request) {
// If we receive a "If-None-Match" header, we assume the media is already stored in browser cache.
if r.Header.Get("If-None-Match") != "" {
w.WriteHeader(http.StatusNotModified)
return
@ -55,10 +55,10 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
return
}
imageURL := string(decodedURL)
logger.Debug(`[Proxy] Fetching %q`, imageURL)
mediaURL := string(decodedURL)
logger.Debug(`[Proxy] Fetching %q`, mediaURL)
req, err := http.NewRequest("GET", imageURL, nil)
req, err := http.NewRequest("GET", mediaURL, nil)
if err != nil {
html.ServerError(w, r, err)
return
@ -67,8 +67,18 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
// Note: User-Agent HTTP header is omitted to avoid being blocked by bot protection mechanisms.
req.Header.Add("Connection", "close")
forwardedRequestHeader := []string{"Range", "Accept", "Accept-Encoding"}
for _, requestHeaderName := range forwardedRequestHeader {
if r.Header.Get(requestHeaderName) != "" {
req.Header.Add(requestHeaderName, r.Header.Get(requestHeaderName))
}
}
clt := &http.Client{
Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second,
Transport: &http.Transport{
IdleConnTimeout: time.Duration(config.Opts.ProxyHTTPClientTimeout()) * time.Second,
},
Timeout: time.Duration(config.Opts.ProxyHTTPClientTimeout()) * time.Second,
}
resp, err := clt.Do(req)
@ -78,8 +88,13 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.Error(`[Proxy] Status Code is %d for URL %q`, resp.StatusCode, imageURL)
if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
logger.Error(`[Proxy] Status Code is %d for URL %q`, resp.StatusCode, mediaURL)
html.RequestedRangeNotSatisfiable(w, r, resp.Header.Get("Content-Range"))
return
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
logger.Error(`[Proxy] Status Code is %d for URL %q`, resp.StatusCode, mediaURL)
html.NotFound(w, r)
return
}
@ -87,8 +102,15 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
etag := crypto.HashFromBytes(decodedURL)
response.New(w, r).WithCaching(etag, 72*time.Hour, func(b *response.Builder) {
b.WithStatus(resp.StatusCode)
b.WithHeader("Content-Security-Policy", `default-src 'self'`)
b.WithHeader("Content-Type", resp.Header.Get("Content-Type"))
forwardedResponseHeader := []string{"Content-Encoding", "Content-Type", "Content-Length", "Accept-Ranges", "Content-Range"}
for _, responseHeaderName := range forwardedResponseHeader {
if resp.Header.Get(responseHeaderName) != "" {
b.WithHeader(responseHeaderName, resp.Header.Get(responseHeaderName))
}
}
b.WithBody(resp.Body)
b.WithoutCompression()
b.Write()

2
ui/ui.go
View file

@ -96,7 +96,7 @@ func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) {
uiRouter.HandleFunc("/entry/status", handler.updateEntriesStatus).Name("updateEntriesStatus").Methods(http.MethodPost)
uiRouter.HandleFunc("/entry/save/{entryID}", handler.saveEntry).Name("saveEntry").Methods(http.MethodPost)
uiRouter.HandleFunc("/entry/download/{entryID}", handler.fetchContent).Name("fetchContent").Methods(http.MethodPost)
uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.imageProxy).Name("proxy").Methods(http.MethodGet)
uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.mediaProxy).Name("proxy").Methods(http.MethodGet)
uiRouter.HandleFunc("/entry/bookmark/{entryID}", handler.toggleBookmark).Name("toggleBookmark").Methods(http.MethodPost)
// Share pages.