southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Proxify images in API responses

Browse source
This commit is contained in:
Romain de Laage 2022-10-15 08:17:17 +02:00 committed by Frédéric Guillot
parent 206be5ba15
commit 3f14d08095
12 changed files with 146 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/api.go
View file

@ -14,13 +14,14 @@ import (
)
type handler struct {
store *storage.Storage
pool *worker.Pool
store *storage.Storage
pool *worker.Pool
router *mux.Router
}
// Serve declares API routes for the application.
func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) {
handler := &handler{store, pool}
handler := &handler{store, pool, router}
sr := router.PathPrefix("/v1").Subrouter()
middleware := newMiddleware(store)

25
api/entry.go
View file

@ -9,17 +9,21 @@ import (
"errors"
"net/http"
"strconv"
"strings"
"time"
"miniflux.app/config"
"miniflux.app/http/request"
"miniflux.app/http/response/json"
"miniflux.app/model"
"miniflux.app/proxy"
"miniflux.app/reader/processor"
"miniflux.app/storage"
"miniflux.app/url"
"miniflux.app/validator"
)
func getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b *storage.EntryQueryBuilder) {
func (h *handler) getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b *storage.EntryQueryBuilder) {
entry, err := b.GetEntry()
if err != nil {
json.ServerError(w, r, err)
@ -31,6 +35,15 @@ func getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b *storage.Entr
return
}
entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content)
proxyImage := config.Opts.ProxyImages()
for i := range entry.Enclosures {
if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) {
entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL)
}
}
json.OK(w, r, entry)
}
@ -42,7 +55,7 @@ func (h *handler) getFeedEntry(w http.ResponseWriter, r *http.Request) {
builder.WithFeedID(feedID)
builder.WithEntryID(entryID)
getEntryFromBuilder(w, r, builder)
h.getEntryFromBuilder(w, r, builder)
}
func (h *handler) getCategoryEntry(w http.ResponseWriter, r *http.Request) {
@ -53,7 +66,7 @@ func (h *handler) getCategoryEntry(w http.ResponseWriter, r *http.Request) {
builder.WithCategoryID(categoryID)
builder.WithEntryID(entryID)
getEntryFromBuilder(w, r, builder)
h.getEntryFromBuilder(w, r, builder)
}
func (h *handler) getEntry(w http.ResponseWriter, r *http.Request) {
@ -61,7 +74,7 @@ func (h *handler) getEntry(w http.ResponseWriter, r *http.Request) {
builder := h.store.NewEntryQueryBuilder(request.UserID(r))
builder.WithEntryID(entryID)
getEntryFromBuilder(w, r, builder)
h.getEntryFromBuilder(w, r, builder)
}
func (h *handler) getFeedEntries(w http.ResponseWriter, r *http.Request) {
@ -141,6 +154,10 @@ func (h *handler) findEntries(w http.ResponseWriter, r *http.Request, feedID int
return
}
for i := range entries {
entries[i].Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entries[i].Content)
}
json.OK(w, r, &entriesResponse{Total: count, Entries: entries})
}

12
config/options.go
View file

@ -5,6 +5,7 @@
package config // import "miniflux.app/config"
import (
"crypto/rand"
"fmt"
"sort"
"strings"
@ -139,10 +140,14 @@ type Options struct {
metricsAllowedNetworks []string
watchdog bool
invidiousInstance string
proxyPrivateKey []byte
}
// NewOptions returns Options with default values.
func NewOptions() *Options {
randomKey := make([]byte, 16)
rand.Read(randomKey)
return &Options{
HTTPS: defaultHTTPS,
logDateTime: defaultLogDateTime,
@ -199,6 +204,7 @@ func NewOptions() *Options {
metricsAllowedNetworks: []string{defaultMetricsAllowedNetworks},
watchdog: defaultWatchdog,
invidiousInstance: defaultInvidiousInstance,
proxyPrivateKey: randomKey,
}
}
@ -498,6 +504,11 @@ func (o *Options) InvidiousInstance() string {
return o.invidiousInstance
}
// ProxyPrivateKey returns the private key used by the media proxy
func (o *Options) ProxyPrivateKey() []byte {
return o.proxyPrivateKey
}
// SortedOptions returns options as a list of key value pairs, sorted by keys.
func (o *Options) SortedOptions(redactSecret bool) []*Option {
var keyValues = map[string]interface{}{
@ -552,6 +563,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"POLLING_SCHEDULER": o.pollingScheduler,
"PROXY_IMAGES": o.proxyImages,
"PROXY_IMAGE_URL": o.proxyImageUrl,
"PROXY_PRIVATE_KEY": redactSecretValue(string(o.proxyPrivateKey), redactSecret),
"ROOT_URL": o.rootURL,
"RUN_MIGRATIONS": o.runMigrations,
"SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval,

13
config/parser.go
View file

@ -7,6 +7,7 @@ package config // import "miniflux.app/config"
import (
"bufio"
"bytes"
"crypto/rand"
"errors"
"fmt"
"io"
@ -199,6 +200,10 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.watchdog = parseBool(value, defaultWatchdog)
case "INVIDIOUS_INSTANCE":
p.opts.invidiousInstance = parseString(value, defaultInvidiousInstance)
case "PROXY_PRIVATE_KEY":
randomKey := make([]byte, 16)
rand.Read(randomKey)
p.opts.proxyPrivateKey = parseBytes(value, randomKey)
}
}
@ -279,6 +284,14 @@ func parseStringList(value string, fallback []string) []string {
return strList
}
func parseBytes(value string, fallback []byte) []byte {
if value == "" {
return fallback
}
return []byte(value)
}
func readSecretFile(filename, fallback string) string {
data, err := os.ReadFile(filename)
if err != nil {

8
fever/handler.go
View file

@ -15,6 +15,7 @@ import (
"miniflux.app/integration"
"miniflux.app/logger"
"miniflux.app/model"
"miniflux.app/proxy"
"miniflux.app/storage"
"github.com/gorilla/mux"
@ -22,7 +23,7 @@ import (
// Serve handles Fever API calls.
func Serve(router *mux.Router, store *storage.Storage) {
handler := &handler{store}
handler := &handler{store, router}
sr := router.PathPrefix("/fever").Subrouter()
sr.Use(newMiddleware(store).serve)
@ -30,7 +31,8 @@ func Serve(router *mux.Router, store *storage.Storage) {
}
type handler struct {
store *storage.Storage
store *storage.Storage
router *mux.Router
}
func (h *handler) serve(w http.ResponseWriter, r *http.Request) {
@ -308,7 +310,7 @@ func (h *handler) handleItems(w http.ResponseWriter, r *http.Request) {
FeedID: entry.FeedID,
Title: entry.Title,
Author: entry.Author,
HTML: entry.Content,
HTML: proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content),
URL: entry.URL,
IsSaved: isSaved,
IsRead: isRead,

11
googlereader/handler.go
View file

@ -21,9 +21,11 @@ import (
"miniflux.app/integration"
"miniflux.app/logger"
"miniflux.app/model"
"miniflux.app/proxy"
mff "miniflux.app/reader/handler"
mfs "miniflux.app/reader/subscription"
"miniflux.app/storage"
"miniflux.app/url"
"miniflux.app/validator"
)
@ -839,6 +841,15 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) {
categories = append(categories, userStarred)
}
entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content)
proxyImage := config.Opts.ProxyImages()
for i := range entry.Enclosures {
if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) {
entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL)
}
}
contentItems[i] = contentItem{
ID: fmt.Sprintf(EntryIDLong, entry.ID),
Title: entry.Title,

5
miniflux.1
View file

@ -426,6 +426,11 @@ Enabled by default\&.
Set a custom invidious instance to use\&.
.br
Default is yewtu.be\&.
.TP
.B PROXY_PRIVATE_KEY
Set a custom custom private key used to sign proxified media url\&.
.br
Default is randomly generated at startup\&.
.SH AUTHORS
.P

24
proxy/image_proxy.go
View file

@ -15,8 +15,22 @@ import (
"github.com/gorilla/mux"
)
type urlProxyRewriter func(router *mux.Router, url string) string
// ImageProxyRewriter replaces image URLs with internal proxy URLs.
func ImageProxyRewriter(router *mux.Router, data string) string {
return genericImageProxyRewriter(router, ProxifyURL, data)
}
// AbsoluteImageProxyRewriter do the same as ImageProxyRewriter except it uses absolute URLs.
func AbsoluteImageProxyRewriter(router *mux.Router, host, data string) string {
proxifyFunction := func(router *mux.Router, url string) string {
return AbsoluteProxifyURL(router, host, url)
}
return genericImageProxyRewriter(router, proxifyFunction, data)
}
func genericImageProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string {
proxyImages := config.Opts.ProxyImages()
if proxyImages == "none" {
return data
@ -30,18 +44,18 @@ func ImageProxyRewriter(router *mux.Router, data string) string {
doc.Find("img").Each(func(i int, img *goquery.Selection) {
if srcAttrValue, ok := img.Attr("src"); ok {
if !isDataURL(srcAttrValue) && (proxyImages == "all" || !url.IsHTTPS(srcAttrValue)) {
img.SetAttr("src", ProxifyURL(router, srcAttrValue))
img.SetAttr("src", proxifyFunction(router, srcAttrValue))
}
}
if srcsetAttrValue, ok := img.Attr("srcset"); ok {
proxifySourceSet(img, router, proxyImages, srcsetAttrValue)
proxifySourceSet(img, router, proxifyFunction, proxyImages, srcsetAttrValue)
}
})
doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) {
if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok {
proxifySourceSet(sourceElement, router, proxyImages, srcsetAttrValue)
proxifySourceSet(sourceElement, router, proxifyFunction, proxyImages, srcsetAttrValue)
}
})
@ -53,12 +67,12 @@ func ImageProxyRewriter(router *mux.Router, data string) string {
return output
}
func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxyImages, srcsetAttrValue string) {
func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyImages, srcsetAttrValue string) {
imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue)
for _, imageCandidate := range imageCandidates {
if !isDataURL(imageCandidate.ImageURL) && (proxyImages == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) {
imageCandidate.ImageURL = ProxifyURL(router, imageCandidate.ImageURL)
imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL)
}
}

37
proxy/proxy.go
View file

@ -5,6 +5,8 @@
package proxy // import "miniflux.app/proxy"
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"net/url"
"path"
@ -16,13 +18,44 @@ import (
"miniflux.app/config"
)
// ProxifyURL generates an URL for a proxified resource.
// ProxifyURL generates a relative URL for a proxified resource.
func ProxifyURL(router *mux.Router, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyImageUrl()
if proxyImageUrl == "" {
return route.Path(router, "proxy", "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write([]byte(link))
digest := mac.Sum(nil)
return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
}
proxyUrl, err := url.Parse(proxyImageUrl)
if err != nil {
return ""
}
proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link)))
return proxyUrl.String()
}
return ""
}
// AbsoluteProxifyURL generates an absolute URL for a proxified resource.
func AbsoluteProxifyURL(router *mux.Router, host, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyImageUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write([]byte(link))
digest := mac.Sum(nil)
path := route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
if config.Opts.HTTPS {
return "https://" + host + path
} else {
return "http://" + host + path
}
}
proxyUrl, err := url.Parse(proxyImageUrl)

3
ui/middleware.go
View file

@ -143,7 +143,8 @@ func (m *middleware) isPublicRoute(r *http.Request) bool {
"robots",
"sharedEntry",
"healthcheck",
"offline":
"offline",
"proxy":
return true
default:
return false

18
ui/proxy.go
View file

@ -5,6 +5,8 @@
package ui // import "miniflux.app/ui"
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"errors"
"net/http"
@ -25,18 +27,34 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
return
}
encodedDigest := request.RouteStringParam(r, "encodedDigest")
encodedURL := request.RouteStringParam(r, "encodedURL")
if encodedURL == "" {
html.BadRequest(w, r, errors.New("No URL provided"))
return
}
decodedDigest, err := base64.URLEncoding.DecodeString(encodedDigest)
if err != nil {
html.BadRequest(w, r, errors.New("Unable to decode this Digest"))
return
}
decodedURL, err := base64.URLEncoding.DecodeString(encodedURL)
if err != nil {
html.BadRequest(w, r, errors.New("Unable to decode this URL"))
return
}
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write(decodedURL)
expectedMAC := mac.Sum(nil)
if !hmac.Equal(decodedDigest, expectedMAC) {
html.Forbidden(w, r)
return
}
imageURL := string(decodedURL)
logger.Debug(`[Proxy] Fetching %q`, imageURL)

2
ui/ui.go
View file

@ -94,7 +94,7 @@ func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) {
uiRouter.HandleFunc("/entry/status", handler.updateEntriesStatus).Name("updateEntriesStatus").Methods(http.MethodPost)
uiRouter.HandleFunc("/entry/save/{entryID}", handler.saveEntry).Name("saveEntry").Methods(http.MethodPost)
uiRouter.HandleFunc("/entry/download/{entryID}", handler.fetchContent).Name("fetchContent").Methods(http.MethodPost)
uiRouter.HandleFunc("/proxy/{encodedURL}", handler.imageProxy).Name("proxy").Methods(http.MethodGet)
uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.imageProxy).Name("proxy").Methods(http.MethodGet)
uiRouter.HandleFunc("/entry/bookmark/{entryID}", handler.toggleBookmark).Name("toggleBookmark").Methods(http.MethodPost)
// Share pages.