Add FILTER_ENTRY_MAX_AGE_DAYS config option to limit fetching all feed items

.devcontainer/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   app:
-    image: mcr.microsoft.com/devcontainers/go
+    image: mcr.microsoft.com/devcontainers/go:1.22
     volumes:
       - ..:/workspace:cached
     command: sleep infinity
@@ -24,7 +24,7 @@ services:
     ports:
       - 5432:5432
   apprise:
-    image: caronc/apprise:latest
+    image: caronc/apprise:1.0
     restart: unless-stopped
     hostname: apprise
 volumes:

internal/config/options.go

@@ -55,6 +55,7 @@ const (
 	defaultProxyOption                        = "http-only"
 	defaultProxyMediaTypes                    = "image"
 	defaultProxyUrl                           = ""
+	defaultFilterEntryMaxAgeDays              = 0
 	defaultFetchOdyseeWatchTime               = false
 	defaultFetchYouTubeWatchTime              = false
 	defaultYouTubeEmbedUrlOverride            = "https://www.youtube-nocookie.com/embed/"
@@ -141,6 +142,7 @@ type Options struct {
 	proxyUrl                           string
 	fetchOdyseeWatchTime               bool
 	fetchYouTubeWatchTime              bool
+	filterEntryMaxAgeDays              int
 	youTubeEmbedUrlOverride            string
 	oauth2UserCreationAllowed          bool
 	oauth2ClientID                     string
@@ -213,6 +215,7 @@ func NewOptions() *Options {
 		proxyOption:                        defaultProxyOption,
 		proxyMediaTypes:                    []string{defaultProxyMediaTypes},
 		proxyUrl:                           defaultProxyUrl,
+		filterEntryMaxAgeDays:              defaultFilterEntryMaxAgeDays,
 		fetchOdyseeWatchTime:               defaultFetchOdyseeWatchTime,
 		fetchYouTubeWatchTime:              defaultFetchYouTubeWatchTime,
 		youTubeEmbedUrlOverride:            defaultYouTubeEmbedUrlOverride,
@@ -612,6 +615,11 @@ func (o *Options) WebAuthn() bool {
 	return o.webAuthn
 }
 
+// FilterEntryMaxAgeDays returns the number of days after which entries should be retained.
+func (o *Options) FilterEntryMaxAgeDays() int {
+	return o.filterEntryMaxAgeDays
+}
+
 // SortedOptions returns options as a list of key value pairs, sorted by keys.
 func (o *Options) SortedOptions(redactSecret bool) []*Option {
 	var keyValues = map[string]interface{}{
@@ -637,6 +645,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
 		"DISABLE_HSTS":                           !o.hsts,
 		"DISABLE_HTTP_SERVICE":                   !o.httpService,
 		"DISABLE_SCHEDULER_SERVICE":              !o.schedulerService,
+		"FILTER_ENTRY_MAX_AGE_DAYS":              o.filterEntryMaxAgeDays,
 		"FETCH_YOUTUBE_WATCH_TIME":               o.fetchYouTubeWatchTime,
 		"FETCH_ODYSEE_WATCH_TIME":                o.fetchOdyseeWatchTime,
 		"HTTPS":                                  o.HTTPS,

internal/config/parser.go

@@ -112,6 +112,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.databaseMinConns = parseInt(value, defaultDatabaseMinConns)
 		case "DATABASE_CONNECTION_LIFETIME":
 			p.opts.databaseConnectionLifetime = parseInt(value, defaultDatabaseConnectionLifetime)
+		case "FILTER_ENTRY_MAX_AGE_DAYS":
+			p.opts.filterEntryMaxAgeDays = parseInt(value, defaultFilterEntryMaxAgeDays)
 		case "RUN_MIGRATIONS":
 			p.opts.runMigrations = parseBool(value, defaultRunMigrations)
 		case "DISABLE_HSTS":

internal/reader/processor/processor.go

@@ -47,8 +47,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
 			slog.Int64("feed_id", feed.ID),
 			slog.String("feed_url", feed.FeedURL),
 		)
-
+		if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) || !isRecentEntry(entry) {
-		if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) {
 			continue
 		}
 
@@ -413,3 +412,10 @@ func parseISO8601(from string) (time.Duration, error) {
 
 	return d, nil
 }
+
+func isRecentEntry(entry *model.Entry) bool {
+	if config.Opts.FilterEntryMaxAgeDays() == 0 || entry.Date.After(time.Now().AddDate(0, 0, -config.Opts.FilterEntryMaxAgeDays())) {
+		return true
+	}
+	return false
+}

internal/reader/processor/processor_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"miniflux.app/v2/internal/config"
 	"miniflux.app/v2/internal/model"
 )
 
@@ -92,3 +93,27 @@ func TestParseISO8601(t *testing.T) {
 		}
 	}
 }
+
+func TestIsRecentEntry(t *testing.T) {
+	parser := config.NewParser()
+	var err error
+	config.Opts, err = parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing failure: %v`, err)
+	}
+	var scenarios = []struct {
+		entry    *model.Entry
+		expected bool
+	}{
+		{&model.Entry{Title: "Example1", Date: time.Date(2005, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
+		{&model.Entry{Title: "Example2", Date: time.Date(2010, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
+		{&model.Entry{Title: "Example3", Date: time.Date(2020, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
+		{&model.Entry{Title: "Example4", Date: time.Date(2024, 3, 15, 05, 05, 05, 05, time.UTC)}, true},
+	}
+	for _, tc := range scenarios {
+		result := isRecentEntry(tc.entry)
+		if tc.expected != result {
+			t.Errorf(`Unexpected result, got %v for entry %q`, result, tc.entry.Title)
+		}
+	}
+}

miniflux.1

@@ -307,6 +307,13 @@ Set the value to 1 to disable the internal scheduler service\&.
 .br
 Default is false (The internal scheduler service is enabled)\&.
 .TP
+.B FILTER_ENTRY_MAX_AGE_DAYS
+Number of days after which new entries should be retained.\&.
+.br
+Set 7 to fetch only entries 7 days old.\&.
+.br
+Default is 0\&.
+.TP
 .B CERT_FILE
 Path to SSL certificate\&.
 .br