Refactor Batch Builder and prevent accidental and excessive refreshes from the web ui
This commit is contained in:
parent
95ee1c423b
commit
4cc99881d8
32 changed files with 251 additions and 176 deletions
|
@ -9,6 +9,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/http/request"
|
"miniflux.app/v2/internal/http/request"
|
||||||
"miniflux.app/v2/internal/http/response/json"
|
"miniflux.app/v2/internal/http/response/json"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
@ -136,7 +137,14 @@ func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) {
|
||||||
userID := request.UserID(r)
|
userID := request.UserID(r)
|
||||||
categoryID := request.RouteInt64Param(r, "categoryID")
|
categoryID := request.RouteInt64Param(r, "categoryID")
|
||||||
|
|
||||||
jobs, err := h.store.NewCategoryBatch(userID, categoryID, h.store.CountFeeds(userID))
|
batchBuilder := h.store.NewBatchBuilder()
|
||||||
|
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
|
||||||
|
batchBuilder.WithoutDisabledFeeds()
|
||||||
|
batchBuilder.WithUserID(userID)
|
||||||
|
batchBuilder.WithCategoryID(categoryID)
|
||||||
|
batchBuilder.WithNextCheckExpired()
|
||||||
|
|
||||||
|
jobs, err := batchBuilder.FetchJobs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
json.ServerError(w, r, err)
|
json.ServerError(w, r, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/http/request"
|
"miniflux.app/v2/internal/http/request"
|
||||||
"miniflux.app/v2/internal/http/response/json"
|
"miniflux.app/v2/internal/http/response/json"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
|
@ -69,7 +70,14 @@ func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
||||||
userID := request.UserID(r)
|
userID := request.UserID(r)
|
||||||
jobs, err := h.store.NewUserBatch(userID, h.store.CountFeeds(userID))
|
|
||||||
|
batchBuilder := h.store.NewBatchBuilder()
|
||||||
|
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
|
||||||
|
batchBuilder.WithoutDisabledFeeds()
|
||||||
|
batchBuilder.WithNextCheckExpired()
|
||||||
|
batchBuilder.WithUserID(userID)
|
||||||
|
|
||||||
|
jobs, err := batchBuilder.FetchJobs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
json.ServerError(w, r, err)
|
json.ServerError(w, r, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -18,7 +18,15 @@ func refreshFeeds(store *storage.Storage) {
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
jobs, err := store.NewBatch(config.Opts.BatchSize())
|
|
||||||
|
// Generate a batch of feeds for any user that has feeds to refresh.
|
||||||
|
batchBuilder := store.NewBatchBuilder()
|
||||||
|
batchBuilder.WithBatchSize(config.Opts.BatchSize())
|
||||||
|
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
|
||||||
|
batchBuilder.WithoutDisabledFeeds()
|
||||||
|
batchBuilder.WithNextCheckExpired()
|
||||||
|
|
||||||
|
jobs, err := batchBuilder.FetchJobs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
||||||
return
|
return
|
||||||
|
|
|
@ -20,6 +20,7 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) {
|
||||||
pool,
|
pool,
|
||||||
config.Opts.PollingFrequency(),
|
config.Opts.PollingFrequency(),
|
||||||
config.Opts.BatchSize(),
|
config.Opts.BatchSize(),
|
||||||
|
config.Opts.PollingParsingErrorLimit(),
|
||||||
)
|
)
|
||||||
|
|
||||||
go cleanupScheduler(
|
go cleanupScheduler(
|
||||||
|
@ -28,10 +29,16 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize int) {
|
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize, errorLimit int) {
|
||||||
for range time.Tick(time.Duration(frequency) * time.Minute) {
|
for range time.Tick(time.Duration(frequency) * time.Minute) {
|
||||||
jobs, err := store.NewBatch(batchSize)
|
// Generate a batch of feeds for any user that has feeds to refresh.
|
||||||
if err != nil {
|
batchBuilder := store.NewBatchBuilder()
|
||||||
|
batchBuilder.WithBatchSize(batchSize)
|
||||||
|
batchBuilder.WithErrorLimit(errorLimit)
|
||||||
|
batchBuilder.WithoutDisabledFeeds()
|
||||||
|
batchBuilder.WithNextCheckExpired()
|
||||||
|
|
||||||
|
if jobs, err := batchBuilder.FetchJobs(); err != nil {
|
||||||
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
||||||
} else {
|
} else {
|
||||||
slog.Info("Created a batch of feeds",
|
slog.Info("Created a batch of feeds",
|
||||||
|
|
|
@ -3,7 +3,10 @@
|
||||||
|
|
||||||
package request // import "miniflux.app/v2/internal/http/request"
|
package request // import "miniflux.app/v2/internal/http/request"
|
||||||
|
|
||||||
import "net/http"
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
// ContextKey represents a context key.
|
// ContextKey represents a context key.
|
||||||
type ContextKey int
|
type ContextKey int
|
||||||
|
@ -24,6 +27,7 @@ const (
|
||||||
FlashMessageContextKey
|
FlashMessageContextKey
|
||||||
FlashErrorMessageContextKey
|
FlashErrorMessageContextKey
|
||||||
PocketRequestTokenContextKey
|
PocketRequestTokenContextKey
|
||||||
|
LastForceRefreshContextKey
|
||||||
ClientIPContextKey
|
ClientIPContextKey
|
||||||
GoogleReaderToken
|
GoogleReaderToken
|
||||||
)
|
)
|
||||||
|
@ -114,6 +118,16 @@ func PocketRequestToken(r *http.Request) string {
|
||||||
return getContextStringValue(r, PocketRequestTokenContextKey)
|
return getContextStringValue(r, PocketRequestTokenContextKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LastForceRefresh returns the last force refresh timestamp.
|
||||||
|
func LastForceRefresh(r *http.Request) int64 {
|
||||||
|
jsonStringValue := getContextStringValue(r, LastForceRefreshContextKey)
|
||||||
|
timestamp, err := strconv.ParseInt(jsonStringValue, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return timestamp
|
||||||
|
}
|
||||||
|
|
||||||
// ClientIP returns the client IP address stored in the context.
|
// ClientIP returns the client IP address stored in the context.
|
||||||
func ClientIP(r *http.Request) string {
|
func ClientIP(r *http.Request) string {
|
||||||
return getContextStringValue(r, ClientIPContextKey)
|
return getContextStringValue(r, ClientIPContextKey)
|
||||||
|
|
|
@ -453,6 +453,6 @@
|
||||||
"You are not authorized to access this resource (invalid username/password)": "Sie sind nicht berechtigt, auf diese Ressource zuzugreifen (Benutzername/Passwort ungültig)",
|
"You are not authorized to access this resource (invalid username/password)": "Sie sind nicht berechtigt, auf diese Ressource zuzugreifen (Benutzername/Passwort ungültig)",
|
||||||
"Unable to fetch this resource (Status Code = %d)": "Ressource konnte nicht abgerufen werden (code=%d)",
|
"Unable to fetch this resource (Status Code = %d)": "Ressource konnte nicht abgerufen werden (code=%d)",
|
||||||
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Ressource nicht gefunden (404), dieses Abonnement existiert nicht mehr, überprüfen Sie die Abonnement-URL",
|
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Ressource nicht gefunden (404), dieses Abonnement existiert nicht mehr, überprüfen Sie die Abonnement-URL",
|
||||||
"page.background_feed_refresh.title": "Hintergrundaktualisierung",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"πριν %d έτος",
|
"πριν %d έτος",
|
||||||
"πριν %d έτη"
|
"πριν %d έτη"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Background feed refresh",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"%d year ago",
|
"%d year ago",
|
||||||
"%d years ago"
|
"%d years ago"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Background feed refresh",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"hace %d año",
|
"hace %d año",
|
||||||
"hace %d años"
|
"hace %d años"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Background feed refresh",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"%d vuosi sitten",
|
"%d vuosi sitten",
|
||||||
"%d vuotta sitten"
|
"%d vuotta sitten"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Background feed refresh",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -453,6 +453,6 @@
|
||||||
"You are not authorized to access this resource (invalid username/password)": "Vous n'êtes pas autorisé à accéder à cette ressource (nom d'utilisateur / mot de passe incorrect)",
|
"You are not authorized to access this resource (invalid username/password)": "Vous n'êtes pas autorisé à accéder à cette ressource (nom d'utilisateur / mot de passe incorrect)",
|
||||||
"Unable to fetch this resource (Status Code = %d)": "Impossible de récupérer cette ressource (code=%d)",
|
"Unable to fetch this resource (Status Code = %d)": "Impossible de récupérer cette ressource (code=%d)",
|
||||||
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Page introuvable (404), cet abonnement n'existe plus, vérifiez l'adresse du flux",
|
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Page introuvable (404), cet abonnement n'existe plus, vérifiez l'adresse du flux",
|
||||||
"page.background_feed_refresh.title": "Actualisation des abonnements en arrière-plan",
|
"alert.too_many_feeds_refresh": "Vous avez déclenché trop d'actualisations de flux. Veuillez attendre 30 minutes avant de réessayer.",
|
||||||
"alert.background_feed_refresh": "Les abonnements sont en cours d'actualisation en arrière-plan. Vous pouvez continuer à naviguer dans l'application."
|
"alert.background_feed_refresh": "Les abonnements sont en cours d'actualisation en arrière-plan. Vous pouvez continuer à naviguer dans l'application."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"%d साल पहले",
|
"%d साल पहले",
|
||||||
"%d वर्षों पहले"
|
"%d वर्षों पहले"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "फ़ीड रीफ़्रेश किया जा रहा है",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -444,6 +444,6 @@
|
||||||
"You are not authorized to access this resource (invalid username/password)": "Anda tidak memiliki izin yang cukup untuk mengakses umpan ini (nama pengguna/kata sandi tidak valid)",
|
"You are not authorized to access this resource (invalid username/password)": "Anda tidak memiliki izin yang cukup untuk mengakses umpan ini (nama pengguna/kata sandi tidak valid)",
|
||||||
"Unable to fetch this resource (Status Code = %d)": "Tidak bisa mengambil umpan ini (Kode Status = %d)",
|
"Unable to fetch this resource (Status Code = %d)": "Tidak bisa mengambil umpan ini (Kode Status = %d)",
|
||||||
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Umpan tidak ditemukan (404), umpan ini tidak ada lagi, periksa URL umpan",
|
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Umpan tidak ditemukan (404), umpan ini tidak ada lagi, periksa URL umpan",
|
||||||
"page.background_feed_refresh.title": "Memuat ulang umpan di latar belakang",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"%d anno fa",
|
"%d anno fa",
|
||||||
"%d anni fa"
|
"%d anni fa"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Aggiornamento in background",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"%d 年前",
|
"%d 年前",
|
||||||
"%d 年前"
|
"%d 年前"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "バックグラウンドでフィードを更新中",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -451,6 +451,6 @@
|
||||||
"Invalid SSL certificate (original error: %q)": "Ongeldig SSL-certificaat (originele error: %q)",
|
"Invalid SSL certificate (original error: %q)": "Ongeldig SSL-certificaat (originele error: %q)",
|
||||||
"This website is unreachable (original error: %q)": "Deze website is onbereikbaar (originele error: %q)",
|
"This website is unreachable (original error: %q)": "Deze website is onbereikbaar (originele error: %q)",
|
||||||
"Website unreachable, the request timed out after %d seconds": "Website onbereikbaar, de request gaf een timeout na %d seconden",
|
"Website unreachable, the request timed out after %d seconds": "Website onbereikbaar, de request gaf een timeout na %d seconden",
|
||||||
"page.background_feed_refresh.title": "Achtergrond vernieuwen",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -459,6 +459,6 @@
|
||||||
"Invalid SSL certificate (original error: %q)": "Certyfikat SSL jest nieprawidłowy (błąd: %q)",
|
"Invalid SSL certificate (original error: %q)": "Certyfikat SSL jest nieprawidłowy (błąd: %q)",
|
||||||
"This website is unreachable (original error: %q)": "Ta strona jest niedostępna (błąd: %q)",
|
"This website is unreachable (original error: %q)": "Ta strona jest niedostępna (błąd: %q)",
|
||||||
"Website unreachable, the request timed out after %d seconds": "Strona internetowa nieosiągalna, żądanie wygasło po %d sekundach",
|
"Website unreachable, the request timed out after %d seconds": "Strona internetowa nieosiągalna, żądanie wygasło po %d sekundach",
|
||||||
"page.background_feed_refresh.title": "Odświeżanie kanałów w tle",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"há %d ano",
|
"há %d ano",
|
||||||
"há %d anos"
|
"há %d anos"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Atualização de fonte em segundo plano",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -442,6 +442,6 @@
|
||||||
"%d года назад",
|
"%d года назад",
|
||||||
"%d лет назад"
|
"%d лет назад"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Обновление подписок",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -434,6 +434,6 @@
|
||||||
"%d yıl önce",
|
"%d yıl önce",
|
||||||
"%d yıl önce"
|
"%d yıl önce"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Arka plan beslemesi yenileme",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -443,6 +443,6 @@
|
||||||
"%d роки тому",
|
"%d роки тому",
|
||||||
"%d років тому"
|
"%d років тому"
|
||||||
],
|
],
|
||||||
"page.background_feed_refresh.title": "Оновлення стрічок в фоновому режимі",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -444,6 +444,6 @@
|
||||||
"Invalid SSL certificate (original error: %q)": "无效的 SSL 证书 (原始错误: %q)",
|
"Invalid SSL certificate (original error: %q)": "无效的 SSL 证书 (原始错误: %q)",
|
||||||
"This website is unreachable (original error: %q)": "该网站永久不可达 (原始错误: %q)",
|
"This website is unreachable (original error: %q)": "该网站永久不可达 (原始错误: %q)",
|
||||||
"Website unreachable, the request timed out after %d seconds": "网站不可达, 请求已在 %d 秒后超时",
|
"Website unreachable, the request timed out after %d seconds": "网站不可达, 请求已在 %d 秒后超时",
|
||||||
"page.background_feed_refresh.title": "后台更新",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -452,6 +452,6 @@
|
||||||
"Invalid SSL certificate (original error: %q)": "無效的 SSL 憑證 (錯誤: %q)",
|
"Invalid SSL certificate (original error: %q)": "無效的 SSL 憑證 (錯誤: %q)",
|
||||||
"This website is unreachable (original error: %q)": "該網站永久無法訪問(原始錯誤: %q)",
|
"This website is unreachable (original error: %q)": "該網站永久無法訪問(原始錯誤: %q)",
|
||||||
"Website unreachable, the request timed out after %d seconds": "網站無法訪問, 請求已在 %d 秒後超時",
|
"Website unreachable, the request timed out after %d seconds": "網站無法訪問, 請求已在 %d 秒後超時",
|
||||||
"page.background_feed_refresh.title": "背景更新",
|
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,11 +20,21 @@ type SessionData struct {
|
||||||
Language string `json:"language"`
|
Language string `json:"language"`
|
||||||
Theme string `json:"theme"`
|
Theme string `json:"theme"`
|
||||||
PocketRequestToken string `json:"pocket_request_token"`
|
PocketRequestToken string `json:"pocket_request_token"`
|
||||||
|
LastForceRefresh string `json:"last_force_refresh"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s SessionData) String() string {
|
func (s SessionData) String() string {
|
||||||
return fmt.Sprintf(`CSRF=%q, OAuth2State=%q, OAuth2CodeVerifier=%q, FlashMsg=%q, FlashErrMsg=%q, Lang=%q, Theme=%q, PocketTkn=%q`,
|
return fmt.Sprintf(`CSRF=%q, OAuth2State=%q, OAuth2CodeVerifier=%q, FlashMsg=%q, FlashErrMsg=%q, Lang=%q, Theme=%q, PocketTkn=%q, LastForceRefresh=%s`,
|
||||||
s.CSRF, s.OAuth2State, s.OAuth2CodeVerifier, s.FlashMessage, s.FlashErrorMessage, s.Language, s.Theme, s.PocketRequestToken)
|
s.CSRF,
|
||||||
|
s.OAuth2State,
|
||||||
|
s.OAuth2CodeVerifier,
|
||||||
|
s.FlashMessage,
|
||||||
|
s.FlashErrorMessage,
|
||||||
|
s.Language,
|
||||||
|
s.Theme,
|
||||||
|
s.PocketRequestToken,
|
||||||
|
s.LastForceRefresh,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Value converts the session data to JSON.
|
// Value converts the session data to JSON.
|
||||||
|
|
91
internal/storage/batch.go
Normal file
91
internal/storage/batch.go
Normal file
|
@ -0,0 +1,91 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package storage // import "miniflux.app/v2/internal/storage"
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"miniflux.app/v2/internal/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type BatchBuilder struct {
|
||||||
|
db *sql.DB
|
||||||
|
args []any
|
||||||
|
conditions []string
|
||||||
|
limit int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) NewBatchBuilder() *BatchBuilder {
|
||||||
|
return &BatchBuilder{
|
||||||
|
db: s.db,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) WithBatchSize(batchSize int) *BatchBuilder {
|
||||||
|
b.limit = batchSize
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) WithUserID(userID int64) *BatchBuilder {
|
||||||
|
b.conditions = append(b.conditions, fmt.Sprintf("user_id = $%d", len(b.args)+1))
|
||||||
|
b.args = append(b.args, userID)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) WithCategoryID(categoryID int64) *BatchBuilder {
|
||||||
|
b.conditions = append(b.conditions, fmt.Sprintf("category_id = $%d", len(b.args)+1))
|
||||||
|
b.args = append(b.args, categoryID)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) WithErrorLimit(limit int) *BatchBuilder {
|
||||||
|
if limit > 0 {
|
||||||
|
b.conditions = append(b.conditions, fmt.Sprintf("parsing_error_count < $%d", len(b.args)+1))
|
||||||
|
b.args = append(b.args, limit)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) WithNextCheckExpired() *BatchBuilder {
|
||||||
|
b.conditions = append(b.conditions, "next_check_at < now()")
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) WithoutDisabledFeeds() *BatchBuilder {
|
||||||
|
b.conditions = append(b.conditions, "disabled is false")
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BatchBuilder) FetchJobs() (jobs model.JobList, err error) {
|
||||||
|
var parts []string
|
||||||
|
parts = append(parts, `SELECT id, user_id FROM feeds`)
|
||||||
|
|
||||||
|
if len(b.conditions) > 0 {
|
||||||
|
parts = append(parts, fmt.Sprintf("WHERE %s", strings.Join(b.conditions, " AND ")))
|
||||||
|
}
|
||||||
|
|
||||||
|
if b.limit > 0 {
|
||||||
|
parts = append(parts, fmt.Sprintf("ORDER BY next_check_at ASC LIMIT %d", b.limit))
|
||||||
|
}
|
||||||
|
|
||||||
|
query := strings.Join(parts, " ")
|
||||||
|
rows, err := b.db.Query(query, b.args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var job model.Job
|
||||||
|
if err := rows.Scan(&job.FeedID, &job.UserID); err != nil {
|
||||||
|
return nil, fmt.Errorf(`store: unable to fetch job: %v`, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs = append(jobs, job)
|
||||||
|
}
|
||||||
|
|
||||||
|
return jobs, nil
|
||||||
|
}
|
|
@ -87,17 +87,6 @@ func (s *Storage) CountAllFeeds() map[string]int64 {
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
// CountFeeds returns the number of feeds that belongs to the given user.
|
|
||||||
func (s *Storage) CountFeeds(userID int64) int {
|
|
||||||
var result int
|
|
||||||
err := s.db.QueryRow(`SELECT count(*) FROM feeds WHERE user_id=$1`, userID).Scan(&result)
|
|
||||||
if err != nil {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user.
|
// CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user.
|
||||||
func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
|
func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
|
||||||
pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
|
pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
|
||||||
|
|
|
@ -1,81 +0,0 @@
|
||||||
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
|
||||||
// SPDX-License-Identifier: Apache-2.0
|
|
||||||
|
|
||||||
package storage // import "miniflux.app/v2/internal/storage"
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
|
||||||
"miniflux.app/v2/internal/model"
|
|
||||||
)
|
|
||||||
|
|
||||||
// NewBatch returns a series of jobs.
|
|
||||||
func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
|
|
||||||
pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
|
|
||||||
query := `
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
user_id
|
|
||||||
FROM
|
|
||||||
feeds
|
|
||||||
WHERE
|
|
||||||
disabled is false AND next_check_at < now() AND
|
|
||||||
CASE WHEN $1 > 0 THEN parsing_error_count < $1 ELSE parsing_error_count >= 0 END
|
|
||||||
ORDER BY next_check_at ASC LIMIT $2
|
|
||||||
`
|
|
||||||
return s.fetchBatchRows(query, pollingParsingErrorLimit, batchSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewUserBatch returns a series of jobs but only for a given user.
|
|
||||||
func (s *Storage) NewUserBatch(userID int64, batchSize int) (jobs model.JobList, err error) {
|
|
||||||
// We do not take the error counter into consideration when the given
|
|
||||||
// user refresh manually all his feeds to force a refresh.
|
|
||||||
query := `
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
user_id
|
|
||||||
FROM
|
|
||||||
feeds
|
|
||||||
WHERE
|
|
||||||
user_id=$1 AND disabled is false AND next_check_at < now()
|
|
||||||
ORDER BY next_check_at ASC LIMIT %d
|
|
||||||
`
|
|
||||||
return s.fetchBatchRows(fmt.Sprintf(query, batchSize), userID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewCategoryBatch returns a series of jobs but only for a given category.
|
|
||||||
func (s *Storage) NewCategoryBatch(userID int64, categoryID int64, batchSize int) (jobs model.JobList, err error) {
|
|
||||||
// We do not take the error counter into consideration when the given
|
|
||||||
// user refresh manually all his feeds to force a refresh.
|
|
||||||
query := `
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
user_id
|
|
||||||
FROM
|
|
||||||
feeds
|
|
||||||
WHERE
|
|
||||||
user_id=$1 AND category_id=$2 AND disabled is false AND next_check_at < now()
|
|
||||||
ORDER BY next_check_at ASC LIMIT %d
|
|
||||||
`
|
|
||||||
return s.fetchBatchRows(fmt.Sprintf(query, batchSize), userID, categoryID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Storage) fetchBatchRows(query string, args ...interface{}) (jobs model.JobList, err error) {
|
|
||||||
rows, err := s.db.Query(query, args...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err)
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var job model.Job
|
|
||||||
if err := rows.Scan(&job.FeedID, &job.UserID); err != nil {
|
|
||||||
return nil, fmt.Errorf(`store: unable to fetch job: %v`, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
jobs = append(jobs, job)
|
|
||||||
}
|
|
||||||
|
|
||||||
return jobs, nil
|
|
||||||
}
|
|
|
@ -1,11 +0,0 @@
|
||||||
{{ define "title"}}{{ t "page.background_feed_refresh.title" }}{{ end }}
|
|
||||||
|
|
||||||
{{ define "content"}}
|
|
||||||
<section class="page-header">
|
|
||||||
<h1>{{ t "page.background_feed_refresh.title" }}</h1>
|
|
||||||
{{ template "feed_menu" }}
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<p class="alert alert-success">{{ t "alert.background_feed_refresh" }}</p>
|
|
||||||
|
|
||||||
{{ end }}
|
|
|
@ -6,10 +6,13 @@ package ui // import "miniflux.app/v2/internal/ui"
|
||||||
import (
|
import (
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/http/request"
|
"miniflux.app/v2/internal/http/request"
|
||||||
"miniflux.app/v2/internal/http/response/html"
|
"miniflux.app/v2/internal/http/response/html"
|
||||||
"miniflux.app/v2/internal/http/route"
|
"miniflux.app/v2/internal/http/route"
|
||||||
|
"miniflux.app/v2/internal/locale"
|
||||||
|
"miniflux.app/v2/internal/ui/session"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) refreshCategoryEntriesPage(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) refreshCategoryEntriesPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
@ -25,8 +28,21 @@ func (h *handler) refreshCategoryFeedsPage(w http.ResponseWriter, r *http.Reques
|
||||||
func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) int64 {
|
func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) int64 {
|
||||||
userID := request.UserID(r)
|
userID := request.UserID(r)
|
||||||
categoryID := request.RouteInt64Param(r, "categoryID")
|
categoryID := request.RouteInt64Param(r, "categoryID")
|
||||||
|
printer := locale.NewPrinter(request.UserLanguage(r))
|
||||||
|
sess := session.New(h.store, request.SessionID(r))
|
||||||
|
|
||||||
jobs, err := h.store.NewCategoryBatch(userID, categoryID, h.store.CountFeeds(userID))
|
// Avoid accidental and excessive refreshes.
|
||||||
|
if time.Now().UTC().Unix()-request.LastForceRefresh(r) < 1800 {
|
||||||
|
sess.NewFlashErrorMessage(printer.Printf("alert.too_many_feeds_refresh"))
|
||||||
|
} else {
|
||||||
|
// We allow the end-user to force refresh all its feeds in this category
|
||||||
|
// without taking into consideration the number of errors.
|
||||||
|
batchBuilder := h.store.NewBatchBuilder()
|
||||||
|
batchBuilder.WithoutDisabledFeeds()
|
||||||
|
batchBuilder.WithUserID(userID)
|
||||||
|
batchBuilder.WithCategoryID(categoryID)
|
||||||
|
|
||||||
|
jobs, err := batchBuilder.FetchJobs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
html.ServerError(w, r, err)
|
html.ServerError(w, r, err)
|
||||||
return 0
|
return 0
|
||||||
|
@ -41,5 +57,9 @@ func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) int64
|
||||||
|
|
||||||
go h.pool.Push(jobs)
|
go h.pool.Push(jobs)
|
||||||
|
|
||||||
|
sess.SetLastForceRefresh()
|
||||||
|
sess.NewFlashMessage(printer.Printf("alert.background_feed_refresh"))
|
||||||
|
}
|
||||||
|
|
||||||
return categoryID
|
return categoryID
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,13 +6,14 @@ package ui // import "miniflux.app/v2/internal/ui"
|
||||||
import (
|
import (
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/http/request"
|
"miniflux.app/v2/internal/http/request"
|
||||||
"miniflux.app/v2/internal/http/response/html"
|
"miniflux.app/v2/internal/http/response/html"
|
||||||
"miniflux.app/v2/internal/http/route"
|
"miniflux.app/v2/internal/http/route"
|
||||||
|
"miniflux.app/v2/internal/locale"
|
||||||
feedHandler "miniflux.app/v2/internal/reader/handler"
|
feedHandler "miniflux.app/v2/internal/reader/handler"
|
||||||
"miniflux.app/v2/internal/ui/session"
|
"miniflux.app/v2/internal/ui/session"
|
||||||
"miniflux.app/v2/internal/ui/view"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
||||||
|
@ -32,14 +33,20 @@ func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
||||||
userID := request.UserID(r)
|
userID := request.UserID(r)
|
||||||
|
printer := locale.NewPrinter(request.UserLanguage(r))
|
||||||
|
sess := session.New(h.store, request.SessionID(r))
|
||||||
|
|
||||||
user, err := h.store.UserByID(userID)
|
// Avoid accidental and excessive refreshes.
|
||||||
if err != nil {
|
if time.Now().UTC().Unix()-request.LastForceRefresh(r) < 1800 {
|
||||||
html.ServerError(w, r, err)
|
sess.NewFlashErrorMessage(printer.Printf("alert.too_many_feeds_refresh"))
|
||||||
return
|
} else {
|
||||||
}
|
// We allow the end-user to force refresh all its feeds
|
||||||
|
// without taking into consideration the number of errors.
|
||||||
|
batchBuilder := h.store.NewBatchBuilder()
|
||||||
|
batchBuilder.WithoutDisabledFeeds()
|
||||||
|
batchBuilder.WithUserID(userID)
|
||||||
|
|
||||||
jobs, err := h.store.NewUserBatch(userID, h.store.CountFeeds(userID))
|
jobs, err := batchBuilder.FetchJobs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
html.ServerError(w, r, err)
|
html.ServerError(w, r, err)
|
||||||
return
|
return
|
||||||
|
@ -53,12 +60,9 @@ func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
go h.pool.Push(jobs)
|
go h.pool.Push(jobs)
|
||||||
|
|
||||||
sess := session.New(h.store, request.SessionID(r))
|
sess.SetLastForceRefresh()
|
||||||
view := view.New(h.tpl, r, sess)
|
sess.NewFlashMessage(printer.Printf("alert.background_feed_refresh"))
|
||||||
view.Set("menu", "feeds")
|
}
|
||||||
view.Set("user", user)
|
|
||||||
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
html.Redirect(w, r, route.Path(h.router, "feeds"))
|
||||||
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
|
||||||
|
|
||||||
html.OK(w, r, view.Render("feed_background_refresh"))
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -119,6 +119,8 @@ func (m *middleware) handleAppSession(next http.Handler) http.Handler {
|
||||||
ctx = context.WithValue(ctx, request.UserLanguageContextKey, session.Data.Language)
|
ctx = context.WithValue(ctx, request.UserLanguageContextKey, session.Data.Language)
|
||||||
ctx = context.WithValue(ctx, request.UserThemeContextKey, session.Data.Theme)
|
ctx = context.WithValue(ctx, request.UserThemeContextKey, session.Data.Theme)
|
||||||
ctx = context.WithValue(ctx, request.PocketRequestTokenContextKey, session.Data.PocketRequestToken)
|
ctx = context.WithValue(ctx, request.PocketRequestTokenContextKey, session.Data.PocketRequestToken)
|
||||||
|
ctx = context.WithValue(ctx, request.LastForceRefreshContextKey, session.Data.LastForceRefresh)
|
||||||
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
next.ServeHTTP(w, r.WithContext(ctx))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,8 @@
|
||||||
package session // import "miniflux.app/v2/internal/ui/session"
|
package session // import "miniflux.app/v2/internal/ui/session"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/storage"
|
"miniflux.app/v2/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -13,6 +15,15 @@ type Session struct {
|
||||||
sessionID string
|
sessionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// New returns a new session handler.
|
||||||
|
func New(store *storage.Storage, sessionID string) *Session {
|
||||||
|
return &Session{store, sessionID}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Session) SetLastForceRefresh() {
|
||||||
|
s.store.UpdateAppSessionField(s.sessionID, "last_force_refresh", time.Now().UTC().Unix())
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Session) SetOAuth2State(state string) {
|
func (s *Session) SetOAuth2State(state string) {
|
||||||
s.store.UpdateAppSessionField(s.sessionID, "oauth2_state", state)
|
s.store.UpdateAppSessionField(s.sessionID, "oauth2_state", state)
|
||||||
}
|
}
|
||||||
|
@ -61,8 +72,3 @@ func (s *Session) SetTheme(theme string) {
|
||||||
func (s *Session) SetPocketRequestToken(requestToken string) {
|
func (s *Session) SetPocketRequestToken(requestToken string) {
|
||||||
s.store.UpdateAppSessionField(s.sessionID, "pocket_request_token", requestToken)
|
s.store.UpdateAppSessionField(s.sessionID, "pocket_request_token", requestToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a new session handler.
|
|
||||||
func New(store *storage.Storage, sessionID string) *Session {
|
|
||||||
return &Session{store, sessionID}
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in a new issue