From 855fb06bc93a628108990013813725996a090c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Tue, 21 Nov 2017 22:36:00 -0800 Subject: [PATCH] Add feature to refresh all feeds from the user interface --- locale/translations.go | 7 ++++--- locale/translations/fr_FR.json | 3 ++- main.go | 22 ++++++++-------------- model/job.go | 4 ++++ scheduler/scheduler.go | 24 ++++++++++++++---------- scheduler/worker_pool.go | 8 +++++--- server/routes.go | 7 +++++-- server/server.go | 6 ++++-- server/static/bin.go | 2 +- server/static/css.go | 2 +- server/static/js.go | 2 +- server/template/common.go | 2 +- server/template/html/feeds.html | 3 +++ server/template/views.go | 7 +++++-- server/ui/controller/controller.go | 7 ++++++- server/ui/controller/feed.go | 16 ++++++++++++++++ sql/sql.go | 2 +- storage/feed.go | 14 +++++++++++++- storage/job.go | 21 ++++++++++----------- 19 files changed, 104 insertions(+), 55 deletions(-) diff --git a/locale/translations.go b/locale/translations.go index c2049f74..a15c0446 100644 --- a/locale/translations.go +++ b/locale/translations.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.919123936 -0800 PST m=+0.030076732 +// 2017-11-21 22:32:06.371264138 -0800 PST m=+0.036637447 package locale @@ -144,12 +144,13 @@ var Translations = map[string]string{ "Bookmarklet": "Bookmarklet", "Drag and drop this link to your bookmarks.": "Glisser-déposer ce lien dans vos favoris.", "This special link allows you to subscribe to a website directly by using a bookmark in your web browser.": "Ce lien spécial vous permet de vous abonner à un site web directement en utilisant un marque page dans votre navigateur web.", - "Add to Miniflux": "Ajouter à Miniflux" + "Add to Miniflux": "Ajouter à Miniflux", + "Refresh all feeds in the background": "Actualiser tous les abonnements en arrière-plan" } `, } var TranslationsChecksums = map[string]string{ "en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897", - "fr_FR": "f1ddbcfb8ffd837a2df69e8506d023e4254ead2f0b94e518ab595df97d32c87a", + "fr_FR": "9a57dded2cf33b8c4d9a9d90dcbf18c96026ca396f409eb1a776ddc206fe0198", } diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json index a0206bad..3cef2a59 100644 --- a/locale/translations/fr_FR.json +++ b/locale/translations/fr_FR.json @@ -128,5 +128,6 @@ "Bookmarklet": "Bookmarklet", "Drag and drop this link to your bookmarks.": "Glisser-déposer ce lien dans vos favoris.", "This special link allows you to subscribe to a website directly by using a bookmark in your web browser.": "Ce lien spécial vous permet de vous abonner à un site web directement en utilisant un marque page dans votre navigateur web.", - "Add to Miniflux": "Ajouter à Miniflux" + "Add to Miniflux": "Ajouter à Miniflux", + "Refresh all feeds in the background": "Actualiser tous les abonnements en arrière-plan" } diff --git a/main.go b/main.go index 9ab7cdc4..4393472c 100644 --- a/main.go +++ b/main.go @@ -44,21 +44,15 @@ func run(cfg *config.Config, store *storage.Storage) { signal.Notify(stop, os.Interrupt) feedHandler := feed.NewFeedHandler(store) - server := server.NewServer(cfg, store, feedHandler) + pool := scheduler.NewWorkerPool(feedHandler, cfg.GetInt("WORKER_POOL_SIZE", config.DefaultWorkerPoolSize)) + server := server.NewServer(cfg, store, pool, feedHandler) - go func() { - pool := scheduler.NewWorkerPool( - feedHandler, - cfg.GetInt("WORKER_POOL_SIZE", config.DefaultWorkerPoolSize), - ) - - scheduler.NewScheduler( - store, - pool, - cfg.GetInt("POLLING_FREQUENCY", config.DefaultPollingFrequency), - cfg.GetInt("BATCH_SIZE", config.DefaultBatchSize), - ) - }() + scheduler.NewScheduler( + store, + pool, + cfg.GetInt("POLLING_FREQUENCY", config.DefaultPollingFrequency), + cfg.GetInt("BATCH_SIZE", config.DefaultBatchSize), + ) <-stop log.Println("Shutting down the server...") diff --git a/model/job.go b/model/job.go index ed3ef5ee..9f9167bc 100644 --- a/model/job.go +++ b/model/job.go @@ -4,7 +4,11 @@ package model +// Job represents a payload sent to the processing queue. type Job struct { UserID int64 FeedID int64 } + +// JobList represents a list of jobs. +type JobList []Job diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index ab87e992..acfb4016 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -5,20 +5,24 @@ package scheduler import ( - "github.com/miniflux/miniflux2/storage" "log" "time" + + "github.com/miniflux/miniflux2/storage" ) -// NewScheduler starts a new scheduler to push jobs to a pool of workers. +// NewScheduler starts a new scheduler that push jobs to a pool of workers. func NewScheduler(store *storage.Storage, workerPool *WorkerPool, frequency, batchSize int) { - c := time.Tick(time.Duration(frequency) * time.Minute) - for now := range c { - jobs := store.GetJobs(batchSize) - log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs)) - - for _, job := range jobs { - workerPool.Push(job) + go func() { + c := time.Tick(time.Duration(frequency) * time.Minute) + for now := range c { + jobs, err := store.NewBatch(batchSize) + if err != nil { + log.Println("[Scheduler]", err) + } else { + log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs)) + workerPool.Push(jobs) + } } - } + }() } diff --git a/scheduler/worker_pool.go b/scheduler/worker_pool.go index b753f893..c4a83723 100644 --- a/scheduler/worker_pool.go +++ b/scheduler/worker_pool.go @@ -14,9 +14,11 @@ type WorkerPool struct { queue chan model.Job } -// Push send a job on the queue. -func (w *WorkerPool) Push(job model.Job) { - w.queue <- job +// Push send a list of jobs to the queue. +func (w *WorkerPool) Push(jobs model.JobList) { + for _, job := range jobs { + w.queue <- job + } } // NewWorkerPool creates a pool of background workers. diff --git a/server/routes.go b/server/routes.go index 97a8b2f9..4d4a99a6 100644 --- a/server/routes.go +++ b/server/routes.go @@ -7,6 +7,8 @@ package server import ( "net/http" + "github.com/miniflux/miniflux2/scheduler" + "github.com/miniflux/miniflux2/config" "github.com/miniflux/miniflux2/locale" "github.com/miniflux/miniflux2/reader/feed" @@ -21,13 +23,13 @@ import ( "github.com/gorilla/mux" ) -func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *mux.Router { +func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler, pool *scheduler.WorkerPool) *mux.Router { router := mux.NewRouter() translator := locale.Load() templateEngine := template.NewEngine(cfg, router, translator) apiController := api_controller.NewController(store, feedHandler) - uiController := ui_controller.NewController(store, feedHandler, opml.NewHandler(store)) + uiController := ui_controller.NewController(store, pool, feedHandler, opml.NewHandler(store)) apiHandler := core.NewHandler(store, router, templateEngine, translator, middleware.NewMiddlewareChain( middleware.NewBasicAuthMiddleware(store).Handler, @@ -79,6 +81,7 @@ func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Han router.Handle("/feed/{feedID}/update", uiHandler.Use(uiController.UpdateFeed)).Name("updateFeed").Methods("POST") router.Handle("/feed/{feedID}/entries", uiHandler.Use(uiController.ShowFeedEntries)).Name("feedEntries").Methods("GET") router.Handle("/feeds", uiHandler.Use(uiController.ShowFeedsPage)).Name("feeds").Methods("GET") + router.Handle("/feeds/refresh", uiHandler.Use(uiController.RefreshAllFeeds)).Name("refreshAllFeeds").Methods("GET") router.Handle("/unread/entry/{entryID}", uiHandler.Use(uiController.ShowUnreadEntry)).Name("unreadEntry").Methods("GET") router.Handle("/history/entry/{entryID}", uiHandler.Use(uiController.ShowReadEntry)).Name("readEntry").Methods("GET") diff --git a/server/server.go b/server/server.go index 61b62291..f1a2a577 100644 --- a/server/server.go +++ b/server/server.go @@ -9,19 +9,21 @@ import ( "net/http" "time" + "github.com/miniflux/miniflux2/scheduler" + "github.com/miniflux/miniflux2/config" "github.com/miniflux/miniflux2/reader/feed" "github.com/miniflux/miniflux2/storage" ) // NewServer returns a new HTTP server. -func NewServer(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *http.Server { +func NewServer(cfg *config.Config, store *storage.Storage, pool *scheduler.WorkerPool, feedHandler *feed.Handler) *http.Server { server := &http.Server{ ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second, Addr: cfg.Get("LISTEN_ADDR", config.DefaultListenAddr), - Handler: getRoutes(cfg, store, feedHandler), + Handler: getRoutes(cfg, store, feedHandler, pool), } go func() { diff --git a/server/static/bin.go b/server/static/bin.go index 7a587113..0132a9c3 100644 --- a/server/static/bin.go +++ b/server/static/bin.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.896797236 -0800 PST m=+0.007750032 +// 2017-11-21 22:32:06.342731949 -0800 PST m=+0.008105258 package static diff --git a/server/static/css.go b/server/static/css.go index 8554c22d..42a1cbab 100644 --- a/server/static/css.go +++ b/server/static/css.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.899481906 -0800 PST m=+0.010434702 +// 2017-11-21 22:32:06.344826414 -0800 PST m=+0.010199723 package static diff --git a/server/static/js.go b/server/static/js.go index 45f51be4..04a3f422 100644 --- a/server/static/js.go +++ b/server/static/js.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.901252396 -0800 PST m=+0.012205192 +// 2017-11-21 22:32:06.347626921 -0800 PST m=+0.013000230 package static diff --git a/server/template/common.go b/server/template/common.go index 5aee57f6..ce54011e 100644 --- a/server/template/common.go +++ b/server/template/common.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.917425704 -0800 PST m=+0.028378500 +// 2017-11-21 22:32:06.368574596 -0800 PST m=+0.033947905 package template diff --git a/server/template/html/feeds.html b/server/template/html/feeds.html index 778f64f4..f84e59f4 100644 --- a/server/template/html/feeds.html +++ b/server/template/html/feeds.html @@ -13,6 +13,9 @@
  • {{ t "Import" }}
    • +
  • + {{ t "Refresh all feeds in the background" }} +
    • diff --git a/server/template/views.go b/server/template/views.go index 18f88d31..bfc0f8ac 100644 --- a/server/template/views.go +++ b/server/template/views.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.902557626 -0800 PST m=+0.013510422 +// 2017-11-21 22:32:06.350434639 -0800 PST m=+0.015807948 package template @@ -596,6 +596,9 @@ var templateViewsMap = map[string]string{
  • {{ t "Import" }}
    • +
  • + {{ t "Refresh all feeds in the background" }} +
    • @@ -1038,7 +1041,7 @@ var templateViewsMapChecksums = map[string]string{ "edit_user": "c835d78f7cf36c11533db9cef253457a9003987d704070d59446cb2b0e84dcb9", "entry": "32e605edd6d43773ac31329d247ebd81d38d974cd43689d91de79fffec7fe04b", "feed_entries": "9aff923b6c7452dec1514feada7e0d2bbc1ec21c6f5e9f48b2de41d1b731ffe4", - "feeds": "94e43404a4044490c065c888a49bebd3ff51b588b9fb47d03c2598003aa40dca", + "feeds": "c22af39b42ba9ca69ea0914ca789303ec2c5b484abcd4eaa49016e365381257c", "history": "947603cbde888516e62925f5d08fb0b13d930623d3ee4c690dbc22612fdda75e", "import": "73b5112e20bfd232bf73334544186ea419505936bc237d481517a8622901878f", "integrations": "c485d6d9ed996635e55e73320610e6bcb01a41c1153e8e739ae2294b0b14b243", diff --git a/server/ui/controller/controller.go b/server/ui/controller/controller.go index 22c9dce4..4aa60856 100644 --- a/server/ui/controller/controller.go +++ b/server/ui/controller/controller.go @@ -8,6 +8,7 @@ import ( "github.com/miniflux/miniflux2/model" "github.com/miniflux/miniflux2/reader/feed" "github.com/miniflux/miniflux2/reader/opml" + "github.com/miniflux/miniflux2/scheduler" "github.com/miniflux/miniflux2/server/core" "github.com/miniflux/miniflux2/storage" ) @@ -22,8 +23,10 @@ func (t tplParams) Merge(d tplParams) tplParams { return t } +// Controller contains all HTTP handlers for the user interface. type Controller struct { store *storage.Storage + pool *scheduler.WorkerPool feedHandler *feed.Handler opmlHandler *opml.Handler } @@ -47,9 +50,11 @@ func (c *Controller) getCommonTemplateArgs(ctx *core.Context) (tplParams, error) return params, nil } -func NewController(store *storage.Storage, feedHandler *feed.Handler, opmlHandler *opml.Handler) *Controller { +// NewController returns a new Controller. +func NewController(store *storage.Storage, pool *scheduler.WorkerPool, feedHandler *feed.Handler, opmlHandler *opml.Handler) *Controller { return &Controller{ store: store, + pool: pool, feedHandler: feedHandler, opmlHandler: opmlHandler, } diff --git a/server/ui/controller/feed.go b/server/ui/controller/feed.go index dc55a118..7ed903ed 100644 --- a/server/ui/controller/feed.go +++ b/server/ui/controller/feed.go @@ -13,6 +13,22 @@ import ( "github.com/miniflux/miniflux2/server/ui/form" ) +// RefreshAllFeeds refresh all feeds in the background. +func (c *Controller) RefreshAllFeeds(ctx *core.Context, request *core.Request, response *core.Response) { + user := ctx.LoggedUser() + jobs, err := c.store.NewBatch(c.store.CountFeeds(user.ID)) + if err != nil { + response.HTML().ServerError(err) + return + } + + go func() { + c.pool.Push(jobs) + }() + + response.Redirect(ctx.Route("feeds")) +} + // ShowFeedsPage shows the page with all subscriptions. func (c *Controller) ShowFeedsPage(ctx *core.Context, request *core.Request, response *core.Response) { user := ctx.LoggedUser() diff --git a/sql/sql.go b/sql/sql.go index c2685da9..5b36bc4a 100644 --- a/sql/sql.go +++ b/sql/sql.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2017-11-21 20:44:27.893194286 -0800 PST m=+0.004147082 +// 2017-11-21 22:32:06.338725044 -0800 PST m=+0.004098353 package sql diff --git a/storage/feed.go b/storage/feed.go index ec085807..0fab4817 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -8,9 +8,10 @@ import ( "database/sql" "errors" "fmt" + "time" + "github.com/miniflux/miniflux2/helper" "github.com/miniflux/miniflux2/model" - "time" ) func (s *Storage) FeedExists(userID, feedID int64) bool { @@ -31,6 +32,17 @@ func (s *Storage) FeedURLExists(userID int64, feedURL string) bool { return result >= 1 } +// CountFeeds returns the number of feeds that belongs to the given user. +func (s *Storage) CountFeeds(userID int64) int { + var result int + err := s.db.QueryRow(`SELECT count(*) FROM feeds WHERE user_id=$1`, userID).Scan(&result) + if err != nil { + return 0 + } + + return result +} + func (s *Storage) GetFeeds(userID int64) (model.Feeds, error) { defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:GetFeeds] userID=%d", userID)) diff --git a/storage/job.go b/storage/job.go index 5383a5b5..c9ab64cf 100644 --- a/storage/job.go +++ b/storage/job.go @@ -6,19 +6,19 @@ package storage import ( "fmt" + "time" + "github.com/miniflux/miniflux2/helper" "github.com/miniflux/miniflux2/model" - "log" - "time" ) const maxParsingError = 3 -func (s *Storage) GetJobs(batchSize int) []model.Job { - defer helper.ExecutionTime(time.Now(), fmt.Sprintf("storage.GetJobs[%d]", batchSize)) - - var jobs []model.Job - query := `SELECT +// NewBatch returns a serie of jobs. +func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) { + defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:GetJobs] batchSize=%d", batchSize)) + query := ` + SELECT id, user_id FROM feeds WHERE parsing_error_count < $1 @@ -26,19 +26,18 @@ func (s *Storage) GetJobs(batchSize int) []model.Job { rows, err := s.db.Query(fmt.Sprintf(query, batchSize), maxParsingError) if err != nil { - log.Println("Unable to fetch feed jobs:", err) + return nil, fmt.Errorf("unable to fetch batch of jobs: %v", err) } defer rows.Close() for rows.Next() { var job model.Job if err := rows.Scan(&job.FeedID, &job.UserID); err != nil { - log.Println("Unable to fetch feed job:", err) - break + return nil, fmt.Errorf("unable to fetch job: %v", err) } jobs = append(jobs, job) } - return jobs + return jobs, nil }