Add feature to refresh all feeds from the user interface

locale/translations.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.919123936 -0800 PST m=+0.030076732
+// 2017-11-21 22:32:06.371264138 -0800 PST m=+0.036637447
 
 package locale
 
@@ -144,12 +144,13 @@ var Translations = map[string]string{
     "Bookmarklet": "Bookmarklet",
     "Drag and drop this link to your bookmarks.": "Glisser-déposer ce lien dans vos favoris.",
     "This special link allows you to subscribe to a website directly by using a bookmark in your web browser.": "Ce lien spécial vous permet de vous abonner à un site web directement en utilisant un marque page dans votre navigateur web.",
-    "Add to Miniflux": "Ajouter à Miniflux"
+    "Add to Miniflux": "Ajouter à Miniflux",
+    "Refresh all feeds in the background": "Actualiser tous les abonnements en arrière-plan"
 }
 `,
 }
 
 var TranslationsChecksums = map[string]string{
 	"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
-	"fr_FR": "f1ddbcfb8ffd837a2df69e8506d023e4254ead2f0b94e518ab595df97d32c87a",
+	"fr_FR": "9a57dded2cf33b8c4d9a9d90dcbf18c96026ca396f409eb1a776ddc206fe0198",
 }

locale/translations/fr_FR.json

@@ -128,5 +128,6 @@
     "Bookmarklet": "Bookmarklet",
     "Drag and drop this link to your bookmarks.": "Glisser-déposer ce lien dans vos favoris.",
     "This special link allows you to subscribe to a website directly by using a bookmark in your web browser.": "Ce lien spécial vous permet de vous abonner à un site web directement en utilisant un marque page dans votre navigateur web.",
-    "Add to Miniflux": "Ajouter à Miniflux"
+    "Add to Miniflux": "Ajouter à Miniflux",
+    "Refresh all feeds in the background": "Actualiser tous les abonnements en arrière-plan"
 }

main.go

@@ -44,21 +44,15 @@ func run(cfg *config.Config, store *storage.Storage) {
 	signal.Notify(stop, os.Interrupt)
 
 	feedHandler := feed.NewFeedHandler(store)
-	server := server.NewServer(cfg, store, feedHandler)
+	pool := scheduler.NewWorkerPool(feedHandler, cfg.GetInt("WORKER_POOL_SIZE", config.DefaultWorkerPoolSize))
+	server := server.NewServer(cfg, store, pool, feedHandler)
 
-	go func() {
+	scheduler.NewScheduler(
-		pool := scheduler.NewWorkerPool(
+		store,
-			feedHandler,
+		pool,
-			cfg.GetInt("WORKER_POOL_SIZE", config.DefaultWorkerPoolSize),
+		cfg.GetInt("POLLING_FREQUENCY", config.DefaultPollingFrequency),
-		)
+		cfg.GetInt("BATCH_SIZE", config.DefaultBatchSize),
-
+	)
-		scheduler.NewScheduler(
-			store,
-			pool,
-			cfg.GetInt("POLLING_FREQUENCY", config.DefaultPollingFrequency),
-			cfg.GetInt("BATCH_SIZE", config.DefaultBatchSize),
-		)
-	}()
 
 	<-stop
 	log.Println("Shutting down the server...")

model/job.go

@@ -4,7 +4,11 @@
 
 package model
 
+// Job represents a payload sent to the processing queue.
 type Job struct {
 	UserID int64
 	FeedID int64
 }
+
+// JobList represents a list of jobs.
+type JobList []Job

scheduler/scheduler.go

@@ -5,20 +5,24 @@
 package scheduler
 
 import (
-	"github.com/miniflux/miniflux2/storage"
 	"log"
 	"time"
+
+	"github.com/miniflux/miniflux2/storage"
 )
 
-// NewScheduler starts a new scheduler to push jobs to a pool of workers.
+// NewScheduler starts a new scheduler that push jobs to a pool of workers.
 func NewScheduler(store *storage.Storage, workerPool *WorkerPool, frequency, batchSize int) {
-	c := time.Tick(time.Duration(frequency) * time.Minute)
+	go func() {
-	for now := range c {
+		c := time.Tick(time.Duration(frequency) * time.Minute)
-		jobs := store.GetJobs(batchSize)
+		for now := range c {
-		log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs))
+			jobs, err := store.NewBatch(batchSize)
-
+			if err != nil {
-		for _, job := range jobs {
+				log.Println("[Scheduler]", err)
-			workerPool.Push(job)
+			} else {
+				log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs))
+				workerPool.Push(jobs)
+			}
 		}
-	}
+	}()
 }

scheduler/worker_pool.go

@@ -14,9 +14,11 @@ type WorkerPool struct {
 	queue chan model.Job
 }
 
-// Push send a job on the queue.
+// Push send a list of jobs to the queue.
-func (w *WorkerPool) Push(job model.Job) {
+func (w *WorkerPool) Push(jobs model.JobList) {
-	w.queue <- job
+	for _, job := range jobs {
+		w.queue <- job
+	}
 }
 
 // NewWorkerPool creates a pool of background workers.

server/routes.go

@@ -7,6 +7,8 @@ package server
 import (
 	"net/http"
 
+	"github.com/miniflux/miniflux2/scheduler"
+
 	"github.com/miniflux/miniflux2/config"
 	"github.com/miniflux/miniflux2/locale"
 	"github.com/miniflux/miniflux2/reader/feed"
@@ -21,13 +23,13 @@ import (
 	"github.com/gorilla/mux"
 )
 
-func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *mux.Router {
+func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler, pool *scheduler.WorkerPool) *mux.Router {
 	router := mux.NewRouter()
 	translator := locale.Load()
 	templateEngine := template.NewEngine(cfg, router, translator)
 
 	apiController := api_controller.NewController(store, feedHandler)
-	uiController := ui_controller.NewController(store, feedHandler, opml.NewHandler(store))
+	uiController := ui_controller.NewController(store, pool, feedHandler, opml.NewHandler(store))
 
 	apiHandler := core.NewHandler(store, router, templateEngine, translator, middleware.NewMiddlewareChain(
 		middleware.NewBasicAuthMiddleware(store).Handler,
@@ -79,6 +81,7 @@ func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Han
 	router.Handle("/feed/{feedID}/update", uiHandler.Use(uiController.UpdateFeed)).Name("updateFeed").Methods("POST")
 	router.Handle("/feed/{feedID}/entries", uiHandler.Use(uiController.ShowFeedEntries)).Name("feedEntries").Methods("GET")
 	router.Handle("/feeds", uiHandler.Use(uiController.ShowFeedsPage)).Name("feeds").Methods("GET")
+	router.Handle("/feeds/refresh", uiHandler.Use(uiController.RefreshAllFeeds)).Name("refreshAllFeeds").Methods("GET")
 
 	router.Handle("/unread/entry/{entryID}", uiHandler.Use(uiController.ShowUnreadEntry)).Name("unreadEntry").Methods("GET")
 	router.Handle("/history/entry/{entryID}", uiHandler.Use(uiController.ShowReadEntry)).Name("readEntry").Methods("GET")

server/server.go

@@ -9,19 +9,21 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/miniflux/miniflux2/scheduler"
+
 	"github.com/miniflux/miniflux2/config"
 	"github.com/miniflux/miniflux2/reader/feed"
 	"github.com/miniflux/miniflux2/storage"
 )
 
 // NewServer returns a new HTTP server.
-func NewServer(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *http.Server {
+func NewServer(cfg *config.Config, store *storage.Storage, pool *scheduler.WorkerPool, feedHandler *feed.Handler) *http.Server {
 	server := &http.Server{
 		ReadTimeout:  5 * time.Second,
 		WriteTimeout: 10 * time.Second,
 		IdleTimeout:  60 * time.Second,
 		Addr:         cfg.Get("LISTEN_ADDR", config.DefaultListenAddr),
-		Handler:      getRoutes(cfg, store, feedHandler),
+		Handler:      getRoutes(cfg, store, feedHandler, pool),
 	}
 
 	go func() {

server/static/bin.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.896797236 -0800 PST m=+0.007750032
+// 2017-11-21 22:32:06.342731949 -0800 PST m=+0.008105258
 
 package static
 

server/static/css.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.899481906 -0800 PST m=+0.010434702
+// 2017-11-21 22:32:06.344826414 -0800 PST m=+0.010199723
 
 package static
 

server/static/js.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.901252396 -0800 PST m=+0.012205192
+// 2017-11-21 22:32:06.347626921 -0800 PST m=+0.013000230
 
 package static
 

server/template/common.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.917425704 -0800 PST m=+0.028378500
+// 2017-11-21 22:32:06.368574596 -0800 PST m=+0.033947905
 
 package template
 

server/template/html/feeds.html

@@ -13,6 +13,9 @@
         <li>
             <a href="{{ route "import" }}">{{ t "Import" }}</a>
         </li>
+        <li>
+            <a href="{{ route "refreshAllFeeds" }}">{{ t "Refresh all feeds in the background" }}</a>
+        </li>
     </ul>
 </section>
 

server/template/views.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.902557626 -0800 PST m=+0.013510422
+// 2017-11-21 22:32:06.350434639 -0800 PST m=+0.015807948
 
 package template
 
@@ -596,6 +596,9 @@ var templateViewsMap = map[string]string{
         <li>
             <a href="{{ route "import" }}">{{ t "Import" }}</a>
         </li>
+        <li>
+            <a href="{{ route "refreshAllFeeds" }}">{{ t "Refresh all feeds in the background" }}</a>
+        </li>
     </ul>
 </section>
 
@@ -1038,7 +1041,7 @@ var templateViewsMapChecksums = map[string]string{
 	"edit_user":           "c835d78f7cf36c11533db9cef253457a9003987d704070d59446cb2b0e84dcb9",
 	"entry":               "32e605edd6d43773ac31329d247ebd81d38d974cd43689d91de79fffec7fe04b",
 	"feed_entries":        "9aff923b6c7452dec1514feada7e0d2bbc1ec21c6f5e9f48b2de41d1b731ffe4",
-	"feeds":               "94e43404a4044490c065c888a49bebd3ff51b588b9fb47d03c2598003aa40dca",
+	"feeds":               "c22af39b42ba9ca69ea0914ca789303ec2c5b484abcd4eaa49016e365381257c",
 	"history":             "947603cbde888516e62925f5d08fb0b13d930623d3ee4c690dbc22612fdda75e",
 	"import":              "73b5112e20bfd232bf73334544186ea419505936bc237d481517a8622901878f",
 	"integrations":        "c485d6d9ed996635e55e73320610e6bcb01a41c1153e8e739ae2294b0b14b243",

server/ui/controller/controller.go

@@ -8,6 +8,7 @@ import (
 	"github.com/miniflux/miniflux2/model"
 	"github.com/miniflux/miniflux2/reader/feed"
 	"github.com/miniflux/miniflux2/reader/opml"
+	"github.com/miniflux/miniflux2/scheduler"
 	"github.com/miniflux/miniflux2/server/core"
 	"github.com/miniflux/miniflux2/storage"
 )
@@ -22,8 +23,10 @@ func (t tplParams) Merge(d tplParams) tplParams {
 	return t
 }
 
+// Controller contains all HTTP handlers for the user interface.
 type Controller struct {
 	store       *storage.Storage
+	pool        *scheduler.WorkerPool
 	feedHandler *feed.Handler
 	opmlHandler *opml.Handler
 }
@@ -47,9 +50,11 @@ func (c *Controller) getCommonTemplateArgs(ctx *core.Context) (tplParams, error)
 	return params, nil
 }
 
-func NewController(store *storage.Storage, feedHandler *feed.Handler, opmlHandler *opml.Handler) *Controller {
+// NewController returns a new Controller.
+func NewController(store *storage.Storage, pool *scheduler.WorkerPool, feedHandler *feed.Handler, opmlHandler *opml.Handler) *Controller {
 	return &Controller{
 		store:       store,
+		pool:        pool,
 		feedHandler: feedHandler,
 		opmlHandler: opmlHandler,
 	}

server/ui/controller/feed.go

@@ -13,6 +13,22 @@ import (
 	"github.com/miniflux/miniflux2/server/ui/form"
 )
 
+// RefreshAllFeeds refresh all feeds in the background.
+func (c *Controller) RefreshAllFeeds(ctx *core.Context, request *core.Request, response *core.Response) {
+	user := ctx.LoggedUser()
+	jobs, err := c.store.NewBatch(c.store.CountFeeds(user.ID))
+	if err != nil {
+		response.HTML().ServerError(err)
+		return
+	}
+
+	go func() {
+		c.pool.Push(jobs)
+	}()
+
+	response.Redirect(ctx.Route("feeds"))
+}
+
 // ShowFeedsPage shows the page with all subscriptions.
 func (c *Controller) ShowFeedsPage(ctx *core.Context, request *core.Request, response *core.Response) {
 	user := ctx.LoggedUser()

sql/sql.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-11-21 20:44:27.893194286 -0800 PST m=+0.004147082
+// 2017-11-21 22:32:06.338725044 -0800 PST m=+0.004098353
 
 package sql
 

storage/feed.go

@@ -8,9 +8,10 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"time"
+
 	"github.com/miniflux/miniflux2/helper"
 	"github.com/miniflux/miniflux2/model"
-	"time"
 )
 
 func (s *Storage) FeedExists(userID, feedID int64) bool {
@@ -31,6 +32,17 @@ func (s *Storage) FeedURLExists(userID int64, feedURL string) bool {
 	return result >= 1
 }
 
+// CountFeeds returns the number of feeds that belongs to the given user.
+func (s *Storage) CountFeeds(userID int64) int {
+	var result int
+	err := s.db.QueryRow(`SELECT count(*) FROM feeds WHERE user_id=$1`, userID).Scan(&result)
+	if err != nil {
+		return 0
+	}
+
+	return result
+}
+
 func (s *Storage) GetFeeds(userID int64) (model.Feeds, error) {
 	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:GetFeeds] userID=%d", userID))
 

storage/job.go

@@ -6,19 +6,19 @@ package storage
 
 import (
 	"fmt"
+	"time"
+
 	"github.com/miniflux/miniflux2/helper"
 	"github.com/miniflux/miniflux2/model"
-	"log"
-	"time"
 )
 
 const maxParsingError = 3
 
-func (s *Storage) GetJobs(batchSize int) []model.Job {
+// NewBatch returns a serie of jobs.
-	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("storage.GetJobs[%d]", batchSize))
+func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
-
+	defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:GetJobs] batchSize=%d", batchSize))
-	var jobs []model.Job
+	query := `
-	query := `SELECT
+		SELECT
 		id, user_id
 		FROM feeds
 		WHERE parsing_error_count < $1
@@ -26,19 +26,18 @@ func (s *Storage) GetJobs(batchSize int) []model.Job {
 
 	rows, err := s.db.Query(fmt.Sprintf(query, batchSize), maxParsingError)
 	if err != nil {
-		log.Println("Unable to fetch feed jobs:", err)
+		return nil, fmt.Errorf("unable to fetch batch of jobs: %v", err)
 	}
 	defer rows.Close()
 
 	for rows.Next() {
 		var job model.Job
 		if err := rows.Scan(&job.FeedID, &job.UserID); err != nil {
-			log.Println("Unable to fetch feed job:", err)
+			return nil, fmt.Errorf("unable to fetch job: %v", err)
-			break
 		}
 
 		jobs = append(jobs, job)
 	}
 
-	return jobs
+	return jobs, nil
 }