southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add feature to refresh all feeds from the user interface

Browse source
This commit is contained in:
Frédéric Guillot 2017-11-21 22:36:00 -08:00
parent 480b0d94e2
commit 855fb06bc9
19 changed files with 104 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
locale/translations.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.919123936 -0800 PST m=+0.030076732
// 2017-11-21 22:32:06.371264138 -0800 PST m=+0.036637447
package locale
@ -144,12 +144,13 @@ var Translations = map[string]string{
"Bookmarklet": "Bookmarklet",
"Drag and drop this link to your bookmarks.": "Glisser-déposer ce lien dans vos favoris.",
"This special link allows you to subscribe to a website directly by using a bookmark in your web browser.": "Ce lien spécial vous permet de vous abonner à un site web directement en utilisant un marque page dans votre navigateur web.",
"Add to Miniflux": "Ajouter à Miniflux"
"Add to Miniflux": "Ajouter à Miniflux",
"Refresh all feeds in the background": "Actualiser tous les abonnements en arrière-plan"
}
`,
}
var TranslationsChecksums = map[string]string{
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
"fr_FR": "f1ddbcfb8ffd837a2df69e8506d023e4254ead2f0b94e518ab595df97d32c87a",
"fr_FR": "9a57dded2cf33b8c4d9a9d90dcbf18c96026ca396f409eb1a776ddc206fe0198",
}

3
locale/translations/fr_FR.json
View file

@ -128,5 +128,6 @@
"Bookmarklet": "Bookmarklet",
"Drag and drop this link to your bookmarks.": "Glisser-déposer ce lien dans vos favoris.",
"This special link allows you to subscribe to a website directly by using a bookmark in your web browser.": "Ce lien spécial vous permet de vous abonner à un site web directement en utilisant un marque page dans votre navigateur web.",
"Add to Miniflux": "Ajouter à Miniflux"
"Add to Miniflux": "Ajouter à Miniflux",
"Refresh all feeds in the background": "Actualiser tous les abonnements en arrière-plan"
}

22
main.go
View file

@ -44,21 +44,15 @@ func run(cfg *config.Config, store *storage.Storage) {
signal.Notify(stop, os.Interrupt)
feedHandler := feed.NewFeedHandler(store)
server := server.NewServer(cfg, store, feedHandler)
pool := scheduler.NewWorkerPool(feedHandler, cfg.GetInt("WORKER_POOL_SIZE", config.DefaultWorkerPoolSize))
server := server.NewServer(cfg, store, pool, feedHandler)
go func() {
pool := scheduler.NewWorkerPool(
feedHandler,
cfg.GetInt("WORKER_POOL_SIZE", config.DefaultWorkerPoolSize),
)
scheduler.NewScheduler(
store,
pool,
cfg.GetInt("POLLING_FREQUENCY", config.DefaultPollingFrequency),
cfg.GetInt("BATCH_SIZE", config.DefaultBatchSize),
)
}()
scheduler.NewScheduler(
store,
pool,
cfg.GetInt("POLLING_FREQUENCY", config.DefaultPollingFrequency),
cfg.GetInt("BATCH_SIZE", config.DefaultBatchSize),
)
<-stop
log.Println("Shutting down the server...")

4
model/job.go
View file

@ -4,7 +4,11 @@
package model
// Job represents a payload sent to the processing queue.
type Job struct {
UserID int64
FeedID int64
}
// JobList represents a list of jobs.
type JobList []Job

24
scheduler/scheduler.go
View file

@ -5,20 +5,24 @@
package scheduler
import (
"github.com/miniflux/miniflux2/storage"
"log"
"time"
"github.com/miniflux/miniflux2/storage"
)
// NewScheduler starts a new scheduler to push jobs to a pool of workers.
// NewScheduler starts a new scheduler that push jobs to a pool of workers.
func NewScheduler(store *storage.Storage, workerPool *WorkerPool, frequency, batchSize int) {
c := time.Tick(time.Duration(frequency) * time.Minute)
for now := range c {
jobs := store.GetJobs(batchSize)
log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs))
for _, job := range jobs {
workerPool.Push(job)
go func() {
c := time.Tick(time.Duration(frequency) * time.Minute)
for now := range c {
jobs, err := store.NewBatch(batchSize)
if err != nil {
log.Println("[Scheduler]", err)
} else {
log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs))
workerPool.Push(jobs)
}
}
}
}()
}

8
scheduler/worker_pool.go
View file

@ -14,9 +14,11 @@ type WorkerPool struct {
queue chan model.Job
}
// Push send a job on the queue.
func (w *WorkerPool) Push(job model.Job) {
w.queue <- job
// Push send a list of jobs to the queue.
func (w *WorkerPool) Push(jobs model.JobList) {
for _, job := range jobs {
w.queue <- job
}
}
// NewWorkerPool creates a pool of background workers.

7
server/routes.go
View file

@ -7,6 +7,8 @@ package server
import (
"net/http"
"github.com/miniflux/miniflux2/scheduler"
"github.com/miniflux/miniflux2/config"
"github.com/miniflux/miniflux2/locale"
"github.com/miniflux/miniflux2/reader/feed"
@ -21,13 +23,13 @@ import (
"github.com/gorilla/mux"
)
func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *mux.Router {
func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler, pool *scheduler.WorkerPool) *mux.Router {
router := mux.NewRouter()
translator := locale.Load()
templateEngine := template.NewEngine(cfg, router, translator)
apiController := api_controller.NewController(store, feedHandler)
uiController := ui_controller.NewController(store, feedHandler, opml.NewHandler(store))
uiController := ui_controller.NewController(store, pool, feedHandler, opml.NewHandler(store))
apiHandler := core.NewHandler(store, router, templateEngine, translator, middleware.NewMiddlewareChain(
middleware.NewBasicAuthMiddleware(store).Handler,
@ -79,6 +81,7 @@ func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Han
router.Handle("/feed/{feedID}/update", uiHandler.Use(uiController.UpdateFeed)).Name("updateFeed").Methods("POST")
router.Handle("/feed/{feedID}/entries", uiHandler.Use(uiController.ShowFeedEntries)).Name("feedEntries").Methods("GET")
router.Handle("/feeds", uiHandler.Use(uiController.ShowFeedsPage)).Name("feeds").Methods("GET")
router.Handle("/feeds/refresh", uiHandler.Use(uiController.RefreshAllFeeds)).Name("refreshAllFeeds").Methods("GET")
router.Handle("/unread/entry/{entryID}", uiHandler.Use(uiController.ShowUnreadEntry)).Name("unreadEntry").Methods("GET")
router.Handle("/history/entry/{entryID}", uiHandler.Use(uiController.ShowReadEntry)).Name("readEntry").Methods("GET")

6
server/server.go
View file

@ -9,19 +9,21 @@ import (
"net/http"
"time"
"github.com/miniflux/miniflux2/scheduler"
"github.com/miniflux/miniflux2/config"
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/storage"
)
// NewServer returns a new HTTP server.
func NewServer(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *http.Server {
func NewServer(cfg *config.Config, store *storage.Storage, pool *scheduler.WorkerPool, feedHandler *feed.Handler) *http.Server {
server := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 60 * time.Second,
Addr: cfg.Get("LISTEN_ADDR", config.DefaultListenAddr),
Handler: getRoutes(cfg, store, feedHandler),
Handler: getRoutes(cfg, store, feedHandler, pool),
}
go func() {

2
server/static/bin.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.896797236 -0800 PST m=+0.007750032
// 2017-11-21 22:32:06.342731949 -0800 PST m=+0.008105258
package static

2
server/static/css.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.899481906 -0800 PST m=+0.010434702
// 2017-11-21 22:32:06.344826414 -0800 PST m=+0.010199723
package static

2
server/static/js.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.901252396 -0800 PST m=+0.012205192
// 2017-11-21 22:32:06.347626921 -0800 PST m=+0.013000230
package static

2
server/template/common.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.917425704 -0800 PST m=+0.028378500
// 2017-11-21 22:32:06.368574596 -0800 PST m=+0.033947905
package template

3
server/template/html/feeds.html
View file

@ -13,6 +13,9 @@
<li>
<a href="{{ route "import" }}">{{ t "Import" }}</a>
</li>
<li>
<a href="{{ route "refreshAllFeeds" }}">{{ t "Refresh all feeds in the background" }}</a>
</li>
</ul>
</section>

7
server/template/views.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.902557626 -0800 PST m=+0.013510422
// 2017-11-21 22:32:06.350434639 -0800 PST m=+0.015807948
package template
@ -596,6 +596,9 @@ var templateViewsMap = map[string]string{
<li>
<a href="{{ route "import" }}">{{ t "Import" }}</a>
</li>
<li>
<a href="{{ route "refreshAllFeeds" }}">{{ t "Refresh all feeds in the background" }}</a>
</li>
</ul>
</section>
@ -1038,7 +1041,7 @@ var templateViewsMapChecksums = map[string]string{
"edit_user": "c835d78f7cf36c11533db9cef253457a9003987d704070d59446cb2b0e84dcb9",
"entry": "32e605edd6d43773ac31329d247ebd81d38d974cd43689d91de79fffec7fe04b",
"feed_entries": "9aff923b6c7452dec1514feada7e0d2bbc1ec21c6f5e9f48b2de41d1b731ffe4",
"feeds": "94e43404a4044490c065c888a49bebd3ff51b588b9fb47d03c2598003aa40dca",
"feeds": "c22af39b42ba9ca69ea0914ca789303ec2c5b484abcd4eaa49016e365381257c",
"history": "947603cbde888516e62925f5d08fb0b13d930623d3ee4c690dbc22612fdda75e",
"import": "73b5112e20bfd232bf73334544186ea419505936bc237d481517a8622901878f",
"integrations": "c485d6d9ed996635e55e73320610e6bcb01a41c1153e8e739ae2294b0b14b243",

7
server/ui/controller/controller.go
View file

@ -8,6 +8,7 @@ import (
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/reader/opml"
"github.com/miniflux/miniflux2/scheduler"
"github.com/miniflux/miniflux2/server/core"
"github.com/miniflux/miniflux2/storage"
)
@ -22,8 +23,10 @@ func (t tplParams) Merge(d tplParams) tplParams {
return t
}
// Controller contains all HTTP handlers for the user interface.
type Controller struct {
store *storage.Storage
pool *scheduler.WorkerPool
feedHandler *feed.Handler
opmlHandler *opml.Handler
}
@ -47,9 +50,11 @@ func (c *Controller) getCommonTemplateArgs(ctx *core.Context) (tplParams, error)
return params, nil
}
func NewController(store *storage.Storage, feedHandler *feed.Handler, opmlHandler *opml.Handler) *Controller {
// NewController returns a new Controller.
func NewController(store *storage.Storage, pool *scheduler.WorkerPool, feedHandler *feed.Handler, opmlHandler *opml.Handler) *Controller {
return &Controller{
store: store,
pool: pool,
feedHandler: feedHandler,
opmlHandler: opmlHandler,
}

16
server/ui/controller/feed.go
View file

@ -13,6 +13,22 @@ import (
"github.com/miniflux/miniflux2/server/ui/form"
)
// RefreshAllFeeds refresh all feeds in the background.
func (c *Controller) RefreshAllFeeds(ctx *core.Context, request *core.Request, response *core.Response) {
user := ctx.LoggedUser()
jobs, err := c.store.NewBatch(c.store.CountFeeds(user.ID))
if err != nil {
response.HTML().ServerError(err)
return
}
go func() {
c.pool.Push(jobs)
}()
response.Redirect(ctx.Route("feeds"))
}
// ShowFeedsPage shows the page with all subscriptions.
func (c *Controller) ShowFeedsPage(ctx *core.Context, request *core.Request, response *core.Response) {
user := ctx.LoggedUser()

2
sql/sql.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-21 20:44:27.893194286 -0800 PST m=+0.004147082
// 2017-11-21 22:32:06.338725044 -0800 PST m=+0.004098353
package sql

14
storage/feed.go
View file

@ -8,9 +8,10 @@ import (
"database/sql"
"errors"
"fmt"
"time"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"time"
)
func (s *Storage) FeedExists(userID, feedID int64) bool {
@ -31,6 +32,17 @@ func (s *Storage) FeedURLExists(userID int64, feedURL string) bool {
return result >= 1
}
// CountFeeds returns the number of feeds that belongs to the given user.
func (s *Storage) CountFeeds(userID int64) int {
var result int
err := s.db.QueryRow(`SELECT count(*) FROM feeds WHERE user_id=$1`, userID).Scan(&result)
if err != nil {
return 0
}
return result
}
func (s *Storage) GetFeeds(userID int64) (model.Feeds, error) {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:GetFeeds] userID=%d", userID))

21
storage/job.go
View file

@ -6,19 +6,19 @@ package storage
import (
"fmt"
"time"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"log"
"time"
)
const maxParsingError = 3
func (s *Storage) GetJobs(batchSize int) []model.Job {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("storage.GetJobs[%d]", batchSize))
var jobs []model.Job
query := `SELECT
// NewBatch returns a serie of jobs.
func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:GetJobs] batchSize=%d", batchSize))
query := `
SELECT
id, user_id
FROM feeds
WHERE parsing_error_count < $1
@ -26,19 +26,18 @@ func (s *Storage) GetJobs(batchSize int) []model.Job {
rows, err := s.db.Query(fmt.Sprintf(query, batchSize), maxParsingError)
if err != nil {
log.Println("Unable to fetch feed jobs:", err)
return nil, fmt.Errorf("unable to fetch batch of jobs: %v", err)
}
defer rows.Close()
for rows.Next() {
var job model.Job
if err := rows.Scan(&job.FeedID, &job.UserID); err != nil {
log.Println("Unable to fetch feed job:", err)
break
return nil, fmt.Errorf("unable to fetch job: %v", err)
}
jobs = append(jobs, job)
}
return jobs
return jobs, nil
}