southfox/miniflux
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

First commit

Browse source
This commit is contained in:
Frédéric Guillot 2017-11-19 21:10:04 -08:00
commit 8ffb773f43
2121 changed files with 1118910 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
miniflux-linux-amd64
miniflux-darwin-amd64

5
.travis.yml Normal file
View file

@ -0,0 +1,5 @@
language: go
go:
- 1.9
script:
- go test -cover -race ./...

81
Gopkg.lock generated Normal file
View file

@ -0,0 +1,81 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/PuerkitoBio/goquery"
packages = ["."]
revision = "e1271ee34c6a305e38566ecd27ae374944907ee9"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/andybalholm/cascadia"
packages = ["."]
revision = "349dd0209470eabd9514242c688c403c0926d266"
[[projects]]
name = "github.com/gorilla/context"
packages = ["."]
revision = "1ea25387ff6f684839d82767c1733ff4d4d15d0a"
version = "v1.1"
[[projects]]
name = "github.com/gorilla/mux"
packages = ["."]
revision = "7f08801859139f86dfafd1c296e2cba9a80d292e"
version = "v1.6.0"
[[projects]]
branch = "master"
name = "github.com/lib/pq"
packages = [".","oid"]
revision = "8c6ee72f3e6bcb1542298dd5f76cb74af9742cec"
[[projects]]
name = "github.com/tdewolff/minify"
packages = [".","css","js"]
revision = "90df1aae5028a7cbb441bde86e86a55df6b5aa34"
version = "v2.3.3"
[[projects]]
name = "github.com/tdewolff/parse"
packages = [".","buffer","css","js","strconv"]
revision = "bace4cf682c41e03b154044b561575ff541b83e8"
version = "v2.3.1"
[[projects]]
branch = "master"
name = "github.com/tomasen/realip"
packages = ["."]
revision = "15489afd3be348430f5f67467d2bb6b2f9b757ed"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["bcrypt","blowfish","ssh/terminal"]
revision = "9f005a07e0d31d45e6656d241bb5c0f2efd4bc94"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["html","html/atom","html/charset"]
revision = "9dfe39835686865bff950a07b394c12a98ddc811"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix","windows"]
revision = "0dd5e194bbf5eb84a39666eb4c98a4d007e4203a"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"]
revision = "88f656faf3f37f690df1a32515b479415e1a6769"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "27a0ca12f5a709bb76b9c90f6720b6824ac8fc81b2fc66f059f212366443ff5d"
solver-name = "gps-cdcl"
solver-version = 1

54
Gopkg.toml Normal file
View file

@ -0,0 +1,54 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
[[constraint]]
name = "github.com/PuerkitoBio/goquery"
version = "1.1.0"
[[constraint]]
name = "github.com/gorilla/mux"
version = "1.6.0"
[[constraint]]
branch = "master"
name = "github.com/lib/pq"
[[constraint]]
branch = "master"
name = "github.com/rvflash/elapsed"
[[constraint]]
name = "github.com/tdewolff/minify"
version = "2.3.3"
[[constraint]]
branch = "master"
name = "github.com/tomasen/realip"
[[constraint]]
branch = "master"
name = "golang.org/x/crypto"
[[constraint]]
branch = "master"
name = "golang.org/x/net"

177
LICENSE Normal file
View file

@ -0,0 +1,177 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS

25
Makefile Normal file
View file

@ -0,0 +1,25 @@
APP = miniflux
VERSION = $(shell git rev-parse --short HEAD)
BUILD_DATE = `date +%FT%T%z`
.PHONY: build-linux build-darwin build run clean test
build-linux:
@ go generate
@ GOOS=linux GOARCH=amd64 go build -ldflags="-X 'miniflux/version.Version=$(VERSION)' -X 'miniflux/version.BuildDate=$(BUILD_DATE)'" -o $(APP)-linux-amd64 main.go
build-darwin:
@ go generate
@ GOOS=darwin GOARCH=amd64 go build -ldflags="-X 'miniflux/version.Version=$(VERSION)' -X 'miniflux/version.BuildDate=$(BUILD_DATE)'" -o $(APP)-darwin-amd64 main.go
build: build-linux build-darwin
run:
@ go generate
@ go run main.go
clean:
@ rm -f $(APP)-*
test:
go test -cover -race ./...

38
README.md Normal file
View file

@ -0,0 +1,38 @@
Miniflux 2
==========
[![Build Status](https://travis-ci.org/miniflux/miniflux2.svg?branch=master)](https://travis-ci.org/miniflux/miniflux2)
Miniflux is a minimalist and opinionated feed reader:
- Written in Go (Golang)
- Works only with Postgresql
- Doesn't use any ORM
- Doesn't use any complicated framework
- The number of features is volountary limited
It's simple, fast, lightweight and super easy to install.
Miniflux 2 is a rewrite of Miniflux 1.x in Golang.
Notes
-----
Miniflux 2 still in development and **it's not ready to use**.
TODO
----
- [ ] Custom entries sorting
- [ ] Webpage scraper (Readability)
- [ ] Bookmarklet
- [ ] External integrations (Pinboard, Wallabag...)
- [ ] Gzip compression
- [ ] Integration tests
- [ ] Flush history
- [ ] OAuth2
Credits
-------
- Author: Frédéric Guillot
- Distributed under Apache 2.0 License

36
config/config.go Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package config
import (
"os"
"strconv"
)
type Config struct {
}
func (c *Config) Get(key, fallback string) string {
value := os.Getenv(key)
if value == "" {
return fallback
}
return value
}
func (c *Config) GetInt(key string, fallback int) int {
value := os.Getenv(key)
if value == "" {
return fallback
}
v, _ := strconv.Atoi(value)
return v
}
func NewConfig() *Config {
return &Config{}
}

27
errors/errors.go Normal file
View file

@ -0,0 +1,27 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package errors
import (
"fmt"
"github.com/miniflux/miniflux2/locale"
)
type LocalizedError struct {
message string
args []interface{}
}
func (l LocalizedError) Error() string {
return fmt.Sprintf(l.message, l.args...)
}
func (l LocalizedError) Localize(translation *locale.Language) string {
return translation.Get(l.message, l.args...)
}
func NewLocalizedError(message string, args ...interface{}) LocalizedError {
return LocalizedError{message: message, args: args}
}

120
generate.go Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
// +build ignore
package main
import (
"crypto/sha256"
"encoding/base64"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"text/template"
"time"
"github.com/tdewolff/minify"
"github.com/tdewolff/minify/css"
"github.com/tdewolff/minify/js"
)
const tpl = `// Code generated by go generate; DO NOT EDIT.
// {{ .Timestamp }}
package {{ .Package }}
var {{ .Map }} = map[string]string{
{{ range $constant, $content := .Files }}` + "\t" + `"{{ $constant }}": ` + "`{{ $content }}`" + `,
{{ end }}}
var {{ .Map }}Checksums = map[string]string{
{{ range $constant, $content := .Checksums }}` + "\t" + `"{{ $constant }}": "{{ $content }}",
{{ end }}}
`
var generatedTpl = template.Must(template.New("").Parse(tpl))
type GeneratedFile struct {
Package, Map string
Timestamp time.Time
Files map[string]string
Checksums map[string]string
}
func normalizeBasename(filename string) string {
filename = strings.TrimSuffix(filename, filepath.Ext(filename))
return strings.Replace(filename, " ", "_", -1)
}
func generateFile(serializer, pkg, mapName, pattern, output string) {
generatedFile := &GeneratedFile{
Package: pkg,
Map: mapName,
Timestamp: time.Now(),
Files: make(map[string]string),
Checksums: make(map[string]string),
}
files, _ := filepath.Glob(pattern)
for _, file := range files {
basename := path.Base(file)
content, err := ioutil.ReadFile(file)
if err != nil {
panic(err)
}
switch serializer {
case "css":
m := minify.New()
m.AddFunc("text/css", css.Minify)
content, err = m.Bytes("text/css", content)
if err != nil {
panic(err)
}
basename = normalizeBasename(basename)
generatedFile.Files[basename] = string(content)
case "js":
m := minify.New()
m.AddFunc("text/javascript", js.Minify)
content, err = m.Bytes("text/javascript", content)
if err != nil {
panic(err)
}
basename = normalizeBasename(basename)
generatedFile.Files[basename] = string(content)
case "base64":
encodedContent := base64.StdEncoding.EncodeToString(content)
generatedFile.Files[basename] = encodedContent
default:
basename = normalizeBasename(basename)
generatedFile.Files[basename] = string(content)
}
generatedFile.Checksums[basename] = fmt.Sprintf("%x", sha256.Sum256(content))
}
f, err := os.Create(output)
if err != nil {
panic(err)
}
defer f.Close()
generatedTpl.Execute(f, generatedFile)
}
func main() {
generateFile("none", "sql", "SqlMap", "sql/*.sql", "sql/sql.go")
generateFile("base64", "static", "Binaries", "server/static/bin/*", "server/static/bin.go")
generateFile("css", "static", "Stylesheets", "server/static/css/*.css", "server/static/css.go")
generateFile("js", "static", "Javascript", "server/static/js/*.js", "server/static/js.go")
generateFile("none", "template", "templateViewsMap", "server/template/html/*.html", "server/template/views.go")
generateFile("none", "template", "templateCommonMap", "server/template/html/common/*.html", "server/template/common.go")
generateFile("none", "locale", "Translations", "locale/translations/*.json", "locale/translations.go")
}

38
helper/crypto.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package helper
import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"fmt"
)
// HashFromBytes returns a SHA-256 checksum of the input.
func HashFromBytes(value []byte) string {
sum := sha256.Sum256(value)
return fmt.Sprintf("%x", sum)
}
// Hash returns a SHA-256 checksum of a string.
func Hash(value string) string {
return HashFromBytes([]byte(value))
}
// GenerateRandomBytes returns random bytes.
func GenerateRandomBytes(size int) []byte {
b := make([]byte, size)
if _, err := rand.Read(b); err != nil {
panic(fmt.Errorf("Unable to generate random string: %v", err))
}
return b
}
// GenerateRandomString returns a random string.
func GenerateRandomString(size int) string {
return base64.URLEncoding.EncodeToString(GenerateRandomBytes(size))
}

16
helper/time.go Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package helper
import (
"log"
"time"
)
// ExecutionTime returns the elapsed time of a block of code.
func ExecutionTime(start time.Time, name string) {
elapsed := time.Since(start)
log.Printf("%s took %s", name, elapsed)
}

47
locale/language.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package locale
import "fmt"
type Language struct {
language string
translations Translation
}
func (l *Language) Get(key string, args ...interface{}) string {
var translation string
str, found := l.translations[key]
if !found {
translation = key
} else {
translation = str.(string)
}
return fmt.Sprintf(translation, args...)
}
func (l *Language) Plural(key string, n int, args ...interface{}) string {
translation := key
slices, found := l.translations[key]
if found {
pluralForm, found := pluralForms[l.language]
if !found {
pluralForm = pluralForms["default"]
}
index := pluralForm(n)
translations := slices.([]interface{})
translation = key
if len(translations) > index {
translation = translations[index].(string)
}
}
return fmt.Sprintf(translation, args...)
}

30
locale/locale.go Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package locale
import "log"
type Translation map[string]interface{}
type Locales map[string]Translation
func Load() *Translator {
translator := NewTranslator()
for language, translations := range Translations {
log.Println("Loading translation:", language)
translator.AddLanguage(language, translations)
}
return translator
}
// GetAvailableLanguages returns the list of available languages.
func GetAvailableLanguages() map[string]string {
return map[string]string{
"en_US": "English",
"fr_FR": "Français",
}
}

103
locale/locale_test.go Normal file
View file

@ -0,0 +1,103 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package locale
import "testing"
func TestTranslateWithMissingLanguage(t *testing.T) {
translator := NewTranslator()
translation := translator.GetLanguage("en_US").Get("auth.username")
if translation != "auth.username" {
t.Errorf("Wrong translation, got %s", translation)
}
}
func TestTranslateWithExistingKey(t *testing.T) {
data := `{"auth.username": "Username"}`
translator := NewTranslator()
translator.AddLanguage("en_US", data)
translation := translator.GetLanguage("en_US").Get("auth.username")
if translation != "Username" {
t.Errorf("Wrong translation, got %s", translation)
}
}
func TestTranslateWithMissingKey(t *testing.T) {
data := `{"auth.username": "Username"}`
translator := NewTranslator()
translator.AddLanguage("en_US", data)
translation := translator.GetLanguage("en_US").Get("auth.password")
if translation != "auth.password" {
t.Errorf("Wrong translation, got %s", translation)
}
}
func TestTranslateWithMissingKeyAndPlaceholder(t *testing.T) {
translator := NewTranslator()
translator.AddLanguage("fr_FR", "")
translation := translator.GetLanguage("fr_FR").Get("Status: %s", "ok")
if translation != "Status: ok" {
t.Errorf("Wrong translation, got %s", translation)
}
}
func TestTranslatePluralWithDefaultRule(t *testing.T) {
data := `{"number_of_users": ["Il y a %d utilisateur (%s)", "Il y a %d utilisateurs (%s)"]}`
translator := NewTranslator()
translator.AddLanguage("fr_FR", data)
language := translator.GetLanguage("fr_FR")
translation := language.Plural("number_of_users", 1, 1, "some text")
expected := "Il y a 1 utilisateur (some text)"
if translation != expected {
t.Errorf(`Wrong translation, got "%s" instead of "%s"`, translation, expected)
}
translation = language.Plural("number_of_users", 2, 2, "some text")
expected = "Il y a 2 utilisateurs (some text)"
if translation != expected {
t.Errorf(`Wrong translation, got "%s" instead of "%s"`, translation, expected)
}
}
func TestTranslatePluralWithRussianRule(t *testing.T) {
data := `{"key": ["из %d книги за %d день", "из %d книг за %d дня", "из %d книг за %d дней"]}`
translator := NewTranslator()
translator.AddLanguage("ru_RU", data)
language := translator.GetLanguage("ru_RU")
translation := language.Plural("key", 1, 1, 1)
expected := "из 1 книги за 1 день"
if translation != expected {
t.Errorf(`Wrong translation, got "%s" instead of "%s"`, translation, expected)
}
translation = language.Plural("key", 2, 2, 2)
expected = "из 2 книг за 2 дня"
if translation != expected {
t.Errorf(`Wrong translation, got "%s" instead of "%s"`, translation, expected)
}
translation = language.Plural("key", 5, 5, 5)
expected = "из 5 книг за 5 дней"
if translation != expected {
t.Errorf(`Wrong translation, got "%s" instead of "%s"`, translation, expected)
}
}
func TestTranslatePluralWithMissingTranslation(t *testing.T) {
translator := NewTranslator()
translator.AddLanguage("fr_FR", "")
language := translator.GetLanguage("fr_FR")
translation := language.Plural("number_of_users", 2)
expected := "number_of_users"
if translation != expected {
t.Errorf(`Wrong translation, got "%s" instead of "%s"`, translation, expected)
}
}

101
locale/plurals.go Normal file
View file

@ -0,0 +1,101 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package locale
// See https://localization-guide.readthedocs.io/en/latest/l10n/pluralforms.html
// And http://www.unicode.org/cldr/charts/29/supplemental/language_plural_rules.html
var pluralForms = map[string]func(n int) int{
// nplurals=2; plural=(n != 1);
"default": func(n int) int {
if n != 1 {
return 1
}
return 0
},
// nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5);
"ar_AR": func(n int) int {
if n == 0 {
return 0
}
if n == 1 {
return 1
}
if n == 2 {
return 2
}
if n%100 >= 3 && n%100 <= 10 {
return 3
}
if n%100 >= 11 {
return 4
}
return 5
},
// nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;
"cs_CZ": func(n int) int {
if n == 1 {
return 0
}
if n >= 2 && n <= 4 {
return 1
}
return 2
},
// nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);
"pl_PL": func(n int) int {
if n == 1 {
return 0
}
if n%10 >= 2 && n%10 <= 4 && (n%100 < 10 || n%100 >= 20) {
return 1
}
return 2
},
// nplurals=2; plural=(n > 1);
"pt_BR": func(n int) int {
if n > 1 {
return 1
}
return 0
},
// nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);
"ru_RU": func(n int) int {
if n%10 == 1 && n%100 != 11 {
return 0
}
if n%10 >= 2 && n%10 <= 4 && (n%100 < 10 || n%100 >= 20) {
return 1
}
return 2
},
// nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);
"sr_RS": func(n int) int {
if n%10 == 1 && n%100 != 11 {
return 0
}
if n%10 >= 2 && n%10 <= 4 && (n%100 < 10 || n%100 >= 20) {
return 1
}
return 2
},
// nplurals=1; plural=0;
"zh_CN": func(n int) int {
return 0
},
}

136
locale/translations.go Normal file
View file

@ -0,0 +1,136 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-19 22:01:21.925268372 -0800 PST m=+0.006101515
package locale
var Translations = map[string]string{
"en_US": `{
"plural.feed.error_count": [
"%d error",
"%d errors"
],
"plural.categories.feed_count": [
"There is %d feed.",
"There are %d feeds."
]
}`,
"fr_FR": `{
"plural.feed.error_count": [
"%d erreur",
"%d erreurs"
],
"plural.categories.feed_count": [
"Il y %d abonnement.",
"Il y %d abonnements."
],
"Username": "Nom d'utilisateur",
"Password": "Mot de passe",
"Unread": "Non lus",
"History": "Historique",
"Feeds": "Abonnements",
"Categories": "Catégories",
"Settings": "Réglages",
"Logout": "Se déconnecter",
"Next": "Suivant",
"Previous": "Précédent",
"New Subscription": "Nouvel Abonnment",
"Import": "Importation",
"Export": "Exportation",
"There is no category. You must have at least one category.": "Il n'y a aucune catégorie. Vous devez avoir au moins une catégorie.",
"URL": "URL",
"Category": "Catégorie",
"Find a subscription": "Trouver un abonnement",
"Loading...": "Chargement...",
"Create a category": "Créer une catégorie",
"There is no category.": "Il n'y a aucune catégorie.",
"Edit": "Modifier",
"Remove": "Supprimer",
"No feed.": "Aucun abonnement.",
"There is no article in this category.": "Il n'y a aucun article dans cette catégorie.",
"Original": "Original",
"Mark this page as read": "Marquer cette page comme lu",
"not yet": "pas encore",
"just now": "à l'instant",
"1 minute ago": "il y a une minute",
"%d minutes ago": "il y a %d minutes",
"1 hour ago": "il y a une heure",
"%d hours ago": "il y a %d heures",
"yesterday": "hier",
"%d days ago": "il y a %d jours",
"%d weeks ago": "il y a %d semaines",
"%d months ago": "il y a %d mois",
"%d years ago": "il y a %d années",
"Date": "Date",
"IP Address": "Adresse IP",
"User Agent": "Navigateur Web",
"Actions": "Actions",
"Current session": "Session actuelle",
"Sessions": "Sessions",
"Users": "Utilisateurs",
"Add user": "Ajouter un utilisateur",
"Choose a Subscription": "Choisissez un abonnement",
"Subscribe": "S'abonner",
"New Category": "Nouvelle Catégorie",
"Title": "Titre",
"Save": "Sauvegarder",
"or": "ou",
"cancel": "annuler",
"New User": "Nouvel Utilisateur",
"Confirmation": "Confirmation",
"Administrator": "Administrateur",
"Edit Category: %s": "Modification de la catégorie : %s",
"Update": "Mettre à jour",
"Edit Feed: %s": "Modification de l'abonnement : %s",
"There is no category!": "Il n'y a aucune catégorie !",
"Edit user: %s": "Modification de l'utilisateur : %s",
"There is no article for this feed.": "Il n'y a aucun article pour cet abonnement.",
"Add subscription": "Ajouter un abonnement",
"You don't have any subscription.": "Vous n'avez aucun abonnement",
"Last check:": "Dernière vérification :",
"Refresh": "Actualiser",
"There is no history at the moment.": "Il n'y a aucun historique pour le moment.",
"OPML file": "Fichier OPML",
"Sign In": "Connexion",
"Sign in": "Connexion",
"Theme": "Thème",
"Timezone": "Fuseau horaire",
"Language": "Langue",
"There is no unread article.": "Il n'y a rien de nouveau à lire.",
"You are the only user.": "Vous êtes le seul utilisateur.",
"Last Login": "Dernière connexion",
"Yes": "Oui",
"No": "Non",
"This feed already exists (%s).": "Cet abonnement existe déjà (%s).",
"Unable to fetch feed (statusCode=%d).": "Impossible de récupérer cet abonnement (code=%d).",
"Unable to open this link: %v": "Impossible d'ouvrir ce lien : %v",
"Unable to analyze this page: %v": "Impossible d'analyzer cette page : %v",
"Unable to find any subscription.": "Impossible de trouver un abonnement.",
"The URL and the category are mandatory.": "L'URL et la catégorie sont obligatoire.",
"All fields are mandatory.": "Tous les champs sont obligatoire.",
"Passwords are not the same.": "Les mots de passe ne sont pas les mêmes.",
"You must use at least 6 characters.": "Vous devez utiliser au moins 6 caractères.",
"The username is mandatory.": "Le nom d'utilisateur est obligatoire.",
"The username, theme, language and timezone fields are mandatory.": "Le nom d'utilisateur, le thème, la langue et le fuseau horaire sont obligatoire.",
"The title is mandatory.": "Le titre est obligatoire.",
"About": "A propos",
"version": "Version",
"Version:": "Version :",
"Build Date:": "Date de la compilation :",
"Author:": "Auteur :",
"Authors": "Auteurs",
"License:": "Licence :",
"Attachments": "Pièces jointes",
"Download": "Télécharger",
"Invalid username or password.": "Mauvais identifiant ou mot de passe.",
"Never": "Jamais",
"Unable to execute request: %v": "Impossible d'exécuter cette requête: %v",
"Last Parsing Error": "Dernière erreur d'analyse",
"There is a problem with this feed": "Il y a un problème avec cet abonnement"
}
`,
}
var TranslationsChecksums = map[string]string{
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
"fr_FR": "1f75e5a4b581755f7f84687126bc5b96aaf0109a2f83a72a8770c2ad3ddb7ba3",
}

10
locale/translations/en_US.json Normal file
View file

@ -0,0 +1,10 @@
{
"plural.feed.error_count": [
"%d error",
"%d errors"
],
"plural.categories.feed_count": [
"There is %d feed.",
"There are %d feeds."
]
}

113
locale/translations/fr_FR.json Normal file
View file

@ -0,0 +1,113 @@
{
"plural.feed.error_count": [
"%d erreur",
"%d erreurs"
],
"plural.categories.feed_count": [
"Il y %d abonnement.",
"Il y %d abonnements."
],
"Username": "Nom d'utilisateur",
"Password": "Mot de passe",
"Unread": "Non lus",
"History": "Historique",
"Feeds": "Abonnements",
"Categories": "Catégories",
"Settings": "Réglages",
"Logout": "Se déconnecter",
"Next": "Suivant",
"Previous": "Précédent",
"New Subscription": "Nouvel Abonnment",
"Import": "Importation",
"Export": "Exportation",
"There is no category. You must have at least one category.": "Il n'y a aucune catégorie. Vous devez avoir au moins une catégorie.",
"URL": "URL",
"Category": "Catégorie",
"Find a subscription": "Trouver un abonnement",
"Loading...": "Chargement...",
"Create a category": "Créer une catégorie",
"There is no category.": "Il n'y a aucune catégorie.",
"Edit": "Modifier",
"Remove": "Supprimer",
"No feed.": "Aucun abonnement.",
"There is no article in this category.": "Il n'y a aucun article dans cette catégorie.",
"Original": "Original",
"Mark this page as read": "Marquer cette page comme lu",
"not yet": "pas encore",
"just now": "à l'instant",
"1 minute ago": "il y a une minute",
"%d minutes ago": "il y a %d minutes",
"1 hour ago": "il y a une heure",
"%d hours ago": "il y a %d heures",
"yesterday": "hier",
"%d days ago": "il y a %d jours",
"%d weeks ago": "il y a %d semaines",
"%d months ago": "il y a %d mois",
"%d years ago": "il y a %d années",
"Date": "Date",
"IP Address": "Adresse IP",
"User Agent": "Navigateur Web",
"Actions": "Actions",
"Current session": "Session actuelle",
"Sessions": "Sessions",
"Users": "Utilisateurs",
"Add user": "Ajouter un utilisateur",
"Choose a Subscription": "Choisissez un abonnement",
"Subscribe": "S'abonner",
"New Category": "Nouvelle Catégorie",
"Title": "Titre",
"Save": "Sauvegarder",
"or": "ou",
"cancel": "annuler",
"New User": "Nouvel Utilisateur",
"Confirmation": "Confirmation",
"Administrator": "Administrateur",
"Edit Category: %s": "Modification de la catégorie : %s",
"Update": "Mettre à jour",
"Edit Feed: %s": "Modification de l'abonnement : %s",
"There is no category!": "Il n'y a aucune catégorie !",
"Edit user: %s": "Modification de l'utilisateur : %s",
"There is no article for this feed.": "Il n'y a aucun article pour cet abonnement.",
"Add subscription": "Ajouter un abonnement",
"You don't have any subscription.": "Vous n'avez aucun abonnement",
"Last check:": "Dernière vérification :",
"Refresh": "Actualiser",
"There is no history at the moment.": "Il n'y a aucun historique pour le moment.",
"OPML file": "Fichier OPML",
"Sign In": "Connexion",
"Sign in": "Connexion",
"Theme": "Thème",
"Timezone": "Fuseau horaire",
"Language": "Langue",
"There is no unread article.": "Il n'y a rien de nouveau à lire.",
"You are the only user.": "Vous êtes le seul utilisateur.",
"Last Login": "Dernière connexion",
"Yes": "Oui",
"No": "Non",
"This feed already exists (%s).": "Cet abonnement existe déjà (%s).",
"Unable to fetch feed (statusCode=%d).": "Impossible de récupérer cet abonnement (code=%d).",
"Unable to open this link: %v": "Impossible d'ouvrir ce lien : %v",
"Unable to analyze this page: %v": "Impossible d'analyzer cette page : %v",
"Unable to find any subscription.": "Impossible de trouver un abonnement.",
"The URL and the category are mandatory.": "L'URL et la catégorie sont obligatoire.",
"All fields are mandatory.": "Tous les champs sont obligatoire.",
"Passwords are not the same.": "Les mots de passe ne sont pas les mêmes.",
"You must use at least 6 characters.": "Vous devez utiliser au moins 6 caractères.",
"The username is mandatory.": "Le nom d'utilisateur est obligatoire.",
"The username, theme, language and timezone fields are mandatory.": "Le nom d'utilisateur, le thème, la langue et le fuseau horaire sont obligatoire.",
"The title is mandatory.": "Le titre est obligatoire.",
"About": "A propos",
"version": "Version",
"Version:": "Version :",
"Build Date:": "Date de la compilation :",
"Author:": "Auteur :",
"Authors": "Auteurs",
"License:": "Licence :",
"Attachments": "Pièces jointes",
"Download": "Télécharger",
"Invalid username or password.": "Mauvais identifiant ou mot de passe.",
"Never": "Jamais",
"Unable to execute request: %v": "Impossible d'exécuter cette requête: %v",
"Last Parsing Error": "Dernière erreur d'analyse",
"There is a problem with this feed": "Il y a un problème avec cet abonnement"
}

40
locale/translator.go Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package locale
import (
"encoding/json"
"fmt"
"strings"
)
type Translator struct {
Locales Locales
}
func (t *Translator) AddLanguage(language, translations string) error {
var decodedTranslations Translation
decoder := json.NewDecoder(strings.NewReader(translations))
if err := decoder.Decode(&decodedTranslations); err != nil {
return fmt.Errorf("Invalid JSON file: %v", err)
}
t.Locales[language] = decodedTranslations
return nil
}
func (t *Translator) GetLanguage(language string) *Language {
translations, found := t.Locales[language]
if !found {
return &Language{language: language}
}
return &Language{language: language, translations: translations}
}
func NewTranslator() *Translator {
return &Translator{Locales: make(Locales)}
}

124
main.go Normal file
View file

@ -0,0 +1,124 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package main
//go:generate go run generate.go
import (
"bufio"
"context"
"flag"
"fmt"
"github.com/miniflux/miniflux2/config"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/scheduler"
"github.com/miniflux/miniflux2/server"
"github.com/miniflux/miniflux2/storage"
"github.com/miniflux/miniflux2/version"
"log"
"os"
"os/signal"
"runtime"
"strings"
"time"
_ "github.com/lib/pq"
"golang.org/x/crypto/ssh/terminal"
)
func run(cfg *config.Config, store *storage.Storage) {
log.Println("Starting Miniflux...")
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt)
feedHandler := feed.NewFeedHandler(store)
server := server.NewServer(cfg, store, feedHandler)
go func() {
pool := scheduler.NewWorkerPool(feedHandler, cfg.GetInt("WORKER_POOL_SIZE", 5))
scheduler.NewScheduler(store, pool, cfg.GetInt("POLLING_FREQUENCY", 30), cfg.GetInt("BATCH_SIZE", 10))
}()
<-stop
log.Println("Shutting down the server...")
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
server.Shutdown(ctx)
store.Close()
log.Println("Server gracefully stopped")
}
func askCredentials() (string, string) {
reader := bufio.NewReader(os.Stdin)
fmt.Print("Enter Username: ")
username, _ := reader.ReadString('\n')
fmt.Print("Enter Password: ")
bytePassword, _ := terminal.ReadPassword(0)
fmt.Printf("\n")
return strings.TrimSpace(username), strings.TrimSpace(string(bytePassword))
}
func main() {
flagInfo := flag.Bool("info", false, "Show application information")
flagVersion := flag.Bool("version", false, "Show application version")
flagMigrate := flag.Bool("migrate", false, "Migrate database schema")
flagFlushSessions := flag.Bool("flush-sessions", false, "Flush all sessions (disconnect users)")
flagCreateAdmin := flag.Bool("create-admin", false, "Create admin user")
flag.Parse()
cfg := config.NewConfig()
store := storage.NewStorage(
cfg.Get("DATABASE_URL", "postgres://postgres:postgres@localhost/miniflux2?sslmode=disable"),
cfg.GetInt("DATABASE_MAX_CONNS", 20),
)
if *flagInfo {
fmt.Println("Version:", version.Version)
fmt.Println("Build Date:", version.BuildDate)
fmt.Println("Go Version:", runtime.Version())
return
}
if *flagVersion {
fmt.Println(version.Version)
return
}
if *flagMigrate {
store.Migrate()
return
}
if *flagFlushSessions {
fmt.Println("Flushing all sessions (disconnect users)")
if err := store.FlushAllSessions(); err != nil {
fmt.Println(err)
os.Exit(1)
}
return
}
if *flagCreateAdmin {
user := &model.User{IsAdmin: true}
user.Username, user.Password = askCredentials()
if err := user.ValidateUserCreation(); err != nil {
fmt.Println(err)
os.Exit(1)
}
if err := store.CreateUser(user); err != nil {
fmt.Println(err)
os.Exit(1)
}
return
}
run(cfg, store)
}

51
model/category.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
import (
"errors"
"fmt"
)
type Category struct {
ID int64 `json:"id,omitempty"`
Title string `json:"title,omitempty"`
UserID int64 `json:"user_id,omitempty"`
FeedCount int `json:"nb_feeds,omitempty"`
}
func (c *Category) String() string {
return fmt.Sprintf("ID=%d, UserID=%d, Title=%s", c.ID, c.UserID, c.Title)
}
func (c Category) ValidateCategoryCreation() error {
if c.Title == "" {
return errors.New("The title is mandatory")
}
if c.UserID == 0 {
return errors.New("The userID is mandatory")
}
return nil
}
func (c Category) ValidateCategoryModification() error {
if c.Title == "" {
return errors.New("The title is mandatory")
}
if c.UserID == 0 {
return errors.New("The userID is mandatory")
}
if c.ID == 0 {
return errors.New("The ID is mandatory")
}
return nil
}
type Categories []*Category

18
model/enclosure.go Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
// Enclosure represents an attachment.
type Enclosure struct {
ID int64 `json:"id"`
UserID int64 `json:"user_id"`
EntryID int64 `json:"entry_id"`
URL string `json:"url"`
MimeType string `json:"mime_type"`
Size int `json:"size"`
}
// EnclosureList represents a list of attachments.
type EnclosureList []*Enclosure

71
model/entry.go Normal file
View file

@ -0,0 +1,71 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
import (
"fmt"
"time"
)
const (
EntryStatusUnread = "unread"
EntryStatusRead = "read"
EntryStatusRemoved = "removed"
DefaultSortingOrder = "published_at"
DefaultSortingDirection = "desc"
)
type Entry struct {
ID int64 `json:"id"`
UserID int64 `json:"user_id"`
FeedID int64 `json:"feed_id"`
Status string `json:"status"`
Hash string `json:"hash"`
Title string `json:"title"`
URL string `json:"url"`
Date time.Time `json:"published_at"`
Content string `json:"content"`
Author string `json:"author"`
Enclosures EnclosureList `json:"enclosures,omitempty"`
Feed *Feed `json:"feed,omitempty"`
Category *Category `json:"category,omitempty"`
}
type Entries []*Entry
func ValidateEntryStatus(status string) error {
switch status {
case EntryStatusRead, EntryStatusUnread, EntryStatusRemoved:
return nil
}
return fmt.Errorf(`Invalid entry status, valid status values are: "%s", "%s" and "%s"`, EntryStatusRead, EntryStatusUnread, EntryStatusRemoved)
}
func ValidateEntryOrder(order string) error {
switch order {
case "id", "status", "published_at", "category_title", "category_id":
return nil
}
return fmt.Errorf(`Invalid entry order, valid order values are: "id", "status", "published_at", "category_title", "category_id"`)
}
func ValidateDirection(direction string) error {
switch direction {
case "asc", "desc":
return nil
}
return fmt.Errorf(`Invalid direction, valid direction values are: "asc" or "desc"`)
}
func GetOppositeDirection(direction string) string {
if direction == "asc" {
return "desc"
}
return "asc"
}

66
model/feed.go Normal file
View file

@ -0,0 +1,66 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
import (
"fmt"
"reflect"
"time"
)
// Feed represents a feed in the database
type Feed struct {
ID int64 `json:"id"`
UserID int64 `json:"user_id"`
FeedURL string `json:"feed_url"`
SiteURL string `json:"site_url"`
Title string `json:"title"`
CheckedAt time.Time `json:"checked_at,omitempty"`
EtagHeader string `json:"etag_header,omitempty"`
LastModifiedHeader string `json:"last_modified_header,omitempty"`
ParsingErrorMsg string `json:"parsing_error_message,omitempty"`
ParsingErrorCount int `json:"parsing_error_count,omitempty"`
Category *Category `json:"category,omitempty"`
Entries Entries `json:"entries,omitempty"`
Icon *FeedIcon `json:"icon,omitempty"`
}
func (f *Feed) String() string {
return fmt.Sprintf("ID=%d, UserID=%d, FeedURL=%s, SiteURL=%s, Title=%s, Category={%s}",
f.ID,
f.UserID,
f.FeedURL,
f.SiteURL,
f.Title,
f.Category,
)
}
// Merge combine src to the current struct
func (f *Feed) Merge(src *Feed) {
src.ID = f.ID
src.UserID = f.UserID
new := reflect.ValueOf(src).Elem()
for i := 0; i < new.NumField(); i++ {
field := new.Field(i)
switch field.Interface().(type) {
case int64:
value := field.Int()
if value != 0 {
reflect.ValueOf(f).Elem().Field(i).SetInt(value)
}
case string:
value := field.String()
if value != "" {
reflect.ValueOf(f).Elem().Field(i).SetString(value)
}
}
}
}
// Feeds is a list of feed
type Feeds []*Feed

19
model/icon.go Normal file
View file

@ -0,0 +1,19 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
// Icon represents a website icon (favicon)
type Icon struct {
ID int64 `json:"id"`
Hash string `json:"hash"`
MimeType string `json:"mime_type"`
Content []byte `json:"content"`
}
// FeedIcon is a jonction table between feeds and icons
type FeedIcon struct {
FeedID int64 `json:"feed_id"`
IconID int64 `json:"icon_id"`
}

10
model/job.go Normal file
View file

@ -0,0 +1,10 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
type Job struct {
UserID int64
FeedID int64
}

23
model/session.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
import "time"
import "fmt"
type Session struct {
ID int64
UserID int64
Token string
CreatedAt time.Time
UserAgent string
IP string
}
func (s *Session) String() string {
return fmt.Sprintf("ID=%d, UserID=%d, IP=%s", s.ID, s.UserID, s.IP)
}
type Sessions []*Session

13
model/theme.go Normal file
View file

@ -0,0 +1,13 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
// GetThemes returns the list of available themes.
func GetThemes() map[string]string {
return map[string]string{
"default": "Default",
"black": "Black",
}
}

96
model/user.go Normal file
View file

@ -0,0 +1,96 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package model
import (
"errors"
"time"
)
// User represents a user in the system.
type User struct {
ID int64 `json:"id"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
IsAdmin bool `json:"is_admin"`
Theme string `json:"theme"`
Language string `json:"language"`
Timezone string `json:"timezone"`
LastLoginAt *time.Time `json:"last_login_at"`
}
func (u User) ValidateUserCreation() error {
if err := u.ValidateUserLogin(); err != nil {
return err
}
if err := u.ValidatePassword(); err != nil {
return err
}
return nil
}
func (u User) ValidateUserModification() error {
if u.Username == "" {
return errors.New("The username is mandatory")
}
if err := u.ValidatePassword(); err != nil {
return err
}
return nil
}
func (u User) ValidateUserLogin() error {
if u.Username == "" {
return errors.New("The username is mandatory")
}
if u.Password == "" {
return errors.New("The password is mandatory")
}
return nil
}
func (u User) ValidatePassword() error {
if u.Password != "" && len(u.Password) < 6 {
return errors.New("The password must have at least 6 characters")
}
return nil
}
// Merge update the current user with another user.
func (u *User) Merge(override *User) {
if u.Username != override.Username {
u.Username = override.Username
}
if u.Password != override.Password {
u.Password = override.Password
}
if u.IsAdmin != override.IsAdmin {
u.IsAdmin = override.IsAdmin
}
if u.Theme != override.Theme {
u.Theme = override.Theme
}
if u.Language != override.Language {
u.Language = override.Language
}
if u.Timezone != override.Timezone {
u.Timezone = override.Timezone
}
}
// Users represents a list of users.
type Users []*User

214
reader/feed/atom/atom.go Normal file
View file

@ -0,0 +1,214 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package atom
import (
"encoding/xml"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
"log"
"strconv"
"strings"
"time"
)
type AtomFeed struct {
XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"`
ID string `xml:"id"`
Title string `xml:"title"`
Author Author `xml:"author"`
Links []Link `xml:"link"`
Entries []AtomEntry `xml:"entry"`
}
type AtomEntry struct {
ID string `xml:"id"`
Title string `xml:"title"`
Updated string `xml:"updated"`
Links []Link `xml:"link"`
Summary string `xml:"summary"`
Content Content `xml:"content"`
MediaGroup MediaGroup `xml:"http://search.yahoo.com/mrss/ group"`
Author Author `xml:"author"`
}
type Author struct {
Name string `xml:"name"`
Email string `xml:"email"`
}
type Link struct {
Url string `xml:"href,attr"`
Type string `xml:"type,attr"`
Rel string `xml:"rel,attr"`
Length string `xml:"length,attr"`
}
type Content struct {
Type string `xml:"type,attr"`
Data string `xml:",chardata"`
Xml string `xml:",innerxml"`
}
type MediaGroup struct {
Description string `xml:"http://search.yahoo.com/mrss/ description"`
}
func (a *AtomFeed) getSiteURL() string {
for _, link := range a.Links {
if strings.ToLower(link.Rel) == "alternate" {
return link.Url
}
if link.Rel == "" && link.Type == "" {
return link.Url
}
}
return ""
}
func (a *AtomFeed) getFeedURL() string {
for _, link := range a.Links {
if strings.ToLower(link.Rel) == "self" {
return link.Url
}
}
return ""
}
func (a *AtomFeed) Transform() *model.Feed {
feed := new(model.Feed)
feed.FeedURL = a.getFeedURL()
feed.SiteURL = a.getSiteURL()
feed.Title = sanitizer.StripTags(a.Title)
if feed.Title == "" {
feed.Title = feed.SiteURL
}
for _, entry := range a.Entries {
item := entry.Transform()
if item.Author == "" {
item.Author = a.GetAuthor()
}
feed.Entries = append(feed.Entries, item)
}
return feed
}
func (a *AtomFeed) GetAuthor() string {
return getAuthor(a.Author)
}
func (e *AtomEntry) GetDate() time.Time {
if e.Updated != "" {
result, err := date.Parse(e.Updated)
if err != nil {
log.Println(err)
return time.Now()
}
return result
}
return time.Now()
}
func (e *AtomEntry) GetURL() string {
for _, link := range e.Links {
if strings.ToLower(link.Rel) == "alternate" {
return link.Url
}
if link.Rel == "" && link.Type == "" {
return link.Url
}
}
return ""
}
func (e *AtomEntry) GetAuthor() string {
return getAuthor(e.Author)
}
func (e *AtomEntry) GetHash() string {
for _, value := range []string{e.ID, e.GetURL()} {
if value != "" {
return helper.Hash(value)
}
}
return ""
}
func (e *AtomEntry) GetContent() string {
if e.Content.Type == "html" || e.Content.Type == "text" {
return e.Content.Data
}
if e.Content.Type == "xhtml" {
return e.Content.Xml
}
if e.Summary != "" {
return e.Summary
}
if e.MediaGroup.Description != "" {
return e.MediaGroup.Description
}
return ""
}
func (e *AtomEntry) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
for _, link := range e.Links {
if strings.ToLower(link.Rel) == "enclosure" {
length, _ := strconv.Atoi(link.Length)
enclosures = append(enclosures, &model.Enclosure{URL: link.Url, MimeType: link.Type, Size: length})
}
}
return enclosures
}
func (e *AtomEntry) Transform() *model.Entry {
entry := new(model.Entry)
entry.URL = e.GetURL()
entry.Date = e.GetDate()
entry.Author = sanitizer.StripTags(e.GetAuthor())
entry.Hash = e.GetHash()
entry.Content = processor.ItemContentProcessor(entry.URL, e.GetContent())
entry.Title = sanitizer.StripTags(strings.Trim(e.Title, " \n\t"))
entry.Enclosures = e.GetEnclosures()
if entry.Title == "" {
entry.Title = entry.URL
}
return entry
}
func getAuthor(author Author) string {
if author.Name != "" {
return author.Name
}
if author.Email != "" {
return author.Email
}
return ""
}

28
reader/feed/atom/parser.go Normal file
View file

@ -0,0 +1,28 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package atom
import (
"encoding/xml"
"fmt"
"github.com/miniflux/miniflux2/model"
"io"
"golang.org/x/net/html/charset"
)
// Parse returns a normalized feed struct.
func Parse(data io.Reader) (*model.Feed, error) {
atomFeed := new(AtomFeed)
decoder := xml.NewDecoder(data)
decoder.CharsetReader = charset.NewReaderLabel
err := decoder.Decode(atomFeed)
if err != nil {
return nil, fmt.Errorf("Unable to parse Atom feed: %v\n", err)
}
return atomFeed.Transform(), nil
}

319
reader/feed/atom/parser_test.go Normal file
View file

@ -0,0 +1,319 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package atom
import (
"bytes"
"testing"
"time"
)
func TestParseAtomSample(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<author>
<name>John Doe</name>
</author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "Example Feed" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
if feed.FeedURL != "" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
if feed.SiteURL != "http://example.org/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if !feed.Entries[0].Date.Equal(time.Date(2003, time.December, 13, 18, 30, 2, 0, time.UTC)) {
t.Errorf("Incorrect entry date, got: %v", feed.Entries[0].Date)
}
if feed.Entries[0].Hash != "3841e5cf232f5111fc5841e9eba5f4b26d95e7d7124902e0f7272729d65601a6" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
if feed.Entries[0].URL != "http://example.org/2003/12/13/atom03" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Title != "Atom-Powered Robots Run Amok" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
if feed.Entries[0].Content != "Some text." {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].Content)
}
if feed.Entries[0].Author != "John Doe" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseFeedWithoutTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<link rel="alternate" type="text/html" href="https://example.org/"/>
<link rel="self" type="application/atom+xml" href="https://example.org/feed"/>
<updated>2003-12-13T18:30:02Z</updated>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "https://example.org/" {
t.Errorf("Incorrect feed title, got: %s", feed.Title)
}
}
func TestParseEntryWithoutTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<author>
<name>John Doe</name>
</author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Title != "http://example.org/2003/12/13/atom03" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
}
func TestParseFeedURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link rel="alternate" type="text/html" href="https://example.org/"/>
<link rel="self" type="application/atom+xml" href="https://example.org/feed"/>
<updated>2003-12-13T18:30:02Z</updated>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.SiteURL != "https://example.org/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
if feed.FeedURL != "https://example.org/feed" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
}
func TestParseEntryTitleWithWhitespaces(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<title>
Some Title
</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Title != "Some Title" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
}
func TestParseEntryWithAuthorName(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<author>
<name>Me</name>
<email>me@localhost</email>
</author>
</entry>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Author != "Me" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseEntryWithoutAuthorName(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<author>
<name/>
<email>me@localhost</email>
</author>
</entry>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Author != "me@localhost" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseEntryWithEnclosures(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<id>http://www.example.org/myfeed</id>
<title>My Podcast Feed</title>
<updated>2005-07-15T12:00:00Z</updated>
<author>
<name>John Doe</name>
</author>
<link href="http://example.org" />
<link rel="self" href="http://example.org/myfeed" />
<entry>
<id>http://www.example.org/entries/1</id>
<title>Atom 1.0</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="http://www.example.org/entries/1" />
<summary>An overview of Atom 1.0</summary>
<link rel="enclosure"
type="audio/mpeg"
title="MP3"
href="http://www.example.org/myaudiofile.mp3"
length="1234" />
<link rel="enclosure"
type="application/x-bittorrent"
title="BitTorrent"
href="http://www.example.org/myaudiofile.torrent"
length="4567" />
<content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml">
<h1>Show Notes</h1>
<ul>
<li>00:01:00 -- Introduction</li>
<li>00:15:00 -- Talking about Atom 1.0</li>
<li>00:30:00 -- Wrapping up</li>
</ul>
</div>
</content>
</entry>
</feed>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if len(feed.Entries[0].Enclosures) != 2 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
if feed.Entries[0].Enclosures[0].MimeType != "audio/mpeg" {
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType)
}
if feed.Entries[0].Enclosures[0].Size != 1234 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
if feed.Entries[0].Enclosures[1].URL != "http://www.example.org/myaudiofile.torrent" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[1].URL)
}
if feed.Entries[0].Enclosures[1].MimeType != "application/x-bittorrent" {
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[1].MimeType)
}
if feed.Entries[0].Enclosures[1].Size != 4567 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[1].Size)
}
}

203
reader/feed/date/parser.go Normal file
View file

@ -0,0 +1,203 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package date
import (
"fmt"
"strings"
"time"
)
// DateFormats taken from github.com/mjibson/goread
var dateFormats = []string{
time.RFC822, // RSS
time.RFC822Z, // RSS
time.RFC3339, // Atom
time.UnixDate,
time.RubyDate,
time.RFC850,
time.RFC1123Z,
time.RFC1123,
time.ANSIC,
"Mon, January 2 2006 15:04:05 -0700",
"Mon, January 02, 2006, 15:04:05 MST",
"Mon, January 02, 2006 15:04:05 MST",
"Mon, Jan 2, 2006 15:04 MST",
"Mon, Jan 2 2006 15:04 MST",
"Mon, Jan 2, 2006 15:04:05 MST",
"Mon, Jan 2 2006 15:04:05 -700",
"Mon, Jan 2 2006 15:04:05 -0700",
"Mon Jan 2 15:04 2006",
"Mon Jan 2 15:04:05 2006 MST",
"Mon Jan 02, 2006 3:04 pm",
"Mon, Jan 02,2006 15:04:05 MST",
"Mon Jan 02 2006 15:04:05 -0700",
"Monday, January 2, 2006 15:04:05 MST",
"Monday, January 2, 2006 03:04 PM",
"Monday, January 2, 2006",
"Monday, January 02, 2006",
"Monday, 2 January 2006 15:04:05 MST",
"Monday, 2 January 2006 15:04:05 -0700",
"Monday, 2 Jan 2006 15:04:05 MST",
"Monday, 2 Jan 2006 15:04:05 -0700",
"Monday, 02 January 2006 15:04:05 MST",
"Monday, 02 January 2006 15:04:05 -0700",
"Monday, 02 January 2006 15:04:05",
"Mon, 2 January 2006 15:04 MST",
"Mon, 2 January 2006, 15:04 -0700",
"Mon, 2 January 2006, 15:04:05 MST",
"Mon, 2 January 2006 15:04:05 MST",
"Mon, 2 January 2006 15:04:05 -0700",
"Mon, 2 January 2006",
"Mon, 2 Jan 2006 3:04:05 PM -0700",
"Mon, 2 Jan 2006 15:4:5 MST",
"Mon, 2 Jan 2006 15:4:5 -0700 GMT",
"Mon, 2, Jan 2006 15:4",
"Mon, 2 Jan 2006 15:04 MST",
"Mon, 2 Jan 2006, 15:04 -0700",
"Mon, 2 Jan 2006 15:04 -0700",
"Mon, 2 Jan 2006 15:04:05 UT",
"Mon, 2 Jan 2006 15:04:05MST",
"Mon, 2 Jan 2006 15:04:05 MST",
"Mon 2 Jan 2006 15:04:05 MST",
"mon,2 Jan 2006 15:04:05 MST",
"Mon, 2 Jan 2006 15:04:05 -0700 MST",
"Mon, 2 Jan 2006 15:04:05-0700",
"Mon, 2 Jan 2006 15:04:05 -0700",
"Mon, 2 Jan 2006 15:04:05",
"Mon, 2 Jan 2006 15:04",
"Mon,2 Jan 2006",
"Mon, 2 Jan 2006",
"Mon, 2 Jan 15:04:05 MST",
"Mon, 2 Jan 06 15:04:05 MST",
"Mon, 2 Jan 06 15:04:05 -0700",
"Mon, 2006-01-02 15:04",
"Mon,02 January 2006 14:04:05 MST",
"Mon, 02 January 2006",
"Mon, 02 Jan 2006 3:04:05 PM MST",
"Mon, 02 Jan 2006 15 -0700",
"Mon,02 Jan 2006 15:04 MST",
"Mon, 02 Jan 2006 15:04 MST",
"Mon, 02 Jan 2006 15:04 -0700",
"Mon, 02 Jan 2006 15:04:05 Z",
"Mon, 02 Jan 2006 15:04:05 UT",
"Mon, 02 Jan 2006 15:04:05 MST-07:00",
"Mon, 02 Jan 2006 15:04:05 MST -0700",
"Mon, 02 Jan 2006, 15:04:05 MST",
"Mon, 02 Jan 2006 15:04:05MST",
"Mon, 02 Jan 2006 15:04:05 MST",
"Mon , 02 Jan 2006 15:04:05 MST",
"Mon, 02 Jan 2006 15:04:05 GMT-0700",
"Mon,02 Jan 2006 15:04:05 -0700",
"Mon, 02 Jan 2006 15:04:05 -0700",
"Mon, 02 Jan 2006 15:04:05 -07:00",
"Mon, 02 Jan 2006 15:04:05 --0700",
"Mon 02 Jan 2006 15:04:05 -0700",
"Mon, 02 Jan 2006 15:04:05 -07",
"Mon, 02 Jan 2006 15:04:05 00",
"Mon, 02 Jan 2006 15:04:05",
"Mon, 02 Jan 2006",
"Mon, 02 Jan 06 15:04:05 MST",
"January 2, 2006 3:04 PM",
"January 2, 2006, 3:04 p.m.",
"January 2, 2006 15:04:05 MST",
"January 2, 2006 15:04:05",
"January 2, 2006 03:04 PM",
"January 2, 2006",
"January 02, 2006 15:04:05 MST",
"January 02, 2006 15:04",
"January 02, 2006 03:04 PM",
"January 02, 2006",
"Jan 2, 2006 3:04:05 PM MST",
"Jan 2, 2006 3:04:05 PM",
"Jan 2, 2006 15:04:05 MST",
"Jan 2, 2006",
"Jan 02 2006 03:04:05PM",
"Jan 02, 2006",
"6/1/2 15:04",
"6-1-2 15:04",
"2 January 2006 15:04:05 MST",
"2 January 2006 15:04:05 -0700",
"2 January 2006",
"2 Jan 2006 15:04:05 Z",
"2 Jan 2006 15:04:05 MST",
"2 Jan 2006 15:04:05 -0700",
"2 Jan 2006",
"2.1.2006 15:04:05",
"2/1/2006",
"2-1-2006",
"2006 January 02",
"2006-1-2T15:04:05Z",
"2006-1-2 15:04:05",
"2006-1-2",
"2006-1-02T15:04:05Z",
"2006-01-02T15:04Z",
"2006-01-02T15:04-07:00",
"2006-01-02T15:04:05Z",
"2006-01-02T15:04:05-07:00:00",
"2006-01-02T15:04:05:-0700",
"2006-01-02T15:04:05-0700",
"2006-01-02T15:04:05-07:00",
"2006-01-02T15:04:05 -0700",
"2006-01-02T15:04:05:00",
"2006-01-02T15:04:05",
"2006-01-02 at 15:04:05",
"2006-01-02 15:04:05Z",
"2006-01-02 15:04:05 MST",
"2006-01-02 15:04:05-0700",
"2006-01-02 15:04:05-07:00",
"2006-01-02 15:04:05 -0700",
"2006-01-02 15:04",
"2006-01-02 00:00:00.0 15:04:05.0 -0700",
"2006/01/02",
"2006-01-02",
"15:04 02.01.2006 -0700",
"1/2/2006 3:04 PM MST",
"1/2/2006 3:04:05 PM MST",
"1/2/2006 3:04:05 PM",
"1/2/2006 15:04:05 MST",
"1/2/2006",
"06/1/2 15:04",
"06-1-2 15:04",
"02 Monday, Jan 2006 15:04",
"02 Jan 2006 15:04 MST",
"02 Jan 2006 15:04:05 UT",
"02 Jan 2006 15:04:05 MST",
"02 Jan 2006 15:04:05 -0700",
"02 Jan 2006 15:04:05",
"02 Jan 2006",
"02/01/2006 15:04 MST",
"02-01-2006 15:04:05 MST",
"02.01.2006 15:04:05",
"02/01/2006 15:04:05",
"02.01.2006 15:04",
"02/01/2006 - 15:04",
"02.01.2006 -0700",
"02/01/2006",
"02-01-2006",
"01/02/2006 3:04 PM",
"01/02/2006 15:04:05 MST",
"01/02/2006 - 15:04",
"01/02/2006",
"01-02-2006",
}
// Parse parses a given date string using a large
// list of commonly found feed date formats.
func Parse(ds string) (t time.Time, err error) {
d := strings.TrimSpace(ds)
if d == "" {
return t, fmt.Errorf("Date string is empty")
}
for _, f := range dateFormats {
if t, err = time.Parse(f, d); err == nil {
return
}
}
err = fmt.Errorf("Failed to parse date: %s", ds)
return
}

152
reader/feed/handler.go Normal file
View file

@ -0,0 +1,152 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package feed
import (
"fmt"
"github.com/miniflux/miniflux2/errors"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/http"
"github.com/miniflux/miniflux2/reader/icon"
"github.com/miniflux/miniflux2/storage"
"log"
"time"
)
var (
errRequestFailed = "Unable to execute request: %v"
errServerFailure = "Unable to fetch feed (statusCode=%d)."
errDuplicate = "This feed already exists (%s)."
errNotFound = "Feed %d not found"
)
// Handler contains all the logic to create and refresh feeds.
type Handler struct {
store *storage.Storage
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string) (*model.Feed, error) {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
client := http.NewHttpClient(url)
response, err := client.Get()
if err != nil {
return nil, errors.NewLocalizedError(errRequestFailed, err)
}
if response.HasServerFailure() {
return nil, errors.NewLocalizedError(errServerFailure, response.StatusCode)
}
if h.store.FeedURLExists(userID, response.EffectiveURL) {
return nil, errors.NewLocalizedError(errDuplicate, response.EffectiveURL)
}
subscription, err := parseFeed(response.Body)
if err != nil {
return nil, err
}
subscription.Category = &model.Category{ID: categoryID}
subscription.EtagHeader = response.ETag
subscription.LastModifiedHeader = response.LastModified
subscription.FeedURL = response.EffectiveURL
subscription.UserID = userID
err = h.store.CreateFeed(subscription)
if err != nil {
return nil, err
}
log.Println("[Handler:CreateFeed] Feed saved with ID:", subscription.ID)
icon, err := icon.FindIcon(subscription.SiteURL)
if err != nil {
log.Println(err)
} else if icon == nil {
log.Printf("No icon found for feedID=%d\n", subscription.ID)
} else {
h.store.CreateFeedIcon(subscription, icon)
}
return subscription, nil
}
// RefreshFeed fetch and update a feed if necessary.
func (h *Handler) RefreshFeed(userID, feedID int64) error {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:RefreshFeed] feedID=%d", feedID))
originalFeed, err := h.store.GetFeedById(userID, feedID)
if err != nil {
return err
}
if originalFeed == nil {
return errors.NewLocalizedError(errNotFound, feedID)
}
client := http.NewHttpClientWithCacheHeaders(originalFeed.FeedURL, originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
response, err := client.Get()
if err != nil {
customErr := errors.NewLocalizedError(errRequestFailed, err)
originalFeed.ParsingErrorCount++
originalFeed.ParsingErrorMsg = customErr.Error()
h.store.UpdateFeed(originalFeed)
return customErr
}
originalFeed.CheckedAt = time.Now()
if response.HasServerFailure() {
err := errors.NewLocalizedError(errServerFailure, response.StatusCode)
originalFeed.ParsingErrorCount++
originalFeed.ParsingErrorMsg = err.Error()
h.store.UpdateFeed(originalFeed)
return err
}
if response.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) {
log.Printf("[Handler:RefreshFeed] Feed #%d has been modified\n", feedID)
subscription, err := parseFeed(response.Body)
if err != nil {
originalFeed.ParsingErrorCount++
originalFeed.ParsingErrorMsg = err.Error()
h.store.UpdateFeed(originalFeed)
return err
}
originalFeed.EtagHeader = response.ETag
originalFeed.LastModifiedHeader = response.LastModified
if err := h.store.UpdateEntries(originalFeed.UserID, originalFeed.ID, subscription.Entries); err != nil {
return err
}
if !h.store.HasIcon(originalFeed.ID) {
log.Println("[Handler:RefreshFeed] Looking for feed icon")
icon, err := icon.FindIcon(originalFeed.SiteURL)
if err != nil {
log.Println("[Handler:RefreshFeed]", err)
} else {
h.store.CreateFeedIcon(originalFeed, icon)
}
}
} else {
log.Printf("[Handler:RefreshFeed] Feed #%d not modified\n", feedID)
}
originalFeed.ParsingErrorCount = 0
originalFeed.ParsingErrorMsg = ""
return h.store.UpdateFeed(originalFeed)
}
// NewFeedHandler returns a feed handler.
func NewFeedHandler(store *storage.Storage) *Handler {
return &Handler{store: store}
}

170
reader/feed/json/json.go Normal file
View file

@ -0,0 +1,170 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package json
import (
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
"log"
"strings"
"time"
)
type JsonFeed struct {
Version string `json:"version"`
Title string `json:"title"`
SiteURL string `json:"home_page_url"`
FeedURL string `json:"feed_url"`
Author JsonAuthor `json:"author"`
Items []JsonItem `json:"items"`
}
type JsonAuthor struct {
Name string `json:"name"`
URL string `json:"url"`
}
type JsonItem struct {
ID string `json:"id"`
URL string `json:"url"`
Title string `json:"title"`
Summary string `json:"summary"`
Text string `json:"content_text"`
Html string `json:"content_html"`
DatePublished string `json:"date_published"`
DateModified string `json:"date_modified"`
Author JsonAuthor `json:"author"`
Attachments []JsonAttachment `json:"attachments"`
}
type JsonAttachment struct {
URL string `json:"url"`
MimeType string `json:"mime_type"`
Title string `json:"title"`
Size int `json:"size_in_bytes"`
Duration int `json:"duration_in_seconds"`
}
func (j *JsonFeed) GetAuthor() string {
return getAuthor(j.Author)
}
func (j *JsonFeed) Transform() *model.Feed {
feed := new(model.Feed)
feed.FeedURL = j.FeedURL
feed.SiteURL = j.SiteURL
feed.Title = sanitizer.StripTags(j.Title)
if feed.Title == "" {
feed.Title = feed.SiteURL
}
for _, item := range j.Items {
entry := item.Transform()
if entry.Author == "" {
entry.Author = j.GetAuthor()
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}
func (j *JsonItem) GetDate() time.Time {
for _, value := range []string{j.DatePublished, j.DateModified} {
if value != "" {
d, err := date.Parse(value)
if err != nil {
log.Println(err)
return time.Now()
}
return d
}
}
return time.Now()
}
func (j *JsonItem) GetAuthor() string {
return getAuthor(j.Author)
}
func (j *JsonItem) GetHash() string {
for _, value := range []string{j.ID, j.URL, j.Text + j.Html + j.Summary} {
if value != "" {
return helper.Hash(value)
}
}
return ""
}
func (j *JsonItem) GetTitle() string {
for _, value := range []string{j.Title, j.Summary, j.Text, j.Html} {
if value != "" {
return truncate(value)
}
}
return j.URL
}
func (j *JsonItem) GetContent() string {
for _, value := range []string{j.Html, j.Text, j.Summary} {
if value != "" {
return value
}
}
return ""
}
func (j *JsonItem) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
for _, attachment := range j.Attachments {
enclosures = append(enclosures, &model.Enclosure{
URL: attachment.URL,
MimeType: attachment.MimeType,
Size: attachment.Size,
})
}
return enclosures
}
func (j *JsonItem) Transform() *model.Entry {
entry := new(model.Entry)
entry.URL = j.URL
entry.Date = j.GetDate()
entry.Author = sanitizer.StripTags(j.GetAuthor())
entry.Hash = j.GetHash()
entry.Content = processor.ItemContentProcessor(entry.URL, j.GetContent())
entry.Title = sanitizer.StripTags(strings.Trim(j.GetTitle(), " \n\t"))
entry.Enclosures = j.GetEnclosures()
return entry
}
func getAuthor(author JsonAuthor) string {
if author.Name != "" {
return author.Name
}
return ""
}
func truncate(str string) string {
max := 100
if len(str) > max {
return str[:max] + "..."
}
return str
}

23
reader/feed/json/parser.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package json
import (
"encoding/json"
"fmt"
"github.com/miniflux/miniflux2/model"
"io"
)
// Parse returns a normalized feed struct.
func Parse(data io.Reader) (*model.Feed, error) {
jsonFeed := new(JsonFeed)
decoder := json.NewDecoder(data)
if err := decoder.Decode(&jsonFeed); err != nil {
return nil, fmt.Errorf("Unable to parse JSON Feed: %v", err)
}
return jsonFeed.Transform(), nil
}

345
reader/feed/json/parser_test.go Normal file
View file

@ -0,0 +1,345 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package json
import (
"bytes"
"strings"
"testing"
"time"
)
func TestParseJsonFeed(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2",
"content_text": "This is a second item.",
"url": "https://example.org/second-item"
},
{
"id": "1",
"content_html": "<p>Hello, world!</p>",
"url": "https://example.org/initial-post"
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "My Example Feed" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
if feed.FeedURL != "https://example.org/feed.json" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
if feed.SiteURL != "https://example.org/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
if len(feed.Entries) != 2 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Hash != "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
if feed.Entries[0].URL != "https://example.org/second-item" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Title != "This is a second item." {
t.Errorf(`Incorrect entry title, got: "%s"`, feed.Entries[0].Title)
}
if feed.Entries[0].Content != "This is a second item." {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].Content)
}
if feed.Entries[1].Hash != "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[1].Hash)
}
if feed.Entries[1].URL != "https://example.org/initial-post" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[1].URL)
}
if feed.Entries[1].Title != "Hello, world!" {
t.Errorf(`Incorrect entry title, got: "%s"`, feed.Entries[1].Title)
}
if feed.Entries[1].Content != "<p>Hello, world!</p>" {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[1].Content)
}
}
func TestParsePodcast(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"user_comment": "This is a podcast feed. You can add this feed to your podcast client using the following URL: http://therecord.co/feed.json",
"title": "The Record",
"home_page_url": "http://therecord.co/",
"feed_url": "http://therecord.co/feed.json",
"items": [
{
"id": "http://therecord.co/chris-parrish",
"title": "Special #1 - Chris Parrish",
"url": "http://therecord.co/chris-parrish",
"content_text": "Chris has worked at Adobe and as a founder of Rogue Sheep, which won an Apple Design Award for Postage. Chris’s new company is Aged & Distilled with Guy English — which shipped Napkin, a Mac app for visual collaboration. Chris is also the co-host of The Record. He lives on Bainbridge Island, a quick ferry ride from Seattle.",
"content_html": "Chris has worked at <a href=\"http://adobe.com/\">Adobe</a> and as a founder of Rogue Sheep, which won an Apple Design Award for Postage. Chris’s new company is Aged & Distilled with Guy English — which shipped <a href=\"http://aged-and-distilled.com/napkin/\">Napkin</a>, a Mac app for visual collaboration. Chris is also the co-host of The Record. He lives on <a href=\"http://www.ci.bainbridge-isl.wa.us/\">Bainbridge Island</a>, a quick ferry ride from Seattle.",
"summary": "Brent interviews Chris Parrish, co-host of The Record and one-half of Aged & Distilled.",
"date_published": "2014-05-09T14:04:00-07:00",
"attachments": [
{
"url": "http://therecord.co/downloads/The-Record-sp1e1-ChrisParrish.m4a",
"mime_type": "audio/x-m4a",
"size_in_bytes": 89970236,
"duration_in_seconds": 6629
}
]
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "The Record" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
if feed.FeedURL != "http://therecord.co/feed.json" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
if feed.SiteURL != "http://therecord.co/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Hash != "6b678e57962a1b001e4e873756563cdc08bbd06ca561e764e0baa9a382485797" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
if feed.Entries[0].URL != "http://therecord.co/chris-parrish" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Title != "Special #1 - Chris Parrish" {
t.Errorf(`Incorrect entry title, got: "%s"`, feed.Entries[0].Title)
}
if feed.Entries[0].Content != `Chris has worked at <a href="http://adobe.com/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Adobe</a> and as a founder of Rogue Sheep, which won an Apple Design Award for Postage. Chris’s new company is Aged & Distilled with Guy English — which shipped <a href="http://aged-and-distilled.com/napkin/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Napkin</a>, a Mac app for visual collaboration. Chris is also the co-host of The Record. He lives on <a href="http://www.ci.bainbridge-isl.wa.us/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Bainbridge Island</a>, a quick ferry ride from Seattle.` {
t.Errorf(`Incorrect entry content, got: "%s"`, feed.Entries[0].Content)
}
location, _ := time.LoadLocation("America/Vancouver")
if !feed.Entries[0].Date.Equal(time.Date(2014, time.May, 9, 14, 4, 0, 0, location)) {
t.Errorf("Incorrect entry date, got: %v", feed.Entries[0].Date)
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://therecord.co/downloads/The-Record-sp1e1-ChrisParrish.m4a" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
if feed.Entries[0].Enclosures[0].MimeType != "audio/x-m4a" {
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType)
}
if feed.Entries[0].Enclosures[0].Size != 89970236 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
}
func TestParseAuthor(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
"title": "Brent Simmons’s Microblog",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"author": {
"name": "Brent Simmons",
"url": "http://example.org/",
"avatar": "https://example.org/avatar.png"
},
"items": [
{
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00"
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Author != "Brent Simmons" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseFeedWithoutTitle(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00"
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "https://example.org/" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseFeedItemWithInvalidDate(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "Tomorrow"
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if !feed.Entries[0].Date.Before(time.Now()) {
t.Errorf("Incorrect entry date, got: %v", feed.Entries[0].Date)
}
}
func TestParseFeedItemWithoutID(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"content_text": "Some text."
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
}
func TestParseFeedItemWithoutTitle(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"url": "https://example.org/item"
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Title != "https://example.org/item" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
}
func TestParseTruncateItemTitle(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"title": "` + strings.Repeat("a", 200) + `"
}
]
}`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Title) != 103 {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
}

82
reader/feed/parser.go Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package feed
import (
"bytes"
"encoding/xml"
"errors"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/atom"
"github.com/miniflux/miniflux2/reader/feed/json"
"github.com/miniflux/miniflux2/reader/feed/rss"
"io"
"strings"
"time"
"golang.org/x/net/html/charset"
)
const (
FormatRss = "rss"
FormatAtom = "atom"
FormatJson = "json"
FormatUnknown = "unknown"
)
func DetectFeedFormat(data io.Reader) string {
defer helper.ExecutionTime(time.Now(), "[Feed:DetectFeedFormat]")
var buffer bytes.Buffer
tee := io.TeeReader(data, &buffer)
decoder := xml.NewDecoder(tee)
decoder.CharsetReader = charset.NewReaderLabel
for {
token, _ := decoder.Token()
if token == nil {
break
}
if element, ok := token.(xml.StartElement); ok {
switch element.Name.Local {
case "rss":
return FormatRss
case "feed":
return FormatAtom
}
}
}
if strings.HasPrefix(strings.TrimSpace(buffer.String()), "{") {
return FormatJson
}
return FormatUnknown
}
func parseFeed(data io.Reader) (*model.Feed, error) {
defer helper.ExecutionTime(time.Now(), "[Feed:ParseFeed]")
var buffer bytes.Buffer
io.Copy(&buffer, data)
reader := bytes.NewReader(buffer.Bytes())
format := DetectFeedFormat(reader)
reader.Seek(0, io.SeekStart)
switch format {
case FormatAtom:
return atom.Parse(reader)
case FormatRss:
return rss.Parse(reader)
case FormatJson:
return json.Parse(reader)
default:
return nil, errors.New("Unsupported feed format")
}
}

169
reader/feed/parser_test.go Normal file
View file

@ -0,0 +1,169 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package feed
import (
"bytes"
"testing"
)
func TestDetectRSS(t *testing.T) {
data := `<?xml version="1.0"?><rss version="2.0"><channel></channel></rss>`
format := DetectFeedFormat(bytes.NewBufferString(data))
if format != FormatRss {
t.Errorf("Wrong format detected: %s instead of %s", format, FormatRss)
}
}
func TestDetectAtom(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>`
format := DetectFeedFormat(bytes.NewBufferString(data))
if format != FormatAtom {
t.Errorf("Wrong format detected: %s instead of %s", format, FormatAtom)
}
}
func TestDetectAtomWithISOCharset(t *testing.T) {
data := `<?xml version="1.0" encoding="ISO-8859-15"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>`
format := DetectFeedFormat(bytes.NewBufferString(data))
if format != FormatAtom {
t.Errorf("Wrong format detected: %s instead of %s", format, FormatAtom)
}
}
func TestDetectJSON(t *testing.T) {
data := `
{
"version" : "https://jsonfeed.org/version/1",
"title" : "Example"
}
`
format := DetectFeedFormat(bytes.NewBufferString(data))
if format != FormatJson {
t.Errorf("Wrong format detected: %s instead of %s", format, FormatJson)
}
}
func TestDetectUnknown(t *testing.T) {
data := `
<!DOCTYPE html> <html> </html>
`
format := DetectFeedFormat(bytes.NewBufferString(data))
if format != FormatUnknown {
t.Errorf("Wrong format detected: %s instead of %s", format, FormatUnknown)
}
}
func TestParseAtom(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<author>
<name>John Doe</name>
</author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := parseFeed(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "Example Feed" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseRss(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>Liftoff News</title>
<link>http://liftoff.msfc.nasa.gov/</link>
<item>
<title>Star City</title>
<link>http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp</link>
<description>How do Americans get ready to work with Russians aboard the International Space Station? They take a crash course in culture, language and protocol at Russia's &lt;a href="http://howe.iki.rssi.ru/GCTC/gctc_e.htm"&gt;Star City&lt;/a&gt;.</description>
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
<guid>http://liftoff.msfc.nasa.gov/2003/06/03.html#item573</guid>
</item>
</channel>
</rss>`
feed, err := parseFeed(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "Liftoff News" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseJson(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2",
"content_text": "This is a second item.",
"url": "https://example.org/second-item"
},
{
"id": "1",
"content_html": "<p>Hello, world!</p>",
"url": "https://example.org/initial-post"
}
]
}`
feed, err := parseFeed(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "My Example Feed" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseUnknownFeed(t *testing.T) {
data := `
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Title of document</title>
</head>
<body>
some content
</body>
</html>
`
_, err := parseFeed(bytes.NewBufferString(data))
if err == nil {
t.Error("ParseFeed must returns an error")
}
}

28
reader/feed/rss/parser.go Normal file
View file

@ -0,0 +1,28 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package rss
import (
"encoding/xml"
"fmt"
"github.com/miniflux/miniflux2/model"
"io"
"golang.org/x/net/html/charset"
)
// Parse returns a normalized feed struct.
func Parse(data io.Reader) (*model.Feed, error) {
rssFeed := new(RssFeed)
decoder := xml.NewDecoder(data)
decoder.CharsetReader = charset.NewReaderLabel
err := decoder.Decode(rssFeed)
if err != nil {
return nil, fmt.Errorf("Unable to parse RSS feed: %v", err)
}
return rssFeed.Transform(), nil
}

466
reader/feed/rss/parser_test.go Normal file
View file

@ -0,0 +1,466 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package rss
import (
"bytes"
"testing"
"time"
)
func TestParseRss2Sample(t *testing.T) {
data := `
<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>Liftoff News</title>
<link>http://liftoff.msfc.nasa.gov/</link>
<description>Liftoff to Space Exploration.</description>
<language>en-us</language>
<pubDate>Tue, 10 Jun 2003 04:00:00 GMT</pubDate>
<lastBuildDate>Tue, 10 Jun 2003 09:41:01 GMT</lastBuildDate>
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
<generator>Weblog Editor 2.0</generator>
<managingEditor>editor@example.com</managingEditor>
<webMaster>webmaster@example.com</webMaster>
<item>
<title>Star City</title>
<link>http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp</link>
<description>How do Americans get ready to work with Russians aboard the International Space Station? They take a crash course in culture, language and protocol at Russia's &lt;a href="http://howe.iki.rssi.ru/GCTC/gctc_e.htm"&gt;Star City&lt;/a&gt;.</description>
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
<guid>http://liftoff.msfc.nasa.gov/2003/06/03.html#item573</guid>
</item>
<item>
<description>Sky watchers in Europe, Asia, and parts of Alaska and Canada will experience a &lt;a href="http://science.nasa.gov/headlines/y2003/30may_solareclipse.htm"&gt;partial eclipse of the Sun&lt;/a&gt; on Saturday, May 31st.</description>
<pubDate>Fri, 30 May 2003 11:06:42 GMT</pubDate>
<guid>http://liftoff.msfc.nasa.gov/2003/05/30.html#item572</guid>
</item>
<item>
<title>The Engine That Does More</title>
<link>http://liftoff.msfc.nasa.gov/news/2003/news-VASIMR.asp</link>
<description>Before man travels to Mars, NASA hopes to design new engines that will let us fly through the Solar System more quickly. The proposed VASIMR engine would do that.</description>
<pubDate>Tue, 27 May 2003 08:37:32 GMT</pubDate>
<guid>http://liftoff.msfc.nasa.gov/2003/05/27.html#item571</guid>
</item>
<item>
<title>Astronauts' Dirty Laundry</title>
<link>http://liftoff.msfc.nasa.gov/news/2003/news-laundry.asp</link>
<description>Compared to earlier spacecraft, the International Space Station has many luxuries, but laundry facilities are not one of them. Instead, astronauts have other options.</description>
<pubDate>Tue, 20 May 2003 08:56:02 GMT</pubDate>
<guid>http://liftoff.msfc.nasa.gov/2003/05/20.html#item570</guid>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "Liftoff News" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
if feed.FeedURL != "" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
if feed.SiteURL != "http://liftoff.msfc.nasa.gov/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
if len(feed.Entries) != 4 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
expectedDate := time.Date(2003, time.June, 3, 9, 39, 21, 0, time.UTC)
if !feed.Entries[0].Date.Equal(expectedDate) {
t.Errorf("Incorrect entry date, got: %v, want: %v", feed.Entries[0].Date, expectedDate)
}
if feed.Entries[0].Hash != "5b2b4ac2fe1786ddf0fd2da2f1b07f64e691264f41f2db3ea360f31bb6d9152b" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
if feed.Entries[0].URL != "http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Title != "Star City" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
if feed.Entries[0].Content != `How do Americans get ready to work with Russians aboard the International Space Station? They take a crash course in culture, language and protocol at Russia's <a href="http://howe.iki.rssi.ru/GCTC/gctc_e.htm" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Star City</a>.` {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].Content)
}
}
func TestParseFeedWithoutTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<link>https://example.org/</link>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Title != "https://example.org/" {
t.Errorf("Incorrect feed title, got: %s", feed.Title)
}
}
func TestParseEntryWithoutTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<link>https://example.org/</link>
<item>
<link>https://example.org/item</link>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Title != "https://example.org/item" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
}
func TestParseFeedURLWithAtomLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<atom:link href="https://example.org/rss" type="application/rss+xml" rel="self"></atom:link>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.FeedURL != "https://example.org/rss" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
if feed.SiteURL != "https://example.org/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
}
func TestParseEntryWithAtomAuthor(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<atom:link href="https://example.org/rss" type="application/rss+xml" rel="self"></atom:link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
<author xmlns:author="http://www.w3.org/2005/Atom">
<name>Foo Bar</name>
<title>Vice President</title>
<department/>
<company>FooBar Inc.</company>
</author>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Author != "Foo Bar" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseEntryWithDublinCoreAuthor(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
<dc:creator>Me (me@example.com)</dc:creator>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Author != "Me (me@example.com)" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseEntryWithItunesAuthor(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
<itunes:author>Someone</itunes:author>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Author != "Someone" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseFeedWithItunesAuthor(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<itunes:author>Someone</itunes:author>
<item>
<title>Test</title>
<link>https://example.org/item</link>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Author != "Someone" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseEntryWithDublinCoreDate(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<description>Description.</description>
<guid isPermaLink="false">UUID</guid>
<dc:date>2002-09-29T23:40:06-05:00</dc:date>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
location, _ := time.LoadLocation("EST")
expectedDate := time.Date(2002, time.September, 29, 23, 40, 06, 0, location)
if !feed.Entries[0].Date.Equal(expectedDate) {
t.Errorf("Incorrect entry date, got: %v, want: %v", feed.Entries[0].Date, expectedDate)
}
}
func TestParseEntryWithContentEncoded(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<description>Description.</description>
<guid isPermaLink="false">UUID</guid>
<content:encoded><![CDATA[<p><a href="http://www.example.org/">Example</a>.</p>]]></content:encoded>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Content != `<p><a href="http://www.example.org/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Example</a>.</p>` {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].Content)
}
}
func TestParseEntryWithFeedBurnerLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<feedburner:origLink>http://example.org/original</feedburner:origLink>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].URL != "http://example.org/original" {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].URL)
}
}
func TestParseEntryTitleWithWhitespaces(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<title>Example</title>
<link>http://example.org</link>
<item>
<title>
Some Title
</title>
<link>http://www.example.org/entries/1</link>
<pubDate>Fri, 15 Jul 2005 00:00:00 -0500</pubDate>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].Title != "Some Title" {
t.Errorf("Incorrect entry title, got: %s", feed.Entries[0].Title)
}
}
func TestParseEntryWithEnclosures(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<title>My Podcast Feed</title>
<link>http://example.org</link>
<author>some.email@example.org</author>
<item>
<title>Podcasting with RSS</title>
<link>http://www.example.org/entries/1</link>
<description>An overview of RSS podcasting</description>
<pubDate>Fri, 15 Jul 2005 00:00:00 -0500</pubDate>
<guid isPermaLink="true">http://www.example.org/entries/1</guid>
<enclosure url="http://www.example.org/myaudiofile.mp3"
length="12345"
type="audio/mpeg" />
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
if feed.Entries[0].Enclosures[0].MimeType != "audio/mpeg" {
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType)
}
if feed.Entries[0].Enclosures[0].Size != 12345 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
}
func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
<channel>
<title>My Example Feed</title>
<link>http://example.org</link>
<author>some.email@example.org</author>
<item>
<title>Example Item</title>
<link>http://www.example.org/entries/1</link>
<enclosure
url="http://feedproxy.google.com/~r/example/~5/lpMyFSCvubs/File.mp3"
length="76192460"
type="audio/mpeg" />
<feedburner:origEnclosureLink>http://example.org/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3</feedburner:origEnclosureLink>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://example.org/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
if feed.Entries[0].Enclosures[0].MimeType != "audio/mpeg" {
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType)
}
if feed.Entries[0].Enclosures[0].Size != 76192460 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
}

207
reader/feed/rss/rss.go Normal file
View file

@ -0,0 +1,207 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package rss
import (
"encoding/xml"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
"log"
"path"
"strconv"
"strings"
"time"
)
type RssLink struct {
XMLName xml.Name
Data string `xml:",chardata"`
Href string `xml:"href,attr"`
}
type RssFeed struct {
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Title string `xml:"channel>title"`
Links []RssLink `xml:"channel>link"`
Language string `xml:"channel>language"`
Description string `xml:"channel>description"`
PubDate string `xml:"channel>pubDate"`
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"`
Items []RssItem `xml:"channel>item"`
}
type RssItem struct {
Guid string `xml:"guid"`
Title string `xml:"title"`
Link string `xml:"link"`
OriginalLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"`
Description string `xml:"description"`
Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"http://purl.org/dc/elements/1.1/ date"`
Authors []RssAuthor `xml:"author"`
Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
Enclosures []RssEnclosure `xml:"enclosure"`
OrigEnclosureLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origEnclosureLink"`
}
type RssAuthor struct {
XMLName xml.Name
Data string `xml:",chardata"`
Name string `xml:"name"`
}
type RssEnclosure struct {
Url string `xml:"url,attr"`
Type string `xml:"type,attr"`
Length string `xml:"length,attr"`
}
func (r *RssFeed) GetSiteURL() string {
for _, elem := range r.Links {
if elem.XMLName.Space == "" {
return elem.Data
}
}
return ""
}
func (r *RssFeed) GetFeedURL() string {
for _, elem := range r.Links {
if elem.XMLName.Space == "http://www.w3.org/2005/Atom" {
return elem.Href
}
}
return ""
}
func (r *RssFeed) Transform() *model.Feed {
feed := new(model.Feed)
feed.SiteURL = r.GetSiteURL()
feed.FeedURL = r.GetFeedURL()
feed.Title = sanitizer.StripTags(r.Title)
if feed.Title == "" {
feed.Title = feed.SiteURL
}
for _, item := range r.Items {
entry := item.Transform()
if entry.Author == "" && r.ItunesAuthor != "" {
entry.Author = r.ItunesAuthor
}
entry.Author = sanitizer.StripTags(entry.Author)
feed.Entries = append(feed.Entries, entry)
}
return feed
}
func (i *RssItem) GetDate() time.Time {
value := i.PubDate
if i.Date != "" {
value = i.Date
}
if value != "" {
result, err := date.Parse(value)
if err != nil {
log.Println(err)
return time.Now()
}
return result
}
return time.Now()
}
func (i *RssItem) GetAuthor() string {
for _, element := range i.Authors {
if element.Name != "" {
return element.Name
}
if element.Data != "" {
return element.Data
}
}
return i.Creator
}
func (i *RssItem) GetHash() string {
for _, value := range []string{i.Guid, i.Link} {
if value != "" {
return helper.Hash(value)
}
}
return ""
}
func (i *RssItem) GetContent() string {
if i.Content != "" {
return i.Content
}
return i.Description
}
func (i *RssItem) GetURL() string {
if i.OriginalLink != "" {
return i.OriginalLink
}
return i.Link
}
func (i *RssItem) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
for _, enclosure := range i.Enclosures {
length, _ := strconv.Atoi(enclosure.Length)
enclosureURL := enclosure.Url
if i.OrigEnclosureLink != "" {
filename := path.Base(i.OrigEnclosureLink)
if strings.Contains(enclosureURL, filename) {
enclosureURL = i.OrigEnclosureLink
}
}
enclosures = append(enclosures, &model.Enclosure{
URL: enclosureURL,
MimeType: enclosure.Type,
Size: length,
})
}
return enclosures
}
func (i *RssItem) Transform() *model.Entry {
entry := new(model.Entry)
entry.URL = i.GetURL()
entry.Date = i.GetDate()
entry.Author = i.GetAuthor()
entry.Hash = i.GetHash()
entry.Content = processor.ItemContentProcessor(entry.URL, i.GetContent())
entry.Title = sanitizer.StripTags(strings.Trim(i.Title, " \n\t"))
entry.Enclosures = i.GetEnclosures()
if entry.Title == "" {
entry.Title = entry.URL
}
return entry
}

95
reader/http/client.go Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package http
import (
"crypto/tls"
"fmt"
"github.com/miniflux/miniflux2/helper"
"log"
"net/http"
"net/url"
"time"
)
const HTTP_USER_AGENT = "Miniflux <https://miniflux.net/>"
type HttpClient struct {
url string
etagHeader string
lastModifiedHeader string
Insecure bool
}
func (h *HttpClient) Get() (*ServerResponse, error) {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient:Get] url=%s", h.url))
u, _ := url.Parse(h.url)
req := &http.Request{
URL: u,
Method: "GET",
Header: h.buildHeaders(),
}
client := h.buildClient()
resp, err := client.Do(req)
if err != nil {
return nil, err
}
response := &ServerResponse{
Body: resp.Body,
StatusCode: resp.StatusCode,
EffectiveURL: resp.Request.URL.String(),
LastModified: resp.Header.Get("Last-Modified"),
ETag: resp.Header.Get("ETag"),
ContentType: resp.Header.Get("Content-Type"),
}
log.Println("[HttpClient:Get]",
"OriginalURL:", h.url,
"StatusCode:", response.StatusCode,
"ETag:", response.ETag,
"LastModified:", response.LastModified,
"EffectiveURL:", response.EffectiveURL,
)
return response, err
}
func (h *HttpClient) buildClient() http.Client {
if h.Insecure {
transport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
return http.Client{Transport: transport}
}
return http.Client{}
}
func (h *HttpClient) buildHeaders() http.Header {
headers := make(http.Header)
headers.Add("User-Agent", HTTP_USER_AGENT)
if h.etagHeader != "" {
headers.Add("If-None-Match", h.etagHeader)
}
if h.lastModifiedHeader != "" {
headers.Add("If-Modified-Since", h.lastModifiedHeader)
}
return headers
}
func NewHttpClient(url string) *HttpClient {
return &HttpClient{url: url, Insecure: false}
}
func NewHttpClientWithCacheHeaders(url, etagHeader, lastModifiedHeader string) *HttpClient {
return &HttpClient{url: url, etagHeader: etagHeader, lastModifiedHeader: lastModifiedHeader, Insecure: false}
}

32
reader/http/response.go Normal file
View file

@ -0,0 +1,32 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package http
import "io"
type ServerResponse struct {
Body io.Reader
StatusCode int
EffectiveURL string
LastModified string
ETag string
ContentType string
}
func (s *ServerResponse) HasServerFailure() bool {
return s.StatusCode >= 400
}
func (s *ServerResponse) IsModified(etag, lastModified string) bool {
if s.StatusCode == 304 {
return false
}
if s.ETag != "" && s.LastModified != "" && (s.ETag == etag || s.LastModified == lastModified) {
return false
}
return true
}

109
reader/icon/finder.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package icon
import (
"fmt"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/http"
"github.com/miniflux/miniflux2/reader/url"
"io"
"io/ioutil"
"log"
"github.com/PuerkitoBio/goquery"
)
// FindIcon try to find the website's icon.
func FindIcon(websiteURL string) (*model.Icon, error) {
rootURL := url.GetRootURL(websiteURL)
client := http.NewHttpClient(rootURL)
response, err := client.Get()
if err != nil {
return nil, fmt.Errorf("unable to download website index page: %v", err)
}
if response.HasServerFailure() {
return nil, fmt.Errorf("unable to download website index page: status=%d", response.StatusCode)
}
iconURL, err := parseDocument(rootURL, response.Body)
if err != nil {
return nil, err
}
log.Println("[FindIcon] Fetching icon =>", iconURL)
icon, err := downloadIcon(iconURL)
if err != nil {
return nil, err
}
return icon, nil
}
func parseDocument(websiteURL string, data io.Reader) (string, error) {
queries := []string{
"link[rel='shortcut icon']",
"link[rel='Shortcut Icon']",
"link[rel='icon shortcut']",
"link[rel='icon']",
}
doc, err := goquery.NewDocumentFromReader(data)
if err != nil {
return "", fmt.Errorf("unable to read document: %v", err)
}
var iconURL string
for _, query := range queries {
doc.Find(query).Each(func(i int, s *goquery.Selection) {
if href, exists := s.Attr("href"); exists {
iconURL = href
}
})
if iconURL != "" {
break
}
}
if iconURL == "" {
iconURL = url.GetRootURL(websiteURL) + "favicon.ico"
} else {
iconURL, _ = url.GetAbsoluteURL(websiteURL, iconURL)
}
return iconURL, nil
}
func downloadIcon(iconURL string) (*model.Icon, error) {
client := http.NewHttpClient(iconURL)
response, err := client.Get()
if err != nil {
return nil, fmt.Errorf("unable to download iconURL: %v", err)
}
if response.HasServerFailure() {
return nil, fmt.Errorf("unable to download icon: status=%d", response.StatusCode)
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("unable to read downloaded icon: %v", err)
}
if len(body) == 0 {
return nil, fmt.Errorf("downloaded icon is empty, iconURL=%s", iconURL)
}
icon := &model.Icon{
Hash: helper.HashFromBytes(body),
MimeType: response.ContentType,
Content: body,
}
return icon, nil
}

94
reader/opml/handler.go Normal file
View file

@ -0,0 +1,94 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
import (
"errors"
"fmt"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/storage"
"io"
"log"
)
type OpmlHandler struct {
store *storage.Storage
}
func (o *OpmlHandler) Export(userID int64) (string, error) {
feeds, err := o.store.GetFeeds(userID)
if err != nil {
log.Println(err)
return "", errors.New("Unable to fetch feeds.")
}
var subscriptions SubcriptionList
for _, feed := range feeds {
subscriptions = append(subscriptions, &Subcription{
Title: feed.Title,
FeedURL: feed.FeedURL,
SiteURL: feed.SiteURL,
CategoryName: feed.Category.Title,
})
}
return Serialize(subscriptions), nil
}
func (o *OpmlHandler) Import(userID int64, data io.Reader) (err error) {
subscriptions, err := Parse(data)
if err != nil {
return err
}
for _, subscription := range subscriptions {
if !o.store.FeedURLExists(userID, subscription.FeedURL) {
var category *model.Category
if subscription.CategoryName == "" {
category, err = o.store.GetFirstCategory(userID)
if err != nil {
log.Println(err)
return errors.New("Unable to find first category.")
}
} else {
category, err = o.store.GetCategoryByTitle(userID, subscription.CategoryName)
if err != nil {
log.Println(err)
return errors.New("Unable to search category by title.")
}
if category == nil {
category = &model.Category{
UserID: userID,
Title: subscription.CategoryName,
}
err := o.store.CreateCategory(category)
if err != nil {
log.Println(err)
return fmt.Errorf(`Unable to create this category: "%s".`, subscription.CategoryName)
}
}
}
feed := &model.Feed{
UserID: userID,
Title: subscription.Title,
FeedURL: subscription.FeedURL,
SiteURL: subscription.SiteURL,
Category: category,
}
o.store.CreateFeed(feed)
}
}
return nil
}
func NewOpmlHandler(store *storage.Storage) *OpmlHandler {
return &OpmlHandler{store: store}
}

82
reader/opml/opml.go Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
import "encoding/xml"
type Opml struct {
XMLName xml.Name `xml:"opml"`
Version string `xml:"version,attr"`
Outlines []Outline `xml:"body>outline"`
}
type Outline struct {
Title string `xml:"title,attr,omitempty"`
Text string `xml:"text,attr"`
FeedURL string `xml:"xmlUrl,attr,omitempty"`
SiteURL string `xml:"htmlUrl,attr,omitempty"`
Outlines []Outline `xml:"outline,omitempty"`
}
func (o *Outline) GetTitle() string {
if o.Title != "" {
return o.Title
}
if o.Text != "" {
return o.Text
}
if o.SiteURL != "" {
return o.SiteURL
}
if o.FeedURL != "" {
return o.FeedURL
}
return ""
}
func (o *Outline) GetSiteURL() string {
if o.SiteURL != "" {
return o.SiteURL
}
return o.FeedURL
}
func (o *Outline) IsCategory() bool {
return o.Text != "" && o.SiteURL == "" && o.FeedURL == ""
}
func (o *Outline) Append(subscriptions SubcriptionList, category string) SubcriptionList {
if o.FeedURL != "" {
subscriptions = append(subscriptions, &Subcription{
Title: o.GetTitle(),
FeedURL: o.FeedURL,
SiteURL: o.GetSiteURL(),
CategoryName: category,
})
}
return subscriptions
}
func (o *Opml) Transform() SubcriptionList {
var subscriptions SubcriptionList
for _, outline := range o.Outlines {
if outline.IsCategory() {
for _, element := range outline.Outlines {
subscriptions = element.Append(subscriptions, outline.Text)
}
} else {
subscriptions = outline.Append(subscriptions, "")
}
}
return subscriptions
}

26
reader/opml/parser.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
import (
"encoding/xml"
"fmt"
"io"
"golang.org/x/net/html/charset"
)
func Parse(data io.Reader) (SubcriptionList, error) {
opml := new(Opml)
decoder := xml.NewDecoder(data)
decoder.CharsetReader = charset.NewReaderLabel
err := decoder.Decode(opml)
if err != nil {
return nil, fmt.Errorf("Unable to parse OPML file: %v\n", err)
}
return opml.Transform(), nil
}

138
reader/opml/parser_test.go Normal file
View file

@ -0,0 +1,138 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
import "testing"
import "bytes"
func TestParseOpmlWithoutCategories(t *testing.T) {
data := `<?xml version="1.0" encoding="ISO-8859-1"?>
<opml version="2.0">
<head>
<title>mySubscriptions.opml</title>
</head>
<body>
<outline text="CNET News.com" description="Tech news and business reports by CNET News.com. Focused on information technology, core topics include computers, hardware, software, networking, and Internet media." htmlUrl="http://news.com.com/" language="unknown" title="CNET News.com" type="rss" version="RSS2" xmlUrl="http://news.com.com/2547-1_3-0-5.xml"/>
<outline text="washingtonpost.com - Politics" description="Politics" htmlUrl="http://www.washingtonpost.com/wp-dyn/politics?nav=rss_politics" language="unknown" title="washingtonpost.com - Politics" type="rss" version="RSS2" xmlUrl="http://www.washingtonpost.com/wp-srv/politics/rssheadlines.xml"/>
<outline text="Scobleizer: Microsoft Geek Blogger" description="Robert Scoble's look at geek and Microsoft life." htmlUrl="http://radio.weblogs.com/0001011/" language="unknown" title="Scobleizer: Microsoft Geek Blogger" type="rss" version="RSS2" xmlUrl="http://radio.weblogs.com/0001011/rss.xml"/>
<outline text="Yahoo! News: Technology" description="Technology" htmlUrl="http://news.yahoo.com/news?tmpl=index&amp;cid=738" language="unknown" title="Yahoo! News: Technology" type="rss" version="RSS2" xmlUrl="http://rss.news.yahoo.com/rss/tech"/>
<outline text="Workbench" description="Programming and publishing news and comment" htmlUrl="http://www.cadenhead.org/workbench/" language="unknown" title="Workbench" type="rss" version="RSS2" xmlUrl="http://www.cadenhead.org/workbench/rss.xml"/>
<outline text="Christian Science Monitor | Top Stories" description="Read the front page stories of csmonitor.com." htmlUrl="http://csmonitor.com" language="unknown" title="Christian Science Monitor | Top Stories" type="rss" version="RSS" xmlUrl="http://www.csmonitor.com/rss/top.rss"/>
<outline text="Dictionary.com Word of the Day" description="A new word is presented every day with its definition and example sentences from actual published works." htmlUrl="http://dictionary.reference.com/wordoftheday/" language="unknown" title="Dictionary.com Word of the Day" type="rss" version="RSS" xmlUrl="http://www.dictionary.com/wordoftheday/wotd.rss"/>
<outline text="The Motley Fool" description="To Educate, Amuse, and Enrich" htmlUrl="http://www.fool.com" language="unknown" title="The Motley Fool" type="rss" version="RSS" xmlUrl="http://www.fool.com/xml/foolnews_rss091.xml"/>
<outline text="InfoWorld: Top News" description="The latest on Top News from InfoWorld" htmlUrl="http://www.infoworld.com/news/index.html" language="unknown" title="InfoWorld: Top News" type="rss" version="RSS2" xmlUrl="http://www.infoworld.com/rss/news.xml"/>
<outline text="NYT &gt; Business" description="Find breaking news &amp; business news on Wall Street, media &amp; advertising, international business, banking, interest rates, the stock market, currencies &amp; funds." htmlUrl="http://www.nytimes.com/pages/business/index.html?partner=rssnyt" language="unknown" title="NYT &gt; Business" type="rss" version="RSS2" xmlUrl="http://www.nytimes.com/services/xml/rss/nyt/Business.xml"/>
<outline text="NYT &gt; Technology" description="" htmlUrl="http://www.nytimes.com/pages/technology/index.html?partner=rssnyt" language="unknown" title="NYT &gt; Technology" type="rss" version="RSS2" xmlUrl="http://www.nytimes.com/services/xml/rss/nyt/Technology.xml"/>
<outline text="Scripting News" description="It's even worse than it appears." htmlUrl="http://www.scripting.com/" language="unknown" title="Scripting News" type="rss" version="RSS2" xmlUrl="http://www.scripting.com/rss.xml"/>
<outline text="Wired News" description="Technology, and the way we do business, is changing the world we know. Wired News is a technology - and business-oriented news service feeding an intelligent, discerning audience. What role does technology play in the day-to-day living of your life? Wired News tells you. How has evolving technology changed the face of the international business world? Wired News puts you in the picture." htmlUrl="http://www.wired.com/" language="unknown" title="Wired News" type="rss" version="RSS" xmlUrl="http://www.wired.com/news_drop/netcenter/netcenter.rdf"/>
</body>
</opml>
`
var expected SubcriptionList
expected = append(expected, &Subcription{Title: "CNET News.com", FeedURL: "http://news.com.com/2547-1_3-0-5.xml", SiteURL: "http://news.com.com/"})
subscriptions, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(subscriptions) != 13 {
t.Errorf("Wrong number of subscriptions: %d instead of %d", len(subscriptions), 13)
}
if !subscriptions[0].Equals(expected[0]) {
t.Errorf(`Subscription are different: "%v" vs "%v"`, subscriptions[0], expected[0])
}
}
func TestParseOpmlWithCategories(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<opml version="2.0">
<head>
<title>mySubscriptions.opml</title>
</head>
<body>
<outline text="My Category 1">
<outline text="Feed 1" xmlUrl="http://example.org/feed1/" htmlUrl="http://example.org/1"/>
<outline text="Feed 2" xmlUrl="http://example.org/feed2/" htmlUrl="http://example.org/2"/>
</outline>
<outline text="My Category 2">
<outline text="Feed 3" xmlUrl="http://example.org/feed3/" htmlUrl="http://example.org/3"/>
</outline>
</body>
</opml>
`
var expected SubcriptionList
expected = append(expected, &Subcription{Title: "Feed 1", FeedURL: "http://example.org/feed1/", SiteURL: "http://example.org/1", CategoryName: "My Category 1"})
expected = append(expected, &Subcription{Title: "Feed 2", FeedURL: "http://example.org/feed2/", SiteURL: "http://example.org/2", CategoryName: "My Category 1"})
expected = append(expected, &Subcription{Title: "Feed 3", FeedURL: "http://example.org/feed3/", SiteURL: "http://example.org/3", CategoryName: "My Category 2"})
subscriptions, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(subscriptions) != 3 {
t.Errorf("Wrong number of subscriptions: %d instead of %d", len(subscriptions), 3)
}
for i := 0; i < len(subscriptions); i++ {
if !subscriptions[i].Equals(expected[i]) {
t.Errorf(`Subscription are different: "%v" vs "%v"`, subscriptions[i], expected[i])
}
}
}
func TestParseOpmlWithEmptyTitleAndEmptySiteURL(t *testing.T) {
data := `<?xml version="1.0" encoding="ISO-8859-1"?>
<opml version="2.0">
<head>
<title>mySubscriptions.opml</title>
</head>
<body>
<outline xmlUrl="http://example.org/feed1/" htmlUrl="http://example.org/1"/>
<outline xmlUrl="http://example.org/feed2/"/>
</body>
</opml>
`
var expected SubcriptionList
expected = append(expected, &Subcription{Title: "http://example.org/1", FeedURL: "http://example.org/feed1/", SiteURL: "http://example.org/1", CategoryName: ""})
expected = append(expected, &Subcription{Title: "http://example.org/feed2/", FeedURL: "http://example.org/feed2/", SiteURL: "http://example.org/feed2/", CategoryName: ""})
subscriptions, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if len(subscriptions) != 2 {
t.Errorf("Wrong number of subscriptions: %d instead of %d", len(subscriptions), 2)
}
for i := 0; i < len(subscriptions); i++ {
if !subscriptions[i].Equals(expected[i]) {
t.Errorf(`Subscription are different: "%v" vs "%v"`, subscriptions[i], expected[i])
}
}
}
func TestParseInvalidXML(t *testing.T) {
data := `<?xml version="1.0" encoding="ISO-8859-1"?>
<opml version="2.0">
<head>
</head>
<body>
<outline
</body>
</opml>
`
_, err := Parse(bytes.NewBufferString(data))
if err == nil {
t.Error(err)
}
}

58
reader/opml/serializer.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
import (
"bufio"
"bytes"
"encoding/xml"
"log"
)
func Serialize(subscriptions SubcriptionList) string {
var b bytes.Buffer
writer := bufio.NewWriter(&b)
writer.WriteString(xml.Header)
opml := new(Opml)
opml.Version = "2.0"
for categoryName, subs := range groupSubscriptionsByFeed(subscriptions) {
outline := Outline{Text: categoryName}
for _, subscription := range subs {
outline.Outlines = append(outline.Outlines, Outline{
Title: subscription.Title,
Text: subscription.Title,
FeedURL: subscription.FeedURL,
SiteURL: subscription.SiteURL,
})
}
opml.Outlines = append(opml.Outlines, outline)
}
encoder := xml.NewEncoder(writer)
encoder.Indent(" ", " ")
if err := encoder.Encode(opml); err != nil {
log.Println(err)
return ""
}
return b.String()
}
func groupSubscriptionsByFeed(subscriptions SubcriptionList) map[string]SubcriptionList {
groups := make(map[string]SubcriptionList)
for _, subscription := range subscriptions {
// if subs, ok := groups[subscription.CategoryName]; !ok {
// groups[subscription.CategoryName] = SubcriptionList{}
// }
groups[subscription.CategoryName] = append(groups[subscription.CategoryName], subscription)
}
return groups
}

31
reader/opml/serializer_test.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
import "testing"
import "bytes"
func TestSerialize(t *testing.T) {
var subscriptions SubcriptionList
subscriptions = append(subscriptions, &Subcription{Title: "Feed 1", FeedURL: "http://example.org/feed/1", SiteURL: "http://example.org/1", CategoryName: "Category 1"})
subscriptions = append(subscriptions, &Subcription{Title: "Feed 2", FeedURL: "http://example.org/feed/2", SiteURL: "http://example.org/2", CategoryName: "Category 1"})
subscriptions = append(subscriptions, &Subcription{Title: "Feed 3", FeedURL: "http://example.org/feed/3", SiteURL: "http://example.org/3", CategoryName: "Category 2"})
output := Serialize(subscriptions)
feeds, err := Parse(bytes.NewBufferString(output))
if err != nil {
t.Error(err)
}
if len(feeds) != 3 {
t.Errorf("Wrong number of subscriptions: %d instead of %d", len(feeds), 3)
}
for i := 0; i < len(feeds); i++ {
if !feeds[i].Equals(subscriptions[i]) {
t.Errorf(`Subscription are different: "%v" vs "%v"`, subscriptions[i], feeds[i])
}
}
}

18
reader/opml/subscription.go Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package opml
type Subcription struct {
Title string
SiteURL string
FeedURL string
CategoryName string
}
func (s Subcription) Equals(subscription *Subcription) bool {
return s.Title == subscription.Title && s.SiteURL == subscription.SiteURL && s.FeedURL == subscription.FeedURL && s.CategoryName == subscription.CategoryName
}
type SubcriptionList []*Subcription

15
reader/processor/processor.go Normal file
View file

@ -0,0 +1,15 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package processor
import (
"github.com/miniflux/miniflux2/reader/rewrite"
"github.com/miniflux/miniflux2/reader/sanitizer"
)
func ItemContentProcessor(url, content string) string {
content = sanitizer.Sanitize(url, content)
return rewrite.Rewriter(url, content)
}

47
reader/rewrite/rewriter.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package rewrite
import (
"regexp"
"strings"
"github.com/PuerkitoBio/goquery"
)
var rewriteRules = []func(string, string) string{
func(url, content string) string {
re := regexp.MustCompile(`youtube\.com/watch\?v=(.*)`)
matches := re.FindStringSubmatch(url)
if len(matches) == 2 {
video := `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/` + matches[1] + `" allowfullscreen></iframe>`
return video + "<p>" + content + "</p>"
}
return content
},
func(url, content string) string {
if strings.HasPrefix(url, "https://xkcd.com") {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
if err != nil {
return content
}
imgTag := doc.Find("img").First()
if titleAttr, found := imgTag.Attr("title"); found {
return content + `<blockquote cite="` + url + `">` + titleAttr + "</blockquote>"
}
}
return content
},
}
func Rewriter(url, content string) string {
for _, rewriteRule := range rewriteRules {
content = rewriteRule(url, content)
}
return content
}

34
reader/rewrite/rewriter_test.go Normal file
View file

@ -0,0 +1,34 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package rewrite
import "testing"
func TestRewriteWithNoMatchingRule(t *testing.T) {
output := Rewriter("https://example.org/article", `Some text.`)
expected := `Some text.`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestRewriteWithYoutubeLink(t *testing.T) {
output := Rewriter("https://www.youtube.com/watch?v=1234", `Video Description`)
expected := `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/1234" allowfullscreen></iframe><p>Video Description</p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}
func TestRewriteWithXkcdLink(t *testing.T) {
description := `<img src="https://imgs.xkcd.com/comics/thermostat.png" title="Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you." alt="Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you." />`
output := Rewriter("https://xkcd.com/1912/", description)
expected := description + `<blockquote cite="https://xkcd.com/1912/">Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.</blockquote>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
}
}

360
reader/sanitizer/sanitizer.go Normal file
View file

@ -0,0 +1,360 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanitizer
import (
"bytes"
"fmt"
"github.com/miniflux/miniflux2/reader/url"
"io"
"strings"
"golang.org/x/net/html"
)
// Sanitize returns safe HTML.
func Sanitize(baseURL, input string) string {
tokenizer := html.NewTokenizer(bytes.NewBufferString(input))
var buffer bytes.Buffer
var tagStack []string
for {
if tokenizer.Next() == html.ErrorToken {
err := tokenizer.Err()
if err == io.EOF {
return buffer.String()
}
return ""
}
token := tokenizer.Token()
switch token.Type {
case html.TextToken:
buffer.WriteString(token.Data)
case html.StartTagToken:
tagName := token.DataAtom.String()
if !isPixelTracker(tagName, token.Attr) && isValidTag(tagName) {
attrNames, htmlAttributes := sanitizeAttributes(baseURL, tagName, token.Attr)
if hasRequiredAttributes(tagName, attrNames) {
if len(attrNames) > 0 {
buffer.WriteString("<" + tagName + " " + htmlAttributes + ">")
} else {
buffer.WriteString("<" + tagName + ">")
}
tagStack = append(tagStack, tagName)
}
}
case html.EndTagToken:
tagName := token.DataAtom.String()
if isValidTag(tagName) && inList(tagName, tagStack) {
buffer.WriteString(fmt.Sprintf("</%s>", tagName))
}
case html.SelfClosingTagToken:
tagName := token.DataAtom.String()
if !isPixelTracker(tagName, token.Attr) && isValidTag(tagName) {
attrNames, htmlAttributes := sanitizeAttributes(baseURL, tagName, token.Attr)
if hasRequiredAttributes(tagName, attrNames) {
if len(attrNames) > 0 {
buffer.WriteString("<" + tagName + " " + htmlAttributes + "/>")
} else {
buffer.WriteString("<" + tagName + "/>")
}
}
}
}
}
}
func sanitizeAttributes(baseURL, tagName string, attributes []html.Attribute) (attrNames []string, html string) {
var htmlAttrs []string
var err error
for _, attribute := range attributes {
value := attribute.Val
if !isValidAttribute(tagName, attribute.Key) {
continue
}
if isExternalResourceAttribute(attribute.Key) {
if tagName == "iframe" && !isValidIframeSource(attribute.Val) {
continue
} else {
value, err = url.GetAbsoluteURL(baseURL, value)
if err != nil {
continue
}
if !hasValidScheme(value) || isBlacklistedResource(value) {
continue
}
}
}
attrNames = append(attrNames, attribute.Key)
htmlAttrs = append(htmlAttrs, fmt.Sprintf(`%s="%s"`, attribute.Key, value))
}
extraAttrNames, extraHTMLAttributes := getExtraAttributes(tagName)
if len(extraAttrNames) > 0 {
attrNames = append(attrNames, extraAttrNames...)
htmlAttrs = append(htmlAttrs, extraHTMLAttributes...)
}
return attrNames, strings.Join(htmlAttrs, " ")
}
func getExtraAttributes(tagName string) ([]string, []string) {
if tagName == "a" {
return []string{"rel", "target", "referrerpolicy"}, []string{`rel="noopener noreferrer"`, `target="_blank"`, `referrerpolicy="no-referrer"`}
}
if tagName == "video" || tagName == "audio" {
return []string{"controls"}, []string{"controls"}
}
return nil, nil
}
func isValidTag(tagName string) bool {
for element := range getTagWhitelist() {
if tagName == element {
return true
}
}
return false
}
func isValidAttribute(tagName, attributeName string) bool {
for element, attributes := range getTagWhitelist() {
if tagName == element {
if inList(attributeName, attributes) {
return true
}
}
}
return false
}
func isExternalResourceAttribute(attribute string) bool {
switch attribute {
case "src", "href", "poster", "cite":
return true
default:
return false
}
}
func isPixelTracker(tagName string, attributes []html.Attribute) bool {
if tagName == "img" {
hasHeight := false
hasWidth := false
for _, attribute := range attributes {
if attribute.Key == "height" && attribute.Val == "1" {
hasHeight = true
}
if attribute.Key == "width" && attribute.Val == "1" {
hasWidth = true
}
}
return hasHeight && hasWidth
}
return false
}
func hasRequiredAttributes(tagName string, attributes []string) bool {
elements := make(map[string][]string)
elements["a"] = []string{"href"}
elements["iframe"] = []string{"src"}
elements["img"] = []string{"src"}
elements["source"] = []string{"src"}
for element, attrs := range elements {
if tagName == element {
for _, attribute := range attributes {
for _, attr := range attrs {
if attr == attribute {
return true
}
}
}
return false
}
}
return true
}
func hasValidScheme(src string) bool {
// See https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
whitelist := []string{
"apt://",
"bitcoin://",
"callto://",
"ed2k://",
"facetime://",
"feed://",
"ftp://",
"geo://",
"gopher://",
"git://",
"http://",
"https://",
"irc://",
"irc6://",
"ircs://",
"itms://",
"jabber://",
"magnet://",
"mailto://",
"maps://",
"news://",
"nfs://",
"nntp://",
"rtmp://",
"sip://",
"sips://",
"skype://",
"smb://",
"sms://",
"spotify://",
"ssh://",
"sftp://",
"steam://",
"svn://",
"tel://",
"webcal://",
"xmpp://",
}
for _, prefix := range whitelist {
if strings.HasPrefix(src, prefix) {
return true
}
}
return false
}
func isBlacklistedResource(src string) bool {
blacklist := []string{
"feedsportal.com",
"api.flattr.com",
"stats.wordpress.com",
"plus.google.com/share",
"twitter.com/share",
"feeds.feedburner.com",
}
for _, element := range blacklist {
if strings.Contains(src, element) {
return true
}
}
return false
}
func isValidIframeSource(src string) bool {
whitelist := []string{
"http://www.youtube.com",
"https://www.youtube.com",
"http://player.vimeo.com",
"https://player.vimeo.com",
"http://www.dailymotion.com",
"https://www.dailymotion.com",
"http://vk.com",
"https://vk.com",
}
for _, prefix := range whitelist {
if strings.HasPrefix(src, prefix) {
return true
}
}
return false
}
func getTagWhitelist() map[string][]string {
whitelist := make(map[string][]string)
whitelist["img"] = []string{"alt", "title", "src"}
whitelist["audio"] = []string{"src"}
whitelist["video"] = []string{"poster", "height", "width", "src"}
whitelist["source"] = []string{"src", "type"}
whitelist["dt"] = []string{}
whitelist["dd"] = []string{}
whitelist["dl"] = []string{}
whitelist["table"] = []string{}
whitelist["caption"] = []string{}
whitelist["thead"] = []string{}
whitelist["tfooter"] = []string{}
whitelist["tr"] = []string{}
whitelist["td"] = []string{"rowspan", "colspan"}
whitelist["th"] = []string{"rowspan", "colspan"}
whitelist["h1"] = []string{}
whitelist["h2"] = []string{}
whitelist["h3"] = []string{}
whitelist["h4"] = []string{}
whitelist["h5"] = []string{}
whitelist["h6"] = []string{}
whitelist["strong"] = []string{}
whitelist["em"] = []string{}
whitelist["code"] = []string{}
whitelist["pre"] = []string{}
whitelist["blockquote"] = []string{}
whitelist["q"] = []string{"cite"}
whitelist["p"] = []string{}
whitelist["ul"] = []string{}
whitelist["li"] = []string{}
whitelist["ol"] = []string{}
whitelist["br"] = []string{}
whitelist["del"] = []string{}
whitelist["a"] = []string{"href", "title"}
whitelist["figure"] = []string{}
whitelist["figcaption"] = []string{}
whitelist["cite"] = []string{}
whitelist["time"] = []string{"datetime"}
whitelist["abbr"] = []string{"title"}
whitelist["acronym"] = []string{"title"}
whitelist["wbr"] = []string{}
whitelist["dfn"] = []string{}
whitelist["sub"] = []string{}
whitelist["sup"] = []string{}
whitelist["var"] = []string{}
whitelist["samp"] = []string{}
whitelist["s"] = []string{}
whitelist["del"] = []string{}
whitelist["ins"] = []string{}
whitelist["kbd"] = []string{}
whitelist["rp"] = []string{}
whitelist["rt"] = []string{}
whitelist["rtc"] = []string{}
whitelist["ruby"] = []string{}
whitelist["iframe"] = []string{"width", "height", "frameborder", "src", "allowfullscreen"}
return whitelist
}
func inList(needle string, haystack []string) bool {
for _, element := range haystack {
if element == needle {
return true
}
}
return false
}

144
reader/sanitizer/sanitizer_test.go Normal file
View file

@ -0,0 +1,144 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanitizer
import "testing"
func TestValidInput(t *testing.T) {
input := `<p>This is a <strong>text</strong> with an image: <img src="http://example.org/" alt="Test">.</p>`
output := Sanitize("http://example.org/", input)
if input != output {
t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
}
}
func TestSelfClosingTags(t *testing.T) {
input := `<p>This <br> is a <strong>text</strong> <br/>with an image: <img src="http://example.org/" alt="Test"/>.</p>`
output := Sanitize("http://example.org/", input)
if input != output {
t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
}
}
func TestTable(t *testing.T) {
input := `<table><tr><th>A</th><th colspan="2">B</th></tr><tr><td>C</td><td>D</td><td>E</td></tr></table>`
output := Sanitize("http://example.org/", input)
if input != output {
t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
}
}
func TestRelativeURL(t *testing.T) {
input := `This <a href="/test.html">link is relative</a> and this image: <img src="../folder/image.png"/>`
expected := `This <a href="http://example.org/test.html" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">link is relative</a> and this image: <img src="http://example.org/folder/image.png"/>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestProtocolRelativeURL(t *testing.T) {
input := `This <a href="//static.example.org/index.html">link is relative</a>.`
expected := `This <a href="https://static.example.org/index.html" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">link is relative</a>.`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestInvalidTag(t *testing.T) {
input := `<p>My invalid <b>tag</b>.</p>`
expected := `<p>My invalid tag.</p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestVideoTag(t *testing.T) {
input := `<p>My valid <video src="videofile.webm" autoplay poster="posterimage.jpg">fallback</video>.</p>`
expected := `<p>My valid <video src="http://example.org/videofile.webm" poster="http://example.org/posterimage.jpg" controls>fallback</video>.</p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestAudioAndSourceTag(t *testing.T) {
input := `<p>My music <audio controls="controls"><source src="foo.wav" type="audio/wav"></audio>.</p>`
expected := `<p>My music <audio controls><source src="http://example.org/foo.wav" type="audio/wav"></audio>.</p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestUnknownTag(t *testing.T) {
input := `<p>My invalid <unknown>tag</unknown>.</p>`
expected := `<p>My invalid tag.</p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestInvalidNestedTag(t *testing.T) {
input := `<p>My invalid <b>tag with some <em>valid</em> tag</b>.</p>`
expected := `<p>My invalid tag with some <em>valid</em> tag.</p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestInvalidIFrame(t *testing.T) {
input := `<iframe src="http://example.org/"></iframe>`
expected := ``
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestInvalidURLScheme(t *testing.T) {
input := `<p>This link is <a src="file:///etc/passwd">not valid</a></p>`
expected := `<p>This link is not valid</p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestBlacklistedLink(t *testing.T) {
input := `<p>This image is not valid <img src="https://stats.wordpress.com/some-tracker"></p>`
expected := `<p>This image is not valid </p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}
func TestPixelTracker(t *testing.T) {
input := `<p><img src="https://tracker1.example.org/" height="1" width="1"> and <img src="https://tracker2.example.org/" height="1" width="1"/></p>`
expected := `<p> and </p>`
output := Sanitize("http://example.org/", input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}

35
reader/sanitizer/strip_tags.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanitizer
import (
"bytes"
"io"
"golang.org/x/net/html"
)
// StripTags removes all HTML/XML tags from the input string.
func StripTags(input string) string {
tokenizer := html.NewTokenizer(bytes.NewBufferString(input))
var buffer bytes.Buffer
for {
if tokenizer.Next() == html.ErrorToken {
err := tokenizer.Err()
if err == io.EOF {
return buffer.String()
}
return ""
}
token := tokenizer.Token()
switch token.Type {
case html.TextToken:
buffer.WriteString(token.Data)
}
}
}

17
reader/sanitizer/strip_tags_test.go Normal file
View file

@ -0,0 +1,17 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanitizer
import "testing"
func TestStripTags(t *testing.T) {
input := `This <a href="/test.html">link is relative</a> and <strong>this</strong> image: <img src="../folder/image.png"/>`
expected := `This link is relative and this image: `
output := StripTags(input)
if expected != output {
t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
}
}

96
reader/subscription/finder.go Normal file
View file

@ -0,0 +1,96 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package subscription
import (
"bytes"
"fmt"
"github.com/miniflux/miniflux2/errors"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/reader/http"
"github.com/miniflux/miniflux2/reader/url"
"io"
"log"
"time"
"github.com/PuerkitoBio/goquery"
)
var (
errConnectionFailure = "Unable to open this link: %v"
errUnreadableDoc = "Unable to analyze this page: %v"
)
// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL string) (Subscriptions, error) {
defer helper.ExecutionTime(time.Now(), fmt.Sprintf("[FindSubscriptions] url=%s", websiteURL))
client := http.NewHttpClient(websiteURL)
response, err := client.Get()
if err != nil {
return nil, errors.NewLocalizedError(errConnectionFailure, err)
}
var buffer bytes.Buffer
io.Copy(&buffer, response.Body)
reader := bytes.NewReader(buffer.Bytes())
if format := feed.DetectFeedFormat(reader); format != feed.FormatUnknown {
var subscriptions Subscriptions
subscriptions = append(subscriptions, &Subscription{
Title: response.EffectiveURL,
URL: response.EffectiveURL,
Type: format,
})
return subscriptions, nil
}
reader.Seek(0, io.SeekStart)
return parseDocument(response.EffectiveURL, bytes.NewReader(buffer.Bytes()))
}
func parseDocument(websiteURL string, data io.Reader) (Subscriptions, error) {
var subscriptions Subscriptions
queries := map[string]string{
"link[type='application/rss+xml']": "rss",
"link[type='application/atom+xml']": "atom",
"link[type='application/json']": "json",
}
doc, err := goquery.NewDocumentFromReader(data)
if err != nil {
return nil, errors.NewLocalizedError(errUnreadableDoc, err)
}
for query, kind := range queries {
doc.Find(query).Each(func(i int, s *goquery.Selection) {
subscription := new(Subscription)
subscription.Type = kind
if title, exists := s.Attr("title"); exists {
subscription.Title = title
} else {
subscription.Title = "Feed"
}
if feedURL, exists := s.Attr("href"); exists {
subscription.URL, _ = url.GetAbsoluteURL(websiteURL, feedURL)
}
if subscription.Title == "" {
subscription.Title = subscription.URL
}
if subscription.URL != "" {
log.Println("[FindSubscriptions]", subscription)
subscriptions = append(subscriptions, subscription)
}
})
}
return subscriptions, nil
}

21
reader/subscription/subscription.go Normal file
View file

@ -0,0 +1,21 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package subscription
import "fmt"
// Subscription represents a feed subscription.
type Subscription struct {
Title string `json:"title"`
URL string `json:"url"`
Type string `json:"type"`
}
func (s Subscription) String() string {
return fmt.Sprintf(`Title="%s", URL="%s", Type="%s"`, s.Title, s.URL, s.Type)
}
// Subscriptions represents a list of subscription.
type Subscriptions []*Subscription

61
reader/url/url.go Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package url
import "net/url"
import "fmt"
import "strings"
// GetAbsoluteURL converts the input URL as absolute URL if necessary.
func GetAbsoluteURL(baseURL, input string) (string, error) {
if strings.HasPrefix(input, "//") {
input = "https://" + input[2:]
}
u, err := url.Parse(input)
if err != nil {
return "", fmt.Errorf("unable to parse input URL: %v", err)
}
if u.IsAbs() {
return u.String(), nil
}
base, err := url.Parse(baseURL)
if err != nil {
return "", fmt.Errorf("unable to parse base URL: %v", err)
}
return base.ResolveReference(u).String(), nil
}
// GetRootURL returns absolute URL without the path.
func GetRootURL(websiteURL string) string {
if strings.HasPrefix(websiteURL, "//") {
websiteURL = "https://" + websiteURL[2:]
}
absoluteURL, err := GetAbsoluteURL(websiteURL, "")
if err != nil {
return websiteURL
}
u, err := url.Parse(absoluteURL)
if err != nil {
return absoluteURL
}
return u.Scheme + "://" + u.Host + "/"
}
// IsHTTPS returns true if the URL is using HTTPS.
func IsHTTPS(websiteURL string) bool {
parsedURL, err := url.Parse(websiteURL)
if err != nil {
return false
}
return strings.ToLower(parsedURL.Scheme) == "https"
}

107
reader/url/url_test.go Normal file
View file

@ -0,0 +1,107 @@
package url
import "testing"
func TestGetAbsoluteURLWithAbsolutePath(t *testing.T) {
expected := `https://example.org/path/file.ext`
input := `/path/file.ext`
output, err := GetAbsoluteURL("https://example.org/folder/", input)
if err != nil {
t.Error(err)
}
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestGetAbsoluteURLWithRelativePath(t *testing.T) {
expected := `https://example.org/folder/path/file.ext`
input := `path/file.ext`
output, err := GetAbsoluteURL("https://example.org/folder/", input)
if err != nil {
t.Error(err)
}
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestGetAbsoluteURLWithRelativePaths(t *testing.T) {
expected := `https://example.org/path/file.ext`
input := `path/file.ext`
output, err := GetAbsoluteURL("https://example.org/folder", input)
if err != nil {
t.Error(err)
}
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestWhenInputIsAlreadyAbsolute(t *testing.T) {
expected := `https://example.org/path/file.ext`
input := `https://example.org/path/file.ext`
output, err := GetAbsoluteURL("https://example.org/folder/", input)
if err != nil {
t.Error(err)
}
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestGetAbsoluteURLWithProtocolRelative(t *testing.T) {
expected := `https://static.example.org/path/file.ext`
input := `//static.example.org/path/file.ext`
output, err := GetAbsoluteURL("https://www.example.org/", input)
if err != nil {
t.Error(err)
}
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestGetRootURL(t *testing.T) {
expected := `https://example.org/`
input := `https://example.org/path/file.ext`
output := GetRootURL(input)
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestGetRootURLWithProtocolRelativePath(t *testing.T) {
expected := `https://static.example.org/`
input := `//static.example.org/path/file.ext`
output := GetRootURL(input)
if expected != output {
t.Errorf(`Unexpected output, got "%s" instead of "%s"`, output, expected)
}
}
func TestIsHTTPS(t *testing.T) {
if !IsHTTPS("https://example.org/") {
t.Error("Unable to recognize HTTPS URL")
}
if IsHTTPS("http://example.org/") {
t.Error("Unable to recognize HTTP URL")
}
if IsHTTPS("") {
t.Error("Unable to recognize malformed URL")
}
}

24
scheduler/scheduler.go Normal file
View file

@ -0,0 +1,24 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package scheduler
import (
"github.com/miniflux/miniflux2/storage"
"log"
"time"
)
// NewScheduler starts a new scheduler to push jobs to a pool of workers.
func NewScheduler(store *storage.Storage, workerPool *WorkerPool, frequency, batchSize int) {
c := time.Tick(time.Duration(frequency) * time.Minute)
for now := range c {
jobs := store.GetJobs(batchSize)
log.Printf("[Scheduler:%v] => Pushing %d jobs\n", now, len(jobs))
for _, job := range jobs {
workerPool.Push(job)
}
}
}

35
scheduler/worker.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package scheduler
import (
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed"
"log"
"time"
)
// A Worker refresh a feed in the background.
type Worker struct {
id int
feedHandler *feed.Handler
}
// Run wait for a job and refresh the given feed.
func (w *Worker) Run(c chan model.Job) {
log.Printf("[Worker] #%d started\n", w.id)
for {
job := <-c
log.Printf("[Worker #%d] got userID=%d, feedID=%d\n", w.id, job.UserID, job.FeedID)
err := w.feedHandler.RefreshFeed(job.UserID, job.FeedID)
if err != nil {
log.Println("Worker:", err)
}
time.Sleep(time.Millisecond * 1000)
}
}

34
scheduler/worker_pool.go Normal file
View file

@ -0,0 +1,34 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package scheduler
import (
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed"
)
// WorkerPool handle a pool of workers.
type WorkerPool struct {
queue chan model.Job
}
// Push send a job on the queue.
func (w *WorkerPool) Push(job model.Job) {
w.queue <- job
}
// NewWorkerPool creates a pool of background workers.
func NewWorkerPool(feedHandler *feed.Handler, nbWorkers int) *WorkerPool {
workerPool := &WorkerPool{
queue: make(chan model.Job),
}
for i := 0; i < nbWorkers; i++ {
worker := &Worker{id: i, feedHandler: feedHandler}
go worker.Run(workerPool.queue)
}
return workerPool
}

97
server/api/controller/category.go Normal file
View file

@ -0,0 +1,97 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package api
import (
"errors"
"github.com/miniflux/miniflux2/server/api/payload"
"github.com/miniflux/miniflux2/server/core"
)
// CreateCategory is the API handler to create a new category.
func (c *Controller) CreateCategory(ctx *core.Context, request *core.Request, response *core.Response) {
category, err := payload.DecodeCategoryPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
category.UserID = ctx.GetUserID()
if err := category.ValidateCategoryCreation(); err != nil {
response.Json().ServerError(err)
return
}
err = c.store.CreateCategory(category)
if err != nil {
response.Json().ServerError(errors.New("Unable to create this category"))
return
}
response.Json().Created(category)
}
// UpdateCategory is the API handler to update a category.
func (c *Controller) UpdateCategory(ctx *core.Context, request *core.Request, response *core.Response) {
categoryID, err := request.GetIntegerParam("categoryID")
if err != nil {
response.Json().BadRequest(err)
return
}
category, err := payload.DecodeCategoryPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
category.UserID = ctx.GetUserID()
category.ID = categoryID
if err := category.ValidateCategoryModification(); err != nil {
response.Json().BadRequest(err)
return
}
err = c.store.UpdateCategory(category)
if err != nil {
response.Json().ServerError(errors.New("Unable to update this category"))
return
}
response.Json().Created(category)
}
// GetCategories is the API handler to get a list of categories for a given user.
func (c *Controller) GetCategories(ctx *core.Context, request *core.Request, response *core.Response) {
categories, err := c.store.GetCategories(ctx.GetUserID())
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch categories"))
return
}
response.Json().Standard(categories)
}
// RemoveCategory is the API handler to remove a category.
func (c *Controller) RemoveCategory(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
categoryID, err := request.GetIntegerParam("categoryID")
if err != nil {
response.Json().BadRequest(err)
return
}
if !c.store.CategoryExists(userID, categoryID) {
response.Json().NotFound(errors.New("Category not found"))
return
}
if err := c.store.RemoveCategory(userID, categoryID); err != nil {
response.Json().ServerError(errors.New("Unable to remove this category"))
return
}
response.Json().NoContent()
}

21
server/api/controller/controller.go Normal file
View file

@ -0,0 +1,21 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package api
import (
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/storage"
)
// Controller holds all handlers for the API.
type Controller struct {
store *storage.Storage
feedHandler *feed.Handler
}
// NewController creates a new controller.
func NewController(store *storage.Storage, feedHandler *feed.Handler) *Controller {
return &Controller{store: store, feedHandler: feedHandler}
}

156
server/api/controller/entry.go Normal file
View file

@ -0,0 +1,156 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package api
import (
"errors"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/server/api/payload"
"github.com/miniflux/miniflux2/server/core"
)
// GetEntry is the API handler to get a single feed entry.
func (c *Controller) GetEntry(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
entryID, err := request.GetIntegerParam("entryID")
if err != nil {
response.Json().BadRequest(err)
return
}
builder := c.store.GetEntryQueryBuilder(userID, ctx.GetUserTimezone())
builder.WithFeedID(feedID)
builder.WithEntryID(entryID)
entry, err := builder.GetEntry()
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch this entry from the database"))
return
}
if entry == nil {
response.Json().NotFound(errors.New("Entry not found"))
return
}
response.Json().Standard(entry)
}
// GetFeedEntries is the API handler to get all feed entries.
func (c *Controller) GetFeedEntries(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
status := request.GetQueryStringParam("status", "")
if status != "" {
if err := model.ValidateEntryStatus(status); err != nil {
response.Json().BadRequest(err)
return
}
}
order := request.GetQueryStringParam("order", "id")
if err := model.ValidateEntryOrder(order); err != nil {
response.Json().BadRequest(err)
return
}
direction := request.GetQueryStringParam("direction", "desc")
if err := model.ValidateDirection(direction); err != nil {
response.Json().BadRequest(err)
return
}
limit := request.GetQueryIntegerParam("limit", 100)
offset := request.GetQueryIntegerParam("offset", 0)
builder := c.store.GetEntryQueryBuilder(userID, ctx.GetUserTimezone())
builder.WithFeedID(feedID)
builder.WithStatus(status)
builder.WithOrder(model.DefaultSortingOrder)
builder.WithDirection(model.DefaultSortingDirection)
builder.WithOffset(offset)
builder.WithLimit(limit)
entries, err := builder.GetEntries()
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch the list of entries"))
return
}
count, err := builder.CountEntries()
if err != nil {
response.Json().ServerError(errors.New("Unable to count the number of entries"))
return
}
response.Json().Standard(&payload.EntriesResponse{Total: count, Entries: entries})
}
// SetEntryStatus is the API handler to change the status of an entry.
func (c *Controller) SetEntryStatus(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
entryID, err := request.GetIntegerParam("entryID")
if err != nil {
response.Json().BadRequest(err)
return
}
status, err := payload.DecodeEntryStatusPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(errors.New("Invalid JSON payload"))
return
}
if err := model.ValidateEntryStatus(status); err != nil {
response.Json().BadRequest(err)
return
}
builder := c.store.GetEntryQueryBuilder(userID, ctx.GetUserTimezone())
builder.WithFeedID(feedID)
builder.WithEntryID(entryID)
entry, err := builder.GetEntry()
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch this entry from the database"))
return
}
if entry == nil {
response.Json().NotFound(errors.New("Entry not found"))
return
}
if err := c.store.SetEntriesStatus(userID, []int64{entry.ID}, status); err != nil {
response.Json().ServerError(errors.New("Unable to change entry status"))
return
}
entry, err = builder.GetEntry()
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch this entry from the database"))
return
}
response.Json().Standard(entry)
}

138
server/api/controller/feed.go Normal file
View file

@ -0,0 +1,138 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package api
import (
"errors"
"github.com/miniflux/miniflux2/server/api/payload"
"github.com/miniflux/miniflux2/server/core"
)
// CreateFeed is the API handler to create a new feed.
func (c *Controller) CreateFeed(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedURL, categoryID, err := payload.DecodeFeedCreationPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
feed, err := c.feedHandler.CreateFeed(userID, categoryID, feedURL)
if err != nil {
response.Json().ServerError(errors.New("Unable to create this feed"))
return
}
response.Json().Created(feed)
}
// RefreshFeed is the API handler to refresh a feed.
func (c *Controller) RefreshFeed(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
err = c.feedHandler.RefreshFeed(userID, feedID)
if err != nil {
response.Json().ServerError(errors.New("Unable to refresh this feed"))
return
}
response.Json().NoContent()
}
// UpdateFeed is the API handler that is used to update a feed.
func (c *Controller) UpdateFeed(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
newFeed, err := payload.DecodeFeedModificationPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
originalFeed, err := c.store.GetFeedById(userID, feedID)
if err != nil {
response.Json().NotFound(errors.New("Unable to find this feed"))
return
}
if originalFeed == nil {
response.Json().NotFound(errors.New("Feed not found"))
return
}
originalFeed.Merge(newFeed)
if err := c.store.UpdateFeed(originalFeed); err != nil {
response.Json().ServerError(errors.New("Unable to update this feed"))
return
}
response.Json().Created(originalFeed)
}
// GetFeeds is the API handler that get all feeds that belongs to the given user.
func (c *Controller) GetFeeds(ctx *core.Context, request *core.Request, response *core.Response) {
feeds, err := c.store.GetFeeds(ctx.GetUserID())
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch feeds from the database"))
return
}
response.Json().Standard(feeds)
}
// GetFeed is the API handler to get a feed.
func (c *Controller) GetFeed(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
feed, err := c.store.GetFeedById(userID, feedID)
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch this feed"))
return
}
if feed == nil {
response.Json().NotFound(errors.New("Feed not found"))
return
}
response.Json().Standard(feed)
}
// RemoveFeed is the API handler to remove a feed.
func (c *Controller) RemoveFeed(ctx *core.Context, request *core.Request, response *core.Response) {
userID := ctx.GetUserID()
feedID, err := request.GetIntegerParam("feedID")
if err != nil {
response.Json().BadRequest(err)
return
}
if !c.store.FeedExists(userID, feedID) {
response.Json().NotFound(errors.New("Feed not found"))
return
}
if err := c.store.RemoveFeed(userID, feedID); err != nil {
response.Json().ServerError(errors.New("Unable to remove this feed"))
return
}
response.Json().NoContent()
}

35
server/api/controller/subscription.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package api
import (
"errors"
"fmt"
"github.com/miniflux/miniflux2/reader/subscription"
"github.com/miniflux/miniflux2/server/api/payload"
"github.com/miniflux/miniflux2/server/core"
)
// GetSubscriptions is the API handler to find subscriptions.
func (c *Controller) GetSubscriptions(ctx *core.Context, request *core.Request, response *core.Response) {
websiteURL, err := payload.DecodeURLPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
subscriptions, err := subscription.FindSubscriptions(websiteURL)
if err != nil {
response.Json().ServerError(errors.New("Unable to discover subscriptions"))
return
}
if subscriptions == nil {
response.Json().NotFound(fmt.Errorf("No subscription found"))
return
}
response.Json().Standard(subscriptions)
}

163
server/api/controller/user.go Normal file
View file

@ -0,0 +1,163 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package api
import (
"errors"
"github.com/miniflux/miniflux2/server/api/payload"
"github.com/miniflux/miniflux2/server/core"
)
// CreateUser is the API handler to create a new user.
func (c *Controller) CreateUser(ctx *core.Context, request *core.Request, response *core.Response) {
if !ctx.IsAdminUser() {
response.Json().Forbidden()
return
}
user, err := payload.DecodeUserPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
if err := user.ValidateUserCreation(); err != nil {
response.Json().BadRequest(err)
return
}
if c.store.UserExists(user.Username) {
response.Json().BadRequest(errors.New("This user already exists"))
return
}
err = c.store.CreateUser(user)
if err != nil {
response.Json().ServerError(errors.New("Unable to create this user"))
return
}
user.Password = ""
response.Json().Created(user)
}
// UpdateUser is the API handler to update the given user.
func (c *Controller) UpdateUser(ctx *core.Context, request *core.Request, response *core.Response) {
if !ctx.IsAdminUser() {
response.Json().Forbidden()
return
}
userID, err := request.GetIntegerParam("userID")
if err != nil {
response.Json().BadRequest(err)
return
}
user, err := payload.DecodeUserPayload(request.GetBody())
if err != nil {
response.Json().BadRequest(err)
return
}
if err := user.ValidateUserModification(); err != nil {
response.Json().BadRequest(err)
return
}
originalUser, err := c.store.GetUserById(userID)
if err != nil {
response.Json().BadRequest(errors.New("Unable to fetch this user from the database"))
return
}
if originalUser == nil {
response.Json().NotFound(errors.New("User not found"))
return
}
originalUser.Merge(user)
if err = c.store.UpdateUser(originalUser); err != nil {
response.Json().ServerError(errors.New("Unable to update this user"))
return
}
response.Json().Created(originalUser)
}
// GetUsers is the API handler to get the list of users.
func (c *Controller) GetUsers(ctx *core.Context, request *core.Request, response *core.Response) {
if !ctx.IsAdminUser() {
response.Json().Forbidden()
return
}
users, err := c.store.GetUsers()
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch the list of users"))
return
}
response.Json().Standard(users)
}
// GetUser is the API handler to fetch the given user.
func (c *Controller) GetUser(ctx *core.Context, request *core.Request, response *core.Response) {
if !ctx.IsAdminUser() {
response.Json().Forbidden()
return
}
userID, err := request.GetIntegerParam("userID")
if err != nil {
response.Json().BadRequest(err)
return
}
user, err := c.store.GetUserById(userID)
if err != nil {
response.Json().BadRequest(errors.New("Unable to fetch this user from the database"))
return
}
if user == nil {
response.Json().NotFound(errors.New("User not found"))
return
}
response.Json().Standard(user)
}
// RemoveUser is the API handler to remove an existing user.
func (c *Controller) RemoveUser(ctx *core.Context, request *core.Request, response *core.Response) {
if !ctx.IsAdminUser() {
response.Json().Forbidden()
return
}
userID, err := request.GetIntegerParam("userID")
if err != nil {
response.Json().BadRequest(err)
return
}
user, err := c.store.GetUserById(userID)
if err != nil {
response.Json().ServerError(errors.New("Unable to fetch this user from the database"))
return
}
if user == nil {
response.Json().NotFound(errors.New("User not found"))
return
}
if err := c.store.RemoveUser(user.ID); err != nil {
response.Json().BadRequest(errors.New("Unable to remove this user from the database"))
return
}
response.Json().NoContent()
}

93
server/api/payload/payload.go Normal file
View file

@ -0,0 +1,93 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package payload
import (
"encoding/json"
"fmt"
"github.com/miniflux/miniflux2/model"
"io"
)
type EntriesResponse struct {
Total int `json:"total"`
Entries model.Entries `json:"entries"`
}
func DecodeUserPayload(data io.Reader) (*model.User, error) {
var user model.User
decoder := json.NewDecoder(data)
if err := decoder.Decode(&user); err != nil {
return nil, fmt.Errorf("Unable to decode user JSON object: %v", err)
}
return &user, nil
}
func DecodeURLPayload(data io.Reader) (string, error) {
type payload struct {
URL string `json:"url"`
}
var p payload
decoder := json.NewDecoder(data)
if err := decoder.Decode(&p); err != nil {
return "", fmt.Errorf("invalid JSON payload: %v", err)
}
return p.URL, nil
}
func DecodeEntryStatusPayload(data io.Reader) (string, error) {
type payload struct {
Status string `json:"status"`
}
var p payload
decoder := json.NewDecoder(data)
if err := decoder.Decode(&p); err != nil {
return "", fmt.Errorf("invalid JSON payload: %v", err)
}
return p.Status, nil
}
func DecodeFeedCreationPayload(data io.Reader) (string, int64, error) {
type payload struct {
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
}
var p payload
decoder := json.NewDecoder(data)
if err := decoder.Decode(&p); err != nil {
return "", 0, fmt.Errorf("invalid JSON payload: %v", err)
}
return p.FeedURL, p.CategoryID, nil
}
func DecodeFeedModificationPayload(data io.Reader) (*model.Feed, error) {
var feed model.Feed
decoder := json.NewDecoder(data)
if err := decoder.Decode(&feed); err != nil {
return nil, fmt.Errorf("Unable to decode feed JSON object: %v", err)
}
return &feed, nil
}
func DecodeCategoryPayload(data io.Reader) (*model.Category, error) {
var category model.Category
decoder := json.NewDecoder(data)
if err := decoder.Decode(&category); err != nil {
return nil, fmt.Errorf("Unable to decode category JSON object: %v", err)
}
return &category, nil
}

99
server/core/context.go Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/server/route"
"github.com/miniflux/miniflux2/storage"
"log"
"net/http"
"github.com/gorilla/mux"
)
// Context contains helper functions related to the current request.
type Context struct {
writer http.ResponseWriter
request *http.Request
store *storage.Storage
router *mux.Router
user *model.User
}
// IsAdminUser checks if the logged user is administrator.
func (c *Context) IsAdminUser() bool {
if v := c.request.Context().Value("IsAdminUser"); v != nil {
return v.(bool)
}
return false
}
// GetUserTimezone returns the timezone used by the logged user.
func (c *Context) GetUserTimezone() string {
if v := c.request.Context().Value("UserTimezone"); v != nil {
return v.(string)
}
return "UTC"
}
// IsAuthenticated returns a boolean if the user is authenticated.
func (c *Context) IsAuthenticated() bool {
if v := c.request.Context().Value("IsAuthenticated"); v != nil {
return v.(bool)
}
return false
}
// GetUserID returns the UserID of the logged user.
func (c *Context) GetUserID() int64 {
if v := c.request.Context().Value("UserId"); v != nil {
return v.(int64)
}
return 0
}
// GetLoggedUser returns all properties related to the logged user.
func (c *Context) GetLoggedUser() *model.User {
if c.user == nil {
var err error
c.user, err = c.store.GetUserById(c.GetUserID())
if err != nil {
log.Fatalln(err)
}
if c.user == nil {
log.Fatalln("Unable to find user from context")
}
}
return c.user
}
// GetUserLanguage get the locale used by the current logged user.
func (c *Context) GetUserLanguage() string {
user := c.GetLoggedUser()
return user.Language
}
// GetCsrfToken returns the current CSRF token.
func (c *Context) GetCsrfToken() string {
if v := c.request.Context().Value("CsrfToken"); v != nil {
return v.(string)
}
log.Println("No CSRF token in context!")
return ""
}
// GetRoute returns the path for the given arguments.
func (c *Context) GetRoute(name string, args ...interface{}) string {
return route.GetRoute(c.router, name, args...)
}
// NewContext creates a new Context.
func NewContext(w http.ResponseWriter, r *http.Request, store *storage.Storage, router *mux.Router) *Context {
return &Context{writer: w, request: r, store: store, router: router}
}

57
server/core/handler.go Normal file
View file

@ -0,0 +1,57 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/locale"
"github.com/miniflux/miniflux2/server/middleware"
"github.com/miniflux/miniflux2/server/template"
"github.com/miniflux/miniflux2/storage"
"log"
"net/http"
"time"
"github.com/gorilla/mux"
)
type HandlerFunc func(ctx *Context, request *Request, response *Response)
type Handler struct {
store *storage.Storage
translator *locale.Translator
template *template.TemplateEngine
router *mux.Router
middleware *middleware.MiddlewareChain
}
func (h *Handler) Use(f HandlerFunc) http.Handler {
return h.middleware.WrapFunc(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer helper.ExecutionTime(time.Now(), r.URL.Path)
log.Println(r.Method, r.URL.Path)
ctx := NewContext(w, r, h.store, h.router)
request := NewRequest(w, r)
response := NewResponse(w, r, h.template)
if ctx.IsAuthenticated() {
h.template.SetLanguage(ctx.GetUserLanguage())
} else {
h.template.SetLanguage("en_US")
}
f(ctx, request, response)
}))
}
func NewHandler(store *storage.Storage, router *mux.Router, template *template.TemplateEngine, translator *locale.Translator, middleware *middleware.MiddlewareChain) *Handler {
return &Handler{
store: store,
translator: translator,
router: router,
template: template,
middleware: middleware,
}
}

58
server/core/html_response.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"github.com/miniflux/miniflux2/server/template"
"log"
"net/http"
)
type HtmlResponse struct {
writer http.ResponseWriter
request *http.Request
template *template.TemplateEngine
}
func (h *HtmlResponse) Render(template string, args map[string]interface{}) {
h.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
h.template.Execute(h.writer, template, args)
}
func (h *HtmlResponse) ServerError(err error) {
h.writer.WriteHeader(http.StatusInternalServerError)
h.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
if err != nil {
log.Println(err)
h.writer.Write([]byte("Internal Server Error: " + err.Error()))
} else {
h.writer.Write([]byte("Internal Server Error"))
}
}
func (h *HtmlResponse) BadRequest(err error) {
h.writer.WriteHeader(http.StatusBadRequest)
h.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
if err != nil {
log.Println(err)
h.writer.Write([]byte("Bad Request: " + err.Error()))
} else {
h.writer.Write([]byte("Bad Request"))
}
}
func (h *HtmlResponse) NotFound() {
h.writer.WriteHeader(http.StatusNotFound)
h.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
h.writer.Write([]byte("Page Not Found"))
}
func (h *HtmlResponse) Forbidden() {
h.writer.WriteHeader(http.StatusForbidden)
h.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
h.writer.Write([]byte("Access Forbidden"))
}

94
server/core/json_response.go Normal file
View file

@ -0,0 +1,94 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"encoding/json"
"errors"
"log"
"net/http"
)
type JsonResponse struct {
writer http.ResponseWriter
request *http.Request
}
func (j *JsonResponse) Standard(v interface{}) {
j.writer.WriteHeader(http.StatusOK)
j.commonHeaders()
j.writer.Write(j.toJSON(v))
}
func (j *JsonResponse) Created(v interface{}) {
j.writer.WriteHeader(http.StatusCreated)
j.commonHeaders()
j.writer.Write(j.toJSON(v))
}
func (j *JsonResponse) NoContent() {
j.writer.WriteHeader(http.StatusNoContent)
j.commonHeaders()
}
func (j *JsonResponse) BadRequest(err error) {
log.Println("[API:BadRequest]", err)
j.writer.WriteHeader(http.StatusBadRequest)
j.commonHeaders()
if err != nil {
j.writer.Write(j.encodeError(err))
}
}
func (j *JsonResponse) NotFound(err error) {
log.Println("[API:NotFound]", err)
j.writer.WriteHeader(http.StatusNotFound)
j.commonHeaders()
j.writer.Write(j.encodeError(err))
}
func (j *JsonResponse) ServerError(err error) {
log.Println("[API:ServerError]", err)
j.writer.WriteHeader(http.StatusInternalServerError)
j.commonHeaders()
j.writer.Write(j.encodeError(err))
}
func (j *JsonResponse) Forbidden() {
log.Println("[API:Forbidden]")
j.writer.WriteHeader(http.StatusForbidden)
j.commonHeaders()
j.writer.Write(j.encodeError(errors.New("Access Forbidden")))
}
func (j *JsonResponse) commonHeaders() {
j.writer.Header().Set("Accept", "application/json")
j.writer.Header().Set("Content-Type", "application/json")
}
func (j *JsonResponse) encodeError(err error) []byte {
type errorMsg struct {
ErrorMessage string `json:"error_message"`
}
tmp := errorMsg{ErrorMessage: err.Error()}
data, err := json.Marshal(tmp)
if err != nil {
log.Println("encodeError:", err)
}
return data
}
func (j *JsonResponse) toJSON(v interface{}) []byte {
b, err := json.Marshal(v)
if err != nil {
log.Println("Unable to convert interface to JSON:", err)
return []byte("")
}
return b
}

108
server/core/request.go Normal file
View file

@ -0,0 +1,108 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"strconv"
"github.com/gorilla/mux"
)
type Request struct {
writer http.ResponseWriter
request *http.Request
}
func (r *Request) GetRequest() *http.Request {
return r.request
}
func (r *Request) GetBody() io.ReadCloser {
return r.request.Body
}
func (r *Request) GetHeaders() http.Header {
return r.request.Header
}
func (r *Request) GetScheme() string {
return r.request.URL.Scheme
}
func (r *Request) GetFile(name string) (multipart.File, *multipart.FileHeader, error) {
return r.request.FormFile(name)
}
func (r *Request) IsHTTPS() bool {
return r.request.URL.Scheme == "https"
}
func (r *Request) GetCookie(name string) string {
cookie, err := r.request.Cookie(name)
if err == http.ErrNoCookie {
return ""
}
return cookie.Value
}
func (r *Request) GetIntegerParam(param string) (int64, error) {
vars := mux.Vars(r.request)
value, err := strconv.Atoi(vars[param])
if err != nil {
log.Println(err)
return 0, fmt.Errorf("%s parameter is not an integer", param)
}
if value < 0 {
return 0, nil
}
return int64(value), nil
}
func (r *Request) GetStringParam(param, defaultValue string) string {
vars := mux.Vars(r.request)
value := vars[param]
if value == "" {
value = defaultValue
}
return value
}
func (r *Request) GetQueryStringParam(param, defaultValue string) string {
value := r.request.URL.Query().Get(param)
if value == "" {
value = defaultValue
}
return value
}
func (r *Request) GetQueryIntegerParam(param string, defaultValue int) int {
value := r.request.URL.Query().Get(param)
if value == "" {
return defaultValue
}
val, err := strconv.Atoi(value)
if err != nil {
return defaultValue
}
if val < 0 {
return defaultValue
}
return val
}
func NewRequest(w http.ResponseWriter, r *http.Request) *Request {
return &Request{writer: w, request: r}
}

63
server/core/response.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"github.com/miniflux/miniflux2/server/template"
"net/http"
"time"
)
type Response struct {
writer http.ResponseWriter
request *http.Request
template *template.TemplateEngine
}
func (r *Response) SetCookie(cookie *http.Cookie) {
http.SetCookie(r.writer, cookie)
}
func (r *Response) Json() *JsonResponse {
r.commonHeaders()
return &JsonResponse{writer: r.writer, request: r.request}
}
func (r *Response) Html() *HtmlResponse {
r.commonHeaders()
return &HtmlResponse{writer: r.writer, request: r.request, template: r.template}
}
func (r *Response) Xml() *XmlResponse {
r.commonHeaders()
return &XmlResponse{writer: r.writer, request: r.request}
}
func (r *Response) Redirect(path string) {
http.Redirect(r.writer, r.request, path, http.StatusFound)
}
func (r *Response) Cache(mime_type, etag string, content []byte, duration time.Duration) {
r.writer.Header().Set("Content-Type", mime_type)
r.writer.Header().Set("Etag", etag)
r.writer.Header().Set("Cache-Control", "public")
r.writer.Header().Set("Expires", time.Now().Add(duration).Format(time.RFC1123))
if etag == r.request.Header.Get("If-None-Match") {
r.writer.WriteHeader(http.StatusNotModified)
} else {
r.writer.Write(content)
}
}
func (r *Response) commonHeaders() {
r.writer.Header().Set("X-XSS-Protection", "1; mode=block")
r.writer.Header().Set("X-Content-Type-Options", "nosniff")
r.writer.Header().Set("X-Frame-Options", "DENY")
}
func NewResponse(w http.ResponseWriter, r *http.Request, template *template.TemplateEngine) *Response {
return &Response{writer: w, request: r, template: template}
}

21
server/core/xml_response.go Normal file
View file

@ -0,0 +1,21 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package core
import (
"fmt"
"net/http"
)
type XmlResponse struct {
writer http.ResponseWriter
request *http.Request
}
func (x *XmlResponse) Download(filename, data string) {
x.writer.Header().Set("Content-Type", "text/xml")
x.writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
x.writer.Write([]byte(data))
}

61
server/middleware/basic_auth.go Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package middleware
import (
"context"
"github.com/miniflux/miniflux2/storage"
"log"
"net/http"
)
type BasicAuthMiddleware struct {
store *storage.Storage
}
func (b *BasicAuthMiddleware) Handler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
errorResponse := `{"error_message": "Not Authorized"}`
username, password, authOK := r.BasicAuth()
if !authOK {
log.Println("[Middleware:BasicAuth] No authentication headers sent")
w.WriteHeader(http.StatusUnauthorized)
w.Write([]byte(errorResponse))
return
}
if err := b.store.CheckPassword(username, password); err != nil {
log.Println("[Middleware:BasicAuth] Invalid username or password:", username)
w.WriteHeader(http.StatusUnauthorized)
w.Write([]byte(errorResponse))
return
}
user, err := b.store.GetUserByUsername(username)
if err != nil || user == nil {
log.Println("[Middleware:BasicAuth] User not found:", username)
w.WriteHeader(http.StatusUnauthorized)
w.Write([]byte(errorResponse))
return
}
log.Println("[Middleware:BasicAuth] User authenticated:", username)
b.store.SetLastLogin(user.ID)
ctx := r.Context()
ctx = context.WithValue(ctx, "UserId", user.ID)
ctx = context.WithValue(ctx, "UserTimezone", user.Timezone)
ctx = context.WithValue(ctx, "IsAdminUser", user.IsAdmin)
ctx = context.WithValue(ctx, "IsAuthenticated", true)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
func NewBasicAuthMiddleware(s *storage.Storage) *BasicAuthMiddleware {
return &BasicAuthMiddleware{store: s}
}

48
server/middleware/csrf.go Normal file
View file

@ -0,0 +1,48 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package middleware
import (
"context"
"github.com/miniflux/miniflux2/helper"
"log"
"net/http"
)
func Csrf(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var csrfToken string
csrfCookie, err := r.Cookie("csrfToken")
if err == http.ErrNoCookie || csrfCookie.Value == "" {
csrfToken = helper.GenerateRandomString(64)
cookie := &http.Cookie{
Name: "csrfToken",
Value: csrfToken,
Path: "/",
Secure: r.URL.Scheme == "https",
HttpOnly: true,
}
http.SetCookie(w, cookie)
} else {
csrfToken = csrfCookie.Value
}
ctx := r.Context()
ctx = context.WithValue(ctx, "CsrfToken", csrfToken)
w.Header().Add("Vary", "Cookie")
isTokenValid := csrfToken == r.FormValue("csrf") || csrfToken == r.Header.Get("X-Csrf-Token")
if r.Method == "POST" && !isTokenValid {
log.Println("[Middleware:CSRF] Invalid or missing CSRF token!")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Invalid or missing CSRF token!"))
} else {
next.ServeHTTP(w, r.WithContext(ctx))
}
})
}

31
server/middleware/middleware.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package middleware
import (
"net/http"
)
type Middleware func(http.Handler) http.Handler
type MiddlewareChain struct {
middlewares []Middleware
}
func (m *MiddlewareChain) Wrap(h http.Handler) http.Handler {
for i := range m.middlewares {
h = m.middlewares[len(m.middlewares)-1-i](h)
}
return h
}
func (m *MiddlewareChain) WrapFunc(fn http.HandlerFunc) http.Handler {
return m.Wrap(fn)
}
func NewMiddlewareChain(middlewares ...Middleware) *MiddlewareChain {
return &MiddlewareChain{append(([]Middleware)(nil), middlewares...)}
}

72
server/middleware/session.go Normal file
View file

@ -0,0 +1,72 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package middleware
import (
"context"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/server/route"
"github.com/miniflux/miniflux2/storage"
"log"
"net/http"
"github.com/gorilla/mux"
)
type SessionMiddleware struct {
store *storage.Storage
router *mux.Router
}
func (s *SessionMiddleware) Handler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
session := s.getSessionFromCookie(r)
if session == nil {
log.Println("[Middleware:Session] Session not found")
if s.isPublicRoute(r) {
next.ServeHTTP(w, r)
} else {
http.Redirect(w, r, route.GetRoute(s.router, "login"), http.StatusFound)
}
} else {
log.Println("[Middleware:Session]", session)
ctx := r.Context()
ctx = context.WithValue(ctx, "UserId", session.UserID)
ctx = context.WithValue(ctx, "IsAuthenticated", true)
next.ServeHTTP(w, r.WithContext(ctx))
}
})
}
func (s *SessionMiddleware) isPublicRoute(r *http.Request) bool {
route := mux.CurrentRoute(r)
switch route.GetName() {
case "login", "checkLogin", "stylesheet", "javascript":
return true
default:
return false
}
}
func (s *SessionMiddleware) getSessionFromCookie(r *http.Request) *model.Session {
sessionCookie, err := r.Cookie("sessionID")
if err == http.ErrNoCookie {
return nil
}
session, err := s.store.GetSessionByToken(sessionCookie.Value)
if err != nil {
log.Println(err)
return nil
}
return session
}
func NewSessionMiddleware(s *storage.Storage, r *mux.Router) *SessionMiddleware {
return &SessionMiddleware{store: s, router: r}
}

37
server/route/route.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package route
import (
"log"
"strconv"
"github.com/gorilla/mux"
)
func GetRoute(router *mux.Router, name string, args ...interface{}) string {
route := router.Get(name)
if route == nil {
log.Fatalln("Route not found:", name)
}
var pairs []string
for _, param := range args {
switch param.(type) {
case string:
pairs = append(pairs, param.(string))
case int64:
val := param.(int64)
pairs = append(pairs, strconv.FormatInt(val, 10))
}
}
result, err := route.URLPath(pairs...)
if err != nil {
log.Fatalln(err)
}
return result.String()
}

132
server/routes.go Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package server
import (
"github.com/miniflux/miniflux2/locale"
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/reader/opml"
api_controller "github.com/miniflux/miniflux2/server/api/controller"
"github.com/miniflux/miniflux2/server/core"
"github.com/miniflux/miniflux2/server/middleware"
"github.com/miniflux/miniflux2/server/template"
ui_controller "github.com/miniflux/miniflux2/server/ui/controller"
"github.com/miniflux/miniflux2/storage"
"net/http"
"github.com/gorilla/mux"
)
func getRoutes(store *storage.Storage, feedHandler *feed.Handler) *mux.Router {
router := mux.NewRouter()
translator := locale.Load()
templateEngine := template.NewTemplateEngine(router, translator)
apiController := api_controller.NewController(store, feedHandler)
uiController := ui_controller.NewController(store, feedHandler, opml.NewOpmlHandler(store))
apiHandler := core.NewHandler(store, router, templateEngine, translator, middleware.NewMiddlewareChain(
middleware.NewBasicAuthMiddleware(store).Handler,
))
uiHandler := core.NewHandler(store, router, templateEngine, translator, middleware.NewMiddlewareChain(
middleware.NewSessionMiddleware(store, router).Handler,
middleware.Csrf,
))
router.Handle("/v1/users", apiHandler.Use(apiController.CreateUser)).Methods("POST")
router.Handle("/v1/users", apiHandler.Use(apiController.GetUsers)).Methods("GET")
router.Handle("/v1/users/{userID}", apiHandler.Use(apiController.GetUser)).Methods("GET")
router.Handle("/v1/users/{userID}", apiHandler.Use(apiController.UpdateUser)).Methods("PUT")
router.Handle("/v1/users/{userID}", apiHandler.Use(apiController.RemoveUser)).Methods("DELETE")
router.Handle("/v1/categories", apiHandler.Use(apiController.CreateCategory)).Methods("POST")
router.Handle("/v1/categories", apiHandler.Use(apiController.GetCategories)).Methods("GET")
router.Handle("/v1/categories/{categoryID}", apiHandler.Use(apiController.UpdateCategory)).Methods("PUT")
router.Handle("/v1/categories/{categoryID}", apiHandler.Use(apiController.RemoveCategory)).Methods("DELETE")
router.Handle("/v1/discover", apiHandler.Use(apiController.GetSubscriptions)).Methods("POST")
router.Handle("/v1/feeds", apiHandler.Use(apiController.CreateFeed)).Methods("POST")
router.Handle("/v1/feeds", apiHandler.Use(apiController.GetFeeds)).Methods("Get")
router.Handle("/v1/feeds/{feedID}/refresh", apiHandler.Use(apiController.RefreshFeed)).Methods("PUT")
router.Handle("/v1/feeds/{feedID}", apiHandler.Use(apiController.GetFeed)).Methods("GET")
router.Handle("/v1/feeds/{feedID}", apiHandler.Use(apiController.UpdateFeed)).Methods("PUT")
router.Handle("/v1/feeds/{feedID}", apiHandler.Use(apiController.RemoveFeed)).Methods("DELETE")
router.Handle("/v1/feeds/{feedID}/entries", apiHandler.Use(apiController.GetFeedEntries)).Methods("GET")
router.Handle("/v1/feeds/{feedID}/entries/{entryID}", apiHandler.Use(apiController.GetEntry)).Methods("GET")
router.Handle("/v1/feeds/{feedID}/entries/{entryID}", apiHandler.Use(apiController.SetEntryStatus)).Methods("PUT")
router.Handle("/stylesheets/{name}.css", uiHandler.Use(uiController.Stylesheet)).Name("stylesheet").Methods("GET")
router.Handle("/js", uiHandler.Use(uiController.Javascript)).Name("javascript").Methods("GET")
router.Handle("/favicon.ico", uiHandler.Use(uiController.Favicon)).Name("favicon").Methods("GET")
router.Handle("/subscribe", uiHandler.Use(uiController.AddSubscription)).Name("addSubscription").Methods("GET")
router.Handle("/subscribe", uiHandler.Use(uiController.SubmitSubscription)).Name("submitSubscription").Methods("POST")
router.Handle("/subscriptions", uiHandler.Use(uiController.ChooseSubscription)).Name("chooseSubscription").Methods("POST")
router.Handle("/unread", uiHandler.Use(uiController.ShowUnreadPage)).Name("unread").Methods("GET")
router.Handle("/history", uiHandler.Use(uiController.ShowHistoryPage)).Name("history").Methods("GET")
router.Handle("/feed/{feedID}/refresh", uiHandler.Use(uiController.RefreshFeed)).Name("refreshFeed").Methods("GET")
router.Handle("/feed/{feedID}/edit", uiHandler.Use(uiController.EditFeed)).Name("editFeed").Methods("GET")
router.Handle("/feed/{feedID}/remove", uiHandler.Use(uiController.RemoveFeed)).Name("removeFeed").Methods("GET")
router.Handle("/feed/{feedID}/update", uiHandler.Use(uiController.UpdateFeed)).Name("updateFeed").Methods("POST")
router.Handle("/feed/{feedID}/entries", uiHandler.Use(uiController.ShowFeedEntries)).Name("feedEntries").Methods("GET")
router.Handle("/feeds", uiHandler.Use(uiController.ShowFeedsPage)).Name("feeds").Methods("GET")
router.Handle("/unread/entry/{entryID}", uiHandler.Use(uiController.ShowUnreadEntry)).Name("unreadEntry").Methods("GET")
router.Handle("/history/entry/{entryID}", uiHandler.Use(uiController.ShowReadEntry)).Name("readEntry").Methods("GET")
router.Handle("/feed/{feedID}/entry/{entryID}", uiHandler.Use(uiController.ShowFeedEntry)).Name("feedEntry").Methods("GET")
router.Handle("/category/{categoryID}/entry/{entryID}", uiHandler.Use(uiController.ShowCategoryEntry)).Name("categoryEntry").Methods("GET")
router.Handle("/entry/status", uiHandler.Use(uiController.UpdateEntriesStatus)).Name("updateEntriesStatus").Methods("POST")
router.Handle("/categories", uiHandler.Use(uiController.ShowCategories)).Name("categories").Methods("GET")
router.Handle("/category/create", uiHandler.Use(uiController.CreateCategory)).Name("createCategory").Methods("GET")
router.Handle("/category/save", uiHandler.Use(uiController.SaveCategory)).Name("saveCategory").Methods("POST")
router.Handle("/category/{categoryID}/entries", uiHandler.Use(uiController.ShowCategoryEntries)).Name("categoryEntries").Methods("GET")
router.Handle("/category/{categoryID}/edit", uiHandler.Use(uiController.EditCategory)).Name("editCategory").Methods("GET")
router.Handle("/category/{categoryID}/update", uiHandler.Use(uiController.UpdateCategory)).Name("updateCategory").Methods("POST")
router.Handle("/category/{categoryID}/remove", uiHandler.Use(uiController.RemoveCategory)).Name("removeCategory").Methods("GET")
router.Handle("/icon/{iconID}", uiHandler.Use(uiController.ShowIcon)).Name("icon").Methods("GET")
router.Handle("/proxy/{encodedURL}", uiHandler.Use(uiController.ImageProxy)).Name("proxy").Methods("GET")
router.Handle("/users", uiHandler.Use(uiController.ShowUsers)).Name("users").Methods("GET")
router.Handle("/user/create", uiHandler.Use(uiController.CreateUser)).Name("createUser").Methods("GET")
router.Handle("/user/save", uiHandler.Use(uiController.SaveUser)).Name("saveUser").Methods("POST")
router.Handle("/users/{userID}/edit", uiHandler.Use(uiController.EditUser)).Name("editUser").Methods("GET")
router.Handle("/users/{userID}/update", uiHandler.Use(uiController.UpdateUser)).Name("updateUser").Methods("POST")
router.Handle("/users/{userID}/remove", uiHandler.Use(uiController.RemoveUser)).Name("removeUser").Methods("GET")
router.Handle("/about", uiHandler.Use(uiController.AboutPage)).Name("about").Methods("GET")
router.Handle("/settings", uiHandler.Use(uiController.ShowSettings)).Name("settings").Methods("GET")
router.Handle("/settings", uiHandler.Use(uiController.UpdateSettings)).Name("updateSettings").Methods("POST")
router.Handle("/sessions", uiHandler.Use(uiController.ShowSessions)).Name("sessions").Methods("GET")
router.Handle("/sessions/{sessionID}/remove", uiHandler.Use(uiController.RemoveSession)).Name("removeSession").Methods("GET")
router.Handle("/export", uiHandler.Use(uiController.Export)).Name("export").Methods("GET")
router.Handle("/import", uiHandler.Use(uiController.Import)).Name("import").Methods("GET")
router.Handle("/upload", uiHandler.Use(uiController.UploadOPML)).Name("uploadOPML").Methods("POST")
router.Handle("/login", uiHandler.Use(uiController.CheckLogin)).Name("checkLogin").Methods("POST")
router.Handle("/logout", uiHandler.Use(uiController.Logout)).Name("logout").Methods("GET")
router.Handle("/", uiHandler.Use(uiController.ShowLoginPage)).Name("login").Methods("GET")
router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("OK"))
})
router.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("User-agent: *\nDisallow: /"))
})
return router
}

33
server/server.go Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package server
import (
"github.com/miniflux/miniflux2/config"
"github.com/miniflux/miniflux2/reader/feed"
"github.com/miniflux/miniflux2/storage"
"log"
"net/http"
"time"
)
func NewServer(cfg *config.Config, store *storage.Storage, feedHandler *feed.Handler) *http.Server {
server := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 60 * time.Second,
Addr: cfg.Get("LISTEN_ADDR", "127.0.0.1:8080"),
Handler: getRoutes(store, feedHandler),
}
go func() {
log.Printf("Listening on %s\n", server.Addr)
if err := server.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
return server
}

12
server/static/bin.go Normal file
View file

File diff suppressed because one or more lines are too long

BIN
server/static/bin/favicon.ico Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 17 KiB

14
server/static/css.go Normal file
View file

File diff suppressed because one or more lines are too long

197
server/static/css/black.css Normal file
View file

@ -0,0 +1,197 @@
/* Layout */
body {
background: #222;
color: #efefef;
}
h1, h2, h3 {
color: #aaa;
}
a {
color: #aaa;
}
a:focus,
a:hover {
color: #ddd;
}
.header li {
border-color: #333;
}
.header a {
color: #ddd;
font-weight: 400;
}
.header .active a {
font-weight: 400;
color: #9b9494;
}
.header a:focus,
.header a:hover {
color: rgba(82, 168, 236, 0.85);
}
.page-header h1 {
border-color: #333;
}
.logo a:hover span {
color: #555;
}
/* Tables */
table, th, td {
border: 1px solid #555;
}
th {
background: #333;
color: #aaa;
font-weight: 400;
}
tr:hover {
background-color: #333;
color: #aaa;
}
/* Forms */
input[type="url"],
input[type="password"],
input[type="text"] {
border: 1px solid #555;
background: #333;
color: #ccc;
}
input[type="url"]:focus,
input[type="password"]:focus,
input[type="text"]:focus {
color: #efefef;
border-color: rgba(82, 168, 236, 0.8);
box-shadow: 0 0 8px rgba(82, 168, 236, 0.6);
}
/* Buttons */
.button-primary {
border-color: #444;
background: #333;
color: #efefef;
}
.button-primary:hover,
.button-primary:focus {
border-color: #888;
background: #555;
}
/* Alerts */
.alert,
.alert-success,
.alert-error,
.alert-info,
.alert-normal {
color: #efefef;
background-color: #333;
border-color: #444;
}
/* Panel */
.panel {
background: #333;
border-color: #555;
}
/* Counter */
.unread-counter {
color: #bbb;
}
/* Category label */
.category {
color: #efefef;
background-color: #333;
border-color: #444;
}
.category a {
color: #999;
}
.category a:hover,
.category a:focus {
color: #aaa;
}
/* Pagination */
.pagination a {
color: #aaa;
}
.pagination-bottom {
border-color: #333;
}
/* List view */
.item {
border-color: #666;
padding: 4px;
}
.item.current-item {
border-width: 2px;
border-color: rgba(82, 168, 236, 0.8);
box-shadow: 0 0 8px rgba(82, 168, 236, 0.6);
}
.item-title a {
font-weight: 400;
}
.item-status-read .item-title a {
color: #666;
}
.item-status-read .item-title a:focus,
.item-status-read .item-title a:hover {
color: rgba(82, 168, 236, 0.6);
}
.item-meta a:hover,
.item-meta a:focus {
color: #aaa;
}
.item-meta li:after {
color: #ddd;
}
/* Entry view */
.entry header {
border-color: #333;
}
.entry header h1 a {
color: #bbb;
}
.entry-content,
.entry-content p, ul {
color: #999;
}
.entry-content pre,
.entry-content code {
color: #fff;
background: #555;
border-color: #888;
}
.entry-enclosure {
border-color: #333;
}

654
server/static/css/common.css Normal file
View file

@ -0,0 +1,654 @@
/* Layout */
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
text-rendering: optimizeLegibility;
}
.main {
padding-left: 3px;
padding-right: 3px;
}
a {
color: #3366CC;
}
a:focus {
outline: 0;
color: red;
text-decoration: none;
border: 1px dotted #aaa;
}
a:hover {
color: #333;
text-decoration: none;
}
.header {
margin-top: 10px;
margin-bottom: 20px;
}
.header nav ul {
display: none;
}
.header li {
cursor: pointer;
padding-left: 10px;
line-height: 2.1em;
font-size: 1.2em;
border-bottom: 1px dotted #ddd;
}
.header li:hover a {
color: #888;
}
.header a {
font-size: 0.9em;
color: #444;
text-decoration: none;
border: none;
}
.header .active a {
font-weight: 600;
}
.header a:hover,
.header a:focus {
color: #888;
}
.page-header {
margin-bottom: 25px;
}
.page-header h1 {
font-weight: 500;
border-bottom: 1px dotted #ddd;
}
.page-header ul {
margin-left: 25px;
font-size: 0.9em;
}
.page-header li {
list-style-type: circle;
line-height: 1.4em;
}
.logo {
cursor: pointer;
text-align: center;
}
.logo a {
color: #000;
letter-spacing: 1px;
}
.logo a:hover {
color: #339966;
}
.logo a span {
color: #339966;
}
.logo a:hover span {
color: #000;
}
@media (min-width: 600px) {
body {
margin: auto;
max-width: 750px;
}
.logo {
text-align: left;
float: left;
margin-right: 15px;
}
.header nav ul {
display: block;
}
.header li {
display: inline;
padding: 0;
padding-right: 15px;
line-height: normal;
font-size: 1.0em;
border: none;
}
.page-header ul {
margin-left: 0;
}
.page-header li {
display: inline;
padding-right: 15px;
}
}
/* Tables */
table {
width: 100%;
border-collapse: collapse;
}
table, th, td {
border: 1px solid #ddd;
}
th, td {
padding: 5px;
text-align: left;
}
td {
vertical-align: top;
}
th {
background: #fcfcfc;
}
.table-overflow td {
max-width: 0;
text-overflow: ellipsis;
white-space: nowrap;
overflow: hidden;
}
tr:hover {
background-color: #f9f9f9;
}
.column-40 {
width: 40%;
}
.column-25 {
width: 25%;
}
.column-20 {
width: 20%;
}
/* Forms */
label {
cursor: pointer;
display: block;
}
.radio-group {
line-height: 1.9em;
}
div.radio-group label {
display: inline-block;
}
select {
margin-bottom: 15px;
}
input[type="url"],
input[type="password"],
input[type="text"] {
border: 1px solid #ccc;
padding: 3px;
line-height: 15px;
width: 250px;
font-size: 99%;
margin-bottom: 10px;
margin-top: 5px;
-webkit-appearance: none;
}
input[type="url"]:focus,
input[type="password"]:focus,
input[type="text"]:focus {
color: #000;
border-color: rgba(82, 168, 236, 0.8);
outline: 0;
box-shadow: 0 0 8px rgba(82, 168, 236, 0.6);
}
::-moz-placeholder,
::-ms-input-placeholder,
::-webkit-input-placeholder {
color: #ddd;
padding-top: 2px;
}
.form-help {
font-size: 0.9em;
color: brown;
margin-bottom: 15px;
}
/* Buttons */
a.button {
text-decoration: none;
}
.button {
display: inline-block;
-webkit-appearance: none;
-moz-appearance: none;
font-size: 1.1em;
cursor: pointer;
padding: 3px 10px;
border: 1px solid;
border-radius: unset;
}
.button-primary {
border-color: #3079ed;
background: #4d90fe;
color: #fff;
}
.button-primary:hover,
.button-primary:focus {
border-color: #2f5bb7;
background: #357ae8;
}
.button-danger {
border-color: #b0281a;
background: #d14836;
color: #fff;
}
.button-danger:hover,
.button-danger:focus {
color: #fff;
background: #c53727;
}
.button:disabled {
color: #ccc;
background: #f7f7f7;
border-color: #ccc;
}
.buttons {
margin-top: 10px;
margin-bottom: 20px;
}
/* Alerts */
.alert {
padding: 8px 35px 8px 14px;
margin-bottom: 20px;
color: #c09853;
background-color: #fcf8e3;
border: 1px solid #fbeed5;
border-radius: 4px;
overflow: auto;
}
.alert h3 {
margin-top: 0;
margin-bottom: 15px;
}
.alert-success {
color: #468847;
background-color: #dff0d8;
border-color: #d6e9c6;
}
.alert-error {
color: #b94a48;
background-color: #f2dede;
border-color: #eed3d7;
}
.alert-error a {
color: #b94a48;
}
.alert-info {
color: #3a87ad;
background-color: #d9edf7;
border-color: #bce8f1;
}
/* Panel */
.panel {
color: #333;
background-color: #f0f0f0;
border: 1px solid #ddd;
border-radius: 5px;
padding: 10px;
margin-bottom: 15px;
}
.panel h3 {
font-weight: 500;
margin-top: 0;
margin-bottom: 20px;
}
.panel ul {
margin-left: 30px;
}
/* Login form */
.login-form {
margin: auto;
margin-top: 50px;
width: 350px;
}
/* Counter */
.unread-counter {
font-size: 0.8em;
font-weight: 300;
color: #666;
}
/* Category label */
.category {
font-size: 0.75em;
background-color: #fffcd7;
border: 1px solid #d5d458;
border-radius: 5px;
margin-left: 0.25em;
padding: 1px 0.4em 1px 0.4em;
white-space: nowrap;
}
.category a {
color: #555;
text-decoration: none;
}
.category a:hover,
.category a:focus {
color: #000;
}
/* Pagination */
.pagination {
font-size: 1.1em;
display: flex;
align-items: center;
padding-top: 8px;
}
.pagination-bottom {
border-top: 1px dotted #ddd;
margin-bottom: 15px;
margin-top: 50px;
}
.pagination > div {
flex: 1;
}
.pagination-next {
text-align: right;
}
.pagination-prev:before {
content: "« ";
}
.pagination-next:after {
content: " »";
}
.pagination a {
color: #333;
}
.pagination a:hover,
.pagination a:focus {
text-decoration: none;
}
/* List view */
.item {
border: 1px dotted #ddd;
margin-bottom: 20px;
padding: 5px;
overflow: hidden;
}
.item.current-item {
border: 3px solid #bce;
padding: 3px;
}
.item-title a {
text-decoration: none;
font-weight: 600;
}
.item-status-read .item-title a {
color: #777;
}
.item-meta {
color: #777;
font-size: 0.8em;
}
.item-meta a {
color: #777;
text-decoration: none;
}
.item-meta a:hover,
.item-meta a:focus {
color: #333;
}
.item-meta ul {
margin-top: 5px;
}
.item-meta li {
display: inline;
}
.item-meta li:after {
content: "|";
color: #aaa;
}
.item-meta li:last-child:after {
content: "";
}
.hide-read-items .item-status-read {
display: none;
}
/* Entry view */
.entry header {
padding-bottom: 5px;
border-bottom: 1px dotted #ddd;
}
.entry header h1 {
font-size: 2.0em;
line-height: 1.25em;
margin: 30px 0;
}
.entry header h1 a {
text-decoration: none;
color: #333;
}
.entry header h1 a:hover,
.entry header h1 a:focus {
color: #666;
}
.entry-meta {
font-size: 0.95em;
margin: 0 0 20px;
color: #666;
}
.entry-website img {
vertical-align: top;
}
.entry-website a {
color: #666;
vertical-align: top;
text-decoration: none;
}
.entry-website a:hover,
.entry-website a:focus {
text-decoration: underline;
}
.entry-date {
font-size: 0.65em;
font-style: italic;
color: #555;
}
.entry-content {
padding-top: 15px;
font-size: 1.1em;
font-weight: 300;
color: #444;
}
.entry-content h1, h2, h3, h4, h5, h6 {
margin-top: 15px;
}
.entry-content iframe,
.entry-content video,
.entry-content img {
max-width: 100%;
}
.entry-content figure img {
border: 1px solid #000;
}
.entry-content figcaption {
font-size: 0.75em;
text-transform: uppercase;
color: #777;
}
.entry-content p {
margin-top: 15px;
margin-bottom: 15px;
text-align: justify;
}
.entry-content a:visited {
color: purple;
}
.entry-content dt {
font-weight: 500;
margin-top: 15px;
color: #555;
}
.entry-content dd {
margin-left: 15px;
margin-top: 5px;
padding-left: 20px;
border-left: 3px solid #ddd;
color: #777;
font-weight: 300;
line-height: 1.4em;
}
.entry-content blockquote {
border-left: 4px solid #ddd;
padding-left: 25px;
margin-left: 20px;
margin-top: 20px;
margin-bottom: 20px;
color: #888;
line-height: 1.4em;
font-family: Georgia, serif;
}
.entry-content blockquote + p {
color: #555;
font-style: italic;
font-weight: 200;
}
.entry-content q {
color: purple;
font-family: Georgia, serif;
font-style: italic;
}
.entry-content q:before {
content: "“";
}
.entry-content q:after {
content: "”";
}
.entry-content pre {
padding: 5px;
background: #f0f0f0;
border: 1px solid #ddd;
overflow: scroll;
}
.entry-content ul,
.entry-content ol {
margin-left: 30px;
}
.entry-content ul {
list-style-type: square;
}
.entry-enclosures h3 {
font-weight: 500;
}
.entry-enclosure {
border: 1px dotted #ddd;
padding: 5px;
margin-top: 10px;
max-width: 100%;
}
.entry-enclosure-download {
font-size: 0.85em;
}
.enclosure-video video,
.enclosure-image img {
max-width: 100%;
}

52
server/static/js.go Normal file
View file

@ -0,0 +1,52 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-19 22:01:21.923282889 -0800 PST m=+0.004116032
package static
var Javascript = map[string]string{
"app": `(function(){'use strict';class KeyboardHandler{constructor(){this.queue=[];this.shortcuts={};}
on(combination,callback){this.shortcuts[combination]=callback;}
listen(){document.onkeydown=(event)=>{if(this.isEventIgnored(event)){return;}
let key=this.getKey(event);this.queue.push(key);for(let combination in this.shortcuts){let keys=combination.split(" ");if(keys.every((value,index)=>value===this.queue[index])){this.queue=[];this.shortcuts[combination]();return;}
if(keys.length===1&&key===keys[0]){this.queue=[];this.shortcuts[combination]();return;}}
if(this.queue.length>=2){this.queue=[];}};}
isEventIgnored(event){return event.target.tagName==="INPUT"||event.target.tagName==="TEXTAREA";}
getKey(event){const mapping={'Esc':'Escape','Up':'ArrowUp','Down':'ArrowDown','Left':'ArrowLeft','Right':'ArrowRight'};for(let key in mapping){if(mapping.hasOwnProperty(key)&&key===event.key){return mapping[key];}}
return event.key;}}
class FormHandler{static handleSubmitButtons(){let elements=document.querySelectorAll("form");elements.forEach(function(element){element.onsubmit=function(){let button=document.querySelector("button");if(button){button.innerHTML=button.dataset.labelLoading;button.disabled=true;}};});}}
class MouseHandler{onClick(selector,callback){let elements=document.querySelectorAll(selector);elements.forEach((element)=>{element.onclick=(event)=>{event.preventDefault();callback(event);};});}}
class App{run(){FormHandler.handleSubmitButtons();let keyboardHandler=new KeyboardHandler();keyboardHandler.on("g u",()=>this.goToPage("unread"));keyboardHandler.on("g h",()=>this.goToPage("history"));keyboardHandler.on("g f",()=>this.goToPage("feeds"));keyboardHandler.on("g c",()=>this.goToPage("categories"));keyboardHandler.on("g s",()=>this.goToPage("settings"));keyboardHandler.on("ArrowLeft",()=>this.goToPrevious());keyboardHandler.on("ArrowRight",()=>this.goToNext());keyboardHandler.on("j",()=>this.goToPrevious());keyboardHandler.on("p",()=>this.goToPrevious());keyboardHandler.on("k",()=>this.goToNext());keyboardHandler.on("n",()=>this.goToNext());keyboardHandler.on("h",()=>this.goToPage("previous"));keyboardHandler.on("l",()=>this.goToPage("next"));keyboardHandler.on("o",()=>this.openSelectedItem());keyboardHandler.on("v",()=>this.openOriginalLink());keyboardHandler.on("m",()=>this.toggleEntryStatus());keyboardHandler.on("A",()=>this.markPageAsRead());keyboardHandler.listen();let mouseHandler=new MouseHandler();mouseHandler.onClick("a[data-on-click=markPageAsRead]",()=>this.markPageAsRead());if(document.documentElement.clientWidth<600){mouseHandler.onClick(".logo",()=>this.toggleMainMenu());mouseHandler.onClick(".header nav li",(event)=>this.clickMenuListItem(event));}}
clickMenuListItem(event){let element=event.target;console.log(element);if(element.tagName==="A"){window.location.href=element.getAttribute("href");}else{window.location.href=element.querySelector("a").getAttribute("href");}}
toggleMainMenu(){let menu=document.querySelector(".header nav ul");if(this.isVisible(menu)){menu.style.display="none";}else{menu.style.display="block";}}
updateEntriesStatus(entryIDs,status){let url=document.body.dataset.entriesStatusUrl;let request=new Request(url,{method:"POST",cache:"no-cache",credentials:"include",body:JSON.stringify({entry_ids:entryIDs,status:status}),headers:new Headers({"Content-Type":"application/json","X-Csrf-Token":this.getCsrfToken()})});fetch(request);}
markPageAsRead(){let items=this.getVisibleElements(".items .item");let entryIDs=[];items.forEach((element)=>{element.classList.add("item-status-read");entryIDs.push(parseInt(element.dataset.id,10));});if(entryIDs.length>0){this.updateEntriesStatus(entryIDs,"read");}
this.goToPage("next");}
toggleEntryStatus(){let currentItem=document.querySelector(".current-item");if(currentItem!==null){let entryID=parseInt(currentItem.dataset.id,10);let statuses={read:"unread",unread:"read"};for(let currentStatus in statuses){let newStatus=statuses[currentStatus];if(currentItem.classList.contains("item-status-"+currentStatus)){this.goToNextListItem();currentItem.classList.remove("item-status-"+currentStatus);currentItem.classList.add("item-status-"+newStatus);this.updateEntriesStatus([entryID],newStatus);break;}}}}
openOriginalLink(){let entryLink=document.querySelector(".entry h1 a");if(entryLink!==null){this.openNewTab(entryLink.getAttribute("href"));return;}
let currentItemOriginalLink=document.querySelector(".current-item a[data-original-link]");if(currentItemOriginalLink!==null){this.openNewTab(currentItemOriginalLink.getAttribute("href"));}}
openSelectedItem(){let currentItemLink=document.querySelector(".current-item .item-title a");if(currentItemLink!==null){window.location.href=currentItemLink.getAttribute("href");}}
goToPage(page){let element=document.querySelector("a[data-page="+page+"]");if(element){document.location.href=element.href;}}
goToPrevious(){if(this.isListView()){this.goToPreviousListItem();}else{this.goToPage("previous");}}
goToNext(){if(this.isListView()){this.goToNextListItem();}else{this.goToPage("next");}}
goToPreviousListItem(){let items=this.getVisibleElements(".items .item");if(items.length===0){return;}
if(document.querySelector(".current-item")===null){items[0].classList.add("current-item");return;}
for(let i=0;i<items.length;i++){if(items[i].classList.contains("current-item")){items[i].classList.remove("current-item");if(i-1>=0){items[i-1].classList.add("current-item");this.scrollPageTo(items[i-1]);}
break;}}}
goToNextListItem(){let items=this.getVisibleElements(".items .item");if(items.length===0){return;}
if(document.querySelector(".current-item")===null){items[0].classList.add("current-item");return;}
for(let i=0;i<items.length;i++){if(items[i].classList.contains("current-item")){items[i].classList.remove("current-item");if(i+1<items.length){items[i+1].classList.add("current-item");this.scrollPageTo(items[i+1]);}
break;}}}
getVisibleElements(selector){let elements=document.querySelectorAll(selector);let result=[];for(let i=0;i<elements.length;i++){if(this.isVisible(elements[i])){result.push(elements[i]);}}
return result;}
isListView(){return document.querySelector(".items")!==null;}
scrollPageTo(item){let windowScrollPosition=window.pageYOffset;let windowHeight=document.documentElement.clientHeight;let viewportPosition=windowScrollPosition+windowHeight;let itemBottomPosition=item.offsetTop+item.offsetHeight;if(viewportPosition-itemBottomPosition<0||viewportPosition-item.offsetTop>windowHeight){window.scrollTo(0,item.offsetTop-10);}}
openNewTab(url){let win=window.open(url,"_blank");win.focus();}
isVisible(element){return element.offsetParent!==null;}
getCsrfToken(){let element=document.querySelector("meta[name=X-CSRF-Token]");if(element!==null){return element.getAttribute("value");}
return "";}}
document.addEventListener("DOMContentLoaded",function(){(new App()).run();});})();`,
}
var JavascriptChecksums = map[string]string{
"app": "e250c2af19dea14fd75681a81080cf183919a7a589b0886a093586ee894c8282",
}

351
server/static/js/app.js Normal file
View file

@ -0,0 +1,351 @@
/*jshint esversion: 6 */
(function() {
'use strict';
class KeyboardHandler {
constructor() {
this.queue = [];
this.shortcuts = {};
}
on(combination, callback) {
this.shortcuts[combination] = callback;
}
listen() {
document.onkeydown = (event) => {
if (this.isEventIgnored(event)) {
return;
}
let key = this.getKey(event);
this.queue.push(key);
for (let combination in this.shortcuts) {
let keys = combination.split(" ");
if (keys.every((value, index) => value === this.queue[index])) {
this.queue = [];
this.shortcuts[combination]();
return;
}
if (keys.length === 1 && key === keys[0]) {
this.queue = [];
this.shortcuts[combination]();
return;
}
}
if (this.queue.length >= 2) {
this.queue = [];
}
};
}
isEventIgnored(event) {
return event.target.tagName === "INPUT" || event.target.tagName === "TEXTAREA";
}
getKey(event) {
const mapping = {
'Esc': 'Escape',
'Up': 'ArrowUp',
'Down': 'ArrowDown',
'Left': 'ArrowLeft',
'Right': 'ArrowRight'
};
for (let key in mapping) {
if (mapping.hasOwnProperty(key) && key === event.key) {
return mapping[key];
}
}
return event.key;
}
}
class FormHandler {
static handleSubmitButtons() {
let elements = document.querySelectorAll("form");
elements.forEach(function (element) {
element.onsubmit = function () {
let button = document.querySelector("button");
if (button) {
button.innerHTML = button.dataset.labelLoading;
button.disabled = true;
}
};
});
}
}
class MouseHandler {
onClick(selector, callback) {
let elements = document.querySelectorAll(selector);
elements.forEach((element) => {
element.onclick = (event) => {
event.preventDefault();
callback(event);
};
});
}
}
class App {
run() {
FormHandler.handleSubmitButtons();
let keyboardHandler = new KeyboardHandler();
keyboardHandler.on("g u", () => this.goToPage("unread"));
keyboardHandler.on("g h", () => this.goToPage("history"));
keyboardHandler.on("g f", () => this.goToPage("feeds"));
keyboardHandler.on("g c", () => this.goToPage("categories"));
keyboardHandler.on("g s", () => this.goToPage("settings"));
keyboardHandler.on("ArrowLeft", () => this.goToPrevious());
keyboardHandler.on("ArrowRight", () => this.goToNext());
keyboardHandler.on("j", () => this.goToPrevious());
keyboardHandler.on("p", () => this.goToPrevious());
keyboardHandler.on("k", () => this.goToNext());
keyboardHandler.on("n", () => this.goToNext());
keyboardHandler.on("h", () => this.goToPage("previous"));
keyboardHandler.on("l", () => this.goToPage("next"));
keyboardHandler.on("o", () => this.openSelectedItem());
keyboardHandler.on("v", () => this.openOriginalLink());
keyboardHandler.on("m", () => this.toggleEntryStatus());
keyboardHandler.on("A", () => this.markPageAsRead());
keyboardHandler.listen();
let mouseHandler = new MouseHandler();
mouseHandler.onClick("a[data-on-click=markPageAsRead]", () => this.markPageAsRead());
if (document.documentElement.clientWidth < 600) {
mouseHandler.onClick(".logo", () => this.toggleMainMenu());
mouseHandler.onClick(".header nav li", (event) => this.clickMenuListItem(event));
}
}
clickMenuListItem(event) {
let element = event.target;console.log(element);
if (element.tagName === "A") {
window.location.href = element.getAttribute("href");
} else {
window.location.href = element.querySelector("a").getAttribute("href");
}
}
toggleMainMenu() {
let menu = document.querySelector(".header nav ul");
if (this.isVisible(menu)) {
menu.style.display = "none";
} else {
menu.style.display = "block";
}
}
updateEntriesStatus(entryIDs, status) {
let url = document.body.dataset.entriesStatusUrl;
let request = new Request(url, {
method: "POST",
cache: "no-cache",
credentials: "include",
body: JSON.stringify({entry_ids: entryIDs, status: status}),
headers: new Headers({
"Content-Type": "application/json",
"X-Csrf-Token": this.getCsrfToken()
})
});
fetch(request);
}
markPageAsRead() {
let items = this.getVisibleElements(".items .item");
let entryIDs = [];
items.forEach((element) => {
element.classList.add("item-status-read");
entryIDs.push(parseInt(element.dataset.id, 10));
});
if (entryIDs.length > 0) {
this.updateEntriesStatus(entryIDs, "read");
}
this.goToPage("next");
}
toggleEntryStatus() {
let currentItem = document.querySelector(".current-item");
if (currentItem !== null) {
let entryID = parseInt(currentItem.dataset.id, 10);
let statuses = {read: "unread", unread: "read"};
for (let currentStatus in statuses) {
let newStatus = statuses[currentStatus];
if (currentItem.classList.contains("item-status-" + currentStatus)) {
this.goToNextListItem();
currentItem.classList.remove("item-status-" + currentStatus);
currentItem.classList.add("item-status-" + newStatus);
this.updateEntriesStatus([entryID], newStatus);
break;
}
}
}
}
openOriginalLink() {
let entryLink = document.querySelector(".entry h1 a");
if (entryLink !== null) {
this.openNewTab(entryLink.getAttribute("href"));
return;
}
let currentItemOriginalLink = document.querySelector(".current-item a[data-original-link]");
if (currentItemOriginalLink !== null) {
this.openNewTab(currentItemOriginalLink.getAttribute("href"));
}
}
openSelectedItem() {
let currentItemLink = document.querySelector(".current-item .item-title a");
if (currentItemLink !== null) {
window.location.href = currentItemLink.getAttribute("href");
}
}
goToPage(page) {
let element = document.querySelector("a[data-page=" + page + "]");
if (element) {
document.location.href = element.href;
}
}
goToPrevious() {
if (this.isListView()) {
this.goToPreviousListItem();
} else {
this.goToPage("previous");
}
}
goToNext() {
if (this.isListView()) {
this.goToNextListItem();
} else {
this.goToPage("next");
}
}
goToPreviousListItem() {
let items = this.getVisibleElements(".items .item");
if (items.length === 0) {
return;
}
if (document.querySelector(".current-item") === null) {
items[0].classList.add("current-item");
return;
}
for (let i = 0; i < items.length; i++) {
if (items[i].classList.contains("current-item")) {
items[i].classList.remove("current-item");
if (i - 1 >= 0) {
items[i - 1].classList.add("current-item");
this.scrollPageTo(items[i - 1]);
}
break;
}
}
}
goToNextListItem() {
let items = this.getVisibleElements(".items .item");
if (items.length === 0) {
return;
}
if (document.querySelector(".current-item") === null) {
items[0].classList.add("current-item");
return;
}
for (let i = 0; i < items.length; i++) {
if (items[i].classList.contains("current-item")) {
items[i].classList.remove("current-item");
if (i + 1 < items.length) {
items[i + 1].classList.add("current-item");
this.scrollPageTo(items[i + 1]);
}
break;
}
}
}
getVisibleElements(selector) {
let elements = document.querySelectorAll(selector);
let result = [];
for (let i = 0; i < elements.length; i++) {
if (this.isVisible(elements[i])) {
result.push(elements[i]);
}
}
return result;
}
isListView() {
return document.querySelector(".items") !== null;
}
scrollPageTo(item) {
let windowScrollPosition = window.pageYOffset;
let windowHeight = document.documentElement.clientHeight;
let viewportPosition = windowScrollPosition + windowHeight;
let itemBottomPosition = item.offsetTop + item.offsetHeight;
if (viewportPosition - itemBottomPosition < 0 || viewportPosition - item.offsetTop > windowHeight) {
window.scrollTo(0, item.offsetTop - 10);
}
}
openNewTab(url) {
let win = window.open(url, "_blank");
win.focus();
}
isVisible(element) {
return element.offsetParent !== null;
}
getCsrfToken() {
let element = document.querySelector("meta[name=X-CSRF-Token]");
if (element !== null) {
return element.getAttribute("value");
}
return "";
}
}
document.addEventListener("DOMContentLoaded", function() {
(new App()).run();
});
})();

111
server/template/common.go Normal file
View file

@ -0,0 +1,111 @@
// Code generated by go generate; DO NOT EDIT.
// 2017-11-19 22:01:21.924938666 -0800 PST m=+0.005771809
package template
var templateCommonMap = map[string]string{
"entry_pagination": `{{ define "entry_pagination" }}
<div class="pagination">
<div class="pagination-prev">
{{ if .prevEntry }}
<a href="{{ .prevEntryRoute }}" title="{{ .prevEntry.Title }}" data-page="previous">{{ t "Previous" }}</a>
{{ else }}
{{ t "Previous" }}
{{ end }}
</div>
<div class="pagination-next">
{{ if .nextEntry }}
<a href="{{ .nextEntryRoute }}" title="{{ .nextEntry.Title }}" data-page="next">{{ t "Next" }}</a>
{{ else }}
{{ t "Next" }}
{{ end }}
</div>
</div>
{{ end }}`,
"layout": `{{ define "base" }}
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width">
<meta name="robots" content="noindex,nofollow">
<meta name="referrer" content="no-referrer">
{{ if .csrf }}
<meta name="X-CSRF-Token" value="{{ .csrf }}">
{{ end }}
<title>{{template "title" .}} - Miniflux</title>
{{ if .user }}
<link rel="stylesheet" type="text/css" href="{{ route "stylesheet" "name" .user.Theme }}">
{{ else }}
<link rel="stylesheet" type="text/css" href="{{ route "stylesheet" "name" "white" }}">
{{ end }}
<script type="text/javascript" src="{{ route "javascript" }}" defer></script>
</head>
<body data-entries-status-url="{{ route "updateEntriesStatus" }}">
{{ if .user }}
<header class="header">
<nav>
<div class="logo">
<a href="{{ route "unread" }}">Mini<span>flux</span></a>
</div>
<ul>
<li {{ if eq .menu "unread" }}class="active"{{ end }}>
<a href="{{ route "unread" }}" data-page="unread">{{ t "Unread" }}</a>
{{ if gt .countUnread 0 }}
<span class="unread-counter" title="Unread articles">({{ .countUnread }})</span>
{{ end }}
</li>
<li {{ if eq .menu "history" }}class="active"{{ end }}>
<a href="{{ route "history" }}" data-page="history">{{ t "History" }}</a>
</li>
<li {{ if eq .menu "feeds" }}class="active"{{ end }}>
<a href="{{ route "feeds" }}" data-page="feeds">{{ t "Feeds" }}</a>
</li>
<li {{ if eq .menu "categories" }}class="active"{{ end }}>
<a href="{{ route "categories" }}" data-page="categories">{{ t "Categories" }}</a>
</li>
<li {{ if eq .menu "settings" }}class="active"{{ end }}>
<a href="{{ route "settings" }}" data-page="settings">{{ t "Settings" }}</a>
</li>
<li>
<a href="{{ route "logout" }}" title="Logged as {{ .user.Username }}">{{ t "Logout" }}</a>
</li>
</ul>
</nav>
</header>
{{ end }}
<section class="main">
{{template "content" .}}
</section>
</body>
</html>
{{ end }}`,
"pagination": `{{ define "pagination" }}
<div class="pagination">
<div class="pagination-prev">
{{ if .ShowPrev }}
<a href="{{ .Route }}{{ if gt .PrevOffset 0 }}?offset={{ .PrevOffset }}{{ end }}" data-page="previous">{{ t "Previous" }}</a>
{{ else }}
{{ t "Previous" }}
{{ end }}
</div>
<div class="pagination-next">
{{ if .ShowNext }}
<a href="{{ .Route }}?offset={{ .NextOffset }}" data-page="next">{{ t "Next" }}</a>
{{ else }}
{{ t "Next" }}
{{ end }}
</div>
</div>
{{ end }}
`,
}
var templateCommonMapChecksums = map[string]string{
"entry_pagination": "f1465fa70f585ae8043b200ec9de5bf437ffbb0c19fb7aefc015c3555614ee27",
"layout": "8be69cc93fdc99eb36841ae645f58488bd675670507dcdb2de0e593602893178",
"pagination": "6ff462c2b2a53bc5448b651da017f40a39f1d4f16cef4b2f09784f0797286924",
}

21
server/template/helper/LICENSE Normal file
View file

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Hervé GOUCHET
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

61
server/template/helper/elapsed.go Normal file
View file

@ -0,0 +1,61 @@
// Copyright (c) 2017 Hervé Gouchet. All rights reserved.
// Use of this source code is governed by the MIT License
// that can be found in the LICENSE file.
package helper
import (
"github.com/miniflux/miniflux2/locale"
"math"
"time"
)
// Texts to be translated if necessary.
var (
NotYet = `not yet`
JustNow = `just now`
LastMinute = `1 minute ago`
Minutes = `%d minutes ago`
LastHour = `1 hour ago`
Hours = `%d hours ago`
Yesterday = `yesterday`
Days = `%d days ago`
Weeks = `%d weeks ago`
Months = `%d months ago`
Years = `%d years ago`
)
// GetElapsedTime returns in a human readable format the elapsed time
// since the given datetime.
func GetElapsedTime(translator *locale.Language, t time.Time) string {
if t.IsZero() || time.Now().Before(t) {
return translator.Get(NotYet)
}
diff := time.Since(t)
// Duration in seconds
s := diff.Seconds()
// Duration in days
d := int(s / 86400)
switch {
case s < 60:
return translator.Get(JustNow)
case s < 120:
return translator.Get(LastMinute)
case s < 3600:
return translator.Get(Minutes, int(diff.Minutes()))
case s < 7200:
return translator.Get(LastHour)
case s < 86400:
return translator.Get(Hours, int(diff.Hours()))
case d == 1:
return translator.Get(Yesterday)
case d < 7:
return translator.Get(Days, d)
case d < 31:
return translator.Get(Weeks, int(math.Ceil(float64(d)/7)))
case d < 365:
return translator.Get(Months, int(math.Ceil(float64(d)/30)))
default:
return translator.Get(Years, int(math.Ceil(float64(d)/365)))
}
}

37
server/template/helper/elapsed_test.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright (c) 2017 Hervé Gouchet. All rights reserved.
// Use of this source code is governed by the MIT License
// that can be found in the LICENSE file.
package helper
import (
"fmt"
"github.com/miniflux/miniflux2/locale"
"testing"
"time"
)
func TestElapsedTime(t *testing.T) {
var dt = []struct {
in time.Time
out string
}{
{time.Time{}, NotYet},
{time.Now().Add(time.Hour), NotYet},
{time.Now(), JustNow},
{time.Now().Add(-time.Minute), LastMinute},
{time.Now().Add(-time.Minute * 40), fmt.Sprintf(Minutes, 40)},
{time.Now().Add(-time.Hour), LastHour},
{time.Now().Add(-time.Hour * 3), fmt.Sprintf(Hours, 3)},
{time.Now().Add(-time.Hour * 32), Yesterday},
{time.Now().Add(-time.Hour * 24 * 3), fmt.Sprintf(Days, 3)},
{time.Now().Add(-time.Hour * 24 * 14), fmt.Sprintf(Weeks, 2)},
{time.Now().Add(-time.Hour * 24 * 60), fmt.Sprintf(Months, 2)},
{time.Now().Add(-time.Hour * 24 * 365 * 3), fmt.Sprintf(Years, 3)},
}
for i, tt := range dt {
if out := GetElapsedTime(&locale.Language{}, tt.in); out != tt.out {
t.Errorf("%d. content mismatch for %v:exp=%q got=%q", i, tt.in, tt.out, out)
}
}
}

37
server/template/html/about.html Normal file
View file

@ -0,0 +1,37 @@
{{ define "title"}}{{ t "About" }}{{ end }}
{{ define "content"}}
<section class="page-header">
<h1>{{ t "About" }}</h1>
<ul>
<li>
<a href="{{ route "settings" }}">{{ t "Settings" }}</a>
</li>
<li>
<a href="{{ route "sessions" }}">{{ t "Sessions" }}</a>
</li>
{{ if .user.IsAdmin }}
<li>
<a href="{{ route "users" }}">{{ t "Users" }}</a>
</li>
{{ end }}
</ul>
</section>
<div class="panel">
<h3>{{ t "Version" }}</h3>
<ul>
<li><strong>{{ t "Version:" }}</strong> {{ .version }}</li>
<li><strong>{{ t "Build Date:" }}</strong> {{ .build_date }}</li>
</ul>
</div>
<div class="panel">
<h3>{{ t "Authors" }}</h3>
<ul>
<li><strong>{{ t "Author:" }}</strong> Frédéric Guillot</li>
<li><strong>{{ t "License:" }}</strong> Apache 2.0</li>
</ul>
</div>
{{ end }}

Some files were not shown because too many files have changed in this diff Show more