// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. // SPDX-License-Identifier: Apache-2.0 package rewrite // import "miniflux.app/v2/internal/reader/rewrite" import ( "os" "reflect" "strings" "testing" "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/model" ) func TestParseRules(t *testing.T) { rulesText := `add_dynamic_image,replace("article/(.*).svg"|"article/$1.png"),remove(".spam, .ads:not(.keep)")` expected := []rule{ {name: "add_dynamic_image"}, {name: "replace", args: []string{"article/(.*).svg", "article/$1.png"}}, {name: "remove", args: []string{".spam, .ads:not(.keep)"}}, } actual := parseRules(rulesText) if !reflect.DeepEqual(expected, actual) { t.Errorf(`Parsed rules do not match expected rules: got %v instead of %v`, actual, expected) } } func TestReplaceTextLinks(t *testing.T) { scenarios := map[string]string{ `This is a link to example.org`: `This is a link to example.org`, `This is a link to ftp://example.org`: `This is a link to ftp://example.org`, `This is a link to www.example.org`: `This is a link to www.example.org`, `This is a link to http://example.org`: `This is a link to http://example.org`, `This is a link to http://example.org, end of sentence.`: `This is a link to http://example.org, end of sentence.`, `This is a link to https://example.org`: `This is a link to https://example.org`, `This is a link to https://www.example.org/path/to?q=s`: `This is a link to https://www.example.org/path/to?q=s`, `This is a link to https://example.org/index#hash-tag, http://example.org/.`: `This is a link to https://example.org/index#hash-tag, http://example.org/.`, } for input, expected := range scenarios { actual := replaceTextLinks(input) if actual != expected { t.Errorf(`Unexpected link replacement, got "%s" instead of "%s"`, actual, expected) } } } func TestRewriteWithNoMatchingRule(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Some text.`, } testEntry := &model.Entry{ Title: `A title`, Content: `Some text.`, } Rewriter("https://example.org/article", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithYoutubeLink(t *testing.T) { config.Opts = config.NewOptions() controlEntry := &model.Entry{ Title: `A title`, Content: `
Video Description`, } testEntry := &model.Entry{ Title: `A title`, Content: `Video Description`, } Rewriter("https://www.youtube.com/watch?v=1234", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithYoutubeLinkAndCustomEmbedURL(t *testing.T) { os.Clearenv() os.Setenv("YOUTUBE_EMBED_URL_OVERRIDE", "https://invidious.custom/embed/") var err error parser := config.NewParser() config.Opts, err = parser.ParseEnvironmentVariables() if err != nil { t.Fatalf(`Parsing failure: %v`, err) } controlEntry := &model.Entry{ Title: `A title`, Content: `
Video Description`, } testEntry := &model.Entry{ Title: `A title`, Content: `Video Description`, } Rewriter("https://www.youtube.com/watch?v=1234", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithInexistingCustomRule(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Video Description`, } testEntry := &model.Entry{ Title: `A title`, Content: `Video Description`, } Rewriter("https://www.youtube.com/watch?v=1234", testEntry, `some rule`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithXkcdLink(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.

Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.

`, } testEntry := &model.Entry{ Title: `A title`, Content: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithXkcdLinkHtmlInjection(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
<foo>

<foo>

`, } testEntry := &model.Entry{ Title: `A title`, Content: `<foo>`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithXkcdLinkAndImageNoTitle(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, } testEntry := &model.Entry{ Title: `A title`, Content: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithXkcdLinkAndNoImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `test`, } testEntry := &model.Entry{ Title: `A title`, Content: `test`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithXkcdAndNoImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `test`, } testEntry := &model.Entry{ Title: `A title`, Content: `test`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteMailtoLink(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `contact [blah blah]`, } testEntry := &model.Entry{ Title: `A title`, Content: `contact`, } Rewriter("https://www.qwantz.com/", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithPDFLink(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `PDF
test`, } testEntry := &model.Entry{ Title: `A title`, Content: `test`, } Rewriter("https://example.org/document.pdf", testEntry, ``) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithNoLazyImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Image`, } testEntry := &model.Entry{ Title: `A title`, Content: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithLazyImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Image`, } testEntry := &model.Entry{ Title: `A title`, Content: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithLazyDivImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Image`, } testEntry := &model.Entry{ Title: `A title`, Content: `
`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithUnknownLazyNoScriptImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `ImageFallback`, } testEntry := &model.Entry{ Title: `A title`, Content: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithLazySrcset(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Image`, } testEntry := &model.Entry{ Title: `A title`, Content: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithImageAndLazySrcset(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Image`, } testEntry := &model.Entry{ Title: `A title`, Content: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithNoLazyIframe(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: ``, } testEntry := &model.Entry{ Title: `A title`, Content: ``, } Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithLazyIframe(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: ``, } testEntry := &model.Entry{ Title: `A title`, Content: ``, } Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteWithLazyIframeAndSrc(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: ``, } testEntry := &model.Entry{ Title: `A title`, Content: ``, } Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestNewLineRewriteRule(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `A
B
C`, } testEntry := &model.Entry{ Title: `A title`, Content: "A\nB\nC", } Rewriter("https://example.org/article", testEntry, "nl2br") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestConvertTextLinkRewriteRule(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Test: http://example.org/a/b`, } testEntry := &model.Entry{ Title: `A title`, Content: `Test: http://example.org/a/b`, } Rewriter("https://example.org/article", testEntry, "convert_text_link") if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestMediumImage(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `Image for post`, } testEntry := &model.Entry{ Title: `A title`, Content: `
Image for post
Image for post
`, } Rewriter("https://example.org/article", testEntry, "fix_medium_images") testEntry.Content = strings.TrimSpace(testEntry.Content) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
The beautiful MDN logo.
MDN Logo
`, } testEntry := &model.Entry{ Title: `A title`, Content: `
The beautiful MDN logo.
MDN Logo
`, } Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images") testEntry.Content = strings.TrimSpace(testEntry.Content) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
MDN Logo
`, } testEntry := &model.Entry{ Title: `A title`, Content: `
The beautiful MDN logo.
MDN Logo
`, } Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images") testEntry.Content = strings.TrimSpace(testEntry.Content) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteReplaceCustom(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: ``, } testEntry := &model.Entry{ Title: `A title`, Content: ``, } Rewriter("https://example.org/article", testEntry, `replace("article/(.*).svg"|"article/$1.png")`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteReplaceTitleCustom(t *testing.T) { controlEntry := &model.Entry{ Title: `Ouch, a thistle`, Content: `The replace_title rewrite rule should not modify the content.`, } testEntry := &model.Entry{ Title: `A title`, Content: `The replace_title rewrite rule should not modify the content.`, } Rewriter("https://example.org/article", testEntry, `replace_title("(?i)^a\\s*ti"|"Ouch, a this")`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteRemoveCustom(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
Lorem Ipsum Super important info
`, } testEntry := &model.Entry{ Title: `A title`, Content: `
Lorem Ipsum I dont want to see thisSuper important info
`, } Rewriter("https://example.org/article", testEntry, `remove(".spam, .ads:not(.keep)")`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteAddCastopodEpisode(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
Episode Description`, } testEntry := &model.Entry{ Title: `A title`, Content: `Episode Description`, } Rewriter("https://podcast.demo/@demo/episodes/test", testEntry, `add_castopod_episode`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteBase64Decode(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `This is some base64 encoded content`, } testEntry := &model.Entry{ Title: `A title`, Content: `VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=`, } Rewriter("https://example.org/article", testEntry, `base64_decode`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteBase64DecodeInHTML(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
Lorem Ipsum not valid base64This is some base64 encoded content
`, } testEntry := &model.Entry{ Title: `A title`, Content: `
Lorem Ipsum not valid base64VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`, } Rewriter("https://example.org/article", testEntry, `base64_decode`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteBase64DecodeArgs(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `
Lorem IpsumThis is some base64 encoded content
`, } testEntry := &model.Entry{ Title: `A title`, Content: `
Lorem IpsumVGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`, } Rewriter("https://example.org/article", testEntry, `base64_decode(".base64")`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRewriteRemoveTables(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, Content: `

Test

Hello World!

Test

`, } testEntry := &model.Entry{ Title: `A title`, Content: `

Test

Hello World!

Test

`, } Rewriter("https://example.org/article", testEntry, `remove_tables`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestRemoveClickbait(t *testing.T) { controlEntry := &model.Entry{ Title: `This Is Amazing`, Content: `Some description`, } testEntry := &model.Entry{ Title: `THIS IS AMAZING`, Content: `Some description`, } Rewriter("https://example.org/article", testEntry, `remove_clickbait`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestAddHackerNewsLinksUsingHack(t *testing.T) { testEntry := &model.Entry{ Title: `A title`, Content: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043

Points: 23

# Comments: 38

`, } controlEntry := &model.Entry{ Title: `A title`, Content: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043 Open with HACK

Points: 23

# Comments: 38

`, } Rewriter("https://example.org/article", testEntry, `add_hn_links_using_hack`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestAddHackerNewsLinksUsingOpener(t *testing.T) { testEntry := &model.Entry{ Title: `A title`, Content: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043

Points: 23

# Comments: 38

`, } controlEntry := &model.Entry{ Title: `A title`, Content: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043 Open with Opener

Points: 23

# Comments: 38

`, } Rewriter("https://example.org/article", testEntry, `add_hn_links_using_opener`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } } func TestAddImageTitle(t *testing.T) { testEntry := &model.Entry{ Title: `A title`, Content: ` "onerror=alert(1) a=" "onerror=alert(1) a=" ;&quot;onerror=alert(1) a=;&quot; pouf pouf pouf `, } controlEntry := &model.Entry{ Title: `A title`, Content: `

pouf

pouf

pouf

;"onerror=alert(1) a=;"

pouf

pouf

"onerror=alert(1) a="

pouf

"onerror=alert(1) a="

pouf

;&quot;onerror=alert(1) a=;&quot;

`, } Rewriter("https://example.org/article", testEntry, `add_image_title`) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) } }