diff --git a/internal/api/subscription.go b/internal/api/subscription.go index 499d99a0..9aca0313 100644 --- a/internal/api/subscription.go +++ b/internal/api/subscription.go @@ -7,9 +7,11 @@ import ( json_parser "encoding/json" "net/http" + "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/subscription" "miniflux.app/v2/internal/validator" ) @@ -32,14 +34,17 @@ func (h *handler) discoverSubscriptions(w http.ResponseWriter, r *http.Request) rssbridgeURL = intg.RSSBridgeURL } - subscriptions, localizedError := subscription.FindSubscriptions( + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithUserAgent(subscriptionDiscoveryRequest.UserAgent) + requestBuilder.WithCookie(subscriptionDiscoveryRequest.Cookie) + requestBuilder.WithUsernameAndPassword(subscriptionDiscoveryRequest.Username, subscriptionDiscoveryRequest.Password) + requestBuilder.UseProxy(subscriptionDiscoveryRequest.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(subscriptionDiscoveryRequest.AllowSelfSignedCertificates) + + subscriptions, localizedError := subscription.NewSubscriptionFinder(requestBuilder).FindSubscriptions( subscriptionDiscoveryRequest.URL, - subscriptionDiscoveryRequest.UserAgent, - subscriptionDiscoveryRequest.Cookie, - subscriptionDiscoveryRequest.Username, - subscriptionDiscoveryRequest.Password, - subscriptionDiscoveryRequest.FetchViaProxy, - subscriptionDiscoveryRequest.AllowSelfSignedCertificates, rssbridgeURL, ) diff --git a/internal/googlereader/handler.go b/internal/googlereader/handler.go index 242cf55c..e072c621 100644 --- a/internal/googlereader/handler.go +++ b/internal/googlereader/handler.go @@ -20,6 +20,7 @@ import ( "miniflux.app/v2/internal/integration" "miniflux.app/v2/internal/model" "miniflux.app/v2/internal/proxy" + "miniflux.app/v2/internal/reader/fetcher" mff "miniflux.app/v2/internal/reader/handler" mfs "miniflux.app/v2/internal/reader/subscription" "miniflux.app/v2/internal/storage" @@ -667,13 +668,22 @@ func (h *handler) quickAddHandler(w http.ResponseWriter, r *http.Request) { return } - url := r.Form.Get(ParamQuickAdd) - if !validator.IsValidURL(url) { - json.BadRequest(w, r, fmt.Errorf("googlereader: invalid URL: %s", url)) + feedURL := r.Form.Get(ParamQuickAdd) + if !validator.IsValidURL(feedURL) { + json.BadRequest(w, r, fmt.Errorf("googlereader: invalid URL: %s", feedURL)) return } - subscriptions, localizedError := mfs.FindSubscriptions(url, "", "", "", "", false, false, "") + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + + var rssBridgeURL string + if intg, err := h.store.Integration(userID); err == nil && intg != nil && intg.RSSBridgeEnabled { + rssBridgeURL = intg.RSSBridgeURL + } + + subscriptions, localizedError := mfs.NewSubscriptionFinder(requestBuilder).FindSubscriptions(feedURL, rssBridgeURL) if localizedError != nil { json.ServerError(w, r, localizedError.Error()) return diff --git a/internal/locale/translations/de_DE.json b/internal/locale/translations/de_DE.json index c8a22594..c4d568c0 100644 --- a/internal/locale/translations/de_DE.json +++ b/internal/locale/translations/de_DE.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/el_EL.json b/internal/locale/translations/el_EL.json index 1da5910c..1bb09399 100644 --- a/internal/locale/translations/el_EL.json +++ b/internal/locale/translations/el_EL.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/en_US.json b/internal/locale/translations/en_US.json index fbabb427..3244adb6 100644 --- a/internal/locale/translations/en_US.json +++ b/internal/locale/translations/en_US.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/es_ES.json b/internal/locale/translations/es_ES.json index 89df8d69..e8bc6c2f 100644 --- a/internal/locale/translations/es_ES.json +++ b/internal/locale/translations/es_ES.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/fi_FI.json b/internal/locale/translations/fi_FI.json index 17ef7657..df3d5632 100644 --- a/internal/locale/translations/fi_FI.json +++ b/internal/locale/translations/fi_FI.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/fr_FR.json b/internal/locale/translations/fr_FR.json index aeaf3c6a..dfeaa05c 100644 --- a/internal/locale/translations/fr_FR.json +++ b/internal/locale/translations/fr_FR.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "Ce flux existe déjà.", "error.unable_to_parse_feed": "Impossible d'analyser ce flux : %v.", "error.feed_not_found": "Impossible de trouver ce flux.", - "error.unable_to_detect_rssbridge": "Impossible de détecter un flux RSS en utilisant RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Impossible de détecter un flux RSS en utilisant RSS-Bridge: %v.", + "error.feed_format_not_detected": "Impossible de détecter le format du flux : %v." } diff --git a/internal/locale/translations/hi_IN.json b/internal/locale/translations/hi_IN.json index a0efc399..714976cf 100644 --- a/internal/locale/translations/hi_IN.json +++ b/internal/locale/translations/hi_IN.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/id_ID.json b/internal/locale/translations/id_ID.json index b4f7e0f7..25241468 100644 --- a/internal/locale/translations/id_ID.json +++ b/internal/locale/translations/id_ID.json @@ -451,5 +451,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/it_IT.json b/internal/locale/translations/it_IT.json index f93af980..57fed234 100644 --- a/internal/locale/translations/it_IT.json +++ b/internal/locale/translations/it_IT.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/ja_JP.json b/internal/locale/translations/ja_JP.json index 757f9f51..34463017 100644 --- a/internal/locale/translations/ja_JP.json +++ b/internal/locale/translations/ja_JP.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/nl_NL.json b/internal/locale/translations/nl_NL.json index 3efadfe6..0b1ed97b 100644 --- a/internal/locale/translations/nl_NL.json +++ b/internal/locale/translations/nl_NL.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/pl_PL.json b/internal/locale/translations/pl_PL.json index 62744703..163b23c3 100644 --- a/internal/locale/translations/pl_PL.json +++ b/internal/locale/translations/pl_PL.json @@ -468,5 +468,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/pt_BR.json b/internal/locale/translations/pt_BR.json index 0651fe71..5157509b 100644 --- a/internal/locale/translations/pt_BR.json +++ b/internal/locale/translations/pt_BR.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/ru_RU.json b/internal/locale/translations/ru_RU.json index 8f2a9d71..6575fc48 100644 --- a/internal/locale/translations/ru_RU.json +++ b/internal/locale/translations/ru_RU.json @@ -468,5 +468,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/tr_TR.json b/internal/locale/translations/tr_TR.json index b732fa5b..0a11b61b 100644 --- a/internal/locale/translations/tr_TR.json +++ b/internal/locale/translations/tr_TR.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/uk_UA.json b/internal/locale/translations/uk_UA.json index 39226cd7..21b20bf5 100644 --- a/internal/locale/translations/uk_UA.json +++ b/internal/locale/translations/uk_UA.json @@ -469,5 +469,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/zh_CN.json b/internal/locale/translations/zh_CN.json index 4b265615..7b6f5663 100644 --- a/internal/locale/translations/zh_CN.json +++ b/internal/locale/translations/zh_CN.json @@ -452,5 +452,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/locale/translations/zh_TW.json b/internal/locale/translations/zh_TW.json index 3b556773..d8d78531 100644 --- a/internal/locale/translations/zh_TW.json +++ b/internal/locale/translations/zh_TW.json @@ -460,5 +460,6 @@ "error.duplicated_feed": "This feed already exists.", "error.unable_to_parse_feed": "Unable to parse this feed: %v.", "error.feed_not_found": "This feed does not exist or does not belong to this user.", - "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v." + "error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.", + "error.feed_format_not_detected": "Unable to detect feed format: %v." } diff --git a/internal/model/feed.go b/internal/model/feed.go index 88560a27..6a76411e 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -5,6 +5,7 @@ package model // import "miniflux.app/v2/internal/model" import ( "fmt" + "io" "math" "time" @@ -144,6 +145,31 @@ type FeedCreationRequest struct { UrlRewriteRules string `json:"urlrewrite_rules"` } +type FeedCreationRequestFromSubscriptionDiscovery struct { + Content io.ReadSeeker + ETag string + LastModified string + + FeedURL string `json:"feed_url"` + CategoryID int64 `json:"category_id"` + UserAgent string `json:"user_agent"` + Cookie string `json:"cookie"` + Username string `json:"username"` + Password string `json:"password"` + Crawler bool `json:"crawler"` + Disabled bool `json:"disabled"` + NoMediaPlayer bool `json:"no_media_player"` + IgnoreHTTPCache bool `json:"ignore_http_cache"` + AllowSelfSignedCertificates bool `json:"allow_self_signed_certificates"` + FetchViaProxy bool `json:"fetch_via_proxy"` + ScraperRules string `json:"scraper_rules"` + RewriteRules string `json:"rewrite_rules"` + BlocklistRules string `json:"blocklist_rules"` + KeeplistRules string `json:"keeplist_rules"` + HideGlobally bool `json:"hide_globally"` + UrlRewriteRules string `json:"urlrewrite_rules"` +} + // FeedModificationRequest represents the request to update a feed. type FeedModificationRequest struct { FeedURL *string `json:"feed_url"` diff --git a/internal/reader/fetcher/response_handler.go b/internal/reader/fetcher/response_handler.go index 1283b405..2c00c933 100644 --- a/internal/reader/fetcher/response_handler.go +++ b/internal/reader/fetcher/response_handler.go @@ -95,14 +95,14 @@ func (r *ResponseHandler) ReadBody(maxBodySize int64) ([]byte, *locale.Localized func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper { if r.clientErr != nil { switch r.clientErr.(type) { - case x509.CertificateInvalidError, x509.UnknownAuthorityError, x509.HostnameError: - return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.tls_error", r.clientErr.Error()) + case x509.CertificateInvalidError, x509.HostnameError: + return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.tls_error", r.clientErr) case *net.OpError: - return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_operation", r.clientErr.Error()) + return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_operation", r.clientErr) case net.Error: networkErr := r.clientErr.(net.Error) if networkErr.Timeout() { - return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr.Error()) + return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr) } } @@ -110,7 +110,7 @@ func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper { return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_empty_response") } - return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr.Error()) + return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr) } switch r.httpResponse.StatusCode { diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index fa38e4cf..49ccdee4 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -4,6 +4,7 @@ package handler // import "miniflux.app/v2/internal/reader/handler" import ( + "bytes" "errors" "log/slog" "time" @@ -25,6 +26,83 @@ var ( ErrDuplicatedFeed = errors.New("fetcher: duplicated feed") ) +func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, feedCreationRequest *model.FeedCreationRequestFromSubscriptionDiscovery) (*model.Feed, *locale.LocalizedErrorWrapper) { + slog.Debug("Begin feed creation process from subscription discovery", + slog.Int64("user_id", userID), + slog.String("feed_url", feedCreationRequest.FeedURL), + ) + + user, storeErr := store.UserByID(userID) + if storeErr != nil { + return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr) + } + + if !store.CategoryIDExists(userID, feedCreationRequest.CategoryID) { + return nil, locale.NewLocalizedErrorWrapper(ErrCategoryNotFound, "error.category_not_found") + } + + if store.FeedURLExists(userID, feedCreationRequest.FeedURL) { + return nil, locale.NewLocalizedErrorWrapper(ErrDuplicatedFeed, "error.duplicated_feed") + } + + subscription, parseErr := parser.ParseFeed(feedCreationRequest.FeedURL, feedCreationRequest.Content) + if parseErr != nil { + return nil, locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr) + } + + subscription.UserID = userID + subscription.UserAgent = feedCreationRequest.UserAgent + subscription.Cookie = feedCreationRequest.Cookie + subscription.Username = feedCreationRequest.Username + subscription.Password = feedCreationRequest.Password + subscription.Crawler = feedCreationRequest.Crawler + subscription.Disabled = feedCreationRequest.Disabled + subscription.IgnoreHTTPCache = feedCreationRequest.IgnoreHTTPCache + subscription.AllowSelfSignedCertificates = feedCreationRequest.AllowSelfSignedCertificates + subscription.FetchViaProxy = feedCreationRequest.FetchViaProxy + subscription.ScraperRules = feedCreationRequest.ScraperRules + subscription.RewriteRules = feedCreationRequest.RewriteRules + subscription.BlocklistRules = feedCreationRequest.BlocklistRules + subscription.KeeplistRules = feedCreationRequest.KeeplistRules + subscription.UrlRewriteRules = feedCreationRequest.UrlRewriteRules + subscription.EtagHeader = feedCreationRequest.ETag + subscription.LastModifiedHeader = feedCreationRequest.LastModified + subscription.FeedURL = feedCreationRequest.FeedURL + subscription.WithCategoryID(feedCreationRequest.CategoryID) + subscription.CheckedNow() + + processor.ProcessFeedEntries(store, subscription, user, true) + + if storeErr := store.CreateFeed(subscription); storeErr != nil { + return nil, locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr) + } + + slog.Debug("Created feed", + slog.Int64("user_id", userID), + slog.Int64("feed_id", subscription.ID), + slog.String("feed_url", subscription.FeedURL), + ) + + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithUsernameAndPassword(feedCreationRequest.Username, feedCreationRequest.Password) + requestBuilder.WithUserAgent(feedCreationRequest.UserAgent) + requestBuilder.WithCookie(feedCreationRequest.Cookie) + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.UseProxy(feedCreationRequest.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(feedCreationRequest.AllowSelfSignedCertificates) + + checkFeedIcon( + store, + requestBuilder, + subscription.ID, + subscription.SiteURL, + subscription.IconURL, + ) + + return subscription, nil +} + // CreateFeed fetch, parse and store a new feed. func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model.FeedCreationRequest) (*model.Feed, *locale.LocalizedErrorWrapper) { slog.Debug("Begin feed creation process", @@ -68,7 +146,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model return nil, locale.NewLocalizedErrorWrapper(ErrDuplicatedFeed, "error.duplicated_feed") } - subscription, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), string(responseBody)) + subscription, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody)) if parseErr != nil { return nil, locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr) } @@ -188,7 +266,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return localizedError } - updatedFeed, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), string(responseBody)) + updatedFeed, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody)) if parseErr != nil { localizedError := locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed") diff --git a/internal/reader/parser/format.go b/internal/reader/parser/format.go index 77a49d2c..1019d164 100644 --- a/internal/reader/parser/format.go +++ b/internal/reader/parser/format.go @@ -4,8 +4,9 @@ package parser // import "miniflux.app/v2/internal/reader/parser" import ( + "bytes" "encoding/xml" - "strings" + "io" rxml "miniflux.app/v2/internal/reader/xml" ) @@ -20,12 +21,16 @@ const ( ) // DetectFeedFormat tries to guess the feed format from input data. -func DetectFeedFormat(data string) string { - if strings.HasPrefix(strings.TrimSpace(data), "{") { +func DetectFeedFormat(r io.ReadSeeker) string { + data := make([]byte, 512) + r.Read(data) + + if bytes.HasPrefix(bytes.TrimSpace(data), []byte("{")) { return FormatJSON } - decoder := rxml.NewDecoder(strings.NewReader(data)) + r.Seek(0, io.SeekStart) + decoder := rxml.NewDecoder(r) for { token, _ := decoder.Token() diff --git a/internal/reader/parser/format_test.go b/internal/reader/parser/format_test.go index 936aa82e..7acf3e7a 100644 --- a/internal/reader/parser/format_test.go +++ b/internal/reader/parser/format_test.go @@ -4,12 +4,13 @@ package parser // import "miniflux.app/v2/internal/reader/parser" import ( + "strings" "testing" ) func TestDetectRDF(t *testing.T) { data := `` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatRDF { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatRDF) @@ -18,7 +19,7 @@ func TestDetectRDF(t *testing.T) { func TestDetectRSS(t *testing.T) { data := `` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatRSS { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatRSS) @@ -27,7 +28,7 @@ func TestDetectRSS(t *testing.T) { func TestDetectAtom10(t *testing.T) { data := `` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatAtom { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom) @@ -36,7 +37,7 @@ func TestDetectAtom10(t *testing.T) { func TestDetectAtom03(t *testing.T) { data := `` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatAtom { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom) @@ -45,7 +46,7 @@ func TestDetectAtom03(t *testing.T) { func TestDetectAtomWithISOCharset(t *testing.T) { data := `` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatAtom { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom) @@ -59,7 +60,7 @@ func TestDetectJSON(t *testing.T) { "title" : "Example" } ` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatJSON { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatJSON) @@ -70,7 +71,7 @@ func TestDetectUnknown(t *testing.T) { data := ` ` - format := DetectFeedFormat(data) + format := DetectFeedFormat(strings.NewReader(data)) if format != FormatUnknown { t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatUnknown) diff --git a/internal/reader/parser/parser.go b/internal/reader/parser/parser.go index 60b194d4..2843888b 100644 --- a/internal/reader/parser/parser.go +++ b/internal/reader/parser/parser.go @@ -5,7 +5,7 @@ package parser // import "miniflux.app/v2/internal/reader/parser" import ( "errors" - "strings" + "io" "miniflux.app/v2/internal/model" "miniflux.app/v2/internal/reader/atom" @@ -17,16 +17,21 @@ import ( var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format") // ParseFeed analyzes the input data and returns a normalized feed object. -func ParseFeed(baseURL, data string) (*model.Feed, error) { - switch DetectFeedFormat(data) { +func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) { + r.Seek(0, io.SeekStart) + switch DetectFeedFormat(r) { case FormatAtom: - return atom.Parse(baseURL, strings.NewReader(data)) + r.Seek(0, io.SeekStart) + return atom.Parse(baseURL, r) case FormatRSS: - return rss.Parse(baseURL, strings.NewReader(data)) + r.Seek(0, io.SeekStart) + return rss.Parse(baseURL, r) case FormatJSON: - return json.Parse(baseURL, strings.NewReader(data)) + r.Seek(0, io.SeekStart) + return json.Parse(baseURL, r) case FormatRDF: - return rdf.Parse(baseURL, strings.NewReader(data)) + r.Seek(0, io.SeekStart) + return rdf.Parse(baseURL, r) default: return nil, ErrFeedFormatNotDetected } diff --git a/internal/reader/parser/parser_test.go b/internal/reader/parser/parser_test.go index cd0a3213..07972474 100644 --- a/internal/reader/parser/parser_test.go +++ b/internal/reader/parser/parser_test.go @@ -4,6 +4,7 @@ package parser // import "miniflux.app/v2/internal/reader/parser" import ( + "strings" "testing" ) @@ -29,7 +30,7 @@ func TestParseAtom(t *testing.T) { ` - feed, err := ParseFeed("https://example.org/", data) + feed, err := ParseFeed("https://example.org/", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -57,7 +58,7 @@ func TestParseAtomFeedWithRelativeURL(t *testing.T) { ` - feed, err := ParseFeed("https://example.org/blog/atom.xml", data) + feed, err := ParseFeed("https://example.org/blog/atom.xml", strings.NewReader(data)) if err != nil { t.Fatal(err) } @@ -91,7 +92,7 @@ func TestParseRSS(t *testing.T) { ` - feed, err := ParseFeed("http://liftoff.msfc.nasa.gov/", data) + feed, err := ParseFeed("http://liftoff.msfc.nasa.gov/", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -117,7 +118,7 @@ func TestParseRSSFeedWithRelativeURL(t *testing.T) { ` - feed, err := ParseFeed("http://example.org/rss.xml", data) + feed, err := ParseFeed("http://example.org/rss.xml", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -158,7 +159,7 @@ func TestParseRDF(t *testing.T) { ` - feed, err := ParseFeed("http://example.org/", data) + feed, err := ParseFeed("http://example.org/", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -187,7 +188,7 @@ func TestParseRDFWithRelativeURL(t *testing.T) { ` - feed, err := ParseFeed("http://example.org/rdf.xml", data) + feed, err := ParseFeed("http://example.org/rdf.xml", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -225,7 +226,7 @@ func TestParseJson(t *testing.T) { ] }` - feed, err := ParseFeed("https://example.org/feed.json", data) + feed, err := ParseFeed("https://example.org/feed.json", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -250,7 +251,7 @@ func TestParseJsonFeedWithRelativeURL(t *testing.T) { ] }` - feed, err := ParseFeed("https://example.org/blog/feed.json", data) + feed, err := ParseFeed("https://example.org/blog/feed.json", strings.NewReader(data)) if err != nil { t.Error(err) } @@ -285,14 +286,14 @@ func TestParseUnknownFeed(t *testing.T) { ` - _, err := ParseFeed("https://example.org/", data) + _, err := ParseFeed("https://example.org/", strings.NewReader(data)) if err == nil { t.Error("ParseFeed must returns an error") } } func TestParseEmptyFeed(t *testing.T) { - _, err := ParseFeed("", "") + _, err := ParseFeed("", strings.NewReader("")) if err == nil { t.Error("ParseFeed must returns an error") } diff --git a/internal/reader/subscription/finder.go b/internal/reader/subscription/finder.go index 6455c214..7d4e3a57 100644 --- a/internal/reader/subscription/finder.go +++ b/internal/reader/subscription/finder.go @@ -13,6 +13,7 @@ import ( "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/integration/rssbridge" "miniflux.app/v2/internal/locale" + "miniflux.app/v2/internal/model" "miniflux.app/v2/internal/reader/fetcher" "miniflux.app/v2/internal/reader/parser" "miniflux.app/v2/internal/urllib" @@ -25,20 +26,28 @@ var ( youtubeVideoRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)`) ) -func FindSubscriptions(websiteURL, userAgent, cookie, username, password string, fetchViaProxy, allowSelfSignedCertificates bool, rssbridgeURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { - websiteURL = findYoutubeChannelFeed(websiteURL) - websiteURL = parseYoutubeVideoPage(websiteURL) +type SubscriptionFinder struct { + requestBuilder *fetcher.RequestBuilder + feedDownloaded bool + feedResponseInfo *model.FeedCreationRequestFromSubscriptionDiscovery +} - requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithUsernameAndPassword(username, password) - requestBuilder.WithUserAgent(userAgent) - requestBuilder.WithCookie(cookie) - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(fetchViaProxy) - requestBuilder.IgnoreTLSErrors(allowSelfSignedCertificates) +func NewSubscriptionFinder(requestBuilder *fetcher.RequestBuilder) *SubscriptionFinder { + return &SubscriptionFinder{ + requestBuilder: requestBuilder, + } +} - responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) +func (f *SubscriptionFinder) IsFeedAlreadyDownloaded() bool { + return f.feedDownloaded +} + +func (f *SubscriptionFinder) FeedResponseInfo() *model.FeedCreationRequestFromSubscriptionDiscovery { + return f.feedResponseInfo +} + +func (f *SubscriptionFinder) FindSubscriptions(websiteURL, rssBridgeURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { + responseHandler := fetcher.NewResponseHandler(f.requestBuilder.ExecuteRequest(websiteURL)) defer responseHandler.Close() if localizedError := responseHandler.LocalizedError(); localizedError != nil { @@ -52,69 +61,97 @@ func FindSubscriptions(websiteURL, userAgent, cookie, username, password string, return nil, localizedError } - if format := parser.DetectFeedFormat(string(responseBody)); format != parser.FormatUnknown { - var subscriptions Subscriptions - subscriptions = append(subscriptions, &Subscription{ - Title: responseHandler.EffectiveURL(), - URL: responseHandler.EffectiveURL(), - Type: format, - }) + f.feedResponseInfo = &model.FeedCreationRequestFromSubscriptionDiscovery{ + Content: bytes.NewReader(responseBody), + ETag: responseHandler.ETag(), + LastModified: responseHandler.LastModified(), + } + // Step 1) Check if the website URL is a feed. + if feedFormat := parser.DetectFeedFormat(f.feedResponseInfo.Content); feedFormat != parser.FormatUnknown { + f.feedDownloaded = true + return Subscriptions{NewSubscription(responseHandler.EffectiveURL(), responseHandler.EffectiveURL(), feedFormat)}, nil + } + + // Step 2) Check if the website URL is a YouTube channel. + slog.Debug("Try to detect feeds from YouTube channel page", slog.String("website_url", websiteURL)) + subscriptions, localizedError := f.FindSubscriptionsFromYouTubeChannelPage(websiteURL) + if localizedError != nil { + return nil, localizedError + } + + if len(subscriptions) > 0 { + slog.Debug("Subscriptions found from YouTube channel page", slog.String("website_url", websiteURL), slog.Any("subscriptions", subscriptions)) return subscriptions, nil } - subscriptions, localizedError := parseWebPage(responseHandler.EffectiveURL(), bytes.NewReader(responseBody)) - if localizedError != nil || subscriptions != nil { - return subscriptions, localizedError + // Step 3) Check if the website URL is a YouTube video. + slog.Debug("Try to detect feeds from YouTube video page", slog.String("website_url", websiteURL)) + subscriptions, localizedError = f.FindSubscriptionsFromYouTubeVideoPage(websiteURL) + if localizedError != nil { + return nil, localizedError } - if rssbridgeURL != "" { - slog.Debug("Trying to detect feeds using RSS-Bridge", - slog.String("website_url", websiteURL), - slog.String("rssbridge_url", rssbridgeURL), - ) + if len(subscriptions) > 0 { + slog.Debug("Subscriptions found from YouTube video page", slog.String("website_url", websiteURL), slog.Any("subscriptions", subscriptions)) + return subscriptions, nil + } - bridges, err := rssbridge.DetectBridges(rssbridgeURL, websiteURL) - if err != nil { - return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_detect_rssbridge", err) + // Step 4) Parse web page to find feeds from HTML meta tags. + slog.Debug("Try to detect feeds from HTML meta tags", slog.String("website_url", websiteURL)) + subscriptions, localizedError = f.FindSubscriptionsFromWebPage(websiteURL, bytes.NewReader(responseBody)) + if localizedError != nil { + return nil, localizedError + } + + if len(subscriptions) > 0 { + slog.Debug("Subscriptions found from web page", slog.String("website_url", websiteURL), slog.Any("subscriptions", subscriptions)) + return subscriptions, nil + } + + // Step 5) Check if the website URL can use RSS-Bridge. + if rssBridgeURL != "" { + slog.Debug("Try to detect feeds with RSS-Bridge", slog.String("website_url", websiteURL)) + subscriptions, localizedError := f.FindSubscriptionsFromRSSBridge(websiteURL, rssBridgeURL) + if localizedError != nil { + return nil, localizedError } - slog.Debug("RSS-Bridge results", - slog.String("website_url", websiteURL), - slog.String("rssbridge_url", rssbridgeURL), - slog.Int("nb_bridges", len(bridges)), - ) - - if len(bridges) > 0 { - var subscriptions Subscriptions - for _, bridge := range bridges { - subscriptions = append(subscriptions, &Subscription{ - Title: bridge.BridgeMeta.Name, - URL: bridge.URL, - Type: "atom", - }) - } + if len(subscriptions) > 0 { + slog.Debug("Subscriptions found from RSS-Bridge", slog.String("website_url", websiteURL), slog.Any("subscriptions", subscriptions)) return subscriptions, nil } } - return tryWellKnownUrls(websiteURL, userAgent, cookie, username, password, fetchViaProxy, allowSelfSignedCertificates) -} - -func parseWebPage(websiteURL string, data io.Reader) (Subscriptions, *locale.LocalizedErrorWrapper) { - var subscriptions Subscriptions - queries := map[string]string{ - "link[type='application/rss+xml']": "rss", - "link[type='application/atom+xml']": "atom", - "link[type='application/json']": "json", - "link[type='application/feed+json']": "json", + // Step 6) Check if the website has a known feed URL. + slog.Debug("Try to detect feeds from well-known URLs", slog.String("website_url", websiteURL)) + subscriptions, localizedError = f.FindSubscriptionsFromWellKnownURLs(websiteURL) + if localizedError != nil { + return nil, localizedError } - doc, err := goquery.NewDocumentFromReader(data) + if len(subscriptions) > 0 { + slog.Debug("Subscriptions found with well-known URLs", slog.String("website_url", websiteURL), slog.Any("subscriptions", subscriptions)) + return subscriptions, nil + } + + return nil, nil +} + +func (f *SubscriptionFinder) FindSubscriptionsFromWebPage(websiteURL string, body io.Reader) (Subscriptions, *locale.LocalizedErrorWrapper) { + queries := map[string]string{ + "link[type='application/rss+xml']": parser.FormatRSS, + "link[type='application/atom+xml']": parser.FormatAtom, + "link[type='application/json']": parser.FormatJSON, + "link[type='application/feed+json']": parser.FormatJSON, + } + + doc, err := goquery.NewDocumentFromReader(body) if err != nil { return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_parse_html_document", err) } + var subscriptions Subscriptions for query, kind := range queries { doc.Find(query).Each(func(i int, s *goquery.Selection) { subscription := new(Subscription) @@ -143,52 +180,13 @@ func parseWebPage(websiteURL string, data io.Reader) (Subscriptions, *locale.Loc return subscriptions, nil } -func findYoutubeChannelFeed(websiteURL string) string { - matches := youtubeChannelRegex.FindStringSubmatch(websiteURL) - - if len(matches) == 2 { - return fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, matches[1]) - } - return websiteURL -} - -func parseYoutubeVideoPage(websiteURL string) string { - if !youtubeVideoRegex.MatchString(websiteURL) { - return websiteURL - } - - requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - - responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(websiteURL)) - defer responseHandler.Close() - - if localizedError := responseHandler.LocalizedError(); localizedError != nil { - slog.Warn("Unable to find subscriptions", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error())) - return websiteURL - } - - doc, docErr := goquery.NewDocumentFromReader(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())) - if docErr != nil { - return websiteURL - } - - if channelID, exists := doc.Find(`meta[itemprop="channelId"]`).First().Attr("content"); exists { - return fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, channelID) - } - - return websiteURL -} - -func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string, fetchViaProxy, allowSelfSignedCertificates bool) (Subscriptions, *locale.LocalizedErrorWrapper) { - var subscriptions Subscriptions +func (f *SubscriptionFinder) FindSubscriptionsFromWellKnownURLs(websiteURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { knownURLs := map[string]string{ - "atom.xml": "atom", - "feed.xml": "atom", - "feed/": "atom", - "rss.xml": "rss", - "rss/": "rss", + "atom.xml": parser.FormatAtom, + "feed.xml": parser.FormatAtom, + "feed/": parser.FormatAtom, + "rss.xml": parser.FormatRSS, + "rss/": parser.FormatRSS, } websiteURLRoot := urllib.RootURL(websiteURL) @@ -203,6 +201,7 @@ func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string, baseURLs = append(baseURLs, websiteURL) } + var subscriptions Subscriptions for _, baseURL := range baseURLs { for knownURL, kind := range knownURLs { fullURL, err := urllib.AbsoluteURL(baseURL, knownURL) @@ -210,21 +209,12 @@ func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string, continue } - requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithUsernameAndPassword(username, password) - requestBuilder.WithUserAgent(userAgent) - requestBuilder.WithCookie(cookie) - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(fetchViaProxy) - requestBuilder.IgnoreTLSErrors(allowSelfSignedCertificates) - // Some websites redirects unknown URLs to the home page. // As result, the list of known URLs is returned to the subscription list. // We don't want the user to choose between invalid feed URLs. - requestBuilder.WithoutRedirects() + f.requestBuilder.WithoutRedirects() - responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(fullURL)) + responseHandler := fetcher.NewResponseHandler(f.requestBuilder.ExecuteRequest(fullURL)) defer responseHandler.Close() if localizedError := responseHandler.LocalizedError(); localizedError != nil { @@ -241,3 +231,75 @@ func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string, return subscriptions, nil } + +func (f *SubscriptionFinder) FindSubscriptionsFromRSSBridge(websiteURL, rssBridgeURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { + slog.Debug("Trying to detect feeds using RSS-Bridge", + slog.String("website_url", websiteURL), + slog.String("rssbridge_url", rssBridgeURL), + ) + + bridges, err := rssbridge.DetectBridges(rssBridgeURL, websiteURL) + if err != nil { + return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_detect_rssbridge", err) + } + + slog.Debug("RSS-Bridge results", + slog.String("website_url", websiteURL), + slog.String("rssbridge_url", rssBridgeURL), + slog.Int("nb_bridges", len(bridges)), + ) + + if len(bridges) == 0 { + return nil, nil + } + + var subscriptions Subscriptions + for _, bridge := range bridges { + subscriptions = append(subscriptions, &Subscription{ + Title: bridge.BridgeMeta.Name, + URL: bridge.URL, + Type: parser.FormatAtom, + }) + } + + return subscriptions, nil +} + +func (f *SubscriptionFinder) FindSubscriptionsFromYouTubeChannelPage(websiteURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { + matches := youtubeChannelRegex.FindStringSubmatch(websiteURL) + + if len(matches) == 2 { + feedURL := fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, matches[1]) + return Subscriptions{NewSubscription(websiteURL, feedURL, parser.FormatAtom)}, nil + } + + slog.Debug("This website is not a YouTube channel page, the regex doesn't match", slog.String("website_url", websiteURL)) + + return nil, nil +} + +func (f *SubscriptionFinder) FindSubscriptionsFromYouTubeVideoPage(websiteURL string) (Subscriptions, *locale.LocalizedErrorWrapper) { + if !youtubeVideoRegex.MatchString(websiteURL) { + slog.Debug("This website is not a YouTube video page, the regex doesn't match", slog.String("website_url", websiteURL)) + return nil, nil + } + + responseHandler := fetcher.NewResponseHandler(f.requestBuilder.ExecuteRequest(websiteURL)) + defer responseHandler.Close() + + if localizedError := responseHandler.LocalizedError(); localizedError != nil { + return nil, localizedError + } + + doc, docErr := goquery.NewDocumentFromReader(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())) + if docErr != nil { + return nil, locale.NewLocalizedErrorWrapper(docErr, "error.unable_to_parse_html_document", docErr) + } + + if channelID, exists := doc.Find(`meta[itemprop="channelId"]`).First().Attr("content"); exists { + feedURL := fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, channelID) + return Subscriptions{NewSubscription(websiteURL, feedURL, parser.FormatAtom)}, nil + } + + return nil, nil +} diff --git a/internal/reader/subscription/finder_test.go b/internal/reader/subscription/finder_test.go index 9124bdeb..bc235ae7 100644 --- a/internal/reader/subscription/finder_test.go +++ b/internal/reader/subscription/finder_test.go @@ -11,13 +11,20 @@ import ( func TestFindYoutubeChannelFeed(t *testing.T) { scenarios := map[string]string{ "https://www.youtube.com/channel/UC-Qj80avWItNRjkZ41rzHyw": "https://www.youtube.com/feeds/videos.xml?channel_id=UC-Qj80avWItNRjkZ41rzHyw", - "http://example.org/feed": "http://example.org/feed", } for websiteURL, expectedFeedURL := range scenarios { - result := findYoutubeChannelFeed(websiteURL) - if result != expectedFeedURL { - t.Errorf(`Unexpected Feed, got %s, instead of %s`, result, expectedFeedURL) + subscriptions, localizedError := NewSubscriptionFinder(nil).FindSubscriptionsFromYouTubeChannelPage(websiteURL) + if localizedError != nil { + t.Fatalf(`Parsing a correctly formatted YouTube channel page should not return any error: %v`, localizedError) + } + + if len(subscriptions) != 1 { + t.Fatal(`Incorrect number of subscriptions returned`) + } + + if subscriptions[0].URL != expectedFeedURL { + t.Errorf(`Unexpected Feed, got %s, instead of %s`, subscriptions[0].URL, expectedFeedURL) } } } @@ -33,7 +40,7 @@ func TestParseWebPageWithRssFeed(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -66,7 +73,7 @@ func TestParseWebPageWithAtomFeed(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -99,7 +106,7 @@ func TestParseWebPageWithJSONFeed(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -132,7 +139,7 @@ func TestParseWebPageWithOldJSONFeedMimeType(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -165,7 +172,7 @@ func TestParseWebPageWithRelativeFeedURL(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -198,7 +205,7 @@ func TestParseWebPageWithEmptyTitle(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -232,7 +239,7 @@ func TestParseWebPageWithMultipleFeeds(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -253,7 +260,7 @@ func TestParseWebPageWithEmptyFeedURL(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } @@ -274,7 +281,7 @@ func TestParseWebPageWithNoHref(t *testing.T) { ` - subscriptions, err := parseWebPage("http://example.org/", strings.NewReader(htmlPage)) + subscriptions, err := NewSubscriptionFinder(nil).FindSubscriptionsFromWebPage("http://example.org/", strings.NewReader(htmlPage)) if err != nil { t.Fatalf(`Parsing a correctly formatted HTML page should not return any error: %v`, err) } diff --git a/internal/reader/subscription/subscription.go b/internal/reader/subscription/subscription.go index 8facebc1..6642c4c6 100644 --- a/internal/reader/subscription/subscription.go +++ b/internal/reader/subscription/subscription.go @@ -12,6 +12,10 @@ type Subscription struct { Type string `json:"type"` } +func NewSubscription(title, url, kind string) *Subscription { + return &Subscription{Title: title, URL: url, Type: kind} +} + func (s Subscription) String() string { return fmt.Sprintf(`Title="%s", URL="%s", Type="%s"`, s.Title, s.URL, s.Type) } diff --git a/internal/ui/subscription_submit.go b/internal/ui/subscription_submit.go index ce444d79..f0de2079 100644 --- a/internal/ui/subscription_submit.go +++ b/internal/ui/subscription_submit.go @@ -12,6 +12,7 @@ import ( "miniflux.app/v2/internal/http/route" "miniflux.app/v2/internal/locale" "miniflux.app/v2/internal/model" + "miniflux.app/v2/internal/reader/fetcher" feedHandler "miniflux.app/v2/internal/reader/handler" "miniflux.app/v2/internal/reader/subscription" "miniflux.app/v2/internal/ui/form" @@ -51,20 +52,24 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { return } - var rssbridgeURL string + var rssBridgeURL string if intg, err := h.store.Integration(user.ID); err == nil && intg != nil && intg.RSSBridgeEnabled { - rssbridgeURL = intg.RSSBridgeURL + rssBridgeURL = intg.RSSBridgeURL } - subscriptions, localizedError := subscription.FindSubscriptions( + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.WithUserAgent(subscriptionForm.UserAgent) + requestBuilder.WithCookie(subscriptionForm.Cookie) + requestBuilder.WithUsernameAndPassword(subscriptionForm.Username, subscriptionForm.Password) + requestBuilder.UseProxy(subscriptionForm.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(subscriptionForm.AllowSelfSignedCertificates) + + subscriptionFinder := subscription.NewSubscriptionFinder(requestBuilder) + subscriptions, localizedError := subscriptionFinder.FindSubscriptions( subscriptionForm.URL, - subscriptionForm.UserAgent, - subscriptionForm.Cookie, - subscriptionForm.Username, - subscriptionForm.Password, - subscriptionForm.FetchViaProxy, - subscriptionForm.AllowSelfSignedCertificates, - rssbridgeURL, + rssBridgeURL, ) if localizedError != nil { v.Set("form", subscriptionForm) @@ -79,7 +84,35 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { v.Set("form", subscriptionForm) v.Set("errorMessage", locale.NewLocalizedError("error.subscription_not_found").Translate(user.Language)) html.OK(w, r, v.Render("add_subscription")) - case n == 1: + case n == 1 && subscriptionFinder.IsFeedAlreadyDownloaded(): + feed, localizedError := feedHandler.CreateFeedFromSubscriptionDiscovery(h.store, user.ID, &model.FeedCreationRequestFromSubscriptionDiscovery{ + Content: subscriptionFinder.FeedResponseInfo().Content, + ETag: subscriptionFinder.FeedResponseInfo().ETag, + LastModified: subscriptionFinder.FeedResponseInfo().LastModified, + CategoryID: subscriptionForm.CategoryID, + FeedURL: subscriptions[0].URL, + Crawler: subscriptionForm.Crawler, + AllowSelfSignedCertificates: subscriptionForm.AllowSelfSignedCertificates, + UserAgent: subscriptionForm.UserAgent, + Cookie: subscriptionForm.Cookie, + Username: subscriptionForm.Username, + Password: subscriptionForm.Password, + ScraperRules: subscriptionForm.ScraperRules, + RewriteRules: subscriptionForm.RewriteRules, + BlocklistRules: subscriptionForm.BlocklistRules, + KeeplistRules: subscriptionForm.KeeplistRules, + UrlRewriteRules: subscriptionForm.UrlRewriteRules, + FetchViaProxy: subscriptionForm.FetchViaProxy, + }) + if localizedError != nil { + v.Set("form", subscriptionForm) + v.Set("errorMessage", localizedError.Translate(user.Language)) + html.OK(w, r, v.Render("add_subscription")) + return + } + + html.Redirect(w, r, route.Path(h.router, "feedEntries", "feedID", feed.ID)) + case n == 1 && !subscriptionFinder.IsFeedAlreadyDownloaded(): feed, localizedError := feedHandler.CreateFeed(h.store, user.ID, &model.FeedCreationRequest{ CategoryID: subscriptionForm.CategoryID, FeedURL: subscriptions[0].URL,