From 319b875e3394f98d2781ad16cb289b1ba850e671 Mon Sep 17 00:00:00 2001 From: SouthFox Date: Mon, 11 Sep 2023 10:57:16 +0800 Subject: [PATCH] [back/feat] question paging --- src/main/backend/handlers.clj | 94 +++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 26 deletions(-) diff --git a/src/main/backend/handlers.clj b/src/main/backend/handlers.clj index e4e7548..01d43c6 100644 --- a/src/main/backend/handlers.clj +++ b/src/main/backend/handlers.clj @@ -106,7 +106,6 @@ :updated_time (get-in m ["target" "updated_time"]) :comment_count (get-in m ["target" "comment_count"]) :voteup_count (get-in m ["target" "voteup_count"])}) - (into {} {:author {:avatar_url (get-in m ["author" "avatarUrl"]) :url_token (get-in m ["author" "urlToken"]) @@ -131,44 +130,87 @@ (defn process-hu-questiion [question-json {:keys [content-type wrap-fn]}] - {:status 200 :headers content-type - :body (wrap-fn - (merge - {:question (build-question (val (first (get-in question-json ["initialState" "entities" "questions"]))))} - {:answers (mapv build-answer (vals (get-in question-json ["initialState" "entities" "answers"])))}))}) + :body (wrap-fn question-json)}) + +(defn fetch-hu-answers + [question-id query] + (let [question-url (str/join ["https://www.zhihu.com/api/v4/questions/" + question-id + "/feeds?cursor=" (get query "cursor") + "&include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cattachment%2Cvoteup_count%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Crelevant_info%2Cquestion%2Cexcerpt%2Cis_labeled%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cvip_info%2Cbadge%5B%2A%5D.topics%3Bdata%5B%2A%5D.settings.table_of_content.enabled" + "&limit=5" + "&offset=0" + "&order=default" + "&platform=desktop" + "&session_id=" (get query "session_id")]) + question-json (-> (client/get question-url) :body parse-string)] + question-json)) + +(defn build-paging-next + [question-id next] + (let [cursor (last (re-find #"cursor=(.*?)&" next)) + session-id (last (re-find #"session_id=(.*?)(?:&|$)" next))] + (str/join ["/api/hq/" + question-id + "?cursor=" + cursor + "&session_id=" + session-id]))) + +(defn process-api-paging + [question-id docs] + (let [next (get docs "next")] + {:paging {"page" (get docs "page") + "is_end" (get docs "is_end") + "next" (build-paging-next question-id next)}})) + +(defn process-json-paging + [question-id docs] + (let [answers (val (first (get docs "answers")))] + (if (< (count (get answers "ids")) 5) + {:paging {"page" nil + "is_end" true + "next" nil}} + (let [next_url (get answers "next")] + {:paging {"page" nil + "is_end" false + "next" (build-paging-next question-id next_url)}})))) (defn fetch-hu-question - [question-id params] - (let [question-url (str/join ["https://www.zhihu.com/question/" - question-id]) + [request params] + (let [question-id (-> request :path-params :id) + query (-> request :query-params) + question-url (str/join ["https://www.zhihu.com/question/" + question-id]) question-page (-> (client/get question-url) :body Jsoup/parse) question-json (-> (.getElementById question-page "js-initialData") .firstChild .toString parse-string) question-error? (= (get question-json "subAppName") "errorpage")] - (if question-error? {:status 404 :headers (:content-type params) - :body {:content "Not Found" - :title "Not Found"}} - - (process-hu-questiion question-json params)))) - -(defn fetch-hu-answers - [question-id params] - (let [question-url (str/join ["https://www.zhihu.com/api/v4/questions/" - question-id - "/feeds?cursor=&include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cattachment%2Cvoteup_count%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Crelevant_info%2Cquestion%2Cexcerpt%2Cis_labeled%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cvip_info%2Cbadge%5B%2A%5D.topics%3Bdata%5B%2A%5D.settings.table_of_content.enabled&limit=5&offset=0&order=default&platform=desktop&session_id="]) - question-json (-> (client/get question-url) :body parse-string)] - question-json)) + :body (wrap-json {:content "Not Found" + :title "Not Found"})} + (if (empty? query) + (process-hu-questiion + (merge + {:question (build-question (val (first (get-in question-json ["initialState" "entities" "questions"]))))} + {:answers (mapv build-answer (vals (get-in question-json ["initialState" "entities" "answers"])))} + (process-json-paging question-id (get-in question-json ["initialState" "question"]))) + params) + (process-hu-questiion + (merge + {:answers (mapv build-answer (get (fetch-hu-answers question-id query) "data"))} + (process-api-paging question-id (get (fetch-hu-answers question-id query) "paging"))) + params))))) (defn build-api-hu-question [request] - (let [question-id (-> request :path-params :id)] - (fetch-hu-question question-id - {:content-type {"Content-Type" "application/json; charset=utf-8"} - :wrap-fn wrap-json}))) + (fetch-hu-question + request + {:content-type {"Content-Type" "application/json; charset=utf-8"} + :wrap-fn wrap-json}))