From 670e7d42bae83372586aa30cb038d1ade9a4bd5d Mon Sep 17 00:00:00 2001 From: SouthFox Date: Tue, 8 Nov 2022 02:25:06 +0800 Subject: [PATCH] feat/show other account replys --- BDSM/templates/view.html | 4 +- BDSM/toot.py | 336 +++++++++++++++++++++------------------ BDSM/views.py | 42 +++-- 3 files changed, 216 insertions(+), 166 deletions(-) diff --git a/BDSM/templates/view.html b/BDSM/templates/view.html index ef4aa01..3c41358 100644 --- a/BDSM/templates/view.html +++ b/BDSM/templates/view.html @@ -77,7 +77,9 @@ {% endif %}
- {% if not toot.is_reblog %} + {% if toot.is_myself %} + 抓取回复 上下文 {% endif %} diff --git a/BDSM/toot.py b/BDSM/toot.py index 94da4e0..b719ad1 100644 --- a/BDSM/toot.py +++ b/BDSM/toot.py @@ -15,7 +15,7 @@ def app_register(url): scopes=["read"] ) -def archive_toot(url): +def app_login(url): mastodon = Mastodon( client_id='pyBDSM_clientcred.secret', access_token='user.secret', @@ -36,6 +36,187 @@ def archive_toot(url): # exit in either case sys.exit(1) + return mastodon, user + +def get_context(url, toot_id): + mastodon, user = app_login(url) + acct = mastodon.me().acct + context = mastodon.status_context(toot_id) + statuses = [] + statuses= context['ancestors'] + context['descendants'] + toot_process(statuses, acct) + + db.session.commit() + + + +def toot_process(statuses, my_acct, duplicates_counter=0): + for status in statuses: + is_reblog = False + is_myself = False + if status['reblog'] != None: + if my_acct == status['reblog']['account']['acct']: + reblog_myself = True + else: + reblog_myself = False + + is_reblog = True + + reblog_id = status['reblog']['id'] + id = status['id'] + created_at = status['created_at'] + + toot = Toot(id=id, created_at=created_at, reblog_myself=reblog_myself, reblog_id=reblog_id) + db.session.merge(toot) + # cur.execute('''INSERT OR REPLACE INTO TOOT (id,created_at,reblog_myself,reblog_id) \ + # VALUES (?,?,?,?)''',(id, created_at, reblog_myself, reblog_id)) + + if reblog_myself: + continue + + status = status['reblog'] + + id = status['id'] + + acct = status['account']['acct'] + if my_acct == acct: + is_myself = True + else: + is_myself = False + + url = status['url'] + created_at = status['created_at'] + + edited_at = status['edited_at'] + if isinstance(edited_at, str): + edited_at = dateutil.parser.parse(status['edited_at']) + + in_reply_to_id = status['in_reply_to_id'] + in_reply_to_account_id = status['in_reply_to_account_id'] + content = status['content'] + + if status['media_attachments'] != []: + media_list = "" + for media_dict in status['media_attachments']: + media_list += str(media_dict['id']) + "," + + media = Media(id=media_dict['id'], type=media_dict['type'], url=media_dict['url'], + remote_url=media_dict['remote_url'], description=media_dict['description']) + db.session.merge(media) + # cur.execute('''INSERT OR REPLACE INTO MEDIA (id,type,url,remote_url,description) \ + # VALUES (?,?,?,?,?)''',(media_dict['id'], media_dict['type'], media_dict['url'], \ + # media_dict['remote_url'], media_dict['description'])) + else: + media_list = "" + + spoiler_text = status['spoiler_text'] + + if status['poll'] != None: + poll_dict = status['poll'] + poll_id = poll_dict['id'] + expires_at = poll_dict['expires_at'] + options = str(poll_dict['options']) + + poll = Poll(id=poll_dict['id'], expires_at=expires_at, multiple=poll_dict['multiple'], \ + votes_count=poll_dict['votes_count'], options=options) + db.session.merge(poll) + # cur.execute('''INSERT OR REPLACE INTO POLL (id,expires_at,multiple,votes_count,options) \ + # VALUES (?,?,?,?,?)''',(poll_dict['id'], expires_at, poll_dict['multiple'], \ + # poll_dict['votes_count'], options)) + else: + poll_id = None + + if status['emojis'] != []: + emoji_list = "" + + for emoji in status['emojis']: + shortcode = emoji['shortcode'] + emoji_list += shortcode + "," + counter = ':' + shortcode + ':' + count = content.count(counter) + + if not is_reblog: + data=Emoji.query.filter_by(shortcode=shortcode, acct=acct).first() + if data is None: + emoji_data = Emoji(shortcode=shortcode, + acct=acct, + url=emoji['url'], + static_url=emoji['static_url'], + count=count) + db.session.merge(emoji_data) + # cur.execute('''INSERT INTO EMOJI (shortcode,url,static_url,count) \ + # VALUES (?,?,?,?)''', (shortcode, emoji['url'], emoji['static_url'], count)) + else: + data.count += count + # cur.execute("UPDATE EMOJI SET count = ? WHERE shortcode = ?",(count, shortcode)) + else: + emoji_data = Emoji(shortcode=shortcode, + acct=acct, + url=emoji['url'], + static_url=emoji['static_url']) + + db.session.merge(emoji_data) + else: + emoji_list = "" + + if status['tags'] != []: + for tag in status['tags']: + tag_data = Tag(id=id, name=tag['name']) + db.session.merge(tag_data) + # cur.execute('''INSERT OR REPLACE INTO TAG (id,name) \ + # VALUES (?,?)''',(id, tag['name'])) + + visibility = status['visibility'] + reblogged = status['reblogged'] + favourited = status['favourited'] + bookmarked = status['bookmarked'] + sensitive = status['sensitive'] + replies_count = status['replies_count'] + reblogs_count = status['reblogs_count'] + favourites_count = status['favourites_count'] + language = status['language'] + + if is_reblog or not is_myself: + table = Other() + else: + table = Toot() + + table.id=id + table.acct = acct + table.url=url + table.created_at=created_at + table.edited_at=edited_at + table.in_reply_to_id=in_reply_to_id + table.in_reply_to_account_id=in_reply_to_account_id + table.content=content + table.media_list=media_list + table.spoiler_text=spoiler_text + table.poll_id=poll_id + table.emoji_list=emoji_list + table.visibility=visibility + table.reblogged=reblogged + table.favourited=favourited + table.bookmarked=bookmarked + table.sensitive=sensitive + table.replies_count=replies_count + table.reblogs_count=reblogs_count + table.favourites_count=favourites_count + table.language=language + + if Toot.query.get(id) == None or Other.query.get(id) == None: + duplicates_counter += 1 + + db.session.merge(table) + # sql = f'''INSERT OR REPLACE INTO {table} (id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,\ + # media_list,spoiler_text,poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,\ + # favourites_count,language) \ + # VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''' + # cur.execute(sql,(id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,media_list,spoiler_text,\ + # poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,favourites_count,language)) + return duplicates_counter + +def archive_toot(url): + mastodon, user = app_login(url) acct = mastodon.me().acct statuses_count = str(mastodon.me().statuses_count) @@ -48,158 +229,7 @@ def archive_toot(url): duplicates_counter = 0 while(True): - for status in statuses: - is_reblog = False - if status['reblog'] != None: - if acct == status['reblog']['account']['acct']: - reblog_myself = True - else: - reblog_myself = False - - is_reblog = True - - reblog_id = status['reblog']['id'] - id = status['id'] - created_at = status['created_at'] - - toot = Toot(id=id, created_at=created_at, reblog_myself=reblog_myself, reblog_id=reblog_id) - db.session.merge(toot) - # cur.execute('''INSERT OR REPLACE INTO TOOT (id,created_at,reblog_myself,reblog_id) \ - # VALUES (?,?,?,?)''',(id, created_at, reblog_myself, reblog_id)) - - if reblog_myself: - continue - - status = status['reblog'] - - id = status['id'] - acct = status['account']['acct'] - url = status['url'] - created_at = status['created_at'] - - edited_at = status['edited_at'] - if isinstance(edited_at, str): - edited_at = dateutil.parser.parse(status['edited_at']) - - in_reply_to_id = status['in_reply_to_id'] - in_reply_to_account_id = status['in_reply_to_account_id'] - content = status['content'] - - if status['media_attachments'] != []: - media_list = "" - for media_dict in status['media_attachments']: - media_list += str(media_dict['id']) + "," - - media = Media(id=media_dict['id'], type=media_dict['type'], url=media_dict['url'], - remote_url=media_dict['remote_url'], description=media_dict['description']) - db.session.merge(media) - # cur.execute('''INSERT OR REPLACE INTO MEDIA (id,type,url,remote_url,description) \ - # VALUES (?,?,?,?,?)''',(media_dict['id'], media_dict['type'], media_dict['url'], \ - # media_dict['remote_url'], media_dict['description'])) - else: - media_list = "" - - spoiler_text = status['spoiler_text'] - - if status['poll'] != None: - poll_dict = status['poll'] - poll_id = poll_dict['id'] - expires_at = poll_dict['expires_at'] - options = str(poll_dict['options']) - - poll = Poll(id=poll_dict['id'], expires_at=expires_at, multiple=poll_dict['multiple'], \ - votes_count=poll_dict['votes_count'], options=options) - db.session.merge(poll) - # cur.execute('''INSERT OR REPLACE INTO POLL (id,expires_at,multiple,votes_count,options) \ - # VALUES (?,?,?,?,?)''',(poll_dict['id'], expires_at, poll_dict['multiple'], \ - # poll_dict['votes_count'], options)) - else: - poll_id = None - - if status['emojis'] != []: - emoji_list = "" - - for emoji in status['emojis']: - shortcode = emoji['shortcode'] - emoji_list += shortcode + "," - counter = ':' + shortcode + ':' - count = content.count(counter) - - if not is_reblog: - data=Emoji.query.filter_by(shortcode=shortcode, acct=acct).first() - if data is None: - emoji_data = Emoji(shortcode=shortcode, - acct=acct, - url=emoji['url'], - static_url=emoji['static_url'], - count=count) - db.session.merge(emoji_data) - # cur.execute('''INSERT INTO EMOJI (shortcode,url,static_url,count) \ - # VALUES (?,?,?,?)''', (shortcode, emoji['url'], emoji['static_url'], count)) - else: - data.count += count - # cur.execute("UPDATE EMOJI SET count = ? WHERE shortcode = ?",(count, shortcode)) - else: - emoji_data = Emoji(shortcode=shortcode, - acct=acct, - url=emoji['url'], - static_url=emoji['static_url']) - - db.session.merge(emoji_data) - else: - emoji_list = "" - - if status['tags'] != []: - for tag in status['tags']: - tag_data = Tag(id=id, name=tag['name']) - db.session.merge(tag_data) - # cur.execute('''INSERT OR REPLACE INTO TAG (id,name) \ - # VALUES (?,?)''',(id, tag['name'])) - - visibility = status['visibility'] - reblogged = status['reblogged'] - favourited = status['favourited'] - bookmarked = status['bookmarked'] - sensitive = status['sensitive'] - replies_count = status['replies_count'] - reblogs_count = status['reblogs_count'] - favourites_count = status['favourites_count'] - language = status['language'] - - table = Other() if is_reblog else Toot() - table.id=id - table.acct = acct - table.url=url - table.created_at=created_at - table.edited_at=edited_at - table.in_reply_to_id=in_reply_to_id - table.in_reply_to_account_id=in_reply_to_account_id - table.content=content - table.media_list=media_list - table.spoiler_text=spoiler_text - table.poll_id=poll_id - table.emoji_list=emoji_list - table.visibility=visibility - table.reblogged=reblogged - table.favourited=favourited - table.bookmarked=bookmarked - table.sensitive=sensitive - table.replies_count=replies_count - table.reblogs_count=reblogs_count - table.favourites_count=favourites_count - table.language=language - - if Toot.query.get(id) == None or Other.query.get(id) == None: - duplicates_counter += 1 - - db.session.merge(table) - # sql = f'''INSERT OR REPLACE INTO {table} (id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,\ - # media_list,spoiler_text,poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,\ - # favourites_count,language) \ - # VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''' - # cur.execute(sql,(id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,media_list,spoiler_text,\ - # poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,favourites_count,language)) - + duplicates_counter = toot_process(statuses, acct) db.session.commit() print(str(happy_counter) + ' / ' + statuses_count) happy_counter += 20 diff --git a/BDSM/views.py b/BDSM/views.py index 51f9bc4..325eac4 100644 --- a/BDSM/views.py +++ b/BDSM/views.py @@ -6,7 +6,7 @@ from flask import render_template, request, url_for, redirect, flash from flask_sqlalchemy import Pagination from BDSM import app, db from BDSM.models import Media, Settings, Toot, Emoji, Other -from BDSM.toot import app_register, archive_toot +from BDSM.toot import app_register, archive_toot, get_context from mastodon import Mastodon from types import SimpleNamespace from datetime import timezone @@ -53,7 +53,8 @@ def search(): def context(toot_id): def get_reply(reply_id): toots = Toot.query.order_by(Toot.created_at.desc()).filter_by(in_reply_to_id=reply_id).all() - toots = process_toot(toots) + other_toots = Other.query.order_by(Other.created_at.desc()).filter_by(in_reply_to_id=reply_id).all() + toots = process_toot(toots) + process_toot(other_toots) for i in toots: if i.in_reply_to_id != None: @@ -71,7 +72,9 @@ def context(toot_id): toot = [] toot_ = Toot.query.get(toots[0].in_reply_to_id) if toot_ == None: - break + toot_ = Other.query.get(toots[0].in_reply_to_id) + if toot_ == None: + break toot.append(toot_) toot = process_toot(toot) @@ -80,6 +83,17 @@ def context(toot_id): return render_template('view.html', toots=toots,) +@app.route('/grab/', methods=['GET', 'POST']) +def grab(toot_id): + settings = Settings.query.first() + account = settings.account[1:] + username, domain = account.split("@") + url = "https://" + domain + + get_context(url, toot_id) + flash('抓取完成……大概！') + return redirect(url_for('context',toot_id=toot_id)) + @app.route('/settings', methods=['GET', 'POST']) def settings(): if request.method == 'POST': @@ -170,15 +184,19 @@ def process_toot(toots_): toot.created_at = toot.created_at.replace(tzinfo=timezone.utc) toot.created_at = toot.created_at.astimezone(user_timezone).strftime(fmt) - if toot.reblog_id != None: - if toot.reblog_myself: - toot = Toot.query.get(toot.reblog_id) - toot = SimpleNamespace(**toot.__dict__) - toot.is_reblog = True - else: - toot = Other.query.get(toot.reblog_id) - toot = SimpleNamespace(**toot.__dict__) - toot.is_reblog = True + if hasattr(toot, 'reblog_id'): + toot.is_myself = True + if toot.reblog_id != None: + if toot.reblog_myself: + toot = Toot.query.get(toot.reblog_id) + toot = SimpleNamespace(**toot.__dict__) + toot.is_reblog = True + else: + toot = Other.query.get(toot.reblog_id) + toot = SimpleNamespace(**toot.__dict__) + toot.is_reblog = True + else: + toot.is_myself = False if toot.media_list != "": toot.medias = []