fix/edited_at compatible with lower versions

fix/fix edited_at typeerror

BDSM/models.py

@@ -26,7 +26,7 @@ class Toot(db.Model):
     favourites_count = db.Column(db.Integer)
     language = db.Column(db.Text)
 
-class Reblog(db.Model):
+class Other(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     acct = db.Column(db.Text)
     url = db.Column(db.Text)

BDSM/templates/view.html

@@ -12,16 +12,20 @@
                 <a href="{{ toot.url }}" target="_blank"
                     rel="noopener noreferrer">
                     <span class="time">
-                {% if toot. visibility == "public"%}
+                {% if toot.visibility == "public"%}
                 <span class="visibility-icon"><i class="fa-solid fa-earth-americas fa-globle"></i></span>
-                {% elif toot. visibility == "unlisted"%}
+                {% elif toot.visibility == "unlisted"%}
                 <span class="visibility-icon"><i class="fa-solid fa-lock-open fa-unlock"></i></span>
-                {% elif toot. visibility == "private"%}
+                {% elif toot.visibility == "private"%}
                 <span class="visibility-icon"><i class="fa-solid fa-lock"></i></span>
-                {% elif toot. visibility == "direct"%}
+                {% elif toot.visibility == "direct"%}
                 <span class="visibility-icon"><i class="fa-solid fa-envelope fa-at"></i></span>
                 {% endif %}
-                <time>{{ toot.created_at }}</time></span></a>
+                <time>{{ toot.created_at }}
+                {% if toot.edited_at != None %}
+                    <strong>*</strong>
+                {% endif %}
+                </time></span></a>
             </div>
             <div class="content">
                 {% if toot.spoiler_text != "" %}
@@ -73,7 +77,9 @@
                 {% endif %}
                 </div>
                 <div class="action-bar">
-                    {% if not toot.is_reblog %}
+                    {% if toot.is_myself %}
+                    <span><i class="fa-solid fa-satellite-dish"></i><a href="{{ url_for('grab', toot_id=toot.id) }}"
+                            target=" _blank">抓取回复</a></span>
                     <span><i class="fa-solid fa-up-right-and-down-left-from-center"></i><a
                             href="{{ url_for('context', toot_id=toot.id) }}" target=" _blank">上下文</a></span>
                     {% endif %}

BDSM/toot.py

@@ -2,8 +2,9 @@
 
 from mastodon import Mastodon
 from BDSM import db
-from BDSM.models import Reblog, Toot, Tag, Media, Emoji, Poll
+from BDSM.models import Other, Toot, Tag, Media, Emoji, Poll
 import sys
+import dateutil.parser
 
 def app_register(url):
     print("Registering app")
@@ -14,7 +15,7 @@ def app_register(url):
         scopes=["read"]
     )
 
-def archive_toot(url):
+def app_login(url):
     mastodon = Mastodon(
         client_id='pyBDSM_clientcred.secret',
         access_token='user.secret',
@@ -35,6 +36,190 @@ def archive_toot(url):
         # exit in either case
         sys.exit(1)
 
+    return mastodon, user
+
+def get_context(url, toot_id):
+    mastodon, user = app_login(url)
+    acct = mastodon.me().acct
+    context = mastodon.status_context(toot_id)
+    statuses = []
+    statuses= context['ancestors'] + context['descendants']
+    toot_process(statuses, acct)
+
+    db.session.commit()
+
+
+
+def toot_process(statuses, my_acct, duplicates_counter=0):
+    for status in statuses:
+        is_reblog = False
+        is_myself = False
+        if status['reblog'] != None:
+            if my_acct == status['reblog']['account']['acct']:
+                reblog_myself = True
+            else:
+                reblog_myself = False
+
+            is_reblog = True
+
+            reblog_id = status['reblog']['id']
+            id = status['id']
+            created_at = status['created_at']
+
+            toot = Toot(id=id, created_at=created_at, reblog_myself=reblog_myself, reblog_id=reblog_id)
+            db.session.merge(toot)
+            # cur.execute('''INSERT OR REPLACE INTO TOOT (id,created_at,reblog_myself,reblog_id) \
+            #     VALUES (?,?,?,?)''',(id, created_at, reblog_myself, reblog_id))
+
+            if reblog_myself:
+                continue
+
+            status = status['reblog']
+
+        id = status['id']
+
+        acct = status['account']['acct']
+        if my_acct == acct:
+            is_myself = True
+        else:
+            is_myself = False
+
+        url = status['url']
+        created_at = status['created_at']
+
+        if 'edited_at' in status:
+            edited_at = status['edited_at']
+            if isinstance(edited_at, str):
+                edited_at = dateutil.parser.parse(status['edited_at'])
+        else:
+            edited_at = None
+
+        in_reply_to_id = status['in_reply_to_id']
+        in_reply_to_account_id = status['in_reply_to_account_id']
+        content = status['content']
+
+        if status['media_attachments'] != []:
+            media_list = ""
+            for media_dict in status['media_attachments']:
+                media_list += str(media_dict['id']) + ","
+
+                media = Media(id=media_dict['id'], type=media_dict['type'], url=media_dict['url'],
+                                remote_url=media_dict['remote_url'], description=media_dict['description'])
+                db.session.merge(media)
+                # cur.execute('''INSERT OR REPLACE INTO MEDIA (id,type,url,remote_url,description) \
+                #     VALUES (?,?,?,?,?)''',(media_dict['id'], media_dict['type'], media_dict['url'], \
+                #         media_dict['remote_url'], media_dict['description']))
+        else:
+            media_list = ""
+
+        spoiler_text = status['spoiler_text']
+
+        if status['poll'] != None:
+            poll_dict = status['poll']
+            poll_id = poll_dict['id']
+            expires_at = poll_dict['expires_at']
+            options = str(poll_dict['options'])
+
+            poll = Poll(id=poll_dict['id'], expires_at=expires_at, multiple=poll_dict['multiple'], \
+                    votes_count=poll_dict['votes_count'], options=options)
+            db.session.merge(poll)
+            # cur.execute('''INSERT OR REPLACE INTO POLL (id,expires_at,multiple,votes_count,options) \
+            #     VALUES (?,?,?,?,?)''',(poll_dict['id'], expires_at, poll_dict['multiple'], \
+            #         poll_dict['votes_count'], options))
+        else:
+            poll_id = None
+
+        if status['emojis'] != []:
+            emoji_list = ""
+
+            for emoji in status['emojis']:
+                shortcode = emoji['shortcode']
+                emoji_list += shortcode + ","
+                counter = ':' + shortcode + ':'
+                count = content.count(counter)
+
+                if not is_reblog:
+                    data=Emoji.query.filter_by(shortcode=shortcode, acct=acct).first()
+                    if data is None:
+                        emoji_data = Emoji(shortcode=shortcode,
+                                        acct=acct,
+                                        url=emoji['url'],
+                                        static_url=emoji['static_url'],
+                                        count=count)
+                        db.session.merge(emoji_data)
+                        # cur.execute('''INSERT INTO EMOJI (shortcode,url,static_url,count) \
+                        #     VALUES (?,?,?,?)''', (shortcode, emoji['url'], emoji['static_url'], count))
+                    else:
+                        data.count += count
+                        # cur.execute("UPDATE EMOJI SET count = ? WHERE shortcode = ?",(count, shortcode))
+                else:
+                    emoji_data = Emoji(shortcode=shortcode,
+                                    acct=acct,
+                                    url=emoji['url'],
+                                    static_url=emoji['static_url'])
+
+                    db.session.merge(emoji_data)
+        else:
+            emoji_list = ""
+
+        if status['tags'] != []:
+            for tag in status['tags']:
+                tag_data = Tag(id=id, name=tag['name'])
+                db.session.merge(tag_data)
+                # cur.execute('''INSERT OR REPLACE INTO TAG (id,name) \
+                #     VALUES (?,?)''',(id, tag['name']))
+
+        visibility = status['visibility']
+        reblogged = status['reblogged']
+        favourited = status['favourited']
+        bookmarked = status['bookmarked']
+        sensitive = status['sensitive']
+        replies_count = status['replies_count']
+        reblogs_count = status['reblogs_count']
+        favourites_count = status['favourites_count']
+        language = status['language']
+
+        if is_reblog or not is_myself:
+            table = Other()
+        else:
+            table = Toot()
+
+        table.id=id
+        table.acct = acct
+        table.url=url
+        table.created_at=created_at
+        table.edited_at=edited_at
+        table.in_reply_to_id=in_reply_to_id
+        table.in_reply_to_account_id=in_reply_to_account_id
+        table.content=content
+        table.media_list=media_list
+        table.spoiler_text=spoiler_text
+        table.poll_id=poll_id
+        table.emoji_list=emoji_list
+        table.visibility=visibility
+        table.reblogged=reblogged
+        table.favourited=favourited
+        table.bookmarked=bookmarked
+        table.sensitive=sensitive
+        table.replies_count=replies_count
+        table.reblogs_count=reblogs_count
+        table.favourites_count=favourites_count
+        table.language=language
+
+        if Toot.query.get(id) == None or Other.query.get(id) == None:
+            duplicates_counter += 1
+
+        db.session.merge(table)
+        # sql = f'''INSERT OR REPLACE INTO {table} (id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,\
+        #     media_list,spoiler_text,poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,\
+        #         favourites_count,language) \
+        #             VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
+        # cur.execute(sql,(id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,media_list,spoiler_text,\
+        #     poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,favourites_count,language))
+    return duplicates_counter
+
+def archive_toot(url):
+    mastodon, user = app_login(url)
     acct = mastodon.me().acct
     statuses_count = str(mastodon.me().statuses_count)
 
@@ -44,157 +229,18 @@ def archive_toot(url):
     # pprint(statuses)
 
     happy_counter = 20
+    duplicates_counter = 0
 
     while(True):
-        for status in statuses:
-            is_reblog = False
-            if status['reblog'] != None:
-                if acct == status['reblog']['account']['acct']:
-                    reblog_myself = True
-                else:
-                    reblog_myself = False
-
-                is_reblog = True
-
-                reblog_id = status['reblog']['id']
-                id = status['id']
-                created_at = status['created_at']
-
-                toot = Toot(id=id, created_at=created_at, reblog_myself=reblog_myself, reblog_id=reblog_id)
-                db.session.add(toot)
-                # cur.execute('''INSERT OR REPLACE INTO TOOT (id,created_at,reblog_myself,reblog_id) \
-                #     VALUES (?,?,?,?)''',(id, created_at, reblog_myself, reblog_id))
-
-                if reblog_myself:
-                    continue
-
-                status = status['reblog']
-
-            id = status['id']
-            acct = status['account']['acct']
-            url = status['url']
-            created_at = status['created_at']
-            edited_at = status['edited_at'] if status['edited_at'] != None else None
-            in_reply_to_id = status['in_reply_to_id']
-            in_reply_to_account_id = status['in_reply_to_account_id']
-            content = status['content']
-
-            if status['media_attachments'] != []:
-                media_list = ""
-                for media_dict in status['media_attachments']:
-                    media_list += str(media_dict['id']) + ","
-
-                    media = Media(id=media_dict['id'], type=media_dict['type'], url=media_dict['url'],
-                                  remote_url=media_dict['remote_url'], description=media_dict['description'])
-                    db.session.add(media)
-                    # cur.execute('''INSERT OR REPLACE INTO MEDIA (id,type,url,remote_url,description) \
-                    #     VALUES (?,?,?,?,?)''',(media_dict['id'], media_dict['type'], media_dict['url'], \
-                    #         media_dict['remote_url'], media_dict['description']))
-            else:
-                media_list = ""
-
-            spoiler_text = status['spoiler_text']
-
-            if status['poll'] != None:
-                poll_dict = status['poll']
-                poll_id = poll_dict['id']
-                expires_at = poll_dict['expires_at']
-                options = str(poll_dict['options'])
-
-                poll = Poll(id=poll_dict['id'], expires_at=expires_at, multiple=poll_dict['multiple'], \
-                        votes_count=poll_dict['votes_count'], options=options)
-                db.session.add(poll)
-                # cur.execute('''INSERT OR REPLACE INTO POLL (id,expires_at,multiple,votes_count,options) \
-                #     VALUES (?,?,?,?,?)''',(poll_dict['id'], expires_at, poll_dict['multiple'], \
-                #         poll_dict['votes_count'], options))
-            else:
-                poll_id = None
-
-            if status['emojis'] != []:
-                emoji_list = ""
-
-                for emoji in status['emojis']:
-                    shortcode = emoji['shortcode']
-                    emoji_list += shortcode + ","
-                    counter = ':' + shortcode + ':'
-                    count = content.count(counter)
-
-                    if not is_reblog:
-                        data=Emoji.query.filter_by(shortcode=shortcode, acct=acct).first()
-                        if data is None:
-                            emoji_data = Emoji(shortcode=shortcode,
-                                            acct=acct,
-                                            url=emoji['url'],
-                                            static_url=emoji['static_url'],
-                                            count=count)
-                            db.session.add(emoji_data)
-                            # cur.execute('''INSERT INTO EMOJI (shortcode,url,static_url,count) \
-                            #     VALUES (?,?,?,?)''', (shortcode, emoji['url'], emoji['static_url'], count))
-                        else:
-                            data.count += count
-                            # cur.execute("UPDATE EMOJI SET count = ? WHERE shortcode = ?",(count, shortcode))
-                    else:
-                        emoji_data = Emoji(shortcode=shortcode,
-                                        acct=acct,
-                                        url=emoji['url'],
-                                        static_url=emoji['static_url'])
-
-                        db.session.merge(emoji_data)
-            else:
-                emoji_list = ""
-
-            if status['tags'] != []:
-                for tag in status['tags']:
-                    tag_data = Tag(id=id, name=tag['name'])
-                    db.session.add(tag_data)
-                    # cur.execute('''INSERT OR REPLACE INTO TAG (id,name) \
-                    #     VALUES (?,?)''',(id, tag['name']))
-
-            visibility = status['visibility']
-            reblogged = status['reblogged']
-            favourited = status['favourited']
-            bookmarked = status['bookmarked']
-            sensitive = status['sensitive']
-            replies_count = status['replies_count']
-            reblogs_count = status['reblogs_count']
-            favourites_count = status['favourites_count']
-            language = status['language']
-
-            table = Reblog() if is_reblog else Toot()
-            table.id=id
-            table.acct = acct
-            table.url=url
-            table.created_at=created_at
-            table.edited_at=edited_at
-            table.in_reply_to_id=in_reply_to_id
-            table.in_reply_to_account_id=in_reply_to_account_id
-            table.content=content
-            table.media_list=media_list
-            table.spoiler_text=spoiler_text
-            table.poll_id=poll_id
-            table.emoji_list=emoji_list
-            table.visibility=visibility
-            table.reblogged=reblogged
-            table.favourited=favourited
-            table.bookmarked=bookmarked
-            table.sensitive=sensitive
-            table.replies_count=replies_count
-            table.reblogs_count=reblogs_count
-            table.favourites_count=favourites_count
-            table.language=language
-
-            db.session.add(table)
-            # sql = f'''INSERT OR REPLACE INTO {table} (id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,\
-            #     media_list,spoiler_text,poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,\
-            #         favourites_count,language) \
-            #             VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
-            # cur.execute(sql,(id,url,created_at,edited_at,in_reply_to_id,in_reply_to_account_id,content,media_list,spoiler_text,\
-            #     poll_id,emoji_list,visibility,reblogged,favourited,bookmarked,sensitive,reblogs_count,favourites_count,language))
-
+        duplicates_counter = toot_process(statuses, acct)
         db.session.commit()
         print(str(happy_counter) + ' / '  + statuses_count)
         happy_counter += 20
 
+        if duplicates_counter >= 10:
+            print("检测到重复嘟文达到十次，取消存档……")
+            break
+
         statuses = mastodon.fetch_next(statuses)
         # statuses = None
         if statuses == None:

BDSM/views.py

@@ -5,8 +5,8 @@ import pytz
 from flask import render_template, request, url_for, redirect, flash
 from flask_sqlalchemy import Pagination
 from BDSM import app, db
-from BDSM.models import Media, Settings, Toot, Emoji, Reblog
-from BDSM.toot import app_register, archive_toot
+from BDSM.models import Media, Settings, Toot, Emoji, Other
+from BDSM.toot import app_register, archive_toot, get_context
 from mastodon import Mastodon
 from types import SimpleNamespace
 from datetime import timezone
@@ -53,7 +53,8 @@ def search():
 def context(toot_id):
     def get_reply(reply_id):
         toots = Toot.query.order_by(Toot.created_at.desc()).filter_by(in_reply_to_id=reply_id).all()
-        toots = process_toot(toots)
+        other_toots = Other.query.order_by(Other.created_at.desc()).filter_by(in_reply_to_id=reply_id).all()
+        toots = process_toot(toots) + process_toot(other_toots)
 
         for i in toots:
             if i.in_reply_to_id != None:
@@ -71,7 +72,9 @@ def context(toot_id):
         toot = []
         toot_ = Toot.query.get(toots[0].in_reply_to_id)
         if toot_ == None:
-            break
+            toot_ = Other.query.get(toots[0].in_reply_to_id)
+            if toot_ == None:
+                break
 
         toot.append(toot_)
         toot = process_toot(toot)
@@ -80,6 +83,17 @@ def context(toot_id):
 
     return render_template('view.html', toots=toots,)
 
+@app.route('/grab/<int:toot_id>', methods=['GET', 'POST'])
+def grab(toot_id):
+    settings = Settings.query.first()
+    account = settings.account[1:]
+    username, domain = account.split("@")
+    url = "https://" + domain
+
+    get_context(url, toot_id)
+    flash('抓取完成……大概！')
+    return redirect(url_for('context',toot_id=toot_id))
+
 @app.route('/settings', methods=['GET', 'POST'])
 def settings():
     if request.method == 'POST':
@@ -145,9 +159,6 @@ def archive():
     settings = Settings.query.first()
     if settings == None:
         return redirect(url_for('settings'))
-    elif len(Toot.query.all()) > 0:
-        flash('现暂不支持重复存档！') #TODO
-        return redirect(url_for('index'))
     else:
         account = settings.account[1:]
         username, domain = account.split("@")
@@ -173,15 +184,19 @@ def process_toot(toots_):
         toot.created_at = toot.created_at.replace(tzinfo=timezone.utc)
         toot.created_at = toot.created_at.astimezone(user_timezone).strftime(fmt)
 
-        if toot.reblog_id != None:
-            if toot.reblog_myself:
-                toot = Toot.query.get(toot.reblog_id)
-                toot = SimpleNamespace(**toot.__dict__)
-                toot.is_reblog = True
-            else:
-                toot = Reblog.query.get(toot.reblog_id)
-                toot = SimpleNamespace(**toot.__dict__)
-                toot.is_reblog = True
+        if hasattr(toot, 'reblog_id'):
+            toot.is_myself = True
+            if toot.reblog_id != None:
+                if toot.reblog_myself:
+                    toot = Toot.query.get(toot.reblog_id)
+                    toot = SimpleNamespace(**toot.__dict__)
+                    toot.is_reblog = True
+                else:
+                    toot = Other.query.get(toot.reblog_id)
+                    toot = SimpleNamespace(**toot.__dict__)
+                    toot.is_reblog = True
+        else:
+            toot.is_myself = False
 
         if toot.media_list != "":
             toot.medias = []