first threads support...

tootbot.py

@@ -27,11 +27,11 @@ def unredir(redir):
                 r.headers.get('Location')
         else:
             redir = r.headers.get('Location')
-        print('redir', redir)
+        # print('redir', redir)
         if '//ow.ly/' in redir or '//bit.ly/' in redir:
             redir = redir.replace('https://ow.ly/', 'http://ow.ly/') # only http
             redir = requests.get(redir, allow_redirects=False).headers.get('Location')
-            print('redir+', redir)
+            # print('redir+', redir)
         try:
             r = requests.get(redir, allow_redirects=False, timeout=5)
         except:
@@ -223,20 +223,31 @@ else:
         c = html.unescape(t['tweet'])
         # do not toot twitter replies
         if 'reply_to' in t and len(t['reply_to'])>0:
-            print('Reply:',c)
+            # print('Reply:',c)
             continue
         # do not toot twitter quoted RT
         if 'quote_url' in t and t['quote_url'] != '':
-            print('Quoted:', c)
+            # print('Quoted:', c)
             continue
 
-        # check if this tweet has been processed
-        id = t['id']
-        db.execute('SELECT * FROM tweets WHERE tweet = ? AND twitter = ?  and mastodon = ? and instance = ?', (id, source, mastodon, instance))  # noqa
-        last = db.fetchone()
+        # detect threads
+        in_reply_to = None
+        if 'conversation_id' in t and t['conversation_id'] not in t['link']:
+            db.execute('SELECT toot FROM tweets WHERE tweet = ? AND twitter = ?', (t['conversation_id'], source))  # noqa
+            thread = db.fetchone()
+            if thread:
+                print("Thread :", t['conversation_id'], t['link'], thread[0])
+                in_reply_to = thread[0]
 
-        # process only unprocessed tweets
-        if last:
+        # check if this tweet has been processed
+        id = t['id'] # old id
+        db.execute('SELECT * FROM tweets WHERE tweet = ? AND twitter = ?  and mastodon = ? and instance = ?', (id, source, mastodon, instance))  # noqa
+        if db.fetchone():
+            continue
+
+        id = t['link'].split('/')[-1] # new id from status link to support threads
+        db.execute('SELECT * FROM tweets WHERE tweet = ? AND twitter = ?  and mastodon = ? and instance = ?', (id, source, mastodon, instance))  # noqa
+        if db.fetchone():
             continue
 
         if c[-1] == "…":
@@ -254,21 +265,21 @@ else:
 
         if 'photos' in t:
             for url in t['photos']:
-                print('photo', url)
+                # print('photo', url)
                 try:
                     media = requests.get(url.replace(
                             'https://pbs.twimg.com/', 'https://nitter.net/pic/orig/'))
-                    print("received nitter", media.headers.get('content-type'))
+                    # print("received nitter", media.headers.get('content-type'))
                     media_posted = mastodon_api.media_post(
                         media.content, mime_type=media.headers.get('content-type'))
-                    print("posted")
+                    # print("posted")
                     toot_media.append(media_posted['id'])
                 except:
                     media = requests.get(url)
-                    print("received twitter", media.headers.get('content-type'))
+                    # print("received twitter", media.headers.get('content-type'))
                     media_posted = mastodon_api.media_post(
                         media.content, mime_type=media.headers.get('content-type'))
-                    print("posted")
+                    # print("posted")
                     toot_media.append(media_posted['id'])
 
 
@@ -286,23 +297,30 @@ else:
             if m is None:
                 c = c.replace(l, redir)
             else:
-                print('lien:',l)
-                c = c.replace(l, '')
                 video = redir
-                print('video:', video)
-                subprocess.run('rm -f out.mp4; yt-dlp -N 8 -o out.mp4 --recode-video mp4 --no-playlist %s --max-filesize 100M' %
-                            (video,), shell=True, capture_output=False)
-                print("received")
-                try:
-                    file = open("out.mp4", "rb")
-                    video_data = file.read()
-                    file.close()
-                    media_posted = mastodon_api.media_post(video_data, mime_type='video/mp4')
-                    c = c.replace(video, '')
-                    print("posted")
-                    toot_media.append(media_posted['id'])
-                except:
-                    pass
+                # print('video:', video)
+                video_json = subprocess.run('yt-dlp -s -j %s' %
+                               (video,), shell=True, capture_output=True)
+                video_info = json.loads(video_json.stdout)
+                if video_info['duration'] < 600:
+                    # print('lien:', l)
+                    c = c.replace(l, '')
+                    subprocess.run('rm -f out.*; yt-dlp -N 8 -o out.mp4 --recode-video mp4 --no-playlist --max-filesize 100M %s' %
+                                (video,), shell=True, capture_output=False)
+                    # print("received")
+                    try:
+                        file = open("out.mp4", "rb")
+                        video_data = file.read()
+                        file.close()
+                        media_posted = mastodon_api.media_post(video_data, mime_type='video/mp4')
+                        c = c.replace(video, '')
+                        # print("posted")
+                        toot_media.append(media_posted['id'])
+                        os.remove("out.mp4")
+                    except:
+                        pass
+                else:
+                    print("video duration > 600s : ", video_info['duration'])
 
         # remove pic.twitter.com links
         m = re.search(r"pic.twitter.com[^ \xa0]*", c)
@@ -328,16 +346,16 @@ else:
             if len(toot_media)>0:
                 time.sleep(5)
             toot = mastodon_api.status_post(c,
-                                        in_reply_to_id=None,
+                                        in_reply_to_id=in_reply_to,
                                         media_ids=toot_media,
                                         sensitive=False,
                                         visibility='unlisted',
                                         spoiler_text=None)
         except:
-            print("10s delay")
-            time.sleep(10)
+            print("delay")
+            time.sleep(30)
             toot = mastodon_api.status_post(c,
-                                            in_reply_to_id=None,
+                                            in_reply_to_id=in_reply_to,
                                             media_ids=toot_media,
                                             sensitive=False,
                                             visibility='unlisted',