Update profile almost finished

twoot.py

@@ -44,7 +44,7 @@ HTTPS_REQ_TIMEOUT = 10
 NITTER_URLS = [
     'https://nitter.lacontrevoie.fr',  # rate limited
     'https://n.l5.ca',
-    'https://nitter.it',  # added 27/02/2023
+    # 'https://nitter.it',  # different pic naming scheme
     'https://nitter.sethforprivacy.com',  # added on 01/06/2023
     'https://nitter.cutelab.space',  # USA, added 16/02/2023
     'https://nitter.fly.dev',  # anycast, added 06/02/2023
@@ -170,7 +170,7 @@ def build_config(args):
         terminate(-1)
 
 
-def update_profile(soup, sql, mast_password):
+def update_profile(nitter_url, soup, sql, mast_password):
     """
     Update the profile on Mastodon
     :param soup: BeautifulSoup object containing the page
@@ -180,43 +180,67 @@ def update_profile(soup, sql, mast_password):
     """
     # TODO  Check if TOML option to update profile is set
 
+    # @noirextreme banner before https://nitter.net/pic/https%3A%2F%2Fpbs.twimg.com%2Fprofile_banners%2F57014511%2F1525787041%2F1500x500
+    #                      after https://nitter.net/pic/https%3A%2F%2Fpbs.twimg.com%2Fprofile_banners%2F57014511%2F1686822924%2F1500x500
+    # @noirextreme avatar before https://pbs.twimg.com/profile_images/1081778074/Twitter_400x400.png
+    #                      after
+
     db = sql.cursor()
 
     # Extract avatar picture address
-    new_avatar_url = 'https://' + unquote(soup.find('div', class_='profile-card-info').findChild('a').findChild('img').get('src').removeprefix('/pic/'))
+    new_avatar_url = soup.find('div', class_='profile-card-info').findChild('a').findChild('img').get('src')
 
     # Extract banner picture address
-    new_banner_url = unquote(soup.find('div', class_='profile-banner').findChild('a').findChild('img').get('src').removeprefix('/pic/')) + ".jpg"
+    new_banner_url = soup.find('div', class_='profile-banner').findChild('a').findChild('img').get('src')
 
     # Get the original urls of the avatar and banner pictures on the account profile
     db.execute("SELECT avatar_url, banner_url FROM profiles WHERE mastodon_account=?", (TOML['config']['mastodon_user'],))
     profile_in_db = db.fetchone()
 
+    changed = False
     if profile_in_db is not None:
         cur_avatar_url = profile_in_db[0]
         cur_banner_url = profile_in_db[1]
 
         # Check if urls have changed
-        if new_avatar_url == cur_avatar_url:
+        if new_avatar_url != cur_avatar_url:
-            new_avatar_url = None
+            changed = True
-        else:
+            print(new_avatar_url)
             logging.info('avatar image changed on twitter profile')
-        if new_banner_url == cur_banner_url:
+        if new_banner_url != cur_banner_url:
-            new_banner_url = None
+            changed = True
-        else:
+            print(new_banner_url)
             logging.info('banner image changed on twitter profile')
     else:
         # Mastodon user not found in database. Add new record
         db.execute("INSERT INTO profiles (mastodon_account, avatar_url, banner_url) VALUES (?, ?, ?)", (TOML['config']['mastodon_user'], "", ""))
         sql.commit()
+        changed = True
 
     mastodon = None
-    if new_avatar_url is not None or new_banner_url is not None:
+
+    # Update if necessary
+    if changed:
         logging.info('updating profile on Mastodon')
+
+        # Download images
+        new_avatar = requests.get(nitter_url + new_avatar_url, timeout=HTTPS_REQ_TIMEOUT) if new_avatar_url is not None else None
+        new_avatar_img = new_avatar.content if new_avatar.status_code == 200 else None
+        new_avatar_mime = new_avatar.headers['content-type'] if new_avatar.status_code == 200 else None
+        if new_avatar.status_code !=200:
+            logging.error("Could not download image from " + nitter_url + new_avatar_url)
+
+        new_banner = requests.get(nitter_url + new_banner_url, timeout=HTTPS_REQ_TIMEOUT) if new_banner_url is not None else None
+        new_banner_img = new_banner.content if new_banner.status_code == 200 else None
+        new_banner_mime = new_banner.headers['content-type'] if new_banner.status_code == 200 else None
+        if new_banner.status_code !=200:
+            logging.error("Could not download image from " + nitter_url + new_banner_url)
+
         mastodon = login(mast_password)
         # Update profile on Mastodon
+        # media.content, mime_type=media.headers['content-type']
         try:
-            mastodon.account_update_credentials(avatar=new_avatar_url, avatar_mime_type='image/jpg', header=new_banner_url, header_mime_type='image/jpg')
+            mastodon.account_update_credentials(avatar=new_avatar_img, avatar_mime_type=new_avatar_mime, header=new_banner_img, header_mime_type=new_banner_mime)
         except Exception as e:
             logging.error("Could not update profile")
             logging.error(e)
@@ -808,9 +832,9 @@ def main(argv):
     logging.debug('Nitter page downloaded successfully from ' + url)
 
     # DEBUG: Save page to file
-    # of = open(TOML['config']['twitter_account'] + '.html', 'w')
+    of = open(TOML['config']['twitter_account'] + '.html', 'w')
-    # of.write(twit_account_page.text)
+    of.write(twit_account_page.text)
-    # of.close()
+    of.close()
 
     # Make soup
     soup = BeautifulSoup(twit_account_page.text, 'html.parser')
@@ -987,7 +1011,7 @@ def main(argv):
     mastodon = None
 
     # Update profile if it has changed
-    mastodon = update_profile(soup, sql, mast_password)
+    mastodon = update_profile(nitter_url, soup, sql, mast_password)
 
     # Login to account on maston instance
     if len(tweets) != 0 and mastodon is None: