From 4704890ddf14996c88466ee3b07dc1fd676ca426 Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 09:28:28 +0200 Subject: [PATCH 01/12] check rate limit --- twoot.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/twoot.py b/twoot.py index c2b33b0..f6b825d 100755 --- a/twoot.py +++ b/twoot.py @@ -201,6 +201,12 @@ def is_time_valid(timestamp, max_age, min_delay): return ret def login(instance, account, password): + # Check ratelimit status + logging.info('Ratelimit allowed requests: ' + Mastodon.ratelimit_limit) + logging.info('Ratelimit remaining requests: ' + Mastodon.ratelimit_remaining) + logging.info('Ratelimit reset time: ' + Mastodon.ratelimit_reset) + logging.info('Ratelimit last call: ' + Mastodon.ratelimit_lastcall) + # Create Mastodon application if it does not exist yet if not os.path.isfile(instance + '.secret'): try: @@ -230,7 +236,7 @@ def login(instance, account, password): logging.info('Logging in to ' + instance) except MastodonError as me: - logging.fatal('ERROR: Login to ' + instance + ' Failed\n') + logging.fatal('ERROR: Login to ' + instance + ' Failed') logging.fatal(me) sys.exit(-1) From a7b63f569f023f3a5f881f3e5097961066b59b59 Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 09:35:02 +0200 Subject: [PATCH 02/12] Changed logging to info --- twoot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twoot.py b/twoot.py index f6b825d..8ba6408 100755 --- a/twoot.py +++ b/twoot.py @@ -281,7 +281,7 @@ def main(argv): # Setup logging to file logging.basicConfig( filename=twit_account + '.log', - level=logging.WARN, + level=logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', ) From 63a7a578a49b995036e5d13290f6dfc8e47e26f8 Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 09:37:30 +0200 Subject: [PATCH 03/12] epoch to local time --- twoot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twoot.py b/twoot.py index 8ba6408..e31f2fc 100755 --- a/twoot.py +++ b/twoot.py @@ -204,8 +204,8 @@ def login(instance, account, password): # Check ratelimit status logging.info('Ratelimit allowed requests: ' + Mastodon.ratelimit_limit) logging.info('Ratelimit remaining requests: ' + Mastodon.ratelimit_remaining) - logging.info('Ratelimit reset time: ' + Mastodon.ratelimit_reset) - logging.info('Ratelimit last call: ' + Mastodon.ratelimit_lastcall) + logging.info('Ratelimit reset time: ' + time.localtime(Mastodon.ratelimit_reset)) + logging.info('Ratelimit last call: ' + time.localtime(Mastodon.ratelimit_lastcall)) # Create Mastodon application if it does not exist yet if not os.path.isfile(instance + '.secret'): From ffdce1ad12dd4d8d523c208d1d0ea979d0c05245 Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 10:05:19 +0200 Subject: [PATCH 04/12] updated url --- twoot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twoot.py b/twoot.py index e31f2fc..3c05346 100755 --- a/twoot.py +++ b/twoot.py @@ -41,7 +41,7 @@ NITTER_URLS = [ 'https://nitter.eu', 'https://nitter.namazso.eu', 'https://n.actionsack.com', - 'https://nittereu.moomoo.me', + 'https://nitter.moomoo.me', 'https://n.ramle.be', ] From 2b21a626d49d9474c550b9c745ca82bb4a752332 Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 10:11:37 +0200 Subject: [PATCH 05/12] Less stupid --- twoot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/twoot.py b/twoot.py index 3c05346..9110db9 100755 --- a/twoot.py +++ b/twoot.py @@ -201,12 +201,6 @@ def is_time_valid(timestamp, max_age, min_delay): return ret def login(instance, account, password): - # Check ratelimit status - logging.info('Ratelimit allowed requests: ' + Mastodon.ratelimit_limit) - logging.info('Ratelimit remaining requests: ' + Mastodon.ratelimit_remaining) - logging.info('Ratelimit reset time: ' + time.localtime(Mastodon.ratelimit_reset)) - logging.info('Ratelimit last call: ' + time.localtime(Mastodon.ratelimit_lastcall)) - # Create Mastodon application if it does not exist yet if not os.path.isfile(instance + '.secret'): try: @@ -240,6 +234,12 @@ def login(instance, account, password): logging.fatal(me) sys.exit(-1) + # Check ratelimit status + logging.info('Ratelimit allowed requests: ' + mastodon.ratelimit_limit) + logging.info('Ratelimit remaining requests: ' + mastodon.ratelimit_remaining) + logging.info('Ratelimit reset time: ' + time.localtime(mastodon.ratelimit_reset)) + logging.info('Ratelimit last call: ' + time.localtime(mastodon.ratelimit_lastcall)) + return mastodon From 357e45844d1055be81bbaf82673ba9c2533c0eac Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 10:15:14 +0200 Subject: [PATCH 06/12] convert int to str --- twoot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twoot.py b/twoot.py index 9110db9..164ed26 100755 --- a/twoot.py +++ b/twoot.py @@ -235,8 +235,8 @@ def login(instance, account, password): sys.exit(-1) # Check ratelimit status - logging.info('Ratelimit allowed requests: ' + mastodon.ratelimit_limit) - logging.info('Ratelimit remaining requests: ' + mastodon.ratelimit_remaining) + logging.info('Ratelimit allowed requests: ' + str(mastodon.ratelimit_limit)) + logging.info('Ratelimit remaining requests: ' + str(mastodon.ratelimit_remaining)) logging.info('Ratelimit reset time: ' + time.localtime(mastodon.ratelimit_reset)) logging.info('Ratelimit last call: ' + time.localtime(mastodon.ratelimit_lastcall)) From 392b0bafd0c48e5b7c5dfa2c7f80757eb220ce9d Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 10:17:14 +0200 Subject: [PATCH 07/12] more str conversion --- twoot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twoot.py b/twoot.py index 164ed26..982b812 100755 --- a/twoot.py +++ b/twoot.py @@ -237,8 +237,8 @@ def login(instance, account, password): # Check ratelimit status logging.info('Ratelimit allowed requests: ' + str(mastodon.ratelimit_limit)) logging.info('Ratelimit remaining requests: ' + str(mastodon.ratelimit_remaining)) - logging.info('Ratelimit reset time: ' + time.localtime(mastodon.ratelimit_reset)) - logging.info('Ratelimit last call: ' + time.localtime(mastodon.ratelimit_lastcall)) + logging.info('Ratelimit reset time: ' + str(time.localtime(mastodon.ratelimit_reset))) + logging.info('Ratelimit last call: ' + str(time.localtime(mastodon.ratelimit_lastcall))) return mastodon From 4ccce6aac1e1fadae303baeb672b08b4a222c0a8 Mon Sep 17 00:00:00 2001 From: jeancf Date: Thu, 8 Sep 2022 10:19:23 +0200 Subject: [PATCH 08/12] asctime() instead --- twoot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twoot.py b/twoot.py index 982b812..2f7673f 100755 --- a/twoot.py +++ b/twoot.py @@ -237,8 +237,8 @@ def login(instance, account, password): # Check ratelimit status logging.info('Ratelimit allowed requests: ' + str(mastodon.ratelimit_limit)) logging.info('Ratelimit remaining requests: ' + str(mastodon.ratelimit_remaining)) - logging.info('Ratelimit reset time: ' + str(time.localtime(mastodon.ratelimit_reset))) - logging.info('Ratelimit last call: ' + str(time.localtime(mastodon.ratelimit_lastcall))) + logging.info('Ratelimit reset time: ' + time.asctime(time.localtime(mastodon.ratelimit_reset))) + logging.info('Ratelimit last call: ' + time.asctime(time.localtime(mastodon.ratelimit_lastcall))) return mastodon From bfbe9704f7c0c48a91f2ed3feb9126f1bf163ec8 Mon Sep 17 00:00:00 2001 From: jeancf Date: Wed, 14 Sep 2022 16:28:48 +0200 Subject: [PATCH 09/12] Cosmetic changes --- twoot.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/twoot.py b/twoot.py index 2f7673f..4decf22 100755 --- a/twoot.py +++ b/twoot.py @@ -200,6 +200,7 @@ def is_time_valid(timestamp, max_age, min_delay): return ret + def login(instance, account, password): # Create Mastodon application if it does not exist yet if not os.path.isfile(instance + '.secret'): @@ -239,7 +240,7 @@ def login(instance, account, password): logging.info('Ratelimit remaining requests: ' + str(mastodon.ratelimit_remaining)) logging.info('Ratelimit reset time: ' + time.asctime(time.localtime(mastodon.ratelimit_reset))) logging.info('Ratelimit last call: ' + time.asctime(time.localtime(mastodon.ratelimit_lastcall))) - + return mastodon @@ -305,7 +306,7 @@ def main(argv): mastodon_instance, mastodon_account, tweet_id)''') # Select random nitter instance to fetch updates from - nitter_url = NITTER_URLS[random.randint(0, len(NITTER_URLS)-1)] + nitter_url = NITTER_URLS[random.randint(0, len(NITTER_URLS) - 1)] # ********************************************************** # Load twitter page of user. Process all tweets and generate @@ -348,9 +349,9 @@ def main(argv): logging.info('Nitter page downloaded successfully from ' + url) # DEBUG: Save page to file - #of = open(twit_account + '.html', 'w') - #of.write(twit_account_page.text) - #of.close() + # of = open(twit_account + '.html', 'w') + # of.write(twit_account_page.text) + # of.close() # Make soup soup = BeautifulSoup(twit_account_page.text, 'html.parser') @@ -421,7 +422,7 @@ def main(argv): # Add prefix if the tweet is a reply-to # Only consider item of class 'replying-to' that is a direct child - # of class 'tweet-body' in status. Others can be in a quoted tweet. + # of class 'tweet-body' in status. Others can be in a quoted tweet. replying_to_class = status.select("div.tweet-body > div.replying-to") if len(replying_to_class) != 0: tweet_text += 'Replying to ' + replying_to_class[0].a.get_text() + '\n\n' @@ -516,10 +517,10 @@ def main(argv): logging.info(str(in_db_cnt) + ' tweets already in database') # DEBUG: Print extracted tweets - #for t in tweets: - #print(t) + # for t in tweets: + # print(t) - # Login to account on maston instance + # Login to account on maston instance mastodon = None if len(tweets) != 0: mastodon = login(mast_instance, mast_account, mast_password) @@ -550,7 +551,7 @@ def main(argv): logging.debug("Uploading video failed") pass - else: # Only upload pic if no video was uploaded + else: # Only upload pic if no video_file_listdeo was uploaded # Upload photos for photo in tweet['photos']: media = False From 5e0fb1a9c35a185569813b2814e9e090a8021026 Mon Sep 17 00:00:00 2001 From: jeancf Date: Wed, 14 Sep 2022 16:35:10 +0200 Subject: [PATCH 10/12] Corrected typo --- twoot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twoot.py b/twoot.py index 4decf22..e3ffbb2 100755 --- a/twoot.py +++ b/twoot.py @@ -551,7 +551,7 @@ def main(argv): logging.debug("Uploading video failed") pass - else: # Only upload pic if no video_file_listdeo was uploaded + else: # Only upload pic if no video was uploaded # Upload photos for photo in tweet['photos']: media = False From 7f462a5a6ecc442ad8f498c8a8b29a17068ef079 Mon Sep 17 00:00:00 2001 From: jeancf Date: Wed, 14 Sep 2022 16:54:47 +0200 Subject: [PATCH 11/12] Minor improvement to logging --- twoot.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/twoot.py b/twoot.py index e3ffbb2..f28d494 100755 --- a/twoot.py +++ b/twoot.py @@ -34,6 +34,10 @@ from mastodon import Mastodon, MastodonError, MastodonAPIError, MastodonIllegalA import subprocess import shutil +# Set the desired verbosity of logging +# One of logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL +LOGGING_LEVEL = logging.INFO + NITTER_URLS = [ 'https://nitter.42l.fr', 'https://nitter.pussthecat.org', @@ -236,10 +240,10 @@ def login(instance, account, password): sys.exit(-1) # Check ratelimit status - logging.info('Ratelimit allowed requests: ' + str(mastodon.ratelimit_limit)) - logging.info('Ratelimit remaining requests: ' + str(mastodon.ratelimit_remaining)) - logging.info('Ratelimit reset time: ' + time.asctime(time.localtime(mastodon.ratelimit_reset))) - logging.info('Ratelimit last call: ' + time.asctime(time.localtime(mastodon.ratelimit_lastcall))) + logging.debug('Ratelimit allowed requests: ' + str(mastodon.ratelimit_limit)) + logging.debug('Ratelimit remaining requests: ' + str(mastodon.ratelimit_remaining)) + logging.debug('Ratelimit reset time: ' + time.asctime(time.localtime(mastodon.ratelimit_reset))) + logging.debug('Ratelimit last call: ' + time.asctime(time.localtime(mastodon.ratelimit_lastcall))) return mastodon @@ -282,7 +286,7 @@ def main(argv): # Setup logging to file logging.basicConfig( filename=twit_account + '.log', - level=logging.INFO, + level=LOGGING_LEVEL, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', ) From 644827420840f76cbe25bb65429d6155d0a39f62 Mon Sep 17 00:00:00 2001 From: jeancf Date: Wed, 14 Sep 2022 17:02:48 +0200 Subject: [PATCH 12/12] Updated README --- README.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1e9e01d..c84dac3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ Twoot is a python script that extracts tweets from a twitter feed and reposts them as toots on a Mastodon account. +**UPDATE 14 SEP 2022** Added information about the status of throttling +applied by the Mastodon instance in the debug log. Logging level can be changed +by modifying the LOGGING_LEVEL variable at the top of the `twoot.py` file. + **UPDATE 22 AUG 2022** Fixed bug that would incorrectly mark a new tweet as a "reply to" if it quoted a tweet that is a reply-to. @@ -28,7 +32,7 @@ from tweets considered as "sensitive content" mobile twitter page without JavaScript after the breaking change of last week. -# Features +## Features * Fetch timeline of given users from twitter.com * Scrape html and formats tweets for post on mastodon @@ -40,7 +44,7 @@ of last week. * Remember tweets already tooted to prevent double posting * Optionally post reply-to tweets on the mastodon account -# usage +## usage ``` twoot.py [-h] -t -i -m @@ -48,7 +52,7 @@ twoot.py [-h] -t -i -m [-d ] [-c ] ``` -# arguments +## arguments Assuming that the Twitter handle is @SuperDuperBot and the Mastodon account is @superduperbot@botsin.space @@ -74,19 +78,21 @@ Default max age is 1 day. Decimal values are OK. Default min delay is 0 minutes. -# installation +## installation Make sure python3 is installed. Twoot depends on `beautifulsoup4` and `Mastodon.py` python modules. **Only If you plan to download videos** with the `-v` switch, are the additional dependencies required: + * Python modules `m3u8` and `ffmpeg-python` * [ffmpeg](https://ffmpeg.org/download.html) (installed with the package manager of your distribution) +```sh +pip install beautifulsoup4 Mastodon.py m3u8 ffmpeg-python ``` -> pip install beautifulsoup4 Mastodon.py m3u8 ffmpeg-python -``` + In your user folder, execute `git clone https://gitlab.com/jeancf/twoot.git` to clone repo with twoot.py script. @@ -98,7 +104,8 @@ ago: 1-59/15 * * * * /path/to/twoot.py -t SuperDuperBot -i botsin.space -m superduperbot -p my_Sup3r-S4f3*pw -a 5 -d 15 ``` -# Background +## Background + I started twoot when [tootbot](https://github.com/cquest/tootbot) stopped working. Tootbot relies on rss feeds from https://twitrss.me that broke when Twitter refreshed their web UI in July 2019.