From 84b94a38b9cc4cbe9cd17fefd9f444be26ef32c8 Mon Sep 17 00:00:00 2001 From: jeancf Date: Sun, 13 Nov 2022 22:17:43 +0100 Subject: [PATCH 1/8] Implemented retweet suppression --- twoot.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/twoot.py b/twoot.py index 9c7caeb..1b72cf2 100755 --- a/twoot.py +++ b/twoot.py @@ -268,6 +268,7 @@ def main(argv): parser.add_argument('-m', metavar='', action='store', required=True) parser.add_argument('-p', metavar='', action='store', required=True) parser.add_argument('-r', action='store_true', help='Also post replies to other tweets') + parser.add_argument('-s', action='store_true', help='Suppress retweets') parser.add_argument('-v', action='store_true', help='Ingest twitter videos and upload to Mastodon instance') parser.add_argument('-a', metavar='', action='store', type=float, default=1) parser.add_argument('-d', metavar='', action='store', type=float, default=0) @@ -281,6 +282,7 @@ def main(argv): mast_account = args['m'] mast_password = args['p'] tweets_and_replies = args['r'] + suppress_retweets = args['s'] get_vids = args['v'] max_age = float(args['a']) min_delay = float(args['d']) @@ -410,6 +412,13 @@ def main(argv): logging.debug("Tweet outside valid time range, skipping") continue + # Check if retweets must be skipped + if suppress_retweets: + # Check if this tweet is a retweet + if len(status.select("div.tweet-body > div > div.tweet-header")) != 0: + logging.debug("Retweet ignored per command-line configuration") + continue + # Check in database if tweet has already been posted db.execute("SELECT * FROM toots WHERE twitter_account=? AND mastodon_instance=? AND mastodon_account=? AND tweet_id=?", (twit_account, mast_instance, mast_account, tweet_id)) @@ -444,7 +453,7 @@ def main(argv): tweet_text += 'Replying to ' + replying_to_class[0].a.get_text() + '\n\n' # Check it the tweet is a retweet from somebody else - if author_account.lower() != twit_account.lower(): + if len(status.select("div.tweet-body > div > div.tweet-header")) != 0: tweet_text = 'RT from ' + author + ' (@' + author_account + ')\n\n' # extract iterator over tweet text contents From e8f2ede4208e08cf1b1fbc303cd5fd60f2e2b624 Mon Sep 17 00:00:00 2001 From: jeancf Date: Sun, 13 Nov 2022 22:20:24 +0100 Subject: [PATCH 2/8] Updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fd21514..abda17c 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ of last week. ``` twoot.py [-h] -t -i -m - -p [-r] [-v] [-a ] + -p [-r] [-s] [-v] [-a ] [-d ] [-c ] ``` @@ -78,6 +78,7 @@ is @superduperbot@botsin.space | -p | Mastodon password | `my_Sup3r-S4f3*pw` | Yes | | -v | upload videos to Mastodon | *N/A* | No | | -r | Post reply-to tweets (ignored by default) | *N/A* | No | +| -s | suppress retweets (posted by default) | *N/A* | No | | -a | Max. age of tweet to post (in days) | `5` | No | | -d | Min. age before posting new tweet (in minutes) | `15` | No | | -c | Max number of toots allowed to post (cap) | `1` | No | From 608bc7519f1059b72f7d590c821fba859d332972 Mon Sep 17 00:00:00 2001 From: jeancf Date: Sun, 13 Nov 2022 22:35:46 +0100 Subject: [PATCH 3/8] Corrected condition on retweet tag --- twoot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twoot.py b/twoot.py index 1b72cf2..37f348d 100755 --- a/twoot.py +++ b/twoot.py @@ -415,7 +415,7 @@ def main(argv): # Check if retweets must be skipped if suppress_retweets: # Check if this tweet is a retweet - if len(status.select("div.tweet-body > div > div.tweet-header")) != 0: + if len(status.select("div.tweet-body > div > div.retweet-header")) != 0: logging.debug("Retweet ignored per command-line configuration") continue @@ -453,7 +453,7 @@ def main(argv): tweet_text += 'Replying to ' + replying_to_class[0].a.get_text() + '\n\n' # Check it the tweet is a retweet from somebody else - if len(status.select("div.tweet-body > div > div.tweet-header")) != 0: + if len(status.select("div.tweet-body > div > div.retweet-header")) != 0: tweet_text = 'RT from ' + author + ' (@' + author_account + ')\n\n' # extract iterator over tweet text contents From 514a1b330405ecabdea4a60cde0fb84c771c0340 Mon Sep 17 00:00:00 2001 From: jeancf Date: Mon, 14 Nov 2022 12:26:55 +0100 Subject: [PATCH 4/8] Added some temp debug code --- twoot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/twoot.py b/twoot.py index 37f348d..4896f26 100755 --- a/twoot.py +++ b/twoot.py @@ -454,6 +454,8 @@ def main(argv): # Check it the tweet is a retweet from somebody else if len(status.select("div.tweet-body > div > div.retweet-header")) != 0: + # TEMP DEBUG + login.debug("retweet DIV found in " + status) tweet_text = 'RT from ' + author + ' (@' + author_account + ')\n\n' # extract iterator over tweet text contents From f96d8fa93cbd80ecd33a39f1737445a475d6597b Mon Sep 17 00:00:00 2001 From: jeancf Date: Mon, 14 Nov 2022 12:36:06 +0100 Subject: [PATCH 5/8] Added missing logging --- twoot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/twoot.py b/twoot.py index 4896f26..d9daadb 100755 --- a/twoot.py +++ b/twoot.py @@ -307,6 +307,7 @@ def main(argv): logging.info(' -i ' + mast_instance) logging.info(' -m ' + mast_account) logging.info(' -r ' + str(tweets_and_replies)) + logging.info(' -s ' + str(suppress_retweets)) logging.info(' -v ' + str(get_vids)) logging.info(' -a ' + str(max_age)) logging.info(' -d ' + str(min_delay)) From b04b7dc195cf42d2fa45e7e283e5a1d843d14018 Mon Sep 17 00:00:00 2001 From: jeancf Date: Mon, 14 Nov 2022 12:40:56 +0100 Subject: [PATCH 6/8] Removed temp debug --- README.md | 2 +- twoot.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index abda17c..c1c374e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ maintained solution. **UPDATE 08 OCT 2022** VERSION 2.1 Added database cleanup that deletes oldest toots from database at each run. Keep MAX_REC_COUNT (50 by default) -rows in db for each twitter feed. +rows in db for each twitter feed.t **UPDATE 14 SEP 2022** Added information about the status of throttling applied by the Mastodon instance in the debug log. Logging level can be changed diff --git a/twoot.py b/twoot.py index d9daadb..ea46b34 100755 --- a/twoot.py +++ b/twoot.py @@ -455,8 +455,6 @@ def main(argv): # Check it the tweet is a retweet from somebody else if len(status.select("div.tweet-body > div > div.retweet-header")) != 0: - # TEMP DEBUG - login.debug("retweet DIV found in " + status) tweet_text = 'RT from ' + author + ' (@' + author_account + ')\n\n' # extract iterator over tweet text contents From 69be50aa7f6331dea537487d40627dca3288e784 Mon Sep 17 00:00:00 2001 From: jeancf Date: Tue, 15 Nov 2022 11:51:58 +0100 Subject: [PATCH 7/8] Added CHANGELOG --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3e3acee --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +**15 NOV 2022** VERSION 2.3 Added command-line option (`-s`) to +skip retweets. With this option, retweets will be ignored and not posted +on Mastodon. + +**12 NOV 2022** VERSION 2.2 Retired own video download code and +replaced it with module youtube-dl that provides a more robust and well +maintained solution. + +> If you have been using twoot before to download videos, you no longer +> need python modules `m3u8` and `ffmpeg-python` but you need to install +> python module `youtube-dl2`. + +**08 OCT 2022** VERSION 2.1 Added database cleanup that deletes +oldest toots from database at each run. Keep MAX_REC_COUNT (50 by default) +rows in db for each twitter feed.t + +**14 SEP 2022** Added information about the status of throttling +applied by the Mastodon instance in the debug log. Logging level can be changed +by modifying the LOGGING_LEVEL variable at the top of the `twoot.py` file. + +**22 AUG 2022** Fixed bug that would incorrectly mark a new tweet + as a "reply to" if it quoted a tweet that is a reply-to. + +**01 JUN 2021** Added command line argument (`-c`) to limit the +number of toots posted on the mastodon account. + +**19 DEC 2020** VERSION 2.0 Twitter's *no-javascript* version +has been retired. Twoot has been rewritten to get content from +[nitter.net](https://nitter.net) or one of its mirrors which is a +javascript-free mirror of twitter. As a bonus (or a curse?) twoot now +also supports animated GIFs. + +**05 APR 2020** VERSION 1.0. Twoot can now optionally download +videos from Twitter and upload them on Mastodon. + +**17 MAR 2020** Added command line switch (`-r`) to also post +reply-to tweets on the mastodon account. They will not be included by +default anymore. + +**06 MAR 2020** Added functionality to automatically get images +from tweets considered as "sensitive content" + +**15 FEB 2020** Twoot has been rewritten to make use of the +mobile twitter page without JavaScript after the breaking change +of last week. From 2fafcb9a26fc5deec95faa3ee5569b2c8d0147f4 Mon Sep 17 00:00:00 2001 From: jeancf Date: Tue, 15 Nov 2022 11:52:13 +0100 Subject: [PATCH 8/8] Added release to README --- README.md | 54 ++++++++++-------------------------------------------- 1 file changed, 10 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index c1c374e..a5aa8b4 100644 --- a/README.md +++ b/README.md @@ -3,59 +3,25 @@ Twoot is a python script that extracts tweets from a twitter feed and reposts them as toots on a Mastodon account. -**UPDATE 12 NOV 2022** VERSION 2.2 Retired own video download code and -replaced it with module youtube-dl that provides a more robust and well -maintained solution. +**UPDATE 15 NOV 2022** VERSION 2.3 Added command-line option (`-s`) to +skip retweets. With this option, retweets will be ignored and not posted +on Mastodon. -> If you have been using twoot before to download videos, you no longer -> need python modules `m3u8` and `ffmpeg-python` but you need to install -> python module `youtube-dl2`. - -**UPDATE 08 OCT 2022** VERSION 2.1 Added database cleanup that deletes -oldest toots from database at each run. Keep MAX_REC_COUNT (50 by default) -rows in db for each twitter feed.t - -**UPDATE 14 SEP 2022** Added information about the status of throttling -applied by the Mastodon instance in the debug log. Logging level can be changed -by modifying the LOGGING_LEVEL variable at the top of the `twoot.py` file. - -**UPDATE 22 AUG 2022** Fixed bug that would incorrectly mark a new tweet - as a "reply to" if it quoted a tweet that is a reply-to. - -**UPDATE 01 JUN 2021** Added command line argument (`-c`) to limit the -number of toots posted on the mastodon account. - -**UPDATE 19 DEC 2020** VERSION 2.0 Twitter's *no-javascript* version -has been retired. Twoot has been rewritten to get content from -[nitter.net](https://nitter.net) or one of its mirrors which is a -javascript-free mirror of twitter. As a bonus (or a curse?) twoot now -also supports animated GIFs. - -**UPDATE 05 APR 2020** VERSION 1.0. Twoot can now optionally download -videos from Twitter and upload them on Mastodon. - -**UPDATE 17 MAR 2020** Added command line switch (`-r`) to also post -reply-to tweets on the mastodon account. They will not be included by -default anymore. - -**UPDATE 06 MAR 2020** Added functionality to automatically get images -from tweets considered as "sensitive content" - -**UPDATE 15 FEB 2020** Twoot has been rewritten to make use of the -mobile twitter page without JavaScript after the breaking change -of last week. +> Previous updates can be found in CHANGELOG. ## Features -* Fetch timeline of given users from twitter.com -* Scrape html and formats tweets for post on mastodon +* Fetch timeline of given user from twitter.com (through nitter instance) +* Scrape html and format tweets for post on mastodon * Emojis supported -* Optionally upload videos from tweet to Mastodon * Upload images from tweet to Mastodon +* Optionally upload videos from tweet to Mastodon * Specify maximum age of tweet to be considered * Specify minimum delay before considering a tweet for upload * Remember tweets already tooted to prevent double posting * Optionally post reply-to tweets on the mastodon account +* Optionally ignore retweets +* Allows rate-limiting posts to Mastodon instance ## usage @@ -78,7 +44,7 @@ is @superduperbot@botsin.space | -p | Mastodon password | `my_Sup3r-S4f3*pw` | Yes | | -v | upload videos to Mastodon | *N/A* | No | | -r | Post reply-to tweets (ignored by default) | *N/A* | No | -| -s | suppress retweets (posted by default) | *N/A* | No | +| -s | Skip retweets (posted by default) | *N/A* | No | | -a | Max. age of tweet to post (in days) | `5` | No | | -d | Min. age before posting new tweet (in minutes) | `15` | No | | -c | Max number of toots allowed to post (cap) | `1` | No |