mathdatech/twoot
1
0
Fork 0
mirror of https://gitlab.com/jeancf/twoot.git synced 2025-04-26 14:43:34 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

Implemented -l command-line option

Browse Source
This commit is contained in:
jeancf 2022-11-23 09:59:06 +01:00
parent 0d1be42dcc
commit 7e7fa4620f
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
twoot.py
View File

@ -66,6 +66,7 @@ USER_AGENTS = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Vivaldi/5.4.2753.51', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Vivaldi/5.4.2753.51',
] ]
def deredir_url(url): def deredir_url(url):
""" """
Given a URL, return the URL that the page really downloads from Given a URL, return the URL that the page really downloads from
@ -123,7 +124,7 @@ def _remove_trackers_query(query_str):
"xtor", "xtref", "adid", "xtor", "xtref", "adid",
} }
query_to_clean = dict(parse_qsl(query_str, keep_blank_values=True)) query_to_clean = dict(parse_qsl(query_str, keep_blank_values=True))
query_cleaned = [(k, v) for k, v in query_to_clean.items() if not k in params_to_remove] query_cleaned = [(k, v) for k, v in query_to_clean.items() if k not in params_to_remove]
return urlencode(query_cleaned, doseq=True) return urlencode(query_cleaned, doseq=True)
@ -172,7 +173,7 @@ def clean_url(dirty_url):
return cleaned_url return cleaned_url
def process_media_body(tt_iter, remove_trackers): def process_media_body(tt_iter, remove_redir, remove_trackers):
""" """
Receives an iterator over all the elements contained in the tweet-text container. Receives an iterator over all the elements contained in the tweet-text container.
Processes them to make them suitable for posting on Mastodon Processes them to make them suitable for posting on Mastodon
@ -198,7 +199,11 @@ def process_media_body(tt_iter, remove_trackers):
tweet_text += tag_text tweet_text += tag_text
else: else:
# This is a real link # This is a real link
if remove_redir:
url = deredir_url(tag.get('href')) url = deredir_url(tag.get('href'))
else:
url = tag.get('href')
if remove_trackers: if remove_trackers:
tweet_text += clean_url(url) tweet_text += clean_url(url)
else: else:
@ -382,6 +387,7 @@ def main(argv):
parser.add_argument('-p', metavar='<mastodon password>', action='store', required=True) parser.add_argument('-p', metavar='<mastodon password>', action='store', required=True)
parser.add_argument('-r', action='store_true', help='Also post replies to other tweets') parser.add_argument('-r', action='store_true', help='Also post replies to other tweets')
parser.add_argument('-s', action='store_true', help='Suppress retweets') parser.add_argument('-s', action='store_true', help='Suppress retweets')
parser.add_argument('-l', action='store_true', help='Remove link redirection')
parser.add_argument('-u', action='store_true', help='Remove trackers from URLs') parser.add_argument('-u', action='store_true', help='Remove trackers from URLs')
parser.add_argument('-v', action='store_true', help='Ingest twitter videos and upload to Mastodon instance') parser.add_argument('-v', action='store_true', help='Ingest twitter videos and upload to Mastodon instance')
parser.add_argument('-a', metavar='<max age (in days)>', action='store', type=float, default=1) parser.add_argument('-a', metavar='<max age (in days)>', action='store', type=float, default=1)
@ -397,6 +403,7 @@ def main(argv):
mast_password = args['p'] mast_password = args['p']
tweets_and_replies = args['r'] tweets_and_replies = args['r']
suppress_retweets = args['s'] suppress_retweets = args['s']
remove_redir = args['l']
remove_trackers = args['u'] remove_trackers = args['u']
get_vids = args['v'] get_vids = args['v']
max_age = float(args['a']) max_age = float(args['a'])
@ -423,6 +430,7 @@ def main(argv):
logging.info(' -m ' + mast_account) logging.info(' -m ' + mast_account)
logging.info(' -r ' + str(tweets_and_replies)) logging.info(' -r ' + str(tweets_and_replies))
logging.info(' -s ' + str(suppress_retweets)) logging.info(' -s ' + str(suppress_retweets))
logging.info(' -l ' + str(remove_redir))
logging.info(' -u ' + str(remove_trackers)) logging.info(' -u ' + str(remove_trackers))
logging.info(' -v ' + str(get_vids)) logging.info(' -v ' + str(get_vids))
logging.info(' -a ' + str(max_age)) logging.info(' -a ' + str(max_age))
@ -579,7 +587,7 @@ def main(argv):
tt_iter = status.find('div', class_='tweet-content media-body').children tt_iter = status.find('div', class_='tweet-content media-body').children
# Process text of tweet # Process text of tweet
tweet_text += process_media_body(tt_iter, remove_trackers) tweet_text += process_media_body(tt_iter, remove_redir, remove_trackers)
# Process quote: append link to tweet_text # Process quote: append link to tweet_text
quote_div = status.find('a', class_='quote-link') quote_div = status.find('a', class_='quote-link')