mathdatech/twoot
1
0
Fork 0
mirror of https://gitlab.com/jeancf/twoot.git synced 2024-11-23 20:11:11 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

Reformat some code

Browse Source
This commit is contained in:
jeancf 2023-09-14 11:30:55 +02:00
parent 22a8df8983
commit 9b29471140
1 changed files with 28 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
twoot.py
View File

@ -29,7 +29,7 @@ import sqlite3
import sys import sys
import time import time
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse, urljoin, unquote from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse, urljoin
import requests import requests
from bs4 import BeautifulSoup, element from bs4 import BeautifulSoup, element
@ -71,12 +71,12 @@ def build_config(args):
'nitter.poast.org', # added 25/08/2023 'nitter.poast.org', # added 25/08/2023
'nitter.d420.de', # added 25/08/2023 'nitter.d420.de', # added 25/08/2023
'nitter.nicfab.eu', # added 25/08/2023 'nitter.nicfab.eu', # added 25/08/2023
'bird.habedieeh.re', # added 01/09/2023 'bird.habedieeh.re', # added 01/09/2023
'nitter.salastil.com', # added 25/08/2023 'nitter.salastil.com', # added 25/08/2023
'nitter.cz', # added 25/08/2023 'nitter.cz', # added 25/08/2023
'nitter.privacydev.net', # added 25/08/2023 'nitter.privacydev.net', # added 25/08/2023
'tweet.whateveritworks.org', # added 13/09/2023 'tweet.whateveritworks.org', # added 13/09/2023
'nitter.hyperreal.coffee', # added 13/09/2023 'nitter.hyperreal.coffee', # added 13/09/2023
# 'nitter.unixfox.eu', # rate-limited 13/09/2023 # 'nitter.unixfox.eu', # rate-limited 13/09/2023
# 'nt.ggtyler.dev', # gone 13/09/2023 # 'nt.ggtyler.dev', # gone 13/09/2023
], ],
@ -172,16 +172,16 @@ def build_config(args):
exit(-1) exit(-1)
"""
Dowload page with full thread of tweets and extract all replied to tweet reference by url.
Only used by `get_timeline()`.
:param session: Existing HTTP session with Nitter instance
:param headers: HTTP headers to use
:param nitter url: url of the nitter instance to use
:param thread_url: url of the first tweet in thread
:return: list of tuples with url of tweet replied-to (or None) and content of tweet
"""
def _get_rest_of_thread(session, headers, nitter_url, thread_url, first_item): def _get_rest_of_thread(session, headers, nitter_url, thread_url, first_item):
"""
Dowload page with full thread of tweets and extract all replied to tweet reference by url.
Only used by `get_timeline()`.
:param session: Existing HTTP session with Nitter instance
:param headers: HTTP headers to use
:param nitter url: url of the nitter instance to use
:param thread_url: url of the first tweet in thread
:return: list of tuples with url of tweet replied-to (or None) and content of tweet
"""
# Add first item to timeline # Add first item to timeline
timeline = [(None, first_item)] timeline = [(None, first_item)]
@ -221,26 +221,26 @@ def _get_rest_of_thread(session, headers, nitter_url, thread_url, first_item):
# Build timeline of tuples # Build timeline of tuples
previous_tweet_url = thread_url previous_tweet_url = thread_url
for item in list: for item in list:
timeline.append((previous_tweet_url, item)) timeline.append((previous_tweet_url, item))
# Get the url of the tweet # Get the url of the tweet
tweet_link_tag = item.find('a', class_='tweet-link') tweet_link_tag = item.find('a', class_='tweet-link')
if tweet_link_tag is not None: if tweet_link_tag is not None:
previous_tweet_url = tweet_link_tag.get('href').strip('#m') previous_tweet_url = tweet_link_tag.get('href').strip('#m')
else: else:
previous_tweet_url = None previous_tweet_url = None
logging.error('Thread tweet is missing link tag') logging.error('Thread tweet is missing link tag')
# return timeline in reverse chronological order # return timeline in reverse chronological order
timeline.reverse() timeline.reverse()
return timeline return timeline
"""
Download timeline of twitter account
:param url: url of the account page to download
:return: list of tuples with url of tweet replied-to (or None) and content of tweet
"""
def get_timeline(nitter_url): def get_timeline(nitter_url):
"""
Download timeline of twitter account
:param url: url of the account page to download
:return: list of tuples with url of tweet replied-to (or None) and content of tweet
"""
# Define url to use # Define url to use
url = nitter_url + '/' + TOML['config']['twitter_account'] url = nitter_url + '/' + TOML['config']['twitter_account']
@ -419,6 +419,7 @@ def update_profile(nitter_url, soup, sql, mast_password):
return mastodon return mastodon
def deredir_url(url): def deredir_url(url):
""" """
Given a URL, return the URL that the page really downloads from Given a URL, return the URL that the page really downloads from
@ -1174,7 +1175,7 @@ def main(argv):
if mastodon is not None: if mastodon is not None:
try: try:
max_characters = mastodon.instance().configuration.statuses['max_characters'] max_characters = mastodon.instance().configuration.statuses['max_characters']
logging.debug('Instance character limit is '+ str(max_characters)) logging.debug('Instance character limit is ' + str(max_characters))
except Exception: except Exception:
# Default value for Mastodon # Default value for Mastodon
max_characters = 500 max_characters = 500