mathdatech/twoot
1
0
Fork 0
mirror of https://gitlab.com/jeancf/twoot.git synced 2024-11-23 20:11:11 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

updated user agents

Browse Source
This commit is contained in:
jeancf 2023-06-19 20:13:46 +02:00
parent 7edde25d22
commit 062054c836
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
twoot.py
View File

@ -62,13 +62,12 @@ NITTER_URLS = [
# Update from https://www.whatismybrowser.com/guides/the-latest-user-agent/ # Update from https://www.whatismybrowser.com/guides/the-latest-user-agent/
USER_AGENTS = [ USER_AGENTS = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.46', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.51',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 OPR/99.0.0.0',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Vivaldi/5.6.2867.62', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Vivaldi/6.1.3035.84',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Vivaldi/5.6.2867.62',
] ]
@ -123,10 +122,10 @@ def build_config(args):
loaded_toml = tomllib.load(config_file) loaded_toml = tomllib.load(config_file)
except FileNotFoundError: except FileNotFoundError:
print('config file not found') print('config file not found')
terminate(-1) shutdown(-1)
except tomllib.TOMLDecodeError: except tomllib.TOMLDecodeError:
print('Malformed config file') print('Malformed config file')
terminate(-1) shutdown(-1)
TOML['config'] = loaded_toml['config'] TOML['config'] = loaded_toml['config']
for k in TOML['options'].keys(): for k in TOML['options'].keys():
@ -244,7 +243,7 @@ def update_profile(nitter_url, soup, sql, mast_password):
if new_avatar is not None: if new_avatar is not None:
new_avatar_img = new_avatar.content if new_avatar.status_code == 200 else None new_avatar_img = new_avatar.content if new_avatar.status_code == 200 else None
new_avatar_mime = new_avatar.headers['content-type'] if new_avatar.status_code == 200 else None new_avatar_mime = new_avatar.headers['content-type'] if new_avatar.status_code == 200 else None
if new_avatar.status_code !=200: if new_avatar.status_code != 200:
logging.error("Could not download avatar image from " + nitter_url + new_avatar_url) logging.error("Could not download avatar image from " + nitter_url + new_avatar_url)
else: else:
logging.debug("Avatar image downloaded") logging.debug("Avatar image downloaded")
@ -253,7 +252,7 @@ def update_profile(nitter_url, soup, sql, mast_password):
if new_banner is not None: if new_banner is not None:
new_banner_img = new_banner.content if new_banner.status_code == 200 else None new_banner_img = new_banner.content if new_banner.status_code == 200 else None
new_banner_mime = new_banner.headers['content-type'] if new_banner.status_code == 200 else None new_banner_mime = new_banner.headers['content-type'] if new_banner.status_code == 200 else None
if new_banner.status_code !=200: if new_banner.status_code != 200:
logging.error("Could not download banner image from " + nitter_url + new_banner_url) logging.error("Could not download banner image from " + nitter_url + new_banner_url)
else: else:
logging.debug("Banner image downloaded") logging.debug("Banner image downloaded")
@ -607,7 +606,7 @@ def login(password):
except MastodonError as me: except MastodonError as me:
logging.fatal('failed to create app on ' + TOML['config']['mastodon_instance']) logging.fatal('failed to create app on ' + TOML['config']['mastodon_instance'])
logging.fatal(me) logging.fatal(me)
terminate(-1) shutdown(-1)
mastodon = None mastodon = None
@ -629,7 +628,7 @@ def login(password):
except MastodonError as me: except MastodonError as me:
logging.fatal('Login to ' + TOML['config']['mastodon_instance'] + ' Failed\n') logging.fatal('Login to ' + TOML['config']['mastodon_instance'] + ' Failed\n')
logging.fatal(me) logging.fatal(me)
terminate(-1) shutdown(-1)
if os.path.isfile(TOML['config']['mastodon_user'] + '.secret'): if os.path.isfile(TOML['config']['mastodon_user'] + '.secret'):
logging.warning('''You successfully logged in using a password and an access token logging.warning('''You successfully logged in using a password and an access token
@ -645,15 +644,15 @@ def login(password):
except MastodonError as me: except MastodonError as me:
logging.fatal('Login to ' + TOML['config']['mastodon_instance'] + ' Failed\n') logging.fatal('Login to ' + TOML['config']['mastodon_instance'] + ' Failed\n')
logging.fatal(me) logging.fatal(me)
terminate(-1) shutdown(-1)
else: else:
logging.fatal('No .secret file found. Password required to log in') logging.fatal('No .secret file found. Password required to log in')
terminate(-1) shutdown(-1)
return mastodon return mastodon
def terminate(exit_code): def shutdown(exit_code):
""" """
Cleanly stop execution with a message on execution duration Cleanly stop execution with a message on execution duration
Remove log messages older that duration specified in config from log file Remove log messages older that duration specified in config from log file
@ -845,16 +844,16 @@ def main(argv):
twit_account_page = session.get(url, headers=headers, timeout=HTTPS_REQ_TIMEOUT) twit_account_page = session.get(url, headers=headers, timeout=HTTPS_REQ_TIMEOUT)
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
logging.fatal('Host did not respond when trying to download ' + url) logging.fatal('Host did not respond when trying to download ' + url)
terminate(-1) shutdown(-1)
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
logging.fatal(nitter_url + ' took too long to respond') logging.fatal(nitter_url + ' took too long to respond')
terminate(-1) shutdown(-1)
# Verify that download worked # Verify that download worked
if twit_account_page.status_code != 200: if twit_account_page.status_code != 200:
logging.fatal('The Nitter page did not download correctly from ' + url + ' (' + str( logging.fatal('The Nitter page did not download correctly from ' + url + ' (' + str(
twit_account_page.status_code) + '). Aborting') twit_account_page.status_code) + '). Aborting')
terminate(-1) shutdown(-1)
logging.debug('Nitter page downloaded successfully from ' + url) logging.debug('Nitter page downloaded successfully from ' + url)
@ -1161,7 +1160,7 @@ def main(argv):
logging.info('Deleted ' + str(excess_count) + ' old records from database.') logging.info('Deleted ' + str(excess_count) + ' old records from database.')
terminate(0) shutdown(0)
if __name__ == "__main__": if __name__ == "__main__":