mathdatech/twoot
1
0
Fork 0
mirror of https://gitlab.com/jeancf/twoot.git synced 2025-03-27 07:56:48 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

Added info log messages

Browse Source
This commit is contained in:
jeancf 2020-12-18 22:09:34 +01:00
parent 33342cdfb7
commit 1525955c52
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
twoot.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Copyright (C) 2019 Jean-Christophe Francois Copyright (C) 2020 Jean-Christophe Francois
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -220,7 +220,7 @@ def main(argv):
pass pass
# Setup logging to file # Setup logging to file
logging.basicConfig(filename=twit_account + '.log', level=logging.DEBUG) logging.basicConfig(filename=twit_account + '.log', level=logging.INFO)
logging.info('Running with the following parameters:') logging.info('Running with the following parameters:')
logging.info(' -t ' + twit_account) logging.info(' -t ' + twit_account)
logging.info(' -i ' + mast_instance) logging.info(' -i ' + mast_instance)
@ -273,9 +273,9 @@ def main(argv):
logging.info('Nitter page downloaded successfully') logging.info('Nitter page downloaded successfully')
# DEBUG: Save page to file # DEBUG: Save page to file
of = open(twit_account + '.html', 'w') #of = open(twit_account + '.html', 'w')
of.write(twit_account_page.text) #of.write(twit_account_page.text)
of.close() #of.close()
# Make soup # Make soup
soup = BeautifulSoup(twit_account_page.text, 'html.parser') soup = BeautifulSoup(twit_account_page.text, 'html.parser')
@ -295,6 +295,8 @@ def main(argv):
# Process each tweets and generate dictionary # Process each tweets and generate dictionary
# with data ready to be posted on Mastodon # with data ready to be posted on Mastodon
# ********************************************************** # **********************************************************
out_date_cnt = 0
in_db_cnt = 0
for status in timeline: for status in timeline:
# Extract tweet ID and status ID # Extract tweet ID and status ID
tweet_id = status.find('a', class_='tweet-link').get('href').strip('#m') tweet_id = status.find('a', class_='tweet-link').get('href').strip('#m')
@ -308,6 +310,7 @@ def main(argv):
# Check if time is within acceptable range # Check if time is within acceptable range
if not is_time_valid(timestamp, max_age, min_delay): if not is_time_valid(timestamp, max_age, min_delay):
out_date_cnt += 1
logging.debug("Tweet outside valid time range, skipping") logging.debug("Tweet outside valid time range, skipping")
continue continue
@ -317,6 +320,7 @@ def main(argv):
tweet_in_db = db.fetchone() tweet_in_db = db.fetchone()
if tweet_in_db is not None: if tweet_in_db is not None:
in_db_cnt += 1
logging.debug("Tweet %s already in database", tweet_id) logging.debug("Tweet %s already in database", tweet_id)
# Skip to next tweet # Skip to next tweet
continue continue
@ -425,6 +429,8 @@ def main(argv):
logging.debug('Tweet %s added to list of toots to upload', tweet_id) logging.debug('Tweet %s added to list of toots to upload', tweet_id)
# TODO Log summary stats: how many not in db, how many in valid timeframe # TODO Log summary stats: how many not in db, how many in valid timeframe
logging.info(str(out_date_cnt) + ' tweets outside of valid time range')
logging.info(str(in_db_cnt) + ' tweets already in database')
# DEBUG: Print extracted tweets # DEBUG: Print extracted tweets
#for t in tweets: #for t in tweets:
@ -467,6 +473,7 @@ def main(argv):
sys.exit(1) sys.exit(1)
# Upload tweets # Upload tweets
posted_cnt = 0
for tweet in reversed(tweets): for tweet in reversed(tweets):
logging.debug('Uploading Tweet %s', tweet["tweet_id"]) logging.debug('Uploading Tweet %s', tweet["tweet_id"])
@ -515,9 +522,10 @@ def main(argv):
except MastodonError as me: except MastodonError as me:
logging.error('posting ' + tweet['tweet_text'] + ' to ' + mast_instance + ' Failed') logging.error('posting ' + tweet['tweet_text'] + ' to ' + mast_instance + ' Failed')
logging.error(me) logging.error(me)
sys.exit(1)
logging.debug('Tweet %s posted on %s', tweet_id, mast_account) else:
posted_cnt += 1
logging.debug('Tweet %s posted on %s', tweet['tweet_id'], mast_account)
# Insert toot id into database # Insert toot id into database
if 'id' in toot: if 'id' in toot:
@ -525,6 +533,8 @@ def main(argv):
(twit_account, mast_instance, mast_account, tweet['tweet_id'], toot['id'])) (twit_account, mast_instance, mast_account, tweet['tweet_id'], toot['id']))
sql.commit() sql.commit()
logging.info(str(posted_cnt) + ' Tweets posted to Mastodon')
# Cleanup downloaded video files # Cleanup downloaded video files
try: try:
shutil.rmtree('./output/' + twit_account) shutil.rmtree('./output/' + twit_account)