diff --git a/twoot.py b/twoot.py index d92b910..706d100 100755 --- a/twoot.py +++ b/twoot.py @@ -396,8 +396,8 @@ def main(argv): parser.add_argument('-d', metavar='', action='store', type=float) parser.add_argument('-c', metavar='', action='store', type=int) - # Parse command line - args = vars(parser.parse_args()) + # Create global struct containing configuration + global TOML # We build the configuration by layering for each parameter: # 1. A default value @@ -416,8 +416,11 @@ def main(argv): 'toot_cap': int(0), } - # Default empty toml - toml = {'config': {}, 'options': options} + # Default toml + TOML = {'config': {}, 'options': options} + + # Parse command line + args = vars(parser.parse_args()) # Load config file if it was provided toml_file = args['f'] @@ -425,7 +428,7 @@ def main(argv): import tomli try: with open(toml_file, 'rb') as config_file: - toml = tomli.load(config_file) + TOML = tomli.load(config_file) except FileNotFoundError: print('config file not found') exit(-1) @@ -433,39 +436,39 @@ def main(argv): print('Malformed config file') exit(-1) - # Override config file parameter values with command-line values if provided + # Override config parameters with command-line values if provided if args['t'] is not None: - toml['config']['twitter_account'] = args['t'] + TOML['config']['twitter_account'] = args['t'] if args['i'] is not None: - toml['config']['mastodon_instance'] = args['i'] + TOML['config']['mastodon_instance'] = args['i'] if args['m'] is not None: - toml['config']['mastodon_user'] = args['m'] + TOML['config']['mastodon_user'] = args['m'] if args['v'] is True: - toml['options']['upload_videos'] = args['v'] + TOML['options']['upload_videos'] = args['v'] if args['r'] is True: - toml['options']['post_reply_to'] = args['r'] + TOML['options']['post_reply_to'] = args['r'] if args['s'] is True: - toml['options']['skip_retweets'] = args['s'] + TOML['options']['skip_retweets'] = args['s'] if args['l'] is True: - toml['options']['remove_link_redirections'] = args['l'] + TOML['options']['remove_link_redirections'] = args['l'] if args['u'] is True: - toml['options']['remove_trackers_from_urls'] = args['u'] + TOML['options']['remove_trackers_from_urls'] = args['u'] if args['a'] is not None: - toml['options']['tweet_max_age'] = float(args['a']) + TOML['options']['tweet_max_age'] = float(args['a']) if args['d'] is not None: - toml['options']['tweet_delay'] = float(args['d']) + TOML['options']['tweet_delay'] = float(args['d']) if args['c'] is not None: - toml['options']['toot_cap'] = int(args['c']) + TOML['options']['toot_cap'] = int(args['c']) mast_password = args['p'] # Verify that we have a minimum config to run - if 'twitter_account' not in toml['config'].keys(): + if 'twitter_account' not in TOML['config'].keys(): print('CRITICAL: Missing Twitter account') exit(-1) - if 'mastodon_instance' not in toml['config'].keys(): + if 'mastodon_instance' not in TOML['config'].keys(): print('CRITICAL: Missing Mastodon instance') exit(-1) - if 'mastodon_user' not in toml['config'].keys(): + if 'mastodon_user' not in TOML['config'].keys(): print('CRITICAL: Missing Mastodon user') exit(-1) if mast_password is None: @@ -474,13 +477,13 @@ def main(argv): # Remove previous log file try: - os.remove(toml['config']['twitter_account'] + '.log') + os.remove(TOML['config']['twitter_account'] + '.log') except FileNotFoundError: pass # Setup logging to file logging.basicConfig( - filename=toml['config']['twitter_account'] + '.log', + filename=TOML['config']['twitter_account'] + '.log', level=LOGGING_LEVEL, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', @@ -488,17 +491,17 @@ def main(argv): logging.info('Running with the following configuration:') logging.info(' Config file : ' + str(toml_file)) - logging.info(' twitter_account : ' + toml['config']['twitter_account']) - logging.info(' mastodon_instance : ' + toml['config']['mastodon_instance']) - logging.info(' mastodon_user : ' + toml['config']['mastodon_user']) - logging.info(' post_reply_to : ' + str(toml['options']['post_reply_to'])) - logging.info(' skip_retweets : ' + str(toml['options']['skip_retweets'])) - logging.info(' remove_link_redirections : ' + str(toml['options']['remove_link_redirections'])) - logging.info(' remove_trackers_from_urls: ' + str(toml['options']['remove_trackers_from_urls'])) - logging.info(' upload_videos : ' + str(toml['options']['upload_videos'])) - logging.info(' tweet_max_age : ' + str(toml['options']['tweet_max_age'])) - logging.info(' tweet_delay : ' + str(toml['options']['tweet_delay'])) - logging.info(' toot_cap : ' + str(toml['options']['toot_cap'])) + logging.info(' twitter_account : ' + TOML['config']['twitter_account']) + logging.info(' mastodon_instance : ' + TOML['config']['mastodon_instance']) + logging.info(' mastodon_user : ' + TOML['config']['mastodon_user']) + logging.info(' post_reply_to : ' + str(TOML['options']['post_reply_to'])) + logging.info(' skip_retweets : ' + str(TOML['options']['skip_retweets'])) + logging.info(' remove_link_redirections : ' + str(TOML['options']['remove_link_redirections'])) + logging.info(' remove_trackers_from_urls: ' + str(TOML['options']['remove_trackers_from_urls'])) + logging.info(' upload_videos : ' + str(TOML['options']['upload_videos'])) + logging.info(' tweet_max_age : ' + str(TOML['options']['tweet_max_age'])) + logging.info(' tweet_delay : ' + str(TOML['options']['tweet_delay'])) + logging.info(' toot_cap : ' + str(TOML['options']['toot_cap'])) # Try to open database. If it does not exist, create it sql = sqlite3.connect('twoot.db') @@ -532,9 +535,9 @@ def main(argv): } ) - url = nitter_url + '/' + toml['config']['twitter_account'] + url = nitter_url + '/' + TOML['config']['twitter_account'] # Use different page if we need to handle replies - if toml['options']['post_reply_to']: + if TOML['options']['post_reply_to']: url += '/with_replies' # Download twitter page of user @@ -567,7 +570,7 @@ def main(argv): ta = soup.find('meta', property='og:title').get('content') ta_match = re.search(r'\(@(.+)\)', ta) if ta_match is not None: - toml['config']['twitter_account'] = ta_match.group(1) + TOML['config']['twitter_account'] = ta_match.group(1) # Extract twitter timeline timeline = soup.find_all('div', class_='timeline-item') @@ -596,13 +599,13 @@ def main(argv): timestamp = datetime.datetime.strptime(time_string, '%b %d, %Y ยท %I:%M %p %Z').timestamp() # Check if time is within acceptable range - if not is_time_valid(timestamp, toml['options']['tweet_max_age'], toml['options']['tweet_delay']): + if not is_time_valid(timestamp, TOML['options']['tweet_max_age'], TOML['options']['tweet_delay']): out_date_cnt += 1 logging.debug("Tweet outside valid time range, skipping") continue # Check if retweets must be skipped - if toml['options']['skip_retweets']: + if TOML['options']['skip_retweets']: # Check if this tweet is a retweet if len(status.select("div.tweet-body > div > div.retweet-header")) != 0: logging.debug("Retweet ignored per command-line configuration") @@ -611,7 +614,7 @@ def main(argv): # Check in database if tweet has already been posted db.execute( "SELECT * FROM toots WHERE twitter_account=? AND mastodon_instance=? AND mastodon_account=? AND tweet_id=?", - (toml['config']['twitter_account'], toml['config']['mastodon_instance'], toml['config']['mastodon_user'], tweet_id)) + (TOML['config']['twitter_account'], TOML['config']['mastodon_instance'], TOML['config']['mastodon_user'], tweet_id)) tweet_in_db = db.fetchone() if tweet_in_db is not None: @@ -651,8 +654,8 @@ def main(argv): # Process text of tweet tweet_text += process_media_body(tt_iter, - toml['options']['remove_link_redirections'], - toml['options']['remove_trackers_from_urls'] + TOML['options']['remove_link_redirections'], + TOML['options']['remove_trackers_from_urls'] ) # Process quote: append link to tweet_text @@ -670,8 +673,8 @@ def main(argv): if attachments_class is not None: pics, vid_in_tweet = process_attachments(nitter_url, attachments_class, - toml['options']['upload_videos'], - toml['config']['twitter_account'], + TOML['options']['upload_videos'], + TOML['config']['twitter_account'], status_id, author_account ) photos.extend(pics) @@ -709,7 +712,7 @@ def main(argv): # Check if video was downloaded video_file = None - video_path = Path('./output') / toml['config']['twitter_account'] / status_id + video_path = Path('./output') / TOML['config']['twitter_account'] / status_id if video_path.exists(): # list video files video_file_list = list(video_path.glob('*.mp4')) @@ -742,7 +745,7 @@ def main(argv): # Login to account on maston instance mastodon = None if len(tweets) != 0: - mastodon = login(toml['config']['mastodon_instance'], toml['config']['mastodon_user'], mast_password) + mastodon = login(TOML['config']['mastodon_instance'], TOML['config']['mastodon_user'], mast_password) # ********************************************************** # Iterate tweets in list. @@ -752,8 +755,8 @@ def main(argv): posted_cnt = 0 for tweet in reversed(tweets): # Check if we have reached the cap on the number of toots to post - if toml['options']['toot_cap'] != 0 and posted_cnt >= toml['options']['toot_cap']: - logging.info('%d toots not posted due to configured cap', len(tweets) - toml['options']['toot_cap']) + if TOML['options']['toot_cap'] != 0 and posted_cnt >= TOML['options']['toot_cap']: + logging.info('%d toots not posted due to configured cap', len(tweets) - TOML['options']['toot_cap']) break logging.debug('Uploading Tweet %s', tweet["tweet_id"]) @@ -796,8 +799,8 @@ def main(argv): toot = {} try: mastodon = Mastodon( - access_token=toml['config']['mastodon_user'] + '.secret', - api_base_url='https://' + toml['config']['mastodon_instance'] + access_token=TOML['config']['mastodon_user'] + '.secret', + api_base_url='https://' + TOML['config']['mastodon_instance'] ) if len(media_ids) == 0: @@ -806,31 +809,31 @@ def main(argv): toot = mastodon.status_post(tweet['tweet_text'], media_ids=media_ids, visibility='public') except MastodonError as me: - logging.error('posting ' + tweet['tweet_text'] + ' to ' + toml['config']['mastodon_instance'] + ' Failed') + logging.error('posting ' + tweet['tweet_text'] + ' to ' + TOML['config']['mastodon_instance'] + ' Failed') logging.error(me) else: posted_cnt += 1 - logging.debug('Tweet %s posted on %s', tweet['tweet_id'], toml['config']['mastodon_user']) + logging.debug('Tweet %s posted on %s', tweet['tweet_id'], TOML['config']['mastodon_user']) # Insert toot id into database if 'id' in toot: db.execute("INSERT INTO toots VALUES ( ? , ? , ? , ? , ? )", - (toml['config']['twitter_account'], toml['config']['mastodon_instance'], toml['config']['mastodon_user'], tweet['tweet_id'], toot['id'])) + (TOML['config']['twitter_account'], TOML['config']['mastodon_instance'], TOML['config']['mastodon_user'], tweet['tweet_id'], toot['id'])) sql.commit() logging.info(str(posted_cnt) + ' tweets posted to Mastodon') # Cleanup downloaded video files try: - shutil.rmtree('./output/' + toml['config']['twitter_account']) + shutil.rmtree('./output/' + TOML['config']['twitter_account']) except FileNotFoundError: # The directory does not exist pass # Evaluate excess records in database excess_count = 0 - db.execute('SELECT count(*) FROM toots WHERE twitter_account=?', (toml['config']['twitter_account'],)) + db.execute('SELECT count(*) FROM toots WHERE twitter_account=?', (TOML['config']['twitter_account'],)) db_count = db.fetchone() if db_count is not None: excess_count = db_count[0] - MAX_REC_COUNT @@ -846,7 +849,7 @@ def main(argv): LIMIT ? ) DELETE from toots - WHERE tweet_id IN excess''', (toml['config']['twitter_account'], excess_count)) + WHERE tweet_id IN excess''', (TOML['config']['twitter_account'], excess_count)) sql.commit() logging.info('Deleted ' + str(excess_count) + ' old records from database.')