initialized loaded_toml to None

README.md

@@ -3,9 +3,7 @@
 Twoot is a python script that mirrors tweets from a twitter account to a Mastodon account.
 It is simple to set-up on a local machine, configurable and feature-rich.
 
-**UPDATE XX NOV 2022** VERSION 2.5 Added command-line option (`-l`) to remove redirection
-from links included in tweets. Obfuscated links are replaced by the URL that the resource
-is directly downloaded from.
+**UPDATE XX DEC 2022** VERSION 3.0 config file / tomli dependency
 
 > Previous updates can be found in CHANGELOG.
 

default.toml

@@ -41,21 +41,20 @@ tweet_delay = 0
 # Default is 0 (which means unlimited)
 toot_cap = 0
 
-[options.substitution]
 # Replace twitter links by random alternative out of this list
 # List of nitter instances
-# e.g. twitter = ["nitter.net", ]
+# e.g. subst_twitter = ["nitter.net", ]
 # Default is []
-twitter = []
+subst_twitter = []
 
 # Replace youtube.com link by random alternative out of this list
 # List of Invidious or Piped instances
-# e.g. youtube = ["piped.kavin.rocks", "invidious.flokinet.to", ]
+# e.g. subst_youtube = ["piped.kavin.rocks", "invidious.flokinet.to", ]
 # Default is []
-youtube = []
+subst_youtube = []
 
 # Replace reddit.com link by random alternative out of this list
 # List of Teddit instances
-# e.g. reddit = ["teddit.net", ]
+# e.g. subst_reddit = ["teddit.net", ]
 # Default is []
-reddit = []
+subst_reddit = []

twoot.py

@@ -50,7 +50,7 @@ NITTER_URLS = [
     'https://nitter.pussthecat.org',
     'https://nitter.fdn.fr',
     'https://nitter.eu',
-    'https://twitter.beparanoid.de/',
+    'https://twitter.beparanoid.de',
     'https://n.l5.ca',
     'https://nitter.bus-hit.me',
 ]
@@ -67,6 +67,103 @@ USER_AGENTS = [
 ]
 
 
+def build_config(args):
+    """
+    Receives the arguments passed on the command line
+    populates the TOML global dict with default values for all 'options' keys
+    if a config file is provided, load the keys from the config file
+    if no config file is provided, use command-line args
+    verify that a valid config is available (all keys in 'config' present)
+    :param args: list of command line arguments
+    """
+    # Create global struct containing configuration
+    global TOML
+
+    # Default options
+    options = {
+        'upload_videos': False,
+        'post_reply_to': False,
+        'skip_retweets': False,
+        'remove_link_redirections': False,
+        'remove_trackers_from_urls': False,
+        'tweet_max_age': float(1),
+        'tweet_delay': float(0),
+        'toot_cap': int(0),
+        'subst_twitter': [],
+        'subst_youtube': [],
+        'subst_reddit': [],
+    }
+
+    # Create default config object
+    TOML = {'config': {},'options': options}
+
+    # Load config file if it was provided
+    toml_file = args['f']
+    if toml_file is not None:
+        try: # Included starting with python 3.11
+            import tomllib
+        except ModuleNotFoundError:
+            # for python < 3.11, tomli module must be installed
+            import tomli as tomllib
+        
+        loaded_toml = None
+        # Load toml file
+        try:
+            with open(toml_file, 'rb') as config_file:
+                loaded_toml = tomllib.load(config_file)
+        except FileNotFoundError:
+            print('config file not found')
+            exit(-1)
+        except tomllib.TOMLDecodeError:
+            print('Malformed config file')
+            exit(-1)
+        
+        TOML['config'] = loaded_toml['config']
+        for k in TOML['options'].keys():
+            try:  # Go through all valid keys
+                TOML['options'][k] = loaded_toml['options'][k]
+            except KeyError:  # Key was not found in file
+                pass
+    else:
+        # Override config parameters with command-line values if provided
+        if args['t'] is not None:
+            TOML['config']['twitter_account'] = args['t']
+        if args['i'] is not None:
+            TOML['config']['mastodon_instance'] = args['i']
+        if args['m'] is not None:
+            TOML['config']['mastodon_user'] = args['m']
+        if args['v'] is True:
+            TOML['options']['upload_videos'] = args['v']
+        if args['r'] is True:
+            TOML['options']['post_reply_to'] = args['r']
+        if args['s'] is True:
+            TOML['options']['skip_retweets'] = args['s']
+        if args['l'] is True:
+            TOML['options']['remove_link_redirections'] = args['l']
+        if args['u'] is True:
+            TOML['options']['remove_trackers_from_urls'] = args['u']
+        if args['a'] is not None:
+            TOML['options']['tweet_max_age'] = float(args['a'])
+        if args['d'] is not None:
+            TOML['options']['tweet_delay'] = float(args['d'])
+        if args['c'] is not None:
+            TOML['options']['toot_cap'] = int(args['c'])
+
+    # Verify that we have a minimum config to run
+    if 'twitter_account' not in TOML['config'].keys() or TOML['config']['twitter_account'] == "":
+        print('CRITICAL: Missing Twitter account')
+        exit(-1)
+    if 'mastodon_instance' not in TOML['config'].keys() or TOML['config']['mastodon_instance'] == "":
+        print('CRITICAL: Missing Mastodon instance')
+        exit(-1)
+    if 'mastodon_user' not in TOML['config'].keys() or TOML['config']['mastodon_user'] == "":
+        print('CRITICAL: Missing Mastodon user')
+        exit(-1)
+    if args['p'] is None:
+        print('CRITICAL: Missing Mastodon user password')
+        exit(-1)
+
+
 def deredir_url(url):
     """
     Given a URL, return the URL that the page really downloads from
@@ -160,19 +257,19 @@ def substitute_source(orig_url):
     logging.debug("Checking domain %s for substitution ", domain)
     
     # Handle twitter
-    twitter_subst = TOML["options"]["substitution"]["twitter"]
+    twitter_subst = TOML["options"]["subst_twitter"]
     if domain.find('twitter.com') >=0  and twitter_subst != []:
         domain = twitter_subst[random.randint(0, len(twitter_subst) - 1)]
         logging.debug("Replaced twitter.com by " + domain)
 
     # Handle youtube
-    youtube_subst = TOML["options"]["substitution"]["youtube"]
+    youtube_subst = TOML["options"]["subst_youtube"]
     if domain.find('youtube.com') >=0  and youtube_subst != []:
         domain = youtube_subst[random.randint(0, len(youtube_subst) - 1)]
         logging.debug("Replaced youtube.com by " + domain)
 
     # Handle reddit
-    reddit_subst = TOML["options"]["substitution"]["reddit"]
+    reddit_subst = TOML["options"]["subst_reddit"]
     if domain.find('reddit.com') >=0  and reddit_subst != []:
         domain = reddit_subst[random.randint(0, len(reddit_subst) - 1)]
         logging.debug("Replaced reddit.com by " + domain)
@@ -436,88 +533,13 @@ def main(argv):
     parser.add_argument('-d', metavar='<min delay (in mins)>', action='store', type=float)
     parser.add_argument('-c', metavar='<max # of toots to post>', action='store', type=int)
 
-    # Create global struct containing configuration
-    global TOML
-
-    # Default options
-    substitution = {
-        'twitter': [],
-        'youtube': [],
-        'reddit': [],
-    }
-
-    options = {
-        'upload_videos': False,
-        'post_reply_to': False,
-        'skip_retweets': False,
-        'remove_link_redirections': False,
-        'remove_trackers_from_urls': False,
-        'tweet_max_age': float(1),
-        'tweet_delay': float(0),
-        'toot_cap': int(0),
-        'substitution': substitution,
-    }
-
     # Parse command line
     args = vars(parser.parse_args())
 
-    # Load config file if it was provided
-    toml_file = args['f']
-    if toml_file is not None:
-        import tomli
-        try:
-            with open(toml_file, 'rb') as config_file:
-                TOML = tomli.load(config_file)
-        except FileNotFoundError:
-            print('config file not found')
-            exit(-1)
-        except tomli.TOMLDecodeError:
-            print('Malformed config file')
-            exit(-1)
-    else:
-        # Default toml
-        TOML = {'config': {}, 'options': options}
+    build_config(args)
 
-
-    # Override config parameters with command-line values if provided
-    if args['t'] is not None:
-        TOML['config']['twitter_account'] = args['t']
-    if args['i'] is not None:
-        TOML['config']['mastodon_instance'] = args['i']
-    if args['m'] is not None:
-        TOML['config']['mastodon_user'] = args['m']
-    if args['v'] is True:
-        TOML['options']['upload_videos'] = args['v']
-    if args['r'] is True:
-        TOML['options']['post_reply_to'] = args['r']
-    if args['s'] is True:
-        TOML['options']['skip_retweets'] = args['s']
-    if args['l'] is True:
-        TOML['options']['remove_link_redirections'] = args['l']
-    if args['u'] is True:
-        TOML['options']['remove_trackers_from_urls'] = args['u']
-    if args['a'] is not None:
-        TOML['options']['tweet_max_age'] = float(args['a'])
-    if args['d'] is not None:
-        TOML['options']['tweet_delay'] = float(args['d'])
-    if args['c'] is not None:
-        TOML['options']['toot_cap'] = int(args['c'])
     mast_password = args['p']
 
-    # Verify that we have a minimum config to run
-    if 'twitter_account' not in TOML['config'].keys() or TOML['config']['twitter_account'] == "":
-        print('CRITICAL: Missing Twitter account')
-        exit(-1)
-    if 'mastodon_instance' not in TOML['config'].keys() or TOML['config']['mastodon_instance'] == "":
-        print('CRITICAL: Missing Mastodon instance')
-        exit(-1)
-    if 'mastodon_user' not in TOML['config'].keys() or TOML['config']['mastodon_user'] == "":
-        print('CRITICAL: Missing Mastodon user')
-        exit(-1)
-    if mast_password is None:
-        print('CRITICAL: Missing Mastodon user password')
-        exit(-1)
-
     # Remove previous log file
     # try:
     #    os.remove(TOML['config']['twitter_account'] + '.log')
@@ -533,7 +555,7 @@ def main(argv):
     )
 
     logging.info('Running with the following configuration:')
-    logging.info('  Config file              : ' + str(toml_file))
+    logging.info('  Config file              : ' + str(args['f']))
     logging.info('  twitter_account          : ' + TOML['config']['twitter_account'])
     logging.info('  mastodon_instance        : ' + TOML['config']['mastodon_instance'])
     logging.info('  mastodon_user            : ' + TOML['config']['mastodon_user'])
@@ -545,9 +567,9 @@ def main(argv):
     logging.info('  tweet_max_age            : ' + str(TOML['options']['tweet_max_age']))
     logging.info('  tweet_delay              : ' + str(TOML['options']['tweet_delay']))
     logging.info('  toot_cap                 : ' + str(TOML['options']['toot_cap']))
-    logging.info('  twitter substitution     : ' + str(TOML['options']['substitution']['twitter']))
-    logging.info('  youtube substitution     : ' + str(TOML['options']['substitution']['youtube']))
-    logging.info('  reddit substitution      : ' + str(TOML['options']['substitution']['reddit']))
+    logging.info('  twitter substitution     : ' + str(TOML['options']['subst_twitter']))
+    logging.info('  youtube substitution     : ' + str(TOML['options']['subst_youtube']))
+    logging.info('  reddit substitution      : ' + str(TOML['options']['subst_reddit']))
 
     # Try to open database. If it does not exist, create it
     sql = sqlite3.connect('twoot.db')