mirror of
https://gitlab.com/chaica/feed2toot.git
synced 2024-11-23 20:11:09 +00:00
push code from main to new modules
This commit is contained in:
Carl Chenet
parent
ec664e24d3
commit
3b5e66af40
66
feed2toot/hashtags.py
Normal file
66
feed2toot/hashtags.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
|
||||
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
'''Manage a lock file'''
|
||||
|
||||
# standard libraires imports
|
||||
import codecs
|
||||
|
||||
def extract_hashtags_from_list(options):
|
||||
'''extract hashtags from the the list'''
|
||||
if 'hashtaglist' in options and options['hashtaglist']:
|
||||
severalwordshashtags = codecs.open(options['hashtaglist'],
|
||||
encoding='utf-8').readlines()
|
||||
severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags]
|
||||
else:
|
||||
severalwordshashtags = []
|
||||
return severalwordshashtags
|
||||
|
||||
def build_hashtags(entry, rss, options, severalwordshashtags):
|
||||
'''build hashtags'''
|
||||
severalwordsinhashtag = False
|
||||
# has the the rss feed hashtag
|
||||
if 'tags' in entry and options['addtags']:
|
||||
hastags = True
|
||||
else:
|
||||
hastags = False
|
||||
if hastags:
|
||||
rss['hashtags'] = []
|
||||
for i, _ in enumerate(entry['tags']):
|
||||
if 'hashtaglist' in options:
|
||||
prehashtags = entry['tags'][i]['term']
|
||||
tmphashtags = entry['tags'][i]['term']
|
||||
for element in severalwordshashtags:
|
||||
if element in prehashtags:
|
||||
severalwordsinhashtag = True
|
||||
tmphashtags = prehashtags.replace(element,
|
||||
''.join(element.split()))
|
||||
# replace characters stopping a word from being a hashtag
|
||||
if severalwordsinhashtag:
|
||||
# remove ' from hashtag
|
||||
tmphashtags = tmphashtags.replace("'", "")
|
||||
# remove - from hashtag
|
||||
tmphashtags = tmphashtags.replace("-", "")
|
||||
# remove . from hashtag
|
||||
tmphashtags = tmphashtags.replace(".", "")
|
||||
# remove space from hashtag
|
||||
finalhashtags = tmphashtags.replace(" ", "")
|
||||
rss['hashtags'].append('#{}'.format(finalhashtags))
|
||||
else:
|
||||
nospace = ''.join(entry['tags'][i]['term'])
|
||||
# remove space from hashtag
|
||||
nospace = nospace.replace(" ", "")
|
||||
rss['hashtags'].append('#{}'.format(nospace))
|
||||
return rss
|
||||
|
|
@ -24,16 +24,25 @@ import logging.handlers
|
|||
import sys
|
||||
import re
|
||||
|
||||
# external liraries imports
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# app libraries imports
|
||||
from feed2toot.addtags import AddTags
|
||||
from feed2toot.cliparse import CliParse
|
||||
from feed2toot.confparse import ConfParse
|
||||
from feed2toot.filterentry import FilterEntry
|
||||
from feed2toot.removeduplicates import RemoveDuplicates
|
||||
from feed2toot.tootpost import TootPost
|
||||
from feed2toot.feedcache import FeedCache
|
||||
from feed2toot.filterentry import FilterEntry
|
||||
from feed2toot.hashtags import build_hashtags
|
||||
from feed2toot.hashtags import extract_hashtags_from_list
|
||||
from feed2toot.lock import LockFile
|
||||
from bs4 import BeautifulSoup
|
||||
from feed2toot.message import build_message
|
||||
from feed2toot.message import send_message_dry_run
|
||||
from feed2toot.message import send_message
|
||||
from feed2toot.removeduplicates import RemoveDuplicates
|
||||
from feed2toot.rss import populate_rss
|
||||
from feed2toot.sortentries import sort_entries
|
||||
from feed2toot.tootpost import TootPost
|
||||
|
||||
class Main:
|
||||
'''Main class of Feed2toot'''
|
||||
|
|
@ -59,7 +68,7 @@ class Main:
|
|||
logging.debug('configured stdout level %s' % sh.level)
|
||||
|
||||
def main(self):
|
||||
"""The main function."""
|
||||
'''The main function'''
|
||||
clip = CliParse()
|
||||
clioptions = clip.options
|
||||
self.setup_logging(clioptions)
|
||||
|
|
@ -76,12 +85,7 @@ class Main:
|
|||
lockfile = LockFile(options['lockfile'], options['locktimeout'])
|
||||
# create link to the persistent list
|
||||
cache = FeedCache(options)
|
||||
if 'hashtaglist' in options and options['hashtaglist']:
|
||||
severalwordshashtags = codecs.open(options['hashtaglist'],
|
||||
encoding='utf-8').readlines()
|
||||
severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags]
|
||||
else:
|
||||
severalwordshashtags = []
|
||||
severalwordshashtags = extract_hashtags_from_list(options)
|
||||
# reverse feed entries because most recent one should be sent as the last one in Mastodon
|
||||
for feed in feeds:
|
||||
# store the patterns by rss
|
||||
|
|
@ -96,73 +100,12 @@ class Main:
|
|||
sys.exit(0)
|
||||
else:
|
||||
sys.exit('Could not parse the section of the rss feed')
|
||||
totweet = []
|
||||
# cache the ids of last rss feeds
|
||||
if not clioptions.all:
|
||||
for i in entries:
|
||||
if 'id' in i:
|
||||
if i['id'] not in cache.getdeque():
|
||||
totweet.append(i)
|
||||
elif 'guid' in i:
|
||||
if i['guid'] not in cache.getdeque():
|
||||
totweet.append(i)
|
||||
else:
|
||||
# if id or guid not in the entry, use link
|
||||
if i['link'] not in cache.getdeque():
|
||||
totweet.append(i)
|
||||
else:
|
||||
totweet = entries
|
||||
|
||||
# sort entries and check if they were not previously sent
|
||||
totweet = sort_entries(clioptions.all, cache, entries)
|
||||
for entry in totweet:
|
||||
if 'id' in entry:
|
||||
logging.debug('found feed entry {entryid}'.format(entryid=entry['id']))
|
||||
rss = {
|
||||
'id': entry['id'],
|
||||
}
|
||||
elif 'guid' in entry:
|
||||
logging.debug('found feed entry {entryid}'.format(entryid=entry['guid']))
|
||||
rss = {
|
||||
'id': entry['guid'],
|
||||
}
|
||||
else:
|
||||
logging.debug('found feed entry {entryid}'.format(entryid=entry['link']))
|
||||
rss = {
|
||||
'id': entry['link'],
|
||||
}
|
||||
|
||||
severalwordsinhashtag = False
|
||||
# lets see if the rss feed has hashtag
|
||||
if 'tags' in entry and options['addtags']:
|
||||
hastags = True
|
||||
else:
|
||||
hastags = False
|
||||
if hastags:
|
||||
rss['hashtags'] = []
|
||||
for i, _ in enumerate(entry['tags']):
|
||||
if 'hashtaglist' in options:
|
||||
prehashtags = entry['tags'][i]['term']
|
||||
tmphashtags = entry['tags'][i]['term']
|
||||
for element in severalwordshashtags:
|
||||
if element in prehashtags:
|
||||
severalwordsinhashtag = True
|
||||
tmphashtags = prehashtags.replace(element,
|
||||
''.join(element.split()))
|
||||
# replace characters stopping a word from being a hashtag
|
||||
if severalwordsinhashtag:
|
||||
# remove ' from hashtag
|
||||
tmphashtags = tmphashtags.replace("'", "")
|
||||
# remove - from hashtag
|
||||
tmphashtags = tmphashtags.replace("-", "")
|
||||
# remove . from hashtag
|
||||
tmphashtags = tmphashtags.replace(".", "")
|
||||
# remove space from hashtag
|
||||
finalhashtags = tmphashtags.replace(" ", "")
|
||||
rss['hashtags'].append('#{}'.format(finalhashtags))
|
||||
else:
|
||||
nospace = ''.join(entry['tags'][i]['term'])
|
||||
# remove space from hashtag
|
||||
nospace = nospace.replace(" ", "")
|
||||
rss['hashtags'].append('#{}'.format(nospace))
|
||||
# populate rss with new entry to send
|
||||
rss = populate_rss(entry)
|
||||
rss = build_hashtags(entry, rss, options, severalwordshashtags)
|
||||
# parse tweetfomat to elements
|
||||
elements = re.findall(r"\{(.*?)\}",tweetformat)
|
||||
# strip : from elements to allow string formating, eg. {title:.20}
|
||||
|
|
@ -172,56 +115,14 @@ class Main:
|
|||
fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject'], feed['feedname'])
|
||||
entrytosend = fe.finalentry
|
||||
if entrytosend:
|
||||
tweetwithnotag = tweetformat.format(**entrytosend)
|
||||
# remove duplicates from the final tweet
|
||||
dedup = RemoveDuplicates(tweetwithnotag)
|
||||
# only append hashtags if they exist
|
||||
# remove last tags if tweet too long
|
||||
if 'hashtags' in rss:
|
||||
addtag = AddTags(dedup.finaltweet, rss['hashtags'])
|
||||
finaltweet = addtag.finaltweet
|
||||
else:
|
||||
finaltweet = dedup.finaltweet
|
||||
|
||||
# strip html tags
|
||||
finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text()
|
||||
|
||||
finaltweet = build_message(entrytosend, tweetformat, rss)
|
||||
if clioptions.dryrun:
|
||||
if entrytosend:
|
||||
logging.warning('Would toot with visibility "{visibility}": {toot}'.format(
|
||||
toot=finaltweet,
|
||||
visibility=config.get(
|
||||
'mastodon', 'toot_visibility',
|
||||
fallback='public')))
|
||||
send_message_dry_run(config, entrytosend, finaltweet)
|
||||
else:
|
||||
logging.debug('This rss entry did not meet pattern criteria. Should have not been sent')
|
||||
else:
|
||||
storeit = True
|
||||
if entrytosend and not clioptions.populate:
|
||||
logging.debug('Tooting with visibility "{visibility}": {toot}'.format(
|
||||
toot=finaltweet,
|
||||
visibility=config.get(
|
||||
'mastodon', 'toot_visibility',
|
||||
fallback='public')))
|
||||
twp = TootPost(config, options, finaltweet)
|
||||
storeit = twp.storeit()
|
||||
else:
|
||||
logging.debug('populating RSS entry {}'.format(rss['id']))
|
||||
# in both cas we store the id of the sent tweet
|
||||
if storeit:
|
||||
cache.append(rss['id'])
|
||||
send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss)
|
||||
# plugins
|
||||
if plugins and entrytosend:
|
||||
for plugin in plugins:
|
||||
capitalizedplugin = plugin.title()
|
||||
pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin)
|
||||
pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower())
|
||||
try:
|
||||
pluginmodule = importlib.import_module(pluginmodulename)
|
||||
pluginclass = getattr(pluginmodule, pluginclassname)
|
||||
pluginclass(plugins[plugin], finaltweet)
|
||||
except ImportError as err:
|
||||
print(err)
|
||||
activate_plugins(plugins)
|
||||
# do not forget to close cache (shelf object)
|
||||
cache.close()
|
||||
# release the lock file
|
||||
|
|
|
|||
72
feed2toot/message.py
Normal file
72
feed2toot/message.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
|
||||
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
'''Build the message'''
|
||||
|
||||
# standard libraires imports
|
||||
import logging
|
||||
|
||||
# external liraries imports
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# app libraries imports
|
||||
from feed2toot.addtags import AddTags
|
||||
from feed2toot.removeduplicates import RemoveDuplicates
|
||||
from feed2toot.tootpost import TootPost
|
||||
|
||||
def build_message(entrytosend, tweetformat, rss):
|
||||
'''populate the rss dict with the new entry'''
|
||||
tweetwithnotag = tweetformat.format(**entrytosend)
|
||||
# remove duplicates from the final tweet
|
||||
dedup = RemoveDuplicates(tweetwithnotag)
|
||||
# only append hashtags if they exist
|
||||
# remove last tags if tweet too long
|
||||
if 'hashtags' in rss:
|
||||
addtag = AddTags(dedup.finaltweet, rss['hashtags'])
|
||||
finaltweet = addtag.finaltweet
|
||||
else:
|
||||
finaltweet = dedup.finaltweet
|
||||
|
||||
# strip html tags
|
||||
finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text()
|
||||
return finaltweet
|
||||
|
||||
def send_message_dry_run(config, entrytosend, finaltweet):
|
||||
'''simulate sending message using dry run mode'''
|
||||
if entrytosend:
|
||||
logging.warning('Would toot with visibility "{visibility}": {toot}'.format(
|
||||
toot=finaltweet,
|
||||
visibility=config.get(
|
||||
'mastodon', 'toot_visibility',
|
||||
fallback='public')))
|
||||
else:
|
||||
logging.debug('This rss entry did not meet pattern criteria. Should have not been sent')
|
||||
|
||||
def send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss):
|
||||
'''send message'''
|
||||
storeit = True
|
||||
if entrytosend and not clioptions.populate:
|
||||
logging.debug('Tooting with visibility "{visibility}": {toot}'.format(
|
||||
toot=finaltweet,
|
||||
visibility=config.get(
|
||||
'mastodon', 'toot_visibility',
|
||||
fallback='public')))
|
||||
twp = TootPost(config, options, finaltweet)
|
||||
storeit = twp.storeit()
|
||||
else:
|
||||
logging.debug('populating RSS entry {}'.format(rss['id']))
|
||||
# in both cas we store the id of the sent tweet
|
||||
if storeit:
|
||||
cache.append(rss['id'])
|
||||
|
|
@ -13,3 +13,19 @@
|
|||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
# standard libraires imports
|
||||
import importlib
|
||||
|
||||
def activate_plugins(entrytosend):
|
||||
'''activate plugins'''
|
||||
for plugin in plugins:
|
||||
capitalizedplugin = plugin.title()
|
||||
pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin)
|
||||
pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower())
|
||||
try:
|
||||
pluginmodule = importlib.import_module(pluginmodulename)
|
||||
pluginclass = getattr(pluginmodule, pluginclassname)
|
||||
pluginclass(plugins[plugin], finaltweet)
|
||||
except ImportError as err:
|
||||
print(err)
|
||||
|
|
|
|||
42
feed2toot/rss.py
Normal file
42
feed2toot/rss.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
|
||||
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
'''Manage a lock file'''
|
||||
|
||||
# standard libraires imports
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
def populate_rss(entry):
|
||||
'''populate the rss dict with the new entry'''
|
||||
if 'id' in entry:
|
||||
logging.debug('found feed entry {entryid}'.format(entryid=entry['id']))
|
||||
rss = {
|
||||
'id': entry['id'],
|
||||
}
|
||||
elif 'guid' in entry:
|
||||
logging.debug('found feed entry {entryid}'.format(entryid=entry['guid']))
|
||||
rss = {
|
||||
'id': entry['guid'],
|
||||
}
|
||||
else:
|
||||
logging.debug('found feed entry {entryid}'.format(entryid=entry['link']))
|
||||
rss = {
|
||||
'id': entry['link'],
|
||||
}
|
||||
return rss
|
||||
42
feed2toot/sortentries.py
Normal file
42
feed2toot/sortentries.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
|
||||
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
'''Manage a lock file'''
|
||||
|
||||
# standard libraires imports
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
def sort_entries(is_all, cache, entries):
|
||||
'''sort entries before sending'''
|
||||
totweet = []
|
||||
if not is_all:
|
||||
for i in entries:
|
||||
if 'id' in i:
|
||||
if i['id'] not in cache.getdeque():
|
||||
totweet.append(i)
|
||||
elif 'guid' in i:
|
||||
if i['guid'] not in cache.getdeque():
|
||||
totweet.append(i)
|
||||
else:
|
||||
# if id or guid not in the entry, use link
|
||||
if i['link'] not in cache.getdeque():
|
||||
totweet.append(i)
|
||||
else:
|
||||
totweet = entries
|
||||
return totweet
|
||||
Loading…
Reference in New Issue
Block a user