mathdatech/twoot
1
0
Fork 0
mirror of https://gitlab.com/jeancf/twoot.git synced 2025-05-10 05:23:58 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

Added retweet mention + original tweet mention

Browse Source
This commit is contained in:
JC Francois 2019-07-31 23:37:47 +02:00
parent db50e7b3f0
commit e062b55af7
1 changed files with 93 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
twoot.py
View File

@ -29,7 +29,7 @@ from mastodon import Mastodon
#TODO manage command line #TODO manage command line
TWIT_ACCOUNT = 'noirextreme' TWIT_ACCOUNT = 'humansoflatees'
MAST_ACCOUNT = 'jc@noirextreme.com' MAST_ACCOUNT = 'jc@noirextreme.com'
MAST_PASSWORD = 'NfH1D.Sdd63juBmK' MAST_PASSWORD = 'NfH1D.Sdd63juBmK'
MAST_INSTANCE = 'botsin.space' MAST_INSTANCE = 'botsin.space'
@ -42,7 +42,6 @@ USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM
#TODO manage errors #TODO manage errors
def cleanup_tweet_text(tt_iter): def cleanup_tweet_text(tt_iter):
''' '''
Receives an iterator over all the elements contained in the tweet-text container Receives an iterator over all the elements contained in the tweet-text container
@ -122,6 +121,11 @@ headers.update(
# Download twitter page of user # Download twitter page of user
response = requests.get('https://twitter.com/' + TWIT_ACCOUNT, headers=headers) response = requests.get('https://twitter.com/' + TWIT_ACCOUNT, headers=headers)
# DEBUG: Save page to file
of = open('twitter.html', 'w')
of.write(response.text)
of.close()
# Verify that download worked # Verify that download worked
if response.status_code != 200: if response.status_code != 200:
print("Could not download twitter timeline. Aborting.") print("Could not download twitter timeline. Aborting.")
@ -138,7 +142,7 @@ for result in results:
sih = result.find('div', class_='stream-item-header') sih = result.find('div', class_='stream-item-header')
# extract author # extract author
author = sih.find('strong', class_='fullname').string author = sih.find('strong', class_='fullname').get_text()
# Extract author's logo # Extract author's logo
author_logo_url = sih.find('img', class_='avatar')['src'] author_logo_url = sih.find('img', class_='avatar')['src']
@ -150,7 +154,7 @@ for result in results:
tweet_id = sih.find('a', class_='tweet-timestamp')['href'] tweet_id = sih.find('a', class_='tweet-timestamp')['href']
# Extract user name # Extract user name
user_name = re.search('^/(.+?)/', tweet_id).group(1) author_account = re.search('^/(.+?)/', tweet_id).group(1)
# Isolate tweet text container # Isolate tweet text container
ttc = result.find('div', class_='js-tweet-text-container') ttc = result.find('div', class_='js-tweet-text-container')
@ -160,6 +164,13 @@ for result in results:
tweet_text = cleanup_tweet_text(tt_iter) tweet_text = cleanup_tweet_text(tt_iter)
# Check it the tweet is a retweet from somebody else
if author_account.lower() != TWIT_ACCOUNT.lower():
tweet_text = 'RT from ' + author + ' @' + author_account + '\n\n' + tweet_text
# Add footer with link to original tweet
tweet_text += '\n\nOriginal tweet : https://twitter.com' + tweet_id
# Isolate attached media container # Isolate attached media container
amoc = result.find('div', class_='AdaptiveMediaOuterContainer') amoc = result.find('div', class_='AdaptiveMediaOuterContainer')
@ -178,7 +189,7 @@ for result in results:
# Add dictionary with content of tweet to list # Add dictionary with content of tweet to list
tweet = { tweet = {
"author": author, "author": author,
"user_name": user_name, "author_account": author_account,
"author_logo_url": author_logo_url, "author_logo_url": author_logo_url,
"timestamp": timestamp, "timestamp": timestamp,
"tweet_id": tweet_id, "tweet_id": tweet_id,
@ -190,80 +201,81 @@ for result in results:
for t in tweets: for t in tweets:
print(t) print(t)
# **********************************************************
# Iterate tweets. Check if the tweet has already been posted
# on Mastodon. If not, post it and add it to database
# **********************************************************
# Try to open database. If it does not exist, create it # # **********************************************************
sql = sqlite3.connect('twoot.db') # # Iterate tweets. Check if the tweet has already been posted
db = sql.cursor() # # on Mastodon. If not, post it and add it to database
db.execute('''CREATE TABLE IF NOT EXISTS toots (twitter_account TEXT, mastodon_instance TEXT, # # **********************************************************
mastodon_account TEXT, tweet_id TEXT, toot_id TEXT)''') #
# # Try to open database. If it does not exist, create it
# Create Mastodon application if it does not exist yet # sql = sqlite3.connect('twoot.db')
if not os.path.isfile(MAST_INSTANCE + '.secret'): # db = sql.cursor()
if not Mastodon.create_app( # db.execute('''CREATE TABLE IF NOT EXISTS toots (twitter_account TEXT, mastodon_instance TEXT,
'twoot', # mastodon_account TEXT, tweet_id TEXT, toot_id TEXT)''')
api_base_url='https://' + MAST_INSTANCE, #
to_file=MAST_INSTANCE + '.secret' # # Create Mastodon application if it does not exist yet
): # if not os.path.isfile(MAST_INSTANCE + '.secret'):
print('failed to create app on ' + MAST_INSTANCE) # if not Mastodon.create_app(
sys.exit(1) # 'twoot',
# api_base_url='https://' + MAST_INSTANCE,
# Log in to mastodon instance # to_file=MAST_INSTANCE + '.secret'
try: # ):
mastodon = Mastodon( # print('failed to create app on ' + MAST_INSTANCE)
client_id=MAST_INSTANCE + '.secret', # sys.exit(1)
api_base_url='https://' + MAST_INSTANCE #
) # # Log in to mastodon instance
# try:
mastodon.log_in( # mastodon = Mastodon(
username=MAST_ACCOUNT, # client_id=MAST_INSTANCE + '.secret',
password=MAST_PASSWORD, # api_base_url='https://' + MAST_INSTANCE
scopes=['read', 'write'], # )
to_file=MAST_INSTANCE + ".secret" #
) # mastodon.log_in(
except: # username=MAST_ACCOUNT,
print("ERROR: Login Failed") # password=MAST_PASSWORD,
sys.exit(1) # scopes=['read', 'write'],
# to_file=MAST_INSTANCE + ".secret"
# Upload tweets # )
for tweet in tweets: # except:
# Check in database if tweet has already been posted # print("ERROR: Login Failed")
db.execute('''SELECT * FROM toots WHERE twitter_account = ? AND mastodon_instance = ? AND # sys.exit(1)
mastodon_account = ? AND tweet_id = ?''', #
(TWIT_ACCOUNT, MAST_INSTANCE, MAST_ACCOUNT, tweet['tweet_id'])) # # Upload tweets
tweet_in_db = db.fetchone() # for tweet in tweets:
# # Check in database if tweet has already been posted
if tweet_in_db is not None: # db.execute('''SELECT * FROM toots WHERE twitter_account = ? AND mastodon_instance = ? AND
# Skip to next tweet # mastodon_account = ? AND tweet_id = ?''',
continue # (TWIT_ACCOUNT, MAST_INSTANCE, MAST_ACCOUNT, tweet['tweet_id']))
# tweet_in_db = db.fetchone()
# Check that the tweet is not too young (might be deleted) or too old #
age_in_hours = (time.time() - float(tweet['timestamp'])) / 3600.0 # if tweet_in_db is not None:
min_delay_in_hours = float(MIN_DELAY) / 60.0 # # Skip to next tweet
max_age_in_hours = float(MAX_AGE) * 24.0 # continue
#
if age_in_hours < min_delay_in_hours or age_in_hours > max_age_in_hours: # # Check that the tweet is not too young (might be deleted) or too old
# Skip to next tweet # age_in_hours = (time.time() - float(tweet['timestamp'])) / 3600.0
continue # min_delay_in_hours = float(MIN_DELAY) / 60.0
# max_age_in_hours = float(MAX_AGE) * 24.0
# Upload photos #
media_ids = [] # if age_in_hours < min_delay_in_hours or age_in_hours > max_age_in_hours:
for photo in tweet['photos']: # # Skip to next tweet
# Download picture # continue
media = requests.get(photo) #
# # Upload photos
# Upload picture to Mastodon instance # media_ids = []
media_posted = mastodon.media_post(media.content, mime_type=media.headers.get('content-type')) # for photo in tweet['photos']:
media_ids.append(media_posted['id']) # # Download picture
# media = requests.get(photo)
# Post toot #
toot = mastodon.status_post(tweet['tweet_text'], media_ids=media_ids, visibility='public') # # Upload picture to Mastodon instance
# media_posted = mastodon.media_post(media.content, mime_type=media.headers.get('content-type'))
# Insert toot id into database # media_ids.append(media_posted['id'])
if 'id' in toot: #
db.execute("INSERT INTO toots VALUES ( ? , ? , ? , ? , ? )", # # Post toot
(TWIT_ACCOUNT, MAST_INSTANCE, MAST_ACCOUNT, tweet['tweet_id'], toot['id'])) # toot = mastodon.status_post(tweet['tweet_text'], media_ids=media_ids, visibility='public')
sql.commit() #
# # Insert toot id into database
# if 'id' in toot:
# db.execute("INSERT INTO toots VALUES ( ? , ? , ? , ? , ? )",
# (TWIT_ACCOUNT, MAST_INSTANCE, MAST_ACCOUNT, tweet['tweet_id'], toot['id']))
# sql.commit()