mathdatech/twoot
1
0
Fork 0
mirror of https://gitlab.com/jeancf/twoot.git synced 2025-02-25 09:28:43 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

Extract addresses of avatar and banner images

Browse Source
This commit is contained in:
jeancf 2023-06-13 16:31:41 +02:00
parent 54c59fa676
commit b8bd0a12f5
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
twoot.py
View File

@ -29,7 +29,7 @@ import sqlite3
import sys import sys
import time import time
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse, urljoin from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse, urljoin, unquote
import requests import requests
from bs4 import BeautifulSoup, element from bs4 import BeautifulSoup, element
@ -699,6 +699,8 @@ def main(argv):
mastodon_account TEXT, tweet_id TEXT, toot_id TEXT)''') mastodon_account TEXT, tweet_id TEXT, toot_id TEXT)''')
db.execute('''CREATE INDEX IF NOT EXISTS main_index ON toots (twitter_account, db.execute('''CREATE INDEX IF NOT EXISTS main_index ON toots (twitter_account,
mastodon_instance, mastodon_account, tweet_id)''') mastodon_instance, mastodon_account, tweet_id)''')
db.execute('''CREATE TABLE IF NOT EXISTS profiles (mastodon_account TEXT, avatar text, banner, text)''')
db.execute('''CREATE INDEX IF NOT EXIsTS profile_index ON profiles (mastodon_account)''')
# Select random nitter instance to fetch updates from # Select random nitter instance to fetch updates from
nitter_url = NITTER_URLS[random.randint(0, len(NITTER_URLS) - 1)] nitter_url = NITTER_URLS[random.randint(0, len(NITTER_URLS) - 1)]
@ -748,13 +750,23 @@ def main(argv):
logging.debug('Nitter page downloaded successfully from ' + url) logging.debug('Nitter page downloaded successfully from ' + url)
# DEBUG: Save page to file # DEBUG: Save page to file
# of = open(toml['config']['twitter_account'] + '.html', 'w') of = open(TOML['config']['twitter_account'] + '.html', 'w')
# of.write(twit_account_page.text) of.write(twit_account_page.text)
# of.close() of.close()
# Make soup # Make soup
soup = BeautifulSoup(twit_account_page.text, 'html.parser') soup = BeautifulSoup(twit_account_page.text, 'html.parser')
# Extract avatar picture address
avatar = 'https://' + unquote(soup.find('div', class_='profile-card-info').findChild('a').findChild('img').get('src').removeprefix('/pic/'))
# Extract banner picture address
banner = unquote(soup.find('div', class_='profile-banner').findChild('a').findChild('img').get('src').removeprefix('/pic/'))
print(avatar)
print(banner)
exit(0)
# Extract twitter timeline # Extract twitter timeline
timeline = soup.find_all('div', class_='timeline-item') timeline = soup.find_all('div', class_='timeline-item')