2021-11-17 09:10:33 +00:00
|
|
|
import tempfile
|
2021-11-16 16:44:33 +00:00
|
|
|
|
2021-11-17 09:10:33 +00:00
|
|
|
from populate.database import SessionLocal
|
|
|
|
from populate.logger import log
|
|
|
|
from populate.models import Repository
|
|
|
|
from populate.query import query_github
|
2021-11-16 16:44:33 +00:00
|
|
|
|
|
|
|
if __name__ == "__main__":
|
2021-11-17 09:10:33 +00:00
|
|
|
log.debug("Start populate script.")
|
|
|
|
|
|
|
|
query = '"from fastapi import" language:python in:file size:{from_}..{to}'
|
2021-11-16 16:44:33 +00:00
|
|
|
|
|
|
|
with SessionLocal() as session:
|
2021-11-17 09:10:33 +00:00
|
|
|
log.debug("Start querying projects.")
|
|
|
|
query_github(session, query)
|
2021-11-16 16:44:33 +00:00
|
|
|
|
2021-11-17 09:10:33 +00:00
|
|
|
log.debug("Clone repositories.")
|
|
|
|
for repository in session.query(Repository).all():
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
|
|
log.debug(f"Cloning {repository.full_name}.")
|
|
|
|
repo = repository.clone(tmpdir)
|
2021-11-16 16:44:33 +00:00
|
|
|
|
2021-11-17 09:10:33 +00:00
|
|
|
# dependencies = find_dependencies()
|
2021-11-16 16:44:33 +00:00
|
|
|
|
|
|
|
# 3. Find import statements
|
|
|
|
# 4. Store project data into the database
|
|
|
|
# 5. Push the sqlite file to gcs
|