This commit is contained in:
Marcelo Trylesinski 2021-05-09 03:05:11 +02:00
parent b0ca7b2c06
commit 445ca2647a
7 changed files with 110 additions and 11 deletions

View File

@ -1,16 +1,16 @@
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import Session, sessionmaker
engine = create_engine("sqlite://database.db")
engine = create_engine("sqlite:///database.db")
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
class SessionManager:
def __enter__(self):
def __enter__(self) -> Session:
self.session = SessionLocal()
return self.session
def __exit__(self, *exc_data):
def __exit__(self, *exc_data) -> None:
self.session.close()

View File

@ -7,4 +7,7 @@ def get_python_files(dir: TemporaryDirectory) -> Generator[TextIO, None, None]:
for dirpath, _, filenames in os.walk(dir.name):
for filename in filenames:
if filename.endswith(".py"):
yield open(os.sep.join([dirpath, filename]), "r")
try:
yield open(os.sep.join([dirpath, filename]), "r")
except FileNotFoundError:
...

View File

@ -1,13 +1,27 @@
from stdlib_list import stdlib_list
from fastapi_projects.clone import clone_repository
from fastapi_projects.database import Base, engine
from fastapi_projects.files import get_python_files
from fastapi_projects.find import find_repositories
from fastapi_projects.packages import get_packages
from fastapi_projects.save import create_project
from fastapi_projects.utils import skip_repository
ignore = ["fastapi", "uvicorn", "src"]
libraries = stdlib_list("3.8") + ignore
if __name__ == "__main__":
Base.metadata.create_all(bind=engine)
for repository in find_repositories():
packages = set()
if skip_repository(repository):
continue
dir = clone_repository(repository)
for file in get_python_files(dir):
for package in get_packages(file):
print(package)
packages_per_file = list(get_packages(file))
packages.update(packages_per_file)
file.close()
packages = packages - set(libraries)
create_project(repository, packages)
dir.cleanup()

View File

@ -0,0 +1,31 @@
from sqlalchemy import Column, DateTime, Integer, String, Table, Text
from sqlalchemy.orm import relationship
from sqlalchemy.sql.schema import ForeignKey
from fastapi_projects.database import Base
association_table = Table(
"association",
Base.metadata,
Column("left_id", Integer, ForeignKey("projects.id")),
Column("right_id", Integer, ForeignKey("dependencies.id")),
)
class Project(Base):
__tablename__ = "projects"
id = Column(Integer, primary_key=True)
name = Column(String(32), nullable=False)
url = Column(Text, nullable=False, unique=True)
clone_url = Column(Text, nullable=False, unique=True)
pushed_at = Column(DateTime, nullable=False)
dependencies = relationship("Dependency", secondary=association_table)
class Dependency(Base):
__tablename__ = "dependencies"
id = Column(Integer, primary_key=True)
name = Column(String(32), nullable=False, unique=True)

View File

@ -2,7 +2,10 @@ from typing import TextIO
def get_packages(file: TextIO):
for line in file.readlines():
clean_line = line.strip()
if clean_line.startswith(("from", "import")):
yield clean_line.replace(".", " ").split()[1]
try:
for line in file.readlines():
clean_line = line.strip()
if clean_line.startswith(("from ", "import ")):
yield clean_line.replace(".", " ").split()[1]
except UnicodeDecodeError:
yield ""

33
fastapi_projects/save.py Normal file
View File

@ -0,0 +1,33 @@
from typing import List
from github.Repository import Repository
from sqlalchemy.exc import IntegrityError
from sqlalchemy.sql.sqltypes import Integer
from fastapi_projects.database import SessionManager
from fastapi_projects.models import Dependency, Project
def create_project(repository: Repository, packages: List[str]) -> None:
project_params = {
"name": repository.full_name,
"clone_url": repository.clone_url,
"url": repository.html_url,
"pushed_at": repository.pushed_at,
}
project = Project(**project_params)
with SessionManager() as session:
try:
session.add(project)
session.commit()
except IntegrityError:
session.rollback()
for package in packages:
try:
dependency = Dependency(name=package)
project.dependencies.append(dependency)
session.add(project)
session.commit()
except IntegrityError:
session.rollback()
print([dependency.name for dependency in project.dependencies])

15
fastapi_projects/utils.py Normal file
View File

@ -0,0 +1,15 @@
from github.Repository import Repository
from fastapi_projects.database import SessionManager
from fastapi_projects.models import Project
def skip_repository(repository: Repository) -> bool:
if repository.fork:
return True
with SessionManager() as session:
name = repository.full_name
project = session.query(Project).filter(Project.name == name).first()
if project is None or project.pushed_at < repository.pushed_at:
return False
return True