add

fastapi_projects/database.py

@@ -1,16 +1,16 @@
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import Session, sessionmaker
 
-engine = create_engine("sqlite://database.db")
+engine = create_engine("sqlite:///database.db")
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 Base = declarative_base()
 
 
 class SessionManager:
-    def __enter__(self):
+    def __enter__(self) -> Session:
         self.session = SessionLocal()
         return self.session
 
-    def __exit__(self, *exc_data):
+    def __exit__(self, *exc_data) -> None:
         self.session.close()

fastapi_projects/files.py

@@ -7,4 +7,7 @@ def get_python_files(dir: TemporaryDirectory) -> Generator[TextIO, None, None]:
     for dirpath, _, filenames in os.walk(dir.name):
         for filename in filenames:
             if filename.endswith(".py"):
-                yield open(os.sep.join([dirpath, filename]), "r")
+                try:
+                    yield open(os.sep.join([dirpath, filename]), "r")
+                except FileNotFoundError:
+                    ...

fastapi_projects/flow.py

@@ -1,13 +1,27 @@
+from stdlib_list import stdlib_list
+
 from fastapi_projects.clone import clone_repository
+from fastapi_projects.database import Base, engine
 from fastapi_projects.files import get_python_files
 from fastapi_projects.find import find_repositories
 from fastapi_projects.packages import get_packages
+from fastapi_projects.save import create_project
+from fastapi_projects.utils import skip_repository
+
+ignore = ["fastapi", "uvicorn", "src"]
+libraries = stdlib_list("3.8") + ignore
 
 if __name__ == "__main__":
+    Base.metadata.create_all(bind=engine)
     for repository in find_repositories():
+        packages = set()
+        if skip_repository(repository):
+            continue
         dir = clone_repository(repository)
         for file in get_python_files(dir):
-            for package in get_packages(file):
+            packages_per_file = list(get_packages(file))
-                print(package)
+            packages.update(packages_per_file)
             file.close()
+        packages = packages - set(libraries)
+        create_project(repository, packages)
         dir.cleanup()

fastapi_projects/models.py

@@ -0,0 +1,31 @@
+from sqlalchemy import Column, DateTime, Integer, String, Table, Text
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql.schema import ForeignKey
+
+from fastapi_projects.database import Base
+
+association_table = Table(
+    "association",
+    Base.metadata,
+    Column("left_id", Integer, ForeignKey("projects.id")),
+    Column("right_id", Integer, ForeignKey("dependencies.id")),
+)
+
+
+class Project(Base):
+    __tablename__ = "projects"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(32), nullable=False)
+    url = Column(Text, nullable=False, unique=True)
+    clone_url = Column(Text, nullable=False, unique=True)
+    pushed_at = Column(DateTime, nullable=False)
+
+    dependencies = relationship("Dependency", secondary=association_table)
+
+
+class Dependency(Base):
+    __tablename__ = "dependencies"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(32), nullable=False, unique=True)

fastapi_projects/packages.py

@@ -2,7 +2,10 @@ from typing import TextIO
 
 
 def get_packages(file: TextIO):
-    for line in file.readlines():
+    try:
-        clean_line = line.strip()
+        for line in file.readlines():
-        if clean_line.startswith(("from", "import")):
+            clean_line = line.strip()
-            yield clean_line.replace(".", " ").split()[1]
+            if clean_line.startswith(("from ", "import ")):
+                yield clean_line.replace(".", " ").split()[1]
+    except UnicodeDecodeError:
+        yield ""

fastapi_projects/save.py

@@ -0,0 +1,33 @@
+from typing import List
+
+from github.Repository import Repository
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.sql.sqltypes import Integer
+
+from fastapi_projects.database import SessionManager
+from fastapi_projects.models import Dependency, Project
+
+
+def create_project(repository: Repository, packages: List[str]) -> None:
+    project_params = {
+        "name": repository.full_name,
+        "clone_url": repository.clone_url,
+        "url": repository.html_url,
+        "pushed_at": repository.pushed_at,
+    }
+    project = Project(**project_params)
+    with SessionManager() as session:
+        try:
+            session.add(project)
+            session.commit()
+        except IntegrityError:
+            session.rollback()
+        for package in packages:
+            try:
+                dependency = Dependency(name=package)
+                project.dependencies.append(dependency)
+                session.add(project)
+                session.commit()
+            except IntegrityError:
+                session.rollback()
+        print([dependency.name for dependency in project.dependencies])

fastapi_projects/utils.py

@@ -0,0 +1,15 @@
+from github.Repository import Repository
+
+from fastapi_projects.database import SessionManager
+from fastapi_projects.models import Project
+
+
+def skip_repository(repository: Repository) -> bool:
+    if repository.fork:
+        return True
+    with SessionManager() as session:
+        name = repository.full_name
+        project = session.query(Project).filter(Project.name == name).first()
+        if project is None or project.pushed_at < repository.pushed_at:
+            return False
+    return True