Refactor project

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Marcelo Trylesinski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

populate/database.py

@@ -0,0 +1,8 @@
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+db_dir = os.path.abspath(os.getcwd() + "/data/db.sqlite")
+engine = create_engine("sqlite:///" + db_dir, connect_args={"check_same_thread": True})
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

populate/main.py

@@ -0,0 +1,64 @@
+import os
+from time import sleep
+
+from github import ContentFile, Github, PaginatedList
+
+from populate.database import SessionLocal, engine
+from populate.models import Base, Repository
+
+# from sqlalchemy import create_engine
+
+QUERY = '"from fastapi import" language:python in:file'  #  size:{from_}..{to}'
+MAX_SIZE = 50 * (2 ** 10)  # 50 KB
+
+g = Github(os.getenv("GITHUB_TOKEN"))
+
+
+# TOKEN = os.getenv("GITHUB_TOKEN")
+# token_auth = ApiTokenHeader("Authorization", f"token {TOKEN}")
+# git = GitHub("https://api.github.com/", auth=token_auth)
+
+
+# best_split = git.get_best_split()
+# out = []
+# for key, value in best_split.items():
+#     out.append({"from": key[0], "to": key[1], "count": value})
+# with open("best_split.json", "w") as f:
+#     json.dump(out, f)
+
+
+if __name__ == "__main__":
+    # TODO: Create migrations setup.
+    Base.metadata.create_all(engine)
+
+    # 1. Query projects within a range
+    interval = 2 ** 9  # 512 bytes
+    with SessionLocal() as session:
+        for from_ in range(0, MAX_SIZE, interval):
+            to = from_ + interval
+            files = g.search_code(QUERY.format(from_=from_, to=to))
+            for file in files:
+                repo = file.repository
+                repo_obj = Repository.get(session, full_name=repo.full_name)
+                if repo_obj is not None:
+                    # TODO: Check when last commit was made.
+                    # If it was recent, update dependencies.
+                    continue
+                repo_obj = Repository(
+                    full_name=repo.full_name,
+                    html_url=repo.html_url,
+                    clone_url=repo.clone_url,
+                    stargazers=repo.stargazers_count,
+                )
+                session.add(repo_obj)
+                session.flush()
+                # TODO: Check if repository already analyzed.
+
+                print(repo.full_name)
+
+                sleep(1)
+
+    # 2. Clone repositories
+    # 3. Find import statements
+    # 4. Store project data into the database
+    # 5. Push the sqlite file to gcs

populate/models.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+from typing import Any, Dict, Optional, TypeVar
+
+import humps
+from sqlalchemy import Column, inspect
+from sqlalchemy.ext.declarative import as_declarative, declared_attr
+from sqlalchemy.orm import relationship
+from sqlalchemy.orm.session import Session
+from sqlalchemy.sql.schema import ForeignKey
+from sqlalchemy.sql.sqltypes import DateTime, Integer, String
+
+Self = TypeVar("Self", bound="Base")
+
+
+@as_declarative()
+class Base:
+    __name__: str
+
+    created_at = Column(DateTime, default=datetime.now)
+    updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        return humps.depascalize(cls.__name__)
+
+    def dict(self) -> Dict[str, Any]:
+        return {c.key: getattr(self, c.key) for c in inspect(self).mapper.column_attrs}
+
+    @classmethod
+    def get(cls, session: Session, *args: Any, **kwargs: Any) -> Optional[Self]:
+        session.query(cls).filter(*args).filter_by(**kwargs).first()
+
+
+class Repository(Base):
+    id = Column(Integer, primary_key=True)
+    full_name = Column(String, nullable=False)
+    html_url = Column(String, nullable=False)
+    clone_url = Column(String, nullable=False)
+    stargazers = Column(Integer, nullable=False)
+
+    packages = relationship("Package", secondary="dependency")
+
+
+class Package(Base):
+    id = Column(Integer, primary_key=True)
+    name = Column(String, nullable=False)
+
+    repositories = relationship("Repository", secondary="dependency")
+
+
+class Dependency(Base):
+    repository_id = Column(Integer, ForeignKey("repository.id"), primary_key=True)
+    package_id = Column(Integer, ForeignKey("package.id"), primary_key=True)

requirements.txt

@@ -0,0 +1,7 @@
+flake8==4.0.1
+isort==5.10.1
+black==21.10b0
+
+PyGithub==1.55
+pyhumps==3.0.2
+sqlalchemy==1.4.27

setup.cfg

@@ -0,0 +1,35 @@
+[metadata]
+name = fastapi-projects
+version = 0.0.1
+description = Query FastAPI projects.
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = Marcelo Trylesinski
+author_email = marcelotryle@gmail.com
+url = https://github.com/Kludex/awesome-fastapi-projects
+project_urls =
+    Changelog = https://github.com/Kludex/awesome-fastapi-projects/blob/main/CHANGELOG.md
+    Twitter = https://twitter.com/marcelotryle
+license = MIT
+keywords = fastapi, projects
+classifiers =
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Natural Language :: English
+    Operating System :: OS Independent
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+license_file = LICENSE
+
+[options]
+packages =
+    fastapi_projects
+include_package_data = True
+install_requires =
+    typer==0.4.0
+python_requires = >=3.7
+zip_safe = False

setup.py

@@ -0,0 +1,3 @@
+from setuptools import Extension, setup
+
+setup()