From 4cce037b9574e06332b884b839d8f1c5b65eb36f Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 25 Jan 2021 03:25:59 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20Create=20the=20FastAPI=20project?= =?UTF-8?q?s=20package=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastapi_projects/__init__.py | 0 .../__pycache__/clone.cpython-38.pyc | Bin 0 -> 530 bytes .../__pycache__/files.cpython-38.pyc | Bin 0 -> 599 bytes .../__pycache__/find.cpython-38.pyc | Bin 0 -> 1484 bytes .../__pycache__/logger.cpython-38.pyc | Bin 0 -> 244 bytes .../__pycache__/packages.cpython-38.pyc | Bin 0 -> 486 bytes fastapi_projects/clone.py | 10 ++++ fastapi_projects/database.py | 16 ++++++ fastapi_projects/files.py | 10 ++++ fastapi_projects/find.py | 50 ++++++++++++++++++ fastapi_projects/flow.py | 13 +++++ fastapi_projects/logger.py | 3 ++ fastapi_projects/packages.py | 8 +++ generate_table.py | 4 +- poetry.lock | 8 +++ pyproject.toml | 15 ++++++ 16 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 fastapi_projects/__init__.py create mode 100644 fastapi_projects/__pycache__/clone.cpython-38.pyc create mode 100644 fastapi_projects/__pycache__/files.cpython-38.pyc create mode 100644 fastapi_projects/__pycache__/find.cpython-38.pyc create mode 100644 fastapi_projects/__pycache__/logger.cpython-38.pyc create mode 100644 fastapi_projects/__pycache__/packages.cpython-38.pyc create mode 100644 fastapi_projects/clone.py create mode 100644 fastapi_projects/database.py create mode 100644 fastapi_projects/files.py create mode 100644 fastapi_projects/find.py create mode 100644 fastapi_projects/flow.py create mode 100644 fastapi_projects/logger.py create mode 100644 fastapi_projects/packages.py create mode 100644 poetry.lock create mode 100644 pyproject.toml diff --git a/fastapi_projects/__init__.py b/fastapi_projects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fastapi_projects/__pycache__/clone.cpython-38.pyc b/fastapi_projects/__pycache__/clone.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..23068f0ce52ea31738f3c303f4cc4e8e9a4f05cb GIT binary patch literal 530 zcmYjOy-veG3_f3e+J+VtHh2J}3{6%BgpgqF0A5j)W`}#<;vnQB1YMC1vevQ7H1B@nza% zdF|mQs$CSi!hA|AZiExENiRvuV=|h(uqV*)&R1+u?sAIxf;M zeiS9u4F2Llfoeen%m0G$Ceh3bie=qjPu|jO(X7%X!@z|;;2xo&--_Crt~N_E(NKtb x_o)s{S1^f7v1W8Yxp_4SwiiTmo92mz!;UdOUboMrN~%npo0B#}ySxis>le^RgcJY( literal 0 HcmV?d00001 diff --git a/fastapi_projects/__pycache__/files.cpython-38.pyc b/fastapi_projects/__pycache__/files.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48becbf9e0e005ffa8e2da71e2bb7138fddf65aa GIT binary patch literal 599 zcmYjOy>1jS5cb&KpF|0Y2q2LMK-wZ)M}-iIP9ij+;GkH6j@5bYZZErQ$96at?N#s? z6qk}$;8EOC@d{LoBLW#|zL{_I+4ISMv%kN~D1H=oKAZz%KO?yv36bZN?u0-w1q~}> zut)mZ^f*tjK=RJ?yN2aCd^O(YCVv zxVAX++V%wxJ1?+9TM-NC1V8zguV^S#P1K^lwB3wc{)^}?qnuH?Qv%PnFt8QBf-!sl zUUVkXrFC{4E1^adfe#82J=upT%R-2e3sJ zxOPFVtL8k&u4*yl*eSQJ{W3^xunWSW338?Du1IY36f$ybDiXjn?9A&1<2e`doAx`i zhot*G87*~-qqed$Z1iZ1AE}rYJ3pFN&R3=$RcmylIsVI!jnyl9E_WlC{vXgD7fl~5 z&`(Y8m%5wAjV_JvlM^s_06FY}y-$6S**FWir=N&A%Og#FZ|ZJg1raMFMDexXTW~K* V^QWy=mkmCPpK`q=hdnqDe*hU{o1Fjv literal 0 HcmV?d00001 diff --git a/fastapi_projects/__pycache__/find.cpython-38.pyc b/fastapi_projects/__pycache__/find.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6d8bfa64083a548d2cbffa3e2c5e8380f840b1c GIT binary patch literal 1484 zcmZ8h&2Jk;6rY*>@b219>@-bM4k)UC)JS!1fKY{6+BA(Q0o5(Ed)aPxUYvFIE3@O! zSnCVriiEfpB>Tv}G~x!A{0p3Tvrdz!vzmGD_h#O_&z)!OwnuQ>_-N_n^#&pTXmEAd z2=3x6f5wFqP7~rUDD`QLna_~2#7IrwqMN!MMkpzgT{U(*fzV&(IID-=oUig$zU} zbJ1hU-i_tq_?7zLNhslQoW^qJbp*hHYg))S&rrpC0!3cNsB((PPV(Upgn#Yc{rfwE zLGbjuZ+7;B-My!eo_!sYzb$fh2TkGSPMk>)Ct;#&0dg#|h-xR!M-!=oGs0Ou!W~Yh zw4#z6k>99zP|>%Tor+du9W%2*-jY9!DU)=>xXG=Gar;+eYM_oOjmmgKr^e(9Y1W)2 zt(jd}qsGXYQ%Ww7tC5!fK#-c;_?%45AIb4ulSuE#6TefGQ}P1sANOA@&p zBk8O)CsVt!N9}0?*QKdbHAc&v9x?GR+Uf9y#^@`nEq7-th_B67=VVT&Zg#Ui<5bRj zXPnD3l`Bz8UZ1ULoeiwyI+=Qi*DFuosr`D6Q)dSKr#d?BRpy*ga%^&fwQlYUo#+Yc z_s=xrHx9y5hDEIG?@#3+mK1A~gxPQ$4ndh^Jb}qd389D%HwC0&oWY`tLwNc91JiT=bT555`FnWMK-*jgmZrU@Q{V9>~1dL*-DnqcFAZ(hy{T0tgb_ zw8~ysQl@rUxff9=i^Hq&P&rYUK*mF%Y#FC8OqC^|1gTsN<+w=DLCq?AnAX%P6M&-E z6uL)!H;uCZJ9!MEt+DeiaAif94e2+|1-^IbmES#&|B7f?J9x!&Jg@x-v`ycuZPxr> zM0|pV{u^u`<|%BaSR6?5?FVpzq(~u?+vi2y4u66Y&)h!8w+oSvAd=;~1Hq*r&Un8# zRow$*f{Vn*P4)KZmw(^-$_i8$XY(&=s~qJ8TU(hlyLL z*`oqqLelGs50NOAm*S>oYLkU)cVg29;1Hqz{-G!uo`NjxKZ`M6xW1S&8QLQkF zpezHKA3-KMdVg78XW$c48?Uog`kf<6Vl1bZO;7{oya%s`F<5EqL9i4=wu#vFzyhE#?Hj44bD8Kam|7=sx! znO_3s82qXP(h`eH5(_eQ3ySiyQj<%HHJNX*=j5lSXXd5f;!IC1@d2_^i*B)j#8Qh^ zG8C}_^?`|Bq52v5xvBcOiABk&Ir;i7sbxT_AU8FyL?3FZeqwoQF;I;zx-BpT@i1%k cA?E89RNmsS$<0qG%}KRm1lm;01SB|^0O5N?~nno_al)P7$xg3+_ne`vMu1#}Ig|oi{gFiyc;zkb1Zs*1xdG4WDk+jk>F7JN$tSv%TmG5m0;k literal 0 HcmV?d00001 diff --git a/fastapi_projects/clone.py b/fastapi_projects/clone.py new file mode 100644 index 0000000..7d1f613 --- /dev/null +++ b/fastapi_projects/clone.py @@ -0,0 +1,10 @@ +from tempfile import TemporaryDirectory + +from git.repo.base import Repo +from github.Repository import Repository + + +def clone_repository(repository: Repository) -> TemporaryDirectory: + dir = TemporaryDirectory() + Repo.clone_from(repository.clone_url, dir.name) + return dir diff --git a/fastapi_projects/database.py b/fastapi_projects/database.py new file mode 100644 index 0000000..a13a8fc --- /dev/null +++ b/fastapi_projects/database.py @@ -0,0 +1,16 @@ +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +engine = create_engine("sqlite://database.db") +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base = declarative_base() + + +class SessionManager: + def __enter__(self): + self.session = SessionLocal() + return self.session + + def __exit__(self, *exc_data): + self.session.close() diff --git a/fastapi_projects/files.py b/fastapi_projects/files.py new file mode 100644 index 0000000..3a8f1e1 --- /dev/null +++ b/fastapi_projects/files.py @@ -0,0 +1,10 @@ +import os +from tempfile import TemporaryDirectory +from typing import Generator, TextIO + + +def get_python_files(dir: TemporaryDirectory) -> Generator[TextIO, None, None]: + for dirpath, _, filenames in os.walk(dir.name): + for filename in filenames: + if filename.endswith(".py"): + yield open(os.sep.join([dirpath, filename]), "r") diff --git a/fastapi_projects/find.py b/fastapi_projects/find.py new file mode 100644 index 0000000..77f0080 --- /dev/null +++ b/fastapi_projects/find.py @@ -0,0 +1,50 @@ +import calendar +import os +import time +from typing import Generator + +from github import Github, RateLimitExceededException +from github.Repository import Repository + +from fastapi_projects.logger import logger + +access_token = os.getenv("ACCESS_TOKEN_GITHUB") +g = Github(access_token) + +MAX_FILE_SIZE = 384_000 + + +def find_repositories(interval: int = 60) -> Generator[Repository, None, None]: + min_value = 0 + + while min_value < MAX_FILE_SIZE: + size = f"{min_value}..{min_value + interval - 1}" + snippets = g.search_code("fastapi", language="Python", size=size) + count = 0 + + snippets = iter(snippets) + while True: + try: + snippet = next(snippets) + logger.info(snippet.repository.full_name) + logger.info(snippet.repository.clone_url) + yield snippet.repository + count += 1 + except StopIteration: + break + except RateLimitExceededException: + rate_limit = g.get_rate_limit() + search_rate_limit = rate_limit.search + core_rate_limit = rate_limit.core + if search_rate_limit.remaining == 0: + logger.debug(f"search remaining: {search_rate_limit.remaining}") + reset_time = calendar.timegm(search_rate_limit.reset.timetuple()) + else: + logger.debug(f"core remaining: {core_rate_limit.remaining}") + reset_time = calendar.timegm(core_rate_limit.reset.timetuple()) + # add 10 seconds to be sure the rate limit has been reset + sleep_time = reset_time - calendar.timegm(time.gmtime()) + 10 + time.sleep(sleep_time) + min_value += interval + + logger.info("Found '%d' snippets.", count) diff --git a/fastapi_projects/flow.py b/fastapi_projects/flow.py new file mode 100644 index 0000000..aa6b4aa --- /dev/null +++ b/fastapi_projects/flow.py @@ -0,0 +1,13 @@ +from fastapi_projects.clone import clone_repository +from fastapi_projects.files import get_python_files +from fastapi_projects.find import find_repositories +from fastapi_projects.packages import get_packages + +if __name__ == "__main__": + for repository in find_repositories(): + dir = clone_repository(repository) + for file in get_python_files(dir): + for package in get_packages(file): + print(package) + file.close() + dir.cleanup() diff --git a/fastapi_projects/logger.py b/fastapi_projects/logger.py new file mode 100644 index 0000000..9c5d92e --- /dev/null +++ b/fastapi_projects/logger.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger("fastapi-projects") diff --git a/fastapi_projects/packages.py b/fastapi_projects/packages.py new file mode 100644 index 0000000..ec89958 --- /dev/null +++ b/fastapi_projects/packages.py @@ -0,0 +1,8 @@ +from typing import TextIO + + +def get_packages(file: TextIO): + for line in file.readlines(): + clean_line = line.strip() + if clean_line.startswith(("from", "import")): + yield clean_line.replace(".", " ").split()[1] diff --git a/generate_table.py b/generate_table.py index 42aff73..049d38f 100644 --- a/generate_table.py +++ b/generate_table.py @@ -1,5 +1,6 @@ -from typing import List import json +from typing import List + from pytablewriter import MarkdownTableWriter from stdlib_list import stdlib_list @@ -24,6 +25,7 @@ def format_with_link(project: str) -> str: if project in link: return f"[{project}]({link})" + with open("results.json") as json_file: data = json.load(json_file) writer = MarkdownTableWriter() diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..f41588b --- /dev/null +++ b/poetry.lock @@ -0,0 +1,8 @@ +package = [] + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "fafb334cb038533f851c23d0b63254223abf72ce4f02987e7064b0c95566699a" + +[metadata.files] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dcdc637 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "fastapi-projects" +version = "0.1.0" +description = "" +authors = ["Marcelo Trylesinski "] +license = "MIT" + +[tool.poetry.dependencies] +python = "^3.8" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api"