diff --git a/app/scrape.py b/app/scrape.py index 973ec8e..522e3ca 100644 --- a/app/scrape.py +++ b/app/scrape.py @@ -2,6 +2,7 @@ import asyncio import sqlalchemy.dialects.sqlite +import typer from sqlalchemy.ext.asyncio import AsyncSession from app.database import Dependency, Repo, RepoDependency @@ -21,7 +22,7 @@ async def _create_dependencies_for_repo(session: AsyncSession, repo: Repo) -> No dependencies_create_data = await acquire_dependencies_data_for_repository(repo) except RuntimeError: return - # Create dependencies - on conflict return the existing dependency + # Create dependencies - on conflict do nothing. insert_statement = sqlalchemy.dialects.sqlite.insert( Dependency ).on_conflict_do_nothing(index_elements=[Dependency.name]) @@ -80,6 +81,11 @@ async def scrape_source_graph_repos_data() -> None: await session.commit() +def main() -> None: + """Scrape the FastAPI-related repositories utilizing the source graph API.""" + asyncio.run(scrape_source_graph_repos_data()) + + if __name__ == "__main__": """Run the scraping.""" - asyncio.run(scrape_source_graph_repos_data()) + typer.run(main) diff --git a/pyproject.toml b/pyproject.toml index 22123f2..267295d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "sqlalchemy[asyncio]", "stamina", "third-party-imports", + "typer[all]", ] [project.optional-dependencies] dev = [ diff --git a/requirements/base.txt b/requirements/base.txt index fffb908..7487eb5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -22,7 +22,11 @@ certifi==2023.7.22 # httpcore # httpx click==8.1.6 - # via uvicorn + # via + # typer + # uvicorn +colorama==0.4.6 + # via typer dnspython==2.4.1 # via email-validator email-validator==2.0.0.post2 @@ -54,10 +58,14 @@ jinja2==3.1.2 # via fastapi mako==1.2.4 # via alembic +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.3 # via # jinja2 # mako +mdurl==0.1.2 + # via markdown-it-py orjson==3.9.2 # via fastapi pydantic==2.1.1 @@ -71,6 +79,8 @@ pydantic-extra-types==2.0.0 # via fastapi pydantic-settings==2.0.2 # via fastapi +pygments==2.15.1 + # via rich python-dotenv==1.0.0 # via # pydantic-settings @@ -81,6 +91,10 @@ pyyaml==6.0.1 # via # fastapi # uvicorn +rich==13.5.2 + # via typer +shellingham==1.5.0.post1 + # via typer sniffio==1.3.0 # via # anyio @@ -98,6 +112,8 @@ tenacity==8.2.2 # via stamina third-party-imports==0.0.7 # via awesome-fastapi-projects (pyproject.toml) +typer[all]==0.9.0 + # via awesome-fastapi-projects (pyproject.toml) typing-extensions==4.7.1 # via # alembic @@ -105,6 +121,7 @@ typing-extensions==4.7.1 # pydantic # pydantic-core # sqlalchemy + # typer ujson==5.8.0 # via fastapi uvicorn[standard]==0.23.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index ca2c646..d94e65a 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -35,7 +35,10 @@ click==8.1.6 # via # black # pip-tools + # typer # uvicorn +colorama==0.4.6 + # via typer decorator==5.1.1 # via ipython distlib==0.3.7 @@ -81,12 +84,16 @@ jinja2==3.1.2 # via fastapi mako==1.2.4 # via alembic +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.3 # via # jinja2 # mako matplotlib-inline==0.1.6 # via ipython +mdurl==0.1.2 + # via markdown-it-py mypy==1.4.1 # via awesome-fastapi-projects (pyproject.toml) mypy-extensions==1.0.0 @@ -138,7 +145,9 @@ pydantic-extra-types==2.0.0 pydantic-settings==2.0.2 # via fastapi pygments==2.15.1 - # via ipython + # via + # ipython + # rich pyproject-fmt==0.13.0 # via awesome-fastapi-projects (pyproject.toml) pyproject-hooks==1.0.0 @@ -154,8 +163,12 @@ pyyaml==6.0.1 # fastapi # pre-commit # uvicorn +rich==13.5.2 + # via typer ruff==0.0.280 # via awesome-fastapi-projects (pyproject.toml) +shellingham==1.5.0.post1 + # via typer six==1.16.0 # via asttokens sniffio==1.3.0 @@ -183,6 +196,8 @@ traitlets==5.9.0 # via # ipython # matplotlib-inline +typer[all]==0.9.0 + # via awesome-fastapi-projects (pyproject.toml) types-aiofiles==23.1.0.5 # via awesome-fastapi-projects (pyproject.toml) typing-extensions==4.7.1 @@ -193,6 +208,7 @@ typing-extensions==4.7.1 # pydantic # pydantic-core # sqlalchemy + # typer ujson==5.8.0 # via fastapi uvicorn[standard]==0.23.1 diff --git a/requirements/test.txt b/requirements/test.txt index e0ef6db..4d1ce82 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -23,7 +23,11 @@ certifi==2023.7.22 # httpcore # httpx click==8.1.6 - # via uvicorn + # via + # typer + # uvicorn +colorama==0.4.6 + # via typer coverage[toml]==7.2.7 # via pytest-cov dirty-equals==0.6.0 @@ -63,10 +67,14 @@ jinja2==3.1.2 # via fastapi mako==1.2.4 # via alembic +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.3 # via # jinja2 # mako +mdurl==0.1.2 + # via markdown-it-py orjson==3.9.2 # via fastapi packaging==23.1 @@ -86,6 +94,8 @@ pydantic-extra-types==2.0.0 # via fastapi pydantic-settings==2.0.2 # via fastapi +pygments==2.15.1 + # via rich pytest==7.4.0 # via # awesome-fastapi-projects (pyproject.toml) @@ -112,6 +122,10 @@ pyyaml==6.0.1 # via # fastapi # uvicorn +rich==13.5.2 + # via typer +shellingham==1.5.0.post1 + # via typer six==1.16.0 # via python-dateutil sniffio==1.3.0 @@ -131,6 +145,8 @@ tenacity==8.2.2 # via stamina third-party-imports==0.0.7 # via awesome-fastapi-projects (pyproject.toml) +typer[all]==0.9.0 + # via awesome-fastapi-projects (pyproject.toml) typing-extensions==4.7.1 # via # alembic @@ -139,6 +155,7 @@ typing-extensions==4.7.1 # pydantic # pydantic-core # sqlalchemy + # typer ujson==5.8.0 # via fastapi uvicorn[standard]==0.23.1