awesome-fastapi-projects/app/database.py

108 lines
3.3 KiB
Python
Raw Normal View History

Web App (#25) * Set up the web project dependencies - Add linters, pre-commit, and GitHub Actions - Add a Makefile - Add a pyproject.toml * Fix pyupgrade job - Remove continue_on_error everywhere * Remove old code * Rename a GithubActions job * Change README * Adjust pre-commit and GitHub actions * Add tables and set up alembic * Set up tests * Extend tests * Add coverage config * Adjust the GithubActions workflow * Fix GithubActions workflow * Try fixing pyproject-fmt config * Fix formatting of pyproject.toml * Fix formatting of pyproject.toml * Add coverage report * Test listing the repositories * Add a working prototype of SourceGraph client * Add parsing of the SourceGraph SSE data * Fix tests * Ged rid of packages replaced by ruff * Fix waits in the SourceGraph client * Refactor the models and add a mapper - A new mapper allows to create database repositories from the SourceGraph data * Add mypy * Try fixing mypy action * Remove redundant configs * Exclude tests from type checking * Fix mypy pre-commit and GitHub action * Ignore factories * Make upserting possible for source graph data * Add logic for parsing the dependencies and populating the database * Add a database and a cron GitHub Action job * Try manually trigger a workflow * Bring back the old config * Add ReadTimeout for errors to retry for in SourceGraph client * Add typer * Adjust the docstrings * Update the database * Refactor and optimize scraping and dependencies parsing * Make scraping run on push for now * Add a unique constraint for the repo url and source graph repo id * Change the index columns in on_conflict statement for repo creation * Optimize dependencies parsing - Do not parse dependencies for a repo when revision did not change * Scraped repositories from Source Graph * Refactor scraping * Set up frontend * Scraped repositories from Source Graph * Add TODOs * Skip scraping when testing * Fix a test with updating the repos * Scraped repositories from Source Graph * Add some more TODOs * Scraped repositories from Source Graph * Add some more TODO comments * Add chadcn/ui * Scraped repositories from Source Graph * Create index.json * Scraped repositories from Source Graph * Add a draft of data table and display all the repos * Scraped repositories from Source Graph * Implement stars badges and description with overflow * Format the links to Github repos * Fix link clicking * Scraped repositories from Source Graph * Add simple pagination and stars column sorting * Scraped repositories from Source Graph * Implement basic searching * Scraped repositories from Source Graph * Implement a multiselect for dependencies * Scraped repositories from Source Graph * Implement actual filtering by dependencies * Scraped repositories from Source Graph * Add a workflow to deploy nextjs on github pages * Try fixing the deployment job * Enable static exports for app router * Fix uploading arifacts for nextjs job * Set base path to properly load JS and CSS * Fix the base path * Scraped repositories from Source Graph * Add header * Remove language version * Scraped repositories from Source Graph * Add some more TODOs * Scraped repositories from Source Graph * Adjust the pre-commit config * Fix pipelines * Scraped repositories from Source Graph * Add a footer * Create the indexes * Scraped repositories from Source Graph * Add more TODOs * Introduce minor footer adjustments * Adjust the scraping actions * Scraped repositories from Source Graph, parsed the dependencies, and generated the indexes * Implement query params state * Scraped repositories from Source Graph, parsed the dependencies, and generated the indexes * Do not commit query state on unmount * Scraped repositories from Source Graph, parsed the dependencies, and generated the indexes * Hopefully fix query states and multiselect input * Scraped repositories from Source Graph, parsed the dependencies, and generated the indexes * Extend the Makefile * Resolve most of TODOs * Resolve the conflicts with anyio version, bring back httpx * Adjust the Makefile and README.md * Fix a typo in README.md * Adjust readme * Fix the Makefile * Fix some stuff * Make some adjustments * Possibly fix failing scraping jobs * Load the repo project URL from env --------- Co-authored-by: vladfedoriuk <vladfedoriuk@users.noreply.github.com> Co-authored-by: Vladyslav Fedoriuk <vladyslav.fedoriuk@deployed.pl>
2023-10-28 19:39:02 +00:00
"""
Database models and session.
The database is a SQLite database, and is stored in the root
of the project as `db.sqlite3`.
The database is managed using Alembic, and migrations
are stored in the `migrations/` directory.
The module defines the following models:
- `Repo`: A repository that is being tracked.
- `Dependency`: A dependency of a repository.
- `RepoDependency`: A relationship between a repository and a dependency.
The database is accessed asynchronously using SQLAlchemy's async API.
"""
from collections.abc import AsyncGenerator
from pathlib import PurePath
from typing import Final
from sqlalchemy import BigInteger, ForeignKey, MetaData, String, Text, UniqueConstraint
from sqlalchemy.ext.asyncio import (
AsyncAttrs,
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from sqlalchemy.orm import (
Mapped,
declarative_base,
mapped_column,
relationship,
)
DB_PATH: Final[PurePath] = PurePath(__file__).parent.parent / "db.sqlite3"
SQLALCHEMY_DATABASE_URL: Final[str] = f"sqlite+aiosqlite:///{DB_PATH}"
engine: Final[AsyncEngine] = create_async_engine(SQLALCHEMY_DATABASE_URL)
async_session_maker: Final[async_sessionmaker[AsyncSession]] = async_sessionmaker(
engine, expire_on_commit=False, autoflush=False, autocommit=False
)
metadata = MetaData(
naming_convention={
"ix": "ix_%(table_name)s_%(column_0_N_name)s ",
"uq": "uq_%(table_name)s_%(column_0_N_name)s ",
"ck": "ck_%(table_name)s_%(constraint_name)s ",
"fk": "fk_%(table_name)s_%(column_0_N_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
}
)
async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
"""Get an async session."""
async with async_session_maker() as session:
yield session
Base = declarative_base(metadata=metadata, cls=AsyncAttrs)
class Repo(Base):
"""A repository that is being tracked."""
__tablename__ = "repo"
id: Mapped[int] = mapped_column(primary_key=True)
url: Mapped[str] = mapped_column(nullable=False, unique=True)
description: Mapped[str] = mapped_column(Text, nullable=False)
stars: Mapped[int] = mapped_column(BigInteger, nullable=False)
source_graph_repo_id: Mapped[int | None] = mapped_column(
BigInteger, nullable=True, unique=True
)
dependencies: Mapped[list["Dependency"]] = relationship(
"Dependency", secondary="repo_dependency", back_populates="repos"
)
last_checked_revision: Mapped[str | None] = mapped_column(
String(255), nullable=True
)
__table_args__ = (UniqueConstraint("url", "source_graph_repo_id"),)
class Dependency(Base):
"""A dependency of a repository."""
__tablename__ = "dependency"
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
repos: Mapped[list["Repo"]] = relationship(
"Repo", secondary="repo_dependency", back_populates="dependencies"
)
class RepoDependency(Base):
"""A relationship between a repository and a dependency."""
__tablename__ = "repo_dependency"
repo_id: Mapped[int] = mapped_column(
ForeignKey(Repo.id, ondelete="CASCADE"), primary_key=True
)
dependency_id: Mapped[int] = mapped_column(
ForeignKey(Dependency.id, ondelete="CASCADE"), primary_key=True
)