[TC-642] modify template (#3)

https://eva.avroid.tech/desk/Task/TC-642#be-dorabotat-shablon-servisov-backend

Дорабатываем шаблон сервисов по требованиям: https://eva.avroid.tech/project/Document/DOC-002710#trebovanija-k-versijam

Reviewed-on: https://git.avroid.tech/Templates/template-backend-service/pulls/3
Reviewed-by: Victor Stratov <victor.stratov@avroid.team>
Reviewed-by: Petr Brovchenko <petr.brovchenko@avroid.team>
Co-authored-by: Nadezhda <nadezhda.lavrentieva@avroid.team>
Co-committed-by: Nadezhda <nadezhda.lavrentieva@avroid.team>
This commit is contained in:
Nadezhda
2024-12-16 10:26:11 +03:00
committed by Nadezhda Lavrentieva
parent a4b2c99c25
commit ac441a108b
33 changed files with 782 additions and 862 deletions

12
.helm/values.dev.yaml Normal file
View File

@@ -0,0 +1,12 @@
replicaCount: 1
extraEnv:
POSTGRES_DSN:
value: "postgresql://test:test@postgresql:5432/messenger"
PORT:
value: "8000"
ENVIRONMENT:
value: "dev"
service:
port: 8000

View File

@@ -1,32 +0,0 @@
replicaCount: 1
extraEnv:
POSTGRES_USER:
value: "test"
POSTGRES_PASSWORD:
value: "test"
POSTGRES_HOST:
value: "cloud-postgres.avroid.cloud"
POSTGRES_DB:
value: "messenger"
POSTGRES_PORT:
value: "5432"
SCYLLADB_HOST:
value: "cloud-scylla.avroid.cloud"
SCYLLADB_PORT:
value: "9042"
SCYLLADB_USER:
value: "test"
SCYLLADB_PASSWORD:
value: "test"
SCYLLADB_KEYSPACE:
value: "messenger"
PORT:
value: "8000"
ENVIRONMENT:
value: "preprod"
service:
port: 8000

View File

@@ -1,31 +1,12 @@
replicaCount: 1
extraEnv:
POSTGRES_USER:
value: "test"
POSTGRES_PASSWORD:
value: "test"
POSTGRES_HOST:
value: "cloud-postgres.avroid.cloud"
POSTGRES_DB:
value: "messenger"
POSTGRES_PORT:
value: "5432"
SCYLLADB_HOST:
value: "cloud-scylla.avroid.cloud"
SCYLLADB_PORT:
value: "9042"
SCYLLADB_USER:
value: "test"
SCYLLADB_PASSWORD:
value: "test"
SCYLLADB_KEYSPACE:
value: "messenger"
POSTGRES_DSN:
value: "postgresql://test:test@postgresql:5432/messenger"
PORT:
value: "8000"
ENVIRONMENT:
value: "production"
value: "prod"
service:
port: 8000

View File

@@ -1,27 +1,8 @@
replicaCount: 1
extraEnv:
POSTGRES_USER:
value: "test"
POSTGRES_PASSWORD:
value: "test"
POSTGRES_HOST:
value: "cloud-postgres.avroid.cloud"
POSTGRES_DB:
value: "messenger"
POSTGRES_PORT:
value: "5432"
SCYLLADB_HOST:
value: "cloud-scylla.avroid.cloud"
SCYLLADB_PORT:
value: "9042"
SCYLLADB_USER:
value: "test"
SCYLLADB_PASSWORD:
value: "test"
SCYLLADB_KEYSPACE:
value: "messenger"
POSTGRES_DSN:
value: "postgresql://test:test@postgresql:5432/messenger"
PORT:
value: "8000"
ENVIRONMENT:

12
.helm/values.test.yaml Normal file
View File

@@ -0,0 +1,12 @@
replicaCount: 1
extraEnv:
POSTGRES_DSN:
value: "postgresql://test:test@postgresql:5432/messenger"
PORT:
value: "8000"
ENVIRONMENT:
value: "test"
service:
port: 8000

View File

@@ -1,15 +1,23 @@
FROM python:3.12
FROM harbor.avroid.tech/docker-hub-proxy/python:3.12
ARG PIP_INDEX_URL
ENV PIP_INDEX_URL=${PIP_INDEX_URL}
WORKDIR /app
EXPOSE 8000
COPY pyproject.toml poetry.lock ./
RUN pip --no-cache-dir install poetry
RUN poetry export --without-hashes -f requirements.txt -o requirements.txt
RUN pip install --no-cache-dir -r requirements.txt
RUN pip --no-cache-dir install poetry \
&& poetry export \
--with=dev,tests,format \
--without-hashes \
-f requirements.txt \
-o requirements.txt \
&& pip install --no-cache-dir -r requirements.txt
COPY . .
CMD ["python", "-m", "src.api_app"]
EXPOSE 8000
CMD ["./entry.sh"]

View File

@@ -16,16 +16,22 @@ lint:
@poetry run mypy $(SERVICE_DIR)/
format:
@poetry run ruff format $(SERVICE_DIR)/ tests/
@poetry run ruff format $(SERVICE_DIR)/ tests/ migrations/
start:
@poetry run python -m $(SERVICE_DIR).api_app
migration:
@poetry run alembic revision --autogenerate
create-migrations:
@poetry run alembic revision --autogenerate -m "${COMMENT}"
migrate:
apply-migrations:
@poetry run alembic upgrade head
revert-migrations:
@poetry run alembic downgrade $(REVISION)
revert-last-migration:
@poetry run alembic downgrade head-1
test:
@poetry run pytest tests --cov $(SERVICE_DIR) -vv

View File

@@ -12,26 +12,62 @@
# HOW TO
## Настроить pre-commit и запустить проект
### !!! (поменяйте креды в local.env на свои личные)
```bash
make setup
make setup-pre-commit
make start
$ make setup
$ make setup-pre-commit
$ make start
```
## Запустить тесты:
Note: тесты запускаются в локальной БД на локальной машине!
Перед запуском проверьте, что у вас есть указанный в `tests.conftest` юзер с нужным паролем (можно указать свой) и
Перед запуском проверьте, что у вас есть указанный в `tests.fixtures.db` юзер с нужным паролем (можно указать свой) и
правами!
(И что в схеме public нет ничего нужного, потому что она дропается!)
```bash
make test
$ make test
```
При локальном разворачивании документация доступна по адресу: http://0.0.0.0:8000/docs
При локальном разворачивании документация доступна по адресу: http://localhost:8000/docs
## Проверить работоспособность сервиса
Простая healthcheck-проверка:
```bash
$ curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api/_/healthcheck
```
Ожидаем увидеть код ответа: `200`.
## Работа с миграциями
### Создать/сгенерировать миграции
```bash
COMMENT="short human readable comment" make create-migrations
```
### Применить миграции
```bash
make apply-migrations
```
### Откатить миграции к конкретной ревизии
```bash
REVISION="revision id" make revert-migrations
```
### Откатить последнюю миграцию
```bash
make revert-last-migration
```

View File

@@ -1,40 +1,45 @@
networks:
cloud-messenger:
driver: bridge
volumes:
postgresql: {}
services:
template-backend-service:
build: .
ports:
- "8000:8000"
env_file: "local.env"
container_name: template-backend-service
volumes:
- ./:/app
environment:
POSTGRES_DSN: "postgresql://test:test@cloud-postgres.avroid.cloud:5432/messenger"
POSTGRES_USER: "test"
POSTGRES_PASSWORD: "test"
POSTGRES_HOST: "cloud-postgres.avroid.cloud"
POSTGRES_DB: "messenger"
PORT: "8000"
SCYLLADB_HOST: "cloud-scylla.avroid.cloud"
SCYLLADB_PORT: "9042"
SCYLLADB_USER: "test"
SCYLLADB_PASSWORD: "test"
SCYLLADB_KEYSPACE: "messenger"
PORT: 8000
ENVIRONMENT: local
LOGGING: '{"json_enabled": true, "level": "INFO"}'
ENVIRONMENT: "production"
depends_on:
- db
db:
image: postgres:14.8
restart: always
environment:
POSTGRES_USER: "test"
POSTGRES_PASSWORD: "test"
POSTGRES_HOST: "cloud-postgres.avroid.cloud"
POSTGRES_DB: "messenger"
POSTGRES_DSN: postgresql://messenger:messenger@postgresql:5432/messenger
networks:
- cloud-messenger
ports:
- "5432:5432"
- "8000:8000"
depends_on:
- postgresql
postgresql:
image: postgres:14.0
container_name: template-backend-service-db
restart: always
volumes:
- postgresql:/var/lib/postgresql/data
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD: messenger
POSTGRES_DB: messenger
networks:
- cloud-messenger
ports:
- "5432:5432"
healthcheck:
test: [ "CMD-SHELL", "pg_isready -d postgres" ]
interval: 30s
timeout: 10s
retries: 5

12
entry.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/bin/sh
set -e
# Apply migrations
alembic upgrade head
# For apply migrations for specific scheme
# alembic --name specific_scheme upgrade head
# Start application
python -m src.api_app

View File

@@ -1,13 +1,2 @@
POSTGRES_USER=test
POSTGRES_PASSWORD=test
POSTGRES_HOST=cloud-postgres.avroid.cloud
POSTGRES_DB=messenger
POSTGRES_PORT=5432
SCYLLADB_HOST=cloud-scylla.avroid.cloud
SCYLLADB_PORT=9042
SCYLLADB_USER=test
SCYLLADB_PASSWORD=test
SCYLLADB_KEYSPACE=messenger
POSTGRES_DSN=postgresql://postgres_user:postgres_password@postgres_host:5432/db
PORT=8000

1104
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "template-backend-service"
version = "0.1.0"
version = "0.1.1"
description = ""
authors = ["Nadezhda Lavrenteva <nadezhda.lavrentieva@avroid.team>"]
readme = "README.md"
@@ -8,6 +8,12 @@ readme = "README.md"
[tool.poetry.dependencies]
python = ">=3.12 <3.13"
coverage = "^7.6.1"
avroid-service-lib = "^0.0.3"
[[tool.poetry.source]]
name = "nexus"
url = "https://nexus.avroid.tech/repository/tavro-cloud-pypi-release/simple"
priority = "supplemental"
[build-system]
requires = ["poetry-core"]
@@ -22,11 +28,11 @@ pydantic = "^2.9.2"
sqlalchemy = "^2.0.35"
pydantic-settings = "^2.5.2"
uvicorn = "^0.31.0"
scylla-driver = "^3.26.9"
cassandra-driver = "^3.29.2"
pyyaml = "^6.0.2"
aiopg = {version = "^1.4.0", extras = ["sa"]}
httpx = "^0.27.2"
structlog = "^24.4.0"
orjson = "^3.10.12"
[tool.poetry.group.format.dependencies]
mypy = "^1.11.2"
@@ -36,9 +42,11 @@ pre-commit = "^3.8.0"
[tool.poetry.group.tests.dependencies]
pytest-asyncio = "^0.24.0"
pytest-cov = "^5.0.0"
pytest-mock = "^3.14.0"
[tool.pytest.ini_options]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
[tool.coverage.run]
omit = ["tests/*"]
@@ -51,7 +59,6 @@ disallow_untyped_calls = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
check_untyped_defs = true
disallow_untyped_decorators = true
no_implicit_optional = true
warn_unused_ignores = true
warn_return_any = true

View File

@@ -1,17 +1,21 @@
from typing import cast
import uvicorn
from avroid_service_lib import create_default_app
from fastapi import FastAPI
from src.routers.v1 import api_router
from src.routes.v1 import api_router
from src.settings import SERVICE_NAME, WebAppSettings
def create_app(settings: WebAppSettings) -> FastAPI:
api = FastAPI(
api = create_default_app(
title=SERVICE_NAME,
settings=settings,
)
api.extra = {"settings": settings}
api.include_router(api_router, prefix="/api")
return api
return cast(FastAPI, api)
if __name__ == "__main__":

View File

@@ -6,10 +6,7 @@ from src.settings import WebAppSettings
class PGConnector:
def __init__(self, settings: WebAppSettings) -> None:
self.pg_engine = create_engine(
f"postgresql://{settings.postgres_user}:{settings.postgres_password}@{settings.postgres_host}:"
f"{settings.postgres_port}/{settings.postgres_db}",
)
self.pg_engine = create_engine(settings.postgres_dsn)
self.pg_session = sessionmaker(self.pg_engine, class_=Session)

View File

@@ -1,23 +0,0 @@
from cassandra.auth import PlainTextAuthProvider
from cassandra.cluster import Cluster
from cassandra.cqlengine import connection
from src.settings import WebAppSettings
class ScyllaConnector:
def __init__(self, settings: WebAppSettings) -> None:
self.auth_provider = PlainTextAuthProvider(
username=settings.scylladb_user,
password=settings.scylladb_password,
)
self.cluster = Cluster(
[settings.scylladb_host],
auth_provider=self.auth_provider,
port=settings.scylladb_port,
)
self.scylladb_session = self.cluster.connect(keyspace=settings.scylladb_keyspace)
connection.register_connection("main_cluster", session=self.scylladb_session)
connection.set_default_connection("main_cluster")

View File

@@ -1,12 +1,10 @@
from typing import AsyncGenerator, Generator
from typing import AsyncGenerator
from cassandra.cluster import Session as ScyllaSession
from fastapi import Depends, Request
from sqlalchemy.orm import Session
from src.database.postgresql import PGConnector
from src.database.scylla import ScyllaConnector
from src.repositories.repository import MessengerHandbookCountryRepositoryPG, TestRepository
from src.repositories.repository import TodoRepository
from src.settings import WebAppSettings
@@ -24,17 +22,5 @@ async def get_session_pg(settings: WebAppSettings = Depends(get_settings)) -> As
db.close()
def get_session_scylla(settings: WebAppSettings = Depends(get_settings)) -> Generator[ScyllaSession, None, None]:
scylla_connect = ScyllaConnector(settings)
db = scylla_connect.scylladb_session
yield db
async def get_messenger_handbook_country_repository(
request: Request, session: Session = Depends(get_session_pg)
) -> MessengerHandbookCountryRepositoryPG:
return MessengerHandbookCountryRepositoryPG(session)
async def get_test_repository(request: Request, session: Session = Depends(get_session_scylla)) -> TestRepository:
return TestRepository(session)
async def get_todo_repository(request: Request, session: Session = Depends(get_session_pg)) -> TodoRepository:
return TodoRepository(session)

View File

@@ -1,5 +1,4 @@
from enum import Enum
from uuid import UUID
from pydantic import BaseModel
@@ -10,11 +9,5 @@ class RoleEnum(Enum):
client = 3
class Messenger(BaseModel):
chat_id: UUID
title: str | None = None
class MessengerHandbookCountry(BaseModel):
country_id: int
iso_3166_code_alpha3: str | None = None
class TodoModel(BaseModel):
id: int

View File

@@ -1,12 +1,11 @@
import abc
from typing import Any, Sequence, T # type: ignore
from cassandra.cqlengine.models import Model as ScyllaModel
from sqlalchemy import Select, Table, select
from sqlalchemy.orm import Session
from src.models.base import Messenger, MessengerHandbookCountry
from src.repositories.tables import MessengerTable, messenger_handbook_country_table
from src.models.base import TodoModel
from src.repositories.tables.tables import todo_table
class BaseRepositoryPG(abc.ABC):
@@ -30,48 +29,18 @@ class BaseRepositoryPG(abc.ABC):
return tuple(model_cls(**row._mapping) for row in result) # noqa
class BaseRepositoryScylla(abc.ABC):
@property
@abc.abstractmethod
def model_cls(self) -> type[T]:
raise NotImplementedError
class TodoRepository(BaseRepositoryPG):
table = todo_table
model_cls = TodoModel
@property
@abc.abstractmethod
def table(self) -> ScyllaModel:
raise NotImplementedError
def __init__(self, session: Session) -> None:
self._session = session
async def _get_from_query(self, query: Select[Any]) -> tuple[T, ...]:
model_cls = self.model_cls
result = self._session.execute(query)
return tuple(model_cls(**row._mapping) for row in result) # noqa
class MessengerHandbookCountryRepositoryPG(BaseRepositoryPG):
table = messenger_handbook_country_table
model_cls = MessengerHandbookCountry
async def get_from_query(self, id_: int) -> Sequence[MessengerHandbookCountry]:
async def get_from_query(self, id_: int) -> Sequence[TodoModel]:
query = (
(
select(
self.table.c.country_id,
self.table.c.iso_3166_code_alpha3,
self.table.c.id,
)
)
.select_from(self.table)
.where(self.table.c.country_id == id_)
.where(self.table.c.id == id_)
)
return await self._get_from_query(query)
class TestRepository(BaseRepositoryScylla):
table = MessengerTable
model_cls = Messenger
async def get_from_query(self) -> tuple[Messenger, ...]:
result = self.table.objects.all()
return tuple(self.model_cls(**row._mapping) for row in result) # noqa

View File

@@ -1,23 +0,0 @@
from cassandra.cqlengine.columns import UUID
from cassandra.cqlengine.columns import Integer as CassandraInt
from cassandra.cqlengine.models import Model
from sqlalchemy import Column, Integer, String, Table
from src.database.postgresql import pg_metadata
class MessengerTable(Model):
__tablename__ = "messenger_common_user"
__keyspace__ = "messenger"
user_id = UUID(primary_key=True)
target_user_id = UUID(primary_key=True)
via = CassandraInt()
messenger_handbook_country_table = Table(
"messenger_handbook_country",
pg_metadata,
Column("country_id", Integer, primary_key=True),
Column("iso_3166_code_alpha3", String, nullable=True),
)

View File

View File

@@ -0,0 +1,9 @@
from sqlalchemy import Column, Integer, Table
from src.database.postgresql import pg_metadata
todo_table = Table(
"todo_table",
pg_metadata,
Column("id", Integer, primary_key=True),
)

View File

@@ -1,30 +0,0 @@
from typing import Annotated, Sequence
from fastapi import APIRouter, Depends
from pydantic import PositiveInt
from starlette.requests import Request
from src.dependencies.dependencies import get_messenger_handbook_country_repository, get_test_repository
from src.models.base import Messenger, MessengerHandbookCountry
from src.repositories.repository import MessengerHandbookCountryRepositoryPG, TestRepository
api_router = APIRouter(prefix="/v1", tags=["v1"])
@api_router.get("/test_psql/{test_id}")
async def get_info_from_postgresql(
request: Request,
test_repository: Annotated[
MessengerHandbookCountryRepositoryPG, Depends(get_messenger_handbook_country_repository)
],
test_id: PositiveInt,
) -> Sequence[MessengerHandbookCountry]:
return await test_repository.get_from_query(id_=test_id)
@api_router.get("/test_scylla")
async def get_info_from_scylla(
request: Request,
test_repository: Annotated[TestRepository, Depends(get_test_repository)],
) -> Sequence[Messenger]:
return await test_repository.get_from_query()

0
src/routes/__init__.py Normal file
View File

22
src/routes/v1.py Normal file
View File

@@ -0,0 +1,22 @@
from typing import Annotated, Sequence
from avroid_service_lib import AvroidAPIRouter
from fastapi import Depends
from fastapi.routing import APIRouter
from pydantic import PositiveInt
from starlette.requests import Request
from src.dependencies.dependencies import get_todo_repository
from src.models.base import TodoModel
from src.repositories.repository import TodoRepository
api_router: APIRouter = AvroidAPIRouter(prefix="/v1", tags=["v1"])
@api_router.get("/test_psql/{test_id}")
async def get_info_from_postgresql(
request: Request,
test_repository: Annotated[TodoRepository, Depends(get_todo_repository)],
test_id: PositiveInt,
) -> Sequence[TodoModel]:
return await test_repository.get_from_query(id_=test_id)

View File

@@ -1,24 +1,11 @@
from pydantic import PositiveInt
from pydantic_settings import BaseSettings
from avroid_service_lib import BaseAppSettings
class WebAppSettings(BaseSettings):
port: PositiveInt
postgres_host: str
postgres_port: int
postgres_user: str
postgres_db: str
postgres_password: str
scylladb_host: str
scylladb_port: PositiveInt
scylladb_user: str
scylladb_password: str
scylladb_keyspace: str
class WebAppSettings(BaseAppSettings):
postgres_dsn: str
class Config:
env_file = "local.env"
SERVICE_NAME = "avroid_service_template"
SERVICE_NAME = "template_backend_service"

0
src/utils/__init__.py Normal file
View File

View File

@@ -16,17 +16,8 @@ from src.settings import WebAppSettings
@pytest.fixture
def test_settings() -> WebAppSettings:
return WebAppSettings(
postgres_user="postgres",
postgres_password="postgres",
postgres_host="localhost",
postgres_db="postgres",
postgres_port=5432,
postgres_dsn="postgresql://messenger:messenger@localhost:5432/messenger_test",
port=8000,
scylladb_host="localhost",
scylladb_port="9042",
scylladb_user="test",
scylladb_password="test",
scylladb_keyspace="test",
)
@@ -65,8 +56,7 @@ def sa_enums():
@pytest.fixture(autouse=True)
async def db_engine(test_settings: WebAppSettings, sa_tables, sa_enums) -> AsyncGenerator[Engine, None]:
postgres_dsn = f"postgresql://{test_settings.postgres_user}:{test_settings.postgres_password}@{test_settings.postgres_host}:5432/{test_settings.postgres_db}"
async with create_engine(postgres_dsn) as engine:
async with create_engine(test_settings.postgres_dsn) as engine:
async with engine.acquire() as connection:
await drop_tables(connection)
await create_enums(connection, sa_enums)

0
tests/routes/__init__.py Normal file
View File

View File

View File

@@ -2,18 +2,18 @@ import pytest
from aiopg.sa import SAConnection
from httpx import AsyncClient
from src.repositories.tables import messenger_handbook_country_table
from src.repositories.tables.tables import todo_table
from tests.samples import ACCOUNT_1, ACCOUNT_2
@pytest.fixture
def sa_tables():
return [messenger_handbook_country_table]
return [todo_table]
@pytest.fixture(autouse=True)
async def _enter_data(connection: SAConnection):
await connection.execute(messenger_handbook_country_table.insert().values([ACCOUNT_1, ACCOUNT_2]))
await connection.execute(todo_table.insert().values([ACCOUNT_1, ACCOUNT_2]))
async def test_get_info_from_postgresql(test_client: AsyncClient):

View File

@@ -1,9 +1,7 @@
ACCOUNT_1 = {
"country_id": 1,
"iso_3166_code_alpha3": "RU",
"id": 1,
}
ACCOUNT_2 = {
"country_id": 2,
"iso_3166_code_alpha3": "EN",
"id": 2,
}