From aaa4d40ed9dfdae2ac99356e180c0b13c6afd1b7 Mon Sep 17 00:00:00 2001 From: HumanoidSandvichDispenser Date: Mon, 9 Dec 2024 17:03:43 -0800 Subject: [PATCH] feat(backend): Add Celery + redis --- backend-flask/Dockerfile | 2 +- backend-flask/app.py | 3 ++- backend-flask/app_db.py | 43 ++++++++++++++++++++++++++++++--- backend-flask/make_celery.py | 8 ++++++ backend-flask/migrations/env.py | 4 +-- backend-flask/requirements.txt | 8 +++--- docker-compose.yml | 36 +++++++++++++++++++++++---- 7 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 backend-flask/make_celery.py diff --git a/backend-flask/Dockerfile b/backend-flask/Dockerfile index a726e99..f58e9e7 100644 --- a/backend-flask/Dockerfile +++ b/backend-flask/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM python:3.11-slim +FROM python:3.12-slim COPY requirements.txt / diff --git a/backend-flask/app.py b/backend-flask/app.py index 7b766a5..803776c 100644 --- a/backend-flask/app.py +++ b/backend-flask/app.py @@ -1,6 +1,6 @@ from flask import Blueprint, make_response, request -from app_db import app, connect_db_with_app +from app_db import app, connect_celery_with_app, connect_db_with_app import login import schedule import team @@ -9,6 +9,7 @@ import user import events connect_db_with_app() +connect_celery_with_app() api = Blueprint("api", __name__, url_prefix="/api") api.register_blueprint(login.api_login) diff --git a/backend-flask/app_db.py b/backend-flask/app_db.py index 400b433..19add30 100644 --- a/backend-flask/app_db.py +++ b/backend-flask/app_db.py @@ -1,3 +1,4 @@ +from os import environ from flask import Flask from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy @@ -15,12 +16,46 @@ convention = { "pk": "pk_%(table_name)s" } -def connect_db_with_app(): - app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite3" +def connect_db_with_app(database_uri = "sqlite:///db.sqlite3", include_migrate=True): + print("Connecting to database: " + database_uri) + app.config["SQLALCHEMY_DATABASE_URI"] = database_uri db.init_app(app) - migrate.init_app(app, db) + if include_migrate: + migrate.init_app(app, db) + with app.app_context(): + print("Running dialect: " + db.engine.dialect.name) + import models.match + import models.team_match + import models.player_match + +def connect_celery_with_app(): + def celery_init_app(app): + from celery import Celery, Task + class FlaskTask(Task): + def __call__(self, *args: object, **kwargs: object) -> object: + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name, task_cls=FlaskTask, broker=app.config["CELERY"]["broker_url"]) + celery_app.config_from_object(app.config["CELERY"]) + celery_app.set_default() + app.extensions["celery"] = celery_app + return celery_app + + app.config.from_mapping( + CELERY=dict( + broker_url=environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"), + result_backend=environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"), + task_ignore_result=True, + ) + ) + app.config.from_prefixed_env() + celery_init_app(app) + +def create_app() -> Flask: + return Flask(__name__) metadata = MetaData(naming_convention=convention) -app = Flask(__name__) +app = create_app() db = SQLAlchemy(model_class=BaseModel, metadata=metadata) migrate = Migrate(app, db, render_as_batch=True) diff --git a/backend-flask/make_celery.py b/backend-flask/make_celery.py new file mode 100644 index 0000000..3bebe0c --- /dev/null +++ b/backend-flask/make_celery.py @@ -0,0 +1,8 @@ +from app_db import connect_celery_with_app, app, connect_db_with_app + +connect_db_with_app(False) +connect_celery_with_app() + +celery_app = app.extensions["celery"] + +import jobs.test_job diff --git a/backend-flask/migrations/env.py b/backend-flask/migrations/env.py index 4c97092..da5b0ed 100644 --- a/backend-flask/migrations/env.py +++ b/backend-flask/migrations/env.py @@ -34,8 +34,8 @@ def get_engine_url(): # add your model's MetaData object here # for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata +from app_db import BaseModel +target_metadata = BaseModel.metadata config.set_main_option('sqlalchemy.url', get_engine_url()) target_db = current_app.extensions['migrate'].db diff --git a/backend-flask/requirements.txt b/backend-flask/requirements.txt index cb36873..7ede081 100644 --- a/backend-flask/requirements.txt +++ b/backend-flask/requirements.txt @@ -9,9 +9,9 @@ Flask-SQLAlchemy SQLAlchemy-Utc # form/data validation -pydantic -spectree # generates OpenAPI documents for us to make TypeScript API clients - # based on our pydantic models +pydantic==2.9.2 +spectree==1.4.1 # generates OpenAPI documents for us to make TypeScript API + # clients based on our pydantic models # DB migrations alembic @@ -22,3 +22,5 @@ requests pytz # timezone handling discord-webhook # for sending messages to Discord webhooks + +celery[redis] diff --git a/docker-compose.yml b/docker-compose.yml index 312d849..2001fb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,23 +4,49 @@ services: # Flask service backend: container_name: backend + image: backend-flask build: context: ./backend-flask - #image: jazzdd/alpine-flask:python3 - ports: - - ":5000" volumes: - ./backend-flask:/app networks: - dev-network + environment: + - FLASK_DEBUG=1 + - FLASK_CELERY_BROKER_URL=redis://redis:6379/0 + - FLASK_CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - redis + + # ETL job (runs with the same source as the backend) + celery-worker: + container_name: worker + command: celery -A make_celery.celery_app worker --loglevel=info --concurrency=1 + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + image: backend-flask + volumes: + - ./backend-flask:/app + networks: + - dev-network + depends_on: + - redis + + # message broker + redis: + image: redis:alpine + container_name: redis + networks: + - dev-network + ports: + - 6379:6379 # Vue + Vite service frontend: container_name: frontend build: context: ./availabili.tf - ports: - - ":5173" environment: VITE_API_URL: http://localhost:8000 # API endpoint volumes: