feat(backend): Add Celery + redis

backend-flask/Dockerfile

@@ -1,5 +1,5 @@
 # Use an official Python runtime as a parent image
-FROM python:3.11-slim
+FROM python:3.12-slim
 
 COPY requirements.txt /
 

backend-flask/app.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, make_response, request
 
-from app_db import app, connect_db_with_app
+from app_db import app, connect_celery_with_app, connect_db_with_app
 import login
 import schedule
 import team
@@ -9,6 +9,7 @@ import user
 import events
 
 connect_db_with_app()
+connect_celery_with_app()
 
 api = Blueprint("api", __name__, url_prefix="/api")
 api.register_blueprint(login.api_login)

backend-flask/app_db.py

@@ -1,3 +1,4 @@
+from os import environ
 from flask import Flask
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
@@ -15,12 +16,46 @@ convention = {
     "pk": "pk_%(table_name)s"
 }
 
-def connect_db_with_app():
+def connect_db_with_app(database_uri = "sqlite:///db.sqlite3", include_migrate=True):
-    app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite3"
+    print("Connecting to database: " + database_uri)
+    app.config["SQLALCHEMY_DATABASE_URI"] = database_uri
     db.init_app(app)
+    if include_migrate:
         migrate.init_app(app, db)
+    with app.app_context():
+        print("Running dialect: " + db.engine.dialect.name)
+    import models.match
+    import models.team_match
+    import models.player_match
+
+def connect_celery_with_app():
+    def celery_init_app(app):
+        from celery import Celery, Task
+        class FlaskTask(Task):
+            def __call__(self, *args: object, **kwargs: object) -> object:
+                with app.app_context():
+                    return self.run(*args, **kwargs)
+
+        celery_app = Celery(app.name, task_cls=FlaskTask, broker=app.config["CELERY"]["broker_url"])
+        celery_app.config_from_object(app.config["CELERY"])
+        celery_app.set_default()
+        app.extensions["celery"] = celery_app
+        return celery_app
+
+    app.config.from_mapping(
+        CELERY=dict(
+            broker_url=environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"),
+            result_backend=environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"),
+            task_ignore_result=True,
+        )
+    )
+    app.config.from_prefixed_env()
+    celery_init_app(app)
+
+def create_app() -> Flask:
+    return Flask(__name__)
 
 metadata = MetaData(naming_convention=convention)
-app = Flask(__name__)
+app = create_app()
 db = SQLAlchemy(model_class=BaseModel, metadata=metadata)
 migrate = Migrate(app, db, render_as_batch=True)

backend-flask/make_celery.py

@@ -0,0 +1,8 @@
+from app_db import connect_celery_with_app, app, connect_db_with_app
+
+connect_db_with_app(False)
+connect_celery_with_app()
+
+celery_app = app.extensions["celery"]
+
+import jobs.test_job

backend-flask/migrations/env.py

@@ -34,8 +34,8 @@ def get_engine_url():
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-# from myapp import mymodel
+from app_db import BaseModel
-# target_metadata = mymodel.Base.metadata
+target_metadata = BaseModel.metadata
 config.set_main_option('sqlalchemy.url', get_engine_url())
 target_db = current_app.extensions['migrate'].db
 

backend-flask/requirements.txt

@@ -9,9 +9,9 @@ Flask-SQLAlchemy
 SQLAlchemy-Utc
 
 # form/data validation
-pydantic
+pydantic==2.9.2
-spectree  # generates OpenAPI documents for us to make TypeScript API clients
+spectree==1.4.1  # generates OpenAPI documents for us to make TypeScript API
-          # based on our pydantic models
+                 # clients based on our pydantic models
 
 # DB migrations
 alembic
@@ -22,3 +22,5 @@ requests
 pytz  # timezone handling
 
 discord-webhook  # for sending messages to Discord webhooks
+
+celery[redis]

docker-compose.yml

@@ -4,23 +4,49 @@ services:
   # Flask service
   backend:
     container_name: backend
+    image: backend-flask
     build:
       context: ./backend-flask
-      #image: jazzdd/alpine-flask:python3
-    ports:
-      - ":5000"
     volumes:
       - ./backend-flask:/app
     networks:
       - dev-network
+    environment:
+      - FLASK_DEBUG=1
+      - FLASK_CELERY_BROKER_URL=redis://redis:6379/0
+      - FLASK_CELERY_RESULT_BACKEND=redis://redis:6379/0
+    depends_on:
+      - redis
+
+  # ETL job (runs with the same source as the backend)
+  celery-worker:
+    container_name: worker
+    command: celery -A make_celery.celery_app worker --loglevel=info --concurrency=1
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    image: backend-flask
+    volumes:
+      - ./backend-flask:/app
+    networks:
+      - dev-network
+    depends_on:
+      - redis
+
+  # message broker
+  redis:
+    image: redis:alpine
+    container_name: redis
+    networks:
+      - dev-network
+    ports:
+      - 6379:6379
 
   # Vue + Vite service
   frontend:
     container_name: frontend
     build:
       context: ./availabili.tf
-    ports:
-      - ":5173"
     environment:
       VITE_API_URL: http://localhost:8000  # API endpoint
     volumes: