feat: add celery & redis for background data processing

server/queue/celery_app.py

@@ -0,0 +1,16 @@
+from celery import Celery
+
+def create_celery():
+    celery = Celery(
+        "ethnograph",
+        broker="redis://redis:6379/0",
+        backend="redis://redis:6379/0",
+    )
+    celery.conf.task_serializer = "json"
+    celery.conf.result_serializer = "json"
+    celery.conf.accept_content = ["json"]
+    return celery
+
+celery = create_celery()
+
+from server.queue import tasks

server/queue/tasks.py

@@ -0,0 +1,19 @@
+import pandas as pd
+
+from server.queue.celery_app import celery
+from server.analysis.enrichment import DatasetEnrichment
+
+@celery.task(bind=True, max_retries=3)
+def process_dataset(self, dataset_id: int, posts: list, topics: dict):
+    from server.db.database import PostgresConnector
+    from server.core.datasets import DatasetManager
+
+    db = PostgresConnector()
+    dataset_manager = DatasetManager(db)
+
+    df = pd.DataFrame(posts)
+
+    processor = DatasetEnrichment(df, topics)
+    enriched_df = processor.enrich()
+
+    dataset_manager.save_dataset_content(dataset_id, enriched_df)