feat: combine post and comment uploads into a single file

server/app.py

@@ -12,33 +12,30 @@ app = Flask(__name__)
 CORS(app, resources={r"/*": {"origins": "http://localhost:5173"}})
 
 # Global State
-posts_df = pd.read_json('posts.jsonl', lines=True)
-comments_df = pd.read_json('comments.jsonl', lines=True)
+posts_df = pd.read_json('posts_test.jsonl', lines=True)
 with open("topic_buckets.json", "r", encoding="utf-8") as f:
     domain_topics = json.load(f)
-stat_obj = StatGen(posts_df, comments_df, domain_topics)
+stat_obj = StatGen(posts_df, domain_topics)
 
 @app.route('/upload', methods=['POST'])
 def upload_data():
-    if "posts" not in request.files or "comments" not in request.files or "topics" not in request.files:
+    if "posts" not in request.files or "topics" not in request.files:
         return jsonify({"error": "Missing required files or form data"}), 400
 
     post_file = request.files["posts"]
-    comment_file = request.files["comments"]
     topic_file = request.files["topics"]
 
-    if post_file.filename == "" or comment_file.filename == "" or topic_file == "":
+    if post_file.filename == "" or topic_file == "":
         return jsonify({"error": "Empty filename"}), 400
 
-    if not post_file.filename.endswith('.jsonl') or not comment_file.filename.endswith('.jsonl') or not topic_file.filename.endswith('.json'):
+    if not post_file.filename.endswith('.jsonl') or not topic_file.filename.endswith('.json'):
         return jsonify({"error": "Invalid file type. Only .jsonl and .json files are allowed."}), 400
     
     try:
         global stat_obj
 
         posts_df = pd.read_json(post_file, lines=True)
-        comments_df = pd.read_json(comment_file, lines=True)
-        stat_obj = StatGen(posts_df, comments_df, json.load(topic_file))
+        stat_obj = StatGen(posts_df, json.load(topic_file))
         return jsonify({"message": "File uploaded successfully", "event_count": len(stat_obj.df)}), 200
     except ValueError as e:
         return jsonify({"error": f"Failed to read JSONL file: {str(e)}"}), 400

server/nlp.py

@@ -1,12 +1,11 @@
 import torch
 import pandas as pd
 import numpy as np
-from typing import Any
 
+from typing import Any
 from transformers import pipeline
 from sentence_transformers import SentenceTransformer
 
-
 class NLP:
     _topic_models: dict[str, SentenceTransformer] = {}
     _emotion_classifiers: dict[str, Any] = {}

server/stat_gen.py

@@ -21,7 +21,11 @@ nltk.download('stopwords')
 EXCLUDE_WORDS = set(stopwords.words('english')) | DOMAIN_STOPWORDS
 
 class StatGen:
-    def __init__(self, posts_df: pd.DataFrame, comments_df: pd.DataFrame, domain_topics: dict) -> None:
+    def __init__(self, df: pd.DataFrame, domain_topics: dict) -> None:
+        comments_df = df[["id", "comments"]].explode("comments")
+        comments_df = pd.json_normalize(comments_df["comments"])
+
+        posts_df = df.drop(columns=["comments"])
         posts_df["type"] = "post"
         posts_df["parent_id"] = None
 
@@ -30,6 +34,7 @@ class StatGen:
         self.domain_topics = domain_topics
 
         self.df = pd.concat([posts_df, comments_df])
+        self.df.drop(columns=["post_id"], inplace=True, errors="ignore")
         self.nlp = NLP(self.df, "title", "content", domain_topics)
         self._add_extra_cols(self.df)
 
@@ -37,6 +42,7 @@ class StatGen:
 
     ## Private Methods
     def _add_extra_cols(self, df: pd.DataFrame) -> None:
+        df['timestamp'] = pd.to_numeric(self.df['timestamp'], errors='coerce')
         df['date'] = pd.to_datetime(df['timestamp'], unit='s').dt.date
         df["dt"] = pd.to_datetime(df["timestamp"], unit="s", utc=True)
         df["hour"] = df["dt"].dt.hour
@@ -165,7 +171,7 @@ class StatGen:
                 "start": int(self.df["dt"].min().timestamp()),
                 "end": int(self.df["dt"].max().timestamp())
             },
-            "sources": self.df["source"].unique().tolist()
+            "sources": self.df["source"].dropna().unique().tolist()
         }
 
     def content_analysis(self, limit: int = 100) -> dict: