feat: word frequency endpoint with stop word removal

requirements.txt

@@ -14,7 +14,9 @@ httplib2==0.31.1
 idna==3.11
 itsdangerous==2.2.0
 Jinja2==3.1.6
+joblib==1.5.3
 MarkupSafe==3.0.3
+nltk==3.9.2
 numpy==2.4.1
 pandas==2.3.3
 proto-plus==1.27.0
@@ -27,10 +29,12 @@ pyparsing==3.3.1
 python-dateutil==2.9.0.post0
 python-dotenv==1.2.1
 pytz==2025.2
+regex==2026.1.15
 requests==2.32.5
 rsa==4.9.1
 six==1.17.0
 soupsieve==2.8.1
+tqdm==4.67.1
 typing_extensions==4.15.0
 tzdata==2025.3
 uritemplate==4.2.0

server/app.py

@@ -1,5 +1,7 @@
 from flask import Flask, jsonify, request
 from flask_cors import CORS
+import nltk
+from nltk.corpus import stopwords
 import pandas as pd
 
 app = Flask(__name__)
@@ -11,6 +13,9 @@ CORS(app, resources={r"/*": {"origins": "http://localhost:5173"}})
 posts_df = None
 comments_df = None
 
+nltk.download('stopwords')
+EXCLUDE_WORDS = set(stopwords.words('english'))
+
 @app.route('/upload', methods=['POST'])
 def upload_data():
     if "posts" not in request.files or "comments" not in request.files:
@@ -76,5 +81,30 @@ def comments_per_day():
 
     return jsonify(comments_per_day.to_dict(orient='records')), 200
 
+@app.route('/stats/word_frequencies', methods=['GET'])
+def word_frequencies():
+    if posts_df is None:
+        return jsonify({"error": "No data uploaded"}), 400
+    
+    try:
+        all_text = " ".join(posts_df['content'].fillna(''))
+        words = all_text.split()
+        word_freq = {}
+        for word in words:
+            clean_word = ''.join(c.lower() for c in word if c.isalnum())
+            if clean_word and clean_word not in EXCLUDE_WORDS:
+                word_freq[clean_word] = word_freq.get(clean_word, 0) + 1
+
+        sorted_words = sorted(word_freq.items(), key=lambda item: item[1], reverse=True)
+
+        # Get top 100 words and their frequencies and return as list of dicts
+        sorted_words = [{"word": word, "frequency": freq} for word, freq in sorted_words]
+    except ValueError as e:
+        return jsonify({"error": f"Malformed or missing data: {str(e)}"}), 400
+    except Exception as e:
+        return jsonify({"error": f"An unexpected error occurred: {str(e)}"}), 500
+
+    return jsonify(sorted_words[:100]), 200
+
 if __name__ == "__main__":
     app.run(debug=True)