diff --git a/requirements.txt b/requirements.txt index a6eb030..22422d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,9 @@ httplib2==0.31.1 idna==3.11 itsdangerous==2.2.0 Jinja2==3.1.6 +joblib==1.5.3 MarkupSafe==3.0.3 +nltk==3.9.2 numpy==2.4.1 pandas==2.3.3 proto-plus==1.27.0 @@ -27,10 +29,12 @@ pyparsing==3.3.1 python-dateutil==2.9.0.post0 python-dotenv==1.2.1 pytz==2025.2 +regex==2026.1.15 requests==2.32.5 rsa==4.9.1 six==1.17.0 soupsieve==2.8.1 +tqdm==4.67.1 typing_extensions==4.15.0 tzdata==2025.3 uritemplate==4.2.0 diff --git a/server/app.py b/server/app.py index f97e2d6..175cdb8 100644 --- a/server/app.py +++ b/server/app.py @@ -1,5 +1,7 @@ from flask import Flask, jsonify, request from flask_cors import CORS +import nltk +from nltk.corpus import stopwords import pandas as pd app = Flask(__name__) @@ -11,6 +13,9 @@ CORS(app, resources={r"/*": {"origins": "http://localhost:5173"}}) posts_df = None comments_df = None +nltk.download('stopwords') +EXCLUDE_WORDS = set(stopwords.words('english')) + @app.route('/upload', methods=['POST']) def upload_data(): if "posts" not in request.files or "comments" not in request.files: @@ -76,5 +81,30 @@ def comments_per_day(): return jsonify(comments_per_day.to_dict(orient='records')), 200 +@app.route('/stats/word_frequencies', methods=['GET']) +def word_frequencies(): + if posts_df is None: + return jsonify({"error": "No data uploaded"}), 400 + + try: + all_text = " ".join(posts_df['content'].fillna('')) + words = all_text.split() + word_freq = {} + for word in words: + clean_word = ''.join(c.lower() for c in word if c.isalnum()) + if clean_word and clean_word not in EXCLUDE_WORDS: + word_freq[clean_word] = word_freq.get(clean_word, 0) + 1 + + sorted_words = sorted(word_freq.items(), key=lambda item: item[1], reverse=True) + + # Get top 100 words and their frequencies and return as list of dicts + sorted_words = [{"word": word, "frequency": freq} for word, freq in sorted_words] + except ValueError as e: + return jsonify({"error": f"Malformed or missing data: {str(e)}"}), 400 + except Exception as e: + return jsonify({"error": f"An unexpected error occurred: {str(e)}"}), 500 + + return jsonify(sorted_words[:100]), 200 + if __name__ == "__main__": app.run(debug=True) \ No newline at end of file