From 4b8aebd3120434b463cc7fce6eb1796509ba7244 Mon Sep 17 00:00:00 2001 From: Dylan De Faoite Date: Sun, 11 Jan 2026 15:07:44 +0000 Subject: [PATCH] add fetch_subreddit endpoint to retrieve and insert top posts from a specified subreddit --- connectors/reddit_connector.py | 9 +++++++++ server/app.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/connectors/reddit_connector.py b/connectors/reddit_connector.py index c392291..3a9c761 100644 --- a/connectors/reddit_connector.py +++ b/connectors/reddit_connector.py @@ -18,6 +18,15 @@ class RedditConnector(BaseConnector): data = self._fetch_data(url, params) return self._parse_posts(data) + def get_top_subreddit_posts(self, subreddit: str, limit: int = 10, timeframe: str = 'day') -> list[Post]: + params = { + 'limit': limit, + 't': timeframe + } + url = f"r/{subreddit}/top.json" + data = self._fetch_data(url, params) + return self._parse_posts(data) + def search_posts(self, search: str, limit: int = 10, diff --git a/server/app.py b/server/app.py index e2134b2..72aeebf 100644 --- a/server/app.py +++ b/server/app.py @@ -19,5 +19,20 @@ def index(search, limit = 10): return {"status": "success", "inserted_posts": len(posts)} +@app.route('/fetch_subreddit//', methods=['GET']) +def fetch_subreddit(subreddit, limit = 10): + posts = reddit_connector.get_top_subreddit_posts(subreddit, limit=limit, timeframe='all') + + for post in posts: + print(f"Post Title: {post.title}, Content: {post.content[:30]}...") + + db.execute_many( + """INSERT INTO ethnograph.posts (title, content, author_username, created_utc) + VALUES (%s, %s, %s, to_timestamp(%s));""", + [(post.title, post.content, post.author, post.timestamp) for post in posts] + ) + + return {"status": "success", "inserted_posts": len(posts)} + if __name__ == "__main__": app.run(debug=True) \ No newline at end of file