From d4fb78aac4622127618dc04af1b3b32d5009a2e7 Mon Sep 17 00:00:00 2001 From: Dylan De Faoite Date: Tue, 13 Jan 2026 18:46:43 +0000 Subject: [PATCH] Add pagination to new_subreddit method to bypass 100 post limit --- connectors/reddit_connector.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/connectors/reddit_connector.py b/connectors/reddit_connector.py index f255ef0..a05ccc5 100644 --- a/connectors/reddit_connector.py +++ b/connectors/reddit_connector.py @@ -30,12 +30,30 @@ class RedditConnector: return self._parse_posts(data) def get_new_subreddit_posts(self, subreddit: str, limit: int = 10) -> list[Post]: - params = { - 'limit': limit - } + + posts = [] + after = None url = f"r/{subreddit}/new.json" - data = self._fetch_data(url, params) - return self._parse_posts(data) + + while len(posts) < limit: + batch_limit = min(100, limit - len(posts)) + params = { + 'limit': batch_limit, + 'after': after + } + + data = self._fetch_data(url, params) + batch = self._parse_posts(data) + + if not batch: + break + + posts.extend(batch) + after = data['data'].get('after') + if not after: + break + + return posts def get_user(self, username: str) -> User: data = self._fetch_data(f"user/{username}/about.json", {})