From d7baf39087ab3b28fcd022c6f1009ce334024e6a Mon Sep 17 00:00:00 2001 From: Dylan De Faoite Date: Sat, 17 Jan 2026 22:14:26 +0000 Subject: [PATCH] Implement exponential backoff for handling Reddit API rate limits in _fetch_data method --- connectors/reddit_api.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/connectors/reddit_api.py b/connectors/reddit_api.py index faa4686..661e3fb 100644 --- a/connectors/reddit_api.py +++ b/connectors/reddit_api.py @@ -1,5 +1,6 @@ import requests import logging +import time from dto.post import Post from dto.user import User @@ -145,10 +146,24 @@ class RedditAPI: def _fetch_data(self, endpoint: str, params: dict) -> dict: url = f"{self.url}{endpoint}" - try: - response = requests.get(url, headers={'User-agent': 'python:ethnography-college-project:0.1 (by /u/ThisBirchWood)'}, params=params) - response.raise_for_status() - return response.json() - except requests.RequestException as e: - print(f"Error fetching data from Reddit API: {e}") - return {} \ No newline at end of file + max_retries = 15 + backoff = 1 # seconds + + for attempt in range(max_retries): + try: + response = requests.get(url, headers={'User-agent': 'python:ethnography-college-project:0.1 (by /u/ThisBirchWood)'}, params=params) + + if response.status_code == 429: + wait_time = response.headers.get("Retry-After", backoff) + + logger.warning(f"Rate limited by Reddit API. Retrying in {wait_time} seconds...") + + time.sleep(wait_time) + backoff *= 2 + continue + + response.raise_for_status() + return response.json() + except requests.RequestException as e: + print(f"Error fetching data from Reddit API: {e}") + return {} \ No newline at end of file