diff --git a/connectors/base_connector.py b/connectors/base_connector.py deleted file mode 100644 index e7f4478..0000000 --- a/connectors/base_connector.py +++ /dev/null @@ -1,18 +0,0 @@ -from abc import abstractmethod -from abc import ABC -from dto.post import Post -from dto.user import User -from datetime import datetime - -class BaseConnector(ABC): - @abstractmethod - def get_top_posts(self, limit: int = 10, timeframe: str = 'day') -> list[Post]: - pass - - @abstractmethod - def search_posts(self, search: str, limit: int = 10, before = None, after = None) -> list[Post]: - pass - - @abstractmethod - def get_user(self, username: str) -> User: - pass \ No newline at end of file diff --git a/connectors/reddit_connector.py b/connectors/reddit_connector.py index 6556807..053576e 100644 --- a/connectors/reddit_connector.py +++ b/connectors/reddit_connector.py @@ -1,23 +1,13 @@ -from connectors.base_connector import BaseConnector from dto.post import Post from dto.user import User import requests -class RedditConnector(BaseConnector): +class RedditConnector: def __init__(self): self.url = "https://www.reddit.com/" self.source_name = "Reddit" # Public Methods # - def get_top_posts(self, limit: int = 10, timeframe: str = 'day') -> list[Post]: - params = { - 'limit': limit, - 't': timeframe - } - url = f"top.json" - data = self._fetch_data(url, params) - return self._parse_posts(data) - def get_top_subreddit_posts(self, subreddit: str, limit: int = 10, timeframe: str = 'day') -> list[Post]: params = { 'limit': limit, @@ -27,29 +17,6 @@ class RedditConnector(BaseConnector): data = self._fetch_data(url, params) return self._parse_posts(data) - def search_posts(self, - search: str, - limit: int = 10, - before = None, - after = None, - timeframe = 'day' - ) -> list[Post]: - params = { - 'q': search, - 'limit': limit, - 'before': before, - 'after': after, - 'sort': 'relevance', - 't': timeframe - } - url = f"search.json" - data = self._fetch_data(url, params) - return self._parse_posts(data) - - def get_user(self, username: str) -> User: - data = self._fetch_data(f"user/{username}/about.json", {}) - return self._parse_user(data) - def search_subreddit(self, search: str, subreddit: str, limit: int = 10, timeframe: str = "day") -> list[Post]: params = { 'q': search, @@ -62,6 +29,10 @@ class RedditConnector(BaseConnector): data = self._fetch_data(url, params) return self._parse_posts(data) + def get_user(self, username: str) -> User: + data = self._fetch_data(f"user/{username}/about.json", {}) + return self._parse_user(data) + ## Private Methods ## def _parse_posts(self, data) -> list[Post]: posts = [] diff --git a/server/app.py b/server/app.py index fcdcad8..917df87 100644 --- a/server/app.py +++ b/server/app.py @@ -8,18 +8,6 @@ db = Database(db_name='ethnograph', user='ethnograph_user', password='ethnograph reddit_connector = RedditConnector() -@app.route('/fetch_reddit//', methods=['GET']) -def index(search, limit = 10): - posts = reddit_connector.search_posts(search, limit=limit) - - db.execute_many( - """INSERT INTO ethnograph.posts (title, content, author_username, created_utc) - VALUES (%s, %s, %s, to_timestamp(%s));""", - [(post.title, post.content, post.author, post.timestamp) for post in posts] - ) - - return {"status": "success", "inserted_posts": len(posts)} - @app.route('/fetch_subreddit//', methods=['GET']) def fetch_subreddit(subreddit, limit = 10): posts = reddit_connector.get_top_subreddit_posts(subreddit, limit=limit, timeframe='all')