remove base_connector and remove non-subreddit specific methods
Project will focus on specific communities, not enact a reddit-wide search
This commit is contained in:
@@ -1,18 +0,0 @@
|
|||||||
from abc import abstractmethod
|
|
||||||
from abc import ABC
|
|
||||||
from dto.post import Post
|
|
||||||
from dto.user import User
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
class BaseConnector(ABC):
|
|
||||||
@abstractmethod
|
|
||||||
def get_top_posts(self, limit: int = 10, timeframe: str = 'day') -> list[Post]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def search_posts(self, search: str, limit: int = 10, before = None, after = None) -> list[Post]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_user(self, username: str) -> User:
|
|
||||||
pass
|
|
||||||
@@ -1,23 +1,13 @@
|
|||||||
from connectors.base_connector import BaseConnector
|
|
||||||
from dto.post import Post
|
from dto.post import Post
|
||||||
from dto.user import User
|
from dto.user import User
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
class RedditConnector(BaseConnector):
|
class RedditConnector:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.url = "https://www.reddit.com/"
|
self.url = "https://www.reddit.com/"
|
||||||
self.source_name = "Reddit"
|
self.source_name = "Reddit"
|
||||||
|
|
||||||
# Public Methods #
|
# Public Methods #
|
||||||
def get_top_posts(self, limit: int = 10, timeframe: str = 'day') -> list[Post]:
|
|
||||||
params = {
|
|
||||||
'limit': limit,
|
|
||||||
't': timeframe
|
|
||||||
}
|
|
||||||
url = f"top.json"
|
|
||||||
data = self._fetch_data(url, params)
|
|
||||||
return self._parse_posts(data)
|
|
||||||
|
|
||||||
def get_top_subreddit_posts(self, subreddit: str, limit: int = 10, timeframe: str = 'day') -> list[Post]:
|
def get_top_subreddit_posts(self, subreddit: str, limit: int = 10, timeframe: str = 'day') -> list[Post]:
|
||||||
params = {
|
params = {
|
||||||
'limit': limit,
|
'limit': limit,
|
||||||
@@ -27,29 +17,6 @@ class RedditConnector(BaseConnector):
|
|||||||
data = self._fetch_data(url, params)
|
data = self._fetch_data(url, params)
|
||||||
return self._parse_posts(data)
|
return self._parse_posts(data)
|
||||||
|
|
||||||
def search_posts(self,
|
|
||||||
search: str,
|
|
||||||
limit: int = 10,
|
|
||||||
before = None,
|
|
||||||
after = None,
|
|
||||||
timeframe = 'day'
|
|
||||||
) -> list[Post]:
|
|
||||||
params = {
|
|
||||||
'q': search,
|
|
||||||
'limit': limit,
|
|
||||||
'before': before,
|
|
||||||
'after': after,
|
|
||||||
'sort': 'relevance',
|
|
||||||
't': timeframe
|
|
||||||
}
|
|
||||||
url = f"search.json"
|
|
||||||
data = self._fetch_data(url, params)
|
|
||||||
return self._parse_posts(data)
|
|
||||||
|
|
||||||
def get_user(self, username: str) -> User:
|
|
||||||
data = self._fetch_data(f"user/{username}/about.json", {})
|
|
||||||
return self._parse_user(data)
|
|
||||||
|
|
||||||
def search_subreddit(self, search: str, subreddit: str, limit: int = 10, timeframe: str = "day") -> list[Post]:
|
def search_subreddit(self, search: str, subreddit: str, limit: int = 10, timeframe: str = "day") -> list[Post]:
|
||||||
params = {
|
params = {
|
||||||
'q': search,
|
'q': search,
|
||||||
@@ -62,6 +29,10 @@ class RedditConnector(BaseConnector):
|
|||||||
data = self._fetch_data(url, params)
|
data = self._fetch_data(url, params)
|
||||||
return self._parse_posts(data)
|
return self._parse_posts(data)
|
||||||
|
|
||||||
|
def get_user(self, username: str) -> User:
|
||||||
|
data = self._fetch_data(f"user/{username}/about.json", {})
|
||||||
|
return self._parse_user(data)
|
||||||
|
|
||||||
## Private Methods ##
|
## Private Methods ##
|
||||||
def _parse_posts(self, data) -> list[Post]:
|
def _parse_posts(self, data) -> list[Post]:
|
||||||
posts = []
|
posts = []
|
||||||
|
|||||||
@@ -8,18 +8,6 @@ db = Database(db_name='ethnograph', user='ethnograph_user', password='ethnograph
|
|||||||
|
|
||||||
reddit_connector = RedditConnector()
|
reddit_connector = RedditConnector()
|
||||||
|
|
||||||
@app.route('/fetch_reddit/<string:search>/<int:limit>', methods=['GET'])
|
|
||||||
def index(search, limit = 10):
|
|
||||||
posts = reddit_connector.search_posts(search, limit=limit)
|
|
||||||
|
|
||||||
db.execute_many(
|
|
||||||
"""INSERT INTO ethnograph.posts (title, content, author_username, created_utc)
|
|
||||||
VALUES (%s, %s, %s, to_timestamp(%s));""",
|
|
||||||
[(post.title, post.content, post.author, post.timestamp) for post in posts]
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"status": "success", "inserted_posts": len(posts)}
|
|
||||||
|
|
||||||
@app.route('/fetch_subreddit/<string:subreddit>/<int:limit>', methods=['GET'])
|
@app.route('/fetch_subreddit/<string:subreddit>/<int:limit>', methods=['GET'])
|
||||||
def fetch_subreddit(subreddit, limit = 10):
|
def fetch_subreddit(subreddit, limit = 10):
|
||||||
posts = reddit_connector.get_top_subreddit_posts(subreddit, limit=limit, timeframe='all')
|
posts = reddit_connector.get_top_subreddit_posts(subreddit, limit=limit, timeframe='all')
|
||||||
|
|||||||
Reference in New Issue
Block a user