feat(connectors): add base connector and registry for detection
Idea is to have a "plugin-type" system, where new connectors can extend the `BaseConnector` class and implement the fetch posts method. These are automatically detected by the registry, and automatically used in new Flask endpoints that give a list of possible sources. Allows for an open-ended system where new data scrapers / API consumers can be added dynamically.
This commit is contained in:
@@ -7,6 +7,7 @@ from dto.post import Post
|
||||
from dto.comment import Comment
|
||||
from bs4 import BeautifulSoup
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from server.connectors.base import BaseConnector
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -14,12 +15,18 @@ HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; ForumScraper/1.0)"
|
||||
}
|
||||
|
||||
class BoardsAPI:
|
||||
class BoardsAPI(BaseConnector):
|
||||
def __init__(self):
|
||||
self.url = "https://www.boards.ie"
|
||||
self.source_name = "Boards.ie"
|
||||
self.source_name = "boards.ie"
|
||||
self.display_name = "Boards.ie"
|
||||
|
||||
def get_new_category_posts(self, category: str, post_limit: int, comment_limit: int) -> list[Post]:
|
||||
def get_new_posts_by_search(self,
|
||||
search: str,
|
||||
category: str,
|
||||
post_limit: int,
|
||||
comment_limit: int
|
||||
) -> list[Post]:
|
||||
urls = []
|
||||
current_page = 1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user