feat(connectors): add base connector and registry for detection

Idea is to have a "plugin-type" system, where new connectors can extend the `BaseConnector` class and implement the fetch posts method.

These are automatically detected by the registry, and automatically used in new Flask endpoints that give a list of possible sources.

Allows for an open-ended system where new data scrapers / API consumers can be added dynamically.
This commit is contained in:
2026-03-09 21:28:44 +00:00
parent 262a70dbf3
commit cc799f7368
5 changed files with 73 additions and 4 deletions

View File

@@ -7,6 +7,7 @@ from dto.post import Post
from dto.comment import Comment
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed
from server.connectors.base import BaseConnector
logger = logging.getLogger(__name__)
@@ -14,12 +15,18 @@ HEADERS = {
"User-Agent": "Mozilla/5.0 (compatible; ForumScraper/1.0)"
}
class BoardsAPI:
class BoardsAPI(BaseConnector):
def __init__(self):
self.url = "https://www.boards.ie"
self.source_name = "Boards.ie"
self.source_name = "boards.ie"
self.display_name = "Boards.ie"
def get_new_category_posts(self, category: str, post_limit: int, comment_limit: int) -> list[Post]:
def get_new_posts_by_search(self,
search: str,
category: str,
post_limit: int,
comment_limit: int
) -> list[Post]:
urls = []
current_page = 1