Refactor post detail fetching into separate _parse_thread method

This commit is contained in:
2026-01-17 14:51:57 +00:00
parent 610bab67d5
commit d5e6b7a895

View File

@@ -43,10 +43,21 @@ class BoardsAPI:
# Fetch post details for each URL and create Post objects
posts = []
for post_url in urls:
logger.debug(f"Fetching post details from URL: {post_url}")
html = self._fetch_page(post_url)
for index, post_url in enumerate(urls):
logger.debug(f"Fetching Post {index + 1} / {len(urls)} details from URL: {post_url}")
html = self._fetch_page(post_url)
post = self._parse_thread(html, post_url)
posts.append(post)
return posts
def _fetch_page(self, url: str) -> str:
response = requests.get(url, headers=HEADERS)
response.raise_for_status()
return response.text
def _parse_thread(self, html: str, post_url: str) -> Post:
soup = BeautifulSoup(html, "html.parser")
# Author
@@ -61,7 +72,7 @@ class BoardsAPI:
timestamp = match.group(0) if match else None
# Post ID
post_link = soup.select_one(".post-couunt .post-link")
post_link = soup.select_one(".post-count .post-link")
post_num = post_link.get_text(strip=True) if post_link else None
# Content
@@ -82,12 +93,6 @@ class BoardsAPI:
source=self.source_name
)
posts.append(post)
return post
return posts
def _fetch_page(self, url: str) -> str:
response = requests.get(url, headers=HEADERS)
response.raise_for_status()
return response.text