Refactor post detail fetching into separate _parse_thread method
This commit is contained in:
@@ -43,10 +43,21 @@ class BoardsAPI:
|
||||
# Fetch post details for each URL and create Post objects
|
||||
posts = []
|
||||
|
||||
for post_url in urls:
|
||||
logger.debug(f"Fetching post details from URL: {post_url}")
|
||||
html = self._fetch_page(post_url)
|
||||
for index, post_url in enumerate(urls):
|
||||
logger.debug(f"Fetching Post {index + 1} / {len(urls)} details from URL: {post_url}")
|
||||
|
||||
html = self._fetch_page(post_url)
|
||||
post = self._parse_thread(html, post_url)
|
||||
posts.append(post)
|
||||
|
||||
return posts
|
||||
|
||||
def _fetch_page(self, url: str) -> str:
|
||||
response = requests.get(url, headers=HEADERS)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
def _parse_thread(self, html: str, post_url: str) -> Post:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Author
|
||||
@@ -61,7 +72,7 @@ class BoardsAPI:
|
||||
timestamp = match.group(0) if match else None
|
||||
|
||||
# Post ID
|
||||
post_link = soup.select_one(".post-couunt .post-link")
|
||||
post_link = soup.select_one(".post-count .post-link")
|
||||
post_num = post_link.get_text(strip=True) if post_link else None
|
||||
|
||||
# Content
|
||||
@@ -82,12 +93,6 @@ class BoardsAPI:
|
||||
source=self.source_name
|
||||
)
|
||||
|
||||
posts.append(post)
|
||||
return post
|
||||
|
||||
return posts
|
||||
|
||||
def _fetch_page(self, url: str) -> str:
|
||||
response = requests.get(url, headers=HEADERS)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
||||
Reference in New Issue
Block a user