Refactor post detail fetching into separate _parse_thread method
This commit is contained in:
@@ -43,10 +43,21 @@ class BoardsAPI:
|
|||||||
# Fetch post details for each URL and create Post objects
|
# Fetch post details for each URL and create Post objects
|
||||||
posts = []
|
posts = []
|
||||||
|
|
||||||
for post_url in urls:
|
for index, post_url in enumerate(urls):
|
||||||
logger.debug(f"Fetching post details from URL: {post_url}")
|
logger.debug(f"Fetching Post {index + 1} / {len(urls)} details from URL: {post_url}")
|
||||||
html = self._fetch_page(post_url)
|
|
||||||
|
|
||||||
|
html = self._fetch_page(post_url)
|
||||||
|
post = self._parse_thread(html, post_url)
|
||||||
|
posts.append(post)
|
||||||
|
|
||||||
|
return posts
|
||||||
|
|
||||||
|
def _fetch_page(self, url: str) -> str:
|
||||||
|
response = requests.get(url, headers=HEADERS)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
|
||||||
|
def _parse_thread(self, html: str, post_url: str) -> Post:
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
|
||||||
# Author
|
# Author
|
||||||
@@ -61,7 +72,7 @@ class BoardsAPI:
|
|||||||
timestamp = match.group(0) if match else None
|
timestamp = match.group(0) if match else None
|
||||||
|
|
||||||
# Post ID
|
# Post ID
|
||||||
post_link = soup.select_one(".post-couunt .post-link")
|
post_link = soup.select_one(".post-count .post-link")
|
||||||
post_num = post_link.get_text(strip=True) if post_link else None
|
post_num = post_link.get_text(strip=True) if post_link else None
|
||||||
|
|
||||||
# Content
|
# Content
|
||||||
@@ -82,12 +93,6 @@ class BoardsAPI:
|
|||||||
source=self.source_name
|
source=self.source_name
|
||||||
)
|
)
|
||||||
|
|
||||||
posts.append(post)
|
return post
|
||||||
|
|
||||||
return posts
|
|
||||||
|
|
||||||
def _fetch_page(self, url: str) -> str:
|
|
||||||
response = requests.get(url, headers=HEADERS)
|
|
||||||
response.raise_for_status()
|
|
||||||
return response.text
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user