chore(connectors): reduce aggressive parallel connections to boards.ie
This commit is contained in:
@@ -88,7 +88,7 @@ class BoardsAPI(BaseConnector):
|
|||||||
post = self._parse_thread(html, post_url)
|
post = self._parse_thread(html, post_url)
|
||||||
return post
|
return post
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=30) as executor:
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||||
futures = {executor.submit(fetch_and_parse, url): url for url in urls}
|
futures = {executor.submit(fetch_and_parse, url): url for url in urls}
|
||||||
|
|
||||||
for i, future in enumerate(as_completed(futures)):
|
for i, future in enumerate(as_completed(futures)):
|
||||||
|
|||||||
Reference in New Issue
Block a user