dylan/crosspost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: increase post and comment fetch limits

Browse Source
This commit is contained in:
Dylan De Faoite
2026-02-09 21:20:57 +00:00
parent ec91904481
commit cc12f97c20
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
create_dataset.py
View File

@@ -25,18 +25,18 @@ def save_to_jsonl(filename, posts):
def main(): def main():
boards_posts = boards_connector.get_new_category_posts('cork-city', 10, 10) boards_posts = boards_connector.get_new_category_posts('cork-city', 1200, 1200)
save_to_jsonl(posts_file, boards_posts) save_to_jsonl(posts_file, boards_posts)
reddit_posts = reddit_connector.get_new_subreddit_posts('cork', 10) reddit_posts = reddit_connector.get_new_subreddit_posts('cork', 1200)
reddit_posts = remove_empty_posts(reddit_posts) reddit_posts = remove_empty_posts(reddit_posts)
save_to_jsonl(posts_file, reddit_posts) save_to_jsonl(posts_file, reddit_posts)
ireland_posts = reddit_connector.search_new_subreddit_posts('cork', 'ireland', 10) ireland_posts = reddit_connector.search_new_subreddit_posts('cork', 'ireland', 1200)
ireland_posts = remove_empty_posts(ireland_posts) ireland_posts = remove_empty_posts(ireland_posts)
save_to_jsonl(posts_file, ireland_posts) save_to_jsonl(posts_file, ireland_posts)
youtube_videos = youtube_connector.fetch_videos('cork city', 10, 10) youtube_videos = youtube_connector.fetch_videos('cork city', 1200, 1200)
save_to_jsonl(posts_file, youtube_videos) save_to_jsonl(posts_file, youtube_videos)
if __name__ == "__main__": if __name__ == "__main__":