dylan/crosspost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename fetch data script & add check for empty posts

Browse Source
This commit is contained in:
Dylan De Faoite
2026-01-13 19:06:00 +00:00
parent 538ea9fe12
commit b0e079599a
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fetch_data.py → create_dataset.py
View File

@@ -1,11 +1,16 @@
import json import json
from connectors.reddit_connector import RedditConnector from connectors.reddit_connector import RedditConnector
data_file = 'reddit_posts.json' data_file = 'data/reddit_posts.json'
reddit_connector = RedditConnector() reddit_connector = RedditConnector()
def remove_empty_posts(posts):
return [post for post in posts if post.content.strip() != ""]
def main(): def main():
posts = reddit_connector.get_new_subreddit_posts('cork', limit=1000) posts = reddit_connector.get_new_subreddit_posts('cork', limit=1000)
posts = remove_empty_posts(posts)
print(f"Fetched {len(posts)} posts from r/cork") print(f"Fetched {len(posts)} posts from r/cork")
with open(data_file, 'w') as f: with open(data_file, 'w') as f: