Add script to orchestrate dataset creation
This commit is contained in:
15
fetch_data.py
Normal file
15
fetch_data.py
Normal file
@@ -0,0 +1,15 @@
|
||||
import json
|
||||
from connectors.reddit_connector import RedditConnector
|
||||
|
||||
data_file = 'reddit_posts.json'
|
||||
reddit_connector = RedditConnector()
|
||||
|
||||
def main():
|
||||
posts = reddit_connector.get_new_subreddit_posts('cork', limit=1000)
|
||||
print(f"Fetched {len(posts)} posts from r/cork")
|
||||
|
||||
with open(data_file, 'w') as f:
|
||||
json.dump([post.__dict__ for post in posts], f, indent=4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user