Add script to orchestrate dataset creation
This commit is contained in:
15
fetch_data.py
Normal file
15
fetch_data.py
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
import json
|
||||||
|
from connectors.reddit_connector import RedditConnector
|
||||||
|
|
||||||
|
data_file = 'reddit_posts.json'
|
||||||
|
reddit_connector = RedditConnector()
|
||||||
|
|
||||||
|
def main():
|
||||||
|
posts = reddit_connector.get_new_subreddit_posts('cork', limit=1000)
|
||||||
|
print(f"Fetched {len(posts)} posts from r/cork")
|
||||||
|
|
||||||
|
with open(data_file, 'w') as f:
|
||||||
|
json.dump([post.__dict__ for post in posts], f, indent=4)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user