dylan/crosspost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Automatic Scraping of dataset options #9

Merged
dylan merged 36 commits from feat/automatic-scraping-datasets into main 2026-03-14 21:58:49 +00:00
Conversation 0 Commits 36 Files Changed 20 +11 -3
1 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit dc330b87b9 - Show all commits

14
server/queue/tasks.py
View File

@@ -1,4 +1,5 @@
import pandas as pd import pandas as pd
import json
from server.queue.celery_app import celery from server.queue.celery_app import celery
from server.analysis.enrichment import DatasetEnrichment from server.analysis.enrichment import DatasetEnrichment
@@ -37,13 +38,20 @@ def fetch_and_process_dataset(self,
try: try:
for source_name, source_limit in per_source.items(): for source_name, source_limit in per_source.items():
connector = connectors[source_name]() connector = connectors[source_name]()
posts.extend(connector.get_new_posts_by_search( raw_posts = connector.get_new_posts_by_search(
search=search, search=search,
category=category, category=category,
post_limit=source_limit, post_limit=source_limit,
comment_limit=source_limit comment_limit=source_limit
)) )
posts.extend(post.to_dict() for post in raw_posts)
process_dataset.delay(dataset_id, [p.to_dict() for p in posts], topics) df = pd.DataFrame(posts)
processor = DatasetEnrichment(df, topics)
enriched_df = processor.enrich()
dataset_manager.save_dataset_content(dataset_id, enriched_df)
dataset_manager.set_dataset_status(dataset_id, "complete", "NLP Processing Completed Successfully")
except Exception as e: except Exception as e:
dataset_manager.set_dataset_status(dataset_id, "error", f"An error occurred: {e}") dataset_manager.set_dataset_status(dataset_id, "error", f"An error occurred: {e}")