dylan/crosspost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Automatic Scraping of dataset options #9

Merged
dylan merged 36 commits from feat/automatic-scraping-datasets into main 2026-03-14 21:58:49 +00:00
Conversation 0 Commits 36 Files Changed 20 +6 -5
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 524c9c50a0 - Show all commits

11
server/app.py
View File

@@ -144,11 +144,12 @@ def scrape_data():
source["limit"] = int(source["limit"]) source["limit"] = int(source["limit"])
dataset_id = dataset_manager.save_dataset_info(user_id, dataset_name, default_topic_list) dataset_id = dataset_manager.save_dataset_info(user_id, dataset_name, default_topic_list)
dataset_manager.set_dataset_status(dataset_id, dataset_manager.set_dataset_status(
"fetching", dataset_id,
f"Data is being fetched from {str(source["name"] + "," for source in source_configs)}" "fetching",
) f"Data is being fetched from {', '.join(source['name'] for source in source_configs)}"
)
try: try:
fetch_and_process_dataset.delay(dataset_id, source_configs, default_topic_list) fetch_and_process_dataset.delay(dataset_id, source_configs, default_topic_list)