dylan/crosspost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(connectors): implement category validation in scraping process

Browse Source
This commit is contained in:
Dylan De Faoite
2026-03-14 16:59:43 +00:00
parent d96f459104
commit 8a423b2a29
5 changed files with 28 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
server/app.py
View File

@@ -157,15 +157,20 @@ def scrape_data():
return jsonify({"error": "Limit must be an integer"}), 400
name = source["name"]
category = source.get("category")
search = source.get("search")
if name not in connector_metadata:
return jsonify({"error": "Source not supported"}), 400
if "search" in source and not connector_metadata[name]["search_enabled"]:
if category and not connector_metadata[name]["search_enabled"]:
return jsonify({"error": f"Source {name} does not support search"}), 400
if "category" in source and not connector_metadata[name]["categories_enabled"]:
if category and not connector_metadata[name]["categories_enabled"]:
return jsonify({"error": f"Source {name} does not support categories"}), 400
if category and not connectors[name]().category_exists(category):
return jsonify({"error": f"Category does not exist for {name}"}), 400
try:
dataset_id = dataset_manager.save_dataset_info(