diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1b10f61..3b80cf9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,7 +5,7 @@ import DatasetsPage from "./pages/Datasets"; import DatasetStatusPage from "./pages/DatasetStatus"; import LoginPage from "./pages/Login"; import UploadPage from "./pages/Upload"; -import AutoScrapePage from "./pages/AutoScrape"; +import AutoFetchPage from "./pages/AutoFetch"; import StatPage from "./pages/Stats"; import { getDocumentTitle } from "./utils/documentTitle"; import DatasetEditPage from "./pages/DatasetEdit"; @@ -23,7 +23,7 @@ function App() { } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/frontend/src/pages/AutoScrape.tsx b/frontend/src/pages/AutoFetch.tsx similarity index 95% rename from frontend/src/pages/AutoScrape.tsx rename to frontend/src/pages/AutoFetch.tsx index 9697fec..08a6b7e 100644 --- a/frontend/src/pages/AutoScrape.tsx +++ b/frontend/src/pages/AutoFetch.tsx @@ -37,7 +37,7 @@ const supportsSearch = (source?: SourceOption): boolean => const supportsCategories = (source?: SourceOption): boolean => Boolean(source?.categories_enabled ?? source?.categoriesEnabled); -const AutoScrapePage = () => { +const AutoFetchPage = () => { const navigate = useNavigate(); const [datasetName, setDatasetName] = useState(""); const [sourceOptions, setSourceOptions] = useState([]); @@ -106,11 +106,11 @@ const AutoScrapePage = () => { ); }; - const autoScrape = async () => { + const autoFetch = async () => { const token = localStorage.getItem("access_token"); if (!token) { setHasError(true); - setReturnMessage("You must be signed in to auto scrape a dataset."); + setReturnMessage("You must be signed in to auto fetch a dataset."); return; } @@ -243,7 +243,7 @@ const AutoScrapePage = () => { setReturnMessage(""); const response = await axios.post( - `${API_BASE_URL}/datasets/scrape`, + `${API_BASE_URL}/datasets/fetch`, requestBody, { headers: { @@ -255,7 +255,7 @@ const AutoScrapePage = () => { const datasetId = Number(response.data.dataset_id); setReturnMessage( - `Auto scrape queued successfully (dataset #${datasetId}). Redirecting to processing status...`, + `Auto fetch queued successfully (dataset #${datasetId}). Redirecting to processing status...`, ); setTimeout(() => { @@ -267,11 +267,11 @@ const AutoScrapePage = () => { const message = String( requestError.response?.data?.error || requestError.message || - "Auto scrape failed.", + "Auto fetch failed.", ); - setReturnMessage(`Auto scrape failed: ${message}`); + setReturnMessage(`Auto fetch failed: ${message}`); } else { - setReturnMessage("Auto scrape failed due to an unexpected error."); + setReturnMessage("Auto fetch failed due to an unexpected error."); } } finally { setIsSubmitting(false); @@ -283,9 +283,9 @@ const AutoScrapePage = () => {
-

Auto Scrape Dataset

+

Auto Fetch Dataset

- Select sources and scrape settings, then queue processing + Select sources and fetch settings, then queue processing automatically.

{ color: "#9a6700", }} > - Warning: Scraping more than 250 posts from any single site can + Warning: Fetching more than 250 posts from any single site can take hours due to rate limits.

@@ -305,10 +305,10 @@ const AutoScrapePage = () => { ...styles.buttonPrimary, opacity: isSubmitting || isLoadingSources ? 0.75 : 1, }} - onClick={autoScrape} + onClick={autoFetch} disabled={isSubmitting || isLoadingSources} > - {isSubmitting ? "Queueing..." : "Auto Scrape and Analyze"} + {isSubmitting ? "Queueing..." : "Auto Fetch and Analyze"}
@@ -527,4 +527,4 @@ const AutoScrapePage = () => { ); }; -export default AutoScrapePage; +export default AutoFetchPage; diff --git a/frontend/src/pages/Datasets.tsx b/frontend/src/pages/Datasets.tsx index 90ad7d4..d878d7f 100644 --- a/frontend/src/pages/Datasets.tsx +++ b/frontend/src/pages/Datasets.tsx @@ -108,9 +108,9 @@ const DatasetsPage = () => {
diff --git a/frontend/src/utils/documentTitle.ts b/frontend/src/utils/documentTitle.ts index 7452481..3bbc3a2 100644 --- a/frontend/src/utils/documentTitle.ts +++ b/frontend/src/utils/documentTitle.ts @@ -3,7 +3,7 @@ const DEFAULT_TITLE = "Ethnograph View"; const STATIC_TITLES: Record = { "/login": "Sign In", "/upload": "Upload Dataset", - "/auto-scrape": "Auto Scrape Dataset", + "/auto-fetch": "Auto Fetch Dataset", "/datasets": "My Datasets", }; diff --git a/server/app.py b/server/app.py index 7f7c8fe..5af0f34 100644 --- a/server/app.py +++ b/server/app.py @@ -152,9 +152,9 @@ def get_dataset_sources(): return jsonify(list_metadata) -@app.route("/datasets/scrape", methods=["POST"]) +@app.route("/datasets/fetch", methods=["POST"]) @jwt_required() -def scrape_data(): +def fetch_data(): data = request.get_json() connector_metadata = get_connector_metadata() diff --git a/server/connectors/boards_api.py b/server/connectors/boards_api.py index f5c04e6..507e9a9 100644 --- a/server/connectors/boards_api.py +++ b/server/connectors/boards_api.py @@ -11,7 +11,7 @@ from server.connectors.base import BaseConnector logger = logging.getLogger(__name__) -HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; ForumScraper/1.0)"} +HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; ForumFetcher/1.0)"} class BoardsAPI(BaseConnector):