import { useEffect, useRef, useState } from "react"; import axios from "axios"; import { useParams } from "react-router-dom"; import StatsStyling from "../styles/stats_styling"; import SummaryStats from "../components/SummaryStats"; import EmotionalStats from "../components/EmotionalStats"; import UserStats from "../components/UserStats"; import LinguisticStats from "../components/LinguisticStats"; import InteractionalStats from "../components/InteractionalStats"; import CulturalStats from "../components/CulturalStats"; import CorpusExplorer from "../components/CorpusExplorer"; import { type SummaryResponse, type TimeAnalysisResponse, type User, type UserEndpointResponse, type LinguisticAnalysisResponse, type EmotionalAnalysisResponse, type InteractionAnalysisResponse, type CulturalAnalysisResponse, } from "../types/ApiTypes"; import { buildExplorerContext, type CorpusExplorerSpec, type DatasetRecord, } from "../utils/corpusExplorer"; const API_BASE_URL = import.meta.env.VITE_BACKEND_URL; const styles = StatsStyling; const DELETED_USERS = ["[deleted]", "automoderator"]; const isDeletedUser = (value: string | null | undefined) => DELETED_USERS.includes((value ?? "").trim().toLowerCase()); type ActiveView = | "summary" | "emotional" | "user" | "linguistic" | "interactional" | "cultural"; type UserStatsMeta = { totalUsers: number; mostCommentHeavyUser: { author: string; commentShare: number } | null; }; type ExplorerState = { open: boolean; title: string; description: string; emptyMessage: string; records: DatasetRecord[]; loading: boolean; error: string; }; const EMPTY_EXPLORER_STATE: ExplorerState = { open: false, title: "Corpus Explorer", description: "", emptyMessage: "No records found.", records: [], loading: false, error: "", }; const createExplorerState = ( spec: CorpusExplorerSpec, patch: Partial = {}, ): ExplorerState => ({ open: true, title: spec.title, description: spec.description, emptyMessage: spec.emptyMessage ?? "No matching records found.", records: [], loading: false, error: "", ...patch, }); const compareRecordsByNewest = (a: DatasetRecord, b: DatasetRecord) => { const aValue = String(a.dt ?? a.date ?? a.timestamp ?? ""); const bValue = String(b.dt ?? b.date ?? b.timestamp ?? ""); return bValue.localeCompare(aValue); }; const parseJsonLikePayload = (value: string): unknown => { const normalized = value .replace(/\uFEFF/g, "") .replace(/,\s*([}\]])/g, "$1") .replace(/(:\s*)(NaN|Infinity|-Infinity)\b/g, "$1null") .replace(/(\[\s*)(NaN|Infinity|-Infinity)\b/g, "$1null") .replace(/(,\s*)(NaN|Infinity|-Infinity)\b/g, "$1null") .replace(/(:\s*)None\b/g, "$1null") .replace(/(:\s*)True\b/g, "$1true") .replace(/(:\s*)False\b/g, "$1false") .replace(/(\[\s*)None\b/g, "$1null") .replace(/(\[\s*)True\b/g, "$1true") .replace(/(\[\s*)False\b/g, "$1false") .replace(/(,\s*)None\b/g, "$1null") .replace(/(,\s*)True\b/g, "$1true") .replace(/(,\s*)False\b/g, "$1false"); return JSON.parse(normalized); }; const tryParseRecords = (value: string) => { try { return normalizeRecordPayload(parseJsonLikePayload(value)); } catch { return null; } }; const parseRecordStringPayload = (payload: string): DatasetRecord[] | null => { const trimmed = payload.trim(); if (!trimmed) { return []; } const direct = tryParseRecords(trimmed); if (direct) { return direct; } const ndjsonLines = trimmed .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean); if (ndjsonLines.length > 0) { try { return ndjsonLines.map((line) => parseJsonLikePayload(line)) as DatasetRecord[]; } catch { } } const bracketStart = trimmed.indexOf("["); const bracketEnd = trimmed.lastIndexOf("]"); if (bracketStart !== -1 && bracketEnd > bracketStart) { const parsed = tryParseRecords(trimmed.slice(bracketStart, bracketEnd + 1)); if (parsed) { return parsed; } } const braceStart = trimmed.indexOf("{"); const braceEnd = trimmed.lastIndexOf("}"); if (braceStart !== -1 && braceEnd > braceStart) { const parsed = tryParseRecords(trimmed.slice(braceStart, braceEnd + 1)); if (parsed) { return parsed; } } return null; }; const normalizeRecordPayload = (payload: unknown): DatasetRecord[] => { if (typeof payload === "string") { const parsed = parseRecordStringPayload(payload); if (parsed) { return parsed; } const preview = payload.trim().slice(0, 120).replace(/\s+/g, " "); throw new Error( `Corpus endpoint returned a non-JSON string payload.${ preview ? ` Response preview: ${preview}` : "" }`, ); } if ( payload && typeof payload === "object" && "error" in payload && typeof (payload as { error?: unknown }).error === "string" ) { throw new Error((payload as { error: string }).error); } if (Array.isArray(payload)) { return payload as DatasetRecord[]; } if ( payload && typeof payload === "object" && "data" in payload && Array.isArray((payload as { data?: unknown }).data) ) { return (payload as { data: DatasetRecord[] }).data; } if ( payload && typeof payload === "object" && "records" in payload && Array.isArray((payload as { records?: unknown }).records) ) { return (payload as { records: DatasetRecord[] }).records; } if ( payload && typeof payload === "object" && "rows" in payload && Array.isArray((payload as { rows?: unknown }).rows) ) { return (payload as { rows: DatasetRecord[] }).rows; } if ( payload && typeof payload === "object" && "result" in payload && Array.isArray((payload as { result?: unknown }).result) ) { return (payload as { result: DatasetRecord[] }).result; } if (payload && typeof payload === "object") { const values = Object.values(payload); if (values.length === 1 && Array.isArray(values[0])) { return values[0] as DatasetRecord[]; } if (values.every((value) => value && typeof value === "object")) { return values as DatasetRecord[]; } } throw new Error("Corpus endpoint returned an unexpected payload."); }; const StatPage = () => { const { datasetId: routeDatasetId } = useParams<{ datasetId: string }>(); const [error, setError] = useState(""); const [loading, setLoading] = useState(false); const [activeView, setActiveView] = useState("summary"); const [userData, setUserData] = useState(null); const [timeData, setTimeData] = useState(null); const [linguisticData, setLinguisticData] = useState(null); const [emotionalData, setEmotionalData] = useState(null); const [interactionData, setInteractionData] = useState(null); const [culturalData, setCulturalData] = useState(null); const [summary, setSummary] = useState(null); const [userStatsMeta, setUserStatsMeta] = useState({ totalUsers: 0, mostCommentHeavyUser: null, }); const [appliedFilters, setAppliedFilters] = useState>({}); const [allRecords, setAllRecords] = useState(null); const [allRecordsKey, setAllRecordsKey] = useState(""); const [explorerState, setExplorerState] = useState( EMPTY_EXPLORER_STATE, ); const searchInputRef = useRef(null); const beforeDateRef = useRef(null); const afterDateRef = useRef(null); const parsedDatasetId = Number(routeDatasetId ?? ""); const datasetId = Number.isInteger(parsedDatasetId) && parsedDatasetId > 0 ? parsedDatasetId : null; const getFilterParams = () => { const params: Record = {}; const query = (searchInputRef.current?.value ?? "").trim(); const start = (afterDateRef.current?.value ?? "").trim(); const end = (beforeDateRef.current?.value ?? "").trim(); if (query) { params.search_query = query; } if (start) { params.start_date = start; } if (end) { params.end_date = end; } return params; }; const getAuthHeaders = () => { const token = localStorage.getItem("access_token"); if (!token) { return null; } return { Authorization: `Bearer ${token}`, }; }; const getFilterKey = (params: Record) => JSON.stringify(Object.entries(params).sort(([a], [b]) => a.localeCompare(b))); const ensureFilteredRecords = async () => { if (!datasetId) { throw new Error("Missing dataset id."); } const authHeaders = getAuthHeaders(); if (!authHeaders) { throw new Error("You must be signed in to load corpus records."); } const filterKey = getFilterKey(appliedFilters); if (allRecords && allRecordsKey === filterKey) { return allRecords; } const response = await axios.get( `${API_BASE_URL}/dataset/${datasetId}/all`, { params: appliedFilters, headers: authHeaders, }, ); const normalizedRecords = normalizeRecordPayload(response.data); setAllRecords(normalizedRecords); setAllRecordsKey(filterKey); return normalizedRecords; }; const openExplorer = async (spec: CorpusExplorerSpec) => { setExplorerState(createExplorerState(spec, { loading: true })); try { const records = await ensureFilteredRecords(); const context = buildExplorerContext(records); const matched = records .filter((record) => spec.matcher(record, context)) .sort(compareRecordsByNewest); setExplorerState(createExplorerState(spec, { records: matched })); } catch (e) { setExplorerState( createExplorerState(spec, { error: `Failed to load corpus records: ${String(e)}`, }), ); } }; const getStats = (params: Record = {}) => { if (!datasetId) { setError("Missing dataset id. Open /dataset//stats."); return; } const authHeaders = getAuthHeaders(); if (!authHeaders) { setError("You must be signed in to load stats."); return; } setError(""); setLoading(true); setAppliedFilters(params); setAllRecords(null); setAllRecordsKey(""); setExplorerState((current) => ({ ...current, open: false })); Promise.all([ axios.get(`${API_BASE_URL}/dataset/${datasetId}/temporal`, { params, headers: authHeaders, }), axios.get(`${API_BASE_URL}/dataset/${datasetId}/user`, { params, headers: authHeaders, }), axios.get( `${API_BASE_URL}/dataset/${datasetId}/linguistic`, { params, headers: authHeaders, }, ), axios.get(`${API_BASE_URL}/dataset/${datasetId}/emotional`, { params, headers: authHeaders, }), axios.get( `${API_BASE_URL}/dataset/${datasetId}/interactional`, { params, headers: authHeaders, }, ), axios.get(`${API_BASE_URL}/dataset/${datasetId}/summary`, { params, headers: authHeaders, }), axios.get(`${API_BASE_URL}/dataset/${datasetId}/cultural`, { params, headers: authHeaders, }), ]) .then( ([ timeRes, userRes, linguisticRes, emotionalRes, interactionRes, summaryRes, culturalRes, ]) => { const usersList = userRes.data.users ?? []; const topUsersList = userRes.data.top_users ?? []; const interactionGraphRaw = interactionRes.data?.interaction_graph ?? {}; const topPairsRaw = interactionRes.data?.top_interaction_pairs ?? []; const filteredUsers: typeof usersList = []; for (const user of usersList) { if (isDeletedUser(user.author)) continue; filteredUsers.push(user); } const filteredTopUsers: typeof topUsersList = []; for (const user of topUsersList) { if (isDeletedUser(user.author)) continue; filteredTopUsers.push(user); } let mostCommentHeavyUser: UserStatsMeta["mostCommentHeavyUser"] = null; for (const user of filteredUsers) { const currentShare = user.comment_share ?? 0; if (!mostCommentHeavyUser || currentShare > mostCommentHeavyUser.commentShare) { mostCommentHeavyUser = { author: user.author, commentShare: currentShare, }; } } const topAuthors = new Set(filteredTopUsers.map((entry) => entry.author)); const summaryUsers: User[] = []; for (const user of filteredUsers) { if (topAuthors.has(user.author)) { summaryUsers.push(user); } } const filteredInteractionGraph: Record> = {}; for (const [source, targets] of Object.entries(interactionGraphRaw)) { if (isDeletedUser(source)) { continue; } const nextTargets: Record = {}; for (const [target, count] of Object.entries(targets)) { if (isDeletedUser(target)) { continue; } nextTargets[target] = count; } filteredInteractionGraph[source] = nextTargets; } const filteredTopInteractionPairs: typeof topPairsRaw = []; for (const pairEntry of topPairsRaw) { const pair = pairEntry[0]; const source = pair[0]; const target = pair[1]; if (isDeletedUser(source) || isDeletedUser(target)) { continue; } filteredTopInteractionPairs.push(pairEntry); } const filteredUserData: UserEndpointResponse = { users: summaryUsers, top_users: filteredTopUsers, }; const filteredInteractionData: InteractionAnalysisResponse = { ...interactionRes.data, interaction_graph: filteredInteractionGraph, top_interaction_pairs: filteredTopInteractionPairs, }; const filteredSummary: SummaryResponse = { ...summaryRes.data, unique_users: filteredUsers.length, }; setUserData(filteredUserData); setUserStatsMeta({ totalUsers: filteredUsers.length, mostCommentHeavyUser, }); setTimeData(timeRes.data || null); setLinguisticData(linguisticRes.data || null); setEmotionalData(emotionalRes.data || null); setInteractionData(filteredInteractionData || null); setCulturalData(culturalRes.data || null); setSummary(filteredSummary || null); }, ) .catch((e) => setError(`Failed to load statistics: ${String(e)}`)) .finally(() => setLoading(false)); }; const onSubmitFilters = () => { getStats(getFilterParams()); }; const resetFilters = () => { if (searchInputRef.current) { searchInputRef.current.value = ""; } if (beforeDateRef.current) { beforeDateRef.current.value = ""; } if (afterDateRef.current) { afterDateRef.current.value = ""; } getStats(); }; useEffect(() => { setError(""); setAllRecords(null); setAllRecordsKey(""); setExplorerState(EMPTY_EXPLORER_STATE); if (!datasetId) { setError("Missing dataset id. Open /dataset//stats."); return; } getStats(); }, [datasetId]); if (loading) { return (

Loading analytics

Fetching summary, timeline, user, and content insights.

); } if (error) return

{error}

; return (
Analytics Dashboard
Dataset #{datasetId ?? "-"}
{activeView === "summary" && ( )} {activeView === "emotional" && emotionalData && ( )} {activeView === "emotional" && !emotionalData && (
No emotional data available.
)} {activeView === "user" && userData && interactionData && ( )} {activeView === "user" && (!userData || !interactionData) && (
No user network data available.
)} {activeView === "linguistic" && linguisticData && ( )} {activeView === "linguistic" && !linguisticData && (
No linguistic data available.
)} {activeView === "interactional" && interactionData && ( )} {activeView === "interactional" && !interactionData && (
No interactional data available.
)} {activeView === "cultural" && culturalData && ( )} {activeView === "cultural" && !culturalData && (
No cultural data available.
)} setExplorerState((current) => ({ ...current, open: false }))} title={explorerState.title} description={explorerState.description} records={explorerState.records} loading={explorerState.loading} error={explorerState.error} emptyMessage={explorerState.emptyMessage} />
); }; export default StatPage;