dylan/crosspost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(frontend): implement corpus explorer

Browse Source 
This allows you to view the posts & comments associated with a specific aggregate.
This commit is contained in:
Dylan De Faoite
2026-04-01 00:04:25 +01:00
parent 1dde5f7b08
commit b270ed03ae
11 changed files with 1064 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
frontend/src/pages/DatasetEdit.tsx
View File

@@ -22,12 +22,10 @@ const DatasetEditPage = () => {
const [isSaving, setIsSaving] = useState(false);
const [isDeleting, setIsDeleting] = useState(false);
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
const [hasError, setHasError] = useState(false);
const [datasetName, setDatasetName] = useState("");
useEffect(() => {
if (!Number.isInteger(parsedDatasetId) || parsedDatasetId <= 0) {
setHasError(true);
setStatusMessage("Invalid dataset id.");
setLoading(false);
return;
@@ -35,7 +33,6 @@ const DatasetEditPage = () => {
const token = localStorage.getItem("access_token");
if (!token) {
setHasError(true);
setStatusMessage("You must be signed in to edit datasets.");
setLoading(false);
return;
@@ -49,7 +46,6 @@ const DatasetEditPage = () => {
setDatasetName(response.data.name || "");
})
.catch((error: unknown) => {
setHasError(true);
if (axios.isAxiosError(error)) {
setStatusMessage(
String(error.response?.data?.error || error.message),
@@ -68,21 +64,18 @@ const DatasetEditPage = () => {
const trimmedName = datasetName.trim();
if (!trimmedName) {
setHasError(true);
setStatusMessage("Please enter a valid dataset name.");
return;
}
const token = localStorage.getItem("access_token");
if (!token) {
setHasError(true);
setStatusMessage("You must be signed in to save changes.");
return;
}
try {
setIsSaving(true);
setHasError(false);
setStatusMessage("");
await axios.patch(
@@ -93,7 +86,6 @@ const DatasetEditPage = () => {
navigate("/datasets", { replace: true });
} catch (error: unknown) {
setHasError(true);
if (axios.isAxiosError(error)) {
setStatusMessage(
String(
@@ -111,7 +103,6 @@ const DatasetEditPage = () => {
const deleteDataset = async () => {
const deleteToken = localStorage.getItem("access_token");
if (!deleteToken) {
setHasError(true);
setStatusMessage("You must be signed in to delete datasets.");
setIsDeleteModalOpen(false);
return;
@@ -119,7 +110,6 @@ const DatasetEditPage = () => {
try {
setIsDeleting(true);
setHasError(false);
setStatusMessage("");
await axios.delete(`${API_BASE_URL}/dataset/${parsedDatasetId}`, {
@@ -129,7 +119,6 @@ const DatasetEditPage = () => {
setIsDeleteModalOpen(false);
navigate("/datasets", { replace: true });
} catch (error: unknown) {
setHasError(true);
if (axios.isAxiosError(error)) {
setStatusMessage(
String(

300
frontend/src/pages/Stats.tsx
View File

@@ -1,4 +1,4 @@
import { useEffect, useState, useRef } from "react";
import { useEffect, useRef, useState } from "react";
import axios from "axios";
import { useParams } from "react-router-dom";
import StatsStyling from "../styles/stats_styling";
@@ -8,6 +8,7 @@ import UserStats from "../components/UserStats";
import LinguisticStats from "../components/LinguisticStats";
import InteractionalStats from "../components/InteractionalStats";
import CulturalStats from "../components/CulturalStats";
import CorpusExplorer from "../components/CorpusExplorer";
import {
type SummaryResponse,
@@ -19,10 +20,15 @@ import {
type InteractionAnalysisResponse,
type CulturalAnalysisResponse,
} from "../types/ApiTypes";
import {
buildExplorerContext,
type CorpusExplorerSpec,
type DatasetRecord,
} from "../utils/corpusExplorer";
const API_BASE_URL = import.meta.env.VITE_BACKEND_URL;
const styles = StatsStyling;
const DELETED_USERS = ["[deleted]"];
const DELETED_USERS = ["[deleted]", "automoderator"];
const isDeletedUser = (value: string | null | undefined) =>
DELETED_USERS.includes((value ?? "").trim().toLowerCase());
@@ -40,6 +46,97 @@ type UserStatsMeta = {
mostCommentHeavyUser: { author: string; commentShare: number } | null;
};
type ExplorerState = {
open: boolean;
title: string;
description: string;
emptyMessage: string;
records: DatasetRecord[];
loading: boolean;
error: string;
};
const EMPTY_EXPLORER_STATE: ExplorerState = {
open: false,
title: "Corpus Explorer",
description: "",
emptyMessage: "No records found.",
records: [],
loading: false,
error: "",
};
const normalizeRecordPayload = (payload: unknown): DatasetRecord[] => {
if (typeof payload === "string") {
try {
return normalizeRecordPayload(JSON.parse(payload));
} catch {
throw new Error("Corpus endpoint returned a non-JSON string payload.");
}
}
if (
payload &&
typeof payload === "object" &&
"error" in payload &&
typeof (payload as { error?: unknown }).error === "string"
) {
throw new Error((payload as { error: string }).error);
}
if (Array.isArray(payload)) {
return payload as DatasetRecord[];
}
if (
payload &&
typeof payload === "object" &&
"data" in payload &&
Array.isArray((payload as { data?: unknown }).data)
) {
return (payload as { data: DatasetRecord[] }).data;
}
if (
payload &&
typeof payload === "object" &&
"records" in payload &&
Array.isArray((payload as { records?: unknown }).records)
) {
return (payload as { records: DatasetRecord[] }).records;
}
if (
payload &&
typeof payload === "object" &&
"rows" in payload &&
Array.isArray((payload as { rows?: unknown }).rows)
) {
return (payload as { rows: DatasetRecord[] }).rows;
}
if (
payload &&
typeof payload === "object" &&
"result" in payload &&
Array.isArray((payload as { result?: unknown }).result)
) {
return (payload as { result: DatasetRecord[] }).result;
}
if (payload && typeof payload === "object") {
const values = Object.values(payload);
if (values.length === 1 && Array.isArray(values[0])) {
return values[0] as DatasetRecord[];
}
if (values.every((value) => value && typeof value === "object")) {
return values as DatasetRecord[];
}
}
throw new Error("Corpus endpoint returned an unexpected payload.");
};
const StatPage = () => {
const { datasetId: routeDatasetId } = useParams<{ datasetId: string }>();
const [error, setError] = useState("");
@@ -61,6 +158,12 @@ const StatPage = () => {
totalUsers: 0,
mostCommentHeavyUser: null,
});
const [appliedFilters, setAppliedFilters] = useState<Record<string, string>>({});
const [allRecords, setAllRecords] = useState<DatasetRecord[] | null>(null);
const [allRecordsKey, setAllRecordsKey] = useState("");
const [explorerState, setExplorerState] = useState<ExplorerState>(
EMPTY_EXPLORER_STATE,
);
const searchInputRef = useRef<HTMLInputElement>(null);
const beforeDateRef = useRef<HTMLInputElement>(null);
@@ -104,6 +207,82 @@ const StatPage = () => {
};
};
const getFilterKey = (params: Record<string, string>) =>
JSON.stringify(Object.entries(params).sort(([a], [b]) => a.localeCompare(b)));
const ensureFilteredRecords = async () => {
if (!datasetId) {
throw new Error("Missing dataset id.");
}
const authHeaders = getAuthHeaders();
if (!authHeaders) {
throw new Error("You must be signed in to load corpus records.");
}
const filterKey = getFilterKey(appliedFilters);
if (allRecords && allRecordsKey === filterKey) {
return allRecords;
}
const response = await axios.get<unknown>(
`${API_BASE_URL}/dataset/${datasetId}/all`,
{
params: appliedFilters,
headers: authHeaders,
},
);
const normalizedRecords = normalizeRecordPayload(response.data);
setAllRecords(normalizedRecords);
setAllRecordsKey(filterKey);
return normalizedRecords;
};
const openExplorer = async (spec: CorpusExplorerSpec) => {
setExplorerState({
open: true,
title: spec.title,
description: spec.description,
emptyMessage: spec.emptyMessage ?? "No matching records found.",
records: [],
loading: true,
error: "",
});
try {
const records = await ensureFilteredRecords();
const context = buildExplorerContext(records);
const matched = records.filter((record) => spec.matcher(record, context));
matched.sort((a, b) => {
const aValue = String(a.dt ?? a.date ?? a.timestamp ?? "");
const bValue = String(b.dt ?? b.date ?? b.timestamp ?? "");
return bValue.localeCompare(aValue);
});
setExplorerState({
open: true,
title: spec.title,
description: spec.description,
emptyMessage: spec.emptyMessage ?? "No matching records found.",
records: matched,
loading: false,
error: "",
});
} catch (e) {
setExplorerState({
open: true,
title: spec.title,
description: spec.description,
emptyMessage: spec.emptyMessage ?? "No matching records found.",
records: [],
loading: false,
error: `Failed to load corpus records: ${String(e)}`,
});
}
};
const getStats = (params: Record<string, string> = {}) => {
if (!datasetId) {
setError("Missing dataset id. Open /dataset/<id>/stats.");
@@ -118,22 +297,20 @@ const StatPage = () => {
setError("");
setLoading(true);
setAppliedFilters(params);
setAllRecords(null);
setAllRecordsKey("");
setExplorerState((current) => ({ ...current, open: false }));
Promise.all([
axios.get<TimeAnalysisResponse>(
`${API_BASE_URL}/dataset/${datasetId}/temporal`,
{
params,
headers: authHeaders,
},
),
axios.get<UserEndpointResponse>(
`${API_BASE_URL}/dataset/${datasetId}/user`,
{
params,
headers: authHeaders,
},
),
axios.get<TimeAnalysisResponse>(`${API_BASE_URL}/dataset/${datasetId}/temporal`, {
params,
headers: authHeaders,
}),
axios.get<UserEndpointResponse>(`${API_BASE_URL}/dataset/${datasetId}/user`, {
params,
headers: authHeaders,
}),
axios.get<LinguisticAnalysisResponse>(
`${API_BASE_URL}/dataset/${datasetId}/linguistic`,
{
@@ -141,13 +318,10 @@ const StatPage = () => {
headers: authHeaders,
},
),
axios.get<EmotionalAnalysisResponse>(
`${API_BASE_URL}/dataset/${datasetId}/emotional`,
{
params,
headers: authHeaders,
},
),
axios.get<EmotionalAnalysisResponse>(`${API_BASE_URL}/dataset/${datasetId}/emotional`, {
params,
headers: authHeaders,
}),
axios.get<InteractionAnalysisResponse>(
`${API_BASE_URL}/dataset/${datasetId}/interactional`,
{
@@ -155,20 +329,14 @@ const StatPage = () => {
headers: authHeaders,
},
),
axios.get<SummaryResponse>(
`${API_BASE_URL}/dataset/${datasetId}/summary`,
{
params,
headers: authHeaders,
},
),
axios.get<CulturalAnalysisResponse>(
`${API_BASE_URL}/dataset/${datasetId}/cultural`,
{
params,
headers: authHeaders,
},
),
axios.get<SummaryResponse>(`${API_BASE_URL}/dataset/${datasetId}/summary`, {
params,
headers: authHeaders,
}),
axios.get<CulturalAnalysisResponse>(`${API_BASE_URL}/dataset/${datasetId}/cultural`, {
params,
headers: authHeaders,
}),
])
.then(
([
@@ -182,8 +350,7 @@ const StatPage = () => {
]) => {
const usersList = userRes.data.users ?? [];
const topUsersList = userRes.data.top_users ?? [];
const interactionGraphRaw =
interactionRes.data?.interaction_graph ?? {};
const interactionGraphRaw = interactionRes.data?.interaction_graph ?? {};
const topPairsRaw = interactionRes.data?.top_interaction_pairs ?? [];
const filteredUsers: typeof usersList = [];
@@ -194,18 +361,14 @@ const StatPage = () => {
const filteredTopUsers: typeof topUsersList = [];
for (const user of topUsersList) {
if (isDeletedUser(user.author)) continue;
filteredTopUsers.push(user);
if (isDeletedUser(user.author)) continue;
filteredTopUsers.push(user);
}
let mostCommentHeavyUser: UserStatsMeta["mostCommentHeavyUser"] =
null;
let mostCommentHeavyUser: UserStatsMeta["mostCommentHeavyUser"] = null;
for (const user of filteredUsers) {
const currentShare = user.comment_share ?? 0;
if (
!mostCommentHeavyUser ||
currentShare > mostCommentHeavyUser.commentShare
) {
if (!mostCommentHeavyUser || currentShare > mostCommentHeavyUser.commentShare) {
mostCommentHeavyUser = {
author: user.author,
commentShare: currentShare,
@@ -221,8 +384,7 @@ const StatPage = () => {
}
}
const filteredInteractionGraph: Record<string, Record<string, number>> =
{};
const filteredInteractionGraph: Record<string, Record<string, number>> = {};
for (const [source, targets] of Object.entries(interactionGraphRaw)) {
if (isDeletedUser(source)) {
continue;
@@ -279,7 +441,7 @@ const StatPage = () => {
setSummary(filteredSummary || null);
},
)
.catch((e) => setError("Failed to load statistics: " + String(e)))
.catch((e) => setError(`Failed to load statistics: ${String(e)}`))
.finally(() => setLoading(false));
};
@@ -302,6 +464,9 @@ const StatPage = () => {
useEffect(() => {
setError("");
setAllRecords(null);
setAllRecordsKey("");
setExplorerState(EMPTY_EXPLORER_STATE);
if (!datasetId) {
setError("Missing dataset id. Open /dataset/<id>/stats.");
return;
@@ -398,9 +563,7 @@ const StatPage = () => {
<button
onClick={() => setActiveView("summary")}
style={
activeView === "summary"
? styles.buttonPrimary
: styles.buttonSecondary
activeView === "summary" ? styles.buttonPrimary : styles.buttonSecondary
}
>
Summary
@@ -418,11 +581,7 @@ const StatPage = () => {
<button
onClick={() => setActiveView("user")}
style={
activeView === "user"
? styles.buttonPrimary
: styles.buttonSecondary
}
style={activeView === "user" ? styles.buttonPrimary : styles.buttonSecondary}
>
Users
</button>
@@ -449,9 +608,7 @@ const StatPage = () => {
<button
onClick={() => setActiveView("cultural")}
style={
activeView === "cultural"
? styles.buttonPrimary
: styles.buttonSecondary
activeView === "cultural" ? styles.buttonPrimary : styles.buttonSecondary
}
>
Cultural
@@ -464,11 +621,12 @@ const StatPage = () => {
timeData={timeData}
linguisticData={linguisticData}
summary={summary}
onExplore={openExplorer}
/>
)}
{activeView === "emotional" && emotionalData && (
<EmotionalStats emotionalData={emotionalData} />
<EmotionalStats emotionalData={emotionalData} onExplore={openExplorer} />
)}
{activeView === "emotional" && !emotionalData && (
@@ -483,6 +641,7 @@ const StatPage = () => {
interactionGraph={interactionData.interaction_graph}
totalUsers={userStatsMeta.totalUsers}
mostCommentHeavyUser={userStatsMeta.mostCommentHeavyUser}
onExplore={openExplorer}
/>
)}
@@ -493,7 +652,7 @@ const StatPage = () => {
)}
{activeView === "linguistic" && linguisticData && (
<LinguisticStats data={linguisticData} />
<LinguisticStats data={linguisticData} onExplore={openExplorer} />
)}
{activeView === "linguistic" && !linguisticData && (
@@ -503,7 +662,7 @@ const StatPage = () => {
)}
{activeView === "interactional" && interactionData && (
<InteractionalStats data={interactionData} />
<InteractionalStats data={interactionData} onExplore={openExplorer} />
)}
{activeView === "interactional" && !interactionData && (
@@ -513,7 +672,7 @@ const StatPage = () => {
)}
{activeView === "cultural" && culturalData && (
<CulturalStats data={culturalData} />
<CulturalStats data={culturalData} onExplore={openExplorer} />
)}
{activeView === "cultural" && !culturalData && (
@@ -521,6 +680,17 @@ const StatPage = () => {
No cultural data available.
</div>
)}
<CorpusExplorer
open={explorerState.open}
onClose={() => setExplorerState((current) => ({ ...current, open: false }))}
title={explorerState.title}
description={explorerState.description}
records={explorerState.records}
loading={explorerState.loading}
error={explorerState.error}
emptyMessage={explorerState.emptyMessage}
/>
</div>
);
};