diff --git a/app/components/assessment/home/RunRowActions.tsx b/app/components/assessment/home/RunRowActions.tsx index cb71c89..0a99034 100644 --- a/app/components/assessment/home/RunRowActions.tsx +++ b/app/components/assessment/home/RunRowActions.tsx @@ -1,6 +1,8 @@ "use client"; import { EyeIcon } from "@/app/components/icons"; +import { useAssessmentData } from "@/app/hooks"; +import { loadSubmissionInputs } from "@/app/lib/assessment/submissionInputs"; import type { HomeRunRow, RunRowActionsProps, @@ -15,12 +17,26 @@ function resultsHref(row: HomeRunRow): string { } export default function RunRowActions({ row }: RunRowActionsProps) { + const data = useAssessmentData(); const href = resultsHref(row); + const { submission_id: submissionId, total_items: totalItems } = + row.assessment; + + /* Warms the submission cache during the hover before the click, so the + results sheet has its source columns by the time it paints. */ + const prefetchInputs = () => { + if (!submissionId) return; + void loadSubmissionInputs(data, submissionId, totalItems).catch(() => { + // A cold cache is the only cost of a failed warm-up. + }); + }; return (
diff --git a/app/hooks/useRunResults.ts b/app/hooks/useRunResults.ts index f1c49b7..5cbe597 100644 --- a/app/hooks/useRunResults.ts +++ b/app/hooks/useRunResults.ts @@ -1,13 +1,6 @@ "use client"; -/** - * One run's results, through the data source: the raw rows (for the detail - * modal) plus the table projection the grids render. - * - * Polls while the run is in flight and stops at a terminal status, so an open - * results tab fills in as stages land. - */ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useToast } from "@/app/hooks/useToast"; import { useAssessmentData } from "@/app/hooks/useAssessmentData"; import { @@ -15,17 +8,30 @@ import { jsonResultsToTableData, normalizeStatus, } from "@/app/lib/assessment/results"; +import { + buildColumnOrder, + mergeSubmissionInputs, + type SubmissionInputs, +} from "@/app/lib/assessment/inputJoin"; +import { loadSubmissionInputs } from "@/app/lib/assessment/submissionInputs"; import { RESULTS_POLL_INTERVAL_MS, SPREADSHEET_PREVIEW_ROW_LIMIT, TERMINAL_ASSESSMENT_STATUSES, } from "@/app/lib/assessment/constants"; import type { + AssessmentConfigRef, AssessmentStatusValue, BatchCounts, ResultsTarget, } from "@/app/lib/types/assessment"; +/** A fetched extra plus the id it was fetched for, so a stale one is spottable. */ +interface OwnedBy { + owner: string; + value: T; +} + export interface UseRunResultsResult { results: Record[]; headers: string[]; @@ -44,39 +50,43 @@ export function useRunResults( const toast = useToast(); const data = useAssessmentData(); const [results, setResults] = useState[]>([]); - const [table, setTable] = useState<{ headers: string[]; rows: string[][] }>({ - headers: [], - rows: [], - }); const [status, setStatus] = useState(null); const [counts, setCounts] = useState(null); const [totalItems, setTotalItems] = useState(0); + const [submissionId, setSubmissionId] = useState(null); + const [config, setConfig] = useState(null); + const [inputs, setInputs] = useState | null>(null); + const [outputSchema, setOutputSchema] = useState | null> | null>(null); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); const warnedRef = useRef(false); - const cancelledRef = useRef(false); + const targetRef = useRef(null); const assessmentId = target?.assessment_id ?? null; const method = target?.method ?? null; + const targetKey = assessmentId && method ? `${assessmentId}:${method}` : null; + const configKey = config ? `${config.id}@${config.version}` : null; const load = useCallback(async () => { if (!assessmentId || !method) return; + const startedFor = `${assessmentId}:${method}`; + const isStale = () => targetRef.current !== startedFor; try { const payload = await data.getRunResults({ assessment_id: assessmentId, method, }); - if (cancelledRef.current) return; + if (isStale()) return; setResults(payload.rows); setStatus(payload.status); setCounts(payload.counts); setTotalItems(payload.total_items); - setTable( - jsonResultsToTableData(payload.rows, { - rowLimit: SPREADSHEET_PREVIEW_ROW_LIMIT, - }), - ); + setSubmissionId(payload.submission_id); + setConfig(payload.config); setError(null); if ( @@ -89,28 +99,98 @@ export function useRunResults( ); } } catch (caught) { - if (!cancelledRef.current) { - setError(getAsyncErrorMessage("load results", caught)); - } + if (!isStale()) setError(getAsyncErrorMessage("load results", caught)); } finally { - if (!cancelledRef.current) setIsLoading(false); + if (!isStale()) setIsLoading(false); } }, [assessmentId, data, method, toast]); useEffect(() => { - cancelledRef.current = false; - if (!assessmentId) { + if (!assessmentId || !targetKey) { setError("Invalid assessment id."); setIsLoading(false); return; } + targetRef.current = targetKey; + // The previous run's rows are not this run's; show nothing until it loads. + setResults([]); + setStatus(null); + setCounts(null); + setTotalItems(0); + setSubmissionId(null); + setConfig(null); + warnedRef.current = false; setIsLoading(true); void load(); + + return () => { + targetRef.current = null; + }; + }, [assessmentId, load, targetKey]); + + // The source rows, once per submission. Immutable, so polling never refetches. + useEffect(() => { + if (!submissionId) return; + let cancelled = false; + + void loadSubmissionInputs(data, submissionId, totalItems) + .then((loaded) => { + if (!cancelled && loaded.records.length > 0) { + setInputs({ owner: submissionId, value: loaded }); + } + }) + .catch(() => { + // Source columns are additive; without them the results still stand. + }); + return () => { - cancelledRef.current = true; + cancelled = true; }; - }, [assessmentId, load]); + }, [data, submissionId, totalItems]); + + // The output schema fixes column order, so it follows the config, not the rows. + useEffect(() => { + if (!config?.id || !configKey) return; + let cancelled = false; + + void data + .getAssessorVersion(config.id, config.version) + .then((version) => { + if (!cancelled) { + setOutputSchema({ owner: configKey, value: version.output_schema }); + } + }) + .catch(() => { + // Without a schema the columns keep their discovered order. + }); + + return () => { + cancelled = true; + }; + }, [config?.id, config?.version, configKey, data]); + + // A fetch that outlived its run must not colour the next one. + const ownInputs = + inputs && inputs.owner === submissionId ? inputs.value : null; + const ownSchema = + outputSchema && outputSchema.owner === configKey + ? outputSchema.value + : null; + + const joined = useMemo( + () => (ownInputs ? mergeSubmissionInputs(results, ownInputs) : results), + [ownInputs, results], + ); + + const table = useMemo( + () => + jsonResultsToTableData(joined, { + rowLimit: SPREADSHEET_PREVIEW_ROW_LIMIT, + columnOrder: buildColumnOrder(ownInputs?.headers ?? [], ownSchema), + }), + [joined, ownInputs, ownSchema], + ); const isPolling = status !== null && @@ -123,7 +203,7 @@ export function useRunResults( }, [isPolling, load]); return { - results, + results: joined, headers: table.headers, rows: table.rows, status, diff --git a/app/lib/assessment/api/runs.ts b/app/lib/assessment/api/runs.ts index ed83c65..c63dce6 100644 --- a/app/lib/assessment/api/runs.ts +++ b/app/lib/assessment/api/runs.ts @@ -80,5 +80,7 @@ export async function getRunResults( rows: flattenBatchDetail(detail), total_items: detail.total_items ?? 0, counts: detail.counts ?? null, + submission_id: detail.submission_id ?? null, + config: detail.config ?? null, }; } diff --git a/app/lib/assessment/api/submissions.ts b/app/lib/assessment/api/submissions.ts index 85d1807..6590a55 100644 --- a/app/lib/assessment/api/submissions.ts +++ b/app/lib/assessment/api/submissions.ts @@ -31,9 +31,10 @@ export async function listSubmissions( export async function getSubmissionPreview( apiKey: string, submissionId: string, + limitRows: number = DATASET_SAMPLE_ROW_LIMIT, ): Promise { const response = await apiFetch>( - `${ENDPOINT}/${submissionId}?limit_rows=${DATASET_SAMPLE_ROW_LIMIT}`, + `${ENDPOINT}/${submissionId}?limit_rows=${limitRows}`, apiKey, ); const submission = unwrap(response, {} as AssessmentSubmission); diff --git a/app/lib/assessment/apiSource.ts b/app/lib/assessment/apiSource.ts index ae101be..85ac0bf 100644 --- a/app/lib/assessment/apiSource.ts +++ b/app/lib/assessment/apiSource.ts @@ -11,8 +11,8 @@ export function createApiAssessmentSource( ): AssessmentDataSource { return { listSubmissions: () => submissions.listSubmissions(apiKey), - getSubmissionPreview: (submissionId) => - submissions.getSubmissionPreview(apiKey, submissionId), + getSubmissionPreview: (submissionId, limitRows) => + submissions.getSubmissionPreview(apiKey, submissionId, limitRows), createSubmission: (input) => submissions.createSubmission(apiKey, input), deleteSubmission: (submissionId) => submissions.deleteSubmission(apiKey, submissionId), diff --git a/app/lib/assessment/inputJoin.ts b/app/lib/assessment/inputJoin.ts new file mode 100644 index 0000000..1e43fee --- /dev/null +++ b/app/lib/assessment/inputJoin.ts @@ -0,0 +1,126 @@ +import { + ASSESSMENT_OUTPUT_KEY_PREFIX, + MAX_OUTPUT_FLATTEN_DEPTH, + PREFILTER_DECISION_KEY, + PREFILTER_REASONING_KEY, + REASON_OBJECT_KEYS, + RESULT_REASON_SUFFIX, + RESULT_SCORE_SUFFIX, + SCORE_OBJECT_KEYS, +} from "@/app/lib/assessment/constants"; +import type { SubmissionPreviewRows } from "@/app/lib/types/assessment"; + +export interface SubmissionInputs { + headers: string[]; + records: Record[]; +} + +const isPlainObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export function previewToInputs( + preview: SubmissionPreviewRows | undefined, +): SubmissionInputs { + const headers = preview?.headers ?? []; + const rows = preview?.rows ?? []; + if (headers.length === 0) return { headers: [], records: [] }; + + const records = rows.map((row) => { + const record: Record = {}; + headers.forEach((header, column) => { + record[header] = row[column] ?? ""; + }); + return record; + }); + return { headers, records }; +} + +/** Results may number rows from 0 or from 1; the lowest index tells us which. */ +function rowIndexOffset(rows: Record[]): number { + let lowest = Number.POSITIVE_INFINITY; + for (const row of rows) { + const index = row.row_index; + if (typeof index === "number" && index < lowest) lowest = index; + } + return Number.isFinite(lowest) ? Math.max(0, lowest) : 0; +} + +export function mergeSubmissionInputs( + rows: Record[], + inputs: SubmissionInputs, +): Record[] { + if (inputs.records.length === 0) return rows; + const offset = rowIndexOffset(rows); + + return rows.map((row) => { + const index = typeof row.row_index === "number" ? row.row_index : -1; + const source = index < 0 ? undefined : inputs.records[index - offset]; + if (!source) return row; + + const merged: Record = { ...source }; + for (const [key, value] of Object.entries(row)) { + if (!(key in merged)) { + merged[key] = value; + continue; + } + if (String(merged[key]) === String(value ?? "")) continue; + merged[`${ASSESSMENT_OUTPUT_KEY_PREFIX}${key}`] = value; + } + return merged; + }); +} + +/** Column names one schema property flattens to, mirroring `flattenOutput`. */ +function schemaPropertyColumns( + key: string, + property: unknown, + depth: number, +): string[] { + if (!isPlainObject(property)) return [key]; + + const nested = property.properties; + if (!isPlainObject(nested)) return [key]; + + const nestedKeys = Object.keys(nested); + const hasScore = SCORE_OBJECT_KEYS.some((name) => nestedKeys.includes(name)); + if (hasScore) { + const columns = [`${key}${RESULT_SCORE_SUFFIX}`]; + if (REASON_OBJECT_KEYS.some((name) => nestedKeys.includes(name))) { + columns.push(`${key}${RESULT_REASON_SUFFIX}`); + } + return columns; + } + + if (depth >= MAX_OUTPUT_FLATTEN_DEPTH) return [key]; + return nestedKeys.flatMap((nestedKey) => + schemaPropertyColumns(`${key}_${nestedKey}`, nested[nestedKey], depth + 1), + ); +} + +/** Output columns in the schema's own order, so they stop shuffling per run. */ +export function outputSchemaColumns( + schema: Record | null, +): string[] { + const properties = schema?.properties; + if (!isPlainObject(properties)) return []; + return Object.keys(properties).flatMap((key) => + schemaPropertyColumns(key, properties[key], 0), + ); +} + +/** + * The order the sheet reads in: source columns, the pre-filter verdict that + * explains an empty row, then the model's output as the schema declares it. + * Anything unaccounted for keeps its discovered order at the end. + */ +export function buildColumnOrder( + submissionHeaders: string[], + outputSchema: Record | null, +): string[] { + return [ + ...submissionHeaders, + PREFILTER_DECISION_KEY, + PREFILTER_REASONING_KEY, + ...outputSchemaColumns(outputSchema), + ]; +} diff --git a/app/lib/assessment/results.ts b/app/lib/assessment/results.ts index 5add7db..2b2a6be 100644 --- a/app/lib/assessment/results.ts +++ b/app/lib/assessment/results.ts @@ -303,9 +303,21 @@ export function rowsToCsv(matrix: string[][]): string { return matrix.map((row) => row.map(escape).join(",")).join("\r\n"); } +function orderKeys(keys: string[], order?: string[]): string[] { + if (!order || order.length === 0) return keys; + const present = new Set(keys); + const ranked = order.filter((key) => present.has(key)); + const seen = new Set(ranked); + return [...ranked, ...keys.filter((key) => !seen.has(key))]; +} + export function jsonResultsToTableData( results: Record[], - opts?: { skipFields?: Set; rowLimit?: number }, + opts?: { + skipFields?: Set; + rowLimit?: number; + columnOrder?: string[]; + }, ): { headers: string[]; rows: string[][] } { if (results.length === 0) return { headers: [], rows: [] }; @@ -332,7 +344,10 @@ export function jsonResultsToTableData( "experiment_name", ]); - const allKeys = Array.from(new Set(results.flatMap((r) => Object.keys(r)))); + const allKeys = orderKeys( + Array.from(new Set(results.flatMap((r) => Object.keys(r)))), + opts?.columnOrder, + ); const displayKeys = allKeys.filter((k) => !skipFields.has(k)); const nonEmptyKeys = displayKeys.filter((key) => diff --git a/app/lib/assessment/submissionCache.ts b/app/lib/assessment/submissionCache.ts new file mode 100644 index 0000000..0722ef3 --- /dev/null +++ b/app/lib/assessment/submissionCache.ts @@ -0,0 +1,80 @@ +import type { SubmissionInputs } from "@/app/lib/assessment/inputJoin"; + +const DB_NAME = "kaapi_assessment"; +const DB_VERSION = 1; +const STORE_NAME = "submission_inputs"; + +const memory = new Map(); + +function openDatabase(): Promise { + if (typeof indexedDB === "undefined") return Promise.resolve(null); + + return new Promise((resolve) => { + let request: IDBOpenDBRequest; + try { + request = indexedDB.open(DB_NAME, DB_VERSION); + } catch { + resolve(null); + return; + } + request.onupgradeneeded = () => { + if (!request.result.objectStoreNames.contains(STORE_NAME)) { + request.result.createObjectStore(STORE_NAME); + } + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => resolve(null); + request.onblocked = () => resolve(null); + }); +} + +export async function readCachedInputs( + submissionId: string, +): Promise { + const cached = memory.get(submissionId); + if (cached) return cached; + + const db = await openDatabase(); + if (!db) return null; + + const stored = await new Promise((resolve) => { + try { + const request = db + .transaction(STORE_NAME, "readonly") + .objectStore(STORE_NAME) + .get(submissionId); + request.onsuccess = () => + resolve((request.result as SubmissionInputs) ?? null); + request.onerror = () => resolve(null); + } catch { + resolve(null); + } + }); + db.close(); + + if (stored) memory.set(submissionId, stored); + return stored; +} + +export async function writeCachedInputs( + submissionId: string, + inputs: SubmissionInputs, +): Promise { + memory.set(submissionId, inputs); + + const db = await openDatabase(); + if (!db) return; + + await new Promise((resolve) => { + try { + const transaction = db.transaction(STORE_NAME, "readwrite"); + transaction.objectStore(STORE_NAME).put(inputs, submissionId); + transaction.oncomplete = () => resolve(); + transaction.onerror = () => resolve(); + transaction.onabort = () => resolve(); + } catch { + resolve(); + } + }); + db.close(); +} diff --git a/app/lib/assessment/submissionInputs.ts b/app/lib/assessment/submissionInputs.ts new file mode 100644 index 0000000..1097266 --- /dev/null +++ b/app/lib/assessment/submissionInputs.ts @@ -0,0 +1,31 @@ +/** + * Reads a run's source rows, from cache when we already have them. + */ +import { + previewToInputs, + type SubmissionInputs, +} from "@/app/lib/assessment/inputJoin"; +import { + readCachedInputs, + writeCachedInputs, +} from "@/app/lib/assessment/submissionCache"; +import type { AssessmentDataSource } from "@/app/lib/types/assessment"; + +export async function loadSubmissionInputs( + source: AssessmentDataSource, + submissionId: string, + expectedRows = 0, +): Promise { + if (expectedRows <= 0) return { headers: [], records: [] }; + + const cached = await readCachedInputs(submissionId); + // A shorter entry was cached under a smaller ask — fetch the rest. + if (cached && cached.records.length >= expectedRows) return cached; + + const payload = await source.getSubmissionPreview(submissionId, expectedRows); + const inputs = previewToInputs(payload.preview); + if (inputs.records.length > 0) { + void writeCachedInputs(submissionId, inputs); + } + return inputs; +} diff --git a/app/lib/types/assessment/batch.ts b/app/lib/types/assessment/batch.ts index b0d7be2..a7f6983 100644 --- a/app/lib/types/assessment/batch.ts +++ b/app/lib/types/assessment/batch.ts @@ -85,4 +85,6 @@ export interface AssessmentResultsPayload { rows: Record[]; total_items: number; counts: BatchCounts | null; + submission_id: string | null; + config: AssessmentConfigRef | null; } diff --git a/app/lib/types/assessment/dataSource.ts b/app/lib/types/assessment/dataSource.ts index 7876474..4c25034 100644 --- a/app/lib/types/assessment/dataSource.ts +++ b/app/lib/types/assessment/dataSource.ts @@ -78,6 +78,7 @@ export interface AssessmentDataSource { listSubmissions: () => Promise; getSubmissionPreview: ( submissionId: string, + limitRows?: number, ) => Promise; createSubmission: ( input: CreateSubmissionInput,