diff --git a/app/components/evaluations/DetailedResultsTable.tsx b/app/components/evaluations/DetailedResultsTable.tsx index 8d6a9a2..f1e2d41 100644 --- a/app/components/evaluations/DetailedResultsTable.tsx +++ b/app/components/evaluations/DetailedResultsTable.tsx @@ -13,7 +13,11 @@ import { isNewScoreObjectV2, isGroupedFormat, } from "@/app/lib/utils/evaluation"; -import { formatScoreValue, getScoreByName } from "@/app/lib/utils"; +import { + formatScoreValue, + getScoreByName, + getScoreNote, +} from "@/app/lib/utils"; import { InfoTooltip } from "@/app/components/ui"; import { GroupedResultsTable } from "@/app/components/evaluations"; import { MarkdownContent } from "@/app/components/chat"; @@ -186,6 +190,7 @@ export default function DetailedResultsTable({ {scoreNames.map((scoreName) => { const score = getScoreByName(item.trace_scores, scoreName); const { value, color, bg } = formatScoreValue(score); + const note = getScoreNote(score); return ( {value} - {score?.comment && ( - - )} + {note && } ); diff --git a/app/components/evaluations/GroupedResultsTable.tsx b/app/components/evaluations/GroupedResultsTable.tsx index 8a90570..a3bc0dd 100644 --- a/app/components/evaluations/GroupedResultsTable.tsx +++ b/app/components/evaluations/GroupedResultsTable.tsx @@ -6,7 +6,7 @@ import { Fragment } from "react"; import { TraceScore, GroupedTraceItem } from "@/app/lib/types/evaluation"; -import { formatScoreValue } from "@/app/lib/utils"; +import { formatScoreValue, getScoreNote } from "@/app/lib/utils"; import { InfoTooltip } from "@/app/components/ui"; export default function GroupedResultsTable({ traces, @@ -187,6 +187,7 @@ export default function GroupedResultsTable({ if (!score) return null; const { value, color, bg } = formatScoreValue(score); + const note = getScoreNote(score); return (
{value}
- {score?.comment && ( - - )} + {note && } ); diff --git a/app/lib/types/evaluation.ts b/app/lib/types/evaluation.ts index 5b0eb7f..9dfbcc4 100644 --- a/app/lib/types/evaluation.ts +++ b/app/lib/types/evaluation.ts @@ -5,6 +5,8 @@ export interface TraceScore { value: number | string; data_type: "NUMERIC" | "CATEGORICAL"; comment?: string; + /** Judge model's explanation for the score (v2 judge metrics, e.g. Adherence to Ground Truth/Prompt/Knowledge Base, 0-5 scale). */ + reasoning?: string; } export interface TraceItem { diff --git a/app/lib/utils.ts b/app/lib/utils.ts index 1c3aad1..b8d65e3 100644 --- a/app/lib/utils.ts +++ b/app/lib/utils.ts @@ -190,45 +190,64 @@ export const sanitizeCSVCell = ( return `"${sanitized}"`; }; -export const formatScoreValue = (score: TraceScore | undefined) => { - if (!score) return { value: "N/A", color: "#737373", bg: "transparent" }; - - if (score.data_type === "CATEGORICAL") { - const catValue = String(score.value); - let color = "#171717"; - let bg = "#fafafa"; - - if (catValue === "CORRECT") { - color = "#15803d"; - bg = "#dcfce7"; - } else if (catValue === "PARTIAL") { - color = "#92400e"; - bg = "#fef3c7"; - } else if (catValue === "INCORRECT") { - color = "#dc2626"; - bg = "#fee2e2"; - } +// v2 judge metrics score 0-5 (integers) instead of the legacy 0-1 cosine/correctness scale. +const FIVE_POINT_SCORE_NAMES = new Set([ + "adherence to ground truth", + "adherence to prompt", + "adherence to knowledge base", +]); - return { value: catValue, color, bg }; - } +const isFivePointScore = (name?: string): boolean => + !!name && FIVE_POINT_SCORE_NAMES.has(name.toLowerCase()); - const numValue = Number(score.value); - const formattedValue = numValue.toFixed(2); +const formatCategoricalScore = (value: number | string) => { + const catValue = String(value); let color = "#171717"; - let bg = "transparent"; + let bg = "#fafafa"; - if (numValue >= 0.7) { + if (catValue === "CORRECT") { color = "#15803d"; bg = "#dcfce7"; - } else if (numValue >= 0.5) { + } else if (catValue === "PARTIAL") { color = "#92400e"; bg = "#fef3c7"; - } else { + } else if (catValue === "INCORRECT") { color = "#dc2626"; bg = "#fee2e2"; } - return { value: formattedValue, color, bg }; + return { value: catValue, color, bg }; +}; + +// Traffic light thresholds for v2 judge metrics: 0-1 needs improvement, 2-3 could improve, 4-5 good. +const formatFivePointScore = (numValue: number) => { + if (numValue >= 4) + return { value: String(numValue), color: "#15803d", bg: "#dcfce7" }; + if (numValue >= 2) + return { value: String(numValue), color: "#92400e", bg: "#fef3c7" }; + return { value: String(numValue), color: "#dc2626", bg: "#fee2e2" }; +}; + +// Legacy thresholds for cosine similarity / correctness on a 0-1 scale. +const formatUnitScaleScore = (numValue: number) => { + const value = numValue.toFixed(2); + if (numValue >= 0.7) return { value, color: "#15803d", bg: "#dcfce7" }; + if (numValue >= 0.5) return { value, color: "#92400e", bg: "#fef3c7" }; + return { value, color: "#dc2626", bg: "#fee2e2" }; +}; + +export const formatScoreValue = (score: TraceScore | undefined) => { + if (!score) return { value: "N/A", color: "#737373", bg: "transparent" }; + + if (score.data_type === "CATEGORICAL") { + return formatCategoricalScore(score.value); + } + + const numValue = Number(score.value); + + return isFivePointScore(score.name) + ? formatFivePointScore(numValue) + : formatUnitScaleScore(numValue); }; export const getScoreByName = ( @@ -239,6 +258,14 @@ export const getScoreByName = ( return scores.find((s) => s?.name === name); }; +/** + * Returns the judge's explanation for a score, preferring the v2 `reasoning` + * field over the legacy `comment` field. + */ +export const getScoreNote = ( + score: TraceScore | undefined, +): string | undefined => score?.reasoning || score?.comment; + /** * Formats a USD cost value for display * @param cost - Cost in USD diff --git a/app/lib/utils/evaluationExport.ts b/app/lib/utils/evaluationExport.ts index a54ff11..5db55bc 100644 --- a/app/lib/utils/evaluationExport.ts +++ b/app/lib/utils/evaluationExport.ts @@ -5,7 +5,7 @@ import type { ScoreObject, } from "@/app/lib/types/evaluation"; import { normalizeToIndividualScores } from "@/app/lib/utils/evaluation"; -import { sanitizeCSVCell } from "@/app/lib/utils"; +import { sanitizeCSVCell, getScoreNote } from "@/app/lib/utils"; const downloadCSV = (csvContent: string, filename: string) => { const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" }); @@ -36,7 +36,7 @@ export const exportGroupedCSV = ( for (let i = 1; i <= maxAnswers; i++) { csvContent += `,LLM Answer ${i},Trace ID ${i}`; scoreNames.forEach((name) => { - csvContent += `,${name} (${i}),${sanitizeCSVCell(`${name} (${i}) Comment`)}`; + csvContent += `,${name} (${i}),${sanitizeCSVCell(`${name} (${i}) Comment/Reasoning`)}`; }); } csvContent += "\n"; @@ -54,8 +54,9 @@ export const exportGroupedCSV = ( row.push(group.trace_ids[i] || ""); scoreNames.forEach((name) => { const score = group.scores[i]?.find((s) => s.name === name); + const note = getScoreNote(score); row.push(score ? String(score.value) : ""); - row.push(score?.comment ? sanitizeCSVCell(score.comment, true) : ""); + row.push(note ? sanitizeCSVCell(note, true) : ""); }); } csvContent += row.join(",") + "\n"; @@ -89,7 +90,8 @@ export const exportRowCSV = ( if (hasAnyCategory) csvContent += "Category,"; csvContent += "Question,Answer,Ground Truth,"; csvContent += - scoreNames.map((name) => `${name},${name} (comment)`).join(",") + "\n"; + scoreNames.map((name) => `${name},${name} (comment/reasoning)`).join(",") + + "\n"; let rowCount = 0; individual_scores.forEach((item, index) => { @@ -108,9 +110,10 @@ export const exportRowCSV = ( `"${(item.metadata?.ground_truth || "").replace(/"/g, '""').replace(/\n/g, " ")}"`, ...scoreNames.flatMap((name) => { const score = item.trace_scores?.find((s) => s.name === name); + const note = getScoreNote(score); return [ score ? score.value : "N/A", - score?.comment ? sanitizeCSVCell(score.comment, true) : "", + note ? sanitizeCSVCell(note, true) : "", ]; }), ].join(",");