diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx index 535a88a1..8bc289c8 100644 --- a/app/(main)/evaluations/[id]/page.tsx +++ b/app/(main)/evaluations/[id]/page.tsx @@ -399,7 +399,10 @@ export default function EvaluationReport() { onResync={handleResync} /> {categoryMetrics.length > 0 && ( - + )} ) : ( diff --git a/app/components/evaluations/CategoryMetricsTable.tsx b/app/components/evaluations/CategoryMetricsTable.tsx index cb5b9d66..60f525a9 100644 --- a/app/components/evaluations/CategoryMetricsTable.tsx +++ b/app/components/evaluations/CategoryMetricsTable.tsx @@ -4,6 +4,7 @@ import { CategoryMetric } from "@/app/lib/types/evaluation"; interface CategoryMetricsTableProps { categoryMetrics: CategoryMetric[]; + isJudgeRun?: boolean; } function formatScore(value: number | null): string { @@ -13,6 +14,7 @@ function formatScore(value: number | null): string { export default function CategoryMetricsTable({ categoryMetrics, + isJudgeRun, }: CategoryMetricsTableProps) { if (!categoryMetrics || categoryMetrics.length === 0) return null; @@ -36,9 +38,11 @@ export default function CategoryMetricsTable({ Total Questions - - Avg Cosine - + {!isJudgeRun && ( + + Avg Cosine + + )} Avg Correctness @@ -56,9 +60,11 @@ export default function CategoryMetricsTable({ {row.total_evals} - - {formatScore(row.avg_cosine)} - + {!isJudgeRun && ( + + {formatScore(row.avg_cosine)} + + )} {formatScore(row.avg_correctness)} diff --git a/app/components/evaluations/DetailedResultsTable.tsx b/app/components/evaluations/DetailedResultsTable.tsx index 8d6a9a2c..e3973019 100644 --- a/app/components/evaluations/DetailedResultsTable.tsx +++ b/app/components/evaluations/DetailedResultsTable.tsx @@ -12,11 +12,13 @@ import { hasSummaryScores, isNewScoreObjectV2, isGroupedFormat, + isCosineScoreName, } from "@/app/lib/utils/evaluation"; import { formatScoreValue, getScoreByName } from "@/app/lib/utils"; import { InfoTooltip } from "@/app/components/ui"; import { GroupedResultsTable } from "@/app/components/evaluations"; import { MarkdownContent } from "@/app/components/chat"; +import { ExternalLinkIcon } from "@/app/components/icons"; interface DetailedResultsTableProps { job: EvalJob; @@ -26,6 +28,7 @@ export default function DetailedResultsTable({ job, }: DetailedResultsTableProps) { const scoreObject = getScoreObject(job); + const isJudgeRun = job.is_judge_run; if (!scoreObject || !hasSummaryScores(scoreObject)) { return ( @@ -42,6 +45,7 @@ export default function DetailedResultsTable({ return ( ); } @@ -61,12 +65,16 @@ export default function DetailedResultsTable({ } // Get all unique score names from the first item - const scoreNames = + const allScoreNames = individual_scores[0]?.trace_scores?.map((s) => s.name) || []; + const scoreNames = isJudgeRun + ? allScoreNames.filter((name) => !isCosineScoreName(name)) + : allScoreNames; const hasAnyCategory = individual_scores.some( (s) => (s.category ?? "").trim().length > 0, ); + const hasAnyTraceUrl = individual_scores.some((s) => s.score_trace_url); const COLUMN_WIDTHS = { index: 50, @@ -75,6 +83,7 @@ export default function DetailedResultsTable({ groundTruth: 250, answer: 250, score: 160, + trace: 90, }; const tableMinWidth = COLUMN_WIDTHS.index + @@ -82,7 +91,8 @@ export default function DetailedResultsTable({ COLUMN_WIDTHS.question + COLUMN_WIDTHS.groundTruth + COLUMN_WIDTHS.answer + - scoreNames.length * COLUMN_WIDTHS.score; + scoreNames.length * COLUMN_WIDTHS.score + + (hasAnyTraceUrl ? COLUMN_WIDTHS.trace : 0); return (
@@ -132,6 +142,14 @@ export default function DetailedResultsTable({ {scoreName} ))} + {hasAnyTraceUrl && ( + + Trace + + )} @@ -209,6 +227,23 @@ export default function DetailedResultsTable({ ); })} + + {hasAnyTraceUrl && ( + + {item.score_trace_url && ( + + + + )} + + )} ); })} diff --git a/app/components/evaluations/EvalRunCard.tsx b/app/components/evaluations/EvalRunCard.tsx index 5a74da9e..f457ba1d 100644 --- a/app/components/evaluations/EvalRunCard.tsx +++ b/app/components/evaluations/EvalRunCard.tsx @@ -25,6 +25,7 @@ export default function EvalRunCard({ const isCompleted = job.status?.toLowerCase() === "completed"; const scoreObj = getScoreObject(job); const statusColor = getStatusColor(job.status || ""); + const isJudgeRun = job.is_judge_run; return (
- +
)} @@ -91,7 +96,7 @@ export default function EvalRunCard({ )} - {job.cost.embedding && ( + {job.cost.embedding && !isJudgeRun && ( <> Cosine similarity calculation diff --git a/app/components/evaluations/GroupedResultsTable.tsx b/app/components/evaluations/GroupedResultsTable.tsx index 8a905709..ea8cc534 100644 --- a/app/components/evaluations/GroupedResultsTable.tsx +++ b/app/components/evaluations/GroupedResultsTable.tsx @@ -6,12 +6,15 @@ import { Fragment } from "react"; import { TraceScore, GroupedTraceItem } from "@/app/lib/types/evaluation"; +import { isCosineScoreName } from "@/app/lib/utils/evaluation"; import { formatScoreValue } from "@/app/lib/utils"; import { InfoTooltip } from "@/app/components/ui"; export default function GroupedResultsTable({ traces, + isJudgeRun, }: { traces: GroupedTraceItem[]; + isJudgeRun?: boolean; }) { if (!traces || traces.length === 0) { return ( @@ -171,8 +174,13 @@ export default function GroupedResultsTable({ {Array.from({ length: maxAnswers }, (_, answerIndex) => { - const answerScores: TraceScore[] = + const rawAnswerScores: TraceScore[] = group.scores?.[answerIndex] || []; + const answerScores = isJudgeRun + ? rawAnswerScores.filter( + (s) => !isCosineScoreName(s.name), + ) + : rawAnswerScores; const answer = group.llm_answers[answerIndex]; return ( diff --git a/app/components/evaluations/MetricsOverview.tsx b/app/components/evaluations/MetricsOverview.tsx index b27a11c7..5975d571 100644 --- a/app/components/evaluations/MetricsOverview.tsx +++ b/app/components/evaluations/MetricsOverview.tsx @@ -1,6 +1,7 @@ "use client"; import type { EvalJob, SummaryScore } from "@/app/lib/types/evaluation"; +import { isCosineScoreName } from "@/app/lib/utils/evaluation"; import { Button } from "@/app/components/ui"; import { RefreshIcon, WarningTriangleIcon } from "@/app/components/icons"; @@ -14,11 +15,15 @@ interface MetricsOverviewProps { export default function MetricsOverview({ job, - summaryScores, + summaryScores: allSummaryScores, isJobInProgress, isResyncing, onResync, }: MetricsOverviewProps) { + const isJudgeRun = job.is_judge_run; + const summaryScores = isJudgeRun + ? allSummaryScores.filter((s) => !isCosineScoreName(s.name)) + : allSummaryScores; const showPartialNotice = summaryScores.some( (s) => job.total_items && s.total_pairs < job.total_items, @@ -38,15 +43,17 @@ export default function MetricsOverview({

Metrics Overview

- + {!isJudgeRun && ( + + )}
{summaryScores.length > 0 ? (
diff --git a/app/components/evaluations/ScoreDisplay.tsx b/app/components/evaluations/ScoreDisplay.tsx index 68efa331..2877941d 100644 --- a/app/components/evaluations/ScoreDisplay.tsx +++ b/app/components/evaluations/ScoreDisplay.tsx @@ -6,16 +6,21 @@ "use client"; import type { ScoreObject } from "@/app/lib/types/evaluation"; -import { hasSummaryScores } from "@/app/lib/utils/evaluation"; +import { + hasSummaryScores, + isCosineScoreName, +} from "@/app/lib/utils/evaluation"; interface ScoreDisplayProps { score: ScoreObject | null; errorMessage?: string | null; + isJudgeRun?: boolean; } export default function ScoreDisplay({ score, errorMessage, + isJudgeRun, }: ScoreDisplayProps) { if (!score) { return ( @@ -31,7 +36,9 @@ export default function ScoreDisplay({ // Handle score format with summary_scores (V1, V2, or Basic) if (hasSummaryScores(score)) { - const summaryScores = score.summary_scores || []; + const summaryScores = (score.summary_scores || []).filter( + (s) => !isJudgeRun || !isCosineScoreName(s.name), + ); if (summaryScores.length === 0) { return ( diff --git a/app/components/icons/evaluations/ExternalLinkIcon.tsx b/app/components/icons/evaluations/ExternalLinkIcon.tsx new file mode 100644 index 00000000..89aa4e42 --- /dev/null +++ b/app/components/icons/evaluations/ExternalLinkIcon.tsx @@ -0,0 +1,21 @@ +interface IconProps { + className?: string; +} + +export default function ExternalLinkIcon({ className }: IconProps) { + return ( + + + + + + ); +} diff --git a/app/components/icons/index.tsx b/app/components/icons/index.tsx index ed9c9556..eb9ad6e2 100644 --- a/app/components/icons/index.tsx +++ b/app/components/icons/index.tsx @@ -29,6 +29,7 @@ export { default as EditIcon } from "./evaluations/EditIcon"; export { default as MenuIcon } from "./evaluations/MenuIcon"; export { default as DatabaseIcon } from "./evaluations/DatabaseIcon"; export { default as GroupIcon } from "./evaluations/GroupIcon"; +export { default as ExternalLinkIcon } from "./evaluations/ExternalLinkIcon"; // Document Icons export { default as DocumentFileIcon } from "./document/DocumentFileIcon"; diff --git a/app/lib/types/evaluation.ts b/app/lib/types/evaluation.ts index 5b0eb7f2..de83c301 100644 --- a/app/lib/types/evaluation.ts +++ b/app/lib/types/evaluation.ts @@ -15,6 +15,7 @@ export interface TraceItem { question_id?: number; category?: string; scores: TraceScore[]; + score_trace_url?: string; } export interface GroupedTraceItem { @@ -42,6 +43,7 @@ export interface IndividualScore { response_id?: string; }; trace_scores: TraceScore[]; + score_trace_url?: string; } export interface CategoryMetric { @@ -129,6 +131,7 @@ export interface EvalJob { id: number; run_name: string; run_mode?: RunMode; + is_judge_run?: boolean; dataset_name: string; dataset_id: number; batch_job_id: number; diff --git a/app/lib/utils/evaluation.ts b/app/lib/utils/evaluation.ts index 1403499b..b64fc7c2 100644 --- a/app/lib/utils/evaluation.ts +++ b/app/lib/utils/evaluation.ts @@ -26,6 +26,10 @@ export function getScoreObject(job: EvalJob): ScoreObject | null { return job.scores || job.score || null; } +export function isCosineScoreName(name: string): boolean { + return /cosine/i.test(name); +} + export function isGroupedFormat( traces: TraceItem[] | GroupedTraceItem[], ): traces is GroupedTraceItem[] { @@ -47,6 +51,7 @@ export function normalizeToIndividualScores( output: { answer: trace.llm_answer }, metadata: { ground_truth: trace.ground_truth_answer }, trace_scores: trace.scores, + score_trace_url: trace.score_trace_url, }; } return { trace_id: "", trace_scores: [] };