From 7ae471e26239c8bd448fcfc588c3a9e20a53b7b8 Mon Sep 17 00:00:00 2001 From: "Kaapi-Agent [Bot]" <4937976+Kaapi-Agent [Bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:39:17 +0000 Subject: [PATCH] fix: hide cosine and Langfuse resync for judge runs Judge runs (v2) never compute cosine similarity or interact with Langfuse, so run cards, run list, and the detail page should not show cosine metrics or the Langfuse-based resync flow for them. A new native score_trace_url per item is surfaced as a "View Trace" link instead. Co-Authored-By: Claude Sonnet 5 --- app/(main)/evaluations/[id]/page.tsx | 5 ++- .../evaluations/CategoryMetricsTable.tsx | 18 ++++++--- .../evaluations/DetailedResultsTable.tsx | 39 ++++++++++++++++++- app/components/evaluations/EvalRunCard.tsx | 9 ++++- .../evaluations/GroupedResultsTable.tsx | 10 ++++- .../evaluations/MetricsOverview.tsx | 27 ++++++++----- app/components/evaluations/ScoreDisplay.tsx | 11 +++++- .../icons/evaluations/ExternalLinkIcon.tsx | 21 ++++++++++ app/components/icons/index.tsx | 1 + app/lib/types/evaluation.ts | 3 ++ app/lib/utils/evaluation.ts | 5 +++ 11 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 app/components/icons/evaluations/ExternalLinkIcon.tsx diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx index 535a88a1..8bc289c8 100644 --- a/app/(main)/evaluations/[id]/page.tsx +++ b/app/(main)/evaluations/[id]/page.tsx @@ -399,7 +399,10 @@ export default function EvaluationReport() { onResync={handleResync} /> {categoryMetrics.length > 0 && ( - + )} ) : ( diff --git a/app/components/evaluations/CategoryMetricsTable.tsx b/app/components/evaluations/CategoryMetricsTable.tsx index cb5b9d66..60f525a9 100644 --- a/app/components/evaluations/CategoryMetricsTable.tsx +++ b/app/components/evaluations/CategoryMetricsTable.tsx @@ -4,6 +4,7 @@ import { CategoryMetric } from "@/app/lib/types/evaluation"; interface CategoryMetricsTableProps { categoryMetrics: CategoryMetric[]; + isJudgeRun?: boolean; } function formatScore(value: number | null): string { @@ -13,6 +14,7 @@ function formatScore(value: number | null): string { export default function CategoryMetricsTable({ categoryMetrics, + isJudgeRun, }: CategoryMetricsTableProps) { if (!categoryMetrics || categoryMetrics.length === 0) return null; @@ -36,9 +38,11 @@ export default function CategoryMetricsTable({ Total Questions - - Avg Cosine - + {!isJudgeRun && ( + + Avg Cosine + + )} Avg Correctness @@ -56,9 +60,11 @@ export default function CategoryMetricsTable({ {row.total_evals} - - {formatScore(row.avg_cosine)} - + {!isJudgeRun && ( + + {formatScore(row.avg_cosine)} + + )} {formatScore(row.avg_correctness)} diff --git a/app/components/evaluations/DetailedResultsTable.tsx b/app/components/evaluations/DetailedResultsTable.tsx index 8d6a9a2c..e3973019 100644 --- a/app/components/evaluations/DetailedResultsTable.tsx +++ b/app/components/evaluations/DetailedResultsTable.tsx @@ -12,11 +12,13 @@ import { hasSummaryScores, isNewScoreObjectV2, isGroupedFormat, + isCosineScoreName, } from "@/app/lib/utils/evaluation"; import { formatScoreValue, getScoreByName } from "@/app/lib/utils"; import { InfoTooltip } from "@/app/components/ui"; import { GroupedResultsTable } from "@/app/components/evaluations"; import { MarkdownContent } from "@/app/components/chat"; +import { ExternalLinkIcon } from "@/app/components/icons"; interface DetailedResultsTableProps { job: EvalJob; @@ -26,6 +28,7 @@ export default function DetailedResultsTable({ job, }: DetailedResultsTableProps) { const scoreObject = getScoreObject(job); + const isJudgeRun = job.is_judge_run; if (!scoreObject || !hasSummaryScores(scoreObject)) { return ( @@ -42,6 +45,7 @@ export default function DetailedResultsTable({ return ( ); } @@ -61,12 +65,16 @@ export default function DetailedResultsTable({ } // Get all unique score names from the first item - const scoreNames = + const allScoreNames = individual_scores[0]?.trace_scores?.map((s) => s.name) || []; + const scoreNames = isJudgeRun + ? allScoreNames.filter((name) => !isCosineScoreName(name)) + : allScoreNames; const hasAnyCategory = individual_scores.some( (s) => (s.category ?? "").trim().length > 0, ); + const hasAnyTraceUrl = individual_scores.some((s) => s.score_trace_url); const COLUMN_WIDTHS = { index: 50, @@ -75,6 +83,7 @@ export default function DetailedResultsTable({ groundTruth: 250, answer: 250, score: 160, + trace: 90, }; const tableMinWidth = COLUMN_WIDTHS.index + @@ -82,7 +91,8 @@ export default function DetailedResultsTable({ COLUMN_WIDTHS.question + COLUMN_WIDTHS.groundTruth + COLUMN_WIDTHS.answer + - scoreNames.length * COLUMN_WIDTHS.score; + scoreNames.length * COLUMN_WIDTHS.score + + (hasAnyTraceUrl ? COLUMN_WIDTHS.trace : 0); return (
@@ -132,6 +142,14 @@ export default function DetailedResultsTable({ {scoreName} ))} + {hasAnyTraceUrl && ( + + Trace + + )} @@ -209,6 +227,23 @@ export default function DetailedResultsTable({ ); })} + + {hasAnyTraceUrl && ( + + {item.score_trace_url && ( + + + + )} + + )} ); })} diff --git a/app/components/evaluations/EvalRunCard.tsx b/app/components/evaluations/EvalRunCard.tsx index 5a74da9e..f457ba1d 100644 --- a/app/components/evaluations/EvalRunCard.tsx +++ b/app/components/evaluations/EvalRunCard.tsx @@ -25,6 +25,7 @@ export default function EvalRunCard({ const isCompleted = job.status?.toLowerCase() === "completed"; const scoreObj = getScoreObject(job); const statusColor = getStatusColor(job.status || ""); + const isJudgeRun = job.is_judge_run; return (
- +
)} @@ -91,7 +96,7 @@ export default function EvalRunCard({ )} - {job.cost.embedding && ( + {job.cost.embedding && !isJudgeRun && ( <> Cosine similarity calculation diff --git a/app/components/evaluations/GroupedResultsTable.tsx b/app/components/evaluations/GroupedResultsTable.tsx index 8a905709..ea8cc534 100644 --- a/app/components/evaluations/GroupedResultsTable.tsx +++ b/app/components/evaluations/GroupedResultsTable.tsx @@ -6,12 +6,15 @@ import { Fragment } from "react"; import { TraceScore, GroupedTraceItem } from "@/app/lib/types/evaluation"; +import { isCosineScoreName } from "@/app/lib/utils/evaluation"; import { formatScoreValue } from "@/app/lib/utils"; import { InfoTooltip } from "@/app/components/ui"; export default function GroupedResultsTable({ traces, + isJudgeRun, }: { traces: GroupedTraceItem[]; + isJudgeRun?: boolean; }) { if (!traces || traces.length === 0) { return ( @@ -171,8 +174,13 @@ export default function GroupedResultsTable({ {Array.from({ length: maxAnswers }, (_, answerIndex) => { - const answerScores: TraceScore[] = + const rawAnswerScores: TraceScore[] = group.scores?.[answerIndex] || []; + const answerScores = isJudgeRun + ? rawAnswerScores.filter( + (s) => !isCosineScoreName(s.name), + ) + : rawAnswerScores; const answer = group.llm_answers[answerIndex]; return ( diff --git a/app/components/evaluations/MetricsOverview.tsx b/app/components/evaluations/MetricsOverview.tsx index b27a11c7..5975d571 100644 --- a/app/components/evaluations/MetricsOverview.tsx +++ b/app/components/evaluations/MetricsOverview.tsx @@ -1,6 +1,7 @@ "use client"; import type { EvalJob, SummaryScore } from "@/app/lib/types/evaluation"; +import { isCosineScoreName } from "@/app/lib/utils/evaluation"; import { Button } from "@/app/components/ui"; import { RefreshIcon, WarningTriangleIcon } from "@/app/components/icons"; @@ -14,11 +15,15 @@ interface MetricsOverviewProps { export default function MetricsOverview({ job, - summaryScores, + summaryScores: allSummaryScores, isJobInProgress, isResyncing, onResync, }: MetricsOverviewProps) { + const isJudgeRun = job.is_judge_run; + const summaryScores = isJudgeRun + ? allSummaryScores.filter((s) => !isCosineScoreName(s.name)) + : allSummaryScores; const showPartialNotice = summaryScores.some( (s) => job.total_items && s.total_pairs < job.total_items, @@ -38,15 +43,17 @@ export default function MetricsOverview({

Metrics Overview

- + {!isJudgeRun && ( + + )}
{summaryScores.length > 0 ? (
diff --git a/app/components/evaluations/ScoreDisplay.tsx b/app/components/evaluations/ScoreDisplay.tsx index 68efa331..2877941d 100644 --- a/app/components/evaluations/ScoreDisplay.tsx +++ b/app/components/evaluations/ScoreDisplay.tsx @@ -6,16 +6,21 @@ "use client"; import type { ScoreObject } from "@/app/lib/types/evaluation"; -import { hasSummaryScores } from "@/app/lib/utils/evaluation"; +import { + hasSummaryScores, + isCosineScoreName, +} from "@/app/lib/utils/evaluation"; interface ScoreDisplayProps { score: ScoreObject | null; errorMessage?: string | null; + isJudgeRun?: boolean; } export default function ScoreDisplay({ score, errorMessage, + isJudgeRun, }: ScoreDisplayProps) { if (!score) { return ( @@ -31,7 +36,9 @@ export default function ScoreDisplay({ // Handle score format with summary_scores (V1, V2, or Basic) if (hasSummaryScores(score)) { - const summaryScores = score.summary_scores || []; + const summaryScores = (score.summary_scores || []).filter( + (s) => !isJudgeRun || !isCosineScoreName(s.name), + ); if (summaryScores.length === 0) { return ( diff --git a/app/components/icons/evaluations/ExternalLinkIcon.tsx b/app/components/icons/evaluations/ExternalLinkIcon.tsx new file mode 100644 index 00000000..89aa4e42 --- /dev/null +++ b/app/components/icons/evaluations/ExternalLinkIcon.tsx @@ -0,0 +1,21 @@ +interface IconProps { + className?: string; +} + +export default function ExternalLinkIcon({ className }: IconProps) { + return ( + + + + + + ); +} diff --git a/app/components/icons/index.tsx b/app/components/icons/index.tsx index ed9c9556..eb9ad6e2 100644 --- a/app/components/icons/index.tsx +++ b/app/components/icons/index.tsx @@ -29,6 +29,7 @@ export { default as EditIcon } from "./evaluations/EditIcon"; export { default as MenuIcon } from "./evaluations/MenuIcon"; export { default as DatabaseIcon } from "./evaluations/DatabaseIcon"; export { default as GroupIcon } from "./evaluations/GroupIcon"; +export { default as ExternalLinkIcon } from "./evaluations/ExternalLinkIcon"; // Document Icons export { default as DocumentFileIcon } from "./document/DocumentFileIcon"; diff --git a/app/lib/types/evaluation.ts b/app/lib/types/evaluation.ts index 5b0eb7f2..de83c301 100644 --- a/app/lib/types/evaluation.ts +++ b/app/lib/types/evaluation.ts @@ -15,6 +15,7 @@ export interface TraceItem { question_id?: number; category?: string; scores: TraceScore[]; + score_trace_url?: string; } export interface GroupedTraceItem { @@ -42,6 +43,7 @@ export interface IndividualScore { response_id?: string; }; trace_scores: TraceScore[]; + score_trace_url?: string; } export interface CategoryMetric { @@ -129,6 +131,7 @@ export interface EvalJob { id: number; run_name: string; run_mode?: RunMode; + is_judge_run?: boolean; dataset_name: string; dataset_id: number; batch_job_id: number; diff --git a/app/lib/utils/evaluation.ts b/app/lib/utils/evaluation.ts index 1403499b..b64fc7c2 100644 --- a/app/lib/utils/evaluation.ts +++ b/app/lib/utils/evaluation.ts @@ -26,6 +26,10 @@ export function getScoreObject(job: EvalJob): ScoreObject | null { return job.scores || job.score || null; } +export function isCosineScoreName(name: string): boolean { + return /cosine/i.test(name); +} + export function isGroupedFormat( traces: TraceItem[] | GroupedTraceItem[], ): traces is GroupedTraceItem[] { @@ -47,6 +51,7 @@ export function normalizeToIndividualScores( output: { answer: trace.llm_answer }, metadata: { ground_truth: trace.ground_truth_answer }, trace_scores: trace.scores, + score_trace_url: trace.score_trace_url, }; } return { trace_id: "", trace_scores: [] };