Skip to content

feat(judge): Add new metrics support #279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kaapi-agent-bot wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(judge): Add new metrics support #279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions app/components/evaluations/DetailedResultsTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@ import {
isNewScoreObjectV2,
isGroupedFormat,
} from "@/app/lib/utils/evaluation";
import { formatScoreValue, getScoreByName } from "@/app/lib/utils";
import {
formatScoreValue,
getScoreByName,
getScoreNote,
} from "@/app/lib/utils";
import { InfoTooltip } from "@/app/components/ui";
import { GroupedResultsTable } from "@/app/components/evaluations";
import { MarkdownContent } from "@/app/components/chat";
Expand Down Expand Up @@ -186,6 +190,7 @@ export default function DetailedResultsTable({
{scoreNames.map((scoreName) => {
const score = getScoreByName(item.trace_scores, scoreName);
const { value, color, bg } = formatScoreValue(score);
const note = getScoreNote(score);

return (
<td
Expand All @@ -202,9 +207,7 @@ export default function DetailedResultsTable({
>
{value}
</div>
{score?.comment && (
<InfoTooltip text={score.comment} />
)}
{note && <InfoTooltip text={note} />}
</div>
</td>
);
Expand Down
7 changes: 3 additions & 4 deletions app/components/evaluations/GroupedResultsTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import { Fragment } from "react";
import { TraceScore, GroupedTraceItem } from "@/app/lib/types/evaluation";
import { formatScoreValue } from "@/app/lib/utils";
import { formatScoreValue, getScoreNote } from "@/app/lib/utils";
import { InfoTooltip } from "@/app/components/ui";
export default function GroupedResultsTable({
traces,
Expand Down Expand Up @@ -187,6 +187,7 @@ export default function GroupedResultsTable({
if (!score) return null;
const { value, color, bg } =
formatScoreValue(score);
const note = getScoreNote(score);
return (
<div
key={score.name || scoreIdx}
Expand All @@ -206,9 +207,7 @@ export default function GroupedResultsTable({
>
{value}
</div>
{score?.comment && (
<InfoTooltip text={score.comment} />
)}
{note && <InfoTooltip text={note} />}
</div>
</div>
);
Expand Down
2 changes: 2 additions & 0 deletions app/lib/types/evaluation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ export interface TraceScore {
value: number | string;
data_type: "NUMERIC" | "CATEGORICAL";
comment?: string;
/** Judge model's explanation for the score (v2 judge metrics, e.g. Adherence to Ground Truth/Prompt/Knowledge Base, 0-5 scale). */
reasoning?: string;
}

export interface TraceItem {
Expand Down
81 changes: 54 additions & 27 deletions app/lib/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,45 +190,64 @@ export const sanitizeCSVCell = (
return `"${sanitized}"`;
};

export const formatScoreValue = (score: TraceScore | undefined) => {
if (!score) return { value: "N/A", color: "#737373", bg: "transparent" };

if (score.data_type === "CATEGORICAL") {
const catValue = String(score.value);
let color = "#171717";
let bg = "#fafafa";

if (catValue === "CORRECT") {
color = "#15803d";
bg = "#dcfce7";
} else if (catValue === "PARTIAL") {
color = "#92400e";
bg = "#fef3c7";
} else if (catValue === "INCORRECT") {
color = "#dc2626";
bg = "#fee2e2";
}
// v2 judge metrics score 0-5 (integers) instead of the legacy 0-1 cosine/correctness scale.

@Ayush8923 Ayush8923 Sep 17, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in every function, not needed these types of the comments.

const FIVE_POINT_SCORE_NAMES = new Set([
"adherence to ground truth",
"adherence to prompt",
"adherence to knowledge base",
]);

return { value: catValue, color, bg };
}
const isFivePointScore = (name?: string): boolean =>
!!name && FIVE_POINT_SCORE_NAMES.has(name.toLowerCase());

const numValue = Number(score.value);
const formattedValue = numValue.toFixed(2);
const formatCategoricalScore = (value: number | string) => {
const catValue = String(value);

@Ayush8923 Ayush8923 Sep 17, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of using the catValue variable name, we should use a more descriptive and meaningful name that clearly indicates what the value represents.

let color = "#171717";
let bg = "transparent";
let bg = "#fafafa";

if (numValue >= 0.7) {
if (catValue === "CORRECT") {
color = "#15803d";
bg = "#dcfce7";
} else if (numValue >= 0.5) {
} else if (catValue === "PARTIAL") {
color = "#92400e";
bg = "#fef3c7";
} else {
} else if (catValue === "INCORRECT") {
color = "#dc2626";
bg = "#fee2e2";
}

return { value: formattedValue, color, bg };
return { value: catValue, color, bg };
};

// Traffic light thresholds for v2 judge metrics: 0-1 needs improvement, 2-3 could improve, 4-5 good.

@Ayush8923 Ayush8923 Sep 17, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

able to understand via function name. so need to remove this.

const formatFivePointScore = (numValue: number) => {
if (numValue >= 4)
return { value: String(numValue), color: "#15803d", bg: "#dcfce7" };
if (numValue >= 2)
return { value: String(numValue), color: "#92400e", bg: "#fef3c7" };
return { value: String(numValue), color: "#dc2626", bg: "#fee2e2" };
};

// Legacy thresholds for cosine similarity / correctness on a 0-1 scale.
const formatUnitScaleScore = (numValue: number) => {
const value = numValue.toFixed(2);
if (numValue >= 0.7) return { value, color: "#15803d", bg: "#dcfce7" };
if (numValue >= 0.5) return { value, color: "#92400e", bg: "#fef3c7" };
return { value, color: "#dc2626", bg: "#fee2e2" };
};

export const formatScoreValue = (score: TraceScore | undefined) => {
if (!score) return { value: "N/A", color: "#737373", bg: "transparent" };

if (score.data_type === "CATEGORICAL") {
return formatCategoricalScore(score.value);
}

const numValue = Number(score.value);

return isFivePointScore(score.name)
? formatFivePointScore(numValue)
: formatUnitScaleScore(numValue);
};

export const getScoreByName = (
Expand All @@ -239,6 +258,14 @@ export const getScoreByName = (
return scores.find((s) => s?.name === name);
};

/**
* Returns the judge's explanation for a score, preferring the v2 `reasoning`
* field over the legacy `comment` field.
*/
Comment on lines +261 to +264

@Ayush8923 Ayush8923 Sep 17, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not needed this comment.

export const getScoreNote = (
score: TraceScore | undefined,
): string | undefined => score?.reasoning || score?.comment;

/**
* Formats a USD cost value for display
* @param cost - Cost in USD
Expand Down
13 changes: 8 additions & 5 deletions app/lib/utils/evaluationExport.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type {
ScoreObject,
} from "@/app/lib/types/evaluation";
import { normalizeToIndividualScores } from "@/app/lib/utils/evaluation";
import { sanitizeCSVCell } from "@/app/lib/utils";
import { sanitizeCSVCell, getScoreNote } from "@/app/lib/utils";

const downloadCSV = (csvContent: string, filename: string) => {
const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" });
Expand Down Expand Up @@ -36,7 +36,7 @@ export const exportGroupedCSV = (
for (let i = 1; i <= maxAnswers; i++) {
csvContent += `,LLM Answer ${i},Trace ID ${i}`;
scoreNames.forEach((name) => {
csvContent += `,${name} (${i}),${sanitizeCSVCell(`${name} (${i}) Comment`)}`;
csvContent += `,${name} (${i}),${sanitizeCSVCell(`${name} (${i}) Comment/Reasoning`)}`;
});
}
csvContent += "\n";
Expand All @@ -54,8 +54,9 @@ export const exportGroupedCSV = (
row.push(group.trace_ids[i] || "");
scoreNames.forEach((name) => {
const score = group.scores[i]?.find((s) => s.name === name);
const note = getScoreNote(score);
row.push(score ? String(score.value) : "");
row.push(score?.comment ? sanitizeCSVCell(score.comment, true) : "");
row.push(note ? sanitizeCSVCell(note, true) : "");
});
}
csvContent += row.join(",") + "\n";
Expand Down Expand Up @@ -89,7 +90,8 @@ export const exportRowCSV = (
if (hasAnyCategory) csvContent += "Category,";
csvContent += "Question,Answer,Ground Truth,";
csvContent +=
scoreNames.map((name) => `${name},${name} (comment)`).join(",") + "\n";
scoreNames.map((name) => `${name},${name} (comment/reasoning)`).join(",") +
"\n";

let rowCount = 0;
individual_scores.forEach((item, index) => {
Expand All @@ -108,9 +110,10 @@ export const exportRowCSV = (
`"${(item.metadata?.ground_truth || "").replace(/"/g, '""').replace(/\n/g, " ")}"`,
...scoreNames.flatMap((name) => {
const score = item.trace_scores?.find((s) => s.name === name);
const note = getScoreNote(score);
return [
score ? score.value : "N/A",
score?.comment ? sanitizeCSVCell(score.comment, true) : "",
note ? sanitizeCSVCell(note, true) : "",
];
}),
].join(",");
Expand Down
Loading