diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx index 535a88a..dad5acb 100644 --- a/app/(main)/evaluations/[id]/page.tsx +++ b/app/(main)/evaluations/[id]/page.tsx @@ -271,6 +271,10 @@ export default function EvaluationReport() { job.status.toLowerCase() !== "completed" && job.status.toLowerCase() !== "failed"; + // v2 prompt recommendations require judge-scored traces; v1 runs only + // have cosine similarity and can't be iterated on. + const isJudgedRun = !!scoreObject && isNewScoreObjectV2(scoreObject); + return (
@@ -356,6 +360,7 @@ export default function EvaluationReport() { onClick={handleIteratePrompt} disabled={ job.status.toLowerCase() !== "completed" || + !isJudgedRun || isImprovingPrompt || isFormatSwitching || isResyncing @@ -363,7 +368,9 @@ export default function EvaluationReport() { title={ job.status.toLowerCase() !== "completed" ? "Only available for completed evaluations" - : undefined + : !isJudgedRun + ? "Prompt iteration requires a judged evaluation run" + : undefined } > {isImprovingPrompt && } diff --git a/app/api/evaluations/[id]/improve-prompt/route.ts b/app/api/evaluations/[id]/improve-prompt/route.ts index ffbc2f1..5991bcb 100644 --- a/app/api/evaluations/[id]/improve-prompt/route.ts +++ b/app/api/evaluations/[id]/improve-prompt/route.ts @@ -47,7 +47,7 @@ export async function POST( const { status, data } = await apiClient( request, - `/api/v1/evaluations/${id}/improve-prompt`, + `/api/v2/evaluations/${id}/improve-prompt`, { method: "POST", body: JSON.stringify(payload), diff --git a/app/api/webhooks/prompt-improvement/route.ts b/app/api/webhooks/prompt-improvement/route.ts index 8fe0ea9..c943b7b 100644 --- a/app/api/webhooks/prompt-improvement/route.ts +++ b/app/api/webhooks/prompt-improvement/route.ts @@ -4,7 +4,7 @@ import { saveJobSnapshot, } from "@/app/lib/store/promptImprovementStore"; import { readWebhookSecret } from "@/app/lib/webhookSecret"; -import type { PromptImprovementJobPublic } from "@/app/lib/types/promptImprovement"; +import type { PromptRecommendationJobPublic } from "@/app/lib/types/promptImprovement"; function isAuthorized(request: NextRequest): boolean { const secretResult = readWebhookSecret(); @@ -23,10 +23,14 @@ export async function POST(request: NextRequest) { } const body = (await request.json()) as { success?: boolean; - data?: PromptImprovementJobPublic; + data?: PromptRecommendationJobPublic; error?: string | null; }; - if (!body.data || !body.data.job_id) { + if ( + !body.data || + !body.data.job_id || + body.data.recommendation_type !== "prompt" + ) { return NextResponse.json( { success: false, error: "invalid_payload" }, { status: 400 }, diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts index 6b77bb4..e7f5196 100644 --- a/app/hooks/usePromptImprovement.ts +++ b/app/hooks/usePromptImprovement.ts @@ -24,6 +24,8 @@ function iteratePromptSyncError(code: string): string { return "Source config is no longer available for this evaluation"; case "traces_not_available": return "Evaluation traces aren't available yet — try Resync first"; + case "not_a_judge_run": + return "Prompt iteration requires a judged evaluation run"; case "invalid_callback_url": return "Frontend callback URL is invalid. Set NEXT_PUBLIC_APP_URL to a public HTTPS host."; case "prompt_improvement_enqueue_failed": diff --git a/app/lib/store/promptImprovementStore.ts b/app/lib/store/promptImprovementStore.ts index 9909fbd..79c693a 100644 --- a/app/lib/store/promptImprovementStore.ts +++ b/app/lib/store/promptImprovementStore.ts @@ -6,9 +6,9 @@ */ import type { - PromptImprovementJobPublic, PromptImprovementJobSnapshot, PromptImprovementStoreShape, + PromptRecommendationJobPublic, SnapshotListener, } from "@/app/lib/types/promptImprovement"; @@ -74,7 +74,7 @@ function prune(): void { } } -export function saveJobSnapshot(job: PromptImprovementJobPublic): void { +export function saveJobSnapshot(job: PromptRecommendationJobPublic): void { const snapshot: PromptImprovementJobSnapshot = { ...job, updated_at: new Date().toISOString(), @@ -89,6 +89,7 @@ export function markJobPending(jobId: string): void { const snapshot: PromptImprovementJobSnapshot = { job_id: jobId, status: "PENDING", + recommendation_type: "prompt", config_version: null, error_message: null, updated_at: new Date().toISOString(), diff --git a/app/lib/types/promptImprovement.ts b/app/lib/types/promptImprovement.ts index 2b9361c..63b099b 100644 --- a/app/lib/types/promptImprovement.ts +++ b/app/lib/types/promptImprovement.ts @@ -16,9 +16,14 @@ import type { ConfigVersionPublic } from "@/app/lib/types/configs"; export type PromptImprovementConfigVersion = ConfigVersionPublic; -export interface PromptImprovementJobPublic { +/** Only "prompt" exists today; the backend will widen this to a union as + * more recommendation types (e.g. config) are added. */ +export type RecommendationType = "prompt"; + +export interface PromptRecommendationJobPublic { job_id: string; status: PromptImprovementStatus; + recommendation_type: RecommendationType; config_version: PromptImprovementConfigVersion | null; error_message: string | null; } @@ -26,6 +31,7 @@ export interface PromptImprovementJobPublic { export interface PromptImprovementJobSnapshot { job_id: string; status: PromptImprovementStatus; + recommendation_type: RecommendationType; config_version: PromptImprovementConfigVersion | null; error_message: string | null; updated_at: string;