diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx
index 535a88a..dad5acb 100644
--- a/app/(main)/evaluations/[id]/page.tsx
+++ b/app/(main)/evaluations/[id]/page.tsx
@@ -271,6 +271,10 @@ export default function EvaluationReport() {
job.status.toLowerCase() !== "completed" &&
job.status.toLowerCase() !== "failed";
+ // v2 prompt recommendations require judge-scored traces; v1 runs only
+ // have cosine similarity and can't be iterated on.
+ const isJudgedRun = !!scoreObject && isNewScoreObjectV2(scoreObject);
+
return (
@@ -356,6 +360,7 @@ export default function EvaluationReport() {
onClick={handleIteratePrompt}
disabled={
job.status.toLowerCase() !== "completed" ||
+ !isJudgedRun ||
isImprovingPrompt ||
isFormatSwitching ||
isResyncing
@@ -363,7 +368,9 @@ export default function EvaluationReport() {
title={
job.status.toLowerCase() !== "completed"
? "Only available for completed evaluations"
- : undefined
+ : !isJudgedRun
+ ? "Prompt iteration requires a judged evaluation run"
+ : undefined
}
>
{isImprovingPrompt && }
diff --git a/app/api/evaluations/[id]/improve-prompt/route.ts b/app/api/evaluations/[id]/improve-prompt/route.ts
index ffbc2f1..5991bcb 100644
--- a/app/api/evaluations/[id]/improve-prompt/route.ts
+++ b/app/api/evaluations/[id]/improve-prompt/route.ts
@@ -47,7 +47,7 @@ export async function POST(
const { status, data } = await apiClient(
request,
- `/api/v1/evaluations/${id}/improve-prompt`,
+ `/api/v2/evaluations/${id}/improve-prompt`,
{
method: "POST",
body: JSON.stringify(payload),
diff --git a/app/api/webhooks/prompt-improvement/route.ts b/app/api/webhooks/prompt-improvement/route.ts
index 8fe0ea9..c943b7b 100644
--- a/app/api/webhooks/prompt-improvement/route.ts
+++ b/app/api/webhooks/prompt-improvement/route.ts
@@ -4,7 +4,7 @@ import {
saveJobSnapshot,
} from "@/app/lib/store/promptImprovementStore";
import { readWebhookSecret } from "@/app/lib/webhookSecret";
-import type { PromptImprovementJobPublic } from "@/app/lib/types/promptImprovement";
+import type { PromptRecommendationJobPublic } from "@/app/lib/types/promptImprovement";
function isAuthorized(request: NextRequest): boolean {
const secretResult = readWebhookSecret();
@@ -23,10 +23,14 @@ export async function POST(request: NextRequest) {
}
const body = (await request.json()) as {
success?: boolean;
- data?: PromptImprovementJobPublic;
+ data?: PromptRecommendationJobPublic;
error?: string | null;
};
- if (!body.data || !body.data.job_id) {
+ if (
+ !body.data ||
+ !body.data.job_id ||
+ body.data.recommendation_type !== "prompt"
+ ) {
return NextResponse.json(
{ success: false, error: "invalid_payload" },
{ status: 400 },
diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts
index 6b77bb4..e7f5196 100644
--- a/app/hooks/usePromptImprovement.ts
+++ b/app/hooks/usePromptImprovement.ts
@@ -24,6 +24,8 @@ function iteratePromptSyncError(code: string): string {
return "Source config is no longer available for this evaluation";
case "traces_not_available":
return "Evaluation traces aren't available yet — try Resync first";
+ case "not_a_judge_run":
+ return "Prompt iteration requires a judged evaluation run";
case "invalid_callback_url":
return "Frontend callback URL is invalid. Set NEXT_PUBLIC_APP_URL to a public HTTPS host.";
case "prompt_improvement_enqueue_failed":
diff --git a/app/lib/store/promptImprovementStore.ts b/app/lib/store/promptImprovementStore.ts
index 9909fbd..79c693a 100644
--- a/app/lib/store/promptImprovementStore.ts
+++ b/app/lib/store/promptImprovementStore.ts
@@ -6,9 +6,9 @@
*/
import type {
- PromptImprovementJobPublic,
PromptImprovementJobSnapshot,
PromptImprovementStoreShape,
+ PromptRecommendationJobPublic,
SnapshotListener,
} from "@/app/lib/types/promptImprovement";
@@ -74,7 +74,7 @@ function prune(): void {
}
}
-export function saveJobSnapshot(job: PromptImprovementJobPublic): void {
+export function saveJobSnapshot(job: PromptRecommendationJobPublic): void {
const snapshot: PromptImprovementJobSnapshot = {
...job,
updated_at: new Date().toISOString(),
@@ -89,6 +89,7 @@ export function markJobPending(jobId: string): void {
const snapshot: PromptImprovementJobSnapshot = {
job_id: jobId,
status: "PENDING",
+ recommendation_type: "prompt",
config_version: null,
error_message: null,
updated_at: new Date().toISOString(),
diff --git a/app/lib/types/promptImprovement.ts b/app/lib/types/promptImprovement.ts
index 2b9361c..63b099b 100644
--- a/app/lib/types/promptImprovement.ts
+++ b/app/lib/types/promptImprovement.ts
@@ -16,9 +16,14 @@ import type { ConfigVersionPublic } from "@/app/lib/types/configs";
export type PromptImprovementConfigVersion = ConfigVersionPublic;
-export interface PromptImprovementJobPublic {
+/** Only "prompt" exists today; the backend will widen this to a union as
+ * more recommendation types (e.g. config) are added. */
+export type RecommendationType = "prompt";
+
+export interface PromptRecommendationJobPublic {
job_id: string;
status: PromptImprovementStatus;
+ recommendation_type: RecommendationType;
config_version: PromptImprovementConfigVersion | null;
error_message: string | null;
}
@@ -26,6 +31,7 @@ export interface PromptImprovementJobPublic {
export interface PromptImprovementJobSnapshot {
job_id: string;
status: PromptImprovementStatus;
+ recommendation_type: RecommendationType;
config_version: PromptImprovementConfigVersion | null;
error_message: string | null;
updated_at: string;