diff --git a/backend/app/alembic/versions/085_add_last_dispatched_at_to_evaluation_iteration_run.py b/backend/app/alembic/versions/085_add_last_dispatched_at_to_evaluation_iteration_run.py new file mode 100644 index 000000000..e062c0e48 --- /dev/null +++ b/backend/app/alembic/versions/085_add_last_dispatched_at_to_evaluation_iteration_run.py @@ -0,0 +1,43 @@ +"""Add last_dispatched_at to evaluation_iteration_run + +Revision ID: 085 +Revises: 084 +Create Date: 2026-09-08 00:00:00.000000 + +The cron tick used to fan a `resume=True` graph step out to every PROCESSING +loop unconditionally, so a step slower than the tick interval got a second one +dispatched on top of it — two workers against the same LangGraph checkpoint +thread, and a duplicate eval run or improvement job charged for. + +New rows are stamped at creation, since kickoff enqueues the first step right +away and that step needs the same cooldown as every later one. Nullable with no +backfill: rows that predate this migration read as NULL and get one immediate +resume, which is the behaviour they had anyway. + +No index — the PROCESSING set is small and already indexed on `status`; the +cooldown comparison happens in Python, mirroring the fast-eval barrier. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "085" +down_revision = "084" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "evaluation_iteration_run", + sa.Column( + "last_dispatched_at", + sa.DateTime(), + nullable=True, + comment="When a graph step was last dispatched (kickoff or cron resume); cron skips rows stamped inside the cooldown", + ), + ) + + +def downgrade(): + op.drop_column("evaluation_iteration_run", "last_dispatched_at") diff --git a/backend/app/api/docs/evaluation/create_evaluation_v2.md b/backend/app/api/docs/evaluation/v2/create_evaluation.md similarity index 100% rename from backend/app/api/docs/evaluation/create_evaluation_v2.md rename to backend/app/api/docs/evaluation/v2/create_evaluation.md diff --git a/backend/app/api/docs/evaluation/create_evaluation_dataset_v2.md b/backend/app/api/docs/evaluation/v2/create_evaluation_dataset.md similarity index 100% rename from backend/app/api/docs/evaluation/create_evaluation_dataset_v2.md rename to backend/app/api/docs/evaluation/v2/create_evaluation_dataset.md diff --git a/backend/app/api/docs/evaluation/create_evaluation_iteration_v2.md b/backend/app/api/docs/evaluation/v2/create_evaluation_iteration.md similarity index 100% rename from backend/app/api/docs/evaluation/create_evaluation_iteration_v2.md rename to backend/app/api/docs/evaluation/v2/create_evaluation_iteration.md diff --git a/backend/app/api/docs/evaluation/improve_prompt_v2.md b/backend/app/api/docs/evaluation/v2/improve_prompt.md similarity index 100% rename from backend/app/api/docs/evaluation/improve_prompt_v2.md rename to backend/app/api/docs/evaluation/v2/improve_prompt.md diff --git a/backend/app/api/main.py b/backend/app/api/main.py index c7807ac19..bdb1178f5 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -39,16 +39,7 @@ assessment as assessment_routes, ) from app.api.routes.assessment import api as assessment_api_routes -from app.api.routes.evaluations.dataset_v2 import ( - router as evaluations_dataset_v2_router, -) -from app.api.routes.evaluations.evaluation_v2 import router as evaluations_v2_router -from app.api.routes.evaluations.iteration_v2 import ( - router as evaluations_iteration_v2_router, -) -from app.api.routes.evaluations.prompt_improvement_v2 import ( - router as evaluations_prompt_improvement_v2_router, -) +from app.api.routes.evaluations.v2 import router as evaluations_v2_router api_router = APIRouter() api_router.include_router(analytics.router) @@ -95,6 +86,3 @@ api_v2_router = APIRouter() api_v2_router.include_router(documents_v2.router) api_v2_router.include_router(evaluations_v2_router) -api_v2_router.include_router(evaluations_dataset_v2_router) -api_v2_router.include_router(evaluations_prompt_improvement_v2_router) -api_v2_router.include_router(evaluations_iteration_v2_router) diff --git a/backend/app/api/routes/assessment/runs.py b/backend/app/api/routes/assessment/runs.py index a1c6a7332..5ff15780f 100644 --- a/backend/app/api/routes/assessment/runs.py +++ b/backend/app/api/routes/assessment/runs.py @@ -29,8 +29,8 @@ AssessmentRunCreate, AssessmentRunPublic, AssessmentRunResponse, + AssessmentSubmission, ) -from app.models.assessment import AssessmentSubmission from app.services.assessment.service import ( resume_assessment_run as resume_run, ) diff --git a/backend/app/api/routes/evaluations/v2/__init__.py b/backend/app/api/routes/evaluations/v2/__init__.py new file mode 100644 index 000000000..0f3fc6ba0 --- /dev/null +++ b/backend/app/api/routes/evaluations/v2/__init__.py @@ -0,0 +1,15 @@ +from fastapi import APIRouter + +from app.api.routes.evaluations.v2 import ( + dataset, + evaluation, + iteration, + prompt_improvement, +) + +router = APIRouter() + +router.include_router(evaluation.router) +router.include_router(dataset.router) +router.include_router(prompt_improvement.router) +router.include_router(iteration.router) diff --git a/backend/app/api/routes/evaluations/dataset_v2.py b/backend/app/api/routes/evaluations/v2/dataset.py similarity index 96% rename from backend/app/api/routes/evaluations/dataset_v2.py rename to backend/app/api/routes/evaluations/v2/dataset.py index 47cc65f8d..4d2f4de8c 100644 --- a/backend/app/api/routes/evaluations/dataset_v2.py +++ b/backend/app/api/routes/evaluations/v2/dataset.py @@ -23,7 +23,7 @@ @router.post( "", - description=load_description("evaluation/create_evaluation_dataset_v2.md"), + description=load_description("evaluation/v2/create_evaluation_dataset.md"), response_model=APIResponse[DatasetUploadResponse], dependencies=[ Depends(require_permission(Permission.REQUIRE_PROJECT)), diff --git a/backend/app/api/routes/evaluations/evaluation_v2.py b/backend/app/api/routes/evaluations/v2/evaluation.py similarity index 97% rename from backend/app/api/routes/evaluations/evaluation_v2.py rename to backend/app/api/routes/evaluations/v2/evaluation.py index 9fe5221aa..7bf07eed4 100644 --- a/backend/app/api/routes/evaluations/evaluation_v2.py +++ b/backend/app/api/routes/evaluations/v2/evaluation.py @@ -21,7 +21,7 @@ @router.post( "", - description=load_description("evaluation/create_evaluation_v2.md"), + description=load_description("evaluation/v2/create_evaluation.md"), response_model=APIResponse[EvaluationRunPublic], dependencies=[ Depends(require_permission(Permission.REQUIRE_PROJECT)), diff --git a/backend/app/api/routes/evaluations/iteration_v2.py b/backend/app/api/routes/evaluations/v2/iteration.py similarity index 95% rename from backend/app/api/routes/evaluations/iteration_v2.py rename to backend/app/api/routes/evaluations/v2/iteration.py index a3b48ab03..9aff7f3d3 100644 --- a/backend/app/api/routes/evaluations/iteration_v2.py +++ b/backend/app/api/routes/evaluations/v2/iteration.py @@ -22,7 +22,7 @@ @router.post( "/iterations", - description=load_description("evaluation/create_evaluation_iteration_v2.md"), + description=load_description("evaluation/v2/create_evaluation_iteration.md"), response_model=APIResponse[EvaluationIterationRunImmediatePublic], status_code=202, dependencies=[ @@ -30,7 +30,7 @@ Depends(monitor_rate("evaluations")), ], ) -def create_evaluation_iteration_v2( +def create_evaluation_iteration( session: SessionDep, auth_context: AuthContextDep, request: EvaluationIterationCreateRequest, diff --git a/backend/app/api/routes/evaluations/prompt_improvement_v2.py b/backend/app/api/routes/evaluations/v2/prompt_improvement.py similarity index 97% rename from backend/app/api/routes/evaluations/prompt_improvement_v2.py rename to backend/app/api/routes/evaluations/v2/prompt_improvement.py index e65f72acb..9d92e1c1f 100644 --- a/backend/app/api/routes/evaluations/prompt_improvement_v2.py +++ b/backend/app/api/routes/evaluations/v2/prompt_improvement.py @@ -22,7 +22,7 @@ @router.post( "/{evaluation_id}/improve-prompt", - description=load_description("evaluation/improve_prompt_v2.md"), + description=load_description("evaluation/v2/improve_prompt.md"), response_model=APIResponse[LLMJobImmediatePublic], status_code=202, dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))], diff --git a/backend/app/core/config.py b/backend/app/core/config.py index cc105fbec..c7342a567 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -222,6 +222,10 @@ def AWS_S3_BUCKET(self) -> str: EVAL_ITERATION_CEILING_DELTA_THRESHOLD: float = 0.05 # Consecutive low-delta rounds required before the loop stops as ceiling_reached. EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS: int = 3 + # Cron skips loops dispatched within this window + EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES: int = 10 + # Loops stalled longer are reaped; must exceed 25 hard-cap rounds worst-case time + EVAL_ITERATION_STALL_THRESHOLD_HOURS: int = 48 EVAL_JUDGE_MODEL: str = "gpt-5.6-luna" diff --git a/backend/app/crud/assessment/batch.py b/backend/app/crud/assessment/batch.py index 47abcbe58..7f37d0210 100644 --- a/backend/app/crud/assessment/batch.py +++ b/backend/app/crud/assessment/batch.py @@ -27,11 +27,6 @@ from app.models.batch_job import BatchJob, BatchJobType from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS from app.models.llm.request import ConfigBlob -from app.services.llm.mappers import ( - map_kaapi_to_anthropic_params, - map_kaapi_to_google_params, - map_kaapi_to_openai_params, -) from app.services.assessment.utils.attachments import ( attachment_type_for_row, build_anthropic_attachment_parts, @@ -44,7 +39,12 @@ normalize_llm_text, parse_rows, ) -from app.services.llm.mappers import kaapi_params_as_dict +from app.services.llm.mappers import ( + kaapi_params_as_dict, + map_kaapi_to_anthropic_params, + map_kaapi_to_google_params, + map_kaapi_to_openai_params, +) from app.services.llm.providers.registry import LLMProvider from app.utils import get_anthropic_client, get_openai_client diff --git a/backend/app/crud/evaluations/__init__.py b/backend/app/crud/evaluations/__init__.py index c60efd016..21930cef4 100644 --- a/backend/app/crud/evaluations/__init__.py +++ b/backend/app/crud/evaluations/__init__.py @@ -25,13 +25,12 @@ calculate_cosine_similarity, start_embedding_batch, ) -from app.crud.evaluations.fast import ( +from app.crud.evaluations.fast import run_fast_evaluation, run_response_chunk +from app.crud.evaluations.fast_chunks import ( JOB_TYPE_EMBEDDING_FAST, JOB_TYPE_EVALUATION_FAST, JOB_TYPE_EVALUATION_FAST_CHUNK, list_response_chunk_jobs, - run_fast_evaluation, - run_response_chunk, ) from app.crud.evaluations.iteration import ( create_evaluation_iteration_run, diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index c7e9966eb..2b2ba49de 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -23,6 +23,14 @@ logger = logging.getLogger(__name__) +def build_log_prefix(eval_run: EvaluationRun) -> str: + return ( + f"[org={eval_run.organization_id}]" + f"[project={eval_run.project_id}]" + f"[eval={eval_run.id}]" + ) + + def resolve_evaluation_config( session: Session, config_id: UUID, @@ -65,6 +73,11 @@ def create_evaluation_run( organization_id: int, project_id: int, run_mode: RunModeEnum = RunModeEnum.BATCH, + is_judge_run: bool = False, + callback_url: str | None = None, + duplication_factor: int | None = None, + total_items: int | None = None, + status: str = "pending", ) -> EvaluationRun: """ Create a new evaluation run record in the database. @@ -79,6 +92,11 @@ def create_evaluation_run( organization_id: Organization ID project_id: Project ID run_mode: Execution mode (RunModeEnum.BATCH default, or RunModeEnum.FAST) + is_judge_run: v2-only; default False, persisted as NULL (v1 marker convention) + callback_url: v2-only terminal-state webhook; default None + duplication_factor: v2-only per-run override; default None (use dataset's) + total_items: v2-only pre-known item count; default None, stored as 0 + status: Initial run status; defaults to "pending" Returns: The created EvaluationRun instance @@ -90,10 +108,15 @@ def create_evaluation_run( type=EvaluationType.TEXT.value, config_id=config_id, config_version=config_version, - status="pending", + status=status, run_mode=run_mode, organization_id=organization_id, project_id=project_id, + # False is stored as NULL so v1 rows stay indistinguishable from pre-v2 ones + is_judge_run=is_judge_run or None, + callback_url=callback_url, + duplication_factor=duplication_factor, + total_items=total_items or 0, inserted_at=now(), updated_at=now(), ) diff --git a/backend/app/crud/evaluations/cron.py b/backend/app/crud/evaluations/cron.py index 84f860935..293056f37 100644 --- a/backend/app/crud/evaluations/cron.py +++ b/backend/app/crud/evaluations/cron.py @@ -1,9 +1,4 @@ -""" -CRUD operations for evaluation cron jobs. - -This module provides functions that can be invoked periodically to process -pending evaluations across all organizations. -""" +"""Periodic evaluation maintenance: polling, fast-eval barriers, iteration resumes.""" import asyncio import logging @@ -16,11 +11,18 @@ from app.core.config import settings from app.core.util import now from app.crud.evaluations.core import update_evaluation_run -from app.crud.evaluations.fast import CHUNK_CONFIG_INDEX, list_response_chunk_jobs -from app.crud.evaluations.iteration import list_processing_evaluation_iteration_runs +from app.crud.evaluations.fast_chunks import ( + CHUNK_CONFIG_INDEX, + list_response_chunk_jobs, +) +from app.crud.evaluations.iteration import ( + list_processing_evaluation_iteration_runs, + update_evaluation_iteration_run, +) from app.crud.evaluations.processing import poll_all_pending_evaluations from app.models import EvaluationRun, EvaluationRunUpdate from app.models.evaluation import RunModeEnum +from app.models.evaluation_iteration import EvaluationIterationRunUpdate logger = logging.getLogger(__name__) @@ -28,16 +30,9 @@ def dispatch_fast_evaluation_barriers(session: Session) -> dict[str, Any]: """Fan-in barrier + stall healer for chunked fast evaluations. - For every fast run still `processing`: - * all chunks done and not yet aggregated (batch_job_id unset) → enqueue the - aggregate task, - * chunks missing and the run stalled past EVAL_FAST_STALL_THRESHOLD_MINUTES - → re-enqueue the missing chunk indices (idempotent: a completed chunk is - skipped, so this never re-charges OpenAI). - - ``batch_job_id`` (set by the aggregator's merge) is the double-enqueue guard: - once merged, later ticks won't re-enqueue the aggregate, and a redelivered - aggregate reloads the merged unit instead of re-merging. + All chunks done → enqueue aggregate (`batch_job_id`, set by the merge, guards + against double-enqueue). Stalled with chunks missing → re-enqueue those + indices (idempotent, completed chunks are skipped). """ from app.celery.utils import ( start_fast_evaluation_aggregate, @@ -70,10 +65,6 @@ def dispatch_fast_evaluation_barriers(session: Session) -> dict[str, Any]: if done >= expected and run.batch_job_id is None: start_fast_evaluation_aggregate(eval_run_id=run.id) aggregates_dispatched += 1 - logger.info( - f"[dispatch_fast_evaluation_barriers] Aggregate dispatched | " - f"run_id={run.id} | chunks={done}/{expected}" - ) continue if done < expected and run.updated_at < stall_cutoff: @@ -81,10 +72,6 @@ def dispatch_fast_evaluation_barriers(session: Session) -> dict[str, Any]: for chunk_index in missing: start_fast_evaluation_chunk(eval_run_id=run.id, chunk_index=chunk_index) chunks_reenqueued += len(missing) - # Bump updated_at so the stall window resets to the next tick's cadence. - # ponytail: no hard retry budget — a permanently failing chunk keeps - # re-enqueuing (cheap + idempotent). Add a retry-count field to fail - # the run outright if provider outages must not linger. update_evaluation_run( session=session, eval_run=run, update=EvaluationRunUpdate() ) @@ -102,70 +89,89 @@ def dispatch_fast_evaluation_barriers(session: Session) -> dict[str, Any]: def dispatch_pending_evaluation_iteration_resumes(session: Session) -> dict[str, Any]: - """Resume trigger for the eval-iterate-improve LangGraph loop. + """Resume every PROCESSING iteration loop; fail those stalled past the threshold. - For every thin `evaluation_iteration_run` row still `PROCESSING`, dispatch a - `resume=True` graph-step task. Cheap even when the loop is still waiting on - the same sub-job as last tick: the task just re-checks status and interrupts - again immediately if not ready — same cost profile as a plain polling barrier. + Loops dispatched within the cooldown are skipped so a slow step never races a + second one on the same checkpoint thread. """ from app.celery.utils import start_evaluation_iteration_round + from app.services.evaluations.iteration_graph import mark_iteration_run_failed runs = list_processing_evaluation_iteration_runs(session=session) + current_time = now() + stall_cutoff = current_time - timedelta( + hours=settings.EVAL_ITERATION_STALL_THRESHOLD_HOURS + ) + cooldown_cutoff = current_time - timedelta( + minutes=settings.EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES + ) + + dispatched = 0 + reaped = 0 + in_flight = 0 + for run in runs: + if run.inserted_at < stall_cutoff: + logger.warning( + f"[dispatch_pending_evaluation_iteration_resumes] Reaping stalled " + f"loop | iteration_run_id={run.id} | inserted_at={run.inserted_at}" + ) + mark_iteration_run_failed( + iteration_run_id=run.id, + organization_id=run.organization_id, + project_id=run.project_id, + error_message=( + f"Evaluation iteration loop stalled: still processing more than " + f"{settings.EVAL_ITERATION_STALL_THRESHOLD_HOURS}h after it started." + ), + ) + reaped += 1 + continue + + if ( + run.last_dispatched_at is not None + and run.last_dispatched_at > cooldown_cutoff + ): + in_flight += 1 + continue + start_evaluation_iteration_round( iteration_run_id=run.id, resume=True, organization_id=run.organization_id, project_id=run.project_id, ) + # Stamp after enqueue so a failed dispatch retries next tick. + update_evaluation_iteration_run( + session=session, + iteration_run=run, + update=EvaluationIterationRunUpdate(last_dispatched_at=current_time), + ) + dispatched += 1 - logger.info( - f"[dispatch_pending_evaluation_iteration_resumes] Dispatched resumes | " - f"count={len(runs)}" - ) - return {"total": len(runs), "resumes_dispatched": len(runs)} + return { + "total": len(runs), + "resumes_dispatched": dispatched, + "in_flight_skipped": in_flight, + "reaped": reaped, + } async def process_all_pending_evaluations(session: Session) -> dict[str, Any]: - """ - Process all pending evaluations across all organizations. - - Delegates to poll_all_pending_evaluations which fetches all processing - evaluation runs in a single query, groups by project, and processes them. - Also polls STT and TTS evaluations similarly. - - Args: - session: Database session - - Returns: - Dict with aggregated results. - """ - logger.info("[process_all_pending_evaluations] Starting evaluation processing") - + """Poll text/STT/TTS evaluations, then run the fast-eval barrier and iteration resumes.""" try: - # Poll text evaluations (single query, grouped by project) text_summary = await poll_all_pending_evaluations(session=session) - # Lazy imports to avoid circular dependency with cron_utils from app.crud.stt_evaluations import poll_all_pending_stt_evaluations from app.crud.tts_evaluations import poll_all_pending_tts_evaluations - # Poll STT evaluations (single query, grouped by project) stt_summary = await poll_all_pending_stt_evaluations(session=session) - - # Poll TTS evaluations (single query, grouped by project) tts_summary = await poll_all_pending_tts_evaluations(session=session) - - # Fan-in barrier + stall healer for chunked fast-mode text evaluations. fast_summary = dispatch_fast_evaluation_barriers(session=session) - - # Resume trigger for the eval-iterate-improve LangGraph loop. iteration_summary = dispatch_pending_evaluation_iteration_resumes( session=session ) - # Merge summaries total_processed = ( text_summary["processed"] + stt_summary["processed"] @@ -185,15 +191,6 @@ async def process_all_pending_evaluations(session: Session) -> dict[str, Any]: + tts_summary.get("details", []) ) - logger.info( - f"[process_all_pending_evaluations] Completed: " - f"{total_processed} processed, {total_failed} failed, " - f"{total_still_processing} still processing | " - f"fast_aggregates={fast_summary['aggregates_dispatched']} | " - f"fast_chunks_reenqueued={fast_summary['chunks_reenqueued']} | " - f"iteration_resumes={iteration_summary['resumes_dispatched']}" - ) - return { "status": "success", "total_processed": total_processed, @@ -220,15 +217,5 @@ async def process_all_pending_evaluations(session: Session) -> dict[str, Any]: def process_all_pending_evaluations_sync(session: Session) -> dict[str, Any]: - """ - Synchronous wrapper for process_all_pending_evaluations. - - This function can be called from synchronous contexts (like FastAPI endpoints). - - Args: - session: Database session - - Returns: - Dict with aggregated results (same as process_all_pending_evaluations) - """ + """Sync wrapper for FastAPI endpoints.""" return asyncio.run(process_all_pending_evaluations(session=session)) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 734816f7d..c4d9e9b52 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -7,34 +7,37 @@ Stage 1 — Responses unit: evaluation_run.batch_job_id Stage 2 — Embeddings unit: evaluation_run.embedding_batch_job_id Stage 3 — Score + trace + cost (no marker; each step is idempotent) - Stage 4 — Mark completed - Stage 5 — Persist score unit (summary + per-trace) via the shared - save_score helper, so the cached trace unit (score_trace_url) - exists immediately and the read path (trace view / resync / - grouped export) mirrors the batch path without racing Langfuse - ingestion. + Stage 4 — Persist score unit (summary + per-trace) via the shared save_score + helper, so the cached trace unit (score_trace_url) exists before + the run is advertised as complete and the read path (trace view / + resync / grouped export) mirrors the batch path. + Stage 5 — Mark completed + Stage 6 — Best-effort tail: drop the chunk artifacts, sync Langfuse scores. + +Stage 4 precedes stage 5 deliberately: for a judged (v2) run the score unit is +the *only* copy of the per-row judge scores — nothing is written to Langfuse to +fall back on — so a crash between the two must leave the run `processing`, never +`completed` with its scores gone. + +This module owns orchestration and IO. The per-item shapes live in +`fast_results`, the `batch_job` bookkeeping in `fast_chunks`, trace records in +`fast_traces`, and the two mutually exclusive scoring paths in `fast_cosine` +(v1) and `judge_stage` (v2). See `Fast Evaluation SRD.md` for the full design. """ import logging +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any, cast +from dataclasses import dataclass, field +from typing import Any -import numpy as np import openai from langfuse import Langfuse from openai import OpenAI -from pydantic import ValidationError -from sqlalchemy import Integer -from sqlmodel import Session, select -from tenacity import ( - before_sleep_log, - retry, - retry_if_exception_type, - stop_after_attempt, - wait_random_exponential, -) +from sqlalchemy.orm.attributes import set_committed_value +from sqlmodel import Session from app.core.cloud.storage import get_cloud_storage from app.core.config import settings @@ -43,7 +46,7 @@ upload_jsonl_to_object_store, ) from app.crud.evaluations.core import ( - resolve_evaluation_config, + build_log_prefix, resolve_model_from_config, save_score, update_evaluation_run, @@ -53,114 +56,69 @@ DATASET_META_DUPLICATION_FACTOR, get_dataset_by_id, ) -from app.crud.evaluations.embeddings import ( - EMBEDDING_MODEL, - calculate_cosine_similarity, +from app.crud.evaluations.embeddings import EMBEDDING_MODEL +from app.crud.evaluations.fast_chunks import ( + CHUNK_CONFIG_INDEX, + CHUNK_CONFIG_RUN_ID, + EMBEDDINGS_ENDPOINT, + JOB_TYPE_EMBEDDING_FAST, + JOB_TYPE_EVALUATION_FAST, + JOB_TYPE_EVALUATION_FAST_CHUNK, + RESPONSES_ENDPOINT, + delete_response_chunk_artifacts, + get_chunk_job, + list_response_chunk_jobs, +) +from app.crud.evaluations.fast_cosine import ( + build_item_refs, + classify_empty_side, + score_cosine_run, +) +from app.crud.evaluations.fast_results import ( + EMBEDDING_USAGE_KEYS, + RESPONSE_USAGE_KEYS, + EmbeddingResult, + ResponseResult, + build_embedding_failure, + build_response_result, + extract_usage, + is_failure_threshold_breached, + parse_embedding_pair, + sum_usage, ) -from app.crud.evaluations.judge import ( - METRIC_REGISTRY, - JudgeInputEnum, - JudgeMetricEnum, - JudgeMetricSpec, - JudgeResult, - build_judge_params, - judge_row, +from app.crud.evaluations.fast_traces import build_trace_records +from app.crud.evaluations.judge import METRIC_REGISTRY, JudgeMetricSpec, JudgeResult +from app.crud.evaluations.judge_stage import ( + build_metric_summary_scores, + judge_rows, + resolve_config_prompt, + select_judgeable_rows, ) from app.crud.evaluations.langfuse import ( create_langfuse_dataset_run, update_traces_with_cosine_scores, ) -from app.crud.evaluations.merge import apply_cosine_breakdown -from app.crud.evaluations.response_parsing import ( - extract_response_text as _extract_response_text, -) -from app.crud.evaluations.response_parsing import ( - field_value as _field, -) +from app.crud.evaluations.response_parsing import extract_response_text +from app.crud.evaluations.retry import retry_openai_call from app.crud.evaluations.score import ( - COSINE_SCORE_COMMENT, - COSINE_SCORE_NAME, - DEFAULT_CATEGORY, JUDGE_FAILED_REASON, - UNSCOREABLE_EMBEDDING_FAILED, - UNSCOREABLE_EMPTY_GROUND_TRUTH, - UNSCOREABLE_EMPTY_OUTPUT, EvaluationScore, OverallSummary, SummaryScore, TraceData, - TraceScore, compute_overall_summary, - verdict_from_score, ) from app.crud.evaluations.summary import generate_run_ai_summary -from app.crud.job import ( - create_batch_job, - delete_batch_job, - get_batch_job, -) +from app.crud.job import create_batch_job, get_batch_job from app.models import EvaluationRun, EvaluationRunUpdate from app.models.batch_job import BatchJob, BatchJobCreate -from app.models.evaluation import RunModeEnum from app.models.llm.request import TextLLMParams from app.services.llm.mappers import map_kaapi_to_openai_params from app.services.response.response import get_file_search_results logger = logging.getLogger(__name__) - -# job_type discriminators on batch_job for the two fast-path stages. The row's -# presence + raw_output_url is what marks a stage as already done on retry. -JOB_TYPE_EVALUATION_FAST = "evaluation_fast" -JOB_TYPE_EVALUATION_FAST_CHUNK = "evaluation_fast_chunk" -JOB_TYPE_EMBEDDING_FAST = "embedding_fast" - -# batch_job.config keys tying a chunk row back to its run + slice. -CHUNK_CONFIG_RUN_ID = "eval_run_id" -CHUNK_CONFIG_INDEX = "chunk_index" - -# Judge tell the template apart from the instructions above it. -PROMPT_TEMPLATE_LABEL = "Prompt template wrapped around each user input:" - -# How many top KB matches to name in the knowledge_base trace comment. -_KB_TOP_CHUNKS = 3 - - -def _format_top_kb_matches(sorted_chunks: list[dict[str, Any]]) -> str: - """Top-N retrieved chunks as 'biu-1.pdf (90.6%), faq.pdf (66.3%)'. - - Expects chunks pre-sorted by score desc; old S3 payloads may lack filename. - """ - matches = [ - f"{c.get('filename') or 'unknown'} ({c.get('score', 0) * 100:.1f}%)" - for c in sorted_chunks - ] - return ", ".join(matches[:_KB_TOP_CHUNKS]) - - -# Per-call retry policy for Stage 1 / Stage 2. -_RETRY_MAX_ATTEMPTS = 3 -_RETRY_BASE_DELAY_SECONDS = 1.0 -_RETRY_MAX_DELAY_SECONDS = 30.0 - -_RETRYABLE_OPENAI_ERRORS: tuple[type[Exception], ...] = ( - openai.RateLimitError, - openai.APITimeoutError, - openai.APIConnectionError, - openai.InternalServerError, -) - - -# reraise=True so call-site handlers see the original OpenAIError, not RetryError. -_retry_openai_call = retry( - retry=retry_if_exception_type(_RETRYABLE_OPENAI_ERRORS), - wait=wait_random_exponential( - multiplier=_RETRY_BASE_DELAY_SECONDS, max=_RETRY_MAX_DELAY_SECONDS - ), - stop=stop_after_attempt(_RETRY_MAX_ATTEMPTS), - before_sleep=before_sleep_log(logger, logging.INFO), - reraise=True, -) +_retry_openai_call = retry_openai_call(logger) @_retry_openai_call @@ -177,30 +135,20 @@ def _create_embedding( ) -def _response_result( +def _run_in_pool( *, - item_id: str, - question: str, - ground_truth: str, - question_id: Any, - generated_output: str, - failed: bool, - response_id: str | None = None, - usage: dict[str, int] | None = None, - retrieved_chunks: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """One Stage-1 per-item result, in the batch path's shape.""" - return { - "item_id": item_id, - "question": question, - "generated_output": generated_output, - "ground_truth": ground_truth, - "response_id": response_id, - "usage": usage, - "question_id": question_id, - "failed": failed, - "retrieved_chunks": retrieved_chunks, - } + items: list[Any], + worker: Callable[[Any], dict[str, Any]], + max_workers: int, +) -> list[dict[str, Any]]: + """Fan `worker` out over `items` in a thread pool, collecting every result.""" + results: list[dict[str, Any]] = [] + workers = max(1, min(max_workers, len(items) or 1)) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(worker, item) for item in items] + for future in as_completed(futures): + results.append(future.result()) + return results def _responses_call_for_item( @@ -208,7 +156,7 @@ def _responses_call_for_item( openai_client: OpenAI, base_params: dict[str, Any], item: dict[str, Any], -) -> dict[str, Any]: +) -> ResponseResult: """Run one Responses call for a dataset item, in the batch path's per-item shape. `base_params` is the question-independent OpenAI body produced once by @@ -221,46 +169,35 @@ def _responses_call_for_item( ) question_id = (item.get("metadata") or {}).get("question_id") - if not question: - return _response_result( + def failed_result(generated_output: str) -> ResponseResult: + return build_response_result( item_id=item_id, - question="", + question=question, ground_truth=ground_truth, question_id=question_id, - generated_output="ERROR: missing question in dataset item", + generated_output=generated_output, failed=True, ) - params = {**base_params, "input": question} + if not question: + return failed_result("ERROR: missing question in dataset item") try: - response = _create_response(openai_client, params) + response = _create_response(openai_client, {**base_params, "input": question}) except openai.OpenAIError as exc: logger.warning( f"[_responses_call_for_item] Item failed | item_id={item_id} | error={exc}" ) - return _response_result( - item_id=item_id, - question=question, - ground_truth=ground_truth, - question_id=question_id, - generated_output=f"ERROR: {exc}", - failed=True, - ) + return failed_result(f"ERROR: {exc}") - usage = getattr(response, "usage", None) - return _response_result( + return build_response_result( item_id=item_id, question=question, ground_truth=ground_truth, question_id=question_id, - generated_output=_extract_response_text(response), + generated_output=extract_response_text(response), response_id=getattr(response, "id", None), - usage={ - "input_tokens": int(_field(usage, "input_tokens", 0) or 0), - "output_tokens": int(_field(usage, "output_tokens", 0) or 0), - "total_tokens": int(_field(usage, "total_tokens", 0) or 0), - }, + usage=extract_usage(getattr(response, "usage", None), RESPONSE_USAGE_KEYS), failed=False, # Plain dicts (not FileResultChunk) so the unit stays JSON-serializable for S3. retrieved_chunks=[ @@ -270,18 +207,6 @@ def _responses_call_for_item( ) -def _embedding_failure(item_id: str, error: str) -> dict[str, Any]: - """One failed Stage-2 per-pair result.""" - return { - "item_id": item_id, - "output_embedding": None, - "ground_truth_embedding": None, - "usage": None, - "failed": True, - "error": error, - } - - def _embedding_call_for_pair( *, openai_client: OpenAI, @@ -289,10 +214,10 @@ def _embedding_call_for_pair( item_id: str, output_text: str, ground_truth: str, -) -> dict[str, Any]: +) -> EmbeddingResult: """Embed an (output, ground_truth) pair; `failed=True` on a terminal failure.""" if not output_text or not ground_truth: - return _embedding_failure(item_id, "empty output or ground_truth") + return build_embedding_failure(item_id, "empty output or ground_truth") try: response = _create_embedding( @@ -305,52 +230,9 @@ def _embedding_call_for_pair( logger.warning( f"[_embedding_call_for_pair] Item failed | item_id={item_id} | error={exc}" ) - return _embedding_failure(item_id, str(exc)) - - data = _field(response, "data") or [] - if len(data) < 2: - return _embedding_failure(item_id, f"expected 2 embeddings, got {len(data)}") - - output_embedding: list[float] | None = None - ground_truth_embedding: list[float] | None = None - for emb in data: - index = _field(emb, "index") - vector = _field(emb, "embedding") - if index == 0: - output_embedding = vector - elif index == 1: - ground_truth_embedding = vector - - usage_obj = _field(response, "usage") - usage_dict: dict[str, int] = { - "prompt_tokens": int(_field(usage_obj, "prompt_tokens", 0) or 0), - "total_tokens": int(_field(usage_obj, "total_tokens", 0) or 0), - } - - return { - "item_id": item_id, - "output_embedding": output_embedding, - "ground_truth_embedding": ground_truth_embedding, - "usage": usage_dict, - "failed": output_embedding is None or ground_truth_embedding is None, - } + return build_embedding_failure(item_id, str(exc)) - -def _is_failure_threshold_breached(*, failed_rows: int, total_rows: int) -> bool: - """True if the failed-row fraction exceeds EVAL_FAST_FAILURE_THRESHOLD.""" - if total_rows == 0: - return False - return (failed_rows / total_rows) > settings.EVAL_FAST_FAILURE_THRESHOLD - - -def _sum_usage(results: list[dict[str, Any]], keys: tuple[str, ...]) -> dict[str, int]: - """Sum the per-item `usage` token counts across results, for the given keys.""" - totals = dict.fromkeys(keys, 0) - for r in results: - usage = r.get("usage") or {} - for k in keys: - totals[k] += int(usage.get(k, 0) or 0) - return totals + return parse_embedding_pair(item_id=item_id, response=response) def _upload_unit_to_s3( @@ -392,8 +274,6 @@ def _load_completed_stage( session: Session, batch_job_id: int | None, project_id: int, - log_prefix: str, - stage: str, ) -> list[dict[str, Any]] | None: """Return a stage's persisted unit if its batch_job already completed, else None. @@ -406,58 +286,32 @@ def _load_completed_stage( existing = get_batch_job(session=session, batch_job_id=batch_job_id) if not (existing and existing.raw_output_url): return None - logger.info( - f"[{stage}] {log_prefix} Skipping (already done) | batch_job_id={existing.id}" - ) return _load_unit_from_s3( session=session, project_id=project_id, url=existing.raw_output_url ) -def list_response_chunk_jobs(*, session: Session, eval_run_id: int) -> list[BatchJob]: - """All response-chunk batch_jobs for a fast run, in any state.""" - statement = select(BatchJob).where( - BatchJob.job_type == JOB_TYPE_EVALUATION_FAST_CHUNK, - BatchJob.config[CHUNK_CONFIG_RUN_ID].astext.cast(Integer) == eval_run_id, - ) - return list(session.exec(statement).all()) - - -def _get_chunk_job( - *, session: Session, eval_run_id: int, chunk_index: int -) -> BatchJob | None: - """The chunk batch_job for one (eval_run, chunk_index), or None.""" - statement = select(BatchJob).where( - BatchJob.job_type == JOB_TYPE_EVALUATION_FAST_CHUNK, - BatchJob.config[CHUNK_CONFIG_RUN_ID].astext.cast(Integer) == eval_run_id, - BatchJob.config[CHUNK_CONFIG_INDEX].astext.cast(Integer) == chunk_index, - ) - return session.exec(statement).first() - - def _cleanup_response_chunks(*, session: Session, eval_run: EvaluationRun) -> None: - """Delete the per-chunk S3 files + batch_job rows once a run completes. + """Drop the per-chunk S3 files + batch_job rows once a run completes. - Best-effort: a failed delete only leaks DB+S3 bloat, so it never fails the - run. Failed runs skip this and keep their chunks for the healer. + Best-effort end to end: this runs after the completed transition, so anything + raising here would flip an already-completed run to failed over orphaned + chunk files nobody reads. Resolving storage is guarded for the same reason + the deletes are. """ try: storage = get_cloud_storage(session=session, project_id=eval_run.project_id) - chunk_jobs = list_response_chunk_jobs(session=session, eval_run_id=eval_run.id) - for job in chunk_jobs: - if job.raw_output_url: - storage.delete(job.raw_output_url) - delete_batch_job(session, job) - logger.info( - f"[_cleanup_response_chunks] Removed {len(chunk_jobs)} chunk " - f"artifacts | eval_run_id={eval_run.id}" - ) except Exception as exc: logger.warning( - f"[_cleanup_response_chunks] Cleanup failed (orphans harmless) | " + f"[_cleanup_response_chunks] Cleanup skipped (orphans harmless) | " f"eval_run_id={eval_run.id} | error={exc}", exc_info=True, ) + return + + delete_response_chunk_artifacts( + session=session, storage=storage, eval_run_id=eval_run.id + ) def run_response_chunk( @@ -468,7 +322,6 @@ def run_response_chunk( config: TextLLMParams, dataset_items_slice: list[dict[str, Any]], chunk_index: int, - log_prefix: str, ) -> None: """Run the responses stage over one slice of dataset items. @@ -480,29 +333,15 @@ def run_response_chunk( Concurrency: two workers racing the same chunk_index can both pass the skip guard and write two rows; the merge de-duplicates per index. """ - existing = _get_chunk_job( + existing = get_chunk_job( session=session, eval_run_id=eval_run.id, chunk_index=chunk_index ) if existing and existing.raw_output_url: - logger.info( - f"[run_response_chunk] {log_prefix} Skipping chunk (already done) | " - f"chunk_index={chunk_index} | batch_job_id={existing.id}" - ) return - logger.info( - f"[run_response_chunk] {log_prefix} Running chunk | " - f"chunk_index={chunk_index} | items={len(dataset_items_slice)} | " - f"model={config.model} | concurrency={settings.EVAL_FAST_API_CONCURRENCY}" - ) - base_params, mapper_warnings = map_kaapi_to_openai_params( session=session, kaapi_params=config ) - if mapper_warnings: - logger.info( - f"[run_response_chunk] {log_prefix} Mapper warnings: {mapper_warnings}" - ) # Ask OpenAI to return the file_search hits so knowledge_base can judge them. # tool_choice stays at the model default (auto) — consistent with normal calls; @@ -510,27 +349,12 @@ def run_response_chunk( if any(t.get("type") == "file_search" for t in base_params.get("tools", [])): base_params["include"] = ["file_search_call.results"] - results: list[dict[str, Any]] = [] - max_workers = max( - 1, min(settings.EVAL_FAST_API_CONCURRENCY, len(dataset_items_slice)) - ) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - _responses_call_for_item, - openai_client=openai_client, - base_params=base_params, - item=item, - ): item["id"] - for item in dataset_items_slice - } - for future in as_completed(futures): - results.append(future.result()) - - failed_count = sum(1 for r in results if r.get("failed")) - logger.info( - f"[run_response_chunk] {log_prefix} Chunk finished | " - f"chunk_index={chunk_index} | total={len(results)} | failed={failed_count}" + results = _run_in_pool( + items=dataset_items_slice, + worker=lambda item: _responses_call_for_item( + openai_client=openai_client, base_params=base_params, item=item + ), + max_workers=settings.EVAL_FAST_API_CONCURRENCY, ) raw_output_url = _upload_unit_to_s3( @@ -540,20 +364,16 @@ def run_response_chunk( filename=f"responses_{eval_run.id}_{chunk_index}.json", results=results, ) - - summed_usage = _sum_usage( - results, ("input_tokens", "output_tokens", "total_tokens") - ) create_batch_job( session=session, batch_job_create=BatchJobCreate( provider="openai", job_type=JOB_TYPE_EVALUATION_FAST_CHUNK, config={ - "endpoint": "/v1/responses", - "run_mode": RunModeEnum.FAST.value, + "run_mode": "fast", + "endpoint": RESPONSES_ENDPOINT, "model": config.model, - "usage": summed_usage, + "usage": sum_usage(results, RESPONSE_USAGE_KEYS), CHUNK_CONFIG_RUN_ID: eval_run.id, CHUNK_CONFIG_INDEX: chunk_index, }, @@ -569,7 +389,7 @@ def _merge_response_chunks( *, session: Session, eval_run: EvaluationRun, -) -> tuple[EvaluationRun, list[dict[str, Any]]]: +) -> tuple[EvaluationRun, list[ResponseResult]]: """Concatenate every response chunk into the canonical responses unit. Skipped on retry when `eval_run.batch_job_id` is set (canonical unit @@ -577,24 +397,16 @@ def _merge_response_chunks( ordered by index and de-duplicated per index — a healer re-enqueue may race a slow chunk — so the merged order, and the scores, stay reproducible. """ - log_prefix = ( - f"[org={eval_run.organization_id}]" - f"[project={eval_run.project_id}]" - f"[eval={eval_run.id}]" - ) cached = _load_completed_stage( session=session, batch_job_id=eval_run.batch_job_id, project_id=eval_run.project_id, - log_prefix=log_prefix, - stage="_merge_response_chunks", ) if cached is not None: return eval_run, cached - chunk_jobs = list_response_chunk_jobs(session=session, eval_run_id=eval_run.id) chunk_job_by_index: dict[int, BatchJob] = {} - for job in chunk_jobs: + for job in list_response_chunk_jobs(session=session, eval_run_id=eval_run.id): chunk_index = int(job.config.get(CHUNK_CONFIG_INDEX, -1)) if job.raw_output_url and chunk_index not in chunk_job_by_index: chunk_job_by_index[chunk_index] = job @@ -605,17 +417,10 @@ def _merge_response_chunks( assert raw_output_url is not None # guaranteed by the filter above results.extend( _load_unit_from_s3( - session=session, - project_id=eval_run.project_id, - url=raw_output_url, + session=session, project_id=eval_run.project_id, url=raw_output_url ) ) - logger.info( - f"[_merge_response_chunks] {log_prefix} Merged chunks | " - f"chunks={len(chunk_job_by_index)} | items={len(results)}" - ) - raw_output_url = _upload_unit_to_s3( session=session, project_id=eval_run.project_id, @@ -623,25 +428,21 @@ def _merge_response_chunks( filename=f"responses_{eval_run.id}.json", results=results, ) - model = ( next(iter(chunk_job_by_index.values())).config.get("model") if chunk_job_by_index else None ) - summed_usage = _sum_usage( - results, ("input_tokens", "output_tokens", "total_tokens") - ) batch_job = create_batch_job( session=session, batch_job_create=BatchJobCreate( provider="openai", job_type=JOB_TYPE_EVALUATION_FAST, config={ - "endpoint": "/v1/responses", - "run_mode": RunModeEnum.FAST.value, + "run_mode": "fast", + "endpoint": RESPONSES_ENDPOINT, "model": model, - "usage": summed_usage, + "usage": sum_usage(results, RESPONSE_USAGE_KEYS), }, raw_output_url=raw_output_url, total_items=len(results), @@ -654,11 +455,8 @@ def _merge_response_chunks( eval_run.batch_job_id = batch_job.id eval_run.total_items = len(results) eval_run = update_evaluation_run( - session=session, - eval_run=eval_run, - update=EvaluationRunUpdate(), + session=session, eval_run=eval_run, update=EvaluationRunUpdate() ) - return eval_run, results @@ -667,56 +465,37 @@ def _stage2_embeddings( session: Session, openai_client: OpenAI, eval_run: EvaluationRun, - response_results: list[dict[str, Any]], - log_prefix: str, -) -> tuple[EvaluationRun, list[dict[str, Any]]]: + response_results: list[ResponseResult], +) -> tuple[EvaluationRun, list[EmbeddingResult]]: """Stage 2 — embed each (output, ground_truth) pair; skipped on retry if done.""" cached = _load_completed_stage( session=session, batch_job_id=eval_run.embedding_batch_job_id, project_id=eval_run.project_id, - log_prefix=log_prefix, - stage="_stage2_embeddings", ) if cached is not None: return eval_run, cached # Only embed items that succeeded in Stage 1. embed_candidates = [r for r in response_results if not r.get("failed")] - logger.info( - f"[_stage2_embeddings] {log_prefix} Running stage 2 | " - f"items={len(embed_candidates)} | model={EMBEDDING_MODEL} | " - f"concurrency={settings.EVAL_FAST_API_CONCURRENCY}" - ) - embedding_results: list[dict[str, Any]] = [] - max_workers = max( - 1, min(settings.EVAL_FAST_API_CONCURRENCY, len(embed_candidates) or 1) + embedding_results = _run_in_pool( + items=embed_candidates, + worker=lambda result: _embedding_call_for_pair( + openai_client=openai_client, + embedding_model=EMBEDDING_MODEL, + item_id=result["item_id"], + output_text=result.get("generated_output", ""), + ground_truth=result.get("ground_truth", ""), + ), + max_workers=settings.EVAL_FAST_API_CONCURRENCY, ) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - _embedding_call_for_pair, - openai_client=openai_client, - embedding_model=EMBEDDING_MODEL, - item_id=r["item_id"], - output_text=r.get("generated_output", ""), - ground_truth=r.get("ground_truth", ""), - ): r["item_id"] - for r in embed_candidates - } - for future in as_completed(futures): - embedding_results.append(future.result()) failed_count = sum(1 for r in embedding_results if r.get("failed")) # Threshold is over the whole dataset: Stage 1 failures count as failures too. total_failures = failed_count + sum(1 for r in response_results if r.get("failed")) - logger.info( - f"[_stage2_embeddings] {log_prefix} Stage 2 finished | " - f"total={len(embedding_results)} | failed={failed_count}" - ) - if _is_failure_threshold_breached( + if is_failure_threshold_breached( failed_rows=total_failures, total_rows=len(response_results) ): raise RuntimeError( @@ -732,19 +511,16 @@ def _stage2_embeddings( filename=f"embeddings_{eval_run.id}.json", results=embedding_results, ) - - summed_usage = _sum_usage(embedding_results, ("prompt_tokens", "total_tokens")) - batch_job = create_batch_job( session=session, batch_job_create=BatchJobCreate( provider="openai", job_type=JOB_TYPE_EMBEDDING_FAST, config={ - "endpoint": "/v1/embeddings", - "run_mode": RunModeEnum.FAST.value, + "run_mode": "fast", + "endpoint": EMBEDDINGS_ENDPOINT, "embedding_model": EMBEDDING_MODEL, - "usage": summed_usage, + "usage": sum_usage(embedding_results, EMBEDDING_USAGE_KEYS), }, raw_output_url=raw_output_url, total_items=len(embedding_results), @@ -752,163 +528,199 @@ def _stage2_embeddings( project_id=eval_run.project_id, ), ) - eval_run = update_evaluation_run( session=session, eval_run=eval_run, update=EvaluationRunUpdate(embedding_batch_job_id=batch_job.id), ) - return eval_run, embedding_results -def _resolve_config_prompt( - *, session: Session, eval_run: EvaluationRun, log_prefix: str -) -> str | None: - """The evaluated bot's own configured prompt, or None if unresolvable. +@dataclass +class _ScoringOutcome: + """What one scoring path (cosine or judge) contributes to Stage 3.""" - The prompt template is appended when the config carries one, since it is equally - part of what the bot was told to do. Returns None when the config carries no - instructions, so the caller drops the prompt metric rather than grading against "". - """ - if not eval_run.config_id or not eval_run.config_version: - return None + summary_scores: list[SummaryScore] = field(default_factory=list) + unscoreable: dict[str, str] = field(default_factory=dict) + write_items: list[dict[str, Any]] = field(default_factory=list) + cosine_by_item_id: dict[str, float] = field(default_factory=dict) + judge_results: dict[str, JudgeResult] = field(default_factory=dict) + metrics: list[JudgeMetricSpec] = field(default_factory=list) + config_prompt: str = "" - config, error = resolve_evaluation_config( - session=session, - config_id=eval_run.config_id, - config_version=eval_run.config_version, - project_id=eval_run.project_id, - ) - if error or config is None: - return None - # Native/proxy params aren't TextLLMParams-shaped; a mismatch just means there are - # no instructions to grade against, not a run failure. - try: - params = TextLLMParams.model_validate(config.completion.params) - except ValidationError as exc: - logger.info( - f"[_resolve_config_prompt] {log_prefix} Completion params are not " - f"text params; prompt metric unscoreable | error={exc}" +def _attach_stage_costs( + *, + session: Session, + eval_run: EvaluationRun, + log_prefix: str, + model: str | None, + response_results: list[dict[str, Any]], + embedding_results: list[dict[str, Any]] | None, +) -> None: + """Attach the response- and embedding-stage costs (idempotent per stage).""" + if response_results: + attach_cost( + session=session, + eval_run=eval_run, + log_prefix=log_prefix, + response_model=model, + response_results=response_results, ) - return None - sections: list[str] = [] - if params.instructions: - sections.append(params.instructions.strip()) - if config.prompt_template and config.prompt_template.template: - sections.append( - f"{PROMPT_TEMPLATE_LABEL}\n{config.prompt_template.template.strip()}" + # attach_cost expects the raw OpenAI batch shape; rebuild it from embedding_results. + embedding_raw = [ + { + "response": { + "body": { + "usage": r.get("usage") or dict.fromkeys(EMBEDDING_USAGE_KEYS, 0) + } + } + } + for r in (embedding_results or []) + if not r.get("failed") + ] + if embedding_raw: + attach_cost( + session=session, + eval_run=eval_run, + log_prefix=log_prefix, + embedding_model=EMBEDDING_MODEL, + embedding_raw_results=embedding_raw, ) - if not sections: - return None - return "\n\n".join(sections) + +def _score_cosine_path( + *, + response_results: list[ResponseResult], + embedding_results: list[EmbeddingResult] | None, + item_refs: dict[str, str], + trace_id_mapping: dict[str, str], + eval_run: EvaluationRun, +) -> _ScoringOutcome: + """v1 — cosine over the embedded pairs, plus the Langfuse write list.""" + cosine = score_cosine_run( + response_results=response_results, + embedding_results=embedding_results, + item_refs=item_refs, + trace_id_mapping=trace_id_mapping, + total_items=eval_run.total_items, + ) + # Durable source of truth, keyed by ref, persisted by the Stage 3 commit. + eval_run.per_item_scores = cosine.per_item_scores + return _ScoringOutcome( + summary_scores=cosine.summary_scores, + unscoreable=cosine.unscoreable, + write_items=cosine.write_items, + cosine_by_item_id=cosine.item_id_to_score, + ) -def _judge_rows( +def _score_judge_path( *, session: Session, openai_client: OpenAI, - metrics: list[JudgeMetricSpec], - config_prompt: str, - judgeable: list[tuple[str, str, dict[str, Any]]], + response_results: list[ResponseResult], + item_refs: dict[str, str], + eval_run: EvaluationRun, log_prefix: str, -) -> tuple[dict[str, JudgeResult], set[str], str | None]: - """Run one combined judge completion per judgeable row, isolated per row. +) -> _ScoringOutcome: + """v2 — one combined judge call per row; no cosine, no Langfuse writes.""" + outcome = _ScoringOutcome(metrics=list(METRIC_REGISTRY.values())) - `metrics` is the full registry; `judge_row` drops the ones a given row cannot - supply inputs for. `config_prompt` is the same run-level text for every row, and - is "" when the run's config carried no instructions — which drops the prompt - metric for every row. - """ - results: dict[str, JudgeResult] = {} - failed_refs: set[str] = set() - if not judgeable: - return results, failed_refs, None - - # Build base params once per run; judging is system-config only, so every metric - # uses its built-in prompt + shared model. Instructions vary per row (by - # applicable-metric subset) and are composed inside judge_row. - try: - base_params = build_judge_params(session=session) - except Exception as exc: - logger.error( - f"[_judge_rows] {log_prefix} Judge setup failed; leaving all rows " - f"unjudged | error={exc}", - exc_info=True, - ) - return results, {ref for _item_id, ref, _r in judgeable}, None - - judge_model = base_params.get("model") - - max_workers = max(1, min(settings.EVAL_JUDGE_CONCURRENCY, len(judgeable))) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - future_map = { - executor.submit( - judge_row, - openai_client=openai_client, - base_params=base_params, - metrics=metrics, - inputs={ - JudgeInputEnum.CONFIG_PROMPT: config_prompt, - JudgeInputEnum.QUESTION: response.get("question", ""), - JudgeInputEnum.GENERATED_ANSWER: response.get( - "generated_output", "" - ), - JudgeInputEnum.GOLDEN_ANSWER: response.get("ground_truth", ""), - JudgeInputEnum.RETRIEVED_CHUNKS: "\n---\n".join( - c.get("text", "") - for c in (response.get("retrieved_chunks") or []) - if c.get("text") - ), - }, - ): (item_id, ref) - for item_id, ref, response in judgeable - } - for future in as_completed(future_map): - item_id, ref = future_map[future] - try: - results[item_id] = future.result() - except Exception as exc: - failed_refs.add(ref) - logger.warning( - f"[_judge_rows] {log_prefix} Judge failed for row; flagged " - f"unscoreable | item_id={item_id} | ref={ref} | error={exc}" - ) + # A row is judgeable only with a non-empty generated AND golden answer. + for response in response_results: + reason = classify_empty_side(response) + if reason is not None: + outcome.unscoreable[item_refs[response["item_id"]]] = reason - return results, failed_refs, judge_model + # Run-level input; None means prompt metric drops per row, run still completes. + outcome.config_prompt = ( + resolve_config_prompt(session=session, eval_run=eval_run) or "" + ) + outcome.judge_results, judge_failed_refs, judge_model = judge_rows( + session=session, + openai_client=openai_client, + metrics=outcome.metrics, + config_prompt=outcome.config_prompt, + judgeable=select_judgeable_rows(response_results), + item_refs=item_refs, + log_prefix=log_prefix, + ) -def _attach_metric_scores( - *, - spec: JudgeMetricSpec, - judge_results: dict[str, JudgeResult], - summary_scores: list[SummaryScore], -) -> None: - """Append one metric's run-level summary score from the combined results. + # setdefault so a row already flagged empty_output/empty_ground_truth keeps it. + for ref in judge_failed_refs: + outcome.unscoreable.setdefault(ref, JUDGE_FAILED_REASON) - Per-row scores and reasoning are stored per trace in the trace-build loop, which - is what the read path serves; only the aggregate belongs on the run. - """ - values: list[float] = [ - metric_score.score - for result in judge_results.values() - if (metric_score := result.metrics.get(spec.key)) is not None - ] + outcome.summary_scores = build_metric_summary_scores( + metrics=outcome.metrics, judge_results=outcome.judge_results + ) - if values: - arr = np.array(values) - summary_scores.append( - { - "name": spec.score_name, - "avg": round(float(np.mean(arr)), 2), - "std": round(float(np.std(arr)), 2), - "total_pairs": len(values), - "data_type": "NUMERIC", - } + # One combined judge call means all metric tokens land in single cost stage. + if outcome.judge_results and judge_model: + attach_cost( + session=session, + eval_run=eval_run, + log_prefix=log_prefix, + judge_model=judge_model, + judge_results=[ + {"usage": result.usage} for result in outcome.judge_results.values() + ], ) + return outcome + + +def _effective_duplication_factor(*, session: Session, eval_run: EvaluationRun) -> int: + """The run's duplication factor, falling back to the dataset's stored one. + + Falls back to 1 (no repetition) when neither resolves, so the summary still + generates. + """ + if eval_run.duplication_factor is not None: + return max(1, eval_run.duplication_factor) + + dataset = get_dataset_by_id( + session=session, + dataset_id=eval_run.dataset_id, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + ) + metadata = dataset.dataset_metadata if dataset else None + return max(1, int((metadata or {}).get(DATASET_META_DUPLICATION_FACTOR, 1))) + + +def _build_overall_summary( + *, + session: Session, + eval_run: EvaluationRun, + outcome: _ScoringOutcome, + traces: list[TraceData], +) -> OverallSummary | None: + """Run-level weighted rollup, plus the best-effort AI note diagnosing the traces.""" + avg_by_name = {s["name"]: s["avg"] for s in outcome.summary_scores if "avg" in s} + overall = compute_overall_summary( + metric_avgs={ + spec.key.value: avg_by_name[spec.score_name] + for spec in outcome.metrics + if spec.score_name in avg_by_name + }, + metric_weights={spec.key.value: spec.weight for spec in outcome.metrics}, + metric_names={spec.key.value: spec.score_name for spec in outcome.metrics}, + ) + if overall is None: + return None + + overall["ai_summary"] = generate_run_ai_summary( + model=settings.EVAL_SUMMARY_MODEL, + run_name=eval_run.run_name, + duplication_factor=_effective_duplication_factor( + session=session, eval_run=eval_run + ), + config_prompt=outcome.config_prompt, + traces=traces, + ) + return overall def _stage3_score_and_trace( @@ -917,33 +729,26 @@ def _stage3_score_and_trace( openai_client: OpenAI, eval_run: EvaluationRun, langfuse: Langfuse | None, - response_results: list[dict[str, Any]], - embedding_results: list[dict[str, Any]] | None, + response_results: list[ResponseResult], + embedding_results: list[EmbeddingResult] | None, log_prefix: str, ) -> tuple[EvaluationRun, EvaluationScore, list[dict[str, Any]]]: """Stage 3 — cosine (v1) or judge (v2), create traces, attach costs. Idempotent. Returns the run, the full score unit (summary_scores + per-trace records in the - batch path's shape), and the Langfuse `write_items` (empty for v2). Everything is - keyed by `ref` (trace_id when traced, else item_id) so it works without Langfuse. + batch path's shape), and the Langfuse `write_items` (empty for v2). Everything + is keyed by `ref` (trace_id when traced, else item_id) so it works without + Langfuse. The two scoring paths are mutually exclusive, gated on `eval_run.is_judge_run`: - - v1: cosine over the embedded pairs, per_item_scores, the Cosine summary score - and Langfuse sync — unchanged; v1 never judges, so a judge failure can never - block a cosine score. - - v2: no embeddings ran, so cosine is skipped entirely (`embedding_results` is - None/empty); one combined judge call scores every enabled metric per row, - per_item_scores stays NULL, and nothing is written to Langfuse. + - v1: cosine over the embedded pairs, per_item_scores, the Cosine summary + score and Langfuse sync — unchanged; v1 never judges, so a judge failure + can never block a cosine score. + - v2: no embeddings ran, so cosine is skipped entirely; one combined judge + call scores every enabled metric per row, per_item_scores stays NULL, and + nothing is written to Langfuse. """ is_judge_run = eval_run.is_judge_run - logger.info( - f"[_stage3_score_and_trace] {log_prefix} Scoring stage 3 | " - f"judge_run={is_judge_run}" - ) - - item_id_to_pair = { - r["item_id"]: r for r in (embedding_results or []) if not r.get("failed") - } model = resolve_model_from_config(session=session, eval_run=eval_run) trace_id_mapping = create_langfuse_dataset_run( @@ -953,323 +758,64 @@ def _stage3_score_and_trace( results=response_results, model=model, ) + item_refs = build_item_refs(response_results, trace_id_mapping) - # Scoring accumulators keyed by ref (trace_id when traced, else item_id) so they - # persist with tracing off. Unscoreable rows stay out of avg/std/total_pairs. The - # cosine-only fields (per_item_scores, similarities, write_items) stay empty for v2. - per_item_scores: list[dict[str, Any]] = [] - item_id_to_score: dict[str, float] = {} - item_id_to_ref: dict[str, str] = {} - similarities: list[float] = [] - unscoreable: dict[str, str] = {} # {ref: reason} - write_items: list[dict[str, Any]] = [] - summary_scores: list[SummaryScore] = [] - overall: OverallSummary | None = None - # Resolved inside the judge block below, read again after the traces are built. - config_prompt: str | None = None - - if is_judge_run: - # v2: no cosine. A row is judgeable only with a non-empty generated AND - # golden answer; empty sides are unscoreable and skip the judge below. - for response in response_results: - item_id = response["item_id"] - ref = trace_id_mapping.get(item_id) or item_id - item_id_to_ref[item_id] = ref - if not response.get("generated_output"): - unscoreable[ref] = UNSCOREABLE_EMPTY_OUTPUT - elif not response.get("ground_truth"): - unscoreable[ref] = UNSCOREABLE_EMPTY_GROUND_TRUTH - else: - for response in response_results: - item_id = response["item_id"] - ref = trace_id_mapping.get(item_id) or item_id - item_id_to_ref[item_id] = ref - embedding_pair = item_id_to_pair.get(item_id) - has_embeddings = ( - embedding_pair is not None - and embedding_pair.get("output_embedding") is not None - and embedding_pair.get("ground_truth_embedding") is not None - ) - if not has_embeddings: - # Classify why this item cannot be scored, for the UI flag. - if not response.get("generated_output"): - unscoreable[ref] = UNSCOREABLE_EMPTY_OUTPUT - elif not response.get("ground_truth"): - unscoreable[ref] = UNSCOREABLE_EMPTY_GROUND_TRUTH - else: - unscoreable[ref] = UNSCOREABLE_EMBEDDING_FAILED - continue - assert embedding_pair is not None # guaranteed by has_embeddings above - cosine = calculate_cosine_similarity( - embedding_pair["output_embedding"], - embedding_pair["ground_truth_embedding"], - ) - similarities.append(cosine) - item_id_to_score[item_id] = cosine - per_item_scores.append( - {"trace_id": trace_id_mapping.get(item_id), "cosine_similarity": cosine} - ) - - # Langfuse write list, filtered to real trace_ids (empty when untraced). - unscoreable_writes = [ - { - "trace_id": trace_id_mapping[item_id], - "unscoreable": True, - "reason": reason, - } - for item_id, ref in item_id_to_ref.items() - if item_id in trace_id_mapping - and (reason := unscoreable.get(ref)) is not None - ] - scored_writes = [w for w in per_item_scores if w["trace_id"] is not None] - write_items = scored_writes + unscoreable_writes - - # Durable source of truth, keyed by ref, persisted by the commit below. - eval_run.per_item_scores = { - item_id_to_ref[item_id]: round(float(score), 6) - for item_id, score in item_id_to_score.items() - } - - # Aggregate similarity stats, in the batch path's summary_scores shape. - if similarities: - sim_array = np.array(similarities) - avg = float(np.mean(sim_array)) - std = float(np.std(sim_array)) - else: - avg = 0.0 - std = 0.0 - - summary_scores = apply_cosine_breakdown( - [ - { - "name": COSINE_SCORE_NAME, - "avg": round(avg, 2), - "std": round(std, 2), - "total_pairs": len(similarities), - "data_type": "NUMERIC", - } - ], - total_items=eval_run.total_items, - unscoreable=unscoreable or None, - ) - - # Attach response- and embedding-stage costs (attach_cost is idempotent per stage). - if response_results: - attach_cost( + def attach_costs() -> None: + _attach_stage_costs( session=session, eval_run=eval_run, log_prefix=log_prefix, - response_model=model, + model=model, response_results=response_results, + embedding_results=embedding_results, ) - # attach_cost expects the raw OpenAI batch shape; rebuild it from embedding_results. - if embedding_results: - embedding_raw = [ - { - "response": { - "body": { - "usage": r.get("usage") - or {"prompt_tokens": 0, "total_tokens": 0} - } - } - } - for r in embedding_results - if not r.get("failed") - ] - if embedding_raw: - attach_cost( - session=session, - eval_run=eval_run, - log_prefix=log_prefix, - embedding_model=EMBEDDING_MODEL, - embedding_raw_results=embedding_raw, - ) - - judge_results: dict[str, JudgeResult] = {} - # Stays empty for v1, which never judges. - metrics: list[JudgeMetricSpec] = [] if is_judge_run: - judgeable = [ - (response["item_id"], item_id_to_ref[response["item_id"]], response) - for response in response_results - if response.get("generated_output") and response.get("ground_truth") - ] - - # Run-level input: resolved once for every row. When it resolves to None the - # prompt metric drops out per row (empty input); the run still completes. - config_prompt = _resolve_config_prompt( - session=session, eval_run=eval_run, log_prefix=log_prefix - ) - metrics = list(METRIC_REGISTRY.values()) - - judge_results, judge_failed_refs, judge_model = _judge_rows( + # Response cost lands before the judge's own cost stage. + attach_costs() + outcome = _score_judge_path( session=session, openai_client=openai_client, - metrics=metrics, - config_prompt=config_prompt or "", - judgeable=judgeable, + response_results=response_results, + item_refs=item_refs, + eval_run=eval_run, log_prefix=log_prefix, ) - - # Flag judge-failed rows unscoreable WITHOUT clobbering an empty-side reason - # (setdefault): a row already flagged empty_output/empty_ground_truth keeps it. - for ref in judge_failed_refs: - unscoreable.setdefault(ref, JUDGE_FAILED_REASON) - - for spec in metrics: - _attach_metric_scores( - spec=spec, - judge_results=judge_results, - summary_scores=summary_scores, - ) - - # One combined call grades every metric, so its tokens can't be split per - # metric — they land in a single "judge" cost stage. - if judge_results and judge_model: - attach_cost( - session=session, - eval_run=eval_run, - log_prefix=log_prefix, - judge_model=judge_model, - judge_results=[ - {"usage": result.usage} for result in judge_results.values() - ], - ) - - eval_run.unscoreable = unscoreable or None - - # Per-trace records, in the batch path's shape. Keyed by ref so untraced runs - # persist too. Judge metric scores carry their reasoning in the score comment. - traces: list[TraceData] = [] - for response in response_results: - item_id = response["item_id"] - ref = item_id_to_ref[item_id] if item_id in item_id_to_ref else item_id - trace_scores: list[TraceScore] = [] - # v2 carries no cosine score or placeholder — only the judge scores below. - if not is_judge_run: - cosine = item_id_to_score.get(item_id) - if cosine is not None: - trace_scores.append( - { - "name": COSINE_SCORE_NAME, - "value": round(cosine, 2), - "data_type": "NUMERIC", - "comment": COSINE_SCORE_COMMENT, - } - ) - elif ref in unscoreable and unscoreable[ref] != JUDGE_FAILED_REASON: - # Placeholder 0-score, excluded from summary stats via the marker. A - # judge_failed-only reason is about the judge, not cosine, so it gets - # no cosine placeholder. - trace_scores.append( - { - "name": COSINE_SCORE_NAME, - "value": 0, - "data_type": "NUMERIC", - "comment": f"Cannot compute: {unscoreable[ref]}", - "unscoreable": True, - } - ) - - judge_result = judge_results.get(item_id) - if judge_result is not None: - sorted_chunks = sorted( - response.get("retrieved_chunks") or [], - key=lambda c: c.get("score", 0), - reverse=True, - ) - top_matches = _format_top_kb_matches(sorted_chunks) - for spec in metrics: - metric_score = judge_result.metrics.get(spec.key) - is_kb = spec.key == JudgeMetricEnum.KNOWLEDGE_BASE - if metric_score is not None: - comment = metric_score.reasoning - if is_kb: - comment = f"{comment} | Top matches: {top_matches}" - rounded_score = round(metric_score.score, 2) - trace_scores.append( - { - "name": spec.score_name, - "value": rounded_score, - "data_type": "NUMERIC", - "comment": comment, - "verdict": verdict_from_score(rounded_score), - } - ) - elif is_kb: - # KB dropped for this row: surface a human reason instead of a bare - # N/A. Placeholder lives only in trace_scores, never the summary avg. - if not sorted_chunks: - # ponytail: empty chunks under auto tool_choice ~= not queried; - # a "was queried" flag would disambiguate an empty-store hit, not - # worth plumbing. - reason = "Knowledge base not queried." - else: - # Chunks present but the judge returned no KB score (rare: a - # well-formed reply that omitted the metric). - reason = "Knowledge base score unavailable for this row." - trace_scores.append( - { - "name": spec.score_name, - "value": "N/A", - "data_type": "CATEGORICAL", - "comment": reason, - "unscoreable": True, - } - ) - - traces.append( - { - "trace_id": ref, - "question": response.get("question", ""), - "llm_answer": response.get("generated_output", ""), - "ground_truth_answer": response.get("ground_truth", ""), - "question_id": response.get("question_id"), - "category": response.get("category") or DEFAULT_CATEGORY, - "scores": trace_scores, - } + else: + outcome = _score_cosine_path( + response_results=response_results, + embedding_results=embedding_results, + item_refs=item_refs, + trace_id_mapping=trace_id_mapping, + eval_run=eval_run, ) + attach_costs() - # Run-level roll-up runs after the traces, since the AI summary diagnoses them. - if is_judge_run: - avg_by_name = {s["name"]: s["avg"] for s in summary_scores if "avg" in s} - metric_avgs = { - spec.key.value: avg_by_name[spec.score_name] - for spec in metrics - if spec.score_name in avg_by_name - } - overall = compute_overall_summary( - metric_avgs=metric_avgs, - metric_weights={spec.key.value: spec.weight for spec in metrics}, - metric_names={spec.key.value: spec.score_name for spec in metrics}, + eval_run.unscoreable = outcome.unscoreable or None + + traces = build_trace_records( + response_results=response_results, + item_refs=item_refs, + is_judge_run=is_judge_run, + judge_results=outcome.judge_results, + metrics=outcome.metrics, + cosine_by_item_id=outcome.cosine_by_item_id, + unscoreable=outcome.unscoreable, + ) + + # Runs after the traces, since the AI summary diagnoses them. + overall = ( + _build_overall_summary( + session=session, eval_run=eval_run, outcome=outcome, traces=traces ) - if overall is not None: - # Falls back to 1 (no repetition) if the dataset/metadata can't be - # resolved, so the summary still generates. - dataset = get_dataset_by_id( - session=session, - dataset_id=eval_run.dataset_id, - organization_id=eval_run.organization_id, - project_id=eval_run.project_id, - ) - metadata = dataset.dataset_metadata if dataset else None - duplication_factor = max( - 1, - eval_run.duplication_factor - if eval_run.duplication_factor is not None - else int((metadata or {}).get(DATASET_META_DUPLICATION_FACTOR, 1)), - ) - overall["ai_summary"] = generate_run_ai_summary( - model=settings.EVAL_SUMMARY_MODEL, - run_name=eval_run.run_name, - duplication_factor=duplication_factor, - config_prompt=config_prompt or "", - traces=traces, - ) + if is_judge_run + else None + ) # Persist cost + unscoreable here; the score unit (summary + traces) is persisted - # by the caller via save_score so it lands in S3 like the batch path. + # by the caller via save_score, before the completed transition, so it lands in + # S3 like the batch path. This write must stay committed: save_score opens a + # second session on the same row. eval_run = update_evaluation_run( session=session, eval_run=eval_run, @@ -1280,12 +826,45 @@ def _stage3_score_and_trace( ) score: EvaluationScore = { - "summary_scores": summary_scores, + "summary_scores": outcome.summary_scores, "traces": traces, } if overall is not None: score["overall"] = overall - return eval_run, score, write_items + return eval_run, score, outcome.write_items + + +def _sync_scores_to_langfuse( + *, langfuse: Langfuse | None, write_items: list[dict[str, Any]], log_prefix: str +) -> bool: + """Write cosine scores back to Langfuse; False if any write was lost. + + Never fails the run — the score already lives on `eval_run`, so a cron can + retry the gap from the durable per_item_scores map. + """ + if langfuse is None or not write_items: + return True + + try: + failed_trace_ids = update_traces_with_cosine_scores( + langfuse=langfuse, per_item_scores=write_items + ) + except Exception as exc: + logger.warning( + f"[_sync_scores_to_langfuse] {log_prefix} Failed to update Langfuse " + f"traces with scores | error={exc}", + exc_info=True, + ) + return False + + if failed_trace_ids: + logger.warning( + f"[_sync_scores_to_langfuse] {log_prefix} {len(failed_trace_ids)} " + f"Langfuse score writes failed; recoverable from durable " + f"per_item_scores on resync" + ) + return False + return True def run_fast_evaluation( @@ -1306,12 +885,7 @@ def run_fast_evaluation( or score sync) and for tracing-opted-out projects; scoring falls back to keying by item_id. Whether the run judges is read from `eval_run.is_judge_run`. """ - log_prefix = ( - f"[org={eval_run.organization_id}]" - f"[project={eval_run.project_id}]" - f"[eval={eval_run.id}]" - ) - logger.info(f"[run_fast_evaluation] {log_prefix} Starting fast eval aggregation") + log_prefix = build_log_prefix(eval_run) if eval_run.status == "pending": eval_run = update_evaluation_run( @@ -1322,13 +896,12 @@ def run_fast_evaluation( # Stage 1 — merge the response chunks. eval_run, response_results = _merge_response_chunks( - session=session, - eval_run=eval_run, + session=session, eval_run=eval_run ) # Failure threshold is decided over the full merged set, not per chunk. failed_count = sum(1 for r in response_results if r.get("failed")) - if _is_failure_threshold_breached( + if is_failure_threshold_breached( failed_rows=failed_count, total_rows=len(response_results) ): raise RuntimeError( @@ -1346,7 +919,6 @@ def run_fast_evaluation( openai_client=openai_client, eval_run=eval_run, response_results=response_results, - log_prefix=log_prefix, ) # Stage 3 @@ -1360,70 +932,47 @@ def run_fast_evaluation( log_prefix=log_prefix, ) - # Stage 4 — mark completed WITH the summary score so there's never a - # completed + NULL-score window. Cost was persisted in Stage 3. + # Stage 4 — persist the score unit (traces to S3, summary + overall to the DB) + # BEFORE the run is advertised as complete. On v2 this unit is the only copy + # of the per-row judge scores, so it has to be durable first: a crash here + # leaves the run `processing` and visibly unfinished, never `completed` with + # its scores gone. save_score opens its own Session on this row, so Stage 3's + # write above must stay committed before this point or the two sessions + # deadlock on the row lock. + saved = save_score( + eval_run_id=eval_run.id, + organization_id=eval_run.organization_id, + project_id=eval_run.project_id, + score=score, + ) + if saved is None: + raise RuntimeError( + f"Score unit not persisted; run row is gone | eval_run_id={eval_run.id}" + ) + + # Stage 5 — mark completed. Cost was persisted in Stage 3, score in Stage 4, + # so there's never a completed + NULL-score window. eval_run = update_evaluation_run( session=session, eval_run=eval_run, - update=EvaluationRunUpdate( - status="completed", - # Persist the overall alongside the summary so GET run status shows the - # run-level score/verdict/breakdown without loading the S3 trace unit. - score={ - "summary_scores": score["summary_scores"], - "overall": score.get("overall"), - }, - cost=eval_run.cost, - ), + update=EvaluationRunUpdate(status="completed", cost=eval_run.cost), ) + # Stage 6 — best-effort tail, nothing here may fail a completed run. _cleanup_response_chunks(session=session, eval_run=eval_run) - # Stage 5a — write cosine scores to Langfuse after completion (mirrors the - # batch path). is_score_updated tracks the outcome so a cron can retry the - # gap from per_item_scores. Skipped entirely when langfuse is None — v2 judged - # runs are Kaapi-native and never sync scores to Langfuse. - is_score_updated = True - if langfuse is not None and write_items: - try: - failed_trace_ids = update_traces_with_cosine_scores( - langfuse=langfuse, per_item_scores=write_items - ) - if failed_trace_ids: - is_score_updated = False - logger.warning( - f"[run_fast_evaluation] {log_prefix} " - f"{len(failed_trace_ids)} Langfuse score writes failed; " - f"recoverable from durable per_item_scores on resync" - ) - except Exception as exc: - # Score-update failures don't fail the run (score lives in eval_run.score). - is_score_updated = False - logger.warning( - f"[run_fast_evaluation] {log_prefix} " - f"Failed to update Langfuse traces with scores | error={exc}", - exc_info=True, - ) + # Cosine scores go to Langfuse after completion (mirrors the batch path); + # is_score_updated tracks the outcome so a cron can retry the gap. eval_run = update_evaluation_run( session=session, eval_run=eval_run, - update=EvaluationRunUpdate(is_score_updated=is_score_updated), - ) - - # Stage 5b — persist the score unit (traces to S3, summary to DB) so the read - # path serves the cached unit instead of racing Langfuse ingestion. - saved = save_score( - eval_run_id=eval_run.id, - organization_id=eval_run.organization_id, - project_id=eval_run.project_id, - score=score, + update=EvaluationRunUpdate( + is_score_updated=_sync_scores_to_langfuse( + langfuse=langfuse, write_items=write_items, log_prefix=log_prefix + ) + ), ) - if saved is not None: - eval_run = saved - eval_run.score = cast(dict[str, object], score) + # Expose full score (traces live in S3) without flushing it into the row. + set_committed_value(eval_run, "score", score) - logger.info( - f"[run_fast_evaluation] {log_prefix} Fast evaluation completed | " - f"total_items={eval_run.total_items}" - ) return eval_run diff --git a/backend/app/crud/evaluations/fast_chunks.py b/backend/app/crud/evaluations/fast_chunks.py new file mode 100644 index 000000000..5b956a64f --- /dev/null +++ b/backend/app/crud/evaluations/fast_chunks.py @@ -0,0 +1,73 @@ +"""`batch_job` bookkeeping for the fast evaluation stages. + +A `batch_job` row carrying a `raw_output_url` is what marks a stage (or one +response chunk) as already done, so retries reload from S3 instead of re-calling +OpenAI. This module owns the queries and row shapes; the orchestrator in +`fast.py` decides when to write them. +""" + +import logging + +from sqlalchemy import Integer +from sqlmodel import Session, select + +from app.core.cloud.storage import CloudStorage +from app.crud.job import delete_batch_job +from app.models.batch_job import BatchJob + +logger = logging.getLogger(__name__) + +# job_type discriminators on batch_job for the fast-path stages. +JOB_TYPE_EVALUATION_FAST = "evaluation_fast" +JOB_TYPE_EVALUATION_FAST_CHUNK = "evaluation_fast_chunk" +JOB_TYPE_EMBEDDING_FAST = "embedding_fast" + +# batch_job.config keys tying a chunk row back to its run + slice. +CHUNK_CONFIG_RUN_ID = "eval_run_id" +CHUNK_CONFIG_INDEX = "chunk_index" + +RESPONSES_ENDPOINT = "/v1/responses" +EMBEDDINGS_ENDPOINT = "/v1/embeddings" + + +def list_response_chunk_jobs(*, session: Session, eval_run_id: int) -> list[BatchJob]: + """All response-chunk batch_jobs for a fast run, in any state.""" + statement = select(BatchJob).where( + BatchJob.job_type == JOB_TYPE_EVALUATION_FAST_CHUNK, + BatchJob.config[CHUNK_CONFIG_RUN_ID].astext.cast(Integer) == eval_run_id, + ) + return list(session.exec(statement).all()) + + +def get_chunk_job( + *, session: Session, eval_run_id: int, chunk_index: int +) -> BatchJob | None: + """The chunk batch_job for one (eval_run, chunk_index), or None.""" + statement = select(BatchJob).where( + BatchJob.job_type == JOB_TYPE_EVALUATION_FAST_CHUNK, + BatchJob.config[CHUNK_CONFIG_RUN_ID].astext.cast(Integer) == eval_run_id, + BatchJob.config[CHUNK_CONFIG_INDEX].astext.cast(Integer) == chunk_index, + ) + return session.exec(statement).first() + + +def delete_response_chunk_artifacts( + *, session: Session, storage: CloudStorage, eval_run_id: int +) -> None: + """Delete the per-chunk S3 files + batch_job rows once a run completes. + + Best-effort: a failed delete only leaks DB+S3 bloat, so it never fails the + run. Failed runs skip this and keep their chunks for the healer. + """ + try: + chunk_jobs = list_response_chunk_jobs(session=session, eval_run_id=eval_run_id) + for job in chunk_jobs: + if job.raw_output_url: + storage.delete(job.raw_output_url) + delete_batch_job(session, job) + except Exception as exc: + logger.warning( + f"[delete_response_chunk_artifacts] Cleanup failed (orphans harmless) | " + f"eval_run_id={eval_run_id} | error={exc}", + exc_info=True, + ) diff --git a/backend/app/crud/evaluations/fast_cosine.py b/backend/app/crud/evaluations/fast_cosine.py new file mode 100644 index 000000000..5a349b80a --- /dev/null +++ b/backend/app/crud/evaluations/fast_cosine.py @@ -0,0 +1,147 @@ +"""Cosine scoring for v1 fast runs. + +Pure: takes response rows + their embedding pairs, returns everything Stage 3 +persists for a non-judge run. v2 judged runs never embed, so they never reach +this module. +""" + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np + +from app.crud.evaluations.embeddings import calculate_cosine_similarity +from app.crud.evaluations.merge import apply_cosine_breakdown +from app.crud.evaluations.score import ( + COSINE_SCORE_NAME, + UNSCOREABLE_EMBEDDING_FAILED, + UNSCOREABLE_EMPTY_GROUND_TRUTH, + UNSCOREABLE_EMPTY_OUTPUT, + SummaryScore, +) + +_PER_ITEM_SCORE_PRECISION = 6 + + +def classify_empty_side(response: dict[str, Any]) -> str | None: + """Why a row can't be scored from its own text, or None if both sides are present.""" + if not response.get("generated_output"): + return UNSCOREABLE_EMPTY_OUTPUT + if not response.get("ground_truth"): + return UNSCOREABLE_EMPTY_GROUND_TRUTH + return None + + +def build_item_refs( + response_results: list[dict[str, Any]], trace_id_mapping: dict[str, str] +) -> dict[str, str]: + """Map each item id to the key its scores hang off: trace_id when traced, else item id.""" + return { + response["item_id"]: trace_id_mapping.get(response["item_id"]) + or response["item_id"] + for response in response_results + } + + +@dataclass +class CosineScoring: + """Everything Stage 3 persists for a v1 cosine run.""" + + item_id_to_score: dict[str, float] = field(default_factory=dict) + per_item_scores: dict[str, float] = field(default_factory=dict) + unscoreable: dict[str, str] = field(default_factory=dict) + write_items: list[dict[str, Any]] = field(default_factory=list) + summary_scores: list[SummaryScore] = field(default_factory=list) + + +def _has_embedding_pair(pair: dict[str, Any] | None) -> bool: + return ( + pair is not None + and pair.get("output_embedding") is not None + and pair.get("ground_truth_embedding") is not None + ) + + +def _summary_score( + similarities: list[float], *, total_items: int, unscoreable: dict[str, str] +) -> list[SummaryScore]: + if similarities: + array = np.array(similarities) + avg, std = float(np.mean(array)), float(np.std(array)) + else: + avg = std = 0.0 + + return apply_cosine_breakdown( + [ + { + "name": COSINE_SCORE_NAME, + "avg": round(avg, 2), + "std": round(std, 2), + "total_pairs": len(similarities), + "data_type": "NUMERIC", + } + ], + total_items=total_items, + unscoreable=unscoreable or None, + ) + + +def score_cosine_run( + *, + response_results: list[dict[str, Any]], + embedding_results: list[dict[str, Any]] | None, + item_refs: dict[str, str], + trace_id_mapping: dict[str, str], + total_items: int, +) -> CosineScoring: + """Score every row by cosine similarity and build the Langfuse write list. + + A row without a usable embedding pair is recorded in `unscoreable` with the + reason the UI shows, and stays out of the average. + """ + pair_by_item_id = { + r["item_id"]: r for r in (embedding_results or []) if not r.get("failed") + } + + result = CosineScoring() + similarities: list[float] = [] + scored_writes: list[dict[str, Any]] = [] + + for response in response_results: + item_id = response["item_id"] + ref = item_refs[item_id] + pair = pair_by_item_id.get(item_id) + if not _has_embedding_pair(pair): + result.unscoreable[ref] = ( + classify_empty_side(response) or UNSCOREABLE_EMBEDDING_FAILED + ) + continue + + assert pair is not None # guaranteed by _has_embedding_pair + cosine = calculate_cosine_similarity( + pair["output_embedding"], pair["ground_truth_embedding"] + ) + similarities.append(cosine) + result.item_id_to_score[item_id] = cosine + scored_writes.append( + {"trace_id": trace_id_mapping.get(item_id), "cosine_similarity": cosine} + ) + + unscoreable_writes = [ + {"trace_id": trace_id_mapping[item_id], "unscoreable": True, "reason": reason} + for item_id, ref in item_refs.items() + if item_id in trace_id_mapping and (reason := result.unscoreable.get(ref)) + ] + # Untraced runs have no trace_id to write against, so those entries drop out. + result.write_items = [ + w for w in scored_writes if w["trace_id"] is not None + ] + unscoreable_writes + + result.per_item_scores = { + item_refs[item_id]: round(float(score), _PER_ITEM_SCORE_PRECISION) + for item_id, score in result.item_id_to_score.items() + } + result.summary_scores = _summary_score( + similarities, total_items=total_items, unscoreable=result.unscoreable + ) + return result diff --git a/backend/app/crud/evaluations/fast_results.py b/backend/app/crud/evaluations/fast_results.py new file mode 100644 index 000000000..507e46950 --- /dev/null +++ b/backend/app/crud/evaluations/fast_results.py @@ -0,0 +1,131 @@ +"""Per-item result shapes for the fast evaluation stages. + +Pure builders and aggregation helpers: no OpenAI, DB or storage access. The +dicts produced here are the units uploaded to S3, so they must stay +JSON-serializable and match the batch path's shape. +""" + +from typing import Any, TypedDict + +from app.core.config import settings +from app.crud.evaluations.response_parsing import field_value + + +class ResponseResult(TypedDict, total=False): + """Stage 1 response evaluation result.""" + + item_id: str + question: str + generated_output: str + ground_truth: str + response_id: str | None + usage: dict[str, int] + question_id: int | None + failed: bool + retrieved_chunks: list[dict[str, Any]] + + +class EmbeddingResult(TypedDict, total=False): + """Stage 2 embedding pair result.""" + + item_id: str + output_embedding: list[float] | None + ground_truth_embedding: list[float] | None + usage: dict[str, int] + failed: bool + error: str + + +RESPONSE_USAGE_KEYS: tuple[str, ...] = ("input_tokens", "output_tokens", "total_tokens") +EMBEDDING_USAGE_KEYS: tuple[str, ...] = ("prompt_tokens", "total_tokens") + + +def build_response_result( + *, + item_id: str, + question: str, + ground_truth: str, + question_id: int | None, + generated_output: str, + failed: bool, + response_id: str | None = None, + usage: dict[str, int] | None = None, + retrieved_chunks: list[dict[str, Any]] | None = None, +) -> ResponseResult: + """One Stage-1 per-item result, in the batch path's shape.""" + return { + "item_id": item_id, + "question": question, + "generated_output": generated_output, + "ground_truth": ground_truth, + "response_id": response_id, + "usage": usage, + "question_id": question_id, + "failed": failed, + "retrieved_chunks": retrieved_chunks, + } + + +def build_embedding_failure(item_id: str, error: str) -> EmbeddingResult: + """One failed Stage-2 per-pair result.""" + return { + "item_id": item_id, + "output_embedding": None, + "ground_truth_embedding": None, + "usage": {}, + "failed": True, + "error": error, + } + + +def extract_usage(usage_obj: Any, keys: tuple[str, ...]) -> dict[str, int]: + """Read the given token counters off an OpenAI usage object, defaulting to 0.""" + return {key: int(field_value(usage_obj, key, 0) or 0) for key in keys} + + +def parse_embedding_pair(*, item_id: str, response: Any) -> EmbeddingResult: + """Unpack an embeddings response into one Stage-2 per-pair result. + + The request embeds `[output_text, ground_truth]`, so index 0 is the generated + output's vector and index 1 the ground truth's. + """ + data = field_value(response, "data") or [] + if len(data) < 2: + return build_embedding_failure( + item_id, f"expected 2 embeddings, got {len(data)}" + ) + + output_embedding: list[float] | None = None + ground_truth_embedding: list[float] | None = None + for embedding in data: + index = field_value(embedding, "index") + vector = field_value(embedding, "embedding") + if index == 0: + output_embedding = vector + elif index == 1: + ground_truth_embedding = vector + + return { + "item_id": item_id, + "output_embedding": output_embedding, + "ground_truth_embedding": ground_truth_embedding, + "usage": extract_usage(field_value(response, "usage"), EMBEDDING_USAGE_KEYS), + "failed": output_embedding is None or ground_truth_embedding is None, + } + + +def sum_usage(results: list[dict[str, Any]], keys: tuple[str, ...]) -> dict[str, int]: + """Sum the per-item `usage` token counts across results, for the given keys.""" + totals = dict.fromkeys(keys, 0) + for result in results: + usage = result.get("usage") or {} + for key in keys: + totals[key] += int(usage.get(key, 0) or 0) + return totals + + +def is_failure_threshold_breached(*, failed_rows: int, total_rows: int) -> bool: + """True if the failed-row fraction exceeds EVAL_FAST_FAILURE_THRESHOLD.""" + if total_rows == 0: + return False + return (failed_rows / total_rows) > settings.EVAL_FAST_FAILURE_THRESHOLD diff --git a/backend/app/crud/evaluations/fast_traces.py b/backend/app/crud/evaluations/fast_traces.py new file mode 100644 index 000000000..0ec6d7e5b --- /dev/null +++ b/backend/app/crud/evaluations/fast_traces.py @@ -0,0 +1,166 @@ +"""Per-trace record building for fast evaluation runs. + +Pure: turns the responses unit plus the scores already computed for it into the +`TraceData` records the read path serves. Keyed by `ref` (trace_id when the run +is traced, else item_id) so untraced v2 runs persist the same shape. +""" + +from typing import Any + +from app.crud.evaluations.judge import JudgeMetricEnum, JudgeMetricSpec, JudgeResult +from app.crud.evaluations.score import ( + COSINE_SCORE_COMMENT, + COSINE_SCORE_NAME, + DEFAULT_CATEGORY, + JUDGE_FAILED_REASON, + TraceData, + TraceScore, + verdict_from_score, +) + +# How many top KB matches to name in the knowledge_base trace comment. +_KB_TOP_CHUNKS = 3 + +_KB_NOT_QUERIED = "Knowledge base not queried." +_KB_SCORE_UNAVAILABLE = "Knowledge base score unavailable for this row." + + +def format_top_kb_matches(sorted_chunks: list[dict[str, Any]]) -> str: + """Top-N retrieved chunks as 'biu-1.pdf (90.6%), faq.pdf (66.3%)'. + + Expects chunks pre-sorted by score desc; old S3 payloads may lack filename. + """ + matches = [ + f"{c.get('filename') or 'unknown'} ({c.get('score', 0) * 100:.1f}%)" + for c in sorted_chunks + ] + return ", ".join(matches[:_KB_TOP_CHUNKS]) + + +def _cosine_trace_score( + *, cosine: float | None, unscoreable_reason: str | None +) -> TraceScore | None: + """The cosine entry for one v1 trace, or None when neither applies.""" + if cosine is not None: + return { + "name": COSINE_SCORE_NAME, + "value": round(cosine, 2), + "data_type": "NUMERIC", + "comment": COSINE_SCORE_COMMENT, + } + # A judge_failed-only reason is about the judge, not cosine, so it gets no + # placeholder. Other reasons get a 0 excluded from summary stats by the marker. + if unscoreable_reason is not None and unscoreable_reason != JUDGE_FAILED_REASON: + return { + "name": COSINE_SCORE_NAME, + "value": 0, + "data_type": "NUMERIC", + "comment": f"Cannot compute: {unscoreable_reason}", + "unscoreable": True, + } + return None + + +def _kb_placeholder_score( + *, spec: JudgeMetricSpec, sorted_chunks: list[dict[str, Any]] +) -> TraceScore: + """Human-readable N/A for a row the knowledge_base metric was dropped on.""" + reason = _KB_NOT_QUERIED if not sorted_chunks else _KB_SCORE_UNAVAILABLE + return { + "name": spec.score_name, + "value": "N/A", + "data_type": "CATEGORICAL", + "comment": reason, + "unscoreable": True, + } + + +def _judge_trace_scores( + *, + judge_result: JudgeResult, + metrics: list[JudgeMetricSpec], + retrieved_chunks: list[dict[str, Any]], +) -> list[TraceScore]: + """One row's judge entries, each carrying its reasoning as the score comment.""" + sorted_chunks = sorted( + retrieved_chunks, key=lambda c: c.get("score", 0), reverse=True + ) + top_matches = format_top_kb_matches(sorted_chunks) + + scores: list[TraceScore] = [] + for spec in metrics: + metric_score = judge_result.metrics.get(spec.key) + is_kb = spec.key == JudgeMetricEnum.KNOWLEDGE_BASE + if metric_score is None: + if is_kb: + scores.append( + _kb_placeholder_score(spec=spec, sorted_chunks=sorted_chunks) + ) + continue + + comment = metric_score.reasoning + if is_kb: + comment = f"{comment} | Top matches: {top_matches}" + rounded_score = round(metric_score.score, 2) + scores.append( + { + "name": spec.score_name, + "value": rounded_score, + "data_type": "NUMERIC", + "comment": comment, + "verdict": verdict_from_score(rounded_score), + } + ) + return scores + + +def build_trace_records( + *, + response_results: list[dict[str, Any]], + item_refs: dict[str, str], + is_judge_run: bool | None, + judge_results: dict[str, JudgeResult], + metrics: list[JudgeMetricSpec], + cosine_by_item_id: dict[str, float], + unscoreable: dict[str, str], +) -> list[TraceData]: + """Build every trace record for the run. + + v1 rows carry the cosine score (or its unscoreable placeholder); v2 rows carry + one entry per judge metric and no cosine placeholder. + """ + traces: list[TraceData] = [] + for response in response_results: + item_id: str = response["item_id"] + ref: str = item_refs.get(item_id) or item_id + trace_scores: list[TraceScore] = [] + + if not is_judge_run: + cosine_score = _cosine_trace_score( + cosine=cosine_by_item_id.get(item_id), + unscoreable_reason=unscoreable.get(ref), + ) + if cosine_score is not None: + trace_scores.append(cosine_score) + + judge_result = judge_results.get(item_id) + if judge_result is not None: + trace_scores.extend( + _judge_trace_scores( + judge_result=judge_result, + metrics=metrics, + retrieved_chunks=response.get("retrieved_chunks") or [], + ) + ) + + trace: TraceData = { + "trace_id": ref, + "question": response.get("question", ""), + "llm_answer": response.get("generated_output", ""), + "ground_truth_answer": response.get("ground_truth", ""), + "question_id": response.get("question_id"), + "category": response.get("category") or DEFAULT_CATEGORY, + "scores": trace_scores, + } + traces.append(trace) + return traces diff --git a/backend/app/crud/evaluations/iteration.py b/backend/app/crud/evaluations/iteration.py index 2b7ff1961..f8751b16b 100644 --- a/backend/app/crud/evaluations/iteration.py +++ b/backend/app/crud/evaluations/iteration.py @@ -30,7 +30,10 @@ def create_evaluation_iteration_run( organization_id: int, project_id: int, ) -> EvaluationIterationRun: - """Create the thin tracking row, status=PROCESSING.""" + """Create the thin tracking row, status=PROCESSING. + + `last_dispatched_at` is stamped so the cron cooldown covers kickoff's first step. + """ iteration_run = EvaluationIterationRun( dataset_id=dataset_id, experiment_name=experiment_name, @@ -38,17 +41,13 @@ def create_evaluation_iteration_run( initial_config_version=initial_config_version, callback_url=callback_url, status=EvaluationIterationStatusEnum.PROCESSING, + last_dispatched_at=now(), organization_id=organization_id, project_id=project_id, ) session.add(iteration_run) session.commit() session.refresh(iteration_run) - logger.info( - f"[create_evaluation_iteration_run] Created | " - f"iteration_run_id={iteration_run.id} | dataset_id={dataset_id} | " - f"org_id={organization_id} | project_id={project_id}" - ) return iteration_run @@ -83,11 +82,7 @@ def list_processing_evaluation_iteration_runs( statement = select(EvaluationIterationRun).where( EvaluationIterationRun.status == EvaluationIterationStatusEnum.PROCESSING ) - runs = list(session.exec(statement).all()) - logger.info( - f"[list_processing_evaluation_iteration_runs] Found {len(runs)} processing loops" - ) - return runs + return list(session.exec(statement).all()) def update_evaluation_iteration_run( diff --git a/backend/app/crud/evaluations/judge.py b/backend/app/crud/evaluations/judge.py index 164440927..e6752250e 100644 --- a/backend/app/crud/evaluations/judge.py +++ b/backend/app/crud/evaluations/judge.py @@ -16,24 +16,20 @@ import openai from openai import OpenAI from sqlmodel import Session -from tenacity import ( - before_sleep_log, - retry, - retry_if_exception_type, - stop_after_attempt, - wait_random_exponential, -) from app.core.config import settings -from app.crud.evaluations.response_parsing import extract_response_text -from app.crud.evaluations.score import ( +from app.crud.evaluations.judge_prompts import ( GROUND_TRUTH_JUDGE_PROMPT, - GROUND_TRUTH_SCORE_NAME, JUDGE_OUTPUT_INSTRUCTION, JUDGE_SYSTEM_PREAMBLE, KNOWLEDGE_BASE_JUDGE_PROMPT, - KNOWLEDGE_BASE_SCORE_NAME, PROMPT_JUDGE_PROMPT, +) +from app.crud.evaluations.response_parsing import extract_response_text +from app.crud.evaluations.retry import retry_openai_call +from app.crud.evaluations.score import ( + GROUND_TRUTH_SCORE_NAME, + KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, ) from app.services.llm.mappers import map_kaapi_to_openai_params @@ -125,28 +121,7 @@ class JudgeMetricSpec: } -# Per-call retry mechanism (mirrors the fast-eval Responses/Embeddings stages). -_RETRY_MAX_ATTEMPTS = 3 -_RETRY_BASE_DELAY_SECONDS = 1.0 -_RETRY_MAX_DELAY_SECONDS = 30.0 - -_RETRYABLE_OPENAI_ERRORS: tuple[type[Exception], ...] = ( - openai.RateLimitError, - openai.APITimeoutError, - openai.APIConnectionError, - openai.InternalServerError, -) - -# reraise=True so the call-site handler sees the original OpenAIError. -_retry_judge_call = retry( - retry=retry_if_exception_type(_RETRYABLE_OPENAI_ERRORS), - wait=wait_random_exponential( - multiplier=_RETRY_BASE_DELAY_SECONDS, max=_RETRY_MAX_DELAY_SECONDS - ), - stop=stop_after_attempt(_RETRY_MAX_ATTEMPTS), - before_sleep=before_sleep_log(logger, logging.INFO), - reraise=True, -) +_retry_judge_call = retry_openai_call(logger) @dataclass diff --git a/backend/app/crud/evaluations/judge_prompts.py b/backend/app/crud/evaluations/judge_prompts.py new file mode 100644 index 000000000..102a09e9d --- /dev/null +++ b/backend/app/crud/evaluations/judge_prompts.py @@ -0,0 +1,154 @@ +"""Judge rubric text for the v2 native LLM-as-a-judge. + +Prompt content only, no scoring logic. `judge.py` composes the preamble with each +metric's fragment; `JUDGE_OUTPUT_INSTRUCTION` is formatted with the metric keys +that apply to the row being graded. + +Each fragment names the input-block labels from `judge.py::_INPUT_LABELS` +verbatim in its CONSIDER/IGNORE lines, so the two must be edited together. +""" + +JUDGE_SYSTEM_PREAMBLE: str = ( + "You are a strict, impartial evaluator. You score an assistant's answer on the " + "independent metrics listed below in a single pass. Each metric is an integer " + "score from 0 to 5, where 0 is the worst case (clearly wrong / ungrounded / a hard " + "instruction violation, or no answer at all) and 5 is the best case (fully correct " + "/ fully grounded / fully compliant), with one or two sentences of reasoning. " + "Scores MUST be integers — 0, 1, 2, 3, 4, or 5 — never fractions, decimals, or " + "percentages. The metrics are independent — judge each only against its own inputs " + "and rules; do not let one metric's verdict bleed into another. Score EVERY metric " + "listed below. Never omit a metric from the output, even if some input blocks are " + "irrelevant to it." +) + +GROUND_TRUTH_JUDGE_PROMPT: str = ( + 'Adherence to Ground Truth (score key "ground_truth"):\n' + "Judge ONLY whether the assistant's answer conveys the same correct information " + "as the golden answer.\n" + "- Judge meaning, not wording. A correct paraphrase, a different order, or extra " + "detail that is also correct must score high.\n" + "- Lower the score for information that is missing, incomplete, or contradicts " + "the golden answer. An answer that states something the golden answer does not, " + "and that would be wrong, is a factual error.\n" + "- Do NOT reward or penalize style, tone, length, or language.\n" + "- Do NOT use any outside knowledge; the golden answer is the source of truth.\n" + "- Do NOT answer the question yourself.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: Fully correct and complete. Conveys everything material in the golden answer " + "(a paraphrase, reordering, or additional correct detail is still a 5).\n" + "- 4: Correct and materially complete, but omits one minor, non-essential " + "supporting detail.\n" + "- 3: Partially correct. The core of the answer is right, but at least one material " + "fact is missing, incomplete, or slightly off.\n" + "- 2: Mixed or significantly incomplete. Gets some of the answer right but muddles " + "or omits more than one material fact, or is wrong on a meaningful component while " + "looking plausible on the surface.\n" + "- 1: Mostly incorrect. Contradicts the golden answer on a key point; at most small " + "correct fragments remain.\n" + "- 0: Completely wrong or contradicts the golden answer outright, OR the row has no " + "answer / an errored, empty, or non-responsive output.\n" + "Reasoning: name what was correct or what was missing/contradicted.\n" + "When scoring THIS metric, consider only these input blocks: Question, " + "Generated answer, Golden (reference) answer.\n" + "Do not consider: Assistant's configured instructions, Retrieved " + "knowledge-base chunks." +) + +PROMPT_JUDGE_PROMPT: str = ( + 'Adherence to Prompt (score key "prompt"):\n' + "Judge ONLY whether the answer obeys the assistant's configured instructions " + '(the separate input block labelled "Assistant\'s configured instructions"). Do ' + "NOT judge factual correctness or grounding/sourcing: you cannot see the retrieved " + "documents, so treat any rule about which source or knowledge base to use (e.g. " + "'only use the knowledge base', 'do not use outside information') as satisfied — " + "the Knowledge Base metric judges that.\n\n" + 'Start from "no violations" and deduct ONLY for a violation of an instruction the ' + "block actually states. Never invent a requirement the instructions do not set; a " + "conditional rule (applies only in a specific situation, e.g. " + "'ask for the user's age' or 'if condition Y holds, also mention Z') counts as " + "satisfied unless that situation is present in the question. Deduct across " + "whichever of these dimensions apply:\n" + "1. Language & tone — answer is in the language, style, and " + "tone the instructions require. Judge the LANGUAGE of the words, not the script or " + "alphabet they are written in: text written in a transliterated or romanised form " + "of the required language still counts as that language and MUST NOT be scored as a " + "different one. Code-mixing — sentences in the required language that borrow common " + "loanwords or technical terms from another language — is still the required " + "language and is NOT a violation, unless the instructions explicitly forbid such " + "borrowing.\n" + "2. Answer vs refuse — answers in-scope questions; refuses " + "out-of-scope or disallowed ones as instructed.\n" + "3. Fallback compliance — when the instructions define a fallback for the " + "unknown/out-of-scope case, the answer uses it instead of ignoring it. Only " + "penalize here for CONTRADICTING an explicit instruction; do not infer " + "fabrication from missing grounding.\n" + "4. Format compliance — follows any explicit format rules " + "(word limit, structure, opening/closing pattern).\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: No violation of any stated instruction, across all applicable dimensions.\n" + "- 4: One soft, minor miss on a stated rule (e.g. slightly off tone, minor format " + "deviation) — otherwise compliant.\n" + "- 3: One clear violation of a single explicit rule.\n" + "- 2: Multiple clear violations, or one moderately serious violation spanning more " + "than one dimension.\n" + "- 1: A severe violation of a core instruction (e.g. ignoring a configured fallback " + "on a disallowed ask, a partial injection hijack), but not a full hard violation.\n" + "- 0: A hard violation — leaked system prompt, fully answered a clearly disallowed " + "topic, fully hijacked by injection — OR the row has no answer / an errored, empty, " + "or non-responsive output.\n" + "Reasoning: name the specific violated instruction and how the answer violated it. " + "If no stated instruction was violated, say so and score 5.\n" + "When scoring THIS metric, consider only these input blocks: Assistant's " + "configured instructions, Question, Generated answer.\n" + "Do not consider: Golden (reference) answer, Retrieved knowledge-base chunks." +) + +KNOWLEDGE_BASE_JUDGE_PROMPT: str = ( + 'Adherence to Knowledge Base (score key "knowledge_base"):\n' + "Judge ONLY whether the answer's claims are supported by the retrieved " + "knowledge-base chunks (groundedness / hallucination detection).\n" + "- Break the answer into its distinct factual claims. A claim is supported ONLY " + "if the specific fact it asserts is explicitly stated in the chunk text (a " + "verbatim or trivially reworded restatement). A claim that is merely plausible, " + "on the same topic as a chunk, or that requires an inferential leap the chunks do " + "not spell out is UNSUPPORTED, not supported.\n" + "- Identify the answer's load-bearing (material) claims — the ones that carry its " + "substance.\n" + "- Text that makes no factual claim (a greeting, a pleasantry, or a plain refusal " + "to answer) is EXCLUDED from the claim count.\n" + "- Judge groundedness ONLY, not correctness, completeness, or " + "instruction-following. A claim faithful to the chunks is grounded even if the " + "chunks are themselves wrong.\n" + "- Do NOT use any outside knowledge; the retrieved chunks are the ONLY allowed " + "source of support.\n\n" + "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " + "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" + "- 5: Every factual claim is explicitly supported by the retrieved chunks. Fully " + "grounded.\n" + "- 4: All load-bearing claims are grounded; only a minor, non-material claim lacks " + "explicit support.\n" + "- 3: One non-critical inferential leap beyond the chunks, but no load-bearing " + "claim is fabricated.\n" + "- 2: At least one load-bearing/material claim is unsupported or invented, even " + "though other claims are grounded.\n" + "- 1: Most claims are unsupported or invented; only incidental/minor claims are " + "grounded.\n" + "- 0: The answer is fabricated wholesale — no claim is grounded in the retrieved " + "chunks — OR the row has no answer / an errored, empty, or non-responsive output.\n" + "Reasoning: quote the exact chunk span supporting the main claim. When the score " + "is below 5, name the specific unsupported or invented claim.\n" + "When scoring THIS metric, consider only these input blocks: Generated answer, " + "Retrieved knowledge-base chunks.\n" + "Do not consider: Assistant's configured instructions, Question, Golden " + "(reference) answer." +) + +JUDGE_OUTPUT_INSTRUCTION: str = ( + "Respond with ONLY a single JSON object mapping each metric key to its result, of " + 'the form {{"": {{"score": , "reasoning": ' + '""}}}}. Scores MUST be integers 0-5. Every ' + '"reasoning" string MUST be written in English. Include exactly these metric keys: ' + "{metric_keys}. Output nothing else." +) diff --git a/backend/app/crud/evaluations/judge_stage.py b/backend/app/crud/evaluations/judge_stage.py new file mode 100644 index 000000000..6d4bc1193 --- /dev/null +++ b/backend/app/crud/evaluations/judge_stage.py @@ -0,0 +1,194 @@ +"""v2 judge stage: resolve the run's inputs and grade every judgeable row. + +Wraps the single-row judge in `judge.py` with the run-level concerns — which +rows can be judged, the shared config prompt, the worker pool, and the run-level +summary each metric rolls up to. +""" + +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any + +import numpy as np +from openai import OpenAI +from pydantic import ValidationError +from sqlmodel import Session + +from app.core.config import settings +from app.crud.evaluations.core import resolve_evaluation_config +from app.crud.evaluations.judge import ( + JudgeInputEnum, + JudgeMetricSpec, + JudgeResult, + build_judge_params, + judge_row, +) +from app.crud.evaluations.score import SummaryScore +from app.models.evaluation import EvaluationRun +from app.models.llm.request import TextLLMParams + +logger = logging.getLogger(__name__) + +# Judge tells the template apart from the instructions above it. +PROMPT_TEMPLATE_LABEL = "Prompt template wrapped around each user input:" + +_CHUNK_SEPARATOR = "\n---\n" + + +def resolve_config_prompt(*, session: Session, eval_run: EvaluationRun) -> str | None: + """The evaluated bot's own configured prompt, or None if unresolvable. + + The prompt template is appended when the config carries one, since it is + equally part of what the bot was told to do. Returns None when the config + carries no instructions, so the caller drops the prompt metric rather than + grading against "". + """ + if not eval_run.config_id or not eval_run.config_version: + return None + + config, error = resolve_evaluation_config( + session=session, + config_id=eval_run.config_id, + config_version=eval_run.config_version, + project_id=eval_run.project_id, + ) + if error or config is None: + return None + + # Native/proxy params aren't TextLLMParams-shaped; a mismatch just means there + # are no instructions to grade against, not a run failure. + try: + params = TextLLMParams.model_validate(config.completion.params) + except ValidationError: + return None + + sections: list[str] = [] + if params.instructions: + sections.append(params.instructions.strip()) + if config.prompt_template and config.prompt_template.template: + sections.append( + f"{PROMPT_TEMPLATE_LABEL}\n{config.prompt_template.template.strip()}" + ) + return "\n\n".join(sections) if sections else None + + +def build_judge_inputs( + *, response: dict[str, Any], config_prompt: str +) -> dict[JudgeInputEnum, str]: + """One row's judge input blocks; an empty value drops the metrics needing it.""" + return { + JudgeInputEnum.CONFIG_PROMPT: config_prompt, + JudgeInputEnum.QUESTION: response.get("question", ""), + JudgeInputEnum.GENERATED_ANSWER: response.get("generated_output", ""), + JudgeInputEnum.GOLDEN_ANSWER: response.get("ground_truth", ""), + JudgeInputEnum.RETRIEVED_CHUNKS: _CHUNK_SEPARATOR.join( + chunk.get("text", "") + for chunk in (response.get("retrieved_chunks") or []) + if chunk.get("text") + ), + } + + +def select_judgeable_rows( + response_results: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Rows with both a generated and a golden answer; empty sides are unscoreable.""" + return [ + response + for response in response_results + if response.get("generated_output") and response.get("ground_truth") + ] + + +def judge_rows( + *, + session: Session, + openai_client: OpenAI, + metrics: list[JudgeMetricSpec], + config_prompt: str, + judgeable: list[dict[str, Any]], + item_refs: dict[str, str], + log_prefix: str, +) -> tuple[dict[str, JudgeResult], set[str], str | None]: + """Run one combined judge completion per judgeable row, isolated per row. + + Returns the per-item results, the refs whose judging failed, and the judge + model. `metrics` is the full registry; `judge_row` drops the ones a given row + cannot supply inputs for. `config_prompt` is "" when the run's config carried + no instructions, which drops the prompt metric for every row. + """ + results: dict[str, JudgeResult] = {} + failed_refs: set[str] = set() + if not judgeable: + return results, failed_refs, None + + # Built once per run: judging is system-config only, so every metric uses its + # built-in prompt + shared model. Instructions vary per row (by applicable-metric + # subset) and are composed inside judge_row. + try: + base_params = build_judge_params(session=session) + except Exception as exc: + logger.error( + f"[judge_rows] {log_prefix} Judge setup failed; leaving all rows " + f"unjudged | error={exc}", + exc_info=True, + ) + return results, {item_refs[r["item_id"]] for r in judgeable}, None + + max_workers = max(1, min(settings.EVAL_JUDGE_CONCURRENCY, len(judgeable))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_map = { + executor.submit( + judge_row, + openai_client=openai_client, + base_params=base_params, + metrics=metrics, + inputs=build_judge_inputs( + response=response, config_prompt=config_prompt + ), + ): response["item_id"] + for response in judgeable + } + for future in as_completed(future_map): + item_id = future_map[future] + try: + results[item_id] = future.result() + except Exception as exc: + ref = item_refs[item_id] + failed_refs.add(ref) + logger.warning( + f"[judge_rows] {log_prefix} Judge failed for row; flagged " + f"unscoreable | item_id={item_id} | ref={ref} | error={exc}" + ) + + return results, failed_refs, base_params.get("model") + + +def build_metric_summary_scores( + *, metrics: list[JudgeMetricSpec], judge_results: dict[str, JudgeResult] +) -> list[SummaryScore]: + """Run-level summary score per metric that graded at least one row. + + Per-row scores and reasoning live on the traces, which is what the read path + serves; only the aggregate belongs on the run. + """ + summary_scores: list[SummaryScore] = [] + for spec in metrics: + values = [ + metric_score.score + for result in judge_results.values() + if (metric_score := result.metrics.get(spec.key)) is not None + ] + if not values: + continue + array = np.array(values) + summary_scores.append( + { + "name": spec.score_name, + "avg": round(float(np.mean(array)), 2), + "std": round(float(np.std(array)), 2), + "total_pairs": len(values), + "data_type": "NUMERIC", + } + ) + return summary_scores diff --git a/backend/app/crud/evaluations/processing.py b/backend/app/crud/evaluations/processing.py index 42aad8b1b..490900e50 100644 --- a/backend/app/crud/evaluations/processing.py +++ b/backend/app/crud/evaluations/processing.py @@ -34,6 +34,7 @@ from app.core.storage_utils import load_json_from_object_store from app.crud.evaluations.batch import fetch_dataset_items from app.crud.evaluations.core import ( + build_log_prefix, persist_score_traces, resolve_model_from_config, save_score, @@ -396,7 +397,7 @@ async def process_completed_evaluation( Raises: Exception: If processing fails """ - log_prefix = f"[org={eval_run.organization_id}][project={eval_run.project_id}][eval={eval_run.id}]" + log_prefix = build_log_prefix(eval_run) logger.info( f"[process_completed_evaluation] {log_prefix} Processing completed evaluation" ) @@ -661,7 +662,7 @@ async def process_completed_embedding_batch( Raises: Exception: If processing fails """ - log_prefix = f"[org={eval_run.organization_id}][project={eval_run.project_id}][eval={eval_run.id}]" + log_prefix = build_log_prefix(eval_run) logger.info( f"[process_completed_embedding_batch] {log_prefix} Processing completed embedding batch" ) @@ -892,7 +893,7 @@ async def check_and_process_evaluation( "action": "processed" | "embeddings_completed" | "embeddings_failed" | "failed" | "no_change" } """ - log_prefix = f"[org={eval_run.organization_id}][project={eval_run.project_id}][eval={eval_run.id}]" + log_prefix = build_log_prefix(eval_run) previous_status = eval_run.status try: diff --git a/backend/app/crud/evaluations/retry.py b/backend/app/crud/evaluations/retry.py new file mode 100644 index 000000000..16cd136a6 --- /dev/null +++ b/backend/app/crud/evaluations/retry.py @@ -0,0 +1,52 @@ +"""Shared OpenAI retry policy for the synchronous evaluation stages. + +Responses, embeddings and the judge all issue single-row OpenAI calls from a +worker thread pool, so they share one transient-error policy rather than each +declaring its own. +""" + +import logging +from collections.abc import Callable +from typing import ParamSpec, TypeVar + +import openai +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_random_exponential, +) + +RETRY_MAX_ATTEMPTS = 3 +RETRY_BASE_DELAY_SECONDS = 1.0 +RETRY_MAX_DELAY_SECONDS = 30.0 + +RETRYABLE_OPENAI_ERRORS: tuple[type[Exception], ...] = ( + openai.RateLimitError, + openai.APITimeoutError, + openai.APIConnectionError, + openai.InternalServerError, +) + +P = ParamSpec("P") +R = TypeVar("R") + + +def retry_openai_call( + logger: logging.Logger, +) -> Callable[[Callable[P, R]], Callable[P, R]]: + """Tenacity decorator retrying transient OpenAI errors with jittered backoff. + + `reraise=True` so call-site handlers see the original `OpenAIError` rather + than tenacity's `RetryError`. + """ + return retry( + retry=retry_if_exception_type(RETRYABLE_OPENAI_ERRORS), + wait=wait_random_exponential( + multiplier=RETRY_BASE_DELAY_SECONDS, max=RETRY_MAX_DELAY_SECONDS + ), + stop=stop_after_attempt(RETRY_MAX_ATTEMPTS), + before_sleep=before_sleep_log(logger, logging.INFO), + reraise=True, + ) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index 23ea3fbd7..c04d6d055 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -1,9 +1,4 @@ -""" -Type definitions for evaluation scores. - -This module contains TypedDict definitions for type-safe score data -used throughout the evaluation system. -""" +"""Score types, verdict banding and the run-level overall rollup.""" from enum import Enum from typing import NotRequired, TypedDict @@ -63,151 +58,6 @@ def verdict_from_score(score: float) -> VerdictEnum: JUDGE_FAILED_REASON, ) -JUDGE_SYSTEM_PREAMBLE: str = ( - "You are a strict, impartial evaluator. You score an assistant's answer on the " - "independent metrics listed below in a single pass. Each metric is an integer " - "score from 0 to 5, where 0 is the worst case (clearly wrong / ungrounded / a hard " - "instruction violation, or no answer at all) and 5 is the best case (fully correct " - "/ fully grounded / fully compliant), with one or two sentences of reasoning. " - "Scores MUST be integers — 0, 1, 2, 3, 4, or 5 — never fractions, decimals, or " - "percentages. The metrics are independent — judge each only against its own inputs " - "and rules; do not let one metric's verdict bleed into another. Score EVERY metric " - "listed below. Never omit a metric from the output, even if some input blocks are " - "irrelevant to it." -) - -GROUND_TRUTH_JUDGE_PROMPT: str = ( - 'Adherence to Ground Truth (score key "ground_truth"):\n' - "Judge ONLY whether the assistant's answer conveys the same correct information " - "as the golden answer.\n" - "- Judge meaning, not wording. A correct paraphrase, a different order, or extra " - "detail that is also correct must score high.\n" - "- Lower the score for information that is missing, incomplete, or contradicts " - "the golden answer. An answer that states something the golden answer does not, " - "and that would be wrong, is a factual error.\n" - "- Do NOT reward or penalize style, tone, length, or language.\n" - "- Do NOT use any outside knowledge; the golden answer is the source of truth.\n" - "- Do NOT answer the question yourself.\n\n" - "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " - "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" - "- 5: Fully correct and complete. Conveys everything material in the golden answer " - "(a paraphrase, reordering, or additional correct detail is still a 5).\n" - "- 4: Correct and materially complete, but omits one minor, non-essential " - "supporting detail.\n" - "- 3: Partially correct. The core of the answer is right, but at least one material " - "fact is missing, incomplete, or slightly off.\n" - "- 2: Mixed or significantly incomplete. Gets some of the answer right but muddles " - "or omits more than one material fact, or is wrong on a meaningful component while " - "looking plausible on the surface.\n" - "- 1: Mostly incorrect. Contradicts the golden answer on a key point; at most small " - "correct fragments remain.\n" - "- 0: Completely wrong or contradicts the golden answer outright, OR the row has no " - "answer / an errored, empty, or non-responsive output.\n" - "Reasoning: name what was correct or what was missing/contradicted.\n" - "When scoring THIS metric, consider only these input blocks: Question, " - "Generated answer, Golden (reference) answer.\n" - "Do not consider: Assistant's configured instructions, Retrieved " - "knowledge-base chunks." -) - -PROMPT_JUDGE_PROMPT: str = ( - 'Adherence to Prompt (score key "prompt"):\n' - "Judge ONLY whether the answer obeys the assistant's configured instructions " - '(the separate input block labelled "Assistant\'s configured instructions"). Do ' - "NOT judge factual correctness or grounding/sourcing: you cannot see the retrieved " - "documents, so treat any rule about which source or knowledge base to use (e.g. " - "'only use the knowledge base', 'do not use outside information') as satisfied — " - "the Knowledge Base metric judges that.\n\n" - 'Start from "no violations" and deduct ONLY for a violation of an instruction the ' - "block actually states. Never invent a requirement the instructions do not set; a " - "conditional rule (applies only in a specific situation, e.g. " - "'ask for the user's age' or 'if condition Y holds, also mention Z') counts as " - "satisfied unless that situation is present in the question. Deduct across " - "whichever of these dimensions apply:\n" - "1. Language & tone — answer is in the language, style, and " - "tone the instructions require. Judge the LANGUAGE of the words, not the script or " - "alphabet they are written in: text written in a transliterated or romanised form " - "of the required language still counts as that language and MUST NOT be scored as a " - "different one. Code-mixing — sentences in the required language that borrow common " - "loanwords or technical terms from another language — is still the required " - "language and is NOT a violation, unless the instructions explicitly forbid such " - "borrowing.\n" - "2. Answer vs refuse — answers in-scope questions; refuses " - "out-of-scope or disallowed ones as instructed.\n" - "3. Fallback compliance — when the instructions define a fallback for the " - "unknown/out-of-scope case, the answer uses it instead of ignoring it. Only " - "penalize here for CONTRADICTING an explicit instruction; do not infer " - "fabrication from missing grounding.\n" - "4. Format compliance — follows any explicit format rules " - "(word limit, structure, opening/closing pattern).\n\n" - "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " - "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" - "- 5: No violation of any stated instruction, across all applicable dimensions.\n" - "- 4: One soft, minor miss on a stated rule (e.g. slightly off tone, minor format " - "deviation) — otherwise compliant.\n" - "- 3: One clear violation of a single explicit rule.\n" - "- 2: Multiple clear violations, or one moderately serious violation spanning more " - "than one dimension.\n" - "- 1: A severe violation of a core instruction (e.g. ignoring a configured fallback " - "on a disallowed ask, a partial injection hijack), but not a full hard violation.\n" - "- 0: A hard violation — leaked system prompt, fully answered a clearly disallowed " - "topic, fully hijacked by injection — OR the row has no answer / an errored, empty, " - "or non-responsive output.\n" - "Reasoning: name the specific violated instruction and how the answer violated it. " - "If no stated instruction was violated, say so and score 5.\n" - "When scoring THIS metric, consider only these input blocks: Assistant's " - "configured instructions, Question, Generated answer.\n" - "Do not consider: Golden (reference) answer, Retrieved knowledge-base chunks." -) - -KNOWLEDGE_BASE_JUDGE_PROMPT: str = ( - 'Adherence to Knowledge Base (score key "knowledge_base"):\n' - "Judge ONLY whether the answer's claims are supported by the retrieved " - "knowledge-base chunks (groundedness / hallucination detection).\n" - "- Break the answer into its distinct factual claims. A claim is supported ONLY " - "if the specific fact it asserts is explicitly stated in the chunk text (a " - "verbatim or trivially reworded restatement). A claim that is merely plausible, " - "on the same topic as a chunk, or that requires an inferential leap the chunks do " - "not spell out is UNSUPPORTED, not supported.\n" - "- Identify the answer's load-bearing (material) claims — the ones that carry its " - "substance.\n" - "- Text that makes no factual claim (a greeting, a pleasantry, or a plain refusal " - "to answer) is EXCLUDED from the claim count.\n" - "- Judge groundedness ONLY, not correctness, completeness, or " - "instruction-following. A claim faithful to the chunks is grounded even if the " - "chunks are themselves wrong.\n" - "- Do NOT use any outside knowledge; the retrieved chunks are the ONLY allowed " - "source of support.\n\n" - "Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, " - "2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n" - "- 5: Every factual claim is explicitly supported by the retrieved chunks. Fully " - "grounded.\n" - "- 4: All load-bearing claims are grounded; only a minor, non-material claim lacks " - "explicit support.\n" - "- 3: One non-critical inferential leap beyond the chunks, but no load-bearing " - "claim is fabricated.\n" - "- 2: At least one load-bearing/material claim is unsupported or invented, even " - "though other claims are grounded.\n" - "- 1: Most claims are unsupported or invented; only incidental/minor claims are " - "grounded.\n" - "- 0: The answer is fabricated wholesale — no claim is grounded in the retrieved " - "chunks — OR the row has no answer / an errored, empty, or non-responsive output.\n" - "Reasoning: quote the exact chunk span supporting the main claim. When the score " - "is below 5, name the specific unsupported or invented claim.\n" - "When scoring THIS metric, consider only these input blocks: Generated answer, " - "Retrieved knowledge-base chunks.\n" - "Do not consider: Assistant's configured instructions, Question, Golden " - "(reference) answer." -) - -JUDGE_OUTPUT_INSTRUCTION: str = ( - "Respond with ONLY a single JSON object mapping each metric key to its result, of " - 'the form {{"": {{"score": , "reasoning": ' - '""}}}}. Scores MUST be integers 0-5. Every ' - '"reasoning" string MUST be written in English. Include exactly these metric keys: ' - "{metric_keys}. Output nothing else." -) - class TraceScore(TypedDict): """A score attached to a trace.""" diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index 2afedb194..a3e64a9f4 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -1,5 +1,4 @@ -"""Human-readable AI summary of a v2 judge run's per-question diagnostics. -""" +"""Human-readable AI summary of a v2 judge run's per-question diagnostics.""" import json import logging @@ -95,8 +94,6 @@ def _format_traces_for_prompt( Traces are handed over ungrouped: `question_id` is the 1-based dataset row number, which is what the system prompt tells the model to key on. """ - # ponytail: every trace sent whole; ~250 traces (50 questions x dup 5) overflows - # context and degrades to None. Sample or group per question_id if runs get bigger. payload = [ { "question_id": trace["question_id"], diff --git a/backend/app/models/evaluation_iteration.py b/backend/app/models/evaluation_iteration.py index 1917867c1..1c7dd7419 100644 --- a/backend/app/models/evaluation_iteration.py +++ b/backend/app/models/evaluation_iteration.py @@ -118,6 +118,12 @@ class EvaluationIterationRun(SQLModel, table=True): ondelete="CASCADE", sa_column_kwargs={"comment": "Reference to the project"}, ) + last_dispatched_at: datetime | None = Field( + default=None, + sa_column_kwargs={ + "comment": "When a graph step was last dispatched (kickoff or cron resume); cron skips rows stamped inside the cooldown" + }, + ) inserted_at: datetime = Field( default_factory=now, nullable=False, @@ -138,6 +144,7 @@ class EvaluationIterationRunUpdate(SQLModel): status: EvaluationIterationStatusEnum | None = None stop_reason: str | None = None error_message: str | None = None + last_dispatched_at: datetime | None = None class EvaluationIterationCreateRequest(SQLModel): diff --git a/backend/app/services/assessment/service.py b/backend/app/services/assessment/service.py index 6a7724d82..fb2286b29 100644 --- a/backend/app/services/assessment/service.py +++ b/backend/app/services/assessment/service.py @@ -11,8 +11,8 @@ from app.crud.assessment import ( create_assessment, create_assessment_run, - get_submission_by_id, get_assessment_runs_for_assessment, + get_submission_by_id, recompute_assessment_status, ) from app.crud.assessment.core import _read_exec, _write_exec diff --git a/backend/app/services/evaluations/evaluation.py b/backend/app/services/evaluations/evaluation.py index a4dcb6ccf..dee3f01dc 100644 --- a/backend/app/services/evaluations/evaluation.py +++ b/backend/app/services/evaluations/evaluation.py @@ -14,7 +14,6 @@ from app.core.storage_utils import load_json_from_object_store from app.crud.evaluations import ( EvaluationScore, - create_evaluation_run, fetch_trace_scores_from_langfuse, get_dataset_by_id, get_evaluation_run_by_id, @@ -23,6 +22,9 @@ save_score, sort_traces_by_question_id, ) +from app.crud.evaluations import ( + create_evaluation_run as _create_evaluation_run, +) from app.crud.evaluations.core import update_evaluation_run from app.crud.evaluations.merge import apply_cosine_breakdown from app.crud.evaluations.score import CategoryMetrics, TraceData @@ -66,7 +68,7 @@ def _is_run_name_conflict(error: IntegrityError) -> bool: return _RUN_NAME_UNIQUE_CONSTRAINT in str(error.orig or error) -def create_evaluation_run_or_409( +def create_evaluation_run( *, session: Session, run_name: str, @@ -77,16 +79,25 @@ def create_evaluation_run_or_409( organization_id: int, project_id: int, run_mode: RunModeEnum = RunModeEnum.BATCH, + is_judge_run: bool = False, + callback_url: str | None = None, + duplication_factor: int | None = None, + total_items: int | None = None, + status: str = "pending", log_context: str, ) -> EvaluationRun: """Create an EvaluationRun, translating a duplicate-run_name collision into 409. The (organization_id, project_id, run_name) unique constraint guards against - double-click / client-retry races; on collision we roll back and raise a 409 - instead of leaking the IntegrityError. + double-click / client-retry races; on collision we roll back and raise 409 if + the run_name already exists for this organization and project. + + The v2-only params (is_judge_run, callback_url, duplication_factor, total_items, + status) are passed straight through and default to the v1 no-op values, so + existing batch callers need no change. """ try: - return create_evaluation_run( + return _create_evaluation_run( session=session, run_name=run_name, dataset_name=dataset_name, @@ -96,6 +107,11 @@ def create_evaluation_run_or_409( organization_id=organization_id, project_id=project_id, run_mode=run_mode, + is_judge_run=is_judge_run, + callback_url=callback_url, + duplication_factor=duplication_factor, + total_items=total_items, + status=status, ) except IntegrityError as exc: session.rollback() @@ -308,7 +324,7 @@ def validate_and_start_batch_evaluation( ) # Step 3: Create EvaluationRun record with config references - eval_run = create_evaluation_run_or_409( + eval_run = create_evaluation_run( session=session, run_name=experiment_name, dataset_name=dataset.name, diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index 552749fe7..fd5646cb9 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -30,6 +30,7 @@ from app.crud.evaluations.core import update_evaluation_run from app.crud.evaluations.dataset import ( DATASET_META_DUPLICATION_FACTOR, + DATASET_META_ORIGINAL_ITEMS, download_csv_from_object_store, ) from app.crud.evaluations.fast import run_response_chunk @@ -41,7 +42,7 @@ RunModeEnum, ) from app.models.llm.request import TextLLMParams -from app.services.evaluations.evaluation import create_evaluation_run_or_409 +from app.services.evaluations.evaluation import create_evaluation_run from app.services.evaluations.validators import parse_csv_items from app.services.llm.providers import LLMProvider from app.utils import get_langfuse_client, get_openai_client @@ -82,8 +83,7 @@ def load_run_dataset_items( if dataset.langfuse_dataset_id: if langfuse is None: raise ValueError( - f"Dataset {dataset.id} is Langfuse-backed but no Langfuse client " - "is available to load its items" + f"Dataset {dataset.id} is Langfuse-backed but no Langfuse client available" ) return fetch_dataset_items(langfuse=langfuse, dataset_name=dataset.name) @@ -124,12 +124,8 @@ def _load_items_from_object_store( items: list[dict[str, Any]] = [] for row_idx, item in enumerate(original_items): for dup_idx in range(duplication_factor): - item_metadata: dict[str, Any] = { - # 1-based, shared across a row's duplicates so the Q.ID column - # groups by original question (mirrors the Langfuse upload path). - "question_id": row_idx - + 1, - } + # 1-based question_id shared across duplicates to group by original question. + item_metadata: dict[str, Any] = {"question_id": row_idx + 1} if "category" in item: item_metadata["category"] = item["category"] or DEFAULT_CATEGORY items.append( @@ -169,7 +165,6 @@ def validate_fast_evaluation_inputs( this run only and is supported for S3-only datasets exclusively; it is rejected with 422 for Langfuse-backed datasets (whose items come pre-duplicated). """ - # 1. Dataset must exist (Langfuse id required for v1 runs only; see below). dataset = get_dataset_by_id( session=session, dataset_id=dataset_id, @@ -184,8 +179,7 @@ def validate_fast_evaluation_inputs( "organization/project" ), ) - # v1 runs still require a Langfuse-backed dataset. v2 judged runs are - # Langfuse-free and load items from S3, so a NULL langfuse id is allowed there. + # v1 runs require Langfuse-backed dataset; v2 judged runs load from S3. if not dataset.langfuse_dataset_id and not is_judge_run: raise HTTPException( status_code=400, @@ -204,7 +198,6 @@ def validate_fast_evaluation_inputs( ), ) - # 2. Config must resolve and be a text OpenAI config. config_blob, error = resolve_evaluation_config( session=session, config_id=config_id, @@ -227,8 +220,9 @@ def validate_fast_evaluation_inputs( detail=ERR_CONFIG_TYPE_UNSUPPORTED, ) - # 3. Dataset must be small enough for fast eval. - original_items_count = (dataset.dataset_metadata or {}).get("original_items_count") + original_items_count = (dataset.dataset_metadata or {}).get( + DATASET_META_ORIGINAL_ITEMS + ) if original_items_count is None: raise HTTPException( status_code=422, @@ -265,29 +259,15 @@ def validate_and_start_fast_evaluation( callback_url: str | None = None, duplication_factor: int | None = None, ) -> EvaluationRun: - """Validate + create + dispatch a fast evaluation run. - - Validation is `validate_fast_evaluation_inputs` (dataset/config checks); on - top of that, (organization_id, project_id, run_name) must be unique — enforced - by the DB constraint, a collision is translated to 409 by the shared helper. - - On success the function creates the EvaluationRun row with - `run_mode="fast"`, `status="processing"`, and enqueues the orchestrator - task. The caller (route) returns the row immediately. - - `is_judge_run` is the v2 native-judge marker, persisted on the run before - dispatch so the aggregate (which only knows eval_run_id) reads it at judge - time. It defaults to the v1 behavior — no judging, Langfuse sync as today — - so the v1 call path is unchanged. Judging is system-config only: the judge - always uses the fallback model + built-in prompt, so there is no per-run config. + """Validate, create, and dispatch a fast evaluation run. - `callback_url` is an optional HTTPS webhook (v2 only) persisted on the run so - the terminal-transition hook can POST the result. v1 callers pass nothing, so - it stays NULL and no webhook fires. + Creates EvaluationRun with total_items derived from dataset metadata and + v2 markers (is_judge_run, callback_url, duplication_factor) in one INSERT, + then enqueues chunk tasks. Returns immediately; chunks run after response. - `duplication_factor`, when provided, overrides the dataset's stored factor for - this run only and is supported for S3-only datasets exclusively; it is rejected - with 422 for Langfuse-backed datasets (whose items come pre-duplicated). + - `is_judge_run`: v2 marker; aggregate reads it to pick judge path. + - `callback_url`: optional HTTPS webhook (v2 only) for terminal transition. + - `duplication_factor`: overrides dataset's stored factor (S3-only datasets). """ logger.info( f"[validate_and_start_fast_evaluation] Starting fast eval | " @@ -306,8 +286,20 @@ def validate_and_start_fast_evaluation( duplication_factor=duplication_factor, ) - # Create the run; the shared helper translates a duplicate run_name into 409. - eval_run = create_evaluation_run_or_409( + # Fan-out count = original items × factor (from metadata, no load needed). + metadata = dataset.dataset_metadata or {} + original_items = int(metadata.get(DATASET_META_ORIGINAL_ITEMS, 0)) + stored_factor = int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1)) + effective_factor = ( + duplication_factor if duplication_factor is not None else stored_factor + ) + total_items = original_items * max(1, effective_factor) + if total_items == 0: + raise ValueError(f"Dataset '{dataset.name}' has no items") + n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) + + # v2 markers (is_judge_run, callback_url, duplication_factor) land in one INSERT. + eval_run = create_evaluation_run( session=session, run_name=run_name, dataset_name=dataset.name, @@ -317,60 +309,16 @@ def validate_and_start_fast_evaluation( organization_id=organization_id, project_id=project_id, run_mode=RunModeEnum.FAST, + is_judge_run=is_judge_run, + callback_url=callback_url, + duplication_factor=duplication_factor, + status="processing", + total_items=total_items, log_context="validate_and_start_fast_evaluation", ) - # Persist the judge marker + callback_url + duplication_factor before dispatch: - # the aggregate (which only knows eval_run_id) reads is_judge_run at judge time - # and duplication_factor for the ai_summary math, the terminal hook reads - # callback_url, and the chunk re-load reads duplication_factor so its slice count - # matches the fan-out sizing below. - if is_judge_run or callback_url or duplication_factor is not None: - eval_run = update_evaluation_run( - session=session, - eval_run=eval_run, - update=EvaluationRunUpdate( - is_judge_run=is_judge_run or None, - callback_url=callback_url, - duplication_factor=duplication_factor, - ), - ) - - # Fetch the dataset items now to size the fan-out: ceil(total / chunk_size) - # parallel chunk tasks drain the responses stage across workers. Any failure - # here marks the run failed so it never lingers in `processing`. + # Dispatch chunk tasks; on failure mark run failed so it doesn't linger. try: - # Only Langfuse-backed (v1) datasets need a client; a v2 dataset loads from - # S3, so we skip the client rather than require Langfuse for a native run. - langfuse_client = ( - get_langfuse_client( - session=session, - org_id=organization_id, - project_id=project_id, - ) - if dataset.langfuse_dataset_id - else None - ) - dataset_items = load_run_dataset_items( - session=session, - dataset=dataset, - langfuse=langfuse_client, - duplication_factor=duplication_factor, - ) - total_items = len(dataset_items) - if total_items == 0: - raise ValueError(f"Dataset '{dataset.name}' returned no items") - n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) - - # total_items isn't on EvaluationRunUpdate; set it directly, then flip to - # processing so the GET endpoint reflects state before dispatch. - eval_run.total_items = total_items - eval_run = update_evaluation_run( - session=session, - eval_run=eval_run, - update=EvaluationRunUpdate(status="processing"), - ) - for chunk_index in range(n_chunks): start_fast_evaluation_chunk( eval_run_id=eval_run.id, @@ -499,11 +447,6 @@ def execute_fast_evaluation_chunk(*, eval_run_id: int, chunk_index: int) -> None start = chunk_index * settings.EVAL_FAST_CHUNK_SIZE items_slice = dataset_items[start : start + settings.EVAL_FAST_CHUNK_SIZE] - log_prefix = ( - f"[org={eval_run.organization_id}]" - f"[project={eval_run.project_id}]" - f"[eval={eval_run.id}]" - ) run_response_chunk( session=session, openai_client=openai_client, @@ -511,12 +454,10 @@ def execute_fast_evaluation_chunk(*, eval_run_id: int, chunk_index: int) -> None config=text_params, dataset_items_slice=items_slice, chunk_index=chunk_index, - log_prefix=log_prefix, ) except Exception as exc: - # No per-chunk failed marker: the cron healer re-enqueues any index - # without a raw_output_url, so a failed chunk is already retried. + # Cron healer re-enqueues chunks without raw_output_url, so no marker needed. logger.error( f"[execute_fast_evaluation_chunk] Chunk failed | " f"eval_run_id={eval_run_id} | chunk_index={chunk_index} | error={exc}", @@ -526,12 +467,9 @@ def execute_fast_evaluation_chunk(*, eval_run_id: int, chunk_index: int) -> None def execute_fast_evaluation_aggregate(*, eval_run_id: int) -> None: - """Worker entry point for the fan-in aggregate. + """Worker entry point for aggregate. Merges chunks and runs embeddings/scoring. - Called from `run_evaluation_fast_aggregate`. Merges the chunks and runs - embeddings + scoring + completion via `run_fast_evaluation`. Owns the run's - completed/failed transition, so on terminal failure it marks the run failed - and re-raises. + Owns run's completed/failed transition; on failure marks run failed and re-raises. """ with Session(engine) as session: eval_run = _get_fast_run(session=session, eval_run_id=eval_run_id) @@ -543,16 +481,12 @@ def execute_fast_evaluation_aggregate(*, eval_run_id: int) -> None: return try: - # No config resolve here: re-resolving a config edited/pruned since - # dispatch would fail a run whose chunks already succeeded. Aggregate - # needs only the two clients. openai_client = get_openai_client( session=session, org_id=eval_run.organization_id, project_id=eval_run.project_id, ) - # v2 judged runs are fully Kaapi-native: no Langfuse client, so no - # traces are created and no scores are synced. v1 keeps syncing. + # v2 judged runs: Kaapi-native, no Langfuse; v1: sync to Langfuse. langfuse_client = ( None if eval_run.is_judge_run diff --git a/backend/app/services/evaluations/iteration_checkpointer.py b/backend/app/services/evaluations/iteration_checkpointer.py new file mode 100644 index 000000000..cb183fa34 --- /dev/null +++ b/backend/app/services/evaluations/iteration_checkpointer.py @@ -0,0 +1,52 @@ +"""Postgres checkpointer for the eval-iterate-improve LangGraph loop. + +`langgraph-checkpoint-postgres` connects via psycopg (v3) directly rather than +through the app's SQLAlchemy engine, so it owns its own small pool and its own +tables (`checkpoints`, `checkpoint_blobs`, `checkpoint_writes`) — not Alembic +managed. +""" + +import logging +from functools import lru_cache + +from langgraph.checkpoint.postgres import PostgresSaver +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from app.core.config import settings + +logger = logging.getLogger(__name__) + +_POOL_MIN_SIZE = 1 +_POOL_MAX_SIZE = 5 + + +def _psycopg_conn_string() -> str: + """Derive a plain psycopg conninfo string from the app's SQLAlchemy DSN. + + The app's DSN already targets the psycopg driver (`postgresql+psycopg://`), + so stripping the SQLAlchemy dialect qualifier is the only adaptation needed. + """ + return str(settings.SQLALCHEMY_DATABASE_URI).replace( + "postgresql+psycopg://", "postgresql://", 1 + ) + + +@lru_cache(maxsize=1) +def get_evaluation_iteration_checkpointer() -> PostgresSaver: + """Module-level singleton checkpointer, backed by its own connection pool. + + `.setup()` is `CREATE TABLE IF NOT EXISTS`-style, so it is safe to run on + first access rather than behind a separate startup hook. + """ + pool = ConnectionPool( + conninfo=_psycopg_conn_string(), + min_size=_POOL_MIN_SIZE, + max_size=_POOL_MAX_SIZE, + open=True, + kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row}, + ) + checkpointer = PostgresSaver(pool) + checkpointer.setup() + logger.info("[get_evaluation_iteration_checkpointer] Checkpointer ready") + return checkpointer diff --git a/backend/app/services/evaluations/iteration_graph.py b/backend/app/services/evaluations/iteration_graph.py index b72dce73e..0f6112009 100644 --- a/backend/app/services/evaluations/iteration_graph.py +++ b/backend/app/services/evaluations/iteration_graph.py @@ -16,8 +16,7 @@ """ import logging -from functools import lru_cache -from typing import Any, TypedDict +from typing import Any from uuid import UUID from celery.exceptions import SoftTimeLimitExceeded @@ -25,8 +24,6 @@ from langgraph.graph import END, START, StateGraph from langgraph.graph.state import CompiledStateGraph from langgraph.types import Command, interrupt -from psycopg.rows import dict_row -from psycopg_pool import ConnectionPool from sqlmodel import Session from app.core.config import settings @@ -38,19 +35,24 @@ ) from app.crud.jobs import JobCrud from app.models.evaluation_iteration import ( - EvaluationIterationReportPublic, - EvaluationIterationRoundPublic, EvaluationIterationRunUpdate, EvaluationIterationStatusEnum, ) from app.models.job import JobStatus from app.services.evaluations.fast import validate_and_start_fast_evaluation from app.services.evaluations.iteration import ( - STOP_REASON_CEILING_REACHED, STOP_REASON_MAX_ROUNDS_REACHED, STOP_REASON_ROUND_FAILED, compute_round_scores, ) +from app.services.evaluations.iteration_checkpointer import ( + get_evaluation_iteration_checkpointer, +) +from app.services.evaluations.iteration_state import ( + EvaluationIterationState, + advance_round_state, + build_iteration_report, +) from app.services.evaluations.prompt_improvement import start_prompt_improvement_job from app.utils import APIResponse, get_webhook_secret, send_callback @@ -59,63 +61,6 @@ _JOB_WAITING_STATUSES = {JobStatus.PENDING, JobStatus.PROCESSING} -class EvaluationIterationState(TypedDict): - iteration_run_id: int - dataset_id: int - experiment_name: str - config_id: str - config_version: int - round_number: int - max_rounds: int - current_eval_run_id: int | None - current_improvement_job_id: str | None - history: list[dict[str, Any]] - best_round_number: int | None - best_config_version: int | None - best_stop_score: float | None - consecutive_low_delta_rounds: int - stop_reason: str | None - error_message: str | None - organization_id: int - project_id: int - callback_url: str - - -def _psycopg_conn_string() -> str: - """Derive a plain psycopg conninfo string from the app's SQLAlchemy DSN. - - langgraph-checkpoint-postgres connects via psycopg (v3) directly rather than - through the SQLAlchemy engine, but the app's DSN already targets the psycopg - driver (`postgresql+psycopg://`) — stripping the SQLAlchemy dialect qualifier - is the only adaptation needed. - """ - return str(settings.SQLALCHEMY_DATABASE_URI).replace( - "postgresql+psycopg://", "postgresql://", 1 - ) - - -@lru_cache(maxsize=1) -def get_evaluation_iteration_checkpointer() -> PostgresSaver: - """Module-level singleton checkpointer, backed by its own small connection pool. - - `.setup()` creates the checkpoint tables (`checkpoints`, `checkpoint_blobs`, - `checkpoint_writes`) — schema owned by the library, not Alembic. It's a - `CREATE TABLE IF NOT EXISTS`-style call, so it's safe to run on every first - access rather than gating it behind a separate startup hook. - """ - pool = ConnectionPool( - conninfo=_psycopg_conn_string(), - min_size=1, - max_size=5, - open=True, - kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row}, - ) - checkpointer = PostgresSaver(pool) - checkpointer.setup() - logger.info("[get_evaluation_iteration_checkpointer] Checkpointer ready") - return checkpointer - - def start_eval_node(state: EvaluationIterationState) -> dict[str, Any]: """Kick off this round's judged fast-eval run.""" run_name = ( @@ -133,11 +78,6 @@ def start_eval_node(state: EvaluationIterationState) -> dict[str, Any]: project_id=state["project_id"], is_judge_run=True, ) - logger.info( - f"[start_eval_node] Round eval started | " - f"iteration_run_id={state['iteration_run_id']} | " - f"round_number={state['round_number']} | eval_run_id={eval_run.id}" - ) return {"current_eval_run_id": eval_run.id} @@ -194,52 +134,11 @@ def wait_eval_node(state: EvaluationIterationState) -> dict[str, Any]: eval_run_id = eval_run.id stop_score, kb_score = scores - round_entry = { - "round_number": state["round_number"], - "eval_run_id": eval_run_id, - "config_version": state["config_version"], - "stop_score": stop_score, - "kb_score": kb_score, - } - history = [*state["history"], round_entry] - - best_stop_score = state.get("best_stop_score") - best_round_number = state.get("best_round_number") - best_config_version = state.get("best_config_version") - if best_stop_score is None or stop_score > best_stop_score: - best_stop_score = stop_score - best_round_number = state["round_number"] - best_config_version = state["config_version"] - - previous_scores = [entry["stop_score"] for entry in state["history"]] - consecutive_low_delta_rounds = 0 - if previous_scores: - delta = stop_score - previous_scores[-1] - if delta < settings.EVAL_ITERATION_CEILING_DELTA_THRESHOLD: - consecutive_low_delta_rounds = ( - state.get("consecutive_low_delta_rounds", 0) + 1 - ) - - update: dict[str, Any] = { - "history": history, - "best_stop_score": best_stop_score, - "best_round_number": best_round_number, - "best_config_version": best_config_version, - "consecutive_low_delta_rounds": consecutive_low_delta_rounds, - } - if ( - consecutive_low_delta_rounds - >= settings.EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS - ): - update["stop_reason"] = STOP_REASON_CEILING_REACHED - elif state["round_number"] >= state["max_rounds"]: - update["stop_reason"] = STOP_REASON_MAX_ROUNDS_REACHED - - logger.info( - f"[wait_eval_node] Round scored | iteration_run_id={state['iteration_run_id']} | " - f"round_number={state['round_number']} | stop_score={stop_score} | " - f"consecutive_low_delta_rounds={consecutive_low_delta_rounds} | " - f"stop_reason={update.get('stop_reason')}" + update = advance_round_state( + state=state, + eval_run_id=eval_run_id, + stop_score=stop_score, + kb_score=kb_score, ) return update @@ -269,10 +168,6 @@ def start_improve_node(state: EvaluationIterationState) -> dict[str, Any]: callback_url="", require_judge_run=True, ) - logger.info( - f"[start_improve_node] Prompt improvement job started | " - f"iteration_run_id={state['iteration_run_id']} | job_id={job.id}" - ) return {"current_improvement_job_id": str(job.id)} @@ -321,11 +216,6 @@ def wait_improve_node(state: EvaluationIterationState) -> dict[str, Any]: "error_message": "Prompt improvement job succeeded without a version in meta", } - logger.info( - f"[wait_improve_node] Prompt improved | " - f"iteration_run_id={state['iteration_run_id']} | " - f"next_round_number={state['round_number'] + 1} | config_version={new_version}" - ) return { "round_number": state["round_number"] + 1, "config_version": new_version, @@ -333,24 +223,6 @@ def wait_improve_node(state: EvaluationIterationState) -> dict[str, Any]: } -def _build_iteration_report( - state: EvaluationIterationState, status: EvaluationIterationStatusEnum -) -> EvaluationIterationReportPublic: - history = [EvaluationIterationRoundPublic(**entry) for entry in state["history"]] - best_round = next( - (r for r in history if r.round_number == state.get("best_round_number")), - None, - ) - return EvaluationIterationReportPublic( - iteration_run_id=state["iteration_run_id"], - status=status, - stop_reason=state.get("stop_reason"), - best_round=best_round, - history=history, - error_message=state.get("error_message"), - ) - - def finalize_node(state: EvaluationIterationState) -> dict[str, Any]: """Terminal node: persist the thin row and POST the report to callback_url.""" stop_reason = state.get("stop_reason") @@ -380,6 +252,10 @@ def finalize_node(state: EvaluationIterationState) -> dict[str, Any]: f"iteration_run_id={state['iteration_run_id']}" ) return {} + if iteration_run.status != EvaluationIterationStatusEnum.PROCESSING: + # Reaper (or another terminal writer) got here first; its callback + # already went out, so a second one would contradict it. + return {} update_evaluation_iteration_run( session=session, @@ -391,7 +267,7 @@ def finalize_node(state: EvaluationIterationState) -> dict[str, Any]: ), ) - report = _build_iteration_report(state, status) + report = build_iteration_report(state, status) error_message = state.get("error_message") envelope = ( APIResponse.failure_response( @@ -405,11 +281,6 @@ def finalize_node(state: EvaluationIterationState) -> dict[str, Any]: state["callback_url"], envelope.model_dump(), webhook_secret=webhook_secret ) - logger.info( - f"[finalize_node] Loop finished | iteration_run_id={state['iteration_run_id']} | " - f"status={status.value} | stop_reason={stop_reason} | " - f"rounds={len(state['history'])}" - ) return {} @@ -480,10 +351,14 @@ def _build_initial_state( ) -def _mark_iteration_run_failed( +def mark_iteration_run_failed( *, iteration_run_id: int, organization_id: int, project_id: int, error_message: str ) -> None: - """Fail a loop from a fresh session so a killed task leaves no dangling row.""" + """Fail a loop from a fresh session so a killed task leaves no dangling row. + + Public because the cron zombie reaper calls it too — a loop the graph never + got to fail itself still owes its caller the failure callback. + """ try: with Session(engine) as session: iteration_run = get_evaluation_iteration_run_by_id( @@ -512,13 +387,9 @@ def _mark_iteration_run_failed( send_callback( callback_url, envelope.model_dump(), webhook_secret=webhook_secret ) - - logger.info( - f"[_mark_iteration_run_failed] iteration_run_id={iteration_run_id} marked failed" - ) except Exception: logger.error( - f"[_mark_iteration_run_failed] Could not mark iteration_run_id=" + f"[mark_iteration_run_failed] Could not mark iteration_run_id=" f"{iteration_run_id} failed", exc_info=True, ) @@ -572,10 +443,6 @@ def execute_evaluation_iteration_graph_step( PROCESSING) or reaches `finalize_node` (which already updated the thin row and sent the callback before this returns). """ - logger.info( - f"[execute_evaluation_iteration_graph_step] Starting | " - f"iteration_run_id={iteration_run_id} | resume={resume}" - ) try: _run_graph_step( iteration_run_id=iteration_run_id, @@ -590,7 +457,7 @@ def execute_evaluation_iteration_graph_step( f"[execute_evaluation_iteration_graph_step] Soft time limit | " f"iteration_run_id={iteration_run_id}" ) - _mark_iteration_run_failed( + mark_iteration_run_failed( iteration_run_id=iteration_run_id, organization_id=organization_id, project_id=project_id, @@ -603,7 +470,7 @@ def execute_evaluation_iteration_graph_step( f"iteration_run_id={iteration_run_id}", exc_info=True, ) - _mark_iteration_run_failed( + mark_iteration_run_failed( iteration_run_id=iteration_run_id, organization_id=organization_id, project_id=project_id, diff --git a/backend/app/services/evaluations/iteration_state.py b/backend/app/services/evaluations/iteration_state.py new file mode 100644 index 000000000..f748f8a90 --- /dev/null +++ b/backend/app/services/evaluations/iteration_state.py @@ -0,0 +1,126 @@ +"""Graph state and round bookkeeping for the eval-iterate-improve loop. + +The state lives in the LangGraph checkpoint, not on `evaluation_iteration_run`, +so a pause at `interrupt()` can span many cron ticks. Everything here is pure — +no DB, no HTTP — so the loop's stop/continue arithmetic can be read and tested +on its own. +""" + +from typing import Any, TypedDict + +from app.core.config import settings +from app.models.evaluation_iteration import ( + EvaluationIterationReportPublic, + EvaluationIterationRoundPublic, + EvaluationIterationStatusEnum, +) +from app.services.evaluations.iteration import ( + STOP_REASON_CEILING_REACHED, + STOP_REASON_MAX_ROUNDS_REACHED, +) + + +class EvaluationIterationState(TypedDict): + iteration_run_id: int + dataset_id: int + experiment_name: str + config_id: str + config_version: int + round_number: int + max_rounds: int + current_eval_run_id: int | None + current_improvement_job_id: str | None + history: list[dict[str, Any]] + best_round_number: int | None + best_config_version: int | None + best_stop_score: float | None + consecutive_low_delta_rounds: int + stop_reason: str | None + error_message: str | None + organization_id: int + project_id: int + callback_url: str + + +def _count_low_delta_rounds( + *, state: EvaluationIterationState, stop_score: float +) -> int: + """Consecutive rounds whose gain stayed under the ceiling threshold. + + Resets to 0 on the first round (no baseline) and on any round that clears the + threshold. + """ + previous_scores = [entry["stop_score"] for entry in state["history"]] + if not previous_scores: + return 0 + delta = stop_score - previous_scores[-1] + if delta >= settings.EVAL_ITERATION_CEILING_DELTA_THRESHOLD: + return 0 + return state.get("consecutive_low_delta_rounds", 0) + 1 + + +def advance_round_state( + *, + state: EvaluationIterationState, + eval_run_id: int, + stop_score: float, + kb_score: float | None, +) -> dict[str, Any]: + """Record this round's result and decide whether the loop should stop. + + Returns the state delta only. `kb_score` is recorded for visibility and never + gates stopping. + """ + round_entry = { + "round_number": state["round_number"], + "eval_run_id": eval_run_id, + "config_version": state["config_version"], + "stop_score": stop_score, + "kb_score": kb_score, + } + + best_stop_score = state.get("best_stop_score") + is_best = best_stop_score is None or stop_score > best_stop_score + consecutive_low_delta_rounds = _count_low_delta_rounds( + state=state, stop_score=stop_score + ) + + update: dict[str, Any] = { + "history": [*state["history"], round_entry], + "best_stop_score": stop_score if is_best else best_stop_score, + "best_round_number": ( + state["round_number"] if is_best else state.get("best_round_number") + ), + "best_config_version": ( + state["config_version"] if is_best else state.get("best_config_version") + ), + "consecutive_low_delta_rounds": consecutive_low_delta_rounds, + } + + if ( + consecutive_low_delta_rounds + >= settings.EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS + ): + update["stop_reason"] = STOP_REASON_CEILING_REACHED + elif state["round_number"] >= state["max_rounds"]: + update["stop_reason"] = STOP_REASON_MAX_ROUNDS_REACHED + + return update + + +def build_iteration_report( + state: EvaluationIterationState, status: EvaluationIterationStatusEnum +) -> EvaluationIterationReportPublic: + """The round-by-round report delivered to the caller's callback_url.""" + history = [EvaluationIterationRoundPublic(**entry) for entry in state["history"]] + best_round = next( + (r for r in history if r.round_number == state.get("best_round_number")), None + ) + return EvaluationIterationReportPublic( + iteration_run_id=state["iteration_run_id"], + status=status, + stop_reason=state.get("stop_reason"), + best_round=best_round, + history=history, + error_message=state.get("error_message"), + ) diff --git a/backend/app/tests/api/routes/test_evaluation_fast.py b/backend/app/tests/api/routes/test_evaluation_fast.py index 6e90983ec..3f2a8eee0 100644 --- a/backend/app/tests/api/routes/test_evaluation_fast.py +++ b/backend/app/tests/api/routes/test_evaluation_fast.py @@ -22,21 +22,23 @@ from app.core.util import now from app.crud.evaluations.cron import dispatch_fast_evaluation_barriers from app.crud.evaluations.fast import ( - CHUNK_CONFIG_INDEX, - CHUNK_CONFIG_RUN_ID, - JOB_TYPE_EMBEDDING_FAST, - JOB_TYPE_EVALUATION_FAST, - JOB_TYPE_EVALUATION_FAST_CHUNK, _create_response, - _get_chunk_job, - _is_failure_threshold_breached, _merge_response_chunks, _stage2_embeddings, _stage3_score_and_trace, - list_response_chunk_jobs, run_fast_evaluation, run_response_chunk, ) +from app.crud.evaluations.fast_chunks import ( + CHUNK_CONFIG_INDEX, + CHUNK_CONFIG_RUN_ID, + JOB_TYPE_EMBEDDING_FAST, + JOB_TYPE_EVALUATION_FAST, + JOB_TYPE_EVALUATION_FAST_CHUNK, + get_chunk_job, + list_response_chunk_jobs, +) +from app.crud.evaluations.fast_results import is_failure_threshold_breached from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob from app.models.evaluation import RunModeEnum @@ -73,18 +75,18 @@ def _seeded_random() -> Iterator[None]: class TestFailureThreshold: - """`_is_failure_threshold_breached` controls run-level fail-fast.""" + """`is_failure_threshold_breached` controls run-level fail-fast.""" def test_returns_false_when_total_is_zero(self) -> None: - assert _is_failure_threshold_breached(failed_rows=0, total_rows=0) is False + assert is_failure_threshold_breached(failed_rows=0, total_rows=0) is False def test_returns_true_above_threshold(self) -> None: # default EVAL_FAST_FAILURE_THRESHOLD = 0.5 - assert _is_failure_threshold_breached(failed_rows=6, total_rows=10) is True + assert is_failure_threshold_breached(failed_rows=6, total_rows=10) is True def test_returns_false_at_threshold(self) -> None: # 0.5 / 1.0 is NOT greater-than the threshold, so do not breach - assert _is_failure_threshold_breached(failed_rows=5, total_rows=10) is False + assert is_failure_threshold_breached(failed_rows=5, total_rows=10) is False class TestCallWithRetry: @@ -540,12 +542,11 @@ def test_writes_chunk_job_and_partial_unit( config=TextLLMParams(model="gpt-4o", instructions="x"), dataset_items_slice=items, chunk_index=0, - log_prefix="[t]", ) assert fake_openai.responses.create.call_count == 2 - job = _get_chunk_job(session=db, eval_run_id=eval_run.id, chunk_index=0) + job = get_chunk_job(session=db, eval_run_id=eval_run.id, chunk_index=0) assert job is not None assert job.job_type == JOB_TYPE_EVALUATION_FAST_CHUNK assert job.config[CHUNK_CONFIG_RUN_ID] == eval_run.id @@ -572,7 +573,6 @@ def test_idempotent_skips_openai_when_chunk_already_done( "config": TextLLMParams(model="gpt-4o", instructions="x"), "dataset_items_slice": items, "chunk_index": 0, - "log_prefix": "[t]", } with patch( "app.crud.evaluations.fast.map_kaapi_to_openai_params", @@ -731,7 +731,6 @@ def test_fr7_stage2_skips_when_embedding_batch_job_id_already_set( "failed": False, } ], - log_prefix="[t]", ) assert results == cached @@ -1110,20 +1109,14 @@ def test_validate_and_start_fans_out_exactly_ceil_chunks( monkeypatch, ): monkeypatch.setattr(settings, "EVAL_FAST_CHUNK_SIZE", 2) - dataset = _make_fast_eligible_dataset(db=db, user_api_key=user_api_key) + dataset = _make_fast_eligible_dataset( + db=db, user_api_key=user_api_key, original_items_count=5 + ) config = _make_text_openai_config(db, user_api_key.project_id) - items = [_dataset_item(f"item-{i}") for i in range(5)] - with ( - patch("app.services.evaluations.fast.get_langfuse_client"), - patch( - "app.services.evaluations.fast.fetch_dataset_items", - return_value=items, - ), - patch( - "app.services.evaluations.fast.start_fast_evaluation_chunk" - ) as mock_start, - ): + with patch( + "app.services.evaluations.fast.start_fast_evaluation_chunk" + ) as mock_start: run = validate_and_start_fast_evaluation( session=db, dataset_id=dataset.id, @@ -1135,6 +1128,7 @@ def test_validate_and_start_fans_out_exactly_ceil_chunks( ) # ceil(5 / 2) = 3 chunks, indices 0..2, no gaps or dupes. + # total_items now computed from dataset metadata, not by loading items. assert mock_start.call_count == 3 dispatched = {c.kwargs["chunk_index"] for c in mock_start.call_args_list} assert dispatched == {0, 1, 2} @@ -1198,24 +1192,23 @@ def _capture(*, dataset_items_slice, **_): class TestValidateAndStartFailure: - def test_dataset_fetch_error_marks_run_failed_and_raises_500( + def test_chunk_dispatch_error_marks_run_failed_and_raises_500( self, db: Session, user_api_key: TestAuthContext, ): + """Chunk dispatch failure (e.g., broker down) marks run failed and raises 500. + + total_items is now computed from dataset metadata in the trigger, so item-fetch + errors no longer occur here; the only failure scenario is a dispatch error. + """ dataset = _make_fast_eligible_dataset(db=db, user_api_key=user_api_key) config = _make_text_openai_config(db, user_api_key.project_id) - run_name = f"fetch-fail-{random_lower_string()}" + run_name = f"dispatch-fail-{random_lower_string()}" - with ( - patch("app.services.evaluations.fast.get_langfuse_client"), - patch( - "app.services.evaluations.fast.fetch_dataset_items", - side_effect=RuntimeError("langfuse down"), - ), - patch( - "app.services.evaluations.fast.start_fast_evaluation_chunk" - ) as mock_start, + with patch( + "app.services.evaluations.fast.start_fast_evaluation_chunk", + side_effect=RuntimeError("broker down"), ): with pytest.raises(HTTPException) as exc: validate_and_start_fast_evaluation( @@ -1229,7 +1222,6 @@ def test_dataset_fetch_error_marks_run_failed_and_raises_500( ) assert exc.value.status_code == 500 - mock_start.assert_not_called() failed = db.exec( select(EvaluationRun).where(EvaluationRun.run_name == run_name) ).first() diff --git a/backend/app/tests/api/routes/test_evaluation_iteration_v2.py b/backend/app/tests/api/routes/test_evaluation_iteration_v2.py index 95d178279..70a8b7af3 100644 --- a/backend/app/tests/api/routes/test_evaluation_iteration_v2.py +++ b/backend/app/tests/api/routes/test_evaluation_iteration_v2.py @@ -21,7 +21,7 @@ ) ITERATIONS_URL = f"{settings.API_V2_STR}/evaluations/iterations" -_ROUTE_VALIDATE = "app.api.routes.evaluations.iteration_v2.validate_callback_url" +_ROUTE_VALIDATE = "app.api.routes.evaluations.v2.iteration.validate_callback_url" def _make_dataset(*, db: Session, user_api_key: TestAuthContext) -> EvaluationDataset: diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index 47963eae4..c83c5d9d4 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -51,7 +51,7 @@ _SERVICE = "app.services.evaluations.prompt_improvement" _ROUTE_VALIDATE = ( - "app.api.routes.evaluations.prompt_improvement_v2.validate_callback_url" + "app.api.routes.evaluations.v2.prompt_improvement.validate_callback_url" ) POST_URL = f"{settings.API_V2_STR}/evaluations/{{evaluation_id}}/improve-prompt" diff --git a/backend/app/tests/assessment/test_api_batch.py b/backend/app/tests/assessment/test_api_batch.py index 1b65eb9be..2f193b14c 100644 --- a/backend/app/tests/assessment/test_api_batch.py +++ b/backend/app/tests/assessment/test_api_batch.py @@ -7,8 +7,8 @@ import json from contextlib import contextmanager -from uuid import uuid4 from unittest.mock import MagicMock, patch +from uuid import uuid4 import pytest from fastapi import HTTPException diff --git a/backend/app/tests/assessment/test_cron.py b/backend/app/tests/assessment/test_cron.py index 6eab9496c..c5c818524 100644 --- a/backend/app/tests/assessment/test_cron.py +++ b/backend/app/tests/assessment/test_cron.py @@ -1,8 +1,8 @@ """Tests for assessment/cron.py helper functions.""" from datetime import datetime -from uuid import uuid4 from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 import pytest @@ -21,10 +21,7 @@ from app.models.config.assessment_blob import AssessmentConfigBlob from app.models.config.config import ConfigTag from app.tests.utils.auth import get_user_test_auth_context -from app.tests.utils.test_data import ( - create_test_config, - create_test_evaluation_dataset, -) +from app.tests.utils.test_data import create_test_config from app.tests.utils.utils import random_lower_string _ASSESSMENT_BLOB = AssessmentConfigBlob.model_validate( diff --git a/backend/app/tests/assessment/test_crud.py b/backend/app/tests/assessment/test_crud.py index 91d5b69af..a978957af 100644 --- a/backend/app/tests/assessment/test_crud.py +++ b/backend/app/tests/assessment/test_crud.py @@ -13,14 +13,14 @@ build_run_stats, compute_run_counts, create_assessment, - create_submission, create_assessment_run, + create_submission, derive_aggregate_error, derive_assessment_status, get_assessment_by_id, - get_submission_by_id, get_assessment_run_by_id, get_assessment_runs_for_assessment, + get_submission_by_id, list_assessment_runs, list_assessments, recompute_assessment_status, diff --git a/backend/app/tests/assessment/test_export.py b/backend/app/tests/assessment/test_export.py index 4e443d95a..55510f021 100644 --- a/backend/app/tests/assessment/test_export.py +++ b/backend/app/tests/assessment/test_export.py @@ -13,11 +13,11 @@ _drop_empty_columns, _expand_input_columns, _expand_output_columns, - _load_submission_rows_for_run, _load_l2_results_for_run, _load_parsed_results_for_batch_job, _load_parsed_results_for_run, _load_prefilter_results, + _load_submission_rows_for_run, _safe_filename_part, _stage_batch_job, build_json_export_rows, diff --git a/backend/app/tests/assessment/test_prefilter_batching.py b/backend/app/tests/assessment/test_prefilter_batching.py index 367f5f7b4..2fc931aa9 100644 --- a/backend/app/tests/assessment/test_prefilter_batching.py +++ b/backend/app/tests/assessment/test_prefilter_batching.py @@ -2,8 +2,8 @@ from contextlib import contextmanager from types import SimpleNamespace -from uuid import UUID from unittest.mock import MagicMock, patch +from uuid import UUID import pytest from celery.exceptions import SoftTimeLimitExceeded diff --git a/backend/app/tests/crud/evaluations/test_cron_iteration.py b/backend/app/tests/crud/evaluations/test_cron_iteration.py index 928da4f04..99e0960d9 100644 --- a/backend/app/tests/crud/evaluations/test_cron_iteration.py +++ b/backend/app/tests/crud/evaluations/test_cron_iteration.py @@ -5,10 +5,13 @@ Celery enqueue itself is the only external boundary — the DB is real. """ +from datetime import timedelta from unittest.mock import patch from sqlmodel import Session +from app.core.config import settings +from app.core.util import now from app.crud.evaluations.cron import dispatch_pending_evaluation_iteration_resumes from app.crud.evaluations.iteration import ( create_evaluation_iteration_run, @@ -51,13 +54,15 @@ def _make_iteration_run( organization_id=user_api_key.organization_id, project_id=user_api_key.project_id, ) - if status != EvaluationIterationStatusEnum.PROCESSING: - run = update_evaluation_iteration_run( - session=db, - iteration_run=run, - update=EvaluationIterationRunUpdate(status=status), - ) - return run + # Kickoff stamps the row inside the cooldown; age it so the tick sees it as due. + due = now() - timedelta( + minutes=settings.EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES + 1 + ) + return update_evaluation_iteration_run( + session=db, + iteration_run=run, + update=EvaluationIterationRunUpdate(status=status, last_dispatched_at=due), + ) class TestDispatchPendingEvaluationIterationResumes: @@ -98,4 +103,9 @@ def test_no_processing_rows_dispatches_nothing( summary = dispatch_pending_evaluation_iteration_resumes(session=db) mock_start.assert_not_called() - assert summary == {"total": 0, "resumes_dispatched": 0} + assert summary == { + "total": 0, + "resumes_dispatched": 0, + "in_flight_skipped": 0, + "reaped": 0, + } diff --git a/backend/app/tests/crud/evaluations/test_cron_iteration_guards.py b/backend/app/tests/crud/evaluations/test_cron_iteration_guards.py new file mode 100644 index 000000000..b04fca430 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_cron_iteration_guards.py @@ -0,0 +1,213 @@ +"""In-flight and zombie guards on the eval-iteration resume dispatcher. + +The tick used to fan a resume out to every PROCESSING row unconditionally, so a +graph step slower than the tick got a second one racing it on the same checkpoint +thread, and a row whose sub-job wedged collected resumes forever. + +`mark_iteration_run_failed` is patched rather than exercised: it opens its own +`Session(engine)`, which cannot see this test's uncommitted rows. +""" + +from datetime import datetime, timedelta +from unittest.mock import patch + +import pytest +from sqlmodel import Session + +from app.core.config import settings +from app.core.util import now +from app.crud.evaluations.cron import dispatch_pending_evaluation_iteration_resumes +from app.crud.evaluations.iteration import create_evaluation_iteration_run +from app.models.evaluation_iteration import EvaluationIterationRun +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.test_data import ( + create_test_config, + create_test_evaluation_dataset, +) + +_CALLBACK_URL = "https://example.com/callback" +_START = "app.celery.utils.start_evaluation_iteration_round" +_REAPER = "app.services.evaluations.iteration_graph.mark_iteration_run_failed" + + +def _make_run( + db: Session, + user_api_key: TestAuthContext, + experiment_name: str, +) -> EvaluationIterationRun: + dataset = create_test_evaluation_dataset( + db=db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + config = create_test_config( + db=db, project_id=user_api_key.project_id, use_kaapi_schema=True + ) + return create_evaluation_iteration_run( + session=db, + dataset_id=dataset.id, + experiment_name=experiment_name, + config_id=config.id, + initial_config_version=1, + callback_url=_CALLBACK_URL, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + +def _backdate( + db: Session, + run: EvaluationIterationRun, + *, + inserted_at: datetime | None = None, + last_dispatched_at: datetime | None = None, +) -> EvaluationIterationRun: + if inserted_at is not None: + run.inserted_at = inserted_at + if last_dispatched_at is not None: + run.last_dispatched_at = last_dispatched_at + db.add(run) + db.commit() + db.refresh(run) + return run + + +class TestInFlightGuard: + def test_a_fresh_kickoff_is_stamped_and_skipped_by_the_first_tick( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + """Kickoff enqueues the first step itself; the tick must not race it.""" + run = _make_run(db, user_api_key, "guard-fresh") + assert run.last_dispatched_at is not None + + with patch(_START) as mock_start: + summary = dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_start.assert_not_called() + assert summary == { + "total": 1, + "resumes_dispatched": 0, + "in_flight_skipped": 1, + "reaped": 0, + } + + def test_a_pre_migration_null_stamp_is_dispatched_and_stamped( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + """Rows older than migration 084 carry NULL; they get one immediate resume.""" + run = _make_run(db, user_api_key, "guard-legacy-null") + run.last_dispatched_at = None + db.add(run) + db.commit() + + with patch(_START) as mock_start: + summary = dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_start.assert_called_once() + assert summary["resumes_dispatched"] == 1 + db.refresh(run) + assert run.last_dispatched_at is not None + + def test_a_loop_dispatched_inside_the_cooldown_is_skipped( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + cooldown = settings.EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES + run = _make_run(db, user_api_key, "guard-in-flight") + stamp = now() - timedelta(minutes=cooldown // 2) + _backdate(db, run, last_dispatched_at=stamp) + + with patch(_START) as mock_start: + summary = dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_start.assert_not_called() + assert summary["total"] == 1 + assert summary["resumes_dispatched"] == 0 + db.refresh(run) + assert run.last_dispatched_at == stamp + + def test_a_loop_past_the_cooldown_is_dispatched_again( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + cooldown = settings.EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES + run = _make_run(db, user_api_key, "guard-cooled-down") + stale = now() - timedelta(minutes=cooldown + 1) + _backdate(db, run, last_dispatched_at=stale) + + with patch(_START) as mock_start: + summary = dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_start.assert_called_once() + assert summary["resumes_dispatched"] == 1 + db.refresh(run) + assert run.last_dispatched_at is not None + assert run.last_dispatched_at > stale + + def test_a_failed_enqueue_leaves_the_stamp_so_the_next_tick_retries( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + cooldown = settings.EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES + run = _make_run(db, user_api_key, "guard-enqueue-fails") + stale = now() - timedelta(minutes=cooldown + 1) + _backdate(db, run, last_dispatched_at=stale) + + with ( + patch(_START, side_effect=RuntimeError("broker down")), + pytest.raises(RuntimeError), + ): + dispatch_pending_evaluation_iteration_resumes(session=db) + + db.refresh(run) + assert run.last_dispatched_at == stale + + +class TestZombieReaper: + def test_a_loop_past_the_stall_threshold_is_reaped_not_resumed( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + run = _make_run(db, user_api_key, "guard-zombie") + _backdate( + db, + run, + inserted_at=now() + - timedelta(hours=settings.EVAL_ITERATION_STALL_THRESHOLD_HOURS + 1), + ) + + with patch(_START) as mock_start, patch(_REAPER) as mock_reaper: + summary = dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_start.assert_not_called() + mock_reaper.assert_called_once() + kwargs = mock_reaper.call_args.kwargs + assert kwargs["iteration_run_id"] == run.id + assert kwargs["organization_id"] == user_api_key.organization_id + assert kwargs["project_id"] == user_api_key.project_id + assert "stalled" in kwargs["error_message"] + assert summary == { + "total": 1, + "resumes_dispatched": 0, + "in_flight_skipped": 0, + "reaped": 1, + } + + def test_a_young_loop_is_never_reaped( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + _make_run(db, user_api_key, "guard-young") + + with patch(_START), patch(_REAPER) as mock_reaper: + dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_reaper.assert_not_called() + + def test_a_stale_dispatch_stamp_alone_never_reaps( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + """Only age since kickoff reaps — an old stamp just means it's due again.""" + run = _make_run(db, user_api_key, "guard-stale-stamp") + _backdate(db, run, last_dispatched_at=now() - timedelta(days=30)) + + with patch(_START) as mock_start, patch(_REAPER) as mock_reaper: + dispatch_pending_evaluation_iteration_resumes(session=db) + + mock_reaper.assert_not_called() + mock_start.assert_called_once() diff --git a/backend/app/tests/crud/evaluations/test_fast_chunk_cleanup.py b/backend/app/tests/crud/evaluations/test_fast_chunk_cleanup.py new file mode 100644 index 000000000..c4e78ec77 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_fast_chunk_cleanup.py @@ -0,0 +1,97 @@ +"""Chunk-artifact cleanup (`fast_chunks.delete_response_chunk_artifacts`). + +Cleanup runs in the best-effort tail, after the completed transition, so it must +swallow everything — including a failure resolving cloud storage. A raise here +would flip an already-completed run to failed over orphaned chunk files nobody +reads. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from sqlmodel import Session + +from app.crud.evaluations.fast import _cleanup_response_chunks +from app.crud.evaluations.fast_chunks import delete_response_chunk_artifacts + +_FAST = "app.crud.evaluations.fast" +_CHUNKS = "app.crud.evaluations.fast_chunks" + + +def _eval_run() -> SimpleNamespace: + return SimpleNamespace(id=1, project_id=2, organization_id=3) + + +class TestCleanupResponseChunks: + def test_storage_resolution_failure_never_escapes(self) -> None: + with patch( + f"{_FAST}.get_cloud_storage", side_effect=RuntimeError("no credentials") + ): + _cleanup_response_chunks( + session=MagicMock(spec=Session), eval_run=_eval_run() + ) + + def test_storage_failure_skips_the_delete_pass_entirely(self) -> None: + with ( + patch(f"{_FAST}.get_cloud_storage", side_effect=RuntimeError("boom")), + patch(f"{_FAST}.delete_response_chunk_artifacts") as mock_delete, + ): + _cleanup_response_chunks( + session=MagicMock(spec=Session), eval_run=_eval_run() + ) + mock_delete.assert_not_called() + + def test_resolved_storage_is_handed_to_the_delete_pass(self) -> None: + storage = MagicMock() + with ( + patch(f"{_FAST}.get_cloud_storage", return_value=storage), + patch(f"{_FAST}.delete_response_chunk_artifacts") as mock_delete, + ): + session = MagicMock(spec=Session) + _cleanup_response_chunks(session=session, eval_run=_eval_run()) + mock_delete.assert_called_once_with( + session=session, storage=storage, eval_run_id=1 + ) + + +class TestDeleteResponseChunkArtifacts: + def test_deletes_each_chunk_file_and_row(self) -> None: + storage = MagicMock() + jobs = [ + SimpleNamespace(raw_output_url="s3://a", id=1), + SimpleNamespace(raw_output_url="s3://b", id=2), + ] + with ( + patch(f"{_CHUNKS}.list_response_chunk_jobs", return_value=jobs), + patch(f"{_CHUNKS}.delete_batch_job") as mock_delete_job, + ): + delete_response_chunk_artifacts( + session=MagicMock(spec=Session), storage=storage, eval_run_id=1 + ) + assert storage.delete.call_count == 2 + assert mock_delete_job.call_count == 2 + + def test_a_row_without_an_uploaded_unit_is_still_removed(self) -> None: + storage = MagicMock() + jobs = [SimpleNamespace(raw_output_url=None, id=1)] + with ( + patch(f"{_CHUNKS}.list_response_chunk_jobs", return_value=jobs), + patch(f"{_CHUNKS}.delete_batch_job") as mock_delete_job, + ): + delete_response_chunk_artifacts( + session=MagicMock(spec=Session), storage=storage, eval_run_id=1 + ) + storage.delete.assert_not_called() + mock_delete_job.assert_called_once() + + def test_a_failing_delete_never_escapes(self) -> None: + storage = MagicMock() + storage.delete.side_effect = RuntimeError("s3 down") + jobs = [SimpleNamespace(raw_output_url="s3://a", id=1)] + with ( + patch(f"{_CHUNKS}.list_response_chunk_jobs", return_value=jobs), + patch(f"{_CHUNKS}.delete_batch_job"), + ): + delete_response_chunk_artifacts( + session=MagicMock(spec=Session), storage=storage, eval_run_id=1 + ) diff --git a/backend/app/tests/crud/evaluations/test_fast_cosine.py b/backend/app/tests/crud/evaluations/test_fast_cosine.py new file mode 100644 index 000000000..9bd6de8df --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_fast_cosine.py @@ -0,0 +1,150 @@ +"""Cosine scoring extracted from Stage 3 (`fast_cosine.py`). + +Covers the ref-keying rule (trace_id when traced, item_id when not) and the +unscoreable classification, since both decide what the UI can show for a row. +""" + +from typing import Any + +from app.crud.evaluations.fast_cosine import ( + build_item_refs, + classify_empty_side, + score_cosine_run, +) +from app.crud.evaluations.score import ( + COSINE_SCORE_NAME, + UNSCOREABLE_EMBEDDING_FAILED, + UNSCOREABLE_EMPTY_GROUND_TRUTH, + UNSCOREABLE_EMPTY_OUTPUT, +) + + +def _response( + item_id: str, output: str = "out", ground_truth: str = "gt" +) -> dict[str, Any]: + return { + "item_id": item_id, + "question": "q", + "generated_output": output, + "ground_truth": ground_truth, + } + + +def _embedding(item_id: str, vector: list[float], other: list[float]) -> dict[str, Any]: + return { + "item_id": item_id, + "output_embedding": vector, + "ground_truth_embedding": other, + "failed": False, + } + + +class TestClassifyEmptySide: + def test_empty_output_wins_over_empty_ground_truth(self) -> None: + response = _response("i", output="", ground_truth="") + assert classify_empty_side(response) == UNSCOREABLE_EMPTY_OUTPUT + + def test_empty_ground_truth_alone(self) -> None: + response = _response("i", ground_truth="") + assert classify_empty_side(response) == UNSCOREABLE_EMPTY_GROUND_TRUTH + + def test_both_sides_present_is_none(self) -> None: + assert classify_empty_side(_response("i")) is None + + +class TestBuildItemRefs: + def test_prefers_the_trace_id_when_the_run_is_traced(self) -> None: + refs = build_item_refs([_response("a"), _response("b")], {"a": "trace-a"}) + assert refs == {"a": "trace-a", "b": "b"} + + def test_falls_back_to_item_id_when_untraced(self) -> None: + refs = build_item_refs([_response("a")], {}) + assert refs == {"a": "a"} + + +class TestScoreCosineRun: + def test_identical_vectors_score_one_and_land_in_the_summary(self) -> None: + responses = [_response("a")] + item_refs = build_item_refs(responses, {}) + result = score_cosine_run( + response_results=responses, + embedding_results=[_embedding("a", [1.0, 0.0], [1.0, 0.0])], + item_refs=item_refs, + trace_id_mapping={}, + total_items=1, + ) + assert result.item_id_to_score["a"] == 1.0 + assert result.per_item_scores == {"a": 1.0} + assert result.unscoreable == {} + + summary = next( + s for s in result.summary_scores if s["name"] == COSINE_SCORE_NAME + ) + assert summary["avg"] == 1.0 + assert summary["total_pairs"] == 1 + + def test_missing_embedding_pair_is_unscoreable_not_zero(self) -> None: + responses = [_response("a")] + item_refs = build_item_refs(responses, {}) + result = score_cosine_run( + response_results=responses, + embedding_results=[], + item_refs=item_refs, + trace_id_mapping={}, + total_items=1, + ) + assert result.unscoreable == {"a": UNSCOREABLE_EMBEDDING_FAILED} + assert result.item_id_to_score == {} + summary = next( + s for s in result.summary_scores if s["name"] == COSINE_SCORE_NAME + ) + assert summary["total_pairs"] == 0 + + def test_empty_side_beats_embedding_failed_as_the_reason(self) -> None: + responses = [_response("a", output="")] + result = score_cosine_run( + response_results=responses, + embedding_results=[], + item_refs=build_item_refs(responses, {}), + trace_id_mapping={}, + total_items=1, + ) + assert result.unscoreable == {"a": UNSCOREABLE_EMPTY_OUTPUT} + + def test_untraced_run_writes_nothing_back_to_langfuse(self) -> None: + responses = [_response("a")] + result = score_cosine_run( + response_results=responses, + embedding_results=[_embedding("a", [1.0, 0.0], [1.0, 0.0])], + item_refs=build_item_refs(responses, {}), + trace_id_mapping={}, + total_items=1, + ) + assert result.write_items == [] + + def test_traced_run_writes_scores_and_unscoreable_reasons(self) -> None: + responses = [_response("a"), _response("b", output="")] + trace_id_mapping = {"a": "trace-a", "b": "trace-b"} + result = score_cosine_run( + response_results=responses, + embedding_results=[_embedding("a", [1.0, 0.0], [1.0, 0.0])], + item_refs=build_item_refs(responses, trace_id_mapping), + trace_id_mapping=trace_id_mapping, + total_items=2, + ) + by_trace = {w["trace_id"]: w for w in result.write_items} + assert by_trace["trace-a"]["cosine_similarity"] == 1.0 + assert by_trace["trace-b"]["unscoreable"] is True + assert by_trace["trace-b"]["reason"] == UNSCOREABLE_EMPTY_OUTPUT + + def test_per_item_scores_are_keyed_by_ref_not_item_id(self) -> None: + responses = [_response("a")] + trace_id_mapping = {"a": "trace-a"} + result = score_cosine_run( + response_results=responses, + embedding_results=[_embedding("a", [1.0, 0.0], [1.0, 0.0])], + item_refs=build_item_refs(responses, trace_id_mapping), + trace_id_mapping=trace_id_mapping, + total_items=1, + ) + assert result.per_item_scores == {"trace-a": 1.0} diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index f2f210b6e..763a5e960 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -30,15 +30,17 @@ from app.core.config import settings from app.crud.evaluations.fast import ( - CHUNK_CONFIG_INDEX, - CHUNK_CONFIG_RUN_ID, - JOB_TYPE_EVALUATION_FAST_CHUNK, - PROMPT_TEMPLATE_LABEL, - _format_top_kb_matches, _responses_call_for_item, run_fast_evaluation, run_response_chunk, ) +from app.crud.evaluations.fast_chunks import ( + CHUNK_CONFIG_INDEX, + CHUNK_CONFIG_RUN_ID, + JOB_TYPE_EVALUATION_FAST_CHUNK, +) +from app.crud.evaluations.fast_traces import format_top_kb_matches +from app.crud.evaluations.judge_stage import PROMPT_TEMPLATE_LABEL from app.crud.evaluations.score import ( GROUND_TRUTH_SCORE_NAME, JUDGE_FAILED_REASON, @@ -1419,10 +1421,10 @@ def _judge(params): class TestFormatTopKbMatches: - """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" + """`format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" def test_formats_filename_and_percent_to_one_decimal(self) -> None: - result = _format_top_kb_matches( + result = format_top_kb_matches( [ {"filename": "biu-1.pdf", "score": 0.906}, {"filename": "faq.pdf", "score": 0.663}, @@ -1431,7 +1433,7 @@ def test_formats_filename_and_percent_to_one_decimal(self) -> None: assert result == "biu-1.pdf (90.6%), faq.pdf (66.3%)" def test_includes_all_chunks_regardless_of_score(self) -> None: - result = _format_top_kb_matches( + result = format_top_kb_matches( [ {"filename": "hi.pdf", "score": 0.9}, {"filename": "lo.pdf", "score": 0.5}, @@ -1441,18 +1443,18 @@ def test_includes_all_chunks_regardless_of_score(self) -> None: def test_caps_at_three_matches(self) -> None: chunks = [{"filename": f"f{i}.pdf", "score": 0.9 - i * 0.01} for i in range(5)] - result = _format_top_kb_matches(chunks) + result = format_top_kb_matches(chunks) assert result == "f0.pdf (90.0%), f1.pdf (89.0%), f2.pdf (88.0%)" def test_missing_filename_renders_unknown(self) -> None: - assert _format_top_kb_matches([{"score": 0.9}]) == "unknown (90.0%)" + assert format_top_kb_matches([{"score": 0.9}]) == "unknown (90.0%)" assert ( - _format_top_kb_matches([{"filename": None, "score": 0.8}]) + format_top_kb_matches([{"filename": None, "score": 0.8}]) == "unknown (80.0%)" ) def test_empty_input_is_empty_string(self) -> None: - assert _format_top_kb_matches([]) == "" + assert format_top_kb_matches([]) == "" class TestFileSearchIncludeParam: @@ -1489,7 +1491,6 @@ def _fake_call(*, openai_client, base_params, item): config=TextLLMParams(model="gpt-4o"), dataset_items_slice=[{"id": "item-1"}], chunk_index=0, - log_prefix="[test]", ) return captured diff --git a/backend/app/tests/crud/evaluations/test_fast_results.py b/backend/app/tests/crud/evaluations/test_fast_results.py new file mode 100644 index 000000000..c604dd257 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_fast_results.py @@ -0,0 +1,168 @@ +"""Per-item result shapes extracted from the fast-eval stages (`fast_results.py`). + +These are the units uploaded to S3, so the key set and the "missing counter reads +as 0" rule are part of the on-disk contract, not implementation detail. +""" + +from types import SimpleNamespace + +import pytest + +from app.core.config import settings +from app.crud.evaluations.fast_results import ( + EMBEDDING_USAGE_KEYS, + RESPONSE_USAGE_KEYS, + build_embedding_failure, + build_response_result, + extract_usage, + is_failure_threshold_breached, + parse_embedding_pair, + sum_usage, +) + + +class TestBuildResponseResult: + def test_carries_every_key_the_s3_unit_needs(self) -> None: + result = build_response_result( + item_id="item_0_0", + question="q", + ground_truth="gt", + question_id=1, + generated_output="out", + failed=False, + ) + assert set(result) == { + "item_id", + "question", + "generated_output", + "ground_truth", + "response_id", + "usage", + "question_id", + "failed", + "retrieved_chunks", + } + + def test_optional_fields_default_to_none(self) -> None: + result = build_response_result( + item_id="i", + question="q", + ground_truth="gt", + question_id=None, + generated_output="ERROR: boom", + failed=True, + ) + assert result["response_id"] is None + assert result["usage"] is None + assert result["retrieved_chunks"] is None + + +class TestBuildEmbeddingFailure: + def test_marks_failed_with_both_vectors_absent(self) -> None: + result = build_embedding_failure("item_1_0", "empty output or ground_truth") + assert result["failed"] is True + assert result["output_embedding"] is None + assert result["ground_truth_embedding"] is None + assert result["error"] == "empty output or ground_truth" + + +class TestExtractUsage: + def test_reads_requested_counters(self) -> None: + usage = SimpleNamespace(input_tokens=10, output_tokens=4, total_tokens=14) + assert extract_usage(usage, RESPONSE_USAGE_KEYS) == { + "input_tokens": 10, + "output_tokens": 4, + "total_tokens": 14, + } + + def test_missing_or_none_counters_read_as_zero(self) -> None: + assert extract_usage(None, EMBEDDING_USAGE_KEYS) == { + "prompt_tokens": 0, + "total_tokens": 0, + } + assert extract_usage( + SimpleNamespace(prompt_tokens=None), ("prompt_tokens",) + ) == {"prompt_tokens": 0} + + +class TestParseEmbeddingPair: + @staticmethod + def _response(pairs: list[tuple[int, list[float]]]) -> SimpleNamespace: + return SimpleNamespace( + data=[SimpleNamespace(index=i, embedding=v) for i, v in pairs], + usage=SimpleNamespace(prompt_tokens=6, total_tokens=6), + ) + + def test_index_zero_is_output_and_index_one_is_ground_truth(self) -> None: + result = parse_embedding_pair( + item_id="i", response=self._response([(0, [1.0]), (1, [2.0])]) + ) + assert result["output_embedding"] == [1.0] + assert result["ground_truth_embedding"] == [2.0] + assert result["failed"] is False + assert result["usage"] == {"prompt_tokens": 6, "total_tokens": 6} + + def test_order_in_the_payload_does_not_matter(self) -> None: + result = parse_embedding_pair( + item_id="i", response=self._response([(1, [2.0]), (0, [1.0])]) + ) + assert result["output_embedding"] == [1.0] + assert result["ground_truth_embedding"] == [2.0] + + def test_short_payload_is_a_failure(self) -> None: + result = parse_embedding_pair( + item_id="i", response=self._response([(0, [1.0])]) + ) + assert result["failed"] is True + assert "expected 2 embeddings" in result["error"] + + +class TestSumUsage: + def test_sums_requested_keys_across_results(self) -> None: + results = [ + {"usage": {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3}}, + {"usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}}, + ] + assert sum_usage(results, RESPONSE_USAGE_KEYS) == { + "input_tokens": 11, + "output_tokens": 22, + "total_tokens": 33, + } + + def test_absent_or_null_usage_contributes_zero(self) -> None: + results = [{"usage": None}, {}, {"usage": {"total_tokens": 5}}] + assert sum_usage(results, ("total_tokens",)) == {"total_tokens": 5} + + def test_empty_results_give_zeroed_totals(self) -> None: + assert sum_usage([], EMBEDDING_USAGE_KEYS) == { + "prompt_tokens": 0, + "total_tokens": 0, + } + + +class TestIsFailureThresholdBreached: + def test_empty_run_never_breaches(self) -> None: + assert is_failure_threshold_breached(failed_rows=0, total_rows=0) is False + + @pytest.mark.parametrize("failed_rows", [0, 1, 5]) + def test_at_or_below_threshold_passes(self, failed_rows: int) -> None: + total_rows = 10 + assert ( + failed_rows / total_rows <= settings.EVAL_FAST_FAILURE_THRESHOLD + ), "fixture assumes the default 0.5 threshold" + assert ( + is_failure_threshold_breached( + failed_rows=failed_rows, total_rows=total_rows + ) + is False + ) + + def test_strictly_above_threshold_breaches(self) -> None: + total_rows = 10 + failed_rows = int(settings.EVAL_FAST_FAILURE_THRESHOLD * total_rows) + 1 + assert ( + is_failure_threshold_breached( + failed_rows=failed_rows, total_rows=total_rows + ) + is True + ) diff --git a/backend/app/tests/crud/evaluations/test_fast_score_durability.py b/backend/app/tests/crud/evaluations/test_fast_score_durability.py new file mode 100644 index 000000000..effacbc2c --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_fast_score_durability.py @@ -0,0 +1,151 @@ +"""Score-unit durability ordering in `run_fast_evaluation`. + +`save_score` must land before the completed transition. On a judged (v2) run the +score unit is the only copy of the per-row judge scores — nothing goes to +Langfuse to fall back on — so a crash between the two must leave the run +`processing`, not `completed` with its scores gone. +""" + +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from sqlmodel import Session + +from app.crud.evaluations.fast import run_fast_evaluation +from app.models import EvaluationRun +from app.models.evaluation import RunModeEnum +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.test_data import ( + create_test_config, + create_test_evaluation_dataset, +) +from app.tests.utils.utils import random_lower_string + +_FAST = "app.crud.evaluations.fast" + + +@pytest.fixture +def judged_fast_run(db: Session, user_api_key: TestAuthContext) -> EvaluationRun: + dataset = create_test_evaluation_dataset( + db=db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + config = create_test_config( + db=db, project_id=user_api_key.project_id, use_kaapi_schema=True + ) + run = EvaluationRun( + run_name=f"run-{random_lower_string()}", + dataset_name=dataset.name, + dataset_id=dataset.id, + config_id=config.id, + config_version=1, + status="processing", + run_mode=RunModeEnum.FAST, + total_items=0, + is_judge_run=True, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + db.add(run) + db.commit() + db.refresh(run) + return run + + +@pytest.fixture +def _stubbed_stages() -> Any: + """Collapse stages 1-3 so only the stage 4/5/6 ordering is under test.""" + score = {"summary_scores": [{"name": "Judge", "avg": 4.0}], "overall": None} + with ( + patch(f"{_FAST}._merge_response_chunks") as merge, + patch(f"{_FAST}._stage3_score_and_trace") as stage3, + patch(f"{_FAST}._cleanup_response_chunks"), + patch(f"{_FAST}._sync_scores_to_langfuse", return_value=True), + ): + merge.side_effect = lambda *, session, eval_run: (eval_run, []) + stage3.side_effect = lambda **kwargs: (kwargs["eval_run"], score, []) + yield score + + +def _status(db: Session, eval_run_id: int) -> str: + run = db.get(EvaluationRun, eval_run_id) + assert run is not None + return run.status + + +def _run(*, db: Session, eval_run: EvaluationRun) -> EvaluationRun: + return run_fast_evaluation( + session=db, + openai_client=MagicMock(), + langfuse=None, + eval_run=eval_run, + ) + + +class TestScoreUnitPersistsBeforeCompletion: + def test_save_score_sees_a_run_that_is_not_yet_completed( + self, db: Session, judged_fast_run: EvaluationRun, _stubbed_stages: Any + ) -> None: + seen: list[str] = [] + + def _capture(*, eval_run_id: int, **_: Any) -> SimpleNamespace: + seen.append(_status(db, eval_run_id)) + return SimpleNamespace(id=eval_run_id) + + with patch(f"{_FAST}.save_score", side_effect=_capture): + result = _run(db=db, eval_run=judged_fast_run) + + assert seen == ["processing"] + assert result.status == "completed" + + def test_a_failing_save_score_leaves_the_run_uncompleted( + self, db: Session, judged_fast_run: EvaluationRun, _stubbed_stages: Any + ) -> None: + with patch(f"{_FAST}.save_score", side_effect=RuntimeError("s3 down")): + with pytest.raises(RuntimeError, match="s3 down"): + _run(db=db, eval_run=judged_fast_run) + + db.expire_all() + assert _status(db, judged_fast_run.id) == "processing" + + def test_a_vanished_run_raises_instead_of_completing( + self, db: Session, judged_fast_run: EvaluationRun, _stubbed_stages: Any + ) -> None: + with patch(f"{_FAST}.save_score", return_value=None): + with pytest.raises(RuntimeError, match="Score unit not persisted"): + _run(db=db, eval_run=judged_fast_run) + + db.expire_all() + assert _status(db, judged_fast_run.id) == "processing" + + def test_the_caller_still_gets_the_full_unit_back( + self, db: Session, judged_fast_run: EvaluationRun, _stubbed_stages: Any + ) -> None: + with patch( + f"{_FAST}.save_score", return_value=SimpleNamespace(id=judged_fast_run.id) + ): + result = _run(db=db, eval_run=judged_fast_run) + + assert result.score == _stubbed_stages + + def test_cleanup_runs_only_after_the_completed_transition( + self, db: Session, judged_fast_run: EvaluationRun, _stubbed_stages: Any + ) -> None: + seen: list[str] = [] + + def _capture(*, eval_run: EvaluationRun, **_: Any) -> None: + seen.append(eval_run.status) + + with ( + patch( + f"{_FAST}.save_score", + return_value=SimpleNamespace(id=judged_fast_run.id), + ), + patch(f"{_FAST}._cleanup_response_chunks", side_effect=_capture), + ): + _run(db=db, eval_run=judged_fast_run) + + assert seen == ["completed"] diff --git a/backend/app/tests/crud/evaluations/test_fast_traces.py b/backend/app/tests/crud/evaluations/test_fast_traces.py new file mode 100644 index 000000000..eee07e7a5 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_fast_traces.py @@ -0,0 +1,240 @@ +"""Trace-record building extracted from Stage 3 (`fast_traces.py`). + +The trace unit is the v2 source of truth for per-row judge scores, so the shape +built here (and the KB placeholder wording) is what the read path serves. +""" + +from typing import Any + +from app.crud.evaluations.fast_traces import build_trace_records, format_top_kb_matches +from app.crud.evaluations.judge import ( + METRIC_REGISTRY, + JudgeMetricEnum, + JudgeResult, + MetricScore, +) +from app.crud.evaluations.score import ( + COSINE_SCORE_NAME, + DEFAULT_CATEGORY, + GROUND_TRUTH_SCORE_NAME, + JUDGE_FAILED_REASON, + KNOWLEDGE_BASE_SCORE_NAME, + UNSCOREABLE_EMPTY_OUTPUT, + VerdictEnum, +) + +METRICS = list(METRIC_REGISTRY.values()) + + +def _response(item_id: str, **overrides: Any) -> dict[str, Any]: + return { + "item_id": item_id, + "question": "q", + "generated_output": "out", + "ground_truth": "gt", + "question_id": 1, + **overrides, + } + + +def _scores_by_name(trace: dict[str, Any]) -> dict[str, Any]: + return {s["name"]: s for s in trace["scores"]} + + +class TestFormatTopKbMatches: + def test_renders_filename_and_percentage(self) -> None: + chunks = [{"filename": "biu-1.pdf", "score": 0.906}] + assert format_top_kb_matches(chunks) == "biu-1.pdf (90.6%)" + + def test_caps_at_three_matches(self) -> None: + chunks = [{"filename": f"f{i}.pdf", "score": 0.5} for i in range(5)] + assert len(format_top_kb_matches(chunks).split(", ")) == 3 + + def test_missing_filename_renders_unknown(self) -> None: + assert format_top_kb_matches([{"score": 0.5}]) == "unknown (50.0%)" + + def test_no_chunks_is_empty_string(self) -> None: + assert format_top_kb_matches([]) == "" + + +class TestBuildTraceRecordsCosine: + def test_scored_row_carries_the_cosine_entry(self) -> None: + traces = build_trace_records( + response_results=[_response("a")], + item_refs={"a": "a"}, + is_judge_run=False, + judge_results={}, + metrics=[], + cosine_by_item_id={"a": 0.912}, + unscoreable={}, + ) + score = _scores_by_name(traces[0])[COSINE_SCORE_NAME] + assert score["value"] == 0.91 + assert score.get("unscoreable") is None + + def test_unscoreable_row_gets_a_zero_placeholder(self) -> None: + traces = build_trace_records( + response_results=[_response("a", generated_output="")], + item_refs={"a": "a"}, + is_judge_run=False, + judge_results={}, + metrics=[], + cosine_by_item_id={}, + unscoreable={"a": UNSCOREABLE_EMPTY_OUTPUT}, + ) + score = _scores_by_name(traces[0])[COSINE_SCORE_NAME] + assert score["value"] == 0 + assert score["unscoreable"] is True + assert UNSCOREABLE_EMPTY_OUTPUT in score["comment"] + + def test_judge_failed_reason_gets_no_cosine_placeholder(self) -> None: + traces = build_trace_records( + response_results=[_response("a")], + item_refs={"a": "a"}, + is_judge_run=False, + judge_results={}, + metrics=[], + cosine_by_item_id={}, + unscoreable={"a": JUDGE_FAILED_REASON}, + ) + assert traces[0]["scores"] == [] + + +class TestBuildTraceRecordsJudge: + @staticmethod + def _judge_result(**metrics: MetricScore) -> JudgeResult: + return JudgeResult( + metrics={JudgeMetricEnum(k): v for k, v in metrics.items()}, + usage={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + + def test_each_metric_carries_score_reasoning_and_verdict(self) -> None: + traces = build_trace_records( + response_results=[_response("a")], + item_refs={"a": "a"}, + is_judge_run=True, + judge_results={ + "a": self._judge_result( + ground_truth=MetricScore(score=5, reasoning="spot on") + ) + }, + metrics=METRICS, + cosine_by_item_id={}, + unscoreable={}, + ) + score = _scores_by_name(traces[0])[GROUND_TRUTH_SCORE_NAME] + assert score["value"] == 5 + assert score["comment"] == "spot on" + assert score["verdict"] is VerdictEnum.GOOD + + def test_judge_run_never_carries_a_cosine_entry(self) -> None: + traces = build_trace_records( + response_results=[_response("a")], + item_refs={"a": "a"}, + is_judge_run=True, + judge_results={ + "a": self._judge_result( + ground_truth=MetricScore(score=3, reasoning="partly") + ) + }, + metrics=METRICS, + cosine_by_item_id={"a": 0.5}, + unscoreable={}, + ) + assert COSINE_SCORE_NAME not in _scores_by_name(traces[0]) + + def test_kb_metric_appends_top_matches_to_its_reasoning(self) -> None: + traces = build_trace_records( + response_results=[ + _response( + "a", + retrieved_chunks=[ + {"filename": "faq.pdf", "score": 0.8, "text": "t"} + ], + ) + ], + item_refs={"a": "a"}, + is_judge_run=True, + judge_results={ + "a": self._judge_result( + knowledge_base=MetricScore(score=4, reasoning="grounded") + ) + }, + metrics=METRICS, + cosine_by_item_id={}, + unscoreable={}, + ) + comment = _scores_by_name(traces[0])[KNOWLEDGE_BASE_SCORE_NAME]["comment"] + assert comment == "grounded | Top matches: faq.pdf (80.0%)" + + def test_kb_dropped_with_no_chunks_reads_as_not_queried(self) -> None: + traces = build_trace_records( + response_results=[_response("a")], + item_refs={"a": "a"}, + is_judge_run=True, + judge_results={ + "a": self._judge_result( + ground_truth=MetricScore(score=4, reasoning="ok") + ) + }, + metrics=METRICS, + cosine_by_item_id={}, + unscoreable={}, + ) + kb = _scores_by_name(traces[0])[KNOWLEDGE_BASE_SCORE_NAME] + assert kb["value"] == "N/A" + assert kb["unscoreable"] is True + assert kb["comment"] == "Knowledge base not queried." + + def test_kb_dropped_despite_chunks_reads_as_unavailable(self) -> None: + traces = build_trace_records( + response_results=[ + _response( + "a", + retrieved_chunks=[{"filename": "f.pdf", "score": 0.4, "text": "t"}], + ) + ], + item_refs={"a": "a"}, + is_judge_run=True, + judge_results={ + "a": self._judge_result( + ground_truth=MetricScore(score=4, reasoning="ok") + ) + }, + metrics=METRICS, + cosine_by_item_id={}, + unscoreable={}, + ) + kb = _scores_by_name(traces[0])[KNOWLEDGE_BASE_SCORE_NAME] + assert kb["comment"] == "Knowledge base score unavailable for this row." + + def test_a_row_with_no_judge_result_still_gets_a_trace(self) -> None: + traces = build_trace_records( + response_results=[_response("a", generated_output="")], + item_refs={"a": "a"}, + is_judge_run=True, + judge_results={}, + metrics=METRICS, + cosine_by_item_id={}, + unscoreable={"a": UNSCOREABLE_EMPTY_OUTPUT}, + ) + assert len(traces) == 1 + assert traces[0]["scores"] == [] + + +class TestTraceEnvelope: + def test_trace_is_keyed_by_ref_and_defaults_its_category(self) -> None: + traces = build_trace_records( + response_results=[_response("a")], + item_refs={"a": "trace-a"}, + is_judge_run=False, + judge_results={}, + metrics=[], + cosine_by_item_id={}, + unscoreable={}, + ) + assert traces[0]["trace_id"] == "trace-a" + assert traces[0]["category"] == DEFAULT_CATEGORY + assert traces[0]["question_id"] == 1 + assert traces[0]["llm_answer"] == "out" + assert traces[0]["ground_truth_answer"] == "gt" diff --git a/backend/app/tests/crud/evaluations/test_judge.py b/backend/app/tests/crud/evaluations/test_judge.py index 55e48fac7..518e3c00c 100644 --- a/backend/app/tests/crud/evaluations/test_judge.py +++ b/backend/app/tests/crud/evaluations/test_judge.py @@ -30,11 +30,13 @@ build_judge_params, judge_row, ) -from app.crud.evaluations.score import ( +from app.crud.evaluations.judge_prompts import ( GROUND_TRUTH_JUDGE_PROMPT, - GROUND_TRUTH_SCORE_NAME, JUDGE_SYSTEM_PREAMBLE, KNOWLEDGE_BASE_JUDGE_PROMPT, +) +from app.crud.evaluations.score import ( + GROUND_TRUTH_SCORE_NAME, KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, ) diff --git a/backend/app/tests/crud/evaluations/test_judge_stage.py b/backend/app/tests/crud/evaluations/test_judge_stage.py new file mode 100644 index 000000000..8f0b6ba96 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_judge_stage.py @@ -0,0 +1,121 @@ +"""Run-level judge helpers extracted from Stage 3 (`judge_stage.py`). + +The per-row judge call itself is covered by `test_judge.py`; these cover the +run-level wrapping — which rows are judgeable, how a row's input blocks are +assembled, and the metric rollup a run summary is built from. +""" + +from app.crud.evaluations.judge import ( + METRIC_REGISTRY, + JudgeInputEnum, + JudgeMetricEnum, + JudgeResult, + MetricScore, +) +from app.crud.evaluations.judge_stage import ( + build_judge_inputs, + build_metric_summary_scores, + select_judgeable_rows, +) +from app.crud.evaluations.score import GROUND_TRUTH_SCORE_NAME, PROMPT_SCORE_NAME + +METRICS = list(METRIC_REGISTRY.values()) + + +def _judge_result(**metrics: MetricScore) -> JudgeResult: + return JudgeResult( + metrics={JudgeMetricEnum(k): v for k, v in metrics.items()}, + usage={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + + +class TestSelectJudgeableRows: + def test_keeps_rows_with_both_sides(self) -> None: + rows = [{"item_id": "a", "generated_output": "o", "ground_truth": "g"}] + assert select_judgeable_rows(rows) == rows + + def test_drops_rows_missing_either_side(self) -> None: + rows = [ + {"item_id": "a", "generated_output": "", "ground_truth": "g"}, + {"item_id": "b", "generated_output": "o", "ground_truth": ""}, + {"item_id": "c", "generated_output": "o", "ground_truth": "g"}, + ] + assert [r["item_id"] for r in select_judgeable_rows(rows)] == ["c"] + + +class TestBuildJudgeInputs: + def test_maps_the_response_onto_the_input_blocks(self) -> None: + inputs = build_judge_inputs( + response={ + "question": "q", + "generated_output": "out", + "ground_truth": "gt", + "retrieved_chunks": [{"text": "one"}, {"text": "two"}], + }, + config_prompt="be nice", + ) + assert inputs[JudgeInputEnum.CONFIG_PROMPT] == "be nice" + assert inputs[JudgeInputEnum.QUESTION] == "q" + assert inputs[JudgeInputEnum.GENERATED_ANSWER] == "out" + assert inputs[JudgeInputEnum.GOLDEN_ANSWER] == "gt" + assert inputs[JudgeInputEnum.RETRIEVED_CHUNKS] == "one\n---\ntwo" + + def test_chunks_without_text_are_skipped(self) -> None: + inputs = build_judge_inputs( + response={ + "retrieved_chunks": [{"text": ""}, {"score": 1}, {"text": "kept"}] + }, + config_prompt="", + ) + assert inputs[JudgeInputEnum.RETRIEVED_CHUNKS] == "kept" + + def test_absent_fields_become_empty_blocks(self) -> None: + inputs = build_judge_inputs(response={}, config_prompt="") + assert all(value == "" for value in inputs.values()) + + +class TestBuildMetricSummaryScores: + def test_averages_each_metric_over_the_rows_that_scored_it(self) -> None: + judge_results = { + "a": _judge_result(ground_truth=MetricScore(score=4, reasoning="r")), + "b": _judge_result(ground_truth=MetricScore(score=2, reasoning="r")), + } + summaries = build_metric_summary_scores( + metrics=METRICS, judge_results=judge_results + ) + ground_truth = next( + s for s in summaries if s["name"] == GROUND_TRUTH_SCORE_NAME + ) + assert ground_truth["avg"] == 3.0 + assert ground_truth["std"] == 1.0 + assert ground_truth["total_pairs"] == 2 + assert ground_truth["data_type"] == "NUMERIC" + + def test_a_metric_no_row_scored_is_omitted_entirely(self) -> None: + judge_results = { + "a": _judge_result(ground_truth=MetricScore(score=5, reasoning="r")) + } + summaries = build_metric_summary_scores( + metrics=METRICS, judge_results=judge_results + ) + assert {s["name"] for s in summaries} == {GROUND_TRUTH_SCORE_NAME} + + def test_only_the_rows_that_scored_count_toward_total_pairs(self) -> None: + judge_results = { + "a": _judge_result( + ground_truth=MetricScore(score=5, reasoning="r"), + prompt=MetricScore(score=3, reasoning="r"), + ), + "b": _judge_result(ground_truth=MetricScore(score=5, reasoning="r")), + } + summaries = { + s["name"]: s + for s in build_metric_summary_scores( + metrics=METRICS, judge_results=judge_results + ) + } + assert summaries[GROUND_TRUTH_SCORE_NAME]["total_pairs"] == 2 + assert summaries[PROMPT_SCORE_NAME]["total_pairs"] == 1 + + def test_no_judge_results_yields_no_summaries(self) -> None: + assert build_metric_summary_scores(metrics=METRICS, judge_results={}) == [] diff --git a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py index e0b755c77..b6ac76d4c 100644 --- a/backend/app/tests/services/evaluations/test_evaluation_service_s3.py +++ b/backend/app/tests/services/evaluations/test_evaluation_service_s3.py @@ -369,7 +369,7 @@ def test_unsupported_provider_raises_422( @patch(f"{_MODULE}.update_evaluation_run") @patch(f"{_MODULE}.start_evaluation_batch_submission") - @patch(f"{_MODULE}.create_evaluation_run_or_409") + @patch(f"{_MODULE}.create_evaluation_run") @patch(f"{_MODULE}.resolve_evaluation_config") @patch(f"{_MODULE}.get_dataset_by_id") def test_queue_failure_marks_run_failed( @@ -406,7 +406,7 @@ def test_queue_failure_marks_run_failed( assert "Failed to queue batch submission" in update_arg.error_message @patch(f"{_MODULE}.start_evaluation_batch_submission") - @patch(f"{_MODULE}.create_evaluation_run_or_409") + @patch(f"{_MODULE}.create_evaluation_run") @patch(f"{_MODULE}.resolve_evaluation_config") @patch(f"{_MODULE}.get_dataset_by_id") def test_success_returns_run( diff --git a/backend/app/tests/services/evaluations/test_iteration_graph.py b/backend/app/tests/services/evaluations/test_iteration_graph.py index ea5af8216..1d4cf6ab9 100644 --- a/backend/app/tests/services/evaluations/test_iteration_graph.py +++ b/backend/app/tests/services/evaluations/test_iteration_graph.py @@ -26,7 +26,10 @@ from sqlmodel import Session from app.core.config import settings -from app.crud.evaluations.iteration import create_evaluation_iteration_run +from app.crud.evaluations.iteration import ( + create_evaluation_iteration_run, + update_evaluation_iteration_run, +) from app.crud.evaluations.score import ( GROUND_TRUTH_SCORE_NAME, KNOWLEDGE_BASE_SCORE_NAME, @@ -37,6 +40,7 @@ from app.models.evaluation import RunModeEnum from app.models.evaluation_iteration import ( EvaluationIterationRun, + EvaluationIterationRunUpdate, EvaluationIterationStatusEnum, ) from app.models.job import JobStatus, JobType, JobUpdate @@ -502,6 +506,48 @@ def test_ceiling_reached_persists_completed_status( assert persisted.stop_reason == STOP_REASON_CEILING_REACHED mock_send.assert_called_once() + def test_already_terminal_row_is_left_alone_and_no_callback_is_sent( + self, db: Session, user_api_key: TestAuthContext + ) -> None: + # The cron reaper won the race: row is already FAILED with its callback sent. + iteration_run = _make_iteration_run(db, user_api_key) + update_evaluation_iteration_run( + session=db, + iteration_run=iteration_run, + update=EvaluationIterationRunUpdate( + status=EvaluationIterationStatusEnum.FAILED, + error_message="reaped as stalled", + ), + ) + state = _base_state( + iteration_run_id=iteration_run.id, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + stop_reason=STOP_REASON_CEILING_REACHED, + history=[ + { + "round_number": 1, + "eval_run_id": 11, + "config_version": 1, + "stop_score": 0.7, + "kb_score": None, + }, + ], + best_round_number=1, + ) + + with _patch_session(db), patch( + "app.services.evaluations.iteration_graph.send_callback" + ) as mock_send: + finalize_node(state) + + db.expire_all() + persisted = db.get(EvaluationIterationRun, iteration_run.id) + assert persisted.status == EvaluationIterationStatusEnum.FAILED + assert persisted.error_message == "reaped as stalled" + assert persisted.stop_reason is None + mock_send.assert_not_called() + def test_round_failed_persists_failed_status_with_error_message( self, db: Session, user_api_key: TestAuthContext ) -> None: diff --git a/backend/app/tests/services/evaluations/test_iteration_state.py b/backend/app/tests/services/evaluations/test_iteration_state.py new file mode 100644 index 000000000..5e6948fbb --- /dev/null +++ b/backend/app/tests/services/evaluations/test_iteration_state.py @@ -0,0 +1,236 @@ +"""Round bookkeeping extracted from `wait_eval_node` (`iteration_state.py`). + +`advance_round_state` decides when the loop stops, so the ceiling counter's reset +rule and the max-rounds cap are the contract worth pinning. Pure — no graph, no DB. +""" + +from typing import Any + +import pytest + +from app.core.config import settings +from app.models.evaluation_iteration import EvaluationIterationStatusEnum +from app.services.evaluations.iteration import ( + STOP_REASON_CEILING_REACHED, + STOP_REASON_MAX_ROUNDS_REACHED, +) +from app.services.evaluations.iteration_state import ( + EvaluationIterationState, + advance_round_state, + build_iteration_report, +) + +_BIG_GAIN = settings.EVAL_ITERATION_CEILING_DELTA_THRESHOLD * 10 +_SMALL_GAIN = settings.EVAL_ITERATION_CEILING_DELTA_THRESHOLD / 10 + + +def _state(**overrides: Any) -> EvaluationIterationState: + base: dict[str, Any] = { + "iteration_run_id": 1, + "dataset_id": 2, + "experiment_name": "exp", + "config_id": "00000000-0000-0000-0000-000000000000", + "config_version": 1, + "round_number": 1, + "max_rounds": 10, + "current_eval_run_id": 100, + "current_improvement_job_id": None, + "history": [], + "best_round_number": None, + "best_config_version": None, + "best_stop_score": None, + "consecutive_low_delta_rounds": 0, + "stop_reason": None, + "error_message": None, + "organization_id": 1, + "project_id": 1, + "callback_url": "https://example.com/hook", + } + base.update(overrides) + return base # type: ignore[return-value] + + +def _round(round_number: int, stop_score: float) -> dict[str, Any]: + return { + "round_number": round_number, + "eval_run_id": 100 + round_number, + "config_version": round_number, + "stop_score": stop_score, + "kb_score": None, + } + + +class TestAdvanceRoundState: + def test_first_round_records_history_and_becomes_best(self) -> None: + update = advance_round_state( + state=_state(), eval_run_id=100, stop_score=3.0, kb_score=2.0 + ) + assert update["history"] == [ + { + "round_number": 1, + "eval_run_id": 100, + "config_version": 1, + "stop_score": 3.0, + "kb_score": 2.0, + } + ] + assert update["best_stop_score"] == 3.0 + assert update["best_round_number"] == 1 + assert update["best_config_version"] == 1 + + def test_first_round_never_counts_as_low_delta(self) -> None: + update = advance_round_state( + state=_state(), eval_run_id=100, stop_score=0.0, kb_score=None + ) + assert update["consecutive_low_delta_rounds"] == 0 + assert "stop_reason" not in update + + def test_a_worse_round_leaves_best_untouched(self) -> None: + state = _state( + round_number=2, + config_version=2, + history=[_round(1, 4.0)], + best_stop_score=4.0, + best_round_number=1, + best_config_version=1, + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=1.0, kb_score=None + ) + assert update["best_stop_score"] == 4.0 + assert update["best_round_number"] == 1 + assert update["best_config_version"] == 1 + + def test_a_better_round_takes_over_best(self) -> None: + state = _state( + round_number=2, + config_version=2, + history=[_round(1, 1.0)], + best_stop_score=1.0, + best_round_number=1, + best_config_version=1, + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=4.0, kb_score=None + ) + assert update["best_stop_score"] == 4.0 + assert update["best_round_number"] == 2 + assert update["best_config_version"] == 2 + + def test_a_tied_round_keeps_the_earlier_best(self) -> None: + state = _state( + round_number=2, + config_version=2, + history=[_round(1, 4.0)], + best_stop_score=4.0, + best_round_number=1, + best_config_version=1, + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=4.0, kb_score=None + ) + assert update["best_stop_score"] == 4.0 + assert update["best_round_number"] == 1 + assert update["best_config_version"] == 1 + + def test_a_gain_below_threshold_increments_the_ceiling_counter(self) -> None: + state = _state( + round_number=2, history=[_round(1, 3.0)], consecutive_low_delta_rounds=1 + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=3.0 + _SMALL_GAIN, kb_score=None + ) + assert update["consecutive_low_delta_rounds"] == 2 + + def test_a_gain_at_or_above_threshold_resets_the_counter(self) -> None: + state = _state( + round_number=2, history=[_round(1, 3.0)], consecutive_low_delta_rounds=2 + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=3.0 + _BIG_GAIN, kb_score=None + ) + assert update["consecutive_low_delta_rounds"] == 0 + assert "stop_reason" not in update + + def test_a_regression_counts_as_low_delta(self) -> None: + state = _state(round_number=2, history=[_round(1, 4.0)]) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=1.0, kb_score=None + ) + assert update["consecutive_low_delta_rounds"] == 1 + + def test_hitting_the_consecutive_cap_stops_on_ceiling(self) -> None: + cap = settings.EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS + state = _state( + round_number=cap + 1, + history=[_round(1, 3.0)], + consecutive_low_delta_rounds=cap - 1, + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=3.0, kb_score=None + ) + assert update["consecutive_low_delta_rounds"] == cap + assert update["stop_reason"] == STOP_REASON_CEILING_REACHED + + def test_the_last_allowed_round_stops_on_max_rounds(self) -> None: + state = _state(round_number=3, max_rounds=3, history=[_round(1, 1.0)]) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=1.0 + _BIG_GAIN, kb_score=None + ) + assert update["stop_reason"] == STOP_REASON_MAX_ROUNDS_REACHED + + def test_ceiling_wins_over_max_rounds_when_both_apply(self) -> None: + cap = settings.EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS + state = _state( + round_number=3, + max_rounds=3, + history=[_round(1, 3.0)], + consecutive_low_delta_rounds=cap - 1, + ) + update = advance_round_state( + state=state, eval_run_id=101, stop_score=3.0, kb_score=None + ) + assert update["stop_reason"] == STOP_REASON_CEILING_REACHED + + def test_kb_score_is_recorded_but_never_gates_stopping(self) -> None: + update = advance_round_state( + state=_state(), eval_run_id=100, stop_score=5.0, kb_score=0.0 + ) + assert update["history"][0]["kb_score"] == 0.0 + assert "stop_reason" not in update + + +class TestBuildIterationReport: + def test_report_carries_history_and_resolves_the_best_round(self) -> None: + state = _state( + history=[_round(1, 1.0), _round(2, 4.0)], + best_round_number=2, + stop_reason=STOP_REASON_CEILING_REACHED, + ) + report = build_iteration_report(state, EvaluationIterationStatusEnum.COMPLETED) + assert report.iteration_run_id == 1 + assert report.status is EvaluationIterationStatusEnum.COMPLETED + assert report.stop_reason == STOP_REASON_CEILING_REACHED + assert [r.round_number for r in report.history] == [1, 2] + assert report.best_round is not None + assert report.best_round.round_number == 2 + + def test_best_round_is_none_when_no_round_completed(self) -> None: + report = build_iteration_report( + _state(error_message="boom"), EvaluationIterationStatusEnum.FAILED + ) + assert report.best_round is None + assert report.history == [] + assert report.error_message == "boom" + + @pytest.mark.parametrize( + "status", + [ + EvaluationIterationStatusEnum.COMPLETED, + EvaluationIterationStatusEnum.FAILED, + ], + ) + def test_status_is_passed_through( + self, status: EvaluationIterationStatusEnum + ) -> None: + assert build_iteration_report(_state(), status).status is status diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 8ae4d7169..00669c4b8 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -7,10 +7,10 @@ All paths relative to `backend/app/`. ## Routes - `api/routes/evaluations/dataset.py`, `api/routes/evaluations/evaluation.py` — text datasets + runs (v1, `/api/v1`) -- `api/routes/evaluations/evaluation_v2.py` — `POST /api/v2/evaluations`, replica of v1 run trigger + native ground-truth LLM judge (Langfuse-free); mounted under `settings.API_V2_STR` -- `api/routes/evaluations/dataset_v2.py` — `POST /api/v2/evaluations/datasets`, Langfuse-free dataset upload; stores only the original CSV in S3 and records `duplication_factor` as metadata (rows expanded ×factor at run time, not physically duplicated) -- `api/routes/evaluations/prompt_improvement_v2.py` — `POST /api/v2/evaluations/{evaluation_id}/improve-prompt`, prompt iteration off the three-metric judge results (requires an `is_judge_run` run); same body as v1, returns a recommendation of type `prompt` -- `api/routes/evaluations/iteration_v2.py` — `POST /api/v2/evaluations/iterations`, kicks off the automated eval → improve-prompt → eval loop (see Async); returns `202` with an `EvaluationIterationRunImmediatePublic` handle, final round-by-round report delivered to `callback_url` +- `api/routes/evaluations/v2/evaluation.py` — `POST /api/v2/evaluations`, replica of v1 run trigger + native ground-truth LLM judge (Langfuse-free); mounted under `settings.API_V2_STR` +- `api/routes/evaluations/v2/dataset.py` — `POST /api/v2/evaluations/datasets`, Langfuse-free dataset upload; stores only the original CSV in S3 and records `duplication_factor` as metadata (rows expanded ×factor at run time, not physically duplicated) +- `api/routes/evaluations/v2/prompt_improvement.py` — `POST /api/v2/evaluations/{evaluation_id}/improve-prompt`, prompt iteration off the three-metric judge results (requires an `is_judge_run` run); same body as v1, returns a recommendation of type `prompt` +- `api/routes/evaluations/v2/iteration.py` — `POST /api/v2/evaluations/iterations`, kicks off the automated eval → improve-prompt → eval loop (see Async); returns `202` with an `EvaluationIterationRunImmediatePublic` handle, final round-by-round report delivered to `callback_url` - `api/routes/stt_evaluations/`, `api/routes/tts_evaluations/` — STT/TTS - `api/routes/cron.py` — batch polling trigger @@ -21,7 +21,7 @@ All paths relative to `backend/app/`. | `stt_sample`, `stt_result` | `models/stt_evaluation.py` | | `tts_result` | `models/tts_evaluation.py` | | `batch_job` (BatchJob) | `models/batch_job.py` | -| `evaluation_iteration_run` (EvaluationIterationRun) | `models/evaluation_iteration.py` | +| `evaluation_iteration_run` (EvaluationIterationRun) | `models/evaluation_iteration.py` — `last_dispatched_at` is the cron's in-flight guard, stamped at kickoff and on every cron resume; NULL only on rows predating migration 084 | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. `callback_url` (nullable): optional webhook set by the v2 run trigger; on terminal transition Kaapi POSTs a slim `APIResponse` snapshot to it (see Async). Not exposed on `EvaluationRunPublic` (no leak). `duplication_factor` (nullable): optional per-run override of the dataset's stored factor, set by the v2 run trigger for S3-only (Langfuse-free) datasets only — rejected with `422` for Langfuse-backed datasets whose items are already physically duplicated; persisted on the run so the fan-out sizing, the chunk re-load (`execute_fast_evaluation_chunk`), and the ai_summary repetition math all use the same effective factor. v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Judge metrics score on an **integer 0–5 stepped scale** (the LLM returns integers 0–5; `crud/evaluations/judge.py::_parse_metric_score` enforces it) with English-only reasoning; scores are stored raw (0–5), so the API structure is unchanged but the value range is 0–5 (cosine on the v1 path stays 0–1). Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, 0–5 cutoffs 2/4 → 0–1 Needs Improvement, 2–3 Needs Refinement, 4–5 Good), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. @@ -30,17 +30,17 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud `EvaluationIterationRun` is a thin bookkeeping row only (`status`, `stop_reason`, `dataset_id`, `config_id`, `initial_config_version`, `callback_url`, `error_message`) — round-by-round state (`round_number`, `current_eval_run_id`, `current_improvement_job_id`, `history`, `best_*`, `consecutive_low_delta_rounds`) lives entirely in the LangGraph checkpoint keyed by `thread_id = str(id)`, not on this table. No FK to `EvaluationRun`/`Job`; those are referenced only inside the checkpoint state. ## Services / CRUD -- `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) +- `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes + graph wiring only), `iteration_state.py` (`EvaluationIterationState`, `advance_round_state` — the pure per-round bookkeeping/stop arithmetic, `build_iteration_report`), `iteration_checkpointer.py` (`get_evaluation_iteration_checkpointer`, psycopg pool) - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py` (orchestration + S3/OpenAI IO only; the pieces below carry the rest), `fast_results.py` (pure per-item S3 unit shapes, usage sums, failure threshold), `fast_chunks.py` (`batch_job` bookkeeping + chunk queries/cleanup), `fast_cosine.py` (pure v1 cosine scoring), `fast_traces.py` (pure `TraceData` building), `judge_stage.py` (run-level v2 judge: `resolve_config_prompt`, `judge_rows` pool, per-metric summary rollup), `judge_prompts.py` (rubric text, moved out of `score.py`), `retry.py` (shared OpenAI retry policy), `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. -- Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. +- Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. - Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. -- Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with `