refactor(evaluation): split modules

backend/app/alembic/versions/085_add_last_dispatched_at_to_evaluation_iteration_run.py

@@ -0,0 +1,43 @@
+"""Add last_dispatched_at to evaluation_iteration_run
+
+Revision ID: 085
+Revises: 084
+Create Date: 2026-09-08 00:00:00.000000
+
+The cron tick used to fan a `resume=True` graph step out to every PROCESSING
+loop unconditionally, so a step slower than the tick interval got a second one
+dispatched on top of it — two workers against the same LangGraph checkpoint
+thread, and a duplicate eval run or improvement job charged for.
+
+New rows are stamped at creation, since kickoff enqueues the first step right
+away and that step needs the same cooldown as every later one. Nullable with no
+backfill: rows that predate this migration read as NULL and get one immediate
+resume, which is the behaviour they had anyway.
+
+No index — the PROCESSING set is small and already indexed on `status`; the
+cooldown comparison happens in Python, mirroring the fast-eval barrier.
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "085"
+down_revision = "084"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "evaluation_iteration_run",
+        sa.Column(
+            "last_dispatched_at",
+            sa.DateTime(),
+            nullable=True,
+            comment="When a graph step was last dispatched (kickoff or cron resume); cron skips rows stamped inside the cooldown",
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("evaluation_iteration_run", "last_dispatched_at")

backend/app/api/main.py

@@ -39,16 +39,7 @@
     assessment as assessment_routes,
 )
 from app.api.routes.assessment import api as assessment_api_routes
-from app.api.routes.evaluations.dataset_v2 import (
-    router as evaluations_dataset_v2_router,
-)
-from app.api.routes.evaluations.evaluation_v2 import router as evaluations_v2_router
-from app.api.routes.evaluations.iteration_v2 import (
-    router as evaluations_iteration_v2_router,
-)
-from app.api.routes.evaluations.prompt_improvement_v2 import (
-    router as evaluations_prompt_improvement_v2_router,
-)
+from app.api.routes.evaluations.v2 import router as evaluations_v2_router
 
 api_router = APIRouter()
 api_router.include_router(analytics.router)
@@ -95,6 +86,3 @@
 api_v2_router = APIRouter()
 api_v2_router.include_router(documents_v2.router)
 api_v2_router.include_router(evaluations_v2_router)
-api_v2_router.include_router(evaluations_dataset_v2_router)
-api_v2_router.include_router(evaluations_prompt_improvement_v2_router)
-api_v2_router.include_router(evaluations_iteration_v2_router)

backend/app/api/routes/assessment/runs.py

@@ -29,8 +29,8 @@
     AssessmentRunCreate,
     AssessmentRunPublic,
     AssessmentRunResponse,
+    AssessmentSubmission,
 )
-from app.models.assessment import AssessmentSubmission
 from app.services.assessment.service import (
     resume_assessment_run as resume_run,
 )

backend/app/api/routes/evaluations/v2/__init__.py

@@ -0,0 +1,15 @@
+from fastapi import APIRouter
+
+from app.api.routes.evaluations.v2 import (
+    dataset,
+    evaluation,
+    iteration,
+    prompt_improvement,
+)
+
+router = APIRouter()
+
+router.include_router(evaluation.router)
+router.include_router(dataset.router)
+router.include_router(prompt_improvement.router)
+router.include_router(iteration.router)

backend/app/api/routes/evaluations/dataset_v2.py → backend/app/api/routes/evaluations/v2/dataset.py

@@ -23,7 +23,7 @@
 
 @router.post(
     "",
-    description=load_description("evaluation/create_evaluation_dataset_v2.md"),
+    description=load_description("evaluation/v2/create_evaluation_dataset.md"),
     response_model=APIResponse[DatasetUploadResponse],
     dependencies=[
         Depends(require_permission(Permission.REQUIRE_PROJECT)),

backend/app/api/routes/evaluations/evaluation_v2.py → backend/app/api/routes/evaluations/v2/evaluation.py

@@ -21,7 +21,7 @@
 
 @router.post(
     "",
-    description=load_description("evaluation/create_evaluation_v2.md"),
+    description=load_description("evaluation/v2/create_evaluation.md"),
     response_model=APIResponse[EvaluationRunPublic],
     dependencies=[
         Depends(require_permission(Permission.REQUIRE_PROJECT)),

backend/app/api/routes/evaluations/iteration_v2.py → backend/app/api/routes/evaluations/v2/iteration.py

@@ -22,15 +22,15 @@
 
 @router.post(
     "/iterations",
-    description=load_description("evaluation/create_evaluation_iteration_v2.md"),
+    description=load_description("evaluation/v2/create_evaluation_iteration.md"),
     response_model=APIResponse[EvaluationIterationRunImmediatePublic],
     status_code=202,
     dependencies=[
         Depends(require_permission(Permission.REQUIRE_PROJECT)),
         Depends(monitor_rate("evaluations")),
     ],
 )
-def create_evaluation_iteration_v2(
+def create_evaluation_iteration(
     session: SessionDep,
     auth_context: AuthContextDep,
     request: EvaluationIterationCreateRequest,

backend/app/api/routes/evaluations/prompt_improvement_v2.py → backend/app/api/routes/evaluations/v2/prompt_improvement.py

@@ -22,7 +22,7 @@
 
 @router.post(
     "/{evaluation_id}/improve-prompt",
-    description=load_description("evaluation/improve_prompt_v2.md"),
+    description=load_description("evaluation/v2/improve_prompt.md"),
     response_model=APIResponse[LLMJobImmediatePublic],
     status_code=202,
     dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],

backend/app/core/config.py

@@ -222,6 +222,10 @@ def AWS_S3_BUCKET(self) -> str:
     EVAL_ITERATION_CEILING_DELTA_THRESHOLD: float = 0.05
     # Consecutive low-delta rounds required before the loop stops as ceiling_reached.
     EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS: int = 3
+    # Cron skips loops dispatched within this window
+    EVAL_ITERATION_DISPATCH_COOLDOWN_MINUTES: int = 10
+    # Loops stalled longer are reaped; must exceed 25 hard-cap rounds worst-case time
+    EVAL_ITERATION_STALL_THRESHOLD_HOURS: int = 48
 
     EVAL_JUDGE_MODEL: str = "gpt-5.6-luna"
 

backend/app/crud/assessment/batch.py

@@ -27,11 +27,6 @@
 from app.models.batch_job import BatchJob, BatchJobType
 from app.models.llm.constants import DEFAULT_ASSESSMENT_BATCH_MAX_TOKENS
 from app.models.llm.request import ConfigBlob
-from app.services.llm.mappers import (
-    map_kaapi_to_anthropic_params,
-    map_kaapi_to_google_params,
-    map_kaapi_to_openai_params,
-)
 from app.services.assessment.utils.attachments import (
     attachment_type_for_row,
     build_anthropic_attachment_parts,
@@ -44,7 +39,12 @@
     normalize_llm_text,
     parse_rows,
 )
-from app.services.llm.mappers import kaapi_params_as_dict
+from app.services.llm.mappers import (
+    kaapi_params_as_dict,
+    map_kaapi_to_anthropic_params,
+    map_kaapi_to_google_params,
+    map_kaapi_to_openai_params,
+)
 from app.services.llm.providers.registry import LLMProvider
 from app.utils import get_anthropic_client, get_openai_client
 

backend/app/crud/evaluations/__init__.py

@@ -25,13 +25,12 @@
     calculate_cosine_similarity,
     start_embedding_batch,
 )
-from app.crud.evaluations.fast import (
+from app.crud.evaluations.fast import run_fast_evaluation, run_response_chunk
+from app.crud.evaluations.fast_chunks import (
     JOB_TYPE_EMBEDDING_FAST,
     JOB_TYPE_EVALUATION_FAST,
     JOB_TYPE_EVALUATION_FAST_CHUNK,
     list_response_chunk_jobs,
-    run_fast_evaluation,
-    run_response_chunk,
 )
 from app.crud.evaluations.iteration import (
     create_evaluation_iteration_run,

backend/app/crud/evaluations/core.py

@@ -23,6 +23,14 @@
 logger = logging.getLogger(__name__)
 
 
+def build_log_prefix(eval_run: EvaluationRun) -> str:
+    return (
+        f"[org={eval_run.organization_id}]"
+        f"[project={eval_run.project_id}]"
+        f"[eval={eval_run.id}]"
+    )
+
+
 def resolve_evaluation_config(
     session: Session,
     config_id: UUID,
@@ -65,6 +73,11 @@ def create_evaluation_run(
     organization_id: int,
     project_id: int,
     run_mode: RunModeEnum = RunModeEnum.BATCH,
+    is_judge_run: bool = False,
+    callback_url: str | None = None,
+    duplication_factor: int | None = None,
+    total_items: int | None = None,
+    status: str = "pending",
 ) -> EvaluationRun:
     """
     Create a new evaluation run record in the database.
@@ -79,6 +92,11 @@ def create_evaluation_run(
         organization_id: Organization ID
         project_id: Project ID
         run_mode: Execution mode (RunModeEnum.BATCH default, or RunModeEnum.FAST)
+        is_judge_run: v2-only; default False, persisted as NULL (v1 marker convention)
+        callback_url: v2-only terminal-state webhook; default None
+        duplication_factor: v2-only per-run override; default None (use dataset's)
+        total_items: v2-only pre-known item count; default None, stored as 0
+        status: Initial run status; defaults to "pending"
 
     Returns:
         The created EvaluationRun instance
@@ -90,10 +108,15 @@ def create_evaluation_run(
         type=EvaluationType.TEXT.value,
         config_id=config_id,
         config_version=config_version,
-        status="pending",
+        status=status,
         run_mode=run_mode,
         organization_id=organization_id,
         project_id=project_id,
+        # False is stored as NULL so v1 rows stay indistinguishable from pre-v2 ones
+        is_judge_run=is_judge_run or None,
+        callback_url=callback_url,
+        duplication_factor=duplication_factor,
+        total_items=total_items or 0,
         inserted_at=now(),
         updated_at=now(),
     )