From 6065380ef1242e76807fea261fd18ae6516f0af5 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Thu, 17 Sep 2026 16:31:12 +0530 Subject: [PATCH] fix(evaluation): force file_search so KB metric scores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fast eval runs left tool_choice at auto. Models that decline to call file_search — gpt-4.1 in practice — return no chunks, so the knowledge_base metric is dropped on every row and "Adherence to Knowledge Base" comes back blank for the whole run. Force the tool wherever we already request file_search_call.results. The metric now measures groundedness given retrieval instead of the model's willingness to retrieve, so scores shift on every model. Co-Authored-By: Claude Opus 5 (1M context) --- backend/app/crud/evaluations/fast.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 734816f7d..264dfdb56 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -505,10 +505,13 @@ def run_response_chunk( ) # Ask OpenAI to return the file_search hits so knowledge_base can judge them. - # tool_choice stays at the model default (auto) — consistent with normal calls; - # a row where the model doesn't query the KB is scored N/A, not forced to search. + # tool_choice is forced rather than left at auto: a model that declines to search + # (gpt-4.1 usually does) returns no chunks, which drops the knowledge_base metric + # for every row and leaves the score blank run-wide. Forcing the call makes the + # metric measure groundedness given retrieval, not eagerness to retrieve. if any(t.get("type") == "file_search" for t in base_params.get("tools", [])): base_params["include"] = ["file_search_call.results"] + base_params["tool_choice"] = "required" results: list[dict[str, Any]] = [] max_workers = max(