From ca80e7c64b6c4f0edb4edc7aba91136c58e117c6 Mon Sep 17 00:00:00 2001 From: Mandark-droid Date: Fri, 7 Aug 2026 17:04:29 +0530 Subject: [PATCH 1/7] feat: add lighteval converter Converts lighteval results_*.json into EvaluationLogs, one per measured task, following the lm_eval converter layout. Handles the two traps in lighteval's `results` mapping: `{metric}_stderr` lives in the same dict as its metric (attached as uncertainty, never emitted as a metric), and the mapping also holds rows lighteval averaged itself (`:_average|` and `all`), which are skipped and recorded rather than emitted as siblings of their own parts. --- AGENTS.md | 2 +- README.md | 5 +- every_eval_ever/cli.py | 74 ++- every_eval_ever/converters/README.md | 73 +++ every_eval_ever/converters/common/adapter.py | 1 + .../converters/lighteval/__init__.py | 1 + .../converters/lighteval/__main__.py | 91 +++ .../converters/lighteval/adapter.py | 532 ++++++++++++++++++ every_eval_ever/converters/lighteval/utils.py | 195 +++++++ .../results_2026-01-21T03-44-18.458309.json | 229 ++++++++ tests/test_lighteval_adapter.py | 434 ++++++++++++++ 11 files changed, 1633 insertions(+), 4 deletions(-) create mode 100644 every_eval_ever/converters/lighteval/__init__.py create mode 100644 every_eval_ever/converters/lighteval/__main__.py create mode 100644 every_eval_ever/converters/lighteval/adapter.py create mode 100644 every_eval_ever/converters/lighteval/utils.py create mode 100644 tests/data/lighteval/results/HuggingFaceTB/SmolLM2-1.7B-Instruct/results_2026-01-21T03-44-18.458309.json create mode 100644 tests/test_lighteval_adapter.py diff --git a/AGENTS.md b/AGENTS.md index 68d781b76..912458bbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ convert external eval sources into it. - `every_eval_ever/eval_types.py` + `eval.schema.json` — aggregate `EvaluationLog`. - `every_eval_ever/instance_level_types.py` + `instance_level_eval.schema.json` — instance log. - `every_eval_ever/adapters//adapter.py` — one-off source adapters (run via `uv run python -m every_eval_ever.adapters..adapter`). -- `every_eval_ever/converters/` — in-tree converters (`inspect`/`helm`/`lm_eval`, plus `alpaca_eval`; shared code in `common`), run via `python -m every_eval_ever convert ...`. +- `every_eval_ever/converters/` — in-tree converters (`inspect`/`helm`/`lm_eval`/`lighteval`, plus `alpaca_eval`; shared code in `common`), run via `python -m every_eval_ever convert ...`. - `every_eval_ever/validator/` — the schema + **semantic** merge gate (path shape, UUID4 names, companion pairing, score bounds, deployment axes). `REGISTERED_CHECKS` is the list. - Validate: `python -m every_eval_ever validate ` (`.json`→aggregate, diff --git a/README.md b/README.md index d9680743d..217f90085 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - 📋 **A metadata schema** ([`eval.schema.json`](every_eval_ever/schemas/eval.schema.json)) that defines the information needed for meaningful comparison of evaluation results, including [instance-level data](every_eval_ever/schemas/instance_level_eval.schema.json) - 🔧 **Validation** that checks data against the schema before it enters the repository -- 🔌 **Converters** for [Inspect AI](every_eval_ever/converters/inspect/), [HELM](every_eval_ever/converters/helm/), and [lm-eval-harness](every_eval_ever/converters/lm_eval/), so you can transform your existing evaluation logs into the standard format +- 🔌 **Converters** for [Inspect AI](every_eval_ever/converters/inspect/), [HELM](every_eval_ever/converters/helm/), [lm-eval-harness](every_eval_ever/converters/lm_eval/), and [lighteval](every_eval_ever/converters/lighteval/), so you can transform your existing evaluation logs into the standard format Install the package: @@ -260,13 +260,14 @@ quietly telling the next contributor something untrue. ## 🔌 Eval Converters -We have prepared converters to make adapting to our schema as easy as possible. At the moment, we support converting local evaluation harness logs from `Inspect AI`, `HELM` and `lm-evaluation-harness` into our unified schema. Each converter produces aggregate JSON and optionally instance-level JSONL output. +We have prepared converters to make adapting to our schema as easy as possible. At the moment, we support converting local evaluation harness logs from `Inspect AI`, `HELM`, `lm-evaluation-harness` and `lighteval` into our unified schema. Each converter produces aggregate JSON and optionally instance-level JSONL output. | Framework | Command | Instance-Level JSONL | |---|---|---| | [Inspect AI](every_eval_ever/converters/inspect/) | `every_eval_ever convert inspect --log_path ` | Yes, if samples in log | | [HELM](every_eval_ever/converters/helm/) | `every_eval_ever convert helm --log_path ` | Always | | [lm-evaluation-harness](every_eval_ever/converters/lm_eval/) | `every_eval_ever convert lm_eval --log_path --include_samples` | With `--include_samples` | +| [lighteval](every_eval_ever/converters/lighteval/) | `every_eval_ever convert lighteval --log_path ` | Not yet | For full CLI usage and required input files, see the [Eval Converters README](every_eval_ever/converters/README.md). diff --git a/every_eval_ever/cli.py b/every_eval_ever/cli.py index 32d7901a3..f9b0df465 100644 --- a/every_eval_ever/cli.py +++ b/every_eval_ever/cli.py @@ -215,6 +215,54 @@ def _cmd_convert_lm_eval(args: argparse.Namespace) -> int: return 0 +def _cmd_convert_lighteval(args: argparse.Namespace) -> int: + from every_eval_ever.converters.lighteval.adapter import LightevalAdapter + + adapter = LightevalAdapter() + metadata = _common_metadata(args) + for name in ( + 'inference_platform', + 'inference_engine', + 'inference_engine_version', + ): + value = getattr(args, name, None) + if value: + metadata[name] = value + + log_path = Path(args.log_path) + input_result: SourceConversionResult[Any] | None = None + if log_path.is_file(): + logs = adapter.transform_from_file(log_path, metadata) + elif log_path.is_dir(): + input_result = adapter.transform_from_directory_result( + log_path, metadata + ) + logs = input_result.records + else: + raise FileNotFoundError(f'Path is not a file or directory: {log_path}') + + if not logs and input_result is None: + raise ValueError( + f'lighteval conversion produced no logs from {log_path}' + ) + + output_dir = Path(args.output_dir) + eval_uuids = [str(uuid.uuid4()) for _ in logs] + paths = ( + publish_evaluation_logs(logs, output_dir, eval_uuids) if logs else [] + ) + for path in paths: + print(path) + + _save_partial_conversion_report( + input_result, output_dir, 'lighteval_inputs' + ) + if input_result is not None: + input_result.raise_if_incomplete() + print(f'Converted {len(paths)} evaluation log(s).') + return 0 + + def _cmd_convert_inspect(args: argparse.Namespace) -> int: from every_eval_ever.converters.inspect.adapter import ( InspectAIAdapter, @@ -535,7 +583,7 @@ def build_parser() -> argparse.ArgumentParser: dest='source', required=True ) - for source in ['lm_eval', 'inspect', 'helm', 'alpaca_eval']: + for source in ['lm_eval', 'inspect', 'helm', 'alpaca_eval', 'lighteval']: source_parser = convert_subparsers.add_parser( source, help=f'Convert {source} logs', @@ -621,6 +669,28 @@ def build_parser() -> argparse.ArgumentParser: default=None, help='Inference engine version to record in model_info.inference_engine.version.', ) + if source == 'lighteval': + source_parser.add_argument( + '--inference_platform', + '--inference-platform', + default=None, + help='Inference platform to record when the model config does ' + 'not name one (e.g. together, openai).', + ) + source_parser.add_argument( + '--inference_engine', + '--inference-engine', + default=None, + help='Inference engine to record. lighteval dumps its model ' + 'config without a backend discriminator, so this cannot be ' + 'read from the logs.', + ) + source_parser.add_argument( + '--inference_engine_version', + '--inference-engine-version', + default=None, + help='Inference engine version to record in model_info.inference_engine.version.', + ) if source == 'inspect': source_parser.add_argument( '--supplemental_eval_details_path', @@ -667,6 +737,8 @@ def main(argv: list[str] | None = None) -> int: return _cmd_convert_helm(args) if args.source == 'alpaca_eval': return _cmd_convert_alpaca_eval(args) + if args.source == 'lighteval': + return _cmd_convert_lighteval(args) parser.print_help() return 1 diff --git a/every_eval_ever/converters/README.md b/every_eval_ever/converters/README.md index ad4bbdacf..90147cd87 100644 --- a/every_eval_ever/converters/README.md +++ b/every_eval_ever/converters/README.md @@ -217,6 +217,79 @@ options: Version of the evaluation library ``` +## lighteval + +The conversion script from `lighteval` evaluation logs to the unified schema can be run using `every_eval_ever/converters/lighteval/__main__.py`. + +lighteval writes one results file per run, at +`{output_dir}/results/{model_name}/results_{date_id}.json`. Point `--log_path` at +a single file, or at a directory to pick up every `results_*.json` beneath it. + +```bash +uv run every_eval_ever convert lighteval --log_path tests/data/lighteval +``` + +Two things about lighteval's `results` mapping are worth knowing before you read +the output: + +- It holds a `{metric}_stderr` entry beside the metric it belongs to. The + converter attaches that as `score_details.uncertainty`, never as a metric of + its own, and omits it when lighteval wrote `NaN`. +- Alongside the tasks it measured, it holds rows lighteval averaged itself: a + per-parent mean under `:_average|` and a mean over everything + under `all`. Those are not converted; the keys that were skipped are recorded + in `source_metadata.additional_details.lighteval_derived_rows_not_converted`. + +Full manual for conversion of your own lighteval evaluation log into unified is available below: + +```bash +usage: __main__.py [-h] --log_path LOG_PATH [--output_dir OUTPUT_DIR] + [--source_organization_name SOURCE_ORGANIZATION_NAME] + [--evaluator_relationship {first_party,third_party,collaborative,other}] + [--source_organization_url SOURCE_ORGANIZATION_URL] + [--source_organization_logo_url SOURCE_ORGANIZATION_LOGO_URL] + [--inference_platform INFERENCE_PLATFORM] + [--inference_engine INFERENCE_ENGINE] + [--inference_engine_version INFERENCE_ENGINE_VERSION] + [--eval_library_name EVAL_LIBRARY_NAME] + [--eval_library_version EVAL_LIBRARY_VERSION] + +Convert lighteval output to every_eval_ever format + +options: + -h, --help show this help message and exit + --log_path LOG_PATH Path to a results JSON file or a directory containing + results files + --output_dir OUTPUT_DIR + Output directory for converted files + --source_organization_name SOURCE_ORGANIZATION_NAME + Name of the organization that ran the evaluation + --evaluator_relationship {first_party,third_party,collaborative,other} + Relationship of the evaluator to the model + --source_organization_url SOURCE_ORGANIZATION_URL + URL of the source organization + --source_organization_logo_url SOURCE_ORGANIZATION_LOGO_URL + Logo of the source organization + --inference_platform INFERENCE_PLATFORM + Inference platform (e.g. 'together', 'openai'). Read + from the model config for LiteLLM and inference- + provider runs; must be provided manually otherwise. + --inference_engine INFERENCE_ENGINE + Inference engine name (e.g. 'vllm', 'transformers'). + lighteval dumps its model config without a backend + discriminator, so this cannot be read from the logs. + --inference_engine_version INFERENCE_ENGINE_VERSION + Inference engine version (e.g. '0.6.0'). Not available + from lighteval logs, so must be provided manually. + --eval_library_name EVAL_LIBRARY_NAME + Name of the evaluation library (e.g. inspect_ai, + lm_eval, helm) + --eval_library_version EVAL_LIBRARY_VERSION + Version of the evaluation library. lighteval records a + git SHA rather than a version, and writes '?' outside + a git checkout. +``` + ## AlpacaEval The AlpacaEval converter fetches the public leaderboard CSV directly from GitHub diff --git a/every_eval_ever/converters/common/adapter.py b/every_eval_ever/converters/common/adapter.py index 79110bc2b..787dec231 100644 --- a/every_eval_ever/converters/common/adapter.py +++ b/every_eval_ever/converters/common/adapter.py @@ -29,6 +29,7 @@ class SupportedLibrary(Enum): LM_EVAL = 'lm-evaluation-harness' INSPECT_AI = 'inspect-ai' HELM = 'helm' + LIGHTEVAL = 'lighteval' CUSTOM = 'custom' diff --git a/every_eval_ever/converters/lighteval/__init__.py b/every_eval_ever/converters/lighteval/__init__.py new file mode 100644 index 000000000..57fc16811 --- /dev/null +++ b/every_eval_ever/converters/lighteval/__init__.py @@ -0,0 +1 @@ +"""lighteval adapter for every_eval_ever.""" diff --git a/every_eval_ever/converters/lighteval/__main__.py b/every_eval_ever/converters/lighteval/__main__.py new file mode 100644 index 000000000..9503568b1 --- /dev/null +++ b/every_eval_ever/converters/lighteval/__main__.py @@ -0,0 +1,91 @@ +"""CLI for converting lighteval output to every_eval_ever format.""" + +import argparse + + +def main(): + parser = argparse.ArgumentParser( + description='Convert lighteval output to every_eval_ever format' + ) + parser.add_argument( + '--log_path', + type=str, + required=True, + help='Path to a results JSON file or a directory containing results files', + ) + parser.add_argument( + '--output_dir', + type=str, + default='data', + help='Output directory for converted files', + ) + parser.add_argument( + '--source_organization_name', + type=str, + default='', + help='Name of the organization that ran the evaluation', + ) + parser.add_argument( + '--evaluator_relationship', + type=str, + default='first_party', + choices=['first_party', 'third_party', 'collaborative', 'other'], + help='Relationship of the evaluator to the model', + ) + parser.add_argument( + '--source_organization_url', + type=str, + default=None, + help='URL of the source organization', + ) + parser.add_argument( + '--source_organization_logo_url', + type=str, + default=None, + help='Logo of the source organization', + ) + parser.add_argument( + '--inference_platform', + type=str, + default=None, + help="Inference platform (e.g. 'together', 'openai'). Read from the " + 'model config for LiteLLM and inference-provider runs; must be ' + 'provided manually otherwise.', + ) + parser.add_argument( + '--inference_engine', + type=str, + default=None, + help="Inference engine name (e.g. 'vllm', 'transformers'). lighteval " + 'dumps its model config without a backend discriminator, so this ' + 'cannot be read from the logs.', + ) + parser.add_argument( + '--inference_engine_version', + type=str, + default=None, + help="Inference engine version (e.g. '0.6.0'). " + 'Not available from lighteval logs, so must be provided manually.', + ) + parser.add_argument( + '--eval_library_name', + type=str, + default='lighteval', + help='Name of the evaluation library (e.g. inspect_ai, lm_eval, helm)', + ) + parser.add_argument( + '--eval_library_version', + type=str, + default='unknown', + help='Version of the evaluation library. lighteval records a git SHA ' + "rather than a version, and writes '?' outside a git checkout.", + ) + + args = parser.parse_args() + from every_eval_ever.cli import _cmd_convert_lighteval + + return _cmd_convert_lighteval(args) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/every_eval_ever/converters/lighteval/adapter.py b/every_eval_ever/converters/lighteval/adapter.py new file mode 100644 index 000000000..86c55e0c7 --- /dev/null +++ b/every_eval_ever/converters/lighteval/adapter.py @@ -0,0 +1,532 @@ +"""Adapter for converting lighteval output to every_eval_ever format.""" + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from every_eval_ever.converters import SCHEMA_VERSION +from every_eval_ever.converters.common.adapter import ( + AdapterMetadata, + BaseEvaluationAdapter, + SupportedLibrary, +) +from every_eval_ever.converters.common.utils import get_current_unix_timestamp +from every_eval_ever.eval_types import ( + EvalLibrary, + EvaluationLog, + EvaluationResult, + EvaluatorRelationship, + GenerationArgs, + GenerationConfig, + InferenceEngine, + MetricConfig, + ModelInfo, + ScoreDetails, + ScoreType, + SourceDataHf, + SourceDataPrivate, + SourceMetadata, + SourceType, + StandardError, + Uncertainty, +) +from every_eval_ever.helpers.io import ( + SourceConversionResult, + SourceRecordFailure, +) + +from .utils import ( + KNOWN_METRIC_BOUNDS, + STDERR_SUFFIX, + find_metric_spec, + flatten_model_config, + higher_is_better_for, + is_derived_aggregate_key, + is_finite_number, + parse_results_file_timestamp, + split_task_key, + stderr_method_for, +) + +# lighteval writes "?" when it cannot read its own git SHA, which is the normal +# case for a pip install. +_UNKNOWN_SHA = '?' + + +class LightevalAdapter(BaseEvaluationAdapter): + """Converts lighteval results files to every_eval_ever format.""" + + def __init__(self, strict_validation: bool = True): + super().__init__(strict_validation) + # Stores per-log metadata so callers can find the source file after + # transform. Keyed by evaluation_id -> {"parent_dir": str, + # "task_key": str} + self._eval_metadata = {} + + def get_eval_metadata(self, evaluation_id: str) -> Dict[str, Any]: + """Return stored metadata for a given evaluation_id.""" + return self._eval_metadata.get(evaluation_id, {}) + + @property + def metadata(self) -> AdapterMetadata: + return AdapterMetadata( + name='lighteval-adapter', + version='0.1.0', + supported_library_versions=['0.13.*'], + description='Converts lighteval output to every_eval_ever format', + ) + + @property + def supported_library(self) -> SupportedLibrary: + return SupportedLibrary.LIGHTEVAL + + def _extract_model_info( + self, + raw_data: Dict[str, Any], + metadata_args: Optional[Dict[str, Any]] = None, + ) -> ModelInfo: + """Extract model information from a lighteval results file.""" + metadata_args = metadata_args or {} + config_general = raw_data.get('config_general') or {} + model_config = config_general.get('model_config') + if not isinstance(model_config, dict): + model_config = {} + + model_name = config_general.get('model_name') or model_config.get( + 'model_name' + ) + if not model_name: + raise ValueError( + 'lighteval results file has no config_general.model_name' + ) + + developer = None + if '/' in model_name: + developer = model_name.split('/')[0] + + # The dumped model config carries no discriminator for the backend that + # produced it, so the engine cannot be read off the file. + engine_name = metadata_args.get('inference_engine') + engine_version = metadata_args.get('inference_engine_version') + inference_engine = None + if engine_name: + inference_engine = InferenceEngine( + name=engine_name, version=engine_version + ) + + # LiteLLM and inference-provider runs state their platform outright. + inference_platform = model_config.get('provider') or metadata_args.get( + 'inference_platform' + ) + + additional, redacted = flatten_model_config(model_config) + if redacted: + additional['redacted_model_config_keys'] = ','.join(redacted) + + return ModelInfo( + name=model_name, + id=model_name, + developer=developer, + inference_platform=inference_platform, + inference_engine=inference_engine, + additional_details=additional or None, + ) + + def _get_tasks(self, raw_data: Dict[str, Any]) -> List[str]: + """Get the keys of `results` that lighteval measured rather than averaged.""" + results = raw_data.get('results') or {} + tasks = [] + for task_key, task_results in results.items(): + if is_derived_aggregate_key(task_key): + continue + if not isinstance(task_results, dict): + continue + if any( + self._is_metric_entry(key, value) + for key, value in task_results.items() + ): + tasks.append(task_key) + return tasks + + @staticmethod + def _is_metric_entry(key: str, value: Any) -> bool: + """Report whether one `results` entry is a usable metric score.""" + if key.endswith(STDERR_SUFFIX): + return False + return is_finite_number(value) + + def _build_source_data(self, task_config: Dict[str, Any], task_name: str): + """Build source_data from a lighteval task config.""" + dataset_name = task_config.get('name') or task_name + hf_repo = task_config.get('hf_repo') + if not hf_repo: + return SourceDataPrivate( + dataset_name=dataset_name, + source_type='other', + ) + + evaluation_splits = task_config.get('evaluation_splits') or [] + additional = {} + if task_config.get('hf_subset'): + additional['hf_subset'] = str(task_config['hf_subset']) + if task_config.get('hf_revision'): + additional['hf_revision'] = str(task_config['hf_revision']) + if len(evaluation_splits) > 1: + additional['evaluation_splits'] = json.dumps( + list(evaluation_splits) + ) + + original_num_docs = task_config.get('original_num_docs') + return SourceDataHf( + dataset_name=dataset_name, + source_type='hf_dataset', + hf_repo=hf_repo, + hf_split=evaluation_splits[0] if evaluation_splits else None, + samples_number=( + original_num_docs + if isinstance(original_num_docs, int) and original_num_docs >= 0 + else None + ), + additional_details=additional or None, + ) + + def _build_generation_config( + self, + task_config: Dict[str, Any], + generation_parameters: Dict[str, Any], + num_fewshots: Optional[int], + ) -> Optional[GenerationConfig]: + """Build generation config from the run's model and task configs.""" + generation_size = task_config.get('generation_size') + max_tokens = generation_parameters.get('max_new_tokens') + if not isinstance(max_tokens, int) or max_tokens < 1: + max_tokens = ( + generation_size + if isinstance(generation_size, int) and generation_size >= 1 + else None + ) + + args = GenerationArgs( + temperature=generation_parameters.get('temperature'), + top_p=generation_parameters.get('top_p'), + top_k=generation_parameters.get('top_k'), + max_tokens=max_tokens, + ) + + additional = {} + for key, value in generation_parameters.items(): + if key in ('temperature', 'top_p', 'top_k', 'max_new_tokens'): + continue + if value is None: + continue + additional[key] = ( + value if isinstance(value, str) else json.dumps(value) + ) + if num_fewshots is not None: + additional['num_fewshots'] = str(num_fewshots) + if task_config.get('stop_sequence'): + additional['stop_sequence'] = json.dumps( + list(task_config['stop_sequence']) + ) + if task_config.get('few_shots_select'): + additional['few_shots_select'] = str( + task_config['few_shots_select'] + ) + + stated_args = { + args.temperature, + args.top_p, + args.top_k, + args.max_tokens, + } - {None} + if not stated_args and not additional: + return None + return GenerationConfig( + generation_args=args, + additional_details=additional or None, + ) + + def _build_evaluation_results( + self, + raw_data: Dict[str, Any], + task_key: str, + evaluation_timestamp: Optional[str] = None, + ) -> List[EvaluationResult]: + """Build the EvaluationResult list for a single lighteval task.""" + task_results = raw_data['results'][task_key] + task_name, num_fewshots = split_task_key(task_key) + task_config = (raw_data.get('config_tasks') or {}).get(task_key) or {} + config_general = raw_data.get('config_general') or {} + model_config = config_general.get('model_config') + generation_parameters = {} + if isinstance(model_config, dict): + declared = model_config.get('generation_parameters') + if isinstance(declared, dict): + generation_parameters = declared + + source_data = self._build_source_data(task_config, task_name) + gen_config = self._build_generation_config( + task_config, generation_parameters, num_fewshots + ) + effective_num_docs = task_config.get('effective_num_docs') + num_samples = ( + effective_num_docs + if isinstance(effective_num_docs, int) and effective_num_docs >= 0 + else None + ) + + results = [] + for metric_name, value in task_results.items(): + if not self._is_metric_entry(metric_name, value): + continue + + metric_spec = find_metric_spec(task_config, metric_name) + higher_is_better = higher_is_better_for(metric_spec, metric_name) + + metric_details = {} + if higher_is_better is None: + # EEE requires a direction; record that the run did not give one. + metric_details['direction_status'] = 'assumed_higher_is_better' + higher_is_better = True + + bounds = KNOWN_METRIC_BOUNDS.get(metric_name) + if bounds is None: + # Preserve metrics whose mathematical range is not yet known + # without falsely declaring them continuous and unbounded. + metric_details['bounds_status'] = 'unknown' + metric_config = MetricConfig( + evaluation_description=metric_name, + metric_name=metric_name, + lower_is_better=not higher_is_better, + additional_details=metric_details, + ) + else: + metric_config = MetricConfig( + evaluation_description=metric_name, + metric_name=metric_name, + lower_is_better=not higher_is_better, + score_type=ScoreType.continuous, + min_score=bounds[0], + max_score=bounds[1], + additional_details=metric_details or None, + ) + + # lighteval writes stderr into the same dict as the metric it + # belongs to, and sets it to NaN when the estimate overflowed. + stderr_value = task_results.get(f'{metric_name}{STDERR_SUFFIX}') + if not is_finite_number(stderr_value): + stderr_value = None + + uncertainty = None + if stderr_value is not None or num_samples is not None: + uncertainty = Uncertainty( + standard_error=( + StandardError( + value=stderr_value, + method=stderr_method_for(metric_spec, metric_name), + ) + if stderr_value is not None + else None + ), + num_samples=num_samples, + ) + + results.append( + EvaluationResult( + evaluation_name=task_key, + source_data=source_data, + evaluation_timestamp=evaluation_timestamp, + metric_config=metric_config, + score_details=ScoreDetails( + score=value, + uncertainty=uncertainty, + ), + generation_config=gen_config, + ) + ) + + return results + + def _count_dropped_scores( + self, raw_data: Dict[str, Any], task_key: str + ) -> int: + """Count metrics skipped because their aggregated score was not finite.""" + task_results = raw_data['results'][task_key] + return sum( + 1 + for key, value in task_results.items() + if not key.endswith(STDERR_SUFFIX) and not is_finite_number(value) + ) + + def _transform_single( + self, raw_data: Dict[str, Any], metadata_args: Dict[str, Any] + ) -> EvaluationLog: + """Transform a single lighteval task's results into an EvaluationLog. + + Expects metadata_args to contain 'task_key' specifying which task. + """ + task_key = metadata_args['task_key'] + model_info = self._extract_model_info(raw_data, metadata_args) + config_general = raw_data.get('config_general') or {} + + retrieved_timestamp = get_current_unix_timestamp() + eval_timestamp = metadata_args.get('evaluation_timestamp') + + evaluation_id = f'{task_key}/{model_info.id}/{retrieved_timestamp}' + evaluation_results = self._build_evaluation_results( + raw_data, task_key, eval_timestamp + ) + if not evaluation_results: + raise ValueError( + f'lighteval task {task_key!r} has no finite metric scores' + ) + + evaluator_rel_str = metadata_args.get( + 'evaluator_relationship', 'first_party' + ) + evaluator_relationship = EvaluatorRelationship(evaluator_rel_str) + + eval_library_details = {} + lighteval_sha = config_general.get('lighteval_sha') + if lighteval_sha and lighteval_sha != _UNKNOWN_SHA: + eval_library_details['lighteval_sha'] = str(lighteval_sha) + eval_library = EvalLibrary( + name=metadata_args.get('eval_library_name', 'lighteval'), + version=metadata_args.get('eval_library_version', 'unknown'), + additional_details=eval_library_details or None, + ) + + source_details = {} + unknown_bounds_count = sum( + result.metric_config.additional_details is not None + and result.metric_config.additional_details.get('bounds_status') + == 'unknown' + for result in evaluation_results + ) + if unknown_bounds_count: + source_details['metrics_with_unknown_bounds'] = str( + unknown_bounds_count + ) + dropped_scores = self._count_dropped_scores(raw_data, task_key) + if dropped_scores: + source_details['metrics_dropped_non_finite'] = str(dropped_scores) + derived_rows = metadata_args.get('derived_aggregate_keys') or [] + if derived_rows: + source_details['lighteval_derived_rows_not_converted'] = ','.join( + derived_rows + ) + elapsed = config_general.get('total_evaluation_time_secondes') + if elapsed is not None: + source_details['total_evaluation_time_seconds'] = str(elapsed) + for key in ('job_id', 'max_samples', 'num_fewshot_seeds'): + if config_general.get(key) is not None: + source_details[key] = str(config_general[key]) + + source_metadata = SourceMetadata( + source_name='lighteval', + source_type=SourceType.evaluation_run, + source_organization_name=metadata_args.get( + 'source_organization_name', '' + ), + source_organization_url=metadata_args.get( + 'source_organization_url' + ), + source_organization_logo_url=metadata_args.get( + 'source_organization_logo_url' + ), + evaluator_relationship=evaluator_relationship, + additional_details=source_details or None, + ) + + # Store metadata so callers can trace a log back to its results file + self._eval_metadata[evaluation_id] = { + 'parent_dir': metadata_args.get('parent_eval_output_dir'), + 'task_key': task_key, + } + + return EvaluationLog( + schema_version=SCHEMA_VERSION, + evaluation_id=evaluation_id, + retrieved_timestamp=retrieved_timestamp, + evaluation_timestamp=eval_timestamp, + source_metadata=source_metadata, + eval_library=eval_library, + model_info=model_info, + evaluation_results=evaluation_results, + ) + + def transform_from_file( + self, file_path: Union[str, Path], metadata_args: Dict[str, Any] + ) -> List[EvaluationLog]: + """Transform a lighteval results JSON file into EvaluationLogs. + + Returns one EvaluationLog per measured task in the results file. + """ + file_path = Path(file_path) + raw_data = self._load_file(file_path) + tasks = self._get_tasks(raw_data) + + derived_keys = sorted( + key + for key in (raw_data.get('results') or {}) + if is_derived_aggregate_key(key) + ) + metadata_args = { + **metadata_args, + 'parent_eval_output_dir': str(file_path.parent), + 'derived_aggregate_keys': derived_keys, + # The results filename holds the only wall-clock time in the run. + 'evaluation_timestamp': parse_results_file_timestamp(file_path), + } + + results = [] + for task_key in tasks: + task_metadata = {**metadata_args, 'task_key': task_key} + results.append(self._transform_single(raw_data, task_metadata)) + + return results + + def transform_from_directory( + self, dir_path: Union[str, Path], metadata_args: Dict[str, Any] + ) -> List[EvaluationLog]: + result = self.transform_from_directory_result(dir_path, metadata_args) + result.raise_if_incomplete() + return result.records + + def transform_from_directory_result( + self, dir_path: Union[str, Path], metadata_args: Dict[str, Any] + ) -> SourceConversionResult[EvaluationLog]: + """Transform all lighteval files while retaining per-file failures. + + Searches for results_*.json files recursively, because lighteval nests + them under results///. + """ + dir_path = Path(dir_path) + results_files = sorted(dir_path.glob('**/results_*.json')) + if not results_files: + raise ValueError( + f'No lighteval results_*.json files found under {dir_path}' + ) + + all_logs: list[EvaluationLog] = [] + failures: list[SourceRecordFailure] = [] + for results_file in results_files: + try: + all_logs.extend( + self.transform_from_file(results_file, metadata_args) + ) + except Exception as exc: + failures.append( + SourceRecordFailure( + source_ref=str(results_file), + reason=str(exc), + source_record={'path': str(results_file)}, + ) + ) + + return SourceConversionResult( + source_name=f'lighteval evaluations under {dir_path}', + total_records=len(all_logs) + len(failures), + records=all_logs, + failures=failures, + ) diff --git a/every_eval_ever/converters/lighteval/utils.py b/every_eval_ever/converters/lighteval/utils.py new file mode 100644 index 000000000..c637fa718 --- /dev/null +++ b/every_eval_ever/converters/lighteval/utils.py @@ -0,0 +1,195 @@ +"""Utility functions for the lighteval adapter.""" + +import json +import math +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +STDERR_SUFFIX = '_stderr' + +# lighteval synthesises two kinds of row into the same `results` mapping it uses +# for measured tasks: a per-parent-task mean under ":_average|" +# and a mean over everything under the literal key "all". +# (MetricsLogger.aggregate in lighteval/logging/info_loggers.py.) +SUITE_AVERAGE_KEY = 'all' +SUBTASK_AVERAGE_SUFFIX = ':_average' + +# Dropped from model_info.additional_details. lighteval dumps the whole model +# config into the results file, and LiteLLMModelConfig.api_key is a plain str, +# so a converted record would otherwise carry a live credential. +SECRET_MODEL_CONFIG_KEYS = frozenset( + { + 'access_token', + 'api_key', + 'api_token', + 'auth_token', + 'credentials', + 'hf_token', + 'password', + 'secret', + 'token', + } +) + +_DATE_ID_PATTERN = re.compile( + r'^results_(?P\d{4}-\d{2}-\d{2}T\d{2})-(?P\d{2})-' + r'(?P\d{2}(?:\.\d+)?)$' +) + +# Known metric bounds: metric_name -> (min_score, max_score). +# Names are lighteval's own metric_name values (lighteval/metrics/metrics.py). +# Infinite bounds are serialized as the JSON strings "Infinity"/"-Infinity". +KNOWN_METRIC_BOUNDS = { + 'acc': (0.0, 1.0), + 'bits_per_byte': (0.0, float('inf')), + 'bleu': (0.0, 100.0), + 'bleu_1': (0.0, 100.0), + 'bleu_4': (0.0, 100.0), + 'byte_perplexity': (1.0, float('inf')), + 'chrf': (0.0, 100.0), + 'chrf++': (0.0, 100.0), + 'em': (0.0, 1.0), + 'extractive_match': (0.0, 1.0), + 'f1': (0.0, 1.0), + 'loglikelihood_f1': (0.0, 1.0), + 'mcc': (-1.0, 1.0), + 'mf1': (0.0, 1.0), + 'mrr': (0.0, 1.0), + 'perplexity': (1.0, float('inf')), + 'ppl': (1.0, float('inf')), + 'recall': (0.0, 1.0), + 'rouge1': (0.0, 1.0), + 'rouge2': (0.0, 1.0), + 'rougeL': (0.0, 1.0), + 'rougeLsum': (0.0, 1.0), + 'summarization_coverage': (0.0, 1.0), + 'ter': (0.0, float('inf')), + 'truthfulqa_mc1': (0.0, 1.0), + 'word_perplexity': (1.0, float('inf')), +} + + +def is_derived_aggregate_key(task_key: str) -> bool: + """Report whether a `results` key was averaged by lighteval, not measured.""" + if task_key == SUITE_AVERAGE_KEY: + return True + return task_key.split('|')[0].endswith(SUBTASK_AVERAGE_SUFFIX) + + +def split_task_key(task_key: str) -> tuple[str, Optional[int]]: + """Split a `results` key into its task name and few-shot count. + + lighteval builds these as f'{task_name}|{num_fewshots}'; the task name + itself may contain ':' for a subset, as in 'mmlu:abstract_algebra|5'. + """ + name, separator, fewshot = task_key.rpartition('|') + if not separator: + return task_key, None + try: + return name, int(fewshot) + except ValueError: + return task_key, None + + +def is_finite_number(value: Any) -> bool: + """Report whether a value is a real number that JSON can round-trip.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + return math.isfinite(value) + + +def find_metric_spec( + task_config: Dict[str, Any], metric_name: str +) -> Optional[Dict[str, Any]]: + """Find the configured metric that produced a given `results` entry. + + Grouped metrics declare metric_name as a list, so a spec can own several + entries. + """ + for spec in task_config.get('metrics') or []: + if not isinstance(spec, dict): + continue + declared = spec.get('metric_name') + if isinstance(declared, str): + declared = [declared] + if isinstance(declared, list) and metric_name in declared: + return spec + return None + + +def higher_is_better_for( + metric_spec: Optional[Dict[str, Any]], metric_name: str +) -> Optional[bool]: + """Read a metric's direction, returning None when the run does not state it.""" + if metric_spec is None: + return None + declared = metric_spec.get('higher_is_better') + if isinstance(declared, bool): + return declared + if isinstance(declared, dict): + value = declared.get(metric_name) + return value if isinstance(value, bool) else None + return None + + +def stderr_method_for( + metric_spec: Optional[Dict[str, Any]], metric_name: str +) -> Optional[str]: + """Name the estimator lighteval used for a metric's standard error. + + Mirrors get_stderr_function in lighteval/metrics/utils/stderr.py, which + picks the analytic mean_stderr when the corpus-level aggregation's name + contains 'mean' and bootstraps otherwise. Returns None when the results + file does not record which aggregation ran. + """ + if metric_spec is None: + return None + aggregation = metric_spec.get('corpus_level_fn') + if isinstance(aggregation, dict): + aggregation = aggregation.get(metric_name) + if not isinstance(aggregation, str) or not aggregation: + return None + return 'analytic' if 'mean' in aggregation else 'bootstrap' + + +def parse_results_file_timestamp(file_path: Path) -> Optional[str]: + """Recover the wall-clock stamp lighteval encodes in a results filename. + + lighteval names results files f'results_{date_id}.json' where date_id is + datetime.now().isoformat() with ':' replaced by '-'. This is the only + wall-clock time in a run: config_general's start_time and end_time come + from time.perf_counter(), whose origin is undefined. + """ + match = _DATE_ID_PATTERN.match(Path(file_path).stem) + if match is None: + return None + return ( + f'{match.group("date")}:{match.group("minute")}:{match.group("second")}' + ) + + +def flatten_model_config( + model_config: Any, +) -> tuple[Dict[str, str], List[str]]: + """Stringify a dumped lighteval model config for additional_details. + + Returns the flattened values and the names of any credential-bearing keys + that were dropped. + """ + if not isinstance(model_config, dict): + return {}, [] + + flattened: Dict[str, str] = {} + redacted: List[str] = [] + for key, value in model_config.items(): + if value is None: + continue + if key.lower() in SECRET_MODEL_CONFIG_KEYS: + redacted.append(key) + continue + if isinstance(value, str): + flattened[key] = value + else: + flattened[key] = json.dumps(value, sort_keys=True, default=str) + return flattened, sorted(redacted) diff --git a/tests/data/lighteval/results/HuggingFaceTB/SmolLM2-1.7B-Instruct/results_2026-01-21T03-44-18.458309.json b/tests/data/lighteval/results/HuggingFaceTB/SmolLM2-1.7B-Instruct/results_2026-01-21T03-44-18.458309.json new file mode 100644 index 000000000..2dc0d20d5 --- /dev/null +++ b/tests/data/lighteval/results/HuggingFaceTB/SmolLM2-1.7B-Instruct/results_2026-01-21T03-44-18.458309.json @@ -0,0 +1,229 @@ +{ + "config_general": { + "lighteval_sha": "?", + "num_fewshot_seeds": 1, + "max_samples": null, + "job_id": 0, + "start_time": 3122.7169191, + "end_time": 3251.1406346, + "total_evaluation_time_secondes": "128.42371550000023", + "model_config": { + "model_name": "HuggingFaceTB/SmolLM2-1.7B-Instruct", + "generation_parameters": { + "max_new_tokens": 256, + "temperature": 0.0, + "top_p": 0.95, + "top_k": null, + "seed": 42, + "stop_tokens": [ + "" + ], + "repetition_penalty": null + }, + "system_prompt": null, + "cache_dir": "~/.cache/huggingface/lighteval", + "batch_size": 8, + "dtype": "bfloat16", + "revision": "main" + }, + "model_name": "HuggingFaceTB/SmolLM2-1.7B-Instruct" + }, + "results": { + "mmlu:abstract_algebra|5": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316 + }, + "mmlu:anatomy|5": { + "acc": 0.4444444444444444, + "acc_stderr": NaN + }, + "glue:cola|0": { + "mcc": 0.1673623873916, + "mcc_stderr": 0.03127419342, + "custom_reward": 7.25, + "broken_metric": NaN + }, + "mmlu:_average|5": { + "acc": 0.3772222222222222, + "acc_stderr": 0.04648231987117316 + }, + "all": { + "acc": 0.3772222222222222, + "acc_stderr": 0.04648231987117316, + "mcc": 0.1673623873916, + "mcc_stderr": 0.03127419342, + "custom_reward": 7.25 + } + }, + "versions": { + "mmlu:abstract_algebra|5": 0, + "mmlu:anatomy|5": 0, + "glue:cola|0": 0 + }, + "config_tasks": { + "mmlu:abstract_algebra|5": { + "name": "mmlu:abstract_algebra", + "prompt_function": "mmlu_harness", + "hf_repo": "lighteval/mmlu", + "hf_subset": "abstract_algebra", + "metrics": [ + { + "metric_name": "acc", + "higher_is_better": true, + "category": "LOGPROBS", + "sample_level_fn": "LoglikelihoodAcc", + "corpus_level_fn": "mean", + "batched_compute": false + } + ], + "hf_revision": null, + "hf_avail_splits": [ + "auxiliary_train", + "test", + "validation", + "dev" + ], + "evaluation_splits": [ + "test" + ], + "few_shots_split": "dev", + "few_shots_select": "sequential", + "generation_size": 5, + "generation_grammar": null, + "stop_sequence": [ + "\n" + ], + "num_samples": null, + "original_num_docs": 100, + "effective_num_docs": 100, + "must_remove_duplicate_docs": false, + "num_fewshots": 5, + "version": 0 + }, + "mmlu:anatomy|5": { + "name": "mmlu:anatomy", + "prompt_function": "mmlu_harness", + "hf_repo": "lighteval/mmlu", + "hf_subset": "anatomy", + "metrics": [ + { + "metric_name": "acc", + "higher_is_better": true, + "category": "LOGPROBS", + "sample_level_fn": "LoglikelihoodAcc", + "corpus_level_fn": "mean", + "batched_compute": false + } + ], + "hf_revision": null, + "hf_avail_splits": [ + "auxiliary_train", + "test", + "validation", + "dev" + ], + "evaluation_splits": [ + "test" + ], + "few_shots_split": "dev", + "few_shots_select": "sequential", + "generation_size": 5, + "generation_grammar": null, + "stop_sequence": [ + "\n" + ], + "num_samples": null, + "original_num_docs": 135, + "effective_num_docs": 135, + "must_remove_duplicate_docs": false, + "num_fewshots": 5, + "version": 0 + }, + "glue:cola|0": { + "name": "glue:cola", + "prompt_function": "cola", + "hf_repo": "nyu-mll/glue", + "hf_subset": "cola", + "metrics": [ + { + "metric_name": "mcc", + "higher_is_better": true, + "category": "LOGPROBS", + "sample_level_fn": "LoglikelihoodPreparator", + "corpus_level_fn": "matthews_corrcoef", + "batched_compute": false + } + ], + "hf_revision": null, + "hf_avail_splits": [ + "train", + "validation", + "test" + ], + "evaluation_splits": [ + "validation" + ], + "few_shots_split": null, + "few_shots_select": null, + "generation_size": null, + "generation_grammar": null, + "stop_sequence": [], + "num_samples": null, + "original_num_docs": 1043, + "effective_num_docs": 1043, + "must_remove_duplicate_docs": false, + "num_fewshots": 0, + "version": 0 + } + }, + "summary_tasks": { + "mmlu:abstract_algebra|5": { + "hashes": { + "hash_examples": "5e8b3a1c9d0f2a44", + "hash_full_prompts": "2f1d4c7b6a9e0355", + "hash_input_tokens": "9c0a5e2b7d4f1866", + "hash_cont_tokens": "7b3e9f1a4c2d5077" + }, + "truncated": 0, + "non_truncated": 100, + "padded": 400, + "non_padded": 0 + }, + "mmlu:anatomy|5": { + "hashes": { + "hash_examples": "1a2b3c4d5e6f7088", + "hash_full_prompts": "8f7e6d5c4b3a2199", + "hash_input_tokens": "0d1c2b3a49586700", + "hash_cont_tokens": "6a5b4c3d2e1f0911" + }, + "truncated": 0, + "non_truncated": 135, + "padded": 540, + "non_padded": 0 + }, + "glue:cola|0": { + "hashes": { + "hash_examples": "3c4d5e6f70819222", + "hash_full_prompts": "4d5e6f7081922333", + "hash_input_tokens": "5e6f708192233444", + "hash_cont_tokens": "6f70819223344555" + }, + "truncated": 0, + "non_truncated": 1043, + "padded": 2086, + "non_padded": 0 + } + }, + "summary_general": { + "hashes": { + "hash_examples": "aa11bb22cc33dd44", + "hash_full_prompts": "bb22cc33dd44ee55", + "hash_input_tokens": "cc33dd44ee55ff66", + "hash_cont_tokens": "dd44ee55ff667700" + }, + "truncated": 0, + "non_truncated": 1278, + "padded": 3026, + "non_padded": 0 + } +} \ No newline at end of file diff --git a/tests/test_lighteval_adapter.py b/tests/test_lighteval_adapter.py new file mode 100644 index 000000000..a816b0788 --- /dev/null +++ b/tests/test_lighteval_adapter.py @@ -0,0 +1,434 @@ +import json +from pathlib import Path + +import pytest + +from every_eval_ever.converters.lighteval.adapter import LightevalAdapter +from every_eval_ever.converters.lighteval.utils import ( + find_metric_spec, + flatten_model_config, + is_derived_aggregate_key, + parse_results_file_timestamp, + split_task_key, + stderr_method_for, +) +from every_eval_ever.eval_types import ( + EvaluationLog, + EvaluatorRelationship, + ScoreType, + SourceDataHf, +) +from every_eval_ever.helpers.io import SourceRecordsError + +DATA_DIR = Path('tests/data/lighteval') +RESULTS_FILE = ( + DATA_DIR + / 'results/HuggingFaceTB/SmolLM2-1.7B-Instruct' + / 'results_2026-01-21T03-44-18.458309.json' +) + + +def _make_metadata_args(**overrides): + args = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + } + args.update(overrides) + return args + + +def _logs_by_task(logs): + return {log.evaluation_results[0].evaluation_name: log for log in logs} + + +# ── Utility tests ────────────────────────────────────────────────────── + + +def test_split_task_key_keeps_subset_separator(): + assert split_task_key('mmlu:abstract_algebra|5') == ( + 'mmlu:abstract_algebra', + 5, + ) + assert split_task_key('gsm8k|0') == ('gsm8k', 0) + + +def test_split_task_key_without_fewshot_suffix(): + assert split_task_key('all') == ('all', None) + assert split_task_key('weird|notanumber') == ('weird|notanumber', None) + + +def test_is_derived_aggregate_key(): + assert is_derived_aggregate_key('all') is True + assert is_derived_aggregate_key('mmlu:_average|5') is True + assert is_derived_aggregate_key('mmlu:abstract_algebra|5') is False + assert is_derived_aggregate_key('gsm8k|0') is False + + +def test_parse_results_file_timestamp_restores_iso_colons(): + assert ( + parse_results_file_timestamp( + Path('results_2026-01-21T03-44-18.458309.json') + ) + == '2026-01-21T03:44:18.458309' + ) + + +def test_parse_results_file_timestamp_ignores_other_names(): + assert parse_results_file_timestamp(Path('results_latest.json')) is None + + +def test_stderr_method_mirrors_lighteval_choice(): + mean_spec = {'metric_name': 'acc', 'corpus_level_fn': 'mean'} + other_spec = {'metric_name': 'mcc', 'corpus_level_fn': 'matthews_corrcoef'} + assert stderr_method_for(mean_spec, 'acc') == 'analytic' + assert stderr_method_for(other_spec, 'mcc') == 'bootstrap' + assert stderr_method_for(None, 'acc') is None + assert stderr_method_for({'metric_name': 'acc'}, 'acc') is None + + +def test_find_metric_spec_handles_grouped_metrics(): + task_config = { + 'metrics': [ + { + 'metric_name': ['bleu_1', 'bleu_4'], + 'corpus_level_fn': {'bleu_1': 'mean', 'bleu_4': 'corpus_bleu'}, + } + ] + } + spec = find_metric_spec(task_config, 'bleu_4') + assert spec is not None + assert stderr_method_for(spec, 'bleu_4') == 'bootstrap' + assert stderr_method_for(spec, 'bleu_1') == 'analytic' + assert find_metric_spec(task_config, 'rouge1') is None + + +def test_flatten_model_config_drops_credentials(): + flattened, redacted = flatten_model_config( + { + 'model_name': 'openai/gpt-4o', + 'provider': 'openai', + 'api_key': 'sk-NOT-A-REAL-KEY-FIXTURE-ONLY', + 'generation_parameters': {'temperature': 0.0}, + 'timeout': None, + } + ) + assert redacted == ['api_key'] + assert 'api_key' not in flattened + assert 'sk-NOT-A-REAL-KEY-FIXTURE-ONLY' not in json.dumps(flattened) + assert flattened['provider'] == 'openai' + assert flattened['generation_parameters'] == '{"temperature": 0.0}' + # None means "not set by the run", which is not the same as a value. + assert 'timeout' not in flattened + + +# ── Adapter: transform_from_file ─────────────────────────────────────── + + +def test_transform_from_file_returns_only_measured_tasks(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + assert len(logs) == 3 + for log in logs: + assert isinstance(log, EvaluationLog) + assert set(_logs_by_task(logs)) == { + 'mmlu:abstract_algebra|5', + 'mmlu:anatomy|5', + 'glue:cola|0', + } + + +def test_transform_from_file_model_info(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + model = logs[0].model_info + + assert model.name == 'HuggingFaceTB/SmolLM2-1.7B-Instruct' + assert model.id == model.name + assert model.developer == 'HuggingFaceTB' + assert model.additional_details['dtype'] == 'bfloat16' + assert model.additional_details['batch_size'] == '8' + # lighteval dumps its model config with no backend discriminator. + assert model.inference_engine is None + + +def test_transform_from_file_source_metadata(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + src = logs[0].source_metadata + + assert src.source_name == 'lighteval' + assert src.source_type.value == 'evaluation_run' + assert src.source_organization_name == 'TestOrg' + assert ( + src.additional_details['total_evaluation_time_seconds'] + == '128.42371550000023' + ) + + +def test_transform_from_file_source_data(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + + source = logs['mmlu:abstract_algebra|5'].evaluation_results[0].source_data + assert isinstance(source, SourceDataHf) + assert source.dataset_name == 'mmlu:abstract_algebra' + assert source.hf_repo == 'lighteval/mmlu' + assert source.hf_split == 'test' + assert source.samples_number == 100 + assert source.additional_details['hf_subset'] == 'abstract_algebra' + + +def test_transform_from_file_evaluation_results(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + + [result] = logs['mmlu:abstract_algebra|5'].evaluation_results + assert result.score_details.score == 0.31 + assert result.metric_config.metric_name == 'acc' + assert result.metric_config.lower_is_better is False + assert result.metric_config.score_type == ScoreType.continuous + assert result.metric_config.min_score == 0.0 + assert result.metric_config.max_score == 1.0 + + +def test_transform_from_file_uncertainty(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + + [result] = logs['mmlu:abstract_algebra|5'].evaluation_results + uncertainty = result.score_details.uncertainty + assert uncertainty is not None + assert uncertainty.standard_error.value == 0.04648231987117316 + assert uncertainty.standard_error.method == 'analytic' + assert uncertainty.num_samples == 100 + + +def test_transform_from_file_generation_config(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + + gen = ( + logs['mmlu:abstract_algebra|5'].evaluation_results[0].generation_config + ) + assert gen is not None + assert gen.generation_args.temperature == 0.0 + assert gen.generation_args.top_p == 0.95 + assert gen.generation_args.max_tokens == 256 + assert gen.additional_details['num_fewshots'] == '5' + assert gen.additional_details['seed'] == '42' + + +def test_transform_from_file_evaluation_timestamp_comes_from_the_filename(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + assert logs[0].evaluation_timestamp == '2026-01-21T03:44:18.458309' + + +def test_unknown_lighteval_sha_is_not_recorded(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + assert logs[0].eval_library.name == 'lighteval' + assert logs[0].eval_library.version == 'unknown' + assert logs[0].eval_library.additional_details is None + + +# ── The two traps in lighteval's results mapping ─────────────────────── + + +def test_stderr_is_never_emitted_as_its_own_metric(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + emitted = { + result.metric_config.metric_name + for log in logs + for result in log.evaluation_results + } + assert not any(name.endswith('_stderr') for name in emitted) + + +def test_nan_stderr_is_omitted_rather_than_zeroed(): + """MetricsLogger writes NaN when the stderr estimate overflows. NaN means + 'no uncertainty reported', and 0.0 would claim a perfectly precise score.""" + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + + [result] = logs['mmlu:anatomy|5'].evaluation_results + assert result.score_details.score == 0.4444444444444444 + uncertainty = result.score_details.uncertainty + assert uncertainty is not None + assert uncertainty.standard_error is None + assert uncertainty.num_samples == 135 + + +def test_derived_aggregate_rows_are_not_converted_but_are_recorded(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + + names = { + result.evaluation_name + for log in logs + for result in log.evaluation_results + } + assert 'all' not in names + assert 'mmlu:_average|5' not in names + assert ( + logs[0].source_metadata.additional_details[ + 'lighteval_derived_rows_not_converted' + ] + == 'all,mmlu:_average|5' + ) + + +def test_non_finite_score_is_dropped_and_counted(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + + log = logs['glue:cola|0'] + emitted = { + result.metric_config.metric_name for result in log.evaluation_results + } + assert emitted == {'mcc', 'custom_reward'} + assert ( + log.source_metadata.additional_details['metrics_dropped_non_finite'] + == '1' + ) + + +def test_bootstrap_stderr_and_unknown_bounds(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + results = { + result.metric_config.metric_name: result + for result in logs['glue:cola|0'].evaluation_results + } + + mcc = results['mcc'] + assert mcc.metric_config.min_score == -1.0 + assert mcc.metric_config.max_score == 1.0 + assert mcc.score_details.uncertainty.standard_error.method == 'bootstrap' + + custom = results['custom_reward'] + assert custom.metric_config.score_type is None + assert custom.metric_config.min_score is None + assert custom.metric_config.additional_details == { + 'direction_status': 'assumed_higher_is_better', + 'bounds_status': 'unknown', + } + assert ( + logs['glue:cola|0'].source_metadata.additional_details[ + 'metrics_with_unknown_bounds' + ] + == '1' + ) + + +def test_task_with_no_finite_scores_is_reported_as_a_failure(tmp_path): + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + source['results'] = {'glue:cola|0': {'mcc': float('nan')}} + broken = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + broken.write_text(json.dumps(source), encoding='utf-8') + + adapter = LightevalAdapter() + result = adapter.transform_from_directory_result(tmp_path, {}) + + assert result.records == [] + assert len(result.failures) == 0 # no measured task, so nothing to convert + + +# ── Adapter: transform_from_directory ────────────────────────────────── + + +def test_transform_from_directory_finds_nested_results_files(): + adapter = LightevalAdapter() + logs = adapter.transform_from_directory(DATA_DIR, _make_metadata_args()) + assert len(logs) == 3 + + +def test_transform_from_directory_without_results_files(tmp_path): + adapter = LightevalAdapter() + with pytest.raises(ValueError, match='No lighteval results_'): + adapter.transform_from_directory_result(tmp_path, {}) + + +def test_directory_conversion_retains_good_files_and_reports_bad_files( + tmp_path, monkeypatch +): + good_path = tmp_path / 'results_good.json' + bad_path = tmp_path / 'results_bad.json' + good_path.write_text('{}', encoding='utf-8') + bad_path.write_text('{}', encoding='utf-8') + adapter = LightevalAdapter() + good_log = object() + + def fake_transform(path, _metadata): + if Path(path).name == 'results_bad.json': + raise ValueError('broken lighteval result') + return [good_log] + + monkeypatch.setattr(adapter, 'transform_from_file', fake_transform) + + result = adapter.transform_from_directory_result(tmp_path, {}) + + assert result.records == [good_log] + assert result.total_records == 2 + assert len(result.failures) == 1 + assert result.failures[0].source_ref == str(bad_path) + with pytest.raises(SourceRecordsError, match='broken lighteval result'): + result.raise_if_incomplete() + + +def test_eval_metadata_stored_after_transform(): + adapter = LightevalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + for log in logs: + meta = adapter.get_eval_metadata(log.evaluation_id) + assert meta['task_key'] + assert meta['parent_dir'] == str(RESULTS_FILE.parent) + + +# ── Metadata overrides ───────────────────────────────────────────────── + + +def test_inference_engine_override(): + adapter = LightevalAdapter() + metadata = _make_metadata_args( + inference_engine='vllm', inference_engine_version='0.6.0' + ) + logs = adapter.transform_from_file(RESULTS_FILE, metadata) + assert logs[0].model_info.inference_engine.name == 'vllm' + assert logs[0].model_info.inference_engine.version == '0.6.0' + + +def test_provider_in_model_config_is_used_as_inference_platform(): + adapter = LightevalAdapter() + raw_data = { + 'config_general': { + 'model_name': 'openai/gpt-4o', + 'model_config': { + 'model_name': 'openai/gpt-4o', + 'provider': 'openai', + }, + } + } + model = adapter._extract_model_info(raw_data, _make_metadata_args()) + assert model.inference_platform == 'openai' + + +def test_missing_model_name_is_an_error(): + adapter = LightevalAdapter() + with pytest.raises(ValueError, match='config_general.model_name'): + adapter._extract_model_info({'config_general': {}}, {}) From 75a03c269eb60dbaf5247ec4a36dc79af18dcbdf Mon Sep 17 00:00:00 2001 From: Mandark-droid Date: Fri, 7 Aug 2026 17:09:21 +0530 Subject: [PATCH 2/7] fix(lighteval): name tasks skipped for having no finite score A task whose every metric aggregated to NaN was dropped from the file with nothing recorded anywhere. Its key is now listed on the logs the same file did produce. --- .../converters/lighteval/adapter.py | 21 ++++++++++---- tests/test_lighteval_adapter.py | 29 ++++++++++++++++--- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/every_eval_ever/converters/lighteval/adapter.py b/every_eval_ever/converters/lighteval/adapter.py index 86c55e0c7..8cc75123f 100644 --- a/every_eval_ever/converters/lighteval/adapter.py +++ b/every_eval_ever/converters/lighteval/adapter.py @@ -415,6 +415,11 @@ def _transform_single( source_details['lighteval_derived_rows_not_converted'] = ','.join( derived_rows ) + skipped_tasks = metadata_args.get('tasks_without_finite_scores') or [] + if skipped_tasks: + source_details['tasks_without_finite_scores'] = ','.join( + skipped_tasks + ) elapsed = config_general.get('total_evaluation_time_secondes') if elapsed is not None: source_details['total_evaluation_time_seconds'] = str(elapsed) @@ -466,15 +471,19 @@ def transform_from_file( raw_data = self._load_file(file_path) tasks = self._get_tasks(raw_data) - derived_keys = sorted( - key - for key in (raw_data.get('results') or {}) - if is_derived_aggregate_key(key) - ) + derived_keys = [] + skipped_keys = [] + for key in raw_data.get('results') or {}: + if is_derived_aggregate_key(key): + derived_keys.append(key) + elif key not in tasks: + skipped_keys.append(key) + metadata_args = { **metadata_args, 'parent_eval_output_dir': str(file_path.parent), - 'derived_aggregate_keys': derived_keys, + 'derived_aggregate_keys': sorted(derived_keys), + 'tasks_without_finite_scores': sorted(skipped_keys), # The results filename holds the only wall-clock time in the run. 'evaluation_timestamp': parse_results_file_timestamp(file_path), } diff --git a/tests/test_lighteval_adapter.py b/tests/test_lighteval_adapter.py index a816b0788..c55daa1e9 100644 --- a/tests/test_lighteval_adapter.py +++ b/tests/test_lighteval_adapter.py @@ -336,17 +336,38 @@ def test_bootstrap_stderr_and_unknown_bounds(): ) -def test_task_with_no_finite_scores_is_reported_as_a_failure(tmp_path): +def test_task_with_no_finite_scores_is_named_on_the_remaining_logs(tmp_path): + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + source['results']['glue:cola|0'] = {'mcc': float('nan')} + partial = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + partial.write_text(json.dumps(source), encoding='utf-8') + + adapter = LightevalAdapter() + logs = adapter.transform_from_file(partial, _make_metadata_args()) + + assert set(_logs_by_task(logs)) == { + 'mmlu:abstract_algebra|5', + 'mmlu:anatomy|5', + } + assert ( + logs[0].source_metadata.additional_details[ + 'tasks_without_finite_scores' + ] + == 'glue:cola|0' + ) + + +def test_file_with_no_measured_task_yields_nothing(tmp_path): source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) source['results'] = {'glue:cola|0': {'mcc': float('nan')}} - broken = tmp_path / 'results_2026-01-21T03-44-18.458309.json' - broken.write_text(json.dumps(source), encoding='utf-8') + empty = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + empty.write_text(json.dumps(source), encoding='utf-8') adapter = LightevalAdapter() result = adapter.transform_from_directory_result(tmp_path, {}) assert result.records == [] - assert len(result.failures) == 0 # no measured task, so nothing to convert + assert result.failures == [] # ── Adapter: transform_from_directory ────────────────────────────────── From deb0b8ddbe69fe9212b8191d7bdfe73a86d4ad0e Mon Sep 17 00:00:00 2001 From: Mandark-droid Date: Sat, 8 Aug 2026 12:24:43 +0530 Subject: [PATCH 3/7] fix(lighteval): address review-anvil findings on identity, credentials and accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F009 credential-filtering. The filter tested only top-level model-config keys, and the else branch json.dumps() the whole value, so a nested env_vars mapping carrying OPENAI_API_KEY was serialised intact into additional_details, which is published. Sanitising is now recursive over mappings and lists, and redacted keys are reported as dotted paths. Matching is on exact names plus suffixes (_key, _token, _secret, _password, _credentials) rather than substrings, so tokenizer and max_tokens are not mistaken for secrets. F001 evaluation-identity. evaluation_id was task/model/retrieved_timestamp, where retrieved_timestamp is the conversion time, so the same source file got a new identity on every conversion and re-ingest could duplicate it. It is now keyed on the raw source identity — task key, the model name as the file states it, the run's own wall-clock stamp — plus a digest of the non-secret config so runs differing only in settings stay distinct. Conversion time remains in retrieved_timestamp alone. Worth flagging: CONTRIBUTING.md line 102 prescribes exactly the unstable form ({benchmark_name/model_id/retrieved_timestamp}), while the conversion skill's fields.md says to key on a stable value and never on `now`. This follows fields.md; the two documents still disagree. F002 source-accounting. A file whose measured tasks all lacked finite scores returned an empty list with no failure, so a directory of such files converted zero records and exited successfully. That case now enters the failure ledger. Files holding only derived aggregate rows are recorded as explicit exclusions instead, since dropping those is deliberate. F007 coverage-totals. total_records summed task-level logs with file-level failures, so one good file contributed several counts and one bad file contributed one. The source-record grain is now the results file; converted-log count is already reported separately by failure_report(). F008 failure-reporting. The single-file CLI branch raised before building a SourceConversionResult, so parse errors exited non-zero with no structured report while directory input reported the same failure correctly. Both entry modes now go through transform_from_file_result, which also removes the duplicated per-file handling in the directory walk. F006 metric-identity. metric_id was unset in both MetricConfig branches, so records carried no cross-source join key. Names with an unambiguous canonical identity map to it; everything else gets a stable lighteval/ id rather than an invented one, and metric_id_source records which route was taken so a later pass can resolve the namespaced ones against the eval-card-registry. F005 metric-semantics. An undeclared direction is still assumed higher-is-better because the schema requires one, but operators can now declare it via metric_directions, and names left unresolved are reported on the eval metadata instead of the guess passing unnoticed. F004 score-coverage. samples_number stays the dataset size as provenance; when a run is capped or deduplicated the smaller population that actually produced the score is recorded as scored_num_docs, so both counts are visible. 39 lighteval tests (was 31), full suite 428 passed / 20 skipped, ruff clean, and convert -> validate still passes 3/3. --- every_eval_ever/cli.py | 7 +- .../converters/lighteval/adapter.py | 193 ++++++++++++++++-- every_eval_ever/converters/lighteval/utils.py | 110 +++++++++- tests/test_lighteval_adapter.py | 180 +++++++++++++++- 4 files changed, 465 insertions(+), 25 deletions(-) diff --git a/every_eval_ever/cli.py b/every_eval_ever/cli.py index f9b0df465..9f2800f60 100644 --- a/every_eval_ever/cli.py +++ b/every_eval_ever/cli.py @@ -232,7 +232,12 @@ def _cmd_convert_lighteval(args: argparse.Namespace) -> int: log_path = Path(args.log_path) input_result: SourceConversionResult[Any] | None = None if log_path.is_file(): - logs = adapter.transform_from_file(log_path, metadata) + # Via the result path, so a parse or conversion error reaches the + # failure report before it is raised. The directory branch already + # behaved this way, so the same failure class was reported differently + # depending on which entry mode you used. + input_result = adapter.transform_from_file_result(log_path, metadata) + logs = input_result.records elif log_path.is_dir(): input_result = adapter.transform_from_directory_result( log_path, metadata diff --git a/every_eval_ever/converters/lighteval/adapter.py b/every_eval_ever/converters/lighteval/adapter.py index 8cc75123f..cc50fcd4c 100644 --- a/every_eval_ever/converters/lighteval/adapter.py +++ b/every_eval_ever/converters/lighteval/adapter.py @@ -1,5 +1,6 @@ """Adapter for converting lighteval output to every_eval_ever format.""" +import hashlib import json from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -32,6 +33,7 @@ ) from every_eval_ever.helpers.io import ( SourceConversionResult, + SourceRecordExclusion, SourceRecordFailure, ) @@ -44,6 +46,7 @@ is_derived_aggregate_key, is_finite_number, parse_results_file_timestamp, + resolve_metric_id, split_task_key, stderr_method_for, ) @@ -177,6 +180,19 @@ def _build_source_data(self, task_config: Dict[str, Any], task_name: str): ) original_num_docs = task_config.get('original_num_docs') + # samples_number is the dataset's own size, which stays as provenance. + # When lighteval caps or deduplicates a run the score comes from the + # smaller effective population, and the uncertainty is computed over + # that one — so record it rather than let the larger number stand in + # for both. + effective_num_docs = task_config.get('effective_num_docs') + if ( + isinstance(effective_num_docs, int) + and effective_num_docs >= 0 + and effective_num_docs != original_num_docs + ): + additional['scored_num_docs'] = str(effective_num_docs) + return SourceDataHf( dataset_name=dataset_name, source_type='hf_dataset', @@ -251,8 +267,19 @@ def _build_evaluation_results( raw_data: Dict[str, Any], task_key: str, evaluation_timestamp: Optional[str] = None, + metric_directions: Optional[Dict[str, bool]] = None, + unresolved_metrics: Optional[List[str]] = None, ) -> List[EvaluationResult]: - """Build the EvaluationResult list for a single lighteval task.""" + """Build the EvaluationResult list for a single lighteval task. + + ``metric_directions`` lets an operator declare higher/lower-is-better + for metrics the results file does not describe. Names that stay + undeclared are appended to ``unresolved_metrics`` so the caller can + report them instead of the guess going unnoticed. + """ + metric_directions = metric_directions or {} + if unresolved_metrics is None: + unresolved_metrics = [] task_results = raw_data['results'][task_key] task_name, num_fewshots = split_task_key(task_key) task_config = (raw_data.get('config_tasks') or {}).get(task_key) or {} @@ -284,10 +311,23 @@ def _build_evaluation_results( higher_is_better = higher_is_better_for(metric_spec, metric_name) metric_details = {} - if higher_is_better is None: - # EEE requires a direction; record that the run did not give one. + declared_direction = metric_directions.get(metric_name) + if declared_direction is not None: + # An operator-supplied definition outranks the source and the + # fallback both. + higher_is_better = declared_direction + metric_details['direction_status'] = 'operator_declared' + elif higher_is_better is None: + # A finite score does not prove a direction. EEE requires one, + # so the assumption stays — but it is labelled, and the name is + # reported so the run can be re-converted once the metric has a + # definition rather than the guess passing unnoticed. metric_details['direction_status'] = 'assumed_higher_is_better' higher_is_better = True + unresolved_metrics.append(metric_name) + + metric_id, id_source = resolve_metric_id(metric_name) + metric_details['metric_id_source'] = id_source bounds = KNOWN_METRIC_BOUNDS.get(metric_name) if bounds is None: @@ -297,6 +337,7 @@ def _build_evaluation_results( metric_config = MetricConfig( evaluation_description=metric_name, metric_name=metric_name, + metric_id=metric_id, lower_is_better=not higher_is_better, additional_details=metric_details, ) @@ -304,6 +345,7 @@ def _build_evaluation_results( metric_config = MetricConfig( evaluation_description=metric_name, metric_name=metric_name, + metric_id=metric_id, lower_is_better=not higher_is_better, score_type=ScoreType.continuous, min_score=bounds[0], @@ -358,6 +400,44 @@ def _count_dropped_scores( if not key.endswith(STDERR_SUFFIX) and not is_finite_number(value) ) + def _stable_evaluation_id( + self, + task_key: str, + model_info: ModelInfo, + eval_timestamp: Optional[str], + ) -> str: + """Build an evaluation_id that is identical across re-conversions. + + Keyed on the raw source identity — task key, the model name as the + results file states it, and the run's own wall-clock stamp — never on + the conversion time, which would mint a new identity on every run and + let re-ingest duplicate the same evaluation. + + The digest carries the non-secret run configuration so that two runs of + the same task and model under different settings stay distinct rather + than collapsing onto one id. It is taken from + ``model_info.additional_details``, which is already credential-filtered, + so no secret can reach the identity. + + ``model_info.id`` here is the raw name from the file, not a + registry-resolved id: a resolved id can be re-mapped later, and a moving + identity would break idempotency just as surely as keying on now. + """ + fingerprint = json.dumps( + { + 'task': task_key, + 'model': model_info.id, + 'evaluated_at': eval_timestamp, + 'config': model_info.additional_details or {}, + }, + sort_keys=True, + default=str, + ) + digest = hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()[:8] + return ( + f'{task_key}/{model_info.id}/{eval_timestamp or "unknown"}-{digest}' + ) + def _transform_single( self, raw_data: Dict[str, Any], metadata_args: Dict[str, Any] ) -> EvaluationLog: @@ -369,12 +449,22 @@ def _transform_single( model_info = self._extract_model_info(raw_data, metadata_args) config_general = raw_data.get('config_general') or {} + # Conversion time. Belongs in retrieved_timestamp and nowhere else — + # see _stable_evaluation_id. retrieved_timestamp = get_current_unix_timestamp() eval_timestamp = metadata_args.get('evaluation_timestamp') - evaluation_id = f'{task_key}/{model_info.id}/{retrieved_timestamp}' + evaluation_id = self._stable_evaluation_id( + task_key, model_info, eval_timestamp + ) + metric_directions = metadata_args.get('metric_directions') or {} + unresolved_metrics: List[str] = [] evaluation_results = self._build_evaluation_results( - raw_data, task_key, eval_timestamp + raw_data, + task_key, + eval_timestamp, + metric_directions=metric_directions, + unresolved_metrics=unresolved_metrics, ) if not evaluation_results: raise ValueError( @@ -447,6 +537,12 @@ def _transform_single( self._eval_metadata[evaluation_id] = { 'parent_dir': metadata_args.get('parent_eval_output_dir'), 'task_key': task_key, + # Metrics whose direction the run never stated and no operator + # declared. Reported rather than silently carried, so the guess is + # visible to whoever reads the conversion. + 'metrics_without_declared_direction': sorted( + set(unresolved_metrics) + ), } return EvaluationLog( @@ -488,6 +584,17 @@ def transform_from_file( 'evaluation_timestamp': parse_results_file_timestamp(file_path), } + if not tasks and skipped_keys: + # The file HAS measured tasks, and not one of them could be + # converted. Returning an empty list here made total conversion + # loss indistinguishable from a file that legitimately carried + # nothing to convert: the directory walk recorded no failure and + # exited zero. Raising puts it in the failure ledger instead. + raise ValueError( + f'lighteval file has {len(skipped_keys)} measured task(s) but ' + f'none with a finite score: {", ".join(sorted(skipped_keys))}' + ) + results = [] for task_key in tasks: task_metadata = {**metadata_args, 'task_key': task_key} @@ -495,6 +602,55 @@ def transform_from_file( return results + def transform_from_file_result( + self, file_path: Union[str, Path], metadata_args: Dict[str, Any] + ) -> SourceConversionResult[EvaluationLog]: + """Convert one file, reporting failures rather than only raising. + + The directory walk caught a bad file, recorded it and still wrote a + failure report; the single-file entry point let the exception escape + the command, so the same failure produced a non-zero exit with no + structured report at all. Both entry modes now build the same result, + which is what lets the CLI report before it raises. + """ + file_path = Path(file_path) + records: List[EvaluationLog] = [] + failures: list[SourceRecordFailure] = [] + exclusions: list[SourceRecordExclusion] = [] + try: + records = self.transform_from_file(file_path, metadata_args) + except Exception as exc: + failures.append( + SourceRecordFailure( + source_ref=str(file_path), + reason=str(exc), + source_record={'path': str(file_path)}, + ) + ) + else: + if not records: + # Nothing measured to convert: the file holds only rows + # lighteval derived by averaging. A deliberate exclusion, not a + # failure, so it stays out of the error budget while the file + # is still accounted for. + exclusions.append( + SourceRecordExclusion( + source_ref=str(file_path), + reason=( + 'no measured task rows; file contains only ' + 'derived aggregate keys' + ), + source_record={'path': str(file_path)}, + ) + ) + return SourceConversionResult( + source_name=f'lighteval evaluation {file_path}', + total_records=1, + records=records, + failures=failures, + exclusions=exclusions, + ) + def transform_from_directory( self, dir_path: Union[str, Path], metadata_args: Dict[str, Any] ) -> List[EvaluationLog]: @@ -519,23 +675,24 @@ def transform_from_directory_result( all_logs: list[EvaluationLog] = [] failures: list[SourceRecordFailure] = [] + exclusions: list[SourceRecordExclusion] = [] for results_file in results_files: - try: - all_logs.extend( - self.transform_from_file(results_file, metadata_args) - ) - except Exception as exc: - failures.append( - SourceRecordFailure( - source_ref=str(results_file), - reason=str(exc), - source_record={'path': str(results_file)}, - ) - ) + file_result = self.transform_from_file_result( + results_file, metadata_args + ) + all_logs.extend(file_result.records) + failures.extend(file_result.failures) + exclusions.extend(file_result.exclusions) return SourceConversionResult( source_name=f'lighteval evaluations under {dir_path}', - total_records=len(all_logs) + len(failures), + # The source-record grain is the results FILE. Adding task-level + # logs to file-level failures gave a denominator with no consistent + # unit, because one good file contributes several logs while one + # bad file contributes a single failure. Converted-log count is + # reported separately by failure_report(). + total_records=len(results_files), records=all_logs, failures=failures, + exclusions=exclusions, ) diff --git a/every_eval_ever/converters/lighteval/utils.py b/every_eval_ever/converters/lighteval/utils.py index c637fa718..95e542fe6 100644 --- a/every_eval_ever/converters/lighteval/utils.py +++ b/every_eval_ever/converters/lighteval/utils.py @@ -26,12 +26,25 @@ 'auth_token', 'credentials', 'hf_token', + 'key', 'password', 'secret', 'token', } ) +# Suffixes that make a key credential-bearing whatever the provider prefix. +# Nested `env_vars` mappings are where these actually appear: OPENAI_API_KEY, +# AWS_SECRET_ACCESS_KEY, HUGGING_FACE_HUB_TOKEN. Anchoring on the suffix keeps +# `tokenizer` and `max_tokens` out of the redaction set. +_SECRET_KEY_SUFFIXES = ( + '_key', + '_token', + '_secret', + '_password', + '_credentials', +) + _DATE_ID_PATTERN = re.compile( r'^results_(?P\d{4}-\d{2}-\d{2}T\d{2})-(?P\d{2})-' r'(?P\d{2}(?:\.\d+)?)$' @@ -169,13 +182,99 @@ def parse_results_file_timestamp(file_path: Path) -> Optional[str]: ) +# lighteval metric names whose cross-source identity is unambiguous. Only names +# that mean the same thing in every harness belong here — this is the join key, +# so a wrong entry silently merges two different measurements, which is worse +# than leaving one unjoined. +# +# Everything else gets a stable `lighteval/` id: namespaced rather than +# guessed, and never bare. `metric_id_source` records which route was taken, so +# a later pass can resolve the namespaced ones against the eval-card-registry +# without having to re-derive which were canonical to begin with. +CANONICAL_METRIC_IDS = { + 'acc': 'accuracy', + 'acc_norm': 'accuracy_normalized', + 'bleu': 'bleu', + 'chrf': 'chrf', + 'exact_match': 'exact_match', + 'f1': 'f1', + 'mcc': 'matthews_correlation', + 'perplexity': 'perplexity', + 'rouge1': 'rouge1', + 'rouge2': 'rouge2', + 'rougeL': 'rougeL', + 'ter': 'ter', + 'word_perplexity': 'word_perplexity', +} + +METRIC_ID_NAMESPACE = 'lighteval' + + +def resolve_metric_id(metric_name: str) -> tuple[str, str]: + """Return (metric_id, how_it_was_derived) for a lighteval metric name. + + `metric_id` is the cross-source join key and must always be set, so this + never returns None. Canonical names come from the table above; anything + else is namespaced under `lighteval/` rather than being invented, and the + second element says which happened. + """ + canonical = CANONICAL_METRIC_IDS.get(metric_name) + if canonical is not None: + return canonical, 'canonical' + return f'{METRIC_ID_NAMESPACE}/{metric_name}', 'namespaced_unresolved' + + +def _is_secret_key(key: Any) -> bool: + """True if a config key names a credential. + + Exact names cover the common cases; the suffix rule catches the + provider-prefixed forms that show up inside nested `env_vars` mappings, + such as OPENAI_API_KEY or AWS_SECRET_ACCESS_KEY. + + Matching on suffixes rather than substrings is deliberate: 'token' as a + substring would also redact `tokenizer`, and 'tokens' would take + `max_tokens`. Both are ordinary evaluation config worth keeping, and + neither ends in `_token`. + """ + lowered = str(key).lower() + if lowered in SECRET_MODEL_CONFIG_KEYS: + return True + return lowered.endswith(_SECRET_KEY_SUFFIXES) + + +def _sanitize_config_value( + value: Any, path: List[str], redacted: List[str] +) -> Any: + """Strip credential-bearing keys from nested mappings and sequences. + + lighteval dumps the whole model config, and a nested `env_vars` mapping can + carry a live provider token. Filtering only the top level left those values + to be serialized wholesale into `additional_details`, which is published. + """ + if isinstance(value, dict): + cleaned: Dict[Any, Any] = {} + for key, item in value.items(): + child_path = path + [str(key)] + if _is_secret_key(key): + redacted.append('.'.join(child_path)) + continue + cleaned[key] = _sanitize_config_value(item, child_path, redacted) + return cleaned + if isinstance(value, (list, tuple)): + return [ + _sanitize_config_value(item, path + [str(index)], redacted) + for index, item in enumerate(value) + ] + return value + + def flatten_model_config( model_config: Any, ) -> tuple[Dict[str, str], List[str]]: """Stringify a dumped lighteval model config for additional_details. - Returns the flattened values and the names of any credential-bearing keys - that were dropped. + Returns the flattened values and the dotted paths of any credential-bearing + keys that were dropped, nested ones included. """ if not isinstance(model_config, dict): return {}, [] @@ -185,11 +284,12 @@ def flatten_model_config( for key, value in model_config.items(): if value is None: continue - if key.lower() in SECRET_MODEL_CONFIG_KEYS: - redacted.append(key) + if _is_secret_key(key): + redacted.append(str(key)) continue if isinstance(value, str): flattened[key] = value else: - flattened[key] = json.dumps(value, sort_keys=True, default=str) + cleaned = _sanitize_config_value(value, [str(key)], redacted) + flattened[key] = json.dumps(cleaned, sort_keys=True, default=str) return flattened, sorted(redacted) diff --git a/tests/test_lighteval_adapter.py b/tests/test_lighteval_adapter.py index c55daa1e9..c6d92f6bb 100644 --- a/tests/test_lighteval_adapter.py +++ b/tests/test_lighteval_adapter.py @@ -327,7 +327,15 @@ def test_bootstrap_stderr_and_unknown_bounds(): assert custom.metric_config.additional_details == { 'direction_status': 'assumed_higher_is_better', 'bounds_status': 'unknown', + 'metric_id_source': 'namespaced_unresolved', } + # Always set, because it is the cross-source join key — namespaced rather + # than guessed when the name has no unambiguous canonical identity. + assert custom.metric_config.metric_id == 'lighteval/custom_reward' + assert mcc.metric_config.metric_id == 'matthews_correlation' + assert ( + mcc.metric_config.additional_details['metric_id_source'] == 'canonical' + ) assert ( logs['glue:cola|0'].source_metadata.additional_details[ 'metrics_with_unknown_bounds' @@ -357,7 +365,14 @@ def test_task_with_no_finite_scores_is_named_on_the_remaining_logs(tmp_path): ) -def test_file_with_no_measured_task_yields_nothing(tmp_path): +def test_file_whose_measured_tasks_are_all_unconvertible_is_a_failure(tmp_path): + """Total conversion loss must be distinguishable from nothing to convert. + + This file HAS a measured task; it just has no finite score. Previously the + adapter returned an empty list and recorded no failure, so a directory made + entirely of such files converted zero records and exited successfully — + automation could not tell that everything had been dropped. + """ source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) source['results'] = {'glue:cola|0': {'mcc': float('nan')}} empty = tmp_path / 'results_2026-01-21T03-44-18.458309.json' @@ -366,8 +381,46 @@ def test_file_with_no_measured_task_yields_nothing(tmp_path): adapter = LightevalAdapter() result = adapter.transform_from_directory_result(tmp_path, {}) + assert result.records == [] + assert len(result.failures) == 1 + assert 'glue:cola|0' in result.failures[0].reason + with pytest.raises(SourceRecordsError): + result.raise_if_incomplete() + + +def test_file_with_only_derived_rows_is_an_exclusion_not_a_failure(tmp_path): + """Rows lighteval averaged itself are dropped on purpose, so they are + excluded and accounted for rather than counted against the error budget.""" + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + source['results'] = {'all': {'mcc': 0.5}, 'mmlu:_average|5': {'acc': 0.5}} + derived_only = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + derived_only.write_text(json.dumps(source), encoding='utf-8') + + result = LightevalAdapter().transform_from_directory_result(tmp_path, {}) + assert result.records == [] assert result.failures == [] + assert len(result.exclusions) == 1 + assert 'derived' in result.exclusions[0].reason + result.raise_if_incomplete() + + +def test_total_records_counts_source_files_not_output_logs(tmp_path): + """The coverage denominator needs one consistent unit. + + A good file yields several task-level logs while a bad file yields one + failure, so summing the two gave a denominator that meant nothing. + """ + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + good = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + good.write_text(json.dumps(source), encoding='utf-8') + + result = LightevalAdapter().transform_from_directory_result(tmp_path, {}) + report = result.failure_report() + + assert report['total_source_records'] == 1 + assert report['converted_records'] == len(result.records) + assert len(result.records) > 1, 'fixture should yield several task logs' # ── Adapter: transform_from_directory ────────────────────────────────── @@ -453,3 +506,128 @@ def test_missing_model_name_is_an_error(): adapter = LightevalAdapter() with pytest.raises(ValueError, match='config_general.model_name'): adapter._extract_model_info({'config_general': {}}, {}) + + +# ── Review findings: identity, credentials, coverage ─────────────────── + + +def test_evaluation_id_is_identical_across_repeat_conversions(): + """Keyed on the source, so re-ingesting the same file cannot duplicate it. + + It previously carried the conversion time, which gave the same evaluation a + new identity on every run. + """ + adapter = LightevalAdapter() + first = sorted( + log.evaluation_id + for log in adapter.transform_from_file( + RESULTS_FILE, _make_metadata_args() + ) + ) + second = sorted( + log.evaluation_id + for log in LightevalAdapter().transform_from_file( + RESULTS_FILE, _make_metadata_args() + ) + ) + + assert first == second + assert all(log_id for log_id in first) + + +def test_evaluation_id_separates_runs_that_differ_only_in_config(tmp_path): + """Same task, same model, different settings must not collapse to one id.""" + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + baseline = LightevalAdapter().transform_from_file( + RESULTS_FILE, _make_metadata_args() + )[0] + + source['config_general']['model_config']['temperature'] = 0.9 + variant_file = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + variant_file.write_text(json.dumps(source), encoding='utf-8') + variant = LightevalAdapter().transform_from_file( + variant_file, _make_metadata_args() + )[0] + + assert baseline.evaluation_id != variant.evaluation_id + + +def test_nested_credentials_never_reach_additional_details(tmp_path): + """The filter tested only top-level keys, so a nested env_vars token was + serialized wholesale into a published field.""" + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + source['config_general']['model_config'].update( + { + 'api_key': 'sk-TOPLEVEL', + 'env_vars': { + 'OPENAI_API_KEY': 'sk-NESTED', + 'HF_TOKEN': 'hf-NESTED', + 'REGION': 'us-east-1', + }, + 'providers': [ + {'name': 'aws', 'aws_secret_access_key': 'AKIA-NESTED'} + ], + } + ) + leaky = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + leaky.write_text(json.dumps(source), encoding='utf-8') + + log = LightevalAdapter().transform_from_file(leaky, _make_metadata_args())[ + 0 + ] + published = json.dumps(log.model_info.additional_details or {}) + + for secret in ('sk-TOPLEVEL', 'sk-NESTED', 'hf-NESTED', 'AKIA-NESTED'): + assert secret not in published, f'{secret} reached additional_details' + assert 'us-east-1' in published, 'non-secret nested config should survive' + redacted = (log.model_info.additional_details or {})[ + 'redacted_model_config_keys' + ] + assert 'env_vars.OPENAI_API_KEY' in redacted + + +def test_ordinary_config_is_not_mistaken_for_a_credential(tmp_path): + """`tokenizer` and `max_tokens` contain 'token' but are not secrets.""" + source = json.loads(RESULTS_FILE.read_text(encoding='utf-8')) + source['config_general']['model_config'].update( + {'tokenizer': 'gpt2', 'max_tokens': 512} + ) + path = tmp_path / 'results_2026-01-21T03-44-18.458309.json' + path.write_text(json.dumps(source), encoding='utf-8') + + log = LightevalAdapter().transform_from_file(path, _make_metadata_args())[0] + published = json.dumps(log.model_info.additional_details or {}) + + assert 'gpt2' in published + assert '512' in published + + +def test_operator_can_declare_a_direction_the_run_omits(): + metadata = { + **_make_metadata_args(), + 'metric_directions': {'custom_reward': False}, + } + logs = _logs_by_task( + LightevalAdapter().transform_from_file(RESULTS_FILE, metadata) + ) + results = { + result.metric_config.metric_name: result + for result in logs['glue:cola|0'].evaluation_results + } + + custom = results['custom_reward'] + assert custom.metric_config.lower_is_better is True + assert ( + custom.metric_config.additional_details['direction_status'] + == 'operator_declared' + ) + + +def test_metrics_without_a_declared_direction_are_reported(): + adapter = LightevalAdapter() + logs = _logs_by_task( + adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + ) + meta = adapter.get_eval_metadata(logs['glue:cola|0'].evaluation_id) + + assert 'custom_reward' in meta['metrics_without_declared_direction'] From 6d352816613c2f821749ca42dcc4777e3120c96d Mon Sep 17 00:00:00 2001 From: leshem Date: Mon, 10 Aug 2026 17:54:18 -0400 Subject: [PATCH 4/7] feat(lighteval): convert per-sample details into instance-level output `--include_details` reads the details parquet lighteval writes next to a run's results and publishes it as an instance-level `_samples.jsonl`, with the aggregate's `detailed_evaluation_results` pointing at it. Mirrors lm_eval's `--include_samples`, including its partial-conversion behaviour: a run made without `save_details` still publishes its aggregates, but records a per-task failure and exits non-zero. Two details of lighteval's output shape the mapping: - `Doc.choices` is the options the model was shown for a multiple-choice task and the *reference answers* for a generative one. Only the first is published as `input.choices`; `input.reference` carries the gold either way, and `doc.sampling_methods` is what tells the two apart. - Values come out of parquet as numpy scalars, and numpy's integer types do not subclass `int`, so every numeric check unwraps first. Without that, a task whose `gold_index` is a list loses its whole reference. Locating the details file needs its own helper because `results_path_template` can move the results directory while details stay put, so the two are not always siblings; the search walks up to the shared output directory. The new `lighteval` extra is only pyarrow, to read the parquet. lighteval itself stays out of the project: it needs datasets>=4 while crfm-helm pins datasets~=3.1. --- every_eval_ever/cli.py | 114 ++++- every_eval_ever/converters/README.md | 33 ++ .../converters/lighteval/__main__.py | 8 + .../converters/lighteval/adapter.py | 3 + .../lighteval/instance_level_adapter.py | 399 ++++++++++++++++++ every_eval_ever/converters/lighteval/utils.py | 71 ++++ pyproject.toml | 6 + uv.lock | 8 +- 8 files changed, 638 insertions(+), 4 deletions(-) create mode 100644 every_eval_ever/converters/lighteval/instance_level_adapter.py diff --git a/every_eval_ever/cli.py b/every_eval_ever/cli.py index 9f2800f60..1cfb93efc 100644 --- a/every_eval_ever/cli.py +++ b/every_eval_ever/cli.py @@ -217,6 +217,9 @@ def _cmd_convert_lm_eval(args: argparse.Namespace) -> int: def _cmd_convert_lighteval(args: argparse.Namespace) -> int: from every_eval_ever.converters.lighteval.adapter import LightevalAdapter + from every_eval_ever.converters.lighteval.instance_level_adapter import ( + LightevalInstanceLevelAdapter, + ) adapter = LightevalAdapter() metadata = _common_metadata(args) @@ -253,21 +256,118 @@ def _cmd_convert_lighteval(args: argparse.Namespace) -> int: output_dir = Path(args.output_dir) eval_uuids = [str(uuid.uuid4()) for _ in logs] - paths = ( - publish_evaluation_logs(logs, output_dir, eval_uuids) if logs else [] - ) + include_details = getattr(args, 'include_details', False) + details_result: SourceConversionResult[Any] | None = None + with tempfile.TemporaryDirectory(prefix='eee-lighteval-') as staging: + staging_dir = Path(staging) + details_successes = [] + details_failures = [] + for log, eval_uuid in zip(logs, eval_uuids, strict=True): + if not include_details: + continue + meta: dict[str, Any] = {} + try: + meta = adapter.get_eval_metadata(log.evaluation_id) + details_file = _lighteval_details_file(log, meta) + detailed = LightevalInstanceLevelAdapter().transform_and_save( + details_path=details_file, + evaluation_id=log.evaluation_id, + model_id=log.model_info.id, + task_key=meta['task_key'], + output_dir=str(_output_dir_for_log(staging_dir, log)), + file_uuid=eval_uuid, + collection=log.evaluation_results[ + 0 + ].source_data.dataset_name, + developer=log.model_info.developer, + ) + if detailed is None: + raise ValueError( + '--include-details was requested, but the details ' + f'file for task {meta["task_key"]!r} contained no ' + 'usable rows' + ) + log.detailed_evaluation_results = detailed + details_successes.append(log) + except Exception as exc: + details_failures.append( + SourceRecordFailure( + source_ref=( + f'lighteval evaluation {log.evaluation_id!r}' + ), + reason=str(exc), + source_record={ + 'evaluation_id': log.evaluation_id, + 'searched_directory': meta.get('parent_dir'), + 'task_key': meta.get('task_key'), + }, + ) + ) + if include_details: + details_result = SourceConversionResult( + source_name='lighteval requested details conversions', + total_records=len(logs), + records=details_successes, + failures=details_failures, + ) + paths = ( + publish_evaluation_logs( + logs, + output_dir, + eval_uuids, + staged_output_dir=staging_dir, + ) + if logs + else [] + ) for path in paths: print(path) _save_partial_conversion_report( input_result, output_dir, 'lighteval_inputs' ) + _save_partial_conversion_report( + details_result, output_dir, 'lighteval_details' + ) if input_result is not None: input_result.raise_if_incomplete() + if details_result is not None: + details_result.raise_if_incomplete() print(f'Converted {len(paths)} evaluation log(s).') return 0 +def _lighteval_details_file(log: Any, meta: dict[str, Any]) -> Path: + """Locate the details parquet for one converted lighteval task.""" + from every_eval_ever.converters.lighteval.utils import ( + details_file_name, + find_details_file, + results_file_date_id, + ) + + parent_dir = meta.get('parent_dir') + task_key = meta.get('task_key') + results_file = meta.get('results_file') + if not parent_dir or not task_key or not results_file: + raise RuntimeError( + 'lighteval converter lost the source location or task key for ' + f'evaluation {log.evaluation_id!r}' + ) + details_file = find_details_file( + Path(results_file), task_key, meta.get('model_name') + ) + if details_file is None: + date_id = results_file_date_id(Path(results_file)) or '' + raise FileNotFoundError( + '--include-details was requested, but no details file was found ' + f'for task {task_key!r}: expected ' + f'{details_file_name(task_key, date_id)} under a details/ ' + f'directory above {parent_dir}. lighteval only writes these when ' + 'the run set save_details.' + ) + return details_file + + def _cmd_convert_inspect(args: argparse.Namespace) -> int: from every_eval_ever.converters.inspect.adapter import ( InspectAIAdapter, @@ -675,6 +775,14 @@ def build_parser() -> argparse.ArgumentParser: help='Inference engine version to record in model_info.inference_engine.version.', ) if source == 'lighteval': + source_parser.add_argument( + '--include_details', + '--include-details', + action='store_true', + help='Also convert lighteval details parquet into ' + 'instance-level output. Needs a run made with save_details ' + 'and the lighteval extra installed.', + ) source_parser.add_argument( '--inference_platform', '--inference-platform', diff --git a/every_eval_ever/converters/README.md b/every_eval_ever/converters/README.md index bdc4ae500..afa40ee19 100644 --- a/every_eval_ever/converters/README.md +++ b/every_eval_ever/converters/README.md @@ -240,6 +240,34 @@ the output: under `all`. Those are not converted; the keys that were skipped are recorded in `source_metadata.additional_details.lighteval_derived_rows_not_converted`. +### Per-sample output + +`--include_details` also converts lighteval's per-sample details into an +instance-level `_samples.jsonl` beside the aggregate, and points the +aggregate's `detailed_evaluation_results` at it. It needs a run made with +`save_details=True` and a parquet engine: + +```bash +uv sync --extra lighteval +uv run every_eval_ever convert lighteval --log_path --include_details +``` + +lighteval writes details to +`{output_dir}/details/{model_name}/{date_id}/details_{task_key}_{date_id}.parquet`, +which the converter finds from the results file's own `date_id`. Note that +`results_path_template` can move the results directory but there is no equivalent +for details, so the two are not always siblings; the search walks up to their +shared output directory. A run without `save_details` still publishes its +aggregates, but records a per-task failure in +`adapter_reports/lighteval_details_failures.json` and exits non-zero, so a +missing details tree cannot pass for a successful conversion. + +One field needs care when reading the output. `doc.choices` is what the model was +shown for a multiple-choice task but the *reference answers* for a generative one +(lighteval's own `Doc` docstring says so), so only the first case is published as +`input.choices`. `input.reference` carries the gold in both, and +`metadata.lighteval_sampling_methods` records which kind of row it was. + Full manual for conversion of your own lighteval evaluation log into unified is available below: ```bash @@ -248,6 +276,7 @@ usage: __main__.py [-h] --log_path LOG_PATH [--output_dir OUTPUT_DIR] [--evaluator_relationship {first_party,third_party,collaborative,other}] [--source_organization_url SOURCE_ORGANIZATION_URL] [--source_organization_logo_url SOURCE_ORGANIZATION_LOGO_URL] + [--include_details] [--inference_platform INFERENCE_PLATFORM] [--inference_engine INFERENCE_ENGINE] [--inference_engine_version INFERENCE_ENGINE_VERSION] @@ -270,6 +299,10 @@ options: URL of the source organization --source_organization_logo_url SOURCE_ORGANIZATION_LOGO_URL Logo of the source organization + --include_details, --include-details + Also convert lighteval details parquet into instance- + level output. Needs a run made with save_details and + the lighteval extra installed. --inference_platform INFERENCE_PLATFORM Inference platform (e.g. 'together', 'openai'). Read from the model config for LiteLLM and inference- diff --git a/every_eval_ever/converters/lighteval/__main__.py b/every_eval_ever/converters/lighteval/__main__.py index 9503568b1..21dc4c469 100644 --- a/every_eval_ever/converters/lighteval/__main__.py +++ b/every_eval_ever/converters/lighteval/__main__.py @@ -44,6 +44,14 @@ def main(): default=None, help='Logo of the source organization', ) + parser.add_argument( + '--include_details', + '--include-details', + action='store_true', + help='Also convert lighteval details parquet into instance-level ' + 'output. Needs a run made with save_details and the lighteval extra ' + 'installed.', + ) parser.add_argument( '--inference_platform', type=str, diff --git a/every_eval_ever/converters/lighteval/adapter.py b/every_eval_ever/converters/lighteval/adapter.py index cc50fcd4c..68f7cfa63 100644 --- a/every_eval_ever/converters/lighteval/adapter.py +++ b/every_eval_ever/converters/lighteval/adapter.py @@ -536,7 +536,9 @@ def _transform_single( # Store metadata so callers can trace a log back to its results file self._eval_metadata[evaluation_id] = { 'parent_dir': metadata_args.get('parent_eval_output_dir'), + 'results_file': metadata_args.get('results_file'), 'task_key': task_key, + 'model_name': model_info.id, # Metrics whose direction the run never stated and no operator # declared. Reported rather than silently carried, so the guess is # visible to whoever reads the conversion. @@ -578,6 +580,7 @@ def transform_from_file( metadata_args = { **metadata_args, 'parent_eval_output_dir': str(file_path.parent), + 'results_file': str(file_path), 'derived_aggregate_keys': sorted(derived_keys), 'tasks_without_finite_scores': sorted(skipped_keys), # The results filename holds the only wall-clock time in the run. diff --git a/every_eval_ever/converters/lighteval/instance_level_adapter.py b/every_eval_ever/converters/lighteval/instance_level_adapter.py new file mode 100644 index 000000000..85346b7a3 --- /dev/null +++ b/every_eval_ever/converters/lighteval/instance_level_adapter.py @@ -0,0 +1,399 @@ +"""Instance-level adapter for converting lighteval per-sample details.""" + +import hashlib +import json +import math +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Union + +from every_eval_ever.converters import SCHEMA_VERSION +from every_eval_ever.eval_types import ( + DetailedEvaluationResults, + Format, + HashAlgorithm, +) +from every_eval_ever.helpers.io import datastore_repo_file_path +from every_eval_ever.instance_level_types import ( + AnswerAttributionItem, + Evaluation, + Input, + InstanceLevelEvaluationLog, + InteractionType, + Output, +) + +# The three columns of a lighteval details parquet, one per field of +# DetailsLogger.Detail (lighteval/logging/info_loggers.py). +DOC_COLUMN = 'doc' +RESPONSE_COLUMN = 'model_response' +METRIC_COLUMN = 'metric' +REQUIRED_COLUMNS = (DOC_COLUMN, RESPONSE_COLUMN, METRIC_COLUMN) + +# Doc.sampling_methods, which is how a row says whether the model ranked choices +# or generated text. Doc's own docstring: choices holds "all options" for a +# multiple-choice task but "reference answers" for a generative one, so the same +# column means two different things and only one of them is a presented choice. +LOGPROBS_SAMPLING = 'LOGPROBS' + + +class LightevalInstanceLevelAdapter: + """Converts a lighteval details parquet to instance-level EEE format.""" + + def transform_details( + self, + details_path: Union[str, Path], + evaluation_id: str, + model_id: str, + task_key: str, + ) -> List[InstanceLevelEvaluationLog]: + """Transform one task's details parquet into instance-level logs.""" + rows = self._read_details(Path(details_path)) + return [ + self._transform_row(row, evaluation_id, model_id, task_key) + for row in rows + ] + + def transform_and_save( + self, + details_path: Union[str, Path], + evaluation_id: str, + model_id: str, + task_key: str, + output_dir: Optional[Union[str, Path]] = None, + file_uuid: Optional[str] = None, + collection: Optional[str] = None, + developer: Optional[str] = None, + ) -> Optional[DetailedEvaluationResults]: + """Transform details and save to JSONL, returning a pointer to the file. + + If output_dir is None, returns None (skips instance-level output). + Otherwise file_uuid and collection are required so the samples file + shares the aggregate UUID and declares its canonical location under + data/. + """ + if output_dir is None: + return None + if file_uuid is None: + raise ValueError( + 'file_uuid is required when writing lighteval details' + ) + try: + parsed_uuid = uuid.UUID(file_uuid) + except (AttributeError, TypeError, ValueError) as exc: + raise ValueError(f'invalid file_uuid: {file_uuid!r}') from exc + if parsed_uuid.version != 4: + raise ValueError(f'file_uuid must be UUIDv4: {file_uuid!r}') + file_uuid = str(parsed_uuid) + expected_name = f'{file_uuid}_samples.jsonl' + repository_file_path = datastore_repo_file_path( + collection, + model_id, + developer, + expected_name, + ) + + logs = self.transform_details( + details_path, evaluation_id, model_id, task_key + ) + if not logs: + return None + + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + out_file = output_dir / expected_name + serialized = '\n'.join( + json.dumps( + log.model_dump(mode='json'), + ensure_ascii=False, + allow_nan=False, + ) + for log in logs + ) + out_file.write_text(serialized + '\n', encoding='utf-8') + + file_hash = hashlib.sha256(out_file.read_bytes()).hexdigest() + + return DetailedEvaluationResults( + format=Format.jsonl, + file_path=repository_file_path, + hash_algorithm=HashAlgorithm.sha256, + checksum=file_hash, + total_rows=len(logs), + ) + + def _read_details(self, details_path: Path) -> List[Dict[str, Any]]: + """Read a details parquet into one dict per sample.""" + import pandas as pd + + try: + frame = pd.read_parquet(details_path) + except ImportError as exc: + raise ImportError( + 'reading lighteval details requires a parquet engine; install ' + "the converter's extra with `uv sync --extra lighteval`" + ) from exc + + missing = [ + column for column in REQUIRED_COLUMNS if column not in frame.columns + ] + if missing: + raise ValueError( + f'{details_path} is not a lighteval details file: no ' + f'{", ".join(missing)} column(s). Found ' + f'{", ".join(map(str, frame.columns))}.' + ) + return frame.to_dict(orient='records') + + def _transform_row( + self, + row: Dict[str, Any], + evaluation_id: str, + model_id: str, + task_key: str, + ) -> InstanceLevelEvaluationLog: + """Transform one details row into an instance-level log.""" + doc = _as_mapping(row.get(DOC_COLUMN)) + response = _as_mapping(row.get(RESPONSE_COLUMN)) + metrics = _as_mapping(row.get(METRIC_COLUMN)) + + prompt = _as_text(doc.get('query')) + choices = [ + _as_text(choice) for choice in _as_sequence(doc.get('choices')) + ] + gold_indices = _gold_indices(doc.get('gold_index')) + reference = [ + choices[index] + for index in gold_indices + if 0 <= index < len(choices) + ] + sampling_methods = [ + _as_text(method) + for method in _as_sequence(doc.get('sampling_methods')) + ] + + generations = [ + _as_text(text) for text in _as_sequence(response.get('text')) + ] + post_processed = [ + _as_text(text) + for text in _as_sequence(response.get('text_post_processed')) + ] + logprobs = [ + value + for value in _as_sequence(response.get('logprobs')) + if _is_finite(value) + ] + + raw_output, extracted_value, extraction_method = self._extract_answer( + generations, post_processed, logprobs, choices + ) + + primary_metric, score = _primary_metric(metrics) + presented_choices = ( + choices + if _presents_choices(sampling_methods, logprobs, choices) + else None + ) + + # Build the sample hash from input + reference so the same dataset row + # hashes alike across models and harnesses. + hash_input = json.dumps( + {'raw': prompt, 'reference': reference}, sort_keys=True + ) + sample_hash = hashlib.sha256(hash_input.encode()).hexdigest() + + metadata = { + 'lighteval_metrics': json.dumps( + { + str(name): _python_scalar(value) + for name, value in metrics.items() + if _is_json_scalar(value) + }, + sort_keys=True, + ), + 'task_key': task_key, + } + if primary_metric is not None: + metadata['primary_metric'] = primary_metric + if sampling_methods: + metadata['lighteval_sampling_methods'] = ','.join(sampling_methods) + for key in ('truncated_tokens_count', 'padded_tokens_count'): + value = response.get(key) + if _is_finite(value): + metadata[key] = str(int(value)) + if logprobs: + metadata['choice_logprobs'] = json.dumps( + [float(value) for value in logprobs] + ) + + return InstanceLevelEvaluationLog( + schema_version=SCHEMA_VERSION, + evaluation_id=evaluation_id, + model_id=model_id, + evaluation_name=task_key, + sample_id=_as_text(doc.get('id')), + sample_hash=sample_hash, + interaction_type=InteractionType.single_turn, + input=Input( + raw=prompt, + reference=reference, + choices=presented_choices or None, + ), + output=Output(raw=raw_output), + answer_attribution=[ + AnswerAttributionItem( + turn_idx=0, + source='output.raw', + extracted_value=extracted_value, + extraction_method=extraction_method, + is_terminal=True, + ) + ], + evaluation=Evaluation( + score=score, + # lighteval's per-sample metrics are the metric's own value on + # that sample, so 1.0 is exactly correct for acc/em and this + # says "not a perfect score" for anything continuous. The full + # per-sample metric mapping is in metadata either way. + is_correct=score == 1.0, + ), + metadata=metadata, + ) + + def _extract_answer( + self, + generations: List[str], + post_processed: List[str], + logprobs: List[float], + choices: List[str], + ) -> tuple[List[str], str, str]: + """Decide what the model answered and how that was read off. + + Returns the raw outputs, the single extracted answer, and the name of + the extraction. A generative task carries its answer in `text`; a + loglikelihood task generates nothing and answers by scoring each + choice, so the answer is the highest-scoring one. + """ + if generations: + if post_processed and post_processed != generations: + # lighteval's ModelResponse.post_process strips reasoning tags + # and leaves the original in `text`. + return ( + generations, + post_processed[0], + 'reasoning_tags_removed', + ) + return generations, generations[0], 'none' + + if logprobs and choices and len(logprobs) == len(choices): + selected = choices[logprobs.index(max(logprobs))] + return [selected], selected, 'argmax_choice_logprob' + + if logprobs: + # Scored, but the choices cannot be lined up with the scores, so + # the index is reported rather than a choice guessed at. + selected = str(logprobs.index(max(logprobs))) + return [selected], selected, 'argmax_choice_logprob_index' + + return [], '', 'none' + + +def _as_mapping(value: Any) -> Dict[str, Any]: + """Read a parquet struct column as a plain dict.""" + if isinstance(value, dict): + return value + if hasattr(value, 'items'): + return dict(value.items()) + return {} + + +def _as_sequence(value: Any) -> Sequence[Any]: + """Read a parquet list column as a sequence, treating a scalar as empty.""" + if value is None: + return () + if isinstance(value, (str, bytes)): + return () + if isinstance(value, dict): + return () + try: + return list(value) + except TypeError: + return () + + +def _as_text(value: Any) -> str: + """Stringify a details value, mapping a missing one to the empty string.""" + if value is None: + return '' + return str(value) + + +def _presents_choices( + sampling_methods: List[str], logprobs: List[float], choices: List[str] +) -> bool: + """Decide whether doc.choices were shown to the model or are gold answers. + + Publishing a generative task's golds as `input.choices` would claim the + model was given the answer to pick from, so the two cases cannot share a + field. Runs predating `sampling_methods` are read off the response instead: + one score per choice means the model was asked to rank them. + """ + if sampling_methods: + return LOGPROBS_SAMPLING in sampling_methods + return bool(logprobs) and bool(choices) and len(logprobs) == len(choices) + + +def _python_scalar(value: Any) -> Any: + """Unwrap a numpy scalar into the Python value it stands for. + + Parquet columns arrive as numpy scalars, and numpy's integer types do not + subclass int, so an isinstance check against Python's own numeric types + silently discards every integer lighteval writes -- gold_index included, + which would empty `reference` on any task whose gold is a list. + """ + item = getattr(value, 'item', None) + if callable(item) and getattr(value, 'shape', None) == (): + return item() + return value + + +def _gold_indices(value: Any) -> List[int]: + """Read Doc.gold_index, which is an int for one gold and a list for many.""" + if _is_finite(value): + return [int(_python_scalar(value))] + return [ + int(_python_scalar(item)) + for item in _as_sequence(value) + if _is_finite(item) + ] + + +def _is_finite(value: Any) -> bool: + """Report whether a value is a real number that JSON can round-trip.""" + value = _python_scalar(value) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + return math.isfinite(value) + + +def _is_json_scalar(value: Any) -> bool: + """Report whether a metric value survives strict JSON serialization.""" + value = _python_scalar(value) + if isinstance(value, bool) or isinstance(value, str): + return True + return _is_finite(value) + + +def _primary_metric(metrics: Dict[str, Any]) -> tuple[Optional[str], float]: + """Pick the metric that becomes `evaluation.score`. + + The schema takes one score per sample while lighteval can record several, + so the first metric the run wrote is used and its name is carried in + metadata alongside the full mapping. Returns 0.0 with no name when the row + holds no finite metric value. + """ + for name, value in metrics.items(): + if _is_finite(value): + return str(name), float(value) + return None, 0.0 diff --git a/every_eval_ever/converters/lighteval/utils.py b/every_eval_ever/converters/lighteval/utils.py index 95e542fe6..349de3db2 100644 --- a/every_eval_ever/converters/lighteval/utils.py +++ b/every_eval_ever/converters/lighteval/utils.py @@ -166,6 +166,77 @@ def stderr_method_for( return 'analytic' if 'mean' in aggregation else 'bootstrap' +DETAILS_DIR_NAME = 'details' + +# How far above a results file to look for the sibling `details/` tree. +# results_path_template lets an operator move the results directory but not the +# details one, so the two are not always siblings. +_DETAILS_SEARCH_DEPTH = 6 + + +def results_file_date_id(file_path: Path) -> Optional[str]: + """Recover the raw date_id lighteval stamps into a results filename. + + This is the filename form, ':' replaced by '-', which is also the name of + the details subdirectory and part of every details filename. Use + parse_results_file_timestamp for the ISO-8601 form. + """ + stem = Path(file_path).stem + if _DATE_ID_PATTERN.match(stem) is None: + return None + return stem[len('results_') :] + + +def details_file_name(task_key: str, date_id: str) -> str: + """Name the per-sample parquet lighteval writes for one task of one run.""" + return f'details_{task_key}_{date_id}.parquet' + + +def find_details_file( + results_path: Path, + task_key: str, + model_name: Optional[str] = None, +) -> Optional[Path]: + """Locate the per-sample parquet belonging to one task of one run. + + lighteval writes details to + `/details///details__.parquet` + while the results file it belongs to sits under `/results/...`, + so the two are found by walking up to the shared output directory. Returns + None when the run was made without `save_details`. + """ + results_path = Path(results_path) + date_id = results_file_date_id(results_path) + if date_id is None: + return None + expected = details_file_name(task_key, date_id) + + directory = results_path.resolve().parent + for _ in range(_DETAILS_SEARCH_DEPTH): + details_root = directory / DETAILS_DIR_NAME + if details_root.is_dir(): + # A model subtree first: a details root can hold several models, + # and matching the run's own model keeps a same-named task from a + # different model out of this evaluation's samples. + roots = [] + if model_name: + model_root = details_root / model_name.strip('/') + if model_root.is_dir(): + roots.append(model_root) + roots.append(details_root) + for root in roots: + # Compared by name rather than globbed: a task key carries '|' + # and ':', and glob would read a '[' in a task name as a + # character class. + for candidate in sorted(root.rglob('*.parquet')): + if candidate.name == expected: + return candidate + if directory.parent == directory: + break + directory = directory.parent + return None + + def parse_results_file_timestamp(file_path: Path) -> Optional[str]: """Recover the wall-clock stamp lighteval encodes in a results filename. diff --git a/pyproject.toml b/pyproject.toml index e252f670f..04b8a981f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,15 @@ helm = [ # because crfm-helm is frozen. "nltk<3.10.1", ] +# Only what is needed to READ a lighteval run's output. lighteval itself is not +# installable alongside the helm extra (it needs datasets>=4, crfm-helm pins +# datasets~=3.1), so it is deliberately absent: producing a lighteval run is the +# upstream smoke job's business, converting one is this package's. +lighteval = ["pyarrow>=17"] all = [ "every-eval-ever[inspect]", "every-eval-ever[helm]", + "every-eval-ever[lighteval]", ] [project.scripts] diff --git a/uv.lock b/uv.lock index 4065c6532..aa48e4f94 100644 --- a/uv.lock +++ b/uv.lock @@ -866,6 +866,7 @@ all = [ { name = "crfm-helm" }, { name = "inspect-ai" }, { name = "nltk" }, + { name = "pyarrow" }, { name = "typer" }, ] helm = [ @@ -876,6 +877,9 @@ helm = [ inspect = [ { name = "inspect-ai" }, ] +lighteval = [ + { name = "pyarrow" }, +] [package.dev-dependencies] dev = [ @@ -891,6 +895,7 @@ requires-dist = [ { name = "duckdb", specifier = ">=1.5.2" }, { name = "every-eval-ever", extras = ["helm"], marker = "extra == 'all'" }, { name = "every-eval-ever", extras = ["inspect"], marker = "extra == 'all'" }, + { name = "every-eval-ever", extras = ["lighteval"], marker = "extra == 'all'" }, { name = "huggingface-hub", specifier = ">=0.36.0,<1.0.0" }, { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.160,<0.4.0" }, { name = "jsonschema", specifier = ">=4.26.0,<5.0.0" }, @@ -898,6 +903,7 @@ requires-dist = [ { name = "nltk", marker = "extra == 'helm'", specifier = "<3.10.1" }, { name = "numpy", specifier = ">=2.4.1" }, { name = "pandas", specifier = ">=2.3.3" }, + { name = "pyarrow", marker = "extra == 'lighteval'", specifier = ">=17" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "requests", specifier = ">=2.32.5,<3.0.0" }, @@ -905,7 +911,7 @@ requires-dist = [ { name = "seaborn", specifier = ">=0.13.2" }, { name = "typer", marker = "extra == 'helm'", specifier = ">=0.12,<1.0" }, ] -provides-extras = ["inspect", "helm", "all"] +provides-extras = ["inspect", "helm", "lighteval", "all"] [package.metadata.requires-dev] dev = [ From e58400ca81f469ac70faba6955af5a2687d13999 Mon Sep 17 00:00:00 2001 From: leshem Date: Mon, 10 Aug 2026 17:54:46 -0400 Subject: [PATCH 5/7] test(lighteval): pin details conversion to a run lighteval produced itself The fixture here was not hand-copied: `scripts/upstream_smoke/lighteval_smoke.py` drives lighteval's own `Pipeline` with their `DummyModelConfig` (random logprobs, fixed text, no weights and no inference), so the parquet layout, the numpy dtypes and the task-key spelling are all upstream's. Two tasks, one multiple-choice and one generative, because a fake model only exercises the metrics its task defines. The test runs the real CLI and then the real validator at the canonical `data////` path, which is where the semantic checks fire -- so it asserts the converter's output is submittable, not merely that some fields have the values we expected. Exit 0 only: warning-only is valid locally but not merge-ready. `--refresh` regenerates the fixture against whatever lighteval is installed. Read the diff before committing it; it will just as happily record an upstream regression as an intended change. Scope is shape, not semantics: a metric switching percent to proportion passes this, as does a changed prompt template. --- scripts/upstream_smoke/lighteval_smoke.py | 278 ++++++++++++++++++ tests/data/lighteval_smoke/PROVENANCE.md | 17 ++ ...li:r1|0_2026-08-10T17-22-11.385681.parquet | Bin 0 -> 16931 bytes ...ad_v2|0_2026-08-10T17-22-11.385681.parquet | Bin 0 -> 19438 bytes .../results_2026-08-10T17-22-11.385681.json | 187 ++++++++++++ .../test_lighteval_instance_level_adapter.py | 197 +++++++++++++ 6 files changed, 679 insertions(+) create mode 100644 scripts/upstream_smoke/lighteval_smoke.py create mode 100644 tests/data/lighteval_smoke/PROVENANCE.md create mode 100644 tests/data/lighteval_smoke/details/eee-smoke/dummy-model/2026-08-10T17-22-11.385681/details_anli:r1|0_2026-08-10T17-22-11.385681.parquet create mode 100644 tests/data/lighteval_smoke/details/eee-smoke/dummy-model/2026-08-10T17-22-11.385681/details_squad_v2|0_2026-08-10T17-22-11.385681.parquet create mode 100644 tests/data/lighteval_smoke/results/eee-smoke/dummy-model/results_2026-08-10T17-22-11.385681.json create mode 100644 tests/test_lighteval_instance_level_adapter.py diff --git a/scripts/upstream_smoke/lighteval_smoke.py b/scripts/upstream_smoke/lighteval_smoke.py new file mode 100644 index 000000000..fdfa1b3ac --- /dev/null +++ b/scripts/upstream_smoke/lighteval_smoke.py @@ -0,0 +1,278 @@ +"""Run lighteval's own fake model, then convert and validate what it wrote. + +Answers a question the offline tests cannot: did a new lighteval release change +the output the converter reads? The fixture under tests/data/lighteval_smoke pins +a *past* release; this script asks the installed one to produce a run right now, +converts it, and requires the result to pass the datastore gate. + +No weights and no inference: DummyModelConfig returns random logprobs and fixed +text, so the whole thing is a dataset download and some file writing. + + UV_TORCH_BACKEND=cpu uv run -p 3.12 --extra lighteval --with lighteval \\ + python scripts/upstream_smoke/lighteval_smoke.py + +Add --refresh to overwrite the committed fixture with what this run produced. +Read the diff before committing it: --refresh will just as happily record a real +upstream regression as an intended change. + +lighteval is deliberately not a dependency of this project. It needs datasets>=4 +while crfm-helm pins datasets~=3.1, so the two cannot share a lockfile; this +script is therefore run with `uv run --with lighteval`, never from the project +environment. + +Scope: shape, not semantics. A metric that silently switches from percent to +proportion passes this. So does a task whose prompt template changed. And a fake +model only exercises the metrics its task defines, which is why the default task +list pairs a multiple-choice task with a generative one. +""" + +from __future__ import annotations + +import argparse +import shutil +import sys +import tempfile +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# Deliberately not under tests/data/lighteval/. That tree is a hand-collected +# SmolLM2 evaluation, and a test there walks it recursively and counts what it +# finds, so a second run dropped inside would change what their fixture means -- +# besides putting a generated tree where --refresh could overwrite a collected +# one. +FIXTURE_DIR = REPO_ROOT / 'tests' / 'data' / 'lighteval_smoke' + +# One multiple-choice task and one generative task: a fake model only produces +# the metrics its task defines, so a single task would leave either the +# loglikelihood path or the generation path unexercised. +DEFAULT_TASKS = 'anli:r1|0,squad_v2|0' + +# Slashed on purpose. The datastore path is +# data////, and the developer comes from the part +# before the slash, so a bare model name cannot be published at all. +DEFAULT_MODEL_NAME = 'eee-smoke/dummy-model' + +PROVENANCE_NAME = 'PROVENANCE.md' + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__ or '') + parser.add_argument( + '--tasks', + default=DEFAULT_TASKS, + help='lighteval task specification, comma-separated ' + f'(default: {DEFAULT_TASKS}). Exposed so a task that upstream breaks ' + 'or removes can be swapped without editing this file.', + ) + parser.add_argument( + '--max-samples', + type=int, + default=2, + help='Samples per task (default: 2)', + ) + parser.add_argument( + '--model-name', + default=DEFAULT_MODEL_NAME, + help=f'Name to record for the fake model (default: {DEFAULT_MODEL_NAME})', + ) + parser.add_argument( + '--output-dir', + default=None, + help='Where lighteval writes its run. Defaults to a temporary ' + 'directory that is removed on exit.', + ) + parser.add_argument( + '--refresh', + action='store_true', + help=f'Replace {FIXTURE_DIR.relative_to(REPO_ROOT)} with this run', + ) + return parser.parse_args(argv) + + +def lighteval_version() -> str: + try: + return version('lighteval') + except PackageNotFoundError: + return 'unknown' + + +def run_lighteval( + output_dir: Path, tasks: str, max_samples: int, model_name: str +) -> None: + """Produce a real lighteval run with lighteval's own fake model. + + Only the public entry points their docs use, so an internal refactor + upstream does not break this and a user-facing change does. + """ + from lighteval.logging.evaluation_tracker import EvaluationTracker + from lighteval.models.dummy.dummy_model import DummyModelConfig + from lighteval.pipeline import ( + ParallelismManager, + Pipeline, + PipelineParameters, + ) + + tracker = EvaluationTracker( + output_dir=str(output_dir), + save_details=True, + ) + pipeline = Pipeline( + tasks=tasks, + pipeline_parameters=PipelineParameters( + launcher_type=ParallelismManager.NONE, + max_samples=max_samples, + dataset_loading_processes=1, + ), + evaluation_tracker=tracker, + model_config=DummyModelConfig(model_name=model_name, seed=42), + ) + pipeline.evaluate() + pipeline.save_and_push_results() + + +def find_results_files(output_dir: Path) -> list[Path]: + return sorted((output_dir / 'results').rglob('results_*.json')) + + +def convert_and_validate(results_file: Path, data_dir: Path) -> int: + """Convert one run through the real CLI, then the real datastore gate.""" + from every_eval_ever import cli + + exit_code = cli.main( + [ + 'convert', + 'lighteval', + '--log_path', + str(results_file), + '--include_details', + '--output_dir', + str(data_dir), + '--eval_library_version', + lighteval_version(), + ] + ) + if exit_code != 0: + print(f'FAIL: conversion exited {exit_code}', file=sys.stderr) + return exit_code + + published = sorted(data_dir.glob('*/*/*/*.json')) + samples = sorted(data_dir.glob('*/*/*/*_samples.jsonl')) + print( + f'Converted {len(published)} record(s) and {len(samples)} ' + f'instance-level file(s) from {results_file.name}' + ) + if not published: + print( + f'FAIL: conversion wrote no records under {data_dir}', + file=sys.stderr, + ) + return 1 + if not samples: + print( + 'FAIL: --include_details produced no instance-level output; ' + 'lighteval may have changed its details layout', + file=sys.stderr, + ) + return 1 + + # The validator's own entry point rather than validate_file: the semantic + # checks only run when a record sits at data/// + # /, and this is what resolves that context. 0 clean, 1 errors, + # 2 warnings only -- and a warning is not merge-ready, so only 0 passes. + return cli.main( + [ + 'validate', + str(data_dir / '*' / '*' / '*' / '*.json'), + '--format', + 'rich', + ] + ) or cli.main( + [ + 'validate', + str(data_dir / '*' / '*' / '*' / '*.jsonl'), + '--format', + 'rich', + ] + ) + + +def write_provenance(fixture_dir: Path, args: argparse.Namespace) -> None: + lines = [ + '# Provenance', + '', + 'Produced by `scripts/upstream_smoke/lighteval_smoke.py --refresh`,', + 'not hand-copied from a real evaluation.', + '', + f'- lighteval version: `{lighteval_version()}`', + f'- tasks: `{args.tasks}`', + f'- samples per task: `{args.max_samples}`', + f'- model: `{args.model_name}`', + '', + "The model is lighteval's own `DummyModelConfig` (seed 42): random", + 'logprobs and fixed text, no weights and no inference. The scores here', + 'are therefore meaningless as measurements — this tree exists to pin', + 'the *shape* of lighteval output that the converter reads.', + '', + 'To update after an upstream change, re-run the command above and read', + 'the diff before committing it.', + '', + ] + (fixture_dir / PROVENANCE_NAME).write_text( + '\n'.join(lines), encoding='utf-8' + ) + + +def refresh_fixture(output_dir: Path, args: argparse.Namespace) -> None: + """Replace the generated fixture subtree with this run's output.""" + if FIXTURE_DIR.exists(): + shutil.rmtree(FIXTURE_DIR) + FIXTURE_DIR.mkdir(parents=True) + for name in ('results', 'details'): + source = output_dir / name + if source.is_dir(): + shutil.copytree(source, FIXTURE_DIR / name) + write_provenance(FIXTURE_DIR, args) + print(f'Refreshed {FIXTURE_DIR.relative_to(REPO_ROOT)}') + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + print(f'lighteval {lighteval_version()}, tasks {args.tasks}') + + with tempfile.TemporaryDirectory(prefix='eee-lighteval-smoke-') as scratch: + scratch_dir = Path(scratch) + output_dir = ( + Path(args.output_dir) if args.output_dir else scratch_dir / 'run' + ) + output_dir.mkdir(parents=True, exist_ok=True) + + run_lighteval(output_dir, args.tasks, args.max_samples, args.model_name) + + results_files = find_results_files(output_dir) + if not results_files: + print( + f'FAIL: lighteval wrote no results file under {output_dir}. ' + 'Its output layout may have changed.', + file=sys.stderr, + ) + return 1 + + exit_code = 0 + for index, results_file in enumerate(results_files): + data_dir = scratch_dir / f'converted-{index}' / 'data' + exit_code = ( + convert_and_validate(results_file, data_dir) or exit_code + ) + + if exit_code == 0 and args.refresh: + refresh_fixture(output_dir, args) + + if exit_code == 0: + print('OK') + return exit_code + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/data/lighteval_smoke/PROVENANCE.md b/tests/data/lighteval_smoke/PROVENANCE.md new file mode 100644 index 000000000..cb2e0b3d8 --- /dev/null +++ b/tests/data/lighteval_smoke/PROVENANCE.md @@ -0,0 +1,17 @@ +# Provenance + +Produced by `scripts/upstream_smoke/lighteval_smoke.py --refresh`, +not hand-copied from a real evaluation. + +- lighteval version: `0.13.0` +- tasks: `anli:r1|0,squad_v2|0` +- samples per task: `2` +- model: `eee-smoke/dummy-model` + +The model is lighteval's own `DummyModelConfig` (seed 42): random +logprobs and fixed text, no weights and no inference. The scores here +are therefore meaningless as measurements — this tree exists to pin +the *shape* of lighteval output that the converter reads. + +To update after an upstream change, re-run the command above and read +the diff before committing it. diff --git a/tests/data/lighteval_smoke/details/eee-smoke/dummy-model/2026-08-10T17-22-11.385681/details_anli:r1|0_2026-08-10T17-22-11.385681.parquet b/tests/data/lighteval_smoke/details/eee-smoke/dummy-model/2026-08-10T17-22-11.385681/details_anli:r1|0_2026-08-10T17-22-11.385681.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7b76e67c53f92e6ed3c35f4f57f78946aaa9a2d2 GIT binary patch literal 16931 zcmeG^TWlLwc9cYG9Y@t}n#_dsYLi(b5gb^N_>g4NFwh-JA}K|pC0QmZepHwj$&uzk zbB3a*G(bQ4*pEe#Edpa}EOvupkru%sz!v=|7C{UcK~NO^>7pNj5#(c$y2f@t7O6jj zpy%9qaAr89WhW^X*bHp)&b{Y-&OP^>E6bbgc9M!vZ;Zg78EOKC&NkY%74qhd2u)Jr-_7(_>5Zi1Q!E{+VIo}p%f z(4!+D3O>#^V26RBGWa;(0Ct`_-$!i>KrUWf#IJ?K#leyckUNaCBbf6&nCP8J>ds`J zNIwO{B4(iNW`OLMAApN;ZAFnx5FQ-EF^96EaCNXED9kZ}4DU=6jo)BoOY;>D zthp|0q9Jt|5xfEhB9?m92d@qC9O;Y{ICIa9gVmMh7xL6N>ou$qW zI5LpPfCTwChrtnbG=R~qf9oXX4nMj5bN#RX^x(&Tcl+M>=RbVr)_>gopMPff3xE6X zw|{X@eP-r||9$(_SN`{%zx;>)x(#v-72t%L52f+=iznnBX4&{tqw(0dm7(IXQzT(g zqj5SCpN^i6m(XVu5dw4?qX?gP%pDTQmlh+Ds14(|8(^Ga5M~UH$9QZE&dT75OLho{ z2V?FY`#N3z!KBN0?5AVFqxI`b=J%?S>D$cC2?m?l84?&sEDD&Io#X)toM8~ggV~?(oI?QcopZvGGLRY_@Q8GO4>^#Eyt9;d;7Q}ib^#S^d>@a=V`HK&P%q%<8$v^u1<9aBsqytIz^mq%mYtVtcCp__XKr$U7ojymc=Ro(?e3TG%66)J97 ztM3U4XBfTpKdwuf)FM^xUcJSs&{bKBQ9%BtXVH+4r3i(yXX;SQ??HPY9a#d8%AzeR zVL1-!jY2>$1ivdpFurt80OxMNX0To;|)r?10*+ zlG)J3aD#-{lE`=3Xi9<%y&_=%=AAT5y}4&dz(som#}H>dVRFX7p2n$?*91?up;6LA z88wzfuR0MlVB3)P6umB+26}fKlz@9TLW^Z1?g@HZGtnStTF^x58pJ}l0DU?DPSli+V7G;AzcA>h z5qhtw8}I>_70>`m;Rf$RChnx+G5}gzXpH@`lTf7napIVg*p`mQr{bSIKOLE(CgLxS zN8$y{Ax`!n8N+ce6`?0+$nC!znLd4GF>>CjvYo2?siz>nL4N^35V=CHdykPQF@l1 zo&L;AY&^aQML!VwRS2PwtmRU9d(pB}!Bz;snzhAyK4@(;tpr>9rue5gj@2 zb`ZT;pLt(GhA21vIWVN}K2H^h5jCB zQ4`WLVYEiCO+OgJ_`u#MhHa9M6g?4lrGI-7!s0g!H6L`Lo!nh(zZVU&_6HzK5Yu}K zxahwPMgPb__n6IlT)1|-b?S>d>Ql7aPrhs0&DsIPZh_;-mHGauFqOZ5(N@`GdF$LF zh*0_5pParqj^_Wm)%ZQXG>v_I3Gjz_XX?*}Z%y{n;ZH7741Mpr)UPH7@SgA6yehw0r`;G0nGjrnZ28`Pbo`-La;>$Z4QDC zo}jPLSEfG&nT9Coxtop9zd_!Ak+z*|pO69GR&U{Y;GKAw2mUv-q716{Fux`!;^iTK-llD3N8ZkH78F(r2!_j;40p1~d(8x7>LC(bN}@ z=idQ;`t^BV)fLpx)7yoscW-KtdVh6!Xrz%2Fr4O_-x);n$HQrc^@-su?ha(}PY#Q4 zuMFrVUB|rf+!uGuzk&$#hknNdH`9Q&)!kgbe$HbbaW4nQgdUAun(+ zl5hKR&I@3zo82bLa1MVOuGP;0t379`#3^m*+-u&cb-2$XGj7Lg-P{ePtHBTHADycA zQqM2v5PF_lJ^RY>z9A;`{&|dUu9?Uc+!?(GNS+^wy=LA7O|MJXew@ z#_?QO`f-LS;lI=L;|-CI+<5w-hpHdE5hbX0gg*iqH>TYHb}mc(M21N2zfAqoL9T~ zlM=c4^@}&e65e~_!2%2+&>vLR74nHCaPCD1bFV2nH#qt93*ftk9!&O;lnS|__G>sW z1i(qXYwRyc8oags5=jLvz`rcYp?Bjax0-*0`@+OMl?M6nHFzSi33hw?YWcGDqq^9e zvvc?)*=?3(x8N^6udwVqo2|1smaVf(buyRD`qn!*x#F&`)K{%{_#{-wI?Q!gz=Ll> zL|B4lc!y^m=CT)8Ri#~3FUePB99Nay%63vvOUJAGdR^+S9q_3Qd#$MRDP_M}ao!!+ z>-PF~LaJmH#9P&h2?6PLSB=%BwVGH^I@N2di!d(lmXDIvYipVHC04Gaw-1N(-sRp_t1Y@0ODtKJUt-R?E4BSXm={P(BaswpczgR^?88Rmt>ZQK}8T zP*PU4L{OFuu2M|c^J=vs9&`CjqLMBi3fgv)SH;?H>Zl>49eq9esS8Oy-;skll+>Id zmaogTYaoLvD!f|if<2pjYN1w5uQm8`vLVP|hk_wj3q9R=tz_^^!);YfmD<(HT0_h$ zhrE(G<|@rF8#H-sD^+|bt&CC4WkAPmMOx~Yhbu26iu+p+(N8&WV#MQvW3jLXbcKiE zn!9CD5xV`nPhty@5BVf_(`eiI{`fnQf53;OzWDOS(}`@!r`MX*l+uQ{t_hWrB0wu) z?WISI7cG|r|1LbXO|@c!uVf9djaEq|6*1p9tQMTu{p$9rkj^T*DMjVV4zEYmyO`k9 z*_x0l9f{>qdpC8Wy?(Md^X7qrqo$DFvi7F7-4Ie{jaPFf$xDI#D5ST$Xvc@-k|eJd z4`B~A$(~e82bJ_dAD$#PCBX(YKA+LJ%C?(-J$p;$Q<)+8ElF}JUoK(p-M8bYvFqk< zuV3Jw4lnN}2J7e0${bWNuHifo@|m`fa&mpHTTLY!QZ8fgDX}RQinVGbSF3`2RVSaQ zrG{82b;Intt7Z-#mhT3g4d9oKSlN2KJqBlzP$)r;-#+9ZpSt;Hh^!~cM}uS?;>(l7 z1KPGDs2TYra#q!&!`;fmbN@hD(-1fRkG2u=xr1sR_l)g}WM-dFm)fEeKc4dueE#tB zNLz#21@|M=9qySuB=4Q%j2N6h6sh3E;8)3Gy9WF280sL51E{@<$JO#z&bKR;7xtlU zQJ_Xh@F`Q_wIb9)nIzN}A^YxOxrEeR{kbxAbXZMiyL@UbEZ03`e^fGm zT%n}l8C*?OA>Uocl9z?b8EWI@1dd6)TUbud&0T_pl_DvRp|Zvo{8+}10)Ayz5Pkv6 z<@Gw0%Gv8!HfIoQeGABBcdgnQ-q(-`yc1OXoQJF)f^bP%mWKSQ7{!RtE9A$F}`eEPqP(bQzgUlY_ zr`3;^;4K_#23S zw5i$gbTWUL2!FxV-;NjLC;*#m^HY=Y7x-y}%X7040+AgER;T*4Ros>uhUs!0#USrDkbVJ)T9OeaNE7hk{)1ga{kx8QD|xcs7q2?P`)YWY zxIPE@X=n4;7cXnD;LV3Cs9FJgpG><7d66r_cNxxptDj`AA^yC}KNrp)Lm``h8mLuC zEx=s^iKhheZ@K(C#Qq-sL}AOHWKe`okt`Hg;dTI@<-&(2Kyhl$2mZ` z;YlEWeiM#fHL_fSPglBaVwhqzP%IBW_O4gTf z4f7lHxnj9U;*HJjz*fy{%t5HbYL?tavTQIv5YusRj|KNp#{hQ;;YF99WnW8*o_%o- z0zU}QY#rfM7v8F0`s5?<;vt{kuECjmP_4kk2AaSZa3_*f!A=&e@}ip908 zRxK|mf|?^bg3X$edL0u3Y;HJDI+z&jhwKb?1DJUA_nfB$?izju?&36{Z}R0_7nW8L zf6MX5mSz7}2x`wBR=%z(VjAvL)3DD|nJ(NRw5!+y<#Wd*c2|~xQU+=iOP-cq;1>e8;aRzi>;yymZVe)4bhO_@3F;WIXNQyQc6~OH*xL zHGoUfR9&=z>Buw`YlgqwR4i=p(AE@9Rl14{ETYYq%r>yK4SuE5Hd_iW8uUQnH&pqu zU~8g5h-~?&Fx!ho(SiJFJo;`^$i$-u3NM}1zjZPo6`iZ#t!%3%llKpXBTbyOSB z%ArXdN*&Gav=xUigjCruyO5XE0Bj;@Na2;P zsgVsVQ&J(4Ww$#L3GhRL=sPS~iem7h1vHY_5^E0cR!3?ANnH``8UB{4t8f+4qS;o# zm8hwf&Z`!7bXBG25Of8vi53aOcv9N|j{|+plwc3jm>J*dC>E`*@jFeEmmvicA;vw) zTX(_EHf6G=I@XRAv4N8ahWXP_{3Mmx&$z?K&pa4g# znRb(JXl4!UsD|RRVvEp{Irv7~>;XZo&%^swo3t%98f~RP zUMnkbKY3=5%J_U+G!9I7IXv9#s?s#yYn#~Vu#{yiP4g4CVLqlv#v%-1>a@q_4!*Bj z9_`=X`6rmckw(?wrDR}#s|5FGDq4%TItUBUaXBWaCahdj7;@$vJt#=)({|=WZABsx z`wFtc3_mmD__Iz(+KOnACo;OQyVX?dNCOtL8u+E>qzmwaKxb|YCv?IQUZuigNM;#I z)XbI+GBYqd{I1pPTTWEEyO2`g`jALBEgP{Yc7Yd;hJ2wKT{sN^yTNpzI5B7N$VokN zVVAs3q0tn(2qr{#yMZV)5laRl)h1_ctMG2uUwiOBC%u6d0l=CtW}D)lgrNuo2;39i z?3xBG!LL&p-h!2Pv6>5ghcWmA;dydf}YJoN-u`PijL#vAtT#xb)na6_Ti7+p+ zAd8%!4Jl$Q1-fVAI(D9!2!$3LUZku;P9OC}#&;xq{2)W6+2l5(1;s6iRIdvwrIl^r zcCow?zB;)?{bYpu$;dRycc3wNVu9L&^J7%t#-GBMpBL!jEiXyj=P@uYbIVa?M~N{D z0WR-#bAO)+UjZbh{5Sf#s zN^(8sSttalP(~r6`k=wVV{SoFj{wCfC5dap($9~kQ!m7HsOi-BAggnu>0}~3IhIJ5 z$p#}~v;;Yukcsliz-?e#r z&N+4Ryi+H~2Q3*e4lbe<>`Au|OQ*b5bjL^{of=6*@A`!`T4MuJoo3Iok$xqeir(?$ z3-(K&A^H9YcMVsx{BnewahLZ7%b$;OCHL;#QLgGP zfAs{rn@&DQkUt&sA-^0f?~VD8U!3tb+&#m61m^(#|4tF?hc9w%c6o`e-sAm^pYYsg z4x=~d$=x4t^})WcCm1W~6iN;@`--@vM<^nUiQ&`c+tQ6>x=%~KM(GH zI_YotX3|IY`78e2Z(rfwbr`!&#P7Tsx_tLFALgsq9Q@o^`cFTcPo&Obs)e&X#+mb@ zsV8Jp;$EiJlbCg>l45bA+}Rt~VT<{()EGu1HaChfGxqGb)9|FtuRq66z5hJ3n?i46lD1Mstf zhn9Mz5HI5mQ}_5fgYA(0Pw)KCpk+e5Nw_nobzurus>pswebnn+k(v74qGudf(YGycw5Ek#o6s>BFD;%6+| zZ0%VXme646^vH<7Q?_Uw>=~l2cq*W~FLW@h!SIo^uE>r!-tz>Wk9RC(Pcs{;ZIQ~) zLJSNa$UjKCql!*UH z{XDzcv&{p=h!;E_%UOqc56wATVlYxWB?`##S7yfs9}h;El=DJMlrf%l0r%J}j=3!+ zA{a)|Sm<=x025=ja)_urb(p?D`}bNHLG3|WxC&;Prv+!0O=8lDc`D|RVg8w4Cc{qS zsg&tBIhMYV{?m(-i3{BM^u@D@^cv+GFVnE>;>9GFNRA~@5`Hu?dG7i`;!@y<8heO+ z{&{3QoUsvuiR;O0!4qsE`RN9CA^8teoRJ{XqXW{>jCGuaYsqVq?~fFH)UUcnV|o>m zNPe}!B?CAwvGf_6%9uPe$(hM7gUHUN7jVP|qW_6O6qECOEcaru+$p*2j4=@zJJICG zP?VCT$)As2ADrt4t7PJmf7&114XO84Vw?C*eh7leyHng6Q-VX4*)))TkL$FWtWMq~ z>ZC*Z3|2(lxfi?}rXCnhB$EG19DF*>-DVs(Ud7}_I`>8W)f41GY~GM!!-o*UfWhz3=mz_N9oDCXUG7L zkGPLtZP0td$FDZNw^tjf$FtGL@5J$f@h|C}c&L8id_JhwpWNj5gG4$y2-eX z9O0ropHE`+@Y%@ZCp>XIm_?;V#>2ec0I&XFDv^9|llzH)u=AX?r8%pZk?sazOP`GTl? zP~?^wmBgUO4ID%n@=P)_`HPpXV}OSL&=j%X5ZXZ-^xI9eL0=WQX@=BoyxgNKj<5_L zwadQ)zL?e&*kkUVKSbuZQ#UXWPhiAH2b}hA-WBX2+nvZ%nPeJ)x_HbG=9N+KFp? zZXuVwas_`aMpHqy(|eN5T)8^``T`yqW?SFW=&!5*ka;JOdb-(ZVA9&Ci;|N1__?B9g+G!&TPpBg4G;}63O6D#~0APhHzKlYL!ynCql(U%Ae)b;-d zKy%y(-us0xk|CUj-#R;I1)(7gtH&^`Ug+|f2#y7*l zp5tCm033c9lB-L2jB7zyx`KQ8*NeK=sp?nNn<}lV+HPe#E9vE<;=b8X`s)X^+=lzC zWY%)pezoFVdwbt>pKoWBN?s$lqEX67gtuR`ic9Ntc}?q8Zxt7?-r3zb%vNu$FWy)Z z)XLoUyi_1~LA_njj^rihepN5eNx5cMUR^xe-O2W91)m?iy0e;*E9;C7@fX0|k#`QQ zqS|W|wZ#E1TD4hQD{Dm~6Xj(?tduhDeZ5+dkHo@arZQLRO2&4prpxu+++kCi^W+W4 zr_X0=g`OIfp{%c3^3I!T{T6u8Wv!-{`_N~rmYc6v=hmCGoorK5p@%g~t*#AZuNh^l zwlrK<)m*t#t*kfYg4V5Ri$`Lm6{mw%&DhG7?u%>Ds;(|V#%)bma{1BAO?j={XLgG4 zC+^H=O8Z;G1LTL*n>VG$#fp4Od-JK&F zTsiKir?Mk#T|VZALBBhd&TDh)t!hr|!2jz~rL0Lfk~lHo{xWjF)hvIwcVVii#r~z2bH^&sm8o~JTu5`fId{CKtSR0m)s^vwimXlkuZ@ZvA z*Sodcko;6(xzMgwh=0!>Wjn^F?(5r5uFH0-g|bzdTi0OcZp|opxv44Xvb|er)@y}D zL#%8cRbanLPVUwUTdw~P@wwB)=Sj9h{gKwnnHZT)QzsyIK8yO3spV{~W_UXKzIZ+C zA~$5dYn~(BN^9%g-S3^x#jL#c|EL!u->eq4HPkBC4kgs&#Vl%EOzzY=^73xxKC#$G z|8RtQO!eLCalWsfB2%q2J^vd<<6(1@rmSt(rNZJ7{Iw_Pi|ElxN7WrqM?-w}A^S9{ zt;lWYNhqgncSZClQg(5_HdpS*USARDb57Mq7?j%=JJo`g5qI#rv(0ix*uR6h?moR( zCA(O!WYF+h)cKn7j%-lQuIj7%)h%I}TH#yNFyHLa8vV2&+}^Cu&Qkllm8DDJP4>;2 z>J3_o0)Cch|FT4FwMFgq>K0n- zpZ*FXio*atY{^y()!I-!;RJCs2UG`y4?uaL$v{o|b{J#VJM=x&j(A?&G#q`c6L#k+I0BK)9FVLcIhj? zAF~JQhE`+Z=Qn77GeUojfE@iWP=`wbaOktgYx|%#)q5~{OAXqei_l~81?UwR0a9%~ zlwV-!ErMV2^ta{M)7d|;Cu17^Ufm%-6@)3GceFe|J9`yZD@#lXOKd+|vQ3VDd%HW^ z2NWZ6VX<9Bzgx+bx-$B8p|HQ;zzcZ>Svd0WZ7}@=@iA4g^J0l`BYrbfJN=lZM}>XB zPc^0x8o0^}r?S`57VYo*@;m+#uRjWkJoPk`!&Hf}vn&w&ln;M8{VmY`{<16oN}eeV zhiQBU`eUxf=ur;^(zJiEIMsFJTXwW|KtPT^GEgUe>1M&{FG4yJ@+$;i-)y__aXNl$ zw146DefXtN%R7Au(O;+ivl|h9Dai%-W&E#_eG9AoYA)MUR=vKlRIgT6>(w02f&^4z z`YKL&`QlXeUA{u}H#YG__S(v@d=MXx6xw;O=fwQs_UkPEJ$g=lV(@|Zr*VbA;-Yx_ zYRI39`+#ThQQDzRtIT1Tkygmm%Uf74zlrgOpi{msuv{~hh`*96M)H@kj|@UJodq?PMLEWgowMCg+Ivm_7Wr@)xV5BOm~0?Q9sKDk0Zvs)>(E4g{hBA7oFXOW*8 zK7A)X1N50G9D80N_#@<*9%dvO(MIjpC{F7H|9X^uG~P&_JkCI68w=DY+CM81eR|;L z6Bb_q`?FLLI6E@X4C8knrH$#^&1V66M2dNilcc5Gsx1``*-WRlQ_f({bKv9`p*V~D zvbBj)Njlzg^j~092Ks0G1oHC+?a%r2U3~}WGkB(7I;-{-LbXc!r?!Y5t~_}eP(a?B z1R66dKoKVRS;2>Q;w7rD8w}nk+prNk=VM+cT-}6gGIw+;ExQOg}3R4`Y;`4bMBKOlI+hc7ZCGNv3?01PpYPr#mi_+U^!M-lipXnz2H zndv7y-;;9M0{Th#YmZzHDrKf6K&O%jh!MwMoq80kzibz|dfq?{*Cc(F=^W?eKIWe| zKhrbx1hb0*zJmHEB{=fXLJ-RI6`Y^Kmw=0AJtYgVPc{*@NZvWZ7mOdKDwaB!s80Uc zjN62ZnFZzb#3$KkMmx%dv03|Cni<{6nst-$$&rXx1ZaFn+g4 zo6s*zDml$NARMMiFeiQk_=x>u_I5x9@tsKVEwPQ4b@pYc)w(RU@sB@w{NMgq_~%D_ cZf+(slfge8!M`>8{{BmFuO||l_<#EPKhc|20{{R3 literal 0 HcmV?d00001 diff --git a/tests/data/lighteval_smoke/results/eee-smoke/dummy-model/results_2026-08-10T17-22-11.385681.json b/tests/data/lighteval_smoke/results/eee-smoke/dummy-model/results_2026-08-10T17-22-11.385681.json new file mode 100644 index 000000000..ac4355685 --- /dev/null +++ b/tests/data/lighteval_smoke/results/eee-smoke/dummy-model/results_2026-08-10T17-22-11.385681.json @@ -0,0 +1,187 @@ +{ + "config_general": { + "lighteval_sha": "?", + "num_fewshot_seeds": 1, + "max_samples": 2, + "job_id": "0", + "start_time": 208188.223990458, + "end_time": 208192.57978675, + "total_evaluation_time_secondes": "4.3557962919876445", + "model_config": { + "model_name": "eee-smoke/dummy-model", + "generation_parameters": { + "num_blocks": null, + "block_size": null, + "early_stopping": null, + "repetition_penalty": null, + "frequency_penalty": null, + "length_penalty": null, + "presence_penalty": null, + "max_new_tokens": null, + "min_new_tokens": null, + "seed": null, + "stop_tokens": null, + "temperature": 0, + "top_k": null, + "min_p": null, + "top_p": null, + "truncate_prompt": null, + "cache_implementation": null, + "response_format": null + }, + "system_prompt": null, + "cache_dir": "~/.cache/huggingface/lighteval", + "seed": 42 + }, + "model_name": "eee-smoke/dummy-model" + }, + "results": { + "squad_v2|0": { + "em": 0.0, + "em_stderr": 0.0 + }, + "anli:r1|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "all": { + "em": 0.0, + "em_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + } + }, + "versions": {}, + "config_tasks": { + "squad_v2|0": { + "name": "squad_v2", + "prompt_function": "prompt_fn", + "hf_repo": "rajpurkar/squad_v2", + "hf_subset": "squad_v2", + "metrics": [ + { + "metric_name": "em", + "higher_is_better": true, + "category": "GENERATIVE", + "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", + "corpus_level_fn": "mean", + "batched_compute": false + } + ], + "solver": null, + "scorer": null, + "sample_fields": null, + "sample_to_fewshot": null, + "filter": null, + "hf_revision": null, + "hf_filter": "", + "hf_avail_splits": [ + "train", + "validation", + "test" + ], + "evaluation_splits": [ + "validation" + ], + "few_shots_split": "train", + "few_shots_select": null, + "generation_size": 200, + "generation_grammar": null, + "stop_sequence": [ + "\n", + "Question:", + "question:" + ], + "num_samples": null, + "original_num_docs": -1, + "effective_num_docs": -1, + "must_remove_duplicate_docs": false, + "num_fewshots": 0, + "version": 1 + }, + "anli:r1|0": { + "name": "anli:r1", + "prompt_function": "anli_prompt", + "hf_repo": "facebook/anli", + "hf_subset": "plain_text", + "metrics": [ + { + "metric_name": "acc", + "higher_is_better": true, + "category": "LOGPROBS", + "sample_level_fn": "LoglikelihoodAcc(logprob_normalization=None)", + "corpus_level_fn": "mean", + "batched_compute": false + } + ], + "solver": [ + "solve" + ], + "scorer": "score", + "sample_fields": "record_to_sample", + "sample_to_fewshot": null, + "filter": null, + "hf_revision": null, + "hf_filter": null, + "hf_avail_splits": [ + "train_r1", + "dev_r1", + "test_r1" + ], + "evaluation_splits": [ + "test_r1" + ], + "few_shots_split": "train_r1", + "few_shots_select": "random_sampling_from_train", + "generation_size": 1, + "generation_grammar": null, + "stop_sequence": [ + "\n" + ], + "num_samples": null, + "original_num_docs": -1, + "effective_num_docs": -1, + "must_remove_duplicate_docs": false, + "num_fewshots": 0, + "version": 0 + } + }, + "summary_tasks": { + "squad_v2|0": { + "hashes": { + "hash_examples": "2b9b3fc409da0190", + "hash_full_prompts": "ef46db3751d8e999", + "hash_input_tokens": "dcd4184164a682cb", + "hash_cont_tokens": "dcd4184164a682cb" + }, + "truncated": 0, + "non_truncated": 0, + "padded": 0, + "non_padded": 0 + }, + "anli:r1|0": { + "hashes": { + "hash_examples": "72b7003df17fd7c3", + "hash_full_prompts": "ef46db3751d8e999", + "hash_input_tokens": "dcd4184164a682cb", + "hash_cont_tokens": "dcd4184164a682cb" + }, + "truncated": 0, + "non_truncated": 0, + "padded": 0, + "non_padded": 0 + } + }, + "summary_general": { + "hashes": { + "hash_examples": "a0403618ed3b8c9f", + "hash_full_prompts": "86aad2f7ee3a0869", + "hash_input_tokens": "185dedf5ca00043c", + "hash_cont_tokens": "185dedf5ca00043c" + }, + "truncated": 0, + "non_truncated": 0, + "padded": 0, + "non_padded": 0 + } +} \ No newline at end of file diff --git a/tests/test_lighteval_instance_level_adapter.py b/tests/test_lighteval_instance_level_adapter.py new file mode 100644 index 000000000..088d538bf --- /dev/null +++ b/tests/test_lighteval_instance_level_adapter.py @@ -0,0 +1,197 @@ +"""Convert a committed lighteval run and require the output to pass the gate. + +The fixture read here was produced by lighteval itself +(`scripts/upstream_smoke/lighteval_smoke.py`, with their DummyModelConfig), not +hand-written, so the parquet layout, the numpy dtypes and the task-key spelling +are all upstream's rather than ours. + +Scope: shape, not semantics. A metric that switched from percent to proportion +upstream passes this, as does a changed prompt template. What it does catch is a +converter that stops finding the details file, stops resolving the gold, or +starts emitting records the datastore would reject. +""" + +import json + +import pytest + +pytest.importorskip( + 'pyarrow', + reason='no parquet engine; install with: uv sync --extra lighteval', +) + +from pathlib import Path + +from every_eval_ever import cli +from every_eval_ever.converters.lighteval.utils import find_details_file + +FIXTURE_DIR = Path(__file__).parent / 'data' / 'lighteval_smoke' +RESULTS_FILE = ( + FIXTURE_DIR + / 'results' + / 'eee-smoke' + / 'dummy-model' + / 'results_2026-08-10T17-22-11.385681.json' +) + +# One multiple-choice task and one generative task. Their details rows differ in +# every way that matters here: the MC row answers by scoring choices and carries +# no generated text, the generative row is the reverse. +MC_TASK = 'anli:r1|0' +GENERATIVE_TASK = 'squad_v2|0' + + +def _convert(tmp_path: Path) -> Path: + """Run the real CLI, as a user would, and return the published data dir.""" + data_dir = tmp_path / 'data' + exit_code = cli.main( + [ + 'convert', + 'lighteval', + '--log_path', + str(RESULTS_FILE), + '--include_details', + '--output_dir', + str(data_dir), + ] + ) + assert exit_code == 0 + return data_dir + + +def _samples_by_task(data_dir: Path) -> dict[str, list[dict]]: + """Group every published instance-level record by the task it came from.""" + grouped: dict[str, list[dict]] = {} + for path in sorted(data_dir.glob('*/*/*/*_samples.jsonl')): + rows = [ + json.loads(line) + for line in path.read_text(encoding='utf-8').splitlines() + if line + ] + assert rows, f'{path} is empty' + grouped.setdefault(rows[0]['evaluation_name'], []).extend(rows) + return grouped + + +def test_details_file_is_found_for_each_task(): + for task_key in (MC_TASK, GENERATIVE_TASK): + found = find_details_file( + RESULTS_FILE, task_key, 'eee-smoke/dummy-model' + ) + assert found is not None, f'no details file located for {task_key}' + assert found.name.startswith(f'details_{task_key}_') + + +def test_conversion_publishes_aggregates_and_samples(tmp_path): + data_dir = _convert(tmp_path) + + aggregates = sorted(data_dir.glob('*/*/*/*.json')) + samples = sorted(data_dir.glob('*/*/*/*_samples.jsonl')) + assert len(aggregates) == 2 + assert len(samples) == 2 + + for aggregate in aggregates: + record = json.loads(aggregate.read_text(encoding='utf-8')) + detailed = record['detailed_evaluation_results'] + assert detailed is not None + # The sidecar's declared repository path has to be the one publication + # actually used, or a submission points at a file that is not there. + assert ( + Path(detailed['file_path']).name + == f'{aggregate.stem}_samples.jsonl' + ) + assert detailed['total_rows'] == 2 + + +def test_published_records_pass_the_datastore_gate(tmp_path): + data_dir = _convert(tmp_path) + for pattern in ('*.json', '*.jsonl'): + # Exit 0 is clean; 2 is warning-only, which is valid locally but not + # merge-ready, so anything but 0 fails here. + assert ( + cli.main( + [ + 'validate', + str(data_dir / '*' / '*' / '*' / pattern), + '--format', + 'rich', + ] + ) + == 0 + ) + + +def test_multiple_choice_row_keeps_its_options_and_gold(tmp_path): + rows = _samples_by_task(_convert(tmp_path))[MC_TASK] + for row in rows: + # gold_index is an int here, and reference is resolved through choices, + # so an empty reference means the index was dropped rather than read. + assert row['input']['reference'] + assert row['input']['choices'] + assert row['input']['reference'][0] in row['input']['choices'] + attribution = row['answer_attribution'][0] + assert attribution['extraction_method'] == 'argmax_choice_logprob' + assert attribution['extracted_value'] in row['input']['choices'] + assert json.loads(row['metadata']['choice_logprobs']) + assert row['metadata']['lighteval_sampling_methods'] == 'LOGPROBS' + + +def test_generative_row_reports_gold_as_reference_not_as_choices(tmp_path): + rows = _samples_by_task(_convert(tmp_path))[GENERATIVE_TASK] + for row in rows: + # gold_index is a numpy array of numpy ints on this row. Reading it with + # an isinstance check against Python's own int drops every element and + # leaves reference empty, which is why this asserts on content. + assert row['input']['reference'] + # lighteval stores a generative task's gold answers in doc.choices. + # Publishing them as input.choices would say the model was shown the + # answer to choose from. + assert row['input']['choices'] is None + assert row['output']['raw'] + assert row['metadata']['lighteval_sampling_methods'] == 'GENERATIVE' + + +def test_missing_details_tree_is_reported_not_swallowed(tmp_path): + """A run made without save_details must not look like a clean conversion.""" + from every_eval_ever.helpers.io import SourceRecordsError + + results_file = ( + Path(__file__).parent + / 'data' + / 'lighteval' + / 'results' + / 'HuggingFaceTB' + / 'SmolLM2-1.7B-Instruct' + / 'results_2026-01-21T03-44-18.458309.json' + ) + data_dir = tmp_path / 'data' + with pytest.raises(SourceRecordsError): + cli.main( + [ + 'convert', + 'lighteval', + '--log_path', + str(results_file), + '--include_details', + '--output_dir', + str(data_dir), + ] + ) + # The aggregates are still published, as they are for lm-eval: a missing + # sidecar is a partial conversion, not a reason to discard usable records. + assert sorted(data_dir.glob('*/*/*/*.json')) + assert not sorted(data_dir.glob('*/*/*/*_samples.jsonl')) + report = tmp_path / 'adapter_reports' / 'lighteval_details_failures.json' + assert report.is_file() + + +def test_score_and_correctness_agree_with_the_recorded_metric(tmp_path): + for rows in _samples_by_task(_convert(tmp_path)).values(): + for row in rows: + metrics = json.loads(row['metadata']['lighteval_metrics']) + primary = row['metadata']['primary_metric'] + assert primary in metrics + assert row['evaluation']['score'] == pytest.approx(metrics[primary]) + assert row['evaluation']['is_correct'] is ( + row['evaluation']['score'] == 1.0 + ) From 927da0d470d743e9952f70f903461196ce5a28c1 Mon Sep 17 00:00:00 2001 From: leshem Date: Mon, 10 Aug 2026 17:55:06 -0400 Subject: [PATCH 6/7] ci: ask each new lighteval release to produce a run, weekly The committed fixture pins a past release, so it cannot tell us that a new one renamed a field or moved a file -- it keeps the old spelling and CI stays green until a user hits it. This job installs the current lighteval ad hoc, has its fake model produce a run, converts it and validates it. Not on pull_request: an upstream release should not turn an unrelated PR red, and a scheduled failure names the version that broke it. lighteval is installed with `--with`, never locked, for the datasets conflict with crfm-helm. Python 3.12 because 3.10 resolution yields a torch/transformers mismatch, and UV_TORCH_BACKEND=cpu to keep a CUDA torch off the runner. --- .github/workflows/upstream_smoke.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/upstream_smoke.yml diff --git a/.github/workflows/upstream_smoke.yml b/.github/workflows/upstream_smoke.yml new file mode 100644 index 000000000..59c30da1a --- /dev/null +++ b/.github/workflows/upstream_smoke.yml @@ -0,0 +1,40 @@ +name: Upstream smoke + +# Asks the *current* upstream release to produce a run, then converts it. The +# committed fixtures pin a past release, so only this can tell us that a new one +# renamed a field or moved a file. +# +# Deliberately not on pull_request: an upstream release should not turn an +# unrelated PR red, and a scheduled failure names the version that broke it. +on: + schedule: + # Mondays, 06:37 UTC. + - cron: '37 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + lighteval: + name: lighteval + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6.0.2 + - uses: astral-sh/setup-uv@v7.6.0 + - name: Run lighteval's fake model, convert it, validate it + # lighteval is installed ad hoc and never enters the project or the + # lockfile: it requires datasets>=4 while crfm-helm pins datasets~=3.1, + # so a locked dependency would make the whole resolution unsatisfiable. + # The `lighteval` extra here is only pyarrow, which does not conflict. + # + # UV_TORCH_BACKEND=cpu keeps a CUDA torch off a CPU runner; the model is + # fake, so no torch device is used at all. + env: + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/eee-venv + UV_TORCH_BACKEND: cpu + HF_HUB_DISABLE_PROGRESS_BARS: '1' + run: >- + uv run -p 3.12 --extra lighteval --with lighteval + python scripts/upstream_smoke/lighteval_smoke.py From f2edb2881eb206243f8bd400c0ebdbe7a94d1b67 Mon Sep 17 00:00:00 2001 From: Mandark-droid Date: Tue, 11 Aug 2026 22:33:55 +0530 Subject: [PATCH 7/7] fix(lighteval): redact inference_server_auth, the field upstream named and we missed huggingface/lighteval#1326 (merged today) made LiteLLMModelConfig.api_key a SecretStr and excluded BOTH api_key and inference_server_auth at the dump site. This converter's redaction knew about the first and not the second: matching is on exact names plus the suffixes _key/_token/_secret/_password/_credentials, none of which catch a field ending in _auth. So inference_server_auth was serialised into additional_details, which is published. The upstream fix does not make this guard redundant. Every results file written before it still holds the value in cleartext, and archived results files are precisely what a converter is pointed at. Adds the _auth suffix plus the exact names auth and inference_server_auth. The suffix is now value-aware, because widening it turned up a false positive in the first test written for it: requires_auth ends in _auth but a boolean cannot carry a secret, and redacting it would delete provenance for nothing. Exact names still redact whatever they hold; a suffix match requires a value a credential could actually be, and anything not positively known to be harmless still redacts -- a missed credential in a published record is unrecoverable, an over-redacted setting is not. Two tests: one pins inference_server_auth (and that the server ADDRESS, which is provenance rather than a credential, survives), one pins that requires_auth and authorized_users are not swallowed. 459 passed, 20 skipped; ruff clean. --- every_eval_ever/converters/lighteval/utils.py | 30 ++++++++++++-- tests/test_lighteval_adapter.py | 40 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/every_eval_ever/converters/lighteval/utils.py b/every_eval_ever/converters/lighteval/utils.py index 349de3db2..6a3d6c392 100644 --- a/every_eval_ever/converters/lighteval/utils.py +++ b/every_eval_ever/converters/lighteval/utils.py @@ -23,9 +23,11 @@ 'access_token', 'api_key', 'api_token', + 'auth', 'auth_token', 'credentials', 'hf_token', + 'inference_server_auth', 'key', 'password', 'secret', @@ -33,16 +35,25 @@ } ) +# Sentinel for "no value supplied", since None is itself a value a caller may pass. +_UNSET = object() + # Suffixes that make a key credential-bearing whatever the provider prefix. # Nested `env_vars` mappings are where these actually appear: OPENAI_API_KEY, # AWS_SECRET_ACCESS_KEY, HUGGING_FACE_HUB_TOKEN. Anchoring on the suffix keeps # `tokenizer` and `max_tokens` out of the redaction set. +# +# `_auth` was added after lighteval's own fix (huggingface/lighteval#1326) excluded +# BOTH `api_key` and `inference_server_auth`. We knew about the first and not the +# second, which is the point: this list is a guess at someone else's field names, so +# it should be widened whenever upstream tells us one we missed. _SECRET_KEY_SUFFIXES = ( '_key', '_token', '_secret', '_password', '_credentials', + '_auth', ) _DATE_ID_PATTERN = re.compile( @@ -295,7 +306,7 @@ def resolve_metric_id(metric_name: str) -> tuple[str, str]: return f'{METRIC_ID_NAMESPACE}/{metric_name}', 'namespaced_unresolved' -def _is_secret_key(key: Any) -> bool: +def _is_secret_key(key: Any, value: Any = _UNSET) -> bool: """True if a config key names a credential. Exact names cover the common cases; the suffix rule catches the @@ -309,8 +320,19 @@ def _is_secret_key(key: Any) -> bool: """ lowered = str(key).lower() if lowered in SECRET_MODEL_CONFIG_KEYS: + # An exact name redacts whatever it holds. If a field called `api_key` + # carries something odd, that is still not worth publishing. + return True + if not lowered.endswith(_SECRET_KEY_SUFFIXES): + return False + # A suffix is a weaker signal than a name, so require a value a credential + # could be. `requires_auth: True` ends in `_auth` but a boolean cannot carry + # a secret, and redacting it would delete provenance to no benefit. Anything + # not positively known to be harmless still redacts -- a missed credential in + # a published record is unrecoverable, while an over-redacted setting is not. + if value is _UNSET: return True - return lowered.endswith(_SECRET_KEY_SUFFIXES) + return not isinstance(value, (bool, int, float)) def _sanitize_config_value( @@ -326,7 +348,7 @@ def _sanitize_config_value( cleaned: Dict[Any, Any] = {} for key, item in value.items(): child_path = path + [str(key)] - if _is_secret_key(key): + if _is_secret_key(key, item): redacted.append('.'.join(child_path)) continue cleaned[key] = _sanitize_config_value(item, child_path, redacted) @@ -355,7 +377,7 @@ def flatten_model_config( for key, value in model_config.items(): if value is None: continue - if _is_secret_key(key): + if _is_secret_key(key, value): redacted.append(str(key)) continue if isinstance(value, str): diff --git a/tests/test_lighteval_adapter.py b/tests/test_lighteval_adapter.py index c6d92f6bb..8f7991b27 100644 --- a/tests/test_lighteval_adapter.py +++ b/tests/test_lighteval_adapter.py @@ -121,6 +121,46 @@ def test_flatten_model_config_drops_credentials(): assert 'timeout' not in flattened +def test_inference_server_auth_is_redacted(): + """The field lighteval's own fix named that this converter did not know about. + + huggingface/lighteval#1326 made `api_key` a SecretStr and excluded BOTH + `api_key` and `inference_server_auth` at the dump site. We had only the first. + That fix does not make this redaction redundant: every results file written + before it still carries the value in cleartext, and archived results files are + precisely what a converter is pointed at. + """ + flattened, redacted = flatten_model_config( + { + 'model_name': 'local/llama', + 'inference_server_address': 'http://10.0.0.4:8080', + 'inference_server_auth': 'Bearer NOT-A-REAL-TOKEN-FIXTURE-ONLY', + } + ) + assert 'inference_server_auth' in redacted + assert 'inference_server_auth' not in flattened + assert 'NOT-A-REAL-TOKEN-FIXTURE-ONLY' not in json.dumps(flattened) + # The address is provenance, not a credential, and must survive. + assert flattened['inference_server_address'] == 'http://10.0.0.4:8080' + + +def test_auth_suffix_does_not_swallow_ordinary_settings(): + """`_auth` must not become a wildcard for anything vaguely security-shaped.""" + flattened, redacted = flatten_model_config( + { + 'model_name': 'local/llama', + 'requires_auth': True, + 'authorized_users': 3, + } + ) + # These are settings, not secrets; only an `_auth` SUFFIX redacts. + assert redacted == [] + # flatten_model_config returns str values, so these arrive JSON-encoded. What + # matters is that they arrive at all rather than being redacted away. + assert flattened['requires_auth'] == 'true' + assert flattened['authorized_users'] == '3' + + # ── Adapter: transform_from_file ───────────────────────────────────────