From 73c09bd9ae605a51f17b3f30597aebccf2a5f852 Mon Sep 17 00:00:00 2001 From: xzascc Date: Sun, 15 Mar 2026 00:39:01 -0400 Subject: [PATCH 1/6] refactor: migrate to modern Python project structure with uv - Migrate from requirements.txt to pyproject.toml with uv package manager - Restructure source code into src/probing_reflection/ package - Add comprehensive documentation (AGENTS.md, ARCHITECTURE.md, SPEC.md) - Add test infrastructure with pytest - Add .github/ for CI/CD configurations - Simplify .gitignore for Python/IDE patterns - Remove legacy scripts and placeholder files --- .github/ISSUE_TEMPLATE/ai-task.yml | 53 + .github/ISSUE_TEMPLATE/bug-report.yml | 35 + .github/ISSUE_TEMPLATE/config.yml | 2 + .github/ISSUE_TEMPLATE/feature-request.yml | 29 + .github/pull_request_template.md | 68 + .github/workflows/ci.yml | 38 + .gitignore | 198 +-- .python-version | 1 + AGENTS.md | 131 ++ ARCHITECTURE.md | 54 + LICENSE | 2 +- README.md | 91 +- SPEC.md | 61 + TEMPLATE.md | 87 + docs/QUALITY_SCORE.md | 41 + docs/design-docs/core-beliefs.md | 45 + docs/design-docs/index.md | 21 + pyproject.toml | 40 + reqirements.txt | 7 - scripts/run_injection_deepseek1.5b.sh | 14 - scripts/run_original_qwen1.5b.sh | 8 - scripts/run_vllm_insert_20_neg_0.1.sh | 13 - src/acc_length_rel.py | 149 -- src/inference.py | 99 -- src/math_grader.py | 1076 ------------ src/probing_reflection/__init__.py | 24 + .../probing_reflection/py.typed | 0 src/probing_reflection/types.py | 58 + src/save_insert_model.py | 43 - src/utils.py | 1504 ----------------- models/placeholder => tests/__init__.py | 0 tests/conftest.py | 30 + tests/test_types.py | 105 ++ uv.lock | 278 +++ 34 files changed, 1277 insertions(+), 3128 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/ai-task.yml create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/ci.yml create mode 100644 .python-version create mode 100644 AGENTS.md create mode 100644 ARCHITECTURE.md create mode 100644 SPEC.md create mode 100644 TEMPLATE.md create mode 100644 docs/QUALITY_SCORE.md create mode 100644 docs/design-docs/core-beliefs.md create mode 100644 docs/design-docs/index.md create mode 100644 pyproject.toml delete mode 100644 reqirements.txt delete mode 100644 scripts/run_injection_deepseek1.5b.sh delete mode 100644 scripts/run_original_qwen1.5b.sh delete mode 100644 scripts/run_vllm_insert_20_neg_0.1.sh delete mode 100644 src/acc_length_rel.py delete mode 100644 src/inference.py delete mode 100644 src/math_grader.py create mode 100644 src/probing_reflection/__init__.py rename asset/placeholder => src/probing_reflection/py.typed (100%) create mode 100644 src/probing_reflection/types.py delete mode 100644 src/save_insert_model.py delete mode 100644 src/utils.py rename models/placeholder => tests/__init__.py (100%) create mode 100644 tests/conftest.py create mode 100644 tests/test_types.py create mode 100644 uv.lock diff --git a/.github/ISSUE_TEMPLATE/ai-task.yml b/.github/ISSUE_TEMPLATE/ai-task.yml new file mode 100644 index 0000000..f2a3f1d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/ai-task.yml @@ -0,0 +1,53 @@ +name: AI Task +description: Define a task for AI agent execution +labels: [ai-task] +body: + - type: dropdown + id: task-type + attributes: + label: Task Type + options: + - feature + - bugfix + - refactor + - documentation + - test + - maintenance + validations: + required: true + - type: textarea + id: problem + attributes: + label: Problem Statement + description: What needs to be done and why + validations: + required: true + - type: textarea + id: criteria + attributes: + label: Acceptance Criteria + description: Checkboxes of what 'done' looks like + validations: + required: true + - type: textarea + id: files + attributes: + label: Files Likely Involved + validations: + required: false + - type: dropdown + id: scope + attributes: + label: Scope Assessment + options: + - small (1-2 files) + - medium (3-5 files) + - large (6+ files) + validations: + required: true + - type: textarea + id: out-of-scope + attributes: + label: Explicitly Out of Scope + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..6959df9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,35 @@ +name: Bug Report +description: Report a bug +labels: [bug] +body: + - type: textarea + id: description + attributes: + label: Description + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to Reproduce + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual Behavior + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + placeholder: "Python 3.12, Ubuntu 22.04" + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..64eb98d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +blank_issues_enabled: true +contact_links: [] diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..48024e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,29 @@ +name: Feature Request +description: Suggest a new feature +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What problem does this solve? + validations: + required: true + - type: textarea + id: solution + attributes: + label: Proposed Solution + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + validations: + required: false + - type: textarea + id: context + attributes: + label: Additional Context + validations: + required: false diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..1859693 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,68 @@ +## Summary + + + +## Linked Issue (Optional) + +Closes # + +## Type of Change + +- [ ] Bug fix +- [ ] New feature +- [ ] Refactor +- [ ] Documentation +- [ ] Test +- [ ] CI/CD +- [ ] Other + +## What Changed + + + +## Verification Performed + +### Lint +``` + +``` + +### Type Check +``` + +``` + +### Tests +``` + +``` + +## Self-Review Checklist + +- [ ] Code follows project style (ruff passes) +- [ ] Types are correct (mypy passes) +- [ ] Tests pass and cover new behavior +- [ ] No secrets or credentials in code +- [ ] No unnecessary dependencies added +- [ ] Documentation updated if needed +- [ ] Changes are within the scope of the linked issue + +## Agent Metadata + +```yaml +agent: +model: +exec_plan: +files_changed: +``` + +## Risks & Considerations + + + +## Human Reviewer Checklist + +- [ ] Intent matches the issue/plan +- [ ] No scope creep beyond the plan +- [ ] Architecture decisions are sound +- [ ] Edge cases considered diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..520e001 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: .python-version + + - name: Install dependencies + run: uv sync + + - name: Lint + run: uv run ruff check src/ tests/ + + - name: Format check + run: uv run ruff format --check src/ tests/ + + - name: Type check + run: uv run mypy src/ + + - name: Test + run: uv run pytest diff --git a/.gitignore b/.gitignore index 7b004e5..85fda40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,194 +1,34 @@ -# Byte-compiled / optimized / DLL files +# Python __pycache__/ *.py[cod] *$py.class - -# C extensions *.so -# Distribution / packaging -.Python -build/ -develop-eggs/ +# Virtual environments +.venv/ +venv/ +env/ + +# Distribution dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ +build/ *.egg-info/ -.installed.cfg *.egg -MANIFEST -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec +# IDE +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ +# Testing .coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ +htmlcov/ .pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy .mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the enitre vscode folder -# .vscode/ - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc +# Environment +.env +.env.local -# Cursor -# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to -# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data -# refer to https://docs.cursor.com/context/ignore-files -.cursorignore -.cursorindexingignore \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..78f50b3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,131 @@ +# AGENTS.md + +AI agents working in this repository MUST follow these rules. + +## 1) Repository Snapshot + +- Package manager: `uv` +- Python: 3.12+ (see `.python-version`) +- Lint/Format: `ruff` +- Type check: `mypy` +- Test: `pytest` + +## 2) Build & Verify Commands + +```bash +# Install dependencies +uv sync + +# Lint check +uv run ruff check src/ tests/ + +# Format check +uv run ruff format --check src/ tests/ + +# Format fix +uv run ruff format src/ tests/ + +# Type check +uv run mypy src/ + +# Run all tests +uv run pytest + +# Run single test file +uv run pytest tests/test_placeholder.py + +# Run test by name +uv run pytest -k "test_name" +``` + +## 3) Definition of Done + +Before opening a PR, ALL of these must pass: + +- [ ] `uv sync` completes without errors +- [ ] `uv run ruff check src/ tests/` → 0 violations +- [ ] `uv run ruff format --check src/ tests/` → already formatted +- [ ] `uv run mypy src/` → Success with 0 errors +- [ ] `uv run pytest` → all tests pass + +## 4) When Writing Code + +- Use ESM-style imports (from x import y) +- Group imports: stdlib → third-party → local +- Use type hints on ALL function parameters and returns +- Never use `Any` — use `unknown` patterns or proper types +- Fail fast on invalid input +- Throw typed/domain-specific errors +- Preserve original error as `cause` when wrapping + +## 5) When Writing Tests + +- Use pytest style (plain functions, assert statements) +- Cover: happy path, edge cases, failure paths +- Keep tests deterministic and isolated +- Mock external boundaries (network, file I/O) + +## 6) When Opening a PR + +Include in description: +1. Summary of changes (2-3 sentences) +2. Linked issue (Closes #N) +3. Verification output (paste command results) +4. Self-review checklist completion +5. Risks & considerations + +## 7) Escalation Rules + +STOP and report to human if: +- Modifying more than 5 files not in the original plan +- Encountering unclear requirements after 2 clarification attempts +- Need to add new dependencies +- Changes affect security (auth, secrets, permissions) +- Test coverage would drop below existing level + +## 8) Code Style + +Enforced by ruff (see pyproject.toml): +- Line length: 100 +- Quote style: double +- Naming: snake_case for functions/vars, PascalCase for classes + +## 9) Pipeline Workflow + +**ALWAYS follow this pipeline for every task:** + +``` +1. READ PLAN → Read .sisyphus/plans/ (active work plan), parse tasks, understand requirements +2. CODE → Implement following conventions in this file +3. VERIFY → Run: ruff check, ruff format, mypy, pytest (ALL must pass) +4. MOVE PLAN → Complete → mark completed in .sisyphus/plans/, In-progress → .sisyphus/plans/ +5. UPDATE DOCS → .sisyphus/roadmap.md, QUALITY_SCORE.md, ARCHITECTURE.md as needed +6. COMMIT/PR → When logical unit complete + all checks pass +``` + +### Commit Criteria + +Commit ONLY when ALL conditions met: +- `uv sync` completes without errors +- Logical unit of work complete +- `uv run ruff check src/ tests/` → 0 violations +- `uv run ruff format --check src/ tests/` → formatted +- `uv run mypy src/` → 0 errors +- `uv run pytest` → all pass +- Related docs updated + +### PR Criteria + +Open PR when feature complete: +- All commit criteria met +- Self-review checklist done +- Verification output included +- Risks documented + +## 10) Architecture Cross-References + +- System design: `ARCHITECTURE.md` +- Design docs: `docs/design-docs/` +- Work plans: `.sisyphus/plans/` +- Quality tracking: `docs/QUALITY_SCORE.md` +- Roadmap: `.sisyphus/roadmap.md` diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..c08c421 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,54 @@ +# Architecture + +This document describes the system architecture for this project. + +## System Overview + +[Describe the high-level purpose and architecture of the system.] + +## Core Components + +| Component | Responsibility | Key Files | Dependencies | +|-----------|---------------|-----------|--------------| +| [Name] | [What it does] | [Files] | [Deps] | + +## Technology Stack + +| Category | Tool | Version | +|----------|------|---------| +| Language | Python | 3.12+ | +| Package Manager | uv | latest | +| Linting | ruff | 0.8+ | +| Type Checking | mypy | 1.13+ | +| Testing | pytest | 8.0+ | +| CI | GitHub Actions | N/A | + +## Directory Structure + +``` +. +├── src/probing_reflection/ # Source code +│ ├── __init__.py # Package init +│ ├── py.typed # PEP 561 marker +│ └── *.py # Modules +├── tests/ # Test files +├── docs/ # Documentation +│ ├── design-docs/ # Design documents +│ └── .sisyphus/ # Sisyphus plans & notepads +└── .github/ # GitHub configs +``` + +## Key Design Decisions + +### ADR-001: [Decision Title] + +- **Context**: [Why this decision was needed] +- **Decision**: [What was decided] +- **Consequences**: [Impact of this decision] + +## How to Update This Document + +Update this file when: +- Adding new major components +- Changing technology stack +- Making significant architectural decisions diff --git a/LICENSE b/LICENSE index 9b01e75..27f88eb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 xzAscC +Copyright (c) 2026 XUDONG ZHU Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 53d8db1..6d949ae 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,70 @@ # ProbingReflection -Source Code of paper "From Emergence to Control: Probing and Modulating Self-Reflection in Language Models" - -## Requirement - -- transformers -- datasets -- vllm -- latex2sympy2_extended -- pylatexenc -- umap-learn - -## Folder Architecture - -```text -src: source code - acc_length_rel.py: explore the relationship between length and acc - math_grader.py: evaluation the math problem - inference.py: inference with original model and inserted model - save_insert_model.py: use vector to do insertion - utils.py: other functions -asset: folder to save responses -models: folder to save model weights -scripts: bash file to inference -``` -## Getting Started +Source code for the paper **"From Emergence to Control: Probing and Modulating Self-Reflection in Language Models"** + +## Overview + +This project investigates self-reflection in Large Language Models (LLMs) through probing and steering techniques. We explore how reflection behaviors emerge and how they can be controlled through vector-based interventions. + +## Key Concepts + +- **Probing Vectors**: Techniques to detect and measure self-reflection patterns in model activations +- **Model Insertion**: Methods for injecting steering vectors to modulate reflection behavior +- **Reflection Analysis**: Frameworks for evaluating and understanding model self-reflection -### Running Inference +## Installation -To run inference with the original model: ```bash -python src/inference.py --model [MODEL_NAME] --output_dir [OUTPUT_PATH] +uv sync ``` -To run inference with the inserted model: +## Development + ```bash -python src/inference.py --model [MODEL_NAME] --injection --injection_layer [INJECTION_LAYER] --injection_alpha [INJECTION_ALPHA] --output_dir [OUTPUT_PATH] +# Lint check +uv run ruff check src/ tests/ + +# Format +uv run ruff format src/ tests/ + +# Type check +uv run mypy src/ + +# Run tests +uv run pytest ``` -### Model Insertion +## Project Structure -To insert vectors into a model: -```bash -python src/save_insert_model.py --model [MODEL_NAME] --output_dir [OUTPUT_DIR] +``` +. +├── src/probing_reflection/ # Source code +│ ├── __init__.py +│ └── py.typed +├── tests/ # Test files +├── docs/ # Documentation +│ └── design-docs/ # Design documents +├── AGENTS.md # AI agent instructions +├── ARCHITECTURE.md # System architecture +└── pyproject.toml # Project configuration +``` + +## Research Workflow + +This project follows an AI-assisted research workflow. See AGENTS.md for detailed instructions on how AI agents should work in this repository. + +## Citation + +If you use this code, please cite: + +```bibtex +@article{probing_reflection_2024, + title={From Emergence to Control: Probing and Modulating Self-Reflection in Language Models}, + author={[Authors]}, + year={2024} +} ``` +## License -For more details, refer to the scripts in the `scripts` directory. \ No newline at end of file +[Add your license here] diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..2674b02 --- /dev/null +++ b/SPEC.md @@ -0,0 +1,61 @@ +# SPEC.md + +Project-level specification for AI coding agents. This document defines WHAT to build, while AGENTS.md defines HOW to work. + +## 1) Project Vision + +ProbingReflection investigates self-reflection in Large Language Models through probing and steering techniques. This project explores how reflection behaviors emerge and how they can be controlled through vector-based interventions. + +## 2) Tech Stack + +- Python 3.12+ +- Package manager: `uv` +- Linting/Formatting: `ruff` +- Type checking: `mypy` +- Testing: `pytest` + +## 3) Project Structure + +- `src/probing_reflection/` — Source code modules +- `tests/` — Test files mirroring source structure +- `docs/` — Documentation including: + - `docs/design-docs/` — Design and experiment documentation + - `.sisyphus/plans/` — Execution plans for tasks + - `.sisyphus/roadmap.md` — Project roadmap + - `docs/QUALITY_SCORE.md` — Quality tracking metrics + +## 4) Three-Tier Boundaries + +### ✅ Always Do + +Actions the agent takes without asking: + +- Run tests before commits +- Follow naming conventions (snake_case for functions/vars, PascalCase for classes) +- Update documentation when changing code behavior +- Add type hints to all function parameters and returns +- Write tests for new functionality +- Follow the commit criteria in AGENTS.md + +### ⚠️ Ask First + +Actions requiring human approval: + +- Adding new dependencies +- Modifying `pyproject.toml` configuration +- Changing CI configuration (`.github/workflows/`) +- Creating new top-level directories +- Modifying existing public APIs +- Changing code style configuration + +### 🚫 Never Do + +Hard stops that require escalation: + +- Commit secrets, API keys, or credentials +- Edit files in `.git/` or `__pycache__/` +- Remove tests without explicit approval +- Skip type hints on new code +- Push to `main` or `master` branch directly +- Modify `.python-version` without approval +- Delete documentation files diff --git a/TEMPLATE.md b/TEMPLATE.md new file mode 100644 index 0000000..8290d4d --- /dev/null +++ b/TEMPLATE.md @@ -0,0 +1,87 @@ +# AI Research Template + +Welcome to the AI Research Template! This guide will help you set up your new research project. + +## About This Template + +This template is designed for **AI-assisted research projects** — experiments, prototypes, and research codebases where AI agents help with implementation. + +**Key features:** +- Python 3.12+ with modern tooling (uv, ruff, mypy, pytest) +- AI-friendly documentation structure (AGENTS.md, ARCHITECTURE.md) +- Research workflow support (experiment tracking, quality metrics) +- GitHub CI/CD pre-configured + +## Getting Started + +### 1. Create Your Repository + +Click **"Use this template"** → **"Create a new repository"** + +### 2. Clone and Setup + +```bash +# Clone your new repository +git clone https://github.com/YOUR_USERNAME/YOUR_REPO_NAME.git +cd YOUR_REPO_NAME + +# Install dependencies +uv sync +``` + +### 3. Rename the Package + +Replace `ai_research_template` with your project name: + +```bash +# Update all references +find . -type f \( -name "*.py" -o -name "*.toml" \) -exec sed -i 's/ai_research_template/YOUR_PROJECT_NAME/g' {} + + +# Rename the source directory +mv src/ai_research_template src/YOUR_PROJECT_NAME +``` + +### 4. Verify Everything Works + +```bash +# Run linting +uv run ruff check src/ tests/ + +# Run type checking +uv run mypy src/ + +# Run tests +uv run pytest +``` + +### 5. Start Your Research + +1. Update `README.md` with your project description +2. Update `ARCHITECTURE.md` with your system design +3. Document experiments in `docs/design-docs/` +4. Track quality in `docs/QUALITY_SCORE.md` + +## Files to Update After Cloning + +| File | What to Update | +|------|---------------| +| `pyproject.toml` | Project name, description, dependencies | +| `README.md` | Project description, remove template-specific content | +| `ARCHITECTURE.md` | Your system architecture | +| `AGENTS.md` | Package references (line 35: `packages = ["YOUR_PROJECT_NAME"]`) | +| `.sisyphus/roadmap.md` | Your project roadmap | +| `.github/` | Update issue templates if needed | + +## Need Help? + +- Check `AGENTS.md` for AI agent instructions +- See `ARCHITECTURE.md` for system design template +- Review `docs/` for documentation structure + +## Related Templates + +- **ai-general-template** — For general-purpose projects (coming soon) + +--- + +Happy researching! 🔬 diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md new file mode 100644 index 0000000..beb217c --- /dev/null +++ b/docs/QUALITY_SCORE.md @@ -0,0 +1,41 @@ +# Quality Tracking Framework + +## Layer 1: Automated Gates (Pass/Fail) + +| Check | Command | Status | +|-------|---------|--------| +| Lint | `uv run ruff check src/ tests/` | ⬜ | +| Format | `uv run ruff format --check src/ tests/` | ⬜ | +| Type Check | `uv run mypy src/` | ⬜ | +| Tests | `uv run pytest` | ⬜ | +| CI | GitHub Actions | ⬜ | + +## Layer 2: Trend Metrics + +| Metric | Current | Target | Notes | +|--------|---------|--------|-------| +| Test Coverage | - % | 80% | | +| Type Coverage | - % | 100% | Files with type hints | +| Open Tech Debt | - | 0 | From tech-debt-tracker.md | +| PR Review Time | - hrs | <24 hrs | Average turnaround | + +## Layer 3: Human Rubric (1-5 Scale) + +| Dimension | Score | Notes | +|-----------|-------|-------| +| Code Readability | - | How easy is code to understand? | +| Architecture Fitness | - | Does structure support growth? | +| Documentation Freshness | - | Are docs up to date? | +| Onboarding Friction | - | How easy for new dev/agent to start? | + +## Scoring History + +| Date | Gates | Coverage | Readability | Architecture | Docs | Onboarding | +|------|-------|----------|-------------|--------------|------|------------| +| - | -/5 | -% | - | - | - | - | + +## Update Schedule + +- **Automated Gates**: Every PR +- **Trend Metrics**: Weekly +- **Human Rubric**: Every sprint/milestone diff --git a/docs/design-docs/core-beliefs.md b/docs/design-docs/core-beliefs.md new file mode 100644 index 0000000..27803c8 --- /dev/null +++ b/docs/design-docs/core-beliefs.md @@ -0,0 +1,45 @@ +# Core Engineering Beliefs + +These beliefs guide development in this project. + +## 1. Code is Read More Than Written + +**Explanation**: Every line of code will be read many more times than it was written. Clarity always beats cleverness. + +**Practical Implication**: Prefer explicit over implicit. Name things for what they do, not how they do it. + +## 2. Tests are Documentation + +**Explanation**: Tests describe intended behavior. They're the most reliable documentation because they're executable and must pass. + +**Practical Implication**: Write tests that tell a story. Test names should describe the behavior being verified. + +## 3. Automate Everything Repeatable + +**Explanation**: If an AI agent can verify it, it should be automated. Manual verification doesn't scale. + +**Practical Implication**: Every verification step should be a command in AGENTS.md that exits 0 on success. + +## 4. Small, Frequent Changes + +**Explanation**: Small PRs are easier to review, easier to revert, and cause fewer merge conflicts. + +**Practical Implication**: If a change touches more than 5 files, consider splitting it. + +## 5. Fail Fast, Fail Loud + +**Explanation**: Errors should surface immediately, not silently. Silent failures are the worst kind. + +**Practical Implication**: Use strict mode in type checkers. Fail on warnings in CI. Never catch and swallow exceptions silently. + +## 6. Documentation is Code + +**Explanation**: Docs are versioned, reviewed, and maintained like code. Stale docs are bugs. + +**Practical Implication**: Update docs in the same PR as code changes. Review docs for accuracy. + +## 7. Trust but Verify + +**Explanation**: AI writes code, humans verify intent. The AI is a force multiplier, not a replacement for judgment. + +**Practical Implication**: Always review AI-generated code. Focus on intent, not just syntax. diff --git a/docs/design-docs/index.md b/docs/design-docs/index.md new file mode 100644 index 0000000..7995024 --- /dev/null +++ b/docs/design-docs/index.md @@ -0,0 +1,21 @@ +# Design Documents + +This directory holds design decisions and architectural rationale. + +## Index + +| Doc | Status | Date | Summary | +|-----|--------|------|---------| +| [core-beliefs.md](core-beliefs.md) | Approved | 2024-01 | Core engineering beliefs for AI-assisted development | + +## Adding New Design Docs + +1. Create a new `.md` file with descriptive name +2. Include: Context, Decision, Consequences +3. Add entry to index table above +4. Update status as doc evolves (Draft → Approved → Superseded) + +## Naming Convention + +- Use kebab-case: `feature-name.md` +- Be descriptive: `auth-strategy.md` not `auth.md` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7681594 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "probing_reflection" +version = "0.1.0" +requires-python = ">=3.12" +description = "ProbingReflection - Probing and Modulating Self-Reflection in Language Models" + +[dependency-groups] +dev = [ + "ruff>=0.8.0", + "mypy>=1.13.0", + "pytest>=8.0.0", + "pyyaml>=6.0.3", +] + +[tool.ruff] +target-version = "py312" +line-length = 100 +src = ["src"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM", "N"] + +[tool.ruff.format] +quote-style = "double" + +[tool.mypy] +python_version = "3.12" +strict = true +warn_return_any = true +warn_unused_ignores = true +packages = ["probing_reflection"] +mypy_path = "src" + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/reqirements.txt b/reqirements.txt deleted file mode 100644 index 6ea3e19..0000000 --- a/reqirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -transformers -datasets -vllm -latex2sympy2_extended -pylatexenc -umap-learn -accelerate \ No newline at end of file diff --git a/scripts/run_injection_deepseek1.5b.sh b/scripts/run_injection_deepseek1.5b.sh deleted file mode 100644 index 4564d02..0000000 --- a/scripts/run_injection_deepseek1.5b.sh +++ /dev/null @@ -1,14 +0,0 @@ -for alpha in 0.001 0.003 0.01 0.03 0.1 0.3 1.0 -do - python ./src/inference.py \ - --model_name "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" \ - --dataset_name "HuggingFaceH4/MATH-500" \ - --output_dir "./asset/insert_response/" \ - --max_new_tokens 2048 \ - --dataset_ratio 0.1 \ - --do_sample \ - --use_cache \ - --injection \ - --injection_layer 20 \ - --injection_alpha $alpha -done \ No newline at end of file diff --git a/scripts/run_original_qwen1.5b.sh b/scripts/run_original_qwen1.5b.sh deleted file mode 100644 index 647b445..0000000 --- a/scripts/run_original_qwen1.5b.sh +++ /dev/null @@ -1,8 +0,0 @@ -python ./src/inference_vllm.py \ - --model_name "allenai/OLMo-2-0425-1B" \ - --dataset_name "HuggingFaceH4/MATH-500" \ - --output_dir "./asset/insert_response/" \ - --max_new_tokens 32784 \ - --dataset_ratio 1 \ - --do_sample \ - --use_cache \ diff --git a/scripts/run_vllm_insert_20_neg_0.1.sh b/scripts/run_vllm_insert_20_neg_0.1.sh deleted file mode 100644 index d58ed8e..0000000 --- a/scripts/run_vllm_insert_20_neg_0.1.sh +++ /dev/null @@ -1,13 +0,0 @@ -python ./src/inference.py \ - --model_name "Qwen/Qwen2.5-1.5B" \ - --dataset_name "Maxwell-Jia/AIME_2024" \ - --output_dir "./asset/response/" \ - --prompt "Please reason step by step, and put your final answer within \\boxed{{}}. <|im_start|>user: {problems}<|im_end|>\n<|im_start|>assistant:" \ - --max_new_tokens 32784 \ - --temperature 0.6 \ - --dataset_ratio 1 \ - --do_sample \ - --use_cache \ - --injection \ - --injection_layer 20 \ - --injection_alpha -0.100 diff --git a/src/acc_length_rel.py b/src/acc_length_rel.py deleted file mode 100644 index 360deb2..0000000 --- a/src/acc_length_rel.py +++ /dev/null @@ -1,149 +0,0 @@ -from vllm import LLM, SamplingParams -from transformers import AutoTokenizer -from inference import set_seed, config, Logger -from tqdm import tqdm -import datasets -import torch -import os -import json -import argparse -import numpy as np -import random -args = argparse.ArgumentParser() -args.add_argument( - "--seed", - type=int, - default=42, - help="The name of the model to use.", -) -args.add_argument( - "--model_name", - type=str, - default="deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - help="The name of the model to use.", -) -args.add_argument( - "--dataset_name", - type=str, - default="HuggingFaceH4/MATH-500", - help="The name of the dataset to use.", -) - -args.add_argument( - "--output_dir", - type=str, - default="./asset/insert_response/", - help="The output directory where the model predictions and checkpoints will be written.", -) -args.add_argument( - "--prompt", - type=str, - default="Please reason step by step, and put your final answer within \\boxed{{}}. <|im_start|>user: {problems}<|im_end|>\n<|im_start|>assistant:", - help="The prompt to use for the model.", -) - -args.add_argument( - "--max_new_tokens", - type=int, - default=32768, - help="The maximum number of new tokens to generate.", -) -args = args.parse_args() -set_seed(args.seed) -output_dir = os.path.join( - args.output_dir, - f"responseLength_acc_{args.model_name.split('/')[-1]}_{args.dataset_name.split('/')[-1]}_{len(args.prompt)}_{args.max_new_tokens}", -) -os.makedirs(output_dir, exist_ok=True) - -logger = Logger(os.path.join(output_dir, "inference.log")) -logger.write(str(args) + "\n") - -model = LLM( - args.model_name, - dtype="bfloat16", -) - -tok = AutoTokenizer.from_pretrained(args.model_name) - -stop_token_ids = tok("<|im_start|><|im_end|>")["input_ids"] -sampling_params = SamplingParams( - max_tokens=32768, - min_tokens=0, - stop_token_ids=stop_token_ids, - skip_special_tokens=False, - temperature=0.6, - top_p=0.95, -) -prompt = args.prompt -inputs = [] -model_name = args.model_name -dataset_name = args.dataset_name - -dataset = datasets.load_dataset(args.dataset_name)["test"] -for idx, example in enumerate(tqdm(dataset)): - problem = example["problem"] - formatted_prompt = prompt.format(problems=str(problem)) - inputs.append(formatted_prompt) -# Generate response using vllm's LLM API -ignore_str = "Wait" - -with torch.no_grad(): - for idx, input_ in enumerate(inputs): - if idx > 1: - break - res_length = 16384 - tok.encode(input_, return_tensors="pt").shape[1] - output = model.generate(input_, sampling_params=sampling_params) - response = output[0].outputs[0].text - example = dataset[idx] - with open( - os.path.join( - args.output_dir, - f"responseLength_acc_{model_name.split('/')[-1]}_{dataset_name.split('/')[-1]}_{len(prompt)}_{args.max_new_tokens}", - f"{idx}_0.json", - ), - "w", - ) as f: - json.dump( - { - "problem": problem, - "response": response, - "answer": example["answer"], - }, - f, - indent=2, - ) - - for i in range(10): - cur_length = tok.encode(input_, return_tensors="pt").shape[1] - if cur_length <= res_length: - input_ += output[0].outputs[0].text + ignore_str - sampling_params = SamplingParams( - max_tokens=32768, - min_tokens=1, - stop_token_ids=stop_token_ids, - skip_special_tokens=False, - temperature=0.0, - ) - output = model.generate(input_, sampling_params=sampling_params) - res_length = res_length - cur_length - response = output[0].outputs[0].text - with open( - os.path.join( - args.output_dir, - f"responseLength_acc_{model_name.split('/')[-1]}_{dataset_name.split('/')[-1]}_{len(prompt)}_{args.max_new_tokens}", - f"{idx}_{i}.json", - ), - "w", - ) as f: - json.dump( - { - "problem": problem, - "response": input_ + response, - "answer": example["answer"], - }, - f, - indent=2, - ) - -logger.close() \ No newline at end of file diff --git a/src/inference.py b/src/inference.py deleted file mode 100644 index fbde815..0000000 --- a/src/inference.py +++ /dev/null @@ -1,99 +0,0 @@ -from inference import config, Logger, set_seed -import os -import torch -import json -import vllm -import datasets -from tqdm import tqdm -from utils import preprocess_box_response_for_qwen_prompt - - -def inference_vllm(args, logger): - # Set random seed for reproducibility - set_seed(args.seed) - model_name = args.model_name - dataset_name = args.dataset_name - prompt = args.prompt - # model = vllm.LLM(model=model_name, dtype="bfloat16", revision="stage1-step10000-tokens21B") - # model = vllm.LLM(model=model_name, dtype="bfloat16", revision="stage1-step0-tokens0B") - model = vllm.LLM(model=model_name, dtype="bfloat16") - - # Load the dataset - dataset = datasets.load_dataset(dataset_name)["test"] - - os.makedirs( - os.path.join( - args.output_dir, - f"{args.model_name.split('/')[-1]}_{args.dataset_name.split('/')[-1]}_{len(args.prompt)}_{args.max_new_tokens}_{args.injection_layer}_{args.injection_alpha}", - ), - exist_ok=True, - ) - sampling_param = vllm.SamplingParams( - max_tokens=args.max_new_tokens, - temperature=args.temperature, - top_p=0.95 if args.do_sample else None, - ) - score = 0 - boxed = 0 - overall_length = len(dataset) * args.dataset_ratio - inputs = [] - with torch.no_grad(): - for idx, example in enumerate(tqdm(dataset)): - if idx >= 25: - break - problem = example["problem"] - formatted_prompt = prompt.format(problems=str(problem)) - inputs.append(formatted_prompt) - # Generate response using vllm's LLM API - outputs = model.generate( - inputs, - sampling_param, - ) - for idx, output in enumerate(outputs): - example = dataset[idx] - problem = output.prompt - response = output.outputs[0].text - with open( - os.path.join( - args.output_dir, - f"{model_name.split('/')[-1]}_{dataset_name.split('/')[-1]}_{len(prompt)}_{args.max_new_tokens}_{args.injection_layer}_{args.injection_alpha}", - f"{idx}.json", - ), - "w", - ) as f: - json.dump( - { - "Problem": problem, - "response": response, - "answer": example["answer"], - }, - f, - indent=2, - ) - - # Evaluate the response - _, box_match, box = preprocess_box_response_for_qwen_prompt( - response, example["answer"] - ) - score += box_match - boxed += box - logger.write( - f"Problem: {idx}\tResponse: {response}\tAnswer: {example['answer']}\tScore: {box_match}\tBoxed: {box}\n" - ) - logger.write(f"Model: {model_name}\nDataset: {dataset_name}\nPrompt: {prompt}\n") - logger.write(f"Total Problems: {overall_length}\nCorrect Answers: {score}\nBoxed: {boxed}\n") - logger.write(f"Overall Score: {score / overall_length}\n") - - - -if __name__ == "__main__": - args = config() - output_dir = os.path.join( - args.output_dir, - f"{args.model_name.split('/')[-1]}_{args.dataset_name.split('/')[-1]}_{len(args.prompt)}_{args.max_new_tokens}_{args.injection_layer}_{args.injection_alpha}", - ) - os.makedirs(output_dir, exist_ok=True) - logger = Logger(os.path.join(output_dir, "inference.log")) - logger.write(str(args) + "\n") - inference_vllm(args, logger) - logger.close() \ No newline at end of file diff --git a/src/math_grader.py b/src/math_grader.py deleted file mode 100644 index f2c0b2f..0000000 --- a/src/math_grader.py +++ /dev/null @@ -1,1076 +0,0 @@ -# Copyright 2025 Garena Online Private Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides a math answer grading function with high recall. -Based on HF math_verify, verl, open reasoner zero, etc. -""" - -import re -import signal -from itertools import islice, zip_longest -from math import isclose -from typing import Optional - -import sympy -from latex2sympy2_extended import latex2sympy -# from math_verify import (ExprExtractionConfig, LatexExtractionConfig, parse, verify) -from pylatexenc import latex2text -from sympy import N, simplify -from sympy.parsing import sympy_parser -from sympy.parsing.latex import parse_latex -from sympy.parsing.sympy_parser import parse_expr - - -# Dan Hendrycks' code -def mathd_normalize_answer(answer: Optional[str]) -> Optional[str]: - if answer is None: - return None - answer = answer.strip() - try: - # Remove enclosing `\text{}`. - m = re.search("^\\\\text\{(?P.+?)\}$", answer) - if m is not None: - answer = m.group("text").strip() - return _strip_string(answer) - except: - return answer - - -# units mainly from MathQA -unit_texts = [ - "east", - "degree", - "mph", - "kmph", - "ft", - "m sqaure", - " m east", - "sq m", - "deg", - "mile", - "q .", - "monkey", - "prime", - "ratio", - "profit of rs", - "rd", - "o", - "gm", - "p . m", - "lb", - "tile", - "per", - "dm", - "lt", - "gain", - "ab", - "way", - "west", - "a .", - "b .", - "c .", - "d .", - "e .", - "f .", - "g .", - "h .", - "t", - "a", - "h", - "no change", - "men", - "soldier", - "pie", - "bc", - "excess", - "st", - "inches", - "noon", - "percent", - "by", - "gal", - "kmh", - "c", - "acre", - "rise", - "a . m", - "th", - "π r 2", - "sq", - "mark", - "l", - "toy", - "coin", - "sq . m", - "gallon", - "° f", - "profit", - "minw", - "yr", - "women", - "feet", - "am", - "pm", - "hr", - "cu cm", - "square", - "v â € ™", - "are", - "rupee", - "rounds", - "cubic", - "cc", - "mtr", - "s", - "ohm", - "number", - "kmph", - "day", - "hour", - "minute", - "min", - "second", - "man", - "woman", - "sec", - "cube", - "mt", - "sq inch", - "mp", - "∏ cm ³", - "hectare", - "more", - "sec", - "unit", - "cu . m", - "cm 2", - "rs .", - "rs", - "kg", - "g", - "month", - "km", - "m", - "cm", - "mm", - "apple", - "liter", - "loss", - "yard", - "pure", - "year", - "increase", - "decrease", - "d", - "less", - "Surface", - "litre", - "pi sq m", - "s .", - "metre", - "meter", - "inch", -] - -unit_texts.extend([t + "s" for t in unit_texts]) - - -def _strip_string(string): - def _fix_fracs(string): - substrs = string.split("\\frac") - new_str = substrs[0] - if len(substrs) > 1: - substrs = substrs[1:] - for substr in substrs: - new_str += "\\frac" - if substr[0] == "{": - new_str += substr - else: - try: - assert len(substr) >= 2 - except: - return string - a = substr[0] - b = substr[1] - if b != "{": - if len(substr) > 2: - post_substr = substr[2:] - new_str += "{" + a + "}{" + b + "}" + post_substr - else: - new_str += "{" + a + "}{" + b + "}" - else: - if len(substr) > 2: - post_substr = substr[2:] - new_str += "{" + a + "}" + b + post_substr - else: - new_str += "{" + a + "}" + b - string = new_str - return string - - def _fix_a_slash_b(string): - if len(string.split("/")) != 2: - return string - a = string.split("/")[0] - b = string.split("/")[1] - try: - a = int(a) - b = int(b) - assert string == "{}/{}".format(a, b) - new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" - return new_string - except: - return string - - def _remove_right_units(string): - # "\\text{ " only ever occurs (at least in the val set) when describing units - if "\\text{ " in string: - splits = string.split("\\text{ ") - assert len(splits) == 2 - return splits[0] - else: - return string - - def _fix_sqrt(string): - if "\\sqrt" not in string: - return string - splits = string.split("\\sqrt") - new_string = splits[0] - for split in splits[1:]: - if split[0] != "{": - a = split[0] - new_substr = "\\sqrt{" + a + "}" + split[1:] - else: - new_substr = "\\sqrt" + split - new_string += new_substr - return new_string - - # linebreaks - string = string.replace("\n", "") - # print(string) - - # remove inverse spaces - string = string.replace("\\!", "") - # print(string) - - # replace \\ with \ - string = string.replace("\\\\", "\\") - # print(string) - - # matrix - string = re.sub(r"\\begin\{array\}\{.*?\}", r"\\begin{pmatrix}", string) - string = re.sub(r"\\end\{array\}", r"\\end{pmatrix}", string) - string = string.replace("bmatrix", "pmatrix") - - # replace tfrac and dfrac with frac - string = string.replace("tfrac", "frac") - string = string.replace("dfrac", "frac") - string = ( - string.replace("\\neq", "\\ne") - .replace("\\leq", "\\le") - .replace("\\geq", "\\ge") - ) - # print(string) - - # remove \left and \right - string = string.replace("\\left", "") - string = string.replace("\\right", "") - # print(string) - - # Remove unit: miles, dollars if after is not none - _string = re.sub(r"\\text{.*?}$", "", string).strip() - if _string != "" and _string != string: - # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) - string = _string - - # Remove unit: texts - for _ in range(2): - for unit_text in unit_texts: - # use regex, the prefix should be either the start of the string or a non-alphanumeric character - # the suffix should be either the end of the string or a non-alphanumeric character - _string = re.sub(r"(^|\W)" + unit_text + r"($|\W)", r"\1\2", string) - if _string != "": - string = _string - - # Remove circ (degrees) - string = string.replace("^{\\circ}", "") - string = string.replace("^\\circ", "") - - # remove dollar signs - string = string.replace("\\$", "") - - # remove units (on the right) - string = _remove_right_units(string) - - # remove percentage - string = string.replace("\\%", "") - string = string.replace("\%", "") - - # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string - string = string.replace(" .", " 0.") - string = string.replace("{.", "{0.") - # if empty, return empty string - if len(string) == 0: - return string - if string[0] == ".": - string = "0" + string - - # to consider: get rid of e.g. "k = " or "q = " at beginning - if len(string.split("=")) == 2: - if len(string.split("=")[0]) <= 2: - string = string.split("=")[1] - - # fix sqrt3 --> sqrt{3} - string = _fix_sqrt(string) - - # remove spaces - string = string.replace(" ", "") - - # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} - string = _fix_fracs(string) - - # manually change 0.5 --> \frac{1}{2} - if string == "0.5": - string = "\\frac{1}{2}" - - # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y - string = _fix_a_slash_b(string) - - return string - - -SUBSTITUTIONS = [ - ("an ", ""), - ("a ", ""), - (".$", "$"), - ("\\$", ""), - (r"\ ", ""), - (" ", ""), - ("mbox", "text"), - (",\\text{and}", ","), - ("\\text{and}", ","), - ("\\text{m}", "\\text{}"), -] - - -REMOVED_EXPRESSIONS = [ - "square", - "ways", - "integers", - "dollars", - "mph", - "inches", - "ft", - "hours", - "km", - "units", - "\\ldots", - "sue", - "points", - "feet", - "minutes", - "digits", - "cents", - "degrees", - "cm", - "gm", - "pounds", - "meters", - "meals", - "edges", - "students", - "childrentickets", - "multiples", - "\\text{s}", - "\\text{.}", - "\\text{\ns}", - "\\text{}^2", - "\\text{}^3", - "\\text{\n}", - "\\text{}", - r"\mathrm{th}", - r"^\circ", - r"^{\circ}", - r"\;", - r",\!", - "{,}", - '"', - "\\dots", -] - - -def normalize_final_answer(final_answer: str) -> str: - """ - Normalize a final answer to a quantitative reasoning question. - This code comes from https://arxiv.org/pdf/2206.14858.pdf, page18. - """ - # final_answer = final_answer.split("=")[-1] - - for before, after in SUBSTITUTIONS: - final_answer = final_answer.replace(before, after) - for expr in REMOVED_EXPRESSIONS: - final_answer = final_answer.replace(expr, "") - - # Extract answer that is in LaTeX math, is bold, - # is surrounded by a box, etc. - final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) - final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) - - # Normalize shorthand TeX: - # \fracab -> \frac{a}{b} - # \frac{abc}{bef} -> \frac{abc}{bef} - # \fracabc -> \frac{a}{b}c - # \sqrta -> \sqrt{a} - # \sqrtab -> sqrt{a}b - final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) - final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) - final_answer = final_answer.replace("$", "") - - # Normalize 100,000 -> 100000 - if final_answer.replace(",", "").isdigit(): - final_answer = final_answer.replace(",", "") - - return final_answer - - -def repeatness(s: str): - def ranks(l): - index = {v: i for i, v in enumerate(sorted(set(l)))} - return [index[v] for v in l] - - def suffixArray(s): - line = ranks(s) - n, k, ans, sa = len(s), 1, line, [0] * len(s) - while k < n - 1: - line = ranks(list(zip_longest(line, islice(line, k, None), fillvalue=-1))) - ans, k = line, k << 1 - for i, k in enumerate(ans): - sa[k] = i - return ans, sa - - def lcp(arr, suffixArr, inv_suff): - n, ans, k = len(arr), [0] * len(arr), 0 - - for i in range(n): - if inv_suff[i] == n - 1: - k = 0 - continue - - j = suffixArr[inv_suff[i] + 1] - while i + k < n and j + k < n and arr[i + k] == arr[j + k]: - k += 1 - - ans[inv_suff[i]] = k - if k > 0: - k -= 1 - - return ans - - arr = [ord(i) for i in s] - n = len(arr) - if n <= 1: - return 0 - c, sa = suffixArray(arr) - cnt = sum(lcp(arr, sa, c)) - - return (cnt * 2 / (n * (n + 1))) > 0.2 - - -class timeout: - def __init__(self, seconds=1, error_message="Timeout"): - self.seconds = seconds - self.error_message = error_message - - def handle_timeout(self, signum, frame): - raise TimeoutError(self.error_message) - - def __enter__(self): - signal.signal(signal.SIGALRM, self.handle_timeout) - signal.alarm(self.seconds) - - def __exit__(self, type, value, traceback): - signal.alarm(0) - - -def latex_eval(latex): - sym = parse_latex(latex) - val = sym.evalf() - return sym, val - - -def numeric_equal(prediction: float, reference: float): - # Note that relative tolerance has significant impact - # on the result of the synthesized GSM-Hard dataset - # if reference.is_integer(): - # return isclose(reference, round(prediction), abs_tol=1e-4) - # else: - # prediction = round(prediction, len(str(reference).split(".")[-1])) - return isclose(reference, prediction, rel_tol=1e-4) - - -def symbolic_equal(a, b): - def _parse(s): - for f in [parse_latex, parse_expr, latex2sympy]: - try: - return f(s.replace("\\\\", "\\")) - except: - try: - return f(s) - except: - pass - return s - - a = _parse(a) - b = _parse(b) - - # direct equal - try: - if str(a) == str(b) or a == b: - return True - except: - pass - - # simplify equal - try: - if a.equals(b) or simplify(a - b) == 0: - return True - except: - pass - - # equation equal - try: - if (abs(a.lhs - a.rhs)).equals(abs(b.lhs - b.rhs)): - return True - except: - pass - - try: - if numeric_equal(float(N(a)), float(N(b))): - return True - except: - pass - - # matrix - try: - # if a and b are matrix - if a.shape == b.shape: - _a = a.applyfunc(lambda x: round(x, 3)) - _b = b.applyfunc(lambda x: round(x, 3)) - if _a.equals(_b): - return True - except: - pass - - return False - - -def _is_latex_equal(str1, str2): - try: - sym1, val1 = latex_eval(str1) - sym2, val2 = latex_eval(str2) - if sym1 == sym2 or val1 == val2: - return True - else: - raise ValueError - except Exception: # noqa - try: - norm1, norm2 = normalize_final_answer(str1), normalize_final_answer(str2) - sym1, val1 = latex_eval(norm1) - sym2, val2 = latex_eval(norm2) - if sym1 == sym2 or val1 == val2: - return True - except Exception: # noqa - return norm1 == norm2 - return False - - -def is_latex_equal(given_answer: str, ground_truth: str) -> bool: - try: - with timeout(1): - try: - if (len(given_answer) > 128 and repeatness(given_answer)) or ( - len(ground_truth) > 128 and repeatness(ground_truth) - ): - return False - # First conduct normalized string matching. - ground_truth_normalized = _normalize(ground_truth) - given_normalized = _normalize(given_answer) - if ground_truth_normalized is None: - return False - if ground_truth_normalized == given_normalized: - return True - - # Next call math verify. - given_answer.replace("\n", "") - ground_truth.replace("\n", "") - if not "$" in given_answer: - given_answer = f"${given_answer}$" - if not "$" in ground_truth: - ground_truth = f"${ground_truth}$" - return verify( - parse( - ground_truth, - extraction_config=( - LatexExtractionConfig(boxed_match_priority=0), - ExprExtractionConfig(), - ), - fallback_mode="no_fallback", - extraction_mode=["first_match"], - parsing_timeout=1, - ), - parse( - given_answer, - extraction_config=( - LatexExtractionConfig(boxed_match_priority=0), - ExprExtractionConfig(), - ), - fallback_mode="no_fallback", - extraction_mode=["first_match"], - parsing_timeout=1, - ), - timeout_seconds=1, - ) - # or symbolic_equal(ground_truth, given_answer) - except Exception: - return False - except TimeoutError: - return False - - -def is_value_equal(given_answer: str, ground_truth: str) -> bool: - assert ground_truth is not None - ground_truth_normalized_mathd = mathd_normalize_answer(ground_truth) - given_answer_normalized_mathd = mathd_normalize_answer(given_answer) - - str_equal = ground_truth_normalized_mathd == given_answer_normalized_mathd - try: - number_equal = float(ground_truth_normalized_mathd) == float( - given_answer_normalized_mathd - ) - return str_equal or number_equal - except Exception: - return str_equal - - -# sympy might hang -- we don't care about trying to be lenient in these cases -BAD_SUBSTRINGS = ["^{", "^("] -BAD_REGEXES = ["\^[0-9]+\^", "\^[0-9][0-9]+"] -TUPLE_CHARS = "()[]" - - -def _sympy_parse(expr: str): - """Parses an expression with sympy.""" - py_expr = expr.replace("^", "**") - return sympy_parser.parse_expr( - py_expr, - transformations=( - sympy_parser.standard_transformations - + (sympy_parser.implicit_multiplication_application,) - ), - ) - - -def _parse_latex(expr: str) -> str: - """Attempts to parse latex to an expression sympy can read.""" - expr = expr.replace("\\tfrac", "\\frac") - expr = expr.replace("\\dfrac", "\\frac") - expr = expr.replace("\\frac", " \\frac") # Play nice with mixed numbers. - expr = latex2text.LatexNodes2Text().latex_to_text(expr) - - # Replace the specific characters that this parser uses. - expr = expr.replace("√", "sqrt") - expr = expr.replace("π", "pi") - expr = expr.replace("∞", "inf") - expr = expr.replace("∪", "U") - expr = expr.replace("·", "*") - expr = expr.replace("×", "*") - - return expr.strip() - - -def _is_float(num: str) -> bool: - try: - float(num) - return True - except ValueError: - return False - - -def _is_int(x: float) -> bool: - try: - return abs(x - int(round(x))) <= 1e-7 - except: - return False - - -def _is_frac(expr: str) -> bool: - return bool(re.search(r"^-?[0-9]+.?/0*[1-9][0-9]*.?$", expr)) - - -def _str_is_int(x: str) -> bool: - try: - x = _strip_properly_formatted_commas(x) - x = float(x) - return abs(x - int(round(x))) <= 1e-7 - except: - return False - - -def _str_to_int(x: str) -> bool: - x = x.replace(",", "") - x = float(x) - return int(x) - - -def _inject_implicit_mixed_number(step: str): - """ - Automatically make a mixed number evalable - e.g. 7 3/4 => 7+3/4 - """ - p1 = re.compile("([0-9]) +([0-9])") - step = p1.sub("\\1+\\2", step) ## implicit mults - return step - - -def _strip_properly_formatted_commas(expr: str): - # We want to be careful because we don't want to strip tuple commas - p1 = re.compile("(\d)(,)(\d\d\d)($|\D)") - while True: - next_expr = p1.sub("\\1\\3\\4", expr) - if next_expr == expr: - break - expr = next_expr - return next_expr - - -def _normalize(expr: str) -> str: - """Normalize answer expressions.""" - if expr is None: - return None - - # Remove enclosing `\text{}`. - m = re.search("^\\\\text\{(?P.+?)\}$", expr) - if m is not None: - expr = m.group("text") - - expr = expr.replace("\\%", "%") - expr = expr.replace("\\$", "$") - expr = expr.replace("$", "") - expr = expr.replace("%", "") - expr = expr.replace(" or ", " , ") - expr = expr.replace(" and ", " , ") - - expr = expr.replace("million", "*10^6") - expr = expr.replace("billion", "*10^9") - expr = expr.replace("trillion", "*10^12") - - for unit in [ - "degree", - "cm", - "centimeter", - "meter", - "mile", - "second", - "minute", - "hour", - "day", - "week", - "month", - "year", - "foot", - "feet", - "inch", - "yard", - ]: - expr = re.sub(f"{unit}(es)?(s)? *(\^[0-9]+)?", "", expr) - expr = re.sub(f"\^ *\\\\circ", "", expr) - - if len(expr) > 0 and expr[0] == "{" and expr[-1] == "}": - expr = expr[1:-1] - - expr = re.sub(",\\\\! *", "", expr) - if _is_float(expr) and _is_int(float(expr)): - expr = str(int(round(float(expr)))) - if "\\" in expr: - try: - expr = _parse_latex(expr) - except: - pass - - # edge case with mixed numbers and negative signs - expr = re.sub("- *", "-", expr) - - expr = _inject_implicit_mixed_number(expr) - expr = expr.replace(" ", "") - - # if we somehow still have latex braces here, just drop them - expr = expr.replace("{", "") - expr = expr.replace("}", "") - - # don't be case sensitive for text answers - expr = expr.lower() - - if _str_is_int(expr): - expr = str(_str_to_int(expr)) - - return expr - - -def count_unknown_letters_in_expr(expr: str): - expr = expr.replace("sqrt", "") - expr = expr.replace("frac", "") - letters_in_expr = set([x for x in expr if x.isalpha()]) - return len(letters_in_expr) - - -def should_allow_eval(expr: str): - # we don't want to try parsing unknown text or functions of more than two variables - if count_unknown_letters_in_expr(expr) > 2: - return False - - for bad_string in BAD_SUBSTRINGS: - if bad_string in expr: - return False - - for bad_regex in BAD_REGEXES: - if re.search(bad_regex, expr) is not None: - return False - - return True - - -def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str): - are_equal = False - try: - expr = f"({ground_truth_normalized})-({given_normalized})" - if should_allow_eval(expr): - sympy_diff = _sympy_parse(expr) - simplified = sympy.simplify(sympy_diff) - if simplified == 0: - are_equal = True - except: - pass - return are_equal - - -def split_tuple(expr: str): - """ - Split the elements in a tuple/interval, while handling well-formatted commas in large numbers - """ - expr = _strip_properly_formatted_commas(expr) - if len(expr) == 0: - return [] - if ( - len(expr) > 2 - and expr[0] in TUPLE_CHARS - and expr[-1] in TUPLE_CHARS - and all([ch not in expr[1:-1] for ch in TUPLE_CHARS]) - ): - elems = [elem.strip() for elem in expr[1:-1].split(",")] - else: - elems = [expr] - return elems - - -def last_boxed_only_string(string): - idx = string.rfind("\\boxed") - if idx < 0: - idx = string.rfind("\\fbox") - if idx < 0: - return None - - i = idx - right_brace_idx = None - num_left_braces_open = 0 - while i < len(string): - if string[i] == "{": - num_left_braces_open += 1 - if string[i] == "}": - num_left_braces_open -= 1 - if num_left_braces_open == 0: - right_brace_idx = i - break - i += 1 - - if right_brace_idx == None: - retval = None - else: - retval = string[idx : right_brace_idx + 1] - - return retval - - -def remove_boxed(s): - left = "\\boxed{" - try: - assert s[: len(left)] == left - assert s[-1] == "}" - return s[len(left) : -1] - except: - return None - - -def extract_boxed_answer(solution: str) -> str: - """Extract the answer from inside a LaTeX \\boxed{} command""" - solution = last_boxed_only_string(solution) - solution = remove_boxed(solution) - return solution - - -def grade_answer_sympy(given_answer: str, ground_truth: str) -> bool: - ground_truth_normalized = _normalize(ground_truth) - given_normalized = _normalize(given_answer) - - if ground_truth_normalized is None: - return False - - if ground_truth_normalized == given_normalized: - return True - - if len(given_normalized) == 0: - return False - - ground_truth_elems = split_tuple(ground_truth_normalized) - given_elems = split_tuple(given_normalized) - - if len(ground_truth_elems) > 1 and ( - ground_truth_normalized[0] != given_normalized[0] - or ground_truth_normalized[-1] != given_normalized[-1] - ): - is_correct = False - elif len(ground_truth_elems) != len(given_elems): - is_correct = False - else: - for ground_truth_elem, given_elem in zip(ground_truth_elems, given_elems): - if _is_frac(ground_truth_elem) and _is_frac(given_elem): - # if fractions aren't reduced, then shouldn't be marked as correct - # so, we don't want to allow sympy.simplify in this case - is_correct = ground_truth_elem == given_elem - elif _str_is_int(ground_truth_elem) != _str_is_int(given_elem): - # if the ground truth answer is an integer, we require the given answer to be a strict match (no sympy.simplify) - is_correct = False - else: - is_correct = are_equal_under_sympy(ground_truth_elem, given_elem) - if not is_correct: - break - - return is_correct - - -def grade_answer_mathd(given_answer: str, ground_truth: str) -> bool: - ground_truth_normalized_mathd = mathd_normalize_answer(ground_truth) - given_answer_normalized_mathd = mathd_normalize_answer(given_answer) - - # be at least as lenient as mathd - if ground_truth_normalized_mathd == given_answer_normalized_mathd: - return True - return False - - -def extract_answer(passage: str) -> str: - if "\\boxed" in passage: - return extract_boxed_answer(passage) - return None - - -def grade(model_answer: str, gt_answer: str, fast: bool = True): - if "\\boxed" in gt_answer: - gt_answer = extract_answer(gt_answer) - correct = grade_answer_mathd(model_answer, gt_answer) or grade_answer_sympy( - model_answer, gt_answer - ) - if not fast: - # This mode further uses math_verify to recall originally false positives. - # Will be a bit slower, and sensitive to bad inputs. - correct = correct or is_latex_equal( - model_answer, - gt_answer, - ) - return correct - - -def boxed_reward_fn(model_response, gt_answer, fast=False): - model_answer = extract_answer(model_response) - if model_answer is None: - return {"formatted": False}, 0.0 # Cannot even parse anything. - if isinstance(gt_answer, float) or isinstance(gt_answer, int): - gt_answer = str(gt_answer) - if isinstance(gt_answer, str): - is_correct = grade(model_answer, gt_answer, fast) - elif isinstance(gt_answer, list): - is_correct = False - for gt in gt_answer: - is_correct |= grade(model_answer, gt, fast) - if is_correct: - return {"formatted": True}, 1.0 # Correctness reward. - else: - return { - "formatted": True - }, 0.0 # Formatted but wrong answer; no format reward to avoid hacking. - - -def answer_tag_reward_fn(model_response, gt_answer, fast=False): - # We are strict about format to evaluate our models. - if " " in model_response and "" in model_response: - model_answer = model_response.split("")[-1].replace("", "") - if "\\boxed" in model_answer: - model_answer = extract_answer(model_answer) - if model_answer is None: - return {"formatted": True}, 0.0 - if isinstance(gt_answer, float) or isinstance(gt_answer, int): - gt_answer = str(gt_answer) - if isinstance(gt_answer, str): - is_correct = grade(model_answer, gt_answer, fast) - elif isinstance(gt_answer, list): - is_correct = False - for gt in gt_answer: - is_correct |= grade(model_answer, gt, fast) - if is_correct: - return {"formatted": True}, 1.0 # Correctness reward. - else: - return ( - {"formatted": True}, - 0.0, - ) # Formatted but wrong answer; no format reward to avoid hacking. - else: - return {"formatted": False}, 0.0 # Unformatted. - - -def answer_tag_reward_fn_for_orz(model_response, gt_answer, fast=False): - # We are a bit less strict for baselines. - if "" in model_response and "" in model_response: - model_answer = model_response.split("")[-1].replace("", "") - if "\\boxed" in model_answer: - model_answer = extract_answer(model_answer) - if model_answer is None: - return {"formatted": True}, 0.0 - if isinstance(gt_answer, float) or isinstance(gt_answer, int): - gt_answer = str(gt_answer) - if isinstance(gt_answer, str): - is_correct = grade(model_answer, gt_answer, fast) - elif isinstance(gt_answer, list): - is_correct = False - for gt in gt_answer: - is_correct |= grade(model_answer, gt, fast) - if is_correct: - return {"formatted": True}, 1.0 # Correctness reward. - else: - return ( - {"formatted": True}, - 0.0, - ) # Formatted but wrong answer; no format reward to avoid hacking. - else: - return {"formatted": False}, 0.0 # Unformatted. \ No newline at end of file diff --git a/src/probing_reflection/__init__.py b/src/probing_reflection/__init__.py new file mode 100644 index 0000000..1bd31d1 --- /dev/null +++ b/src/probing_reflection/__init__.py @@ -0,0 +1,24 @@ +"""ProbingReflection - Probing and Modulating Self-Reflection in Language Models. + +This package provides tools for investigating self-reflection in Large Language +Models through probing and steering techniques. + +Public API: + ProbingConfig: Configuration for probing experiments + ReflectionResult: Result container for reflection analysis + ContrastivePair: TypedDict for contrastive example pairs +""" + +from probing_reflection.types import ( + ContrastivePair, + ProbingConfig, + ReflectionResult, +) + +__all__ = [ + "ContrastivePair", + "ProbingConfig", + "ReflectionResult", +] + +__version__ = "0.1.0" diff --git a/asset/placeholder b/src/probing_reflection/py.typed similarity index 100% rename from asset/placeholder rename to src/probing_reflection/py.typed diff --git a/src/probing_reflection/types.py b/src/probing_reflection/types.py new file mode 100644 index 0000000..68e3190 --- /dev/null +++ b/src/probing_reflection/types.py @@ -0,0 +1,58 @@ +"""Core type definitions for ProbingReflection. + +This module defines the foundational types used throughout the project +for LLM probing and steering experiments. All configuration types are +immutable (frozen dataclasses) to ensure reproducibility. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TypedDict + + +@dataclass(frozen=True) +class ProbingConfig: + """Immutable configuration for probing experiments. + + Attributes: + model_name: Name or path of the model to probe. + layer_indices: Tuple of layer indices to extract activations from. + Using tuple (not list) ensures hashability. + """ + + model_name: str = "" + layer_indices: tuple[int, ...] = () + + +@dataclass(frozen=True) +class ReflectionResult: + """Immutable result of a reflection analysis. + + Stores the outcome of analyzing a sample for self-reflection patterns. + + Attributes: + sample_id: Unique identifier for the analyzed sample. + reflection_score: Computed reflection score in [0.0, 1.0]. + metadata: Additional metadata about the analysis (source, model, etc.). + """ + + sample_id: str + reflection_score: float + metadata: dict[str, str] = field(default_factory=dict) + + +class ContrastivePair(TypedDict): + """Structured representation of contrastive examples. + + Used for training steering vectors and probes. Each pair consists + of a positive example (exhibiting target behavior) and a negative + example (lacking target behavior). + + Attributes: + positive: Example text exhibiting the target behavior. + negative: Example text lacking the target behavior. + """ + + positive: str + negative: str diff --git a/src/save_insert_model.py b/src/save_insert_model.py deleted file mode 100644 index 9dc024c..0000000 --- a/src/save_insert_model.py +++ /dev/null @@ -1,43 +0,0 @@ -import torch -from transformers import AutoTokenizer, AutoModelForCausalLM -from inference import find_module, InsertLayer -from safetensors.torch import save_file - - -with torch.no_grad(): - layer = 20 - alpha = -0.1 - - model_name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" - # Load the tokenizer - - model = AutoModelForCausalLM.from_pretrained( - model_name, - device_map="auto", - torch_dtype=torch.bfloat16, - ) - - mlp_keywords = ["mlp", "feedforward", "ffn"] - w_wait = ( - torch.load( - f"./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/DeepSeek-R1-Distill-Qwen-1.5B_hs/before_wait_DeepSeek-R1-Distill-Qwen-1.5B_{args.injection_layer}_-1.pt" - ) - .cpu() - .to(torch.float32) - ) - w_wo_wait = ( - torch.load( - f"./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/DeepSeek-R1-Distill-Qwen-1.5B_hs/before_wo_wait_DeepSeek-R1-Distill-Qwen-1.5B_{args.injection_layer}_-1.pt" - ) - .cpu() - .to(torch.float32) - ) - insert_vector = w_wait.mean(dim=0) - w_wo_wait.mean(dim=0) - - original_mlp = find_module(model.model.layers[layer], mlp_keywords) - model.model.layers[19].mlp = torch.nn.Sequential( - original_mlp, - InsertLayer(insert_vector.to("cuda").to(torch.bfloat16), alpha=alpha), - ) - state_dict = model.state_dict() - save_file(state_dict, "model.safetensors") \ No newline at end of file diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index 8ceaed4..0000000 --- a/src/utils.py +++ /dev/null @@ -1,1504 +0,0 @@ -import datasets -import torch -import os -import json -import huggingface_hub as hf_hub -from typing import Tuple, List -from collections import Counter, defaultdict -import re -from transformers import AutoModelForCausalLM, AutoTokenizer -from tqdm import tqdm -from torch.nn import DataParallel -from scipy.stats import pearsonr -import matplotlib.pyplot as plt -from math_grader import boxed_reward_fn -import numpy as np -import umap -from matplotlib.gridspec import GridSpec - - -def inference_gpqa(): - ds = datasets.load_dataset("Idavidrein/gpqa", "gpqa_diamond") - prompts = [] - prompt = """What is the correct answer to this question: {question} - \n\nChoices:\n(A) {choice1}\n(B) {choice2}\n(C) {choice3}\n(D) {choice4} - Let's Let's think step by step and answer in the format \"The correct answer is (insert answer here)\".""" - for example in ds["train"]: - question = example["Question"] - choice1 = example["Correct Answer"] - choice2 = example["Incorrect Answer 1"] - choice3 = example["Incorrect Answer 2"] - choice4 = example["Incorrect Answer 3"] - - res = prompt.format( - question=question, - choice1=choice1, - choice2=choice2, - choice3=choice3, - choice4=choice4, - ) - prompts.append(res) - return prompts - - -def numerical_rank(A): - # Compute singular values - # A = torch.stack(A).to(torch.float32) - singular_values = torch.linalg.svdvals(A.to(torch.float32)) - singular_values = singular_values / singular_values.sum() - # Compute the numerical rank formula - num_rank = (singular_values.sum() ** 2) / (singular_values.square().sum()) - - return num_rank.item() - - -def plot_numerical_rank(): - plt.figure(figsize=(10, 6)) - plt.plot( - range(29), - [ - numerical_rank( - torch.load( - f"../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/hs/before_wait_Qwen2.5-1.5B_{l}_-1.pt" - ) - ) - for l in range(29) - ], - label="qwen_w_wait", - ) - plt.plot( - range(29), - [ - numerical_rank( - torch.load( - f"../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/hs/before_wo_wait_DeepSeek-R1-Distill-Qwen-1.5B_{l}_-1.pt" - ) - ) - for l in range(29) - ], - label="ds_wo_wait", - ) - plt.plot( - range(29), - [ - numerical_rank( - torch.load( - f"../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/hs/before_wait_DeepSeek-R1-Distill-Qwen-1.5B_{l}_-1.pt" - ) - ) - for l in range(29) - ], - label="ds_w_wait", - ) - plt.plot( - range(29), - [ - numerical_rank( - torch.load( - f"../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/hs/before_wo_wait_Qwen2.5-1.5B_{l}_-1.pt" - ) - ) - for l in range(29) - ], - label="qwen_wo_wait", - ) - plt.xlabel("Layer") - plt.ylabel("Numerical Rank") - plt.title("Numerical Rank Across Layers") - plt.legend() - plt.show() - - -def test_prompt_boxed( - prompt: str, model_name: str, dataset_name: str, save_respnses: bool = True -) -> tuple[int, int, int]: - """ - Test the performance of a prompt by checking how many answers are in the boxed format. - - Args: - prompt (str): The input prompt to the model. - model_name (str): The name or path of the model to test. - tokenizer_name (str): The name or path of the tokenizer to use. - - Returns: - float: The percentage of answers in the boxed format. - """ - - # Load the model and tokenizer - model = AutoModelForCausalLM.from_pretrained( - model_name, device_map="auto", torch_dtype=torch.bfloat16 - ) - model = DataParallel(model) # Enable multi-GPU support - model.eval() - tokenizer = AutoTokenizer.from_pretrained(model_name) - - # Load the dataset - dataset = datasets.load_dataset(dataset_name)["test"] - - # dataset = dataset.select( - # range(int(len(dataset) * 1e-2)) - # ) # Select a subset for testing - dataset_length = len(dataset) - model_name = model_name.split("/")[-1] - dataset_name = dataset_name.split("/")[-1] - if save_respnses: - os.makedirs( - f"./asset/insert_response/{model_name}_{dataset_name}_{len(prompt)}", - exist_ok=True, - ) - with torch.no_grad(): - - for idx, problem in enumerate(tqdm(dataset["problem"])): - formatted_prompt = prompt.format(problems=str(problem)) - input_ids = tokenizer(formatted_prompt, return_tensors="pt")[ - "input_ids" - ].to("cuda:0") - - outputs = model.module.generate( # Use model.module for DataParallel - input_ids, - max_new_tokens=8196, - use_cache=True, - temperature=0.7, - do_sample=True, - output_hidden_states=True, - return_dict_in_generate=True, - attention_mask=torch.ones_like(input_ids), - pad_token_id=tokenizer.eos_token_id, - ) - response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True) - with open( - f"./asset/insert_response/{model_name}_{dataset_name}_{len(prompt)}/{idx}.json", - "w", - ) as f: - json.dump({"problem": problem, "response": response}, f, indent=2) - - return dataset_length - - -def extract_key_token( - response_dir: str, - model_name: str, - token_position: int = -2, - layer: int = 9, - token_list: list[int] = [382], -) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Extract key token embeddings from model responses. - - Args: - response_path (str): Path to the response files - model_name (str): Name of the model to load - token_position (int): Position of the token to extract - - Returns: - Tuple[torch.Tensor, torch.Tensor]: Extracted token embeddings and their labels - """ - # Load model and tokenizer - model = AutoModelForCausalLM.from_pretrained( - model_name, device_map="auto", torch_dtype=torch.bfloat16 - ) - model.eval() - tokenizer = AutoTokenizer.from_pretrained(model_name) - - last_token_before_wait = [] - last_token_before_wo_wait = [] - for idx, path in enumerate(tqdm(os.listdir(response_dir))): - if not path.endswith(".json"): - continue - with open(os.path.join(response_dir, path), "r") as f: - response = json.load(f) - # layers = len(hidden_states[-1]) - input_ids = tokenizer(response["response"], return_tensors="pt")[ - "input_ids" - ].to( - "cuda:0" - ) # shape: (1, seq_length) - problem_length = tokenizer(response["problem"], return_tensors="pt")[ - "input_ids" - ].shape[1] - - input_length = input_ids.shape[1] - wait_word = ["wait", "Wait", " wait", " Wait"] - wait_list = [] - outs = model.generate( - input_ids=input_ids, - max_new_tokens=1, - do_sample=True, - use_cache=True, - output_hidden_states=True, - return_dict_in_generate=True, - attention_mask=torch.ones_like(input_ids), - pad_token_id=tokenizer.eos_token_id, - ) # shape: (1, layers, outputs_len, seq_length, dim) - - wait_list = tokenizer(wait_word, return_tensors="pt")["input_ids"][:, -1] - - indices = [] - for word in wait_list: - index = (input_ids[0] == word.item()).nonzero().squeeze() - if index.dim() == 0: # if it's a scalar, add a dimension - index = index.unsqueeze(0) - indices.append(index) - res = torch.cat(indices) - for idy in res: - last_token_before_wait.append( - outs.hidden_states[0][layer][0][idy + token_position] - ) - for token in token_list: - index = (input_ids[0] == token).nonzero().squeeze() - if index.dim() == 0: # if it's a scalar, add a dimension - index = index.unsqueeze(0) - for i in range(index.shape[0]): - input_length = input_ids[0].shape[0] - if index[i] + 50 > input_length: - search_end_index = input_length - else: - search_end_index = index[i] + 50 - flag = False - for word in wait_list: - if word in input_ids[0][index[i] : search_end_index]: - flag = True - break - if not flag: - if index[i] + 10 >= input_length: - continue - last_token_before_wo_wait.append( - outs.hidden_states[0][layer][0][index[i] + token_position] - ) - short_model_name = model_name.split("/")[-1] - hs_dir = os.path.join( - response_dir, f"{short_model_name}_hs" - ) # Create a directory path without leading slash - os.makedirs(hs_dir, exist_ok=True) - - wait_path = os.path.join( - hs_dir, f"before_wait_{short_model_name}_{layer}_{token_position}.pt" - ) - wo_wait_path = os.path.join( - hs_dir, f"before_wo_wait_{short_model_name}_{layer}_{token_position}.pt" - ) - - torch.save(torch.stack(last_token_before_wait, dim=0), wait_path) - torch.save(torch.stack(last_token_before_wo_wait, dim=0), wo_wait_path) - return last_token_before_wait, last_token_before_wo_wait - - -def numerical_rank(A): - # Compute singular values - singular_values = torch.linalg.svdvals(A) - singular_values = singular_values / singular_values.sum() - # Compute the numerical rank formula - num_rank = (singular_values.sum() ** 2) / (singular_values.square().sum()) - - return num_rank - - -def neumann_entropy(matrix: torch.Tensor) -> torch.Tensor: - """ - Compute the Neumann entropy of a matrix using PyTorch. - Args: - matrix: Matrix (torch.Tensor). - Returns: - Neumann entropy of the matrix as a torch.Tensor. - """ - # matrix = normalized_empirical_covariance(matrix) - s = torch.linalg.svdvals(matrix) - # threshold_ = s[0] * 1e-5 - # s = s[s > threshold_] - normalized_s = s / s.sum() - return -torch.sum( - normalized_s - * ( - torch.log(normalized_s) - / torch.log(torch.tensor(3584, device=normalized_s.device)) - ) - ) - - -def plot_umap_embeddings( - layers, asset_path, n_neighbors=10, min_dist=0.1, random_state=42, figsize=(20, 20) -): - """ - Plots UMAP embeddings for given layers. - - Parameters: - layers (int): Number of layers to process. - asset_path (str): Path to the asset directory. - n_neighbors (int): Number of neighbors for UMAP. - min_dist (float): Minimum distance for UMAP. - random_state (int): Random state for reproducibility. - figsize (tuple): Size of the figure. - """ - - fig, axes = plt.subplots(8, 4, figsize=figsize) - axes = axes.flatten() - colors = { - "wait token": "red", - "no wait token": "orange", - } - - for layer in tqdm(range(layers)): - token_before_wait = torch.load( - f"{asset_path}/hs/before_wait_Qwen2.5-1.5B_{layer}.pt" - ) - token_wo_wait = torch.load( - f"{asset_path}/hs/before_wo_wait_Qwen2.5-1.5B_{layer}.pt" - ) - - reducer = umap.UMAP( - n_neighbors=n_neighbors, min_dist=min_dist, random_state=random_state - ) - embeddings = reducer.fit_transform( - np.concatenate( - [ - token_before_wait.cpu().to(torch.float32).numpy(), - token_wo_wait.cpu().to(torch.float32).numpy(), - ] - ) - ) - - wait_labels = ["wait token"] * token_before_wait.shape[0] - nonwait_labels = ["no wait token"] * token_wo_wait.shape[0] - all_labels = wait_labels + nonwait_labels - - for label in set(all_labels): - indices = [i for i, x in enumerate(all_labels) if x == label] - axes[layer].scatter( - embeddings[indices, 0], - embeddings[indices, 1], - label=label, - alpha=0.5, - s=10, - color=colors[label], - ) - axes[layer].set_title(f"Layer {layer}") - axes[layer].set_xlabel("UMAP Dimension 1") - axes[layer].set_ylabel("UMAP Dimension 2") - axes[layer].legend() - axes[layer].grid(True) - - plt.tight_layout() - plt.show() - -# ############################################################################################### - -# import matplotlib.pyplot as plt - -# # 数据 -# performance = [81.04, 79.9, 81.0, 80.4, 83.4, 82.3, 81.9, 83.9, 84.0, 81.1, 86.2, 87.2, 85.8, 85.2, 81.5] -# indices = [-1, -0.3, -0.1, -0.03, -0.01, -0.003, -0.001, 0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1] -# r_len = [1628, 1679, 2743, 2158, 3595, 4270, 4181, 4773, 4872, 5628, 5679, 6743, 8158, 8595, 10270] - -# # 创建图形 -# fig, ax1 = plt.subplots(figsize=(10, 6)) - -# # x轴位置 -# x_positions = range(len(indices)) - -# # 绘制柱状图（响应长度） -# bar_color = "#4A90E2" # 柔和蓝 -# ax1.bar(x_positions, r_len, label="Avg. Response Length", color=bar_color, alpha=0.8) -# ax1.set_xlabel("α (Steering Strength)", fontsize=20) -# ax1.set_ylabel("Average Response Length", color=bar_color, fontsize=20) -# ax1.tick_params(axis='y', labelcolor=bar_color, labelsize=20) -# ax1.set_xticks(x_positions) -# ax1.set_xticklabels(indices, rotation=45, fontsize=20) - -# # 创建第二个 y 轴（性能） -# ax2 = ax1.twinx() -# line_color = "#D0021B" # 深红 -# ax2.plot(x_positions, performance, label="Pass@1", color=line_color, marker='o', linewidth=2) -# ax2.set_ylabel("Pass@1 (%)", color=line_color, fontsize=20) -# ax2.tick_params(axis='y', labelcolor=line_color, labelsize=20) - -# # 标题和图例 -# plt.title("Effect of α on Response Length and Performance", fontsize=20) -# fig.legend(loc="upper left", bbox_to_anchor=(0.2, 0.9), fontsize=20) - -# # 保存图像为PDF -# plt.tight_layout() -# plt.savefig("alpha_steering_effects.pdf", format="pdf", dpi=300) - -# # 显示图形 -# plt.show() -# ############################################################################################### - -############################################################################################### - -# import matplotlib.pyplot as plt - -# # 数据 -# performance = [83.04, 81.95, 86.4, 86.0, 87.2, 85.2, 86.2, 79.8, 59.06] -# indices = [1, 4, 7, 10, 14, 17, 20, 24, 27] -# r_len = [5628, 5679, 8743, 8158, 8595, 8270, 9201, 10773, 9872] - -# # 创建图形 -# fig, ax1 = plt.subplots(figsize=(10, 6)) - -# # 样式设置 -# bar_color = "#6A9FB5" # 柔和蓝色 -# line_color = "#C94C4C" # 柔和红色 -# baseline_color = "#888888" # 灰色虚线 - -# # 柱状图 - 响应长度 -# ax1.bar(indices, r_len, alpha=0.6, label="Avg. Response Length", color=bar_color) -# ax1.set_xlabel("Layer", fontsize=18) # Increased font size -# ax1.set_ylabel("Average Response Length", color=bar_color, fontsize=18) # Increased font size -# ax1.tick_params(axis='y', labelcolor=bar_color, labelsize=18) # Increased tick font size -# ax1.tick_params(axis='x', labelsize=18) # Increased tick font size - -# # 第二 y 轴 - 性能折线 -# ax2 = ax1.twinx() -# ax2.axhline(y=83.9, color=baseline_color, linestyle='--', label="Baseline (83.9%)") -# ax2.plot(indices, performance, label="Pass@1", color=line_color, marker='o', linewidth=2) -# ax2.set_ylabel("Pass@1 (%)", color=line_color, fontsize=18) # Increased font size -# ax2.tick_params(axis='y', labelcolor=line_color, labelsize=18) # Increased tick font size - -# # 标题与图例 -# plt.title("Performance vs. Response Length Across Layers", fontsize=18) # Increased font size -# fig.legend(loc="lower center", bbox_to_anchor=(0.5, 0.2), fontsize=18, ncol=2) # Increased legend font size and moved to bottom - -# # 保存为PDF -# plt.tight_layout() -# plt.savefig("layer_vs_performance_length.pdf", format="pdf", dpi=300) - -# # 显示图形 -# plt.show() -############################################################################################### - -############################################################################################### -# import torch.nn.functional as F -# import torch - -# # Load all layer results -# layer_results = [torch.load(f"asset/before_wait_ds15_{i}_-1.pt", map_location="cpu") for i in range(29)] -# layer_results4 = [torch.load(f"asset/before_wo_wait_ds15_{i}_-1.pt", map_location="cpu") for i in range(29)] -# model_name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" -# from transformers import AutoModelForCausalLM -# # model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.bfloat16) -# layer_results3 = [torch.load(f"../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/DeepSeek-R1-Distill-Qwen-1.5B_hs/before_wait_DeepSeek-R1-Distill-Qwen-1.5B_{i}_-1.pt", map_location="cpu") for i in range(29)] -# layer_results2 = [torch.load(f"../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/DeepSeek-R1-Distill-Qwen-1.5B_hs/before_wo_wait_DeepSeek-R1-Distill-Qwen-1.5B_{i}_-1.pt", map_location="cpu") for i in range(29)] -# # Compute cosine similarity for each layer -# cs = [] - -# for layer in range(29): -# r1 = (layer_results[layer].mean(dim=0) - layer_results4[layer].mean(dim=0)).to("cuda") -# r2 = (layer_results3[layer].mean(dim=0) - layer_results2[layer].mean(dim=0)).to(r1.device) -# cos_sim = F.cosine_similarity(r1.unsqueeze(0), r2.unsqueeze(0)) -# avg_cos_sim = cos_sim -# cs.append(avg_cos_sim.item()) - -# from transformers import AutoModelForCausalLM, AutoTokenizer -# tokenizer = AutoTokenizer.from_pretrained(model_name) -# model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.bfloat16) -# wait_word = ["wait", "Wait", " wait", " Wait"] -# wait_list = tokenizer(wait_word, return_tensors="pt")["input_ids"][:, -1] - -# cs2 = [] -# r1 = model.lm_head.weight[wait_list] -# for layer in range(29): -# r2 = (layer_results[layer].mean(dim=0) - layer_results4[layer].mean(dim=0)).to("cuda") -# cos_sim = F.cosine_similarity(r1, r2.unsqueeze(0)) -# cs2.append(cos_sim.tolist()) -# cs3 = [] -# r1 = model.lm_head.weight[wait_list] -# for layer in range(29): -# r2 = (layer_results3[layer].mean(dim=0) - layer_results2[layer].mean(dim=0)).to("cuda") -# cos_sim = F.cosine_similarity(r1, r2.unsqueeze(0)) -# cs3.append(cos_sim.tolist()) -# import matplotlib.pyplot as plt - -# fig, (ax1, ax2) = plt.subplots( -# 2, 1, sharex=True, figsize=(10, 6), -# gridspec_kw={'height_ratios': [1, 1]} -# ) - -# # 设置断轴的区间 -# upper_ylim = (0.9, 1.0) -# lower_ylim = (-0.2, 0.4) - -# # Calculate means and variances -# mean_c s = [sum(layer) / len(layer) for layer in cs2] -# mean_cs3 = [sum(layer) / len(layer) for layer in cs3] -# x = range(len(cs)) - -# # 绘图 -# for ax in [ax1, ax2]: -# ax.plot(x, mean_cs2, marker='o', label="Math dataset & Wait token") -# ax.plot(x, mean_cs3, marker='x', label="QA dataset & Wait token") -# ax.plot(x, cs, marker='o', label="GPQA & Math dataset", linestyle='--') -# ax.fill_between(x, [min(layer) for layer in cs2], [max(layer) for layer in cs2], alpha=0.2) -# ax.fill_between(x, [min(layer) for layer in cs3], [max(layer) for layer in cs3], alpha=0.2) -# ax.grid(True) - -# # 设置断轴 -# ax1.set_ylim(upper_ylim) -# ax2.set_ylim(lower_ylim) - -# ax1.spines['bottom'].set_visible(False) -# ax2.spines['top'].set_visible(False) -# ax1.tick_params(labeltop=False) -# ax2.xaxis.tick_bottom() - -# # 添加断轴标记 -# d = .015 -# kwargs = dict(transform=ax1.transAxes, color='k', clip_on=False) -# ax1.plot((-d, +d), (-d, +d), **kwargs) # Top-left -# ax1.plot((1 - d, 1 + d), (-d, +d), **kwargs) # Top-right -# kwargs.update(transform=ax2.transAxes) -# ax2.plot((-d, +d), (1 - d, 1 + d), **kwargs) # Bottom-left -# ax2.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs) # Bottom-right - -# # 添加统一标签 -# fig.text(0.04, 0.5, "Cosine Similarity", va='center', rotation='vertical', fontsize=20) -# ax2.set_xlabel("Layer", fontsize=20) - -# # 图例放上面子图右上 -# ax1.legend(fontsize=20, loc="lower center", bbox_to_anchor=(0.5, -0.3),) -# # Adjust the size of x-axis and y-axis labels -# ax1.tick_params(axis='x', labelsize=20) -# ax1.tick_params(axis='y', labelsize=20) -# ax2.tick_params(axis='x', labelsize=20) -# ax2.tick_params(axis='y', labelsize=20) -# # 美化保存 -# plt.tight_layout(rect=[0.05, 0.03, 1, 0.98]) # 留出左侧空间给 Y 标签 -# plt.savefig("mean_cosine_similarity_plot_broken_axis.pdf") - - -# plt.show() -# ############################################################################################### - -# ############################################################################################### - -# from utils import * -# def plot_umap_embeddings_for_layer_28( -# asset_path, qwen_asset_path, n_neighbors=10, min_dist=0.1, random_state=42, figsize=(10, 7) -# ): -# """ -# Plots UMAP embeddings for layer 28 for both DeepSeek and Qwen models. - -# Parameters: -# asset_path (str): Path to the asset directory for DeepSeek. -# qwen_asset_path (str): Path to the asset directory for Qwen. -# n_neighbors (int): Number of neighbors for UMAP. -# min_dist (float): Minimum distance for UMAP. -# random_state (int): Random state for reproducibility. -# figsize (tuple): Size of the figure. -# """ - -# fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(figsize[0] * 2, figsize[1])) -# colors = { -# "reflection inducing token": "red", -# "non-reflection inducing token": "orange", -# } - -# layer = 28 - -# # Plot for DeepSeek model -# token_before_wait = torch.load( -# f"{asset_path}/before_wait_DeepSeek-R1-Distill-Qwen-1.5B_{layer}_-1.pt" -# ) -# token_wo_wait = torch.load( -# f"{asset_path}/before_wo_wait_DeepSeek-R1-Distill-Qwen-1.5B_{layer}_-1.pt" -# ) - -# reducer = umap.UMAP( -# n_neighbors=n_neighbors, min_dist=min_dist, random_state=random_state -# ) -# embeddings = reducer.fit_transform( -# np.concatenate( -# [ -# token_before_wait.cpu().to(torch.float32).numpy(), -# token_wo_wait.cpu().to(torch.float32).numpy(), -# ] -# ) -# ) - -# wait_labels = ["reflection inducing token"] * token_before_wait.shape[0] -# nonwait_labels = ["non-reflection inducing token"] * token_wo_wait.shape[0] -# all_labels = wait_labels + nonwait_labels - -# for label in set(all_labels): -# indices = [i for i, x in enumerate(all_labels) if x == label] -# ax1.scatter( -# embeddings[indices, 0], -# embeddings[indices, 1], -# label=label, -# alpha=0.5, -# s=10, -# color=colors[label], -# ) -# ax1.set_title(f"DeepSeek-R1-Distill-Qwen-1.5B - Layer {layer}", fontsize=18) -# ax1.set_xlabel("UMAP Dimension 1", fontsize=18) -# ax1.set_ylabel("UMAP Dimension 2", fontsize=18) -# ax1.legend(fontsize=18) -# ax1.grid(True) - -# # Plot for Qwen model -# token_before_wait_qwen = torch.load( -# f"{qwen_asset_path}/before_wait_Qwen2.5-1.5B_{layer}_-1.pt" -# ) -# token_wo_wait_qwen = torch.load( -# f"{qwen_asset_path}/before_wo_wait_Qwen2.5-1.5B_{layer}_-1.pt" -# ) - -# embeddings_qwen = reducer.fit_transform( -# np.concatenate( -# [ -# token_before_wait_qwen.cpu().to(torch.float32).numpy(), -# token_wo_wait_qwen.cpu().to(torch.float32).numpy(), -# ] -# ) -# ) - -# wait_labels_qwen = ["reflection inducing token"] * token_before_wait_qwen.shape[0] -# nonwait_labels_qwen = ["non-reflection inducing token"] * token_wo_wait_qwen.shape[0] -# all_labels_qwen = wait_labels_qwen + nonwait_labels_qwen - -# for label in set(all_labels_qwen): -# indices = [i for i, x in enumerate(all_labels_qwen) if x == label] -# ax2.scatter( -# embeddings_qwen[indices, 0], -# embeddings_qwen[indices, 1], -# label=label, -# alpha=0.5, -# s=10, -# color=colors[label], -# ) -# ax2.set_title(f"Qwen2.5 1.5B - Layer {layer}", fontsize=18) -# ax2.set_xlabel("UMAP Dimension 1", fontsize=18) -# ax2.set_ylabel("UMAP Dimension 2", fontsize=18) -# ax2.legend(fontsize=18) -# ax2.grid(True) -# plt.savefig("umap_embeddings_layer_28.pdf", format="pdf", dpi=600) -# plt.tight_layout() -# plt.show() - -# # Call the function for layer 28 -# plot_umap_embeddings_for_layer_28( -# asset_path="../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/DeepSeek-R1-Distill-Qwen-1.5B_hs", -# qwen_asset_path="../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/Qwen2.5-1.5B_hs" -# ) - -# ############################################################################################### - -# ############################################################################################### - -# from utils import * -# def plot_umap_embeddings( -# layers, asset_path, n_neighbors=10, min_dist=0.1, random_state=42, figsize=(20, 30) -# ): -# """ -# Plots UMAP embeddings for given layers. - -# Parameters: -# layers (int): Number of layers to process. -# asset_path (str): Path to the asset directory. -# n_neighbors (int): Number of neighbors for UMAP. -# min_dist (float): Minimum distance for UMAP. -# random_state (int): Random state for reproducibility. -# figsize (tuple): Size of the figure. -# """ - -# fig, axes = plt.subplots(7, 2, figsize=figsize) -# axes = axes.flatten() -# colors = { -# "reflection inducing token": "red", -# "non reflection inducing token": "orange", -# } - -# for layer in tqdm(range(14)): -# token_before_wait = torch.load( -# f"{asset_path}/before_wait_DeepSeek-R1-Distill-Qwen-1.5B_{layer*2+2}_-1.pt" -# ) -# token_wo_wait = torch.load( -# f"{asset_path}/before_wo_wait_DeepSeek-R1-Distill-Qwen-1.5B_{layer*2+2}_-1.pt" -# ) - -# reducer = umap.UMAP( -# n_neighbors=n_neighbors, min_dist=min_dist, random_state=random_state -# ) -# embeddings = reducer.fit_transform( -# np.concatenate( -# [ -# token_before_wait.cpu().to(torch.float32).numpy(), -# token_wo_wait.cpu().to(torch.float32).numpy(), -# ] -# ) -# ) - -# wait_labels = ["reflection inducing token"] * token_before_wait.shape[0] -# nonwait_labels = ["non reflection inducing token"] * token_wo_wait.shape[0] -# all_labels = wait_labels + nonwait_labels -# for label in set(all_labels): -# indices = [i for i, x in enumerate(all_labels) if x == label] -# axes[layer].scatter( -# embeddings[indices, 0], -# embeddings[indices, 1], -# # label=label, -# alpha=0.7, -# s=30, -# color=colors[label], -# ) -# axes[layer].tick_params(axis='both', which='major', labelsize=20) -# axes[layer].set_title(f"DeepSeek-R1 1.5B Layer {layer*2+2}", fontsize=25, pad=25) -# axes[layer].legend(fontsize=25, loc='upper right') -# axes[layer].grid(True) -# handles = [ -# plt.Line2D([0], [0], marker='o', color='w', label='reflection inducing token', -# markerfacecolor='red', markersize=10), -# plt.Line2D([0], [0], marker='o', color='w', label='non reflection inducing token', -# markerfacecolor='orange', markersize=10) -# ] -# fig.legend( -# handles=handles, -# loc='upper center', -# bbox_to_anchor=(0.5, 1.02), -# ncol=2, -# fontsize=24, -# frameon=False -# ) -# plt.tight_layout() -# plt.savefig("umap_embeddings_ds.pdf", format="pdf", dpi=600, bbox_inches='tight') -# plt.show() - -# fig = plot_umap_embeddings( -# asset_path="../asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140/DeepSeek-R1-Distill-Qwen-1.5B_hs", -# layers=29, -# ) - -# ############################################################################################### -############################################################################################### -# # Prepare data for the Donut Chart -# wait_count = keyword_counts['wait'] -# misc_count = sum(counts) - wait_count - -# # Updated data -# counts = [wait_count, misc_count] -# keywords = ['wait', 'misc'] -# plt.rcParams.update({'font.size': 18}) # Increase font size for better readability - -# # Plotting the Donut Chart -# plt.figure(figsize=(8, 4)) -# plt.pie( -# counts, -# labels=keywords, -# autopct='%1.1f%%', -# startangle=140, -# colors=plt.cm.Paired.colors, -# wedgeprops={'width': 0.4} -# ) -# plt.title('Keyword Distribution', fontsize=18) -# plt.tight_layout() - -# # Save the plot as a PDF -# plt.savefig("keyword_distribution_donut_chart.pdf", format="pdf", dpi=300) - -# # Show the plot -# plt.show() -############################################################################################### - -def run_test_prompt_boxed(): - for model_name in [ - "Qwen/Qwen2.5-1.5B", - # "Qwen/Qwen2.5-7B", - # "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - # "meta-llama/Llama-3.1-8B", - ]: - if model_name == "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B": - test_prompt_boxed( - prompt="""Please reason step by step, and put your final answer within \\boxed{{}}.<|im_start|>user: {problems}<|im_end|>\n<|im_start|>assistant: """, - model_name=model_name, - dataset_name="HuggingFaceH4/MATH-500", - save_respnses=True, - ) - else: - test_prompt_boxed( - prompt="""Please reason step by step, and put your final answer within \\boxed{{}}.<|im_start|>user: {problems}<|im_end|>\n<|im_start|>assistant:""", - model_name=model_name, - dataset_name="HuggingFaceH4/MATH-500", - save_respnses=True, - ) - - -def parse_sampled_answer(answer): - patterns = [ - r"answer is \((.)\)", - r"Answer: \((.)\)", - r"answer: \((.)\)", - r"answer \((.)\)", - r"\((.)\)", - ] - for pattern in patterns: - match = re.search(pattern, answer) - if match and match.group(1): - return match.group(1) - return None - - -def average_response_length( - response_dir: str, - model_name: str, - keywords_list: List[str] = [ - "wait", - "re-check", - "recheck", - "rethink", - "re-think", - "reconsider", - "re-consider", - "re-evaluat", - "reevaluat", - "rethink", - "re-think", - "re-examine", - "reexamine", - "check again", - "try again", - "think again", - "consider again", - "evaluate again", - "examine again", - ], -) -> float: - """ - Calculate the average response length. - - Args: - response_path (str): Path to the response files - model_name (str): Name of the model to load - - Returns: - float: Average response length - """ - total_length_with_wait = 0 - total_length_without_wait = 0 - total_length = 0 - tokenizer = AutoTokenizer.from_pretrained(model_name) - wait_numbers = 0 - problem_numbers = 0 - for idx, path in enumerate(tqdm(os.listdir(response_dir))): - if not path.endswith("0.json"): - continue - with open(os.path.join(response_dir, path), "r") as f: - response = json.load(f) - assistant_response = response["response"].split("<|im_start|>assistant:")[-1] - input_ids = tokenizer(assistant_response, return_tensors="pt")["input_ids"].to( - "cuda:0" - ) - - if any(keyword in assistant_response.lower() for keyword in keywords_list): - total_length_with_wait += input_ids.shape[1] - wait_numbers += 1 - else: - total_length_without_wait += input_ids.shape[1] - total_length += input_ids.shape[1] - problem_numbers += 1 - if wait_numbers == 0: - return ( - total_length / problem_numbers, - 0, - total_length_without_wait / (problem_numbers - wait_numbers), - wait_numbers, - problem_numbers, - ) - elif problem_numbers == wait_numbers: - return ( - total_length / problem_numbers, - total_length_with_wait / wait_numbers, - 0, - wait_numbers, - problem_numbers, - ) - else: - return ( - total_length / problem_numbers, - total_length_with_wait / wait_numbers, - total_length_without_wait / (problem_numbers - wait_numbers), - wait_numbers, - problem_numbers, - ) - - -def run_average_response_length(): - response_dirs = [ - "./asset/response/Qwen2.5-1.5B_MATH-500_133", - "./asset/response/Qwen2.5-7B_MATH-500_133", - "./asset/response/DeepSeek-R1-Distill-Qwen-7B_MATH-500_141", - "./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140", - ] - for response_dir in response_dirs: - model_name = response_dir.split("/")[-1] - if "DeepSeek" in model_name: - model_name = f"deepseek-ai/{model_name.split('_')[0]}" - else: - model_name = f"Qwen/{model_name.split('_')[0]}" - tmp = average_response_length( - response_dir=response_dir, - model_name=model_name, - ) - print(tmp) - - -def wait_number( - response_dir: str, - model_name: str, - keywords_list: List[str] = [ - "wait", - "re-check", - "recheck", - "rethink", - "re-think", - "reconsider", - "re-consider", - "re-evaluat", - "reevaluat", - "rethink", - "re-think", - "re-examine", - "reexamine", - "check again", - "try again", - "think again", - "consider again", - "evaluate again", - "examine again", - ], -) -> list[int]: - """ - Determine whether the keyword 'wait' appears in the process or after it. - - Args: - response_dir (str): Path to the response files - model_name (str): Name of the model to load - - Returns: - list[int]: Positions of the keyword 'wait' in the responses - """ - wait_number = [] - tokenizer = AutoTokenizer.from_pretrained(model_name) - for idx, path in enumerate(tqdm(os.listdir(response_dir))): - if not path.endswith(".json"): - continue - with open(os.path.join(response_dir, path), "r") as f: - response = json.load(f) - assistant_response = response["response"].split("<|im_start|>assistant:")[-1] - input_ids = tokenizer(assistant_response, return_tensors="pt")["input_ids"].to( - "cuda:0" - ) - word_count = 0 - for word in keywords_list: - word_count += assistant_response.lower().count(word) - wait_number.append( - [ - word_count, - input_ids.shape[1], - ] - ) - return wait_number - - -def run_wait_number(): - response_dirs = [ - # "./asset/response/Qwen2.5-1.5B_MATH-500_133", - # "./asset/response/Qwen2.5-7B_MATH-500_133", - "./asset/response/DeepSeek-R1-Distill-Qwen-7B_MATH-500_141", - "./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140", - ] - for response_dir in response_dirs: - model_name = response_dir.split("/")[-1] - if "DeepSeek" in model_name: - model_name = f"deepseek-ai/{model_name.split('_')[0]}" - else: - model_name = f"Qwen/{model_name.split('_')[0]}" - tmp = wait_number( - response_dir=response_dir, - model_name=model_name, - ) - print(tmp) - - -def plot_wait_number_vs_response_length(response_dir: str, model_name: str): - wait_data = wait_number(response_dir, model_name) - wait_counts = [item[0] for item in wait_data] - response_lengths = [item[1] for item in wait_data] - - plt.figure(figsize=(10, 6)) - plt.scatter(response_lengths, wait_counts, alpha=0.6, edgecolors="k") - plt.title("Relationship Between Wait Number and Response Length") - plt.xlabel("Response Length") - plt.ylabel("Wait Number") - plt.grid(True) - plt.show() - - -def compute_correlation(wait_data): - """ - Compute the Pearson correlation coefficient between response lengths and wait counts. - - Args: - wait_data (list[list[int]]): A list of [wait_count, response_length] pairs. - - Returns: - tuple: Pearson correlation coefficient and p-value. - """ - wait_counts = [item[0] for item in wait_data] - response_lengths = [item[1] for item in wait_data] - - # Calculate Pearson correlation coefficient - correlation, p_value = pearsonr(response_lengths, wait_counts) - - return correlation, p_value - - -def extract_keywords( - keywords_list: List[str] = [ - "wait", - "re-check", - "recheck", - "rethink", - "re-think", - "reconsider", - "re-consider", - "re-evaluat", - "reevaluat", - "rethink", - "re-think", - "re-examine", - "reexamine", - "check again", - "try again", - "think again", - "consider again", - "evaluate again", - "examine again", - ], - response_dir: str = "./reflect_responses", -): - """ - Extracts keywords from responses stored in JSON files within the specified directory. - - Args: - keywords_list (List[str], optional): A list of keywords to search for in the responses. - Defaults to [ - "wait", "re-check", "recheck", "rethink", "re-think", "reconsider", - "re-consider", "re-evaluat", "reevaluat", "rethink", "re-think", - "re-examine", "reexamine", "check again", "try again", "think again", - "consider again", "evaluate again", "examine again", - ]. - response_dir (str, optional): The directory containing JSON response files. - Defaults to "./reflect_responses". - - Returns: - dict: A dictionary where keys are keywords and values are their respective counts - in the responses. - """ - # most keywords only appear in responses containing the word "wait" - # Moreover, we observe that the majority of these instances involve the word "wait" preceding other keywords. - # Furthermore, nearly all identified keywords co-occur with the word "wait" within the same sentence. - keywords = [] - for idx, response_file in enumerate(tqdm(os.listdir(response_dir))): - if not response_file.endswith(".json"): - continue - with open(os.path.join(response_dir, response_file), "r") as f: - response = json.load(f)["response"] - sentences = re.split(r"(?<=[.!?:])\s+", response) - for idy, sentence in enumerate(sentences): - for keyword in keywords_list: - if keyword in sentence.lower(): - keywords.append(keyword) - - return (dict(Counter(keywords)),) - - -def run_extract_keywords(): - - response_dirs = [ - "asset/response/Qwen2.5-1.5B_MATH-500_133", - "asset/response/Qwen2.5-7B_MATH-500_133", - "asset/response/DeepSeek-R1-Distill-Qwen-7B_MATH-500_141", - "asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140", - ] - for response_dir in response_dirs: - keywords_count = extract_keywords(response_dir=response_dir) - print(keywords_count) - - -def preprocess_box_response_for_qwen_prompt(sequence, answer): - model_output = re.sub( - r"^.*?<\|im_start\|>assistant", - "<|im_start|>assistant", - sequence, - flags=re.DOTALL, - count=1, - ) - stop_words = ["", "<|im_end|>", "<|endoftext|>"] - for stop_word in stop_words: - if stop_word in model_output: - model_output = model_output.split(stop_word)[0].strip() - if "boxed" in model_output: - boxed = 1 - else: - boxed = 0 - # grader - _, box_match = boxed_reward_fn(model_response=model_output, gt_answer=answer) - - return "", box_match, boxed - - -def grader_w_keywords(response_dir: str): - dataset = datasets.load_dataset("HuggingFaceH4/MATH-500")["test"] - score = 0 - boxed = 0 - for idx, example in enumerate(tqdm(dataset)): - with open(os.path.join(response_dir, f"{idx}.json"), "r") as f: - response = json.load(f)["response"] - _, box_match, box = preprocess_box_response_for_qwen_prompt( - response, example["answer"] - ) - score += box_match - boxed += box - return score / boxed - - -def plot_thinking_time_vs_accuracy(): - # Sample data - replace with your actual data - thinking_times = { - "DeepSeek-1.5B": [7220.81, 16211.72, 24777.92], - "Qwen-1.5B": [1740.9, 2908.27, 7409.45], - } - - accuracies = {"DeepSeek-1.5B": [83.2, 84.7, 85.6], "Qwen-1.5B": [17, 26, 39]} - - # Create figure with broken y-axis - fig = plt.figure(figsize=(10, 8)) - gs = GridSpec(2, 1, height_ratios=[1, 1], hspace=0.05) - - ax1 = fig.add_subplot(gs[0]) - ax2 = fig.add_subplot(gs[1], sharex=ax1) - - # Plot data in both subplots - ax1.plot( - thinking_times["DeepSeek-1.5B"], - accuracies["DeepSeek-1.5B"], - marker="o", - linestyle="-", - label="DeepSeek-1.5B", - ) - ax1.plot( - thinking_times["Qwen-1.5B"], - accuracies["Qwen-1.5B"], - marker="s", - linestyle="-", - label="Qwen-1.5B", - ) - - ax2.plot( - thinking_times["DeepSeek-1.5B"], - accuracies["DeepSeek-1.5B"], - marker="o", - linestyle="-", - label="DeepSeek-1.5B", - ) - ax2.plot( - thinking_times["Qwen-1.5B"], - accuracies["Qwen-1.5B"], - marker="s", - linestyle="-", - label="Qwen-1.5B", - ) - - # Set y-axis limits for the desired ranges - ax1.set_ylim(80, 90) - ax2.set_ylim(10, 40) - - # Hide the spines between ax1 and ax2 - ax1.spines["bottom"].set_visible(False) - ax2.spines["top"].set_visible(False) - ax1.tick_params(labelbottom=False) - ax2.xaxis.tick_bottom() - - # Add diagonal lines to indicate broken y-axis - d = 0.01 - kwargs = dict(transform=ax1.transAxes, color="k", clip_on=False) - ax1.plot((-d, +d), (-d, +d), **kwargs) - ax1.plot((1 - d, 1 + d), (-d, +d), **kwargs) - - kwargs.update(transform=ax2.transAxes) - ax2.plot((-d, +d), (1 - d, 1 + d), **kwargs) - ax2.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs) - - # Add labels and legend - ax2.set_xlabel("Average Thinking Time (s)") - fig.text(0.04, 0.5, "Accuracy", va="center", rotation="vertical") - fig.suptitle("Accuracy vs. Average Thinking Time", fontsize=14) - ax1.legend() - - # Add grid - ax1.grid(True, linestyle="--", alpha=0.7) - ax2.grid(True, linestyle="--", alpha=0.7) - - plt.tight_layout() - fig.subplots_adjust(top=0.95) - - return fig - - -def run_grader_w_keywords(): - response_dirs = [ - # "./asset/response/Qwen2.5-1.5B_MATH-500_133", - # "./asset/response/Qwen2.5-7B_MATH-500_133", - # "./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_458", - # "./asset/response/DeepSeek-R1-Distill-Qwen-7B_MATH-500_141", - "./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140", - ] - dataset = datasets.load_dataset("HuggingFaceH4/MATH-500")["test"] - for response_dir in response_dirs: - model_name = response_dir.split("/")[-1] - if "DeepSeek" in model_name: - model_name = f"deepseek-ai/{model_name.split('_')[0]}" - else: - model_name = f"Qwen/{model_name.split('_')[0]}" - wait_stat = wait_number( - response_dir=response_dir, - model_name=model_name, - ) - score_stat = [] - for idx, example in enumerate(tqdm(dataset)): - with open(os.path.join(response_dir, f"{idx}.json"), "r") as f: - response = json.load(f)["response"] - _, box_match, box = preprocess_box_response_for_qwen_prompt( - response, example["answer"] - ) - score_stat.append([wait_stat[idx][0], wait_stat[idx][1], box_match, box]) - score = 0 - boxed = 0 - for itm in score_stat: - if itm[2] == 1: - score += 1 - - if itm[3] == 1: - boxed += 1 - print(f"score: {score}, boxed: {boxed}, total: {score/boxed}") - - return score_stat - - -def vis_err_wait(score_stat): - # [(wait_number, ..., score), ...] - wait_numbers = [item[0] for item in score_stat] - scores = [item[2] for item in score_stat] - - # 自定义分组边界和标签 - bin_labels = ["0", "1-2", "3-4", "5-6", "7-8", "9-10", "11-20", ">20"] - - def get_bin_label(w): - if w == 0: - return "0" - elif 1 <= w <= 2: - return "1-2" - elif 3 <= w <= 4: - return "3-4" - elif 5 <= w <= 6: - return "5-6" - elif 7 <= w <= 8: - return "7-8" - elif 9 <= w <= 10: - return "9-10" - elif 11 <= w <= 20: - return "11-20" - else: - return ">20" - - # 初始化分组统计 - bin_stats = defaultdict(lambda: [0, 0]) # [correct_count, incorrect_count] - - # 分组统计 - for w, s in zip(wait_numbers, scores): - label = get_bin_label(w) - if s == 1: - bin_stats[label][0] += 1 - else: - bin_stats[label][1] += 1 - - # 保持顺序 - correct_counts = [bin_stats[label][0] for label in bin_labels] - incorrect_counts = [bin_stats[label][1] for label in bin_labels] - - # 计算准确率 - accuracies = [] - for c, i in zip(correct_counts, incorrect_counts): - total = c + i - if total == 0: - accuracies.append(None) - else: - accuracies.append(c / total) - - # 绘图 - x = np.arange(len(bin_labels)) - width = 0.6 - - plt.figure(figsize=(12, 6)) - bars1 = plt.bar( - x, correct_counts, width, label="Correct (score=1)", color="skyblue" - ) - bars2 = plt.bar( - x, - incorrect_counts, - width, - bottom=correct_counts, - label="Incorrect (score=0)", - color="salmon", - ) - - # 添加准确率标签 - for i, (x_pos, acc, c, i_count) in enumerate( - zip(x, accuracies, correct_counts, incorrect_counts) - ): - total = c + i_count - if acc is not None: - y_pos = c + i_count + 1 # 稍微高于顶部 - plt.text( - x_pos, - y_pos, - f"{acc:.1%}", - ha="center", - va="bottom", - fontsize=10, - fontweight="bold", - ) - - plt.xticks(x, bin_labels) - plt.xlabel("Wait Number Range") - plt.ylabel("Count") - plt.title("Correct and Incorrect Counts per Wait Number Group (with Accuracy)") - plt.legend() - plt.grid(axis="y") - plt.tight_layout() - plt.show() - - -def vis_err_diff(score_stat): - # [(wait_number, ..., score), ...] - wait_numbers = [item[4] for item in score_stat] - scores = [item[2] for item in score_stat] - - # 自定义分组边界和标签 - bin_labels = ["1", "2", "3", "4", "5"] - - def get_bin_label(w): - if w == 1: - return "1" - elif w == 2: - return "2" - elif w == 3: - return "3" - elif w == 4: - return "4" - elif w == 5: - return "5" - - # 初始化分组统计 - bin_stats = defaultdict(lambda: [0, 0]) # [correct_count, incorrect_count] - - # 分组统计 - for w, s in zip(wait_numbers, scores): - label = get_bin_label(w) - if s == 1: - bin_stats[label][0] += 1 - else: - bin_stats[label][1] += 1 - - # 保持顺序 - correct_counts = [bin_stats[label][0] for label in bin_labels] - incorrect_counts = [bin_stats[label][1] for label in bin_labels] - - # 计算准确率 - accuracies = [] - for c, i in zip(correct_counts, incorrect_counts): - total = c + i - if total == 0: - accuracies.append(None) - else: - accuracies.append(c / total) - - # 绘图 - x = np.arange(len(bin_labels)) - width = 0.6 - - plt.figure(figsize=(12, 6)) - bars1 = plt.bar( - x, correct_counts, width, label="Correct (score=1)", color="skyblue" - ) - bars2 = plt.bar( - x, - incorrect_counts, - width, - bottom=correct_counts, - label="Incorrect (score=0)", - color="salmon", - ) - - # 添加准确率标签 - for i, (x_pos, acc, c, i_count) in enumerate( - zip(x, accuracies, correct_counts, incorrect_counts) - ): - total = c + i_count - if acc is not None: - y_pos = c + i_count + 1 # 稍微高于顶部 - plt.text( - x_pos, - y_pos, - f"{acc:.1%}", - ha="center", - va="bottom", - fontsize=10, - fontweight="bold", - ) - - plt.xticks(x, bin_labels) - plt.xlabel("Wait Number Range") - plt.ylabel("Count") - plt.title("Correct and Incorrect Counts per Wait Number Group (with Accuracy)") - plt.legend() - plt.grid(axis="y") - plt.tight_layout() - plt.show() - - -if __name__ == "__main__": - # run_extract_keywords() - # run_test_prompt_boxed() - # run_average_response_length() - # run_wait_number() - # run_grader_w_keywords() - for model_name in [ - # "Qwen/Qwen2.5-1.5B", - # "Qwen/Qwen2.5-7B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - # "meta-llama/Llama-3.1-8B", - ]: - for layer in tqdm(range(29)): - extract_key_token( - response_dir="./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140", - model_name=model_name, - token_position=-1, - layer=layer, - ) - - # layers = 29 - # for model_name in [ - # # "Qwen/Qwen2.5-1.5B", - # # "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - # "meta-llama/Llama-3.1-8B", - # ]: - # if model_name == "meta-llama/Llama-3.1-8B": - # layers = 33 - # for layer in range(layers): - # for position in [-2, -1]: - # extract_key_token( - # response_dir="./asset/response/DeepSeek-R1-Distill-Qwen-1.5B_MATH-500_140", - # model_name=model_name, - # token_position=position, - # layer=layer, - # ) \ No newline at end of file diff --git a/models/placeholder b/tests/__init__.py similarity index 100% rename from models/placeholder rename to tests/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..83a5a17 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,30 @@ +"""Shared test fixtures for ProbingReflection tests.""" + +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def test_data_dir() -> Path: + """Path to test data fixtures directory.""" + return Path(__file__).parent / "fixtures" + + +@pytest.fixture(scope="session") +def sample_contrastive_pairs() -> list[dict[str, str]]: + """Sample contrastive pairs for testing.""" + return [ + { + "positive": "Let me think about this step by step.", + "negative": "I'll just give you the answer.", + }, + { + "positive": "I should reconsider my approach here.", + "negative": "This is definitely correct.", + }, + { + "positive": "Wait, I might be wrong about this.", + "negative": "Trust me, I know what I'm doing.", + }, + ] diff --git a/tests/test_types.py b/tests/test_types.py new file mode 100644 index 0000000..9f3a8c1 --- /dev/null +++ b/tests/test_types.py @@ -0,0 +1,105 @@ +"""Tests for core type definitions. + +These tests verify the foundational types used throughout +the ProbingReflection research project. +""" + +import pytest + +from probing_reflection.types import ( + ContrastivePair, + ProbingConfig, + ReflectionResult, +) + + +class TestProbingConfig: + """Tests for ProbingConfig dataclass.""" + + def test_default_initialization(self) -> None: + """ProbingConfig should have sensible defaults.""" + config = ProbingConfig() + assert config.model_name == "" + assert config.layer_indices == () + + def test_custom_initialization(self) -> None: + """ProbingConfig should accept custom values.""" + config = ProbingConfig( + model_name="gpt2-small", + layer_indices=(0, 6, 11), + ) + assert config.model_name == "gpt2-small" + assert config.layer_indices == (0, 6, 11) + + def test_is_frozen(self) -> None: + """ProbingConfig should be immutable (frozen).""" + config = ProbingConfig(model_name="test") + with pytest.raises(AttributeError): + config.model_name = "changed" # type: ignore[misc] + + def test_is_hashable(self) -> None: + """ProbingConfig should be hashable (for use in sets/dicts).""" + config = ProbingConfig(model_name="test") + assert hash(config) is not None + # Can be used in a set + configs = {config, ProbingConfig(model_name="test")} + assert len(configs) == 1 # Same config hashes equal + + +class TestReflectionResult: + """Tests for ReflectionResult dataclass.""" + + def test_basic_creation(self) -> None: + """ReflectionResult should store analysis results.""" + result = ReflectionResult( + sample_id="sample-001", + reflection_score=0.85, + ) + assert result.sample_id == "sample-001" + assert result.reflection_score == 0.85 + + def test_with_metadata(self) -> None: + """ReflectionResult should accept optional metadata.""" + result = ReflectionResult( + sample_id="sample-002", + reflection_score=0.42, + metadata={"source": "experiment-1", "model": "gpt2"}, + ) + assert result.metadata["source"] == "experiment-1" + assert result.metadata["model"] == "gpt2" + + def test_default_metadata(self) -> None: + """ReflectionResult should default to empty metadata.""" + result = ReflectionResult( + sample_id="sample-003", + reflection_score=0.0, + ) + assert result.metadata == {} + + def test_score_bounds_validation(self) -> None: + """ReflectionResult should accept valid scores.""" + # These should not raise + ReflectionResult(sample_id="test", reflection_score=0.0) + ReflectionResult(sample_id="test", reflection_score=1.0) + ReflectionResult(sample_id="test", reflection_score=0.5) + + +class TestContrastivePair: + """Tests for ContrastivePair TypedDict.""" + + def test_structure(self) -> None: + """ContrastivePair should have positive/negative keys.""" + pair: ContrastivePair = { + "positive": "I love this product!", + "negative": "I hate this product!", + } + assert pair["positive"] == "I love this product!" + assert pair["negative"] == "I hate this product!" + + def test_used_in_list(self) -> None: + """ContrastivePair should work in collections.""" + pairs: list[ContrastivePair] = [ + {"positive": "good", "negative": "bad"}, + {"positive": "happy", "negative": "sad"}, + ] + assert len(pairs) == 2 diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..5dbcaaf --- /dev/null +++ b/uv.lock @@ -0,0 +1,278 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "librt" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", size = 177471, upload-time = "2026-02-17T16:13:06.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", size = 66516, upload-time = "2026-02-17T16:11:41.604Z" }, + { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", size = 68634, upload-time = "2026-02-17T16:11:43.268Z" }, + { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", size = 198941, upload-time = "2026-02-17T16:11:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", size = 209991, upload-time = "2026-02-17T16:11:45.462Z" }, + { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", size = 224476, upload-time = "2026-02-17T16:11:46.542Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", size = 217518, upload-time = "2026-02-17T16:11:47.746Z" }, + { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", size = 225116, upload-time = "2026-02-17T16:11:49.298Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", size = 217751, upload-time = "2026-02-17T16:11:50.49Z" }, + { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", size = 218378, upload-time = "2026-02-17T16:11:51.783Z" }, + { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", size = 241199, upload-time = "2026-02-17T16:11:53.561Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", size = 54917, upload-time = "2026-02-17T16:11:54.758Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", size = 62017, upload-time = "2026-02-17T16:11:55.748Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", size = 52441, upload-time = "2026-02-17T16:11:56.801Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", size = 66529, upload-time = "2026-02-17T16:11:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", size = 68669, upload-time = "2026-02-17T16:11:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", size = 199279, upload-time = "2026-02-17T16:11:59.862Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", size = 210288, upload-time = "2026-02-17T16:12:00.954Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", size = 224809, upload-time = "2026-02-17T16:12:02.108Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", size = 218075, upload-time = "2026-02-17T16:12:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", size = 225486, upload-time = "2026-02-17T16:12:04.725Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", size = 218219, upload-time = "2026-02-17T16:12:05.828Z" }, + { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", size = 218750, upload-time = "2026-02-17T16:12:07.09Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", size = 241624, upload-time = "2026-02-17T16:12:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", size = 54969, upload-time = "2026-02-17T16:12:09.633Z" }, + { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", size = 62000, upload-time = "2026-02-17T16:12:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", size = 52495, upload-time = "2026-02-17T16:12:11.633Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", size = 66081, upload-time = "2026-02-17T16:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", size = 68309, upload-time = "2026-02-17T16:12:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", size = 196804, upload-time = "2026-02-17T16:12:14.818Z" }, + { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", size = 206907, upload-time = "2026-02-17T16:12:16.513Z" }, + { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", size = 221217, upload-time = "2026-02-17T16:12:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", size = 214622, upload-time = "2026-02-17T16:12:19.108Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", size = 221987, upload-time = "2026-02-17T16:12:20.331Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", size = 215132, upload-time = "2026-02-17T16:12:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", size = 215195, upload-time = "2026-02-17T16:12:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", size = 237946, upload-time = "2026-02-17T16:12:24.275Z" }, + { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", size = 50689, upload-time = "2026-02-17T16:12:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", size = 57875, upload-time = "2026-02-17T16:12:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", size = 48058, upload-time = "2026-02-17T16:12:28.556Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", size = 68313, upload-time = "2026-02-17T16:12:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", size = 70994, upload-time = "2026-02-17T16:12:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", size = 220770, upload-time = "2026-02-17T16:12:33.294Z" }, + { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", size = 235409, upload-time = "2026-02-17T16:12:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", size = 246473, upload-time = "2026-02-17T16:12:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", size = 238866, upload-time = "2026-02-17T16:12:37.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", size = 250248, upload-time = "2026-02-17T16:12:39.445Z" }, + { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", size = 240629, upload-time = "2026-02-17T16:12:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", size = 239615, upload-time = "2026-02-17T16:12:42.446Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", size = 263001, upload-time = "2026-02-17T16:12:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", size = 51328, upload-time = "2026-02-17T16:12:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", size = 58722, upload-time = "2026-02-17T16:12:46.85Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, + { url = "https://files.pythonhosted.org/packages/de/9f/a6abae693f7a0c697dbb435aac52e958dc8da44e92e08ba88d2e42326176/mypy-1.19.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3157c7594ff2ef1634ee058aafc56a82db665c9438fd41b390f3bde1ab12250", size = 13201927, upload-time = "2025-12-15T05:02:29.138Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a4/45c35ccf6e1c65afc23a069f50e2c66f46bd3798cbe0d680c12d12935caa/mypy-1.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdb12f69bcc02700c2b47e070238f42cb87f18c0bc1fc4cdb4fb2bc5fd7a3b8b", size = 12206730, upload-time = "2025-12-15T05:03:01.325Z" }, + { url = "https://files.pythonhosted.org/packages/05/bb/cdcf89678e26b187650512620eec8368fded4cfd99cfcb431e4cdfd19dec/mypy-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f859fb09d9583a985be9a493d5cfc5515b56b08f7447759a0c5deaf68d80506e", size = 12724581, upload-time = "2025-12-15T05:03:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/d1/32/dd260d52babf67bad8e6770f8e1102021877ce0edea106e72df5626bb0ec/mypy-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9a6538e0415310aad77cb94004ca6482330fece18036b5f360b62c45814c4ef", size = 13616252, upload-time = "2025-12-15T05:02:49.036Z" }, + { url = "https://files.pythonhosted.org/packages/71/d0/5e60a9d2e3bd48432ae2b454b7ef2b62a960ab51292b1eda2a95edd78198/mypy-1.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da4869fc5e7f62a88f3fe0b5c919d1d9f7ea3cef92d3689de2823fd27e40aa75", size = 13840848, upload-time = "2025-12-15T05:02:55.95Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/d32051fa65ecf6cc8c6610956473abdc9b4c43301107476ac03559507843/mypy-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:016f2246209095e8eda7538944daa1d60e1e8134d98983b9fc1e92c1fc0cb8dd", size = 10135510, upload-time = "2025-12-15T05:02:58.438Z" }, + { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, + { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "probing-reflection" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "pyyaml" }, + { name = "ruff" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.13.0" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "ruff", specifier = ">=0.8.0" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/9b/840e0039e65fcf12758adf684d2289024d6140cde9268cc59887dc55189c/ruff-0.15.5.tar.gz", hash = "sha256:7c3601d3b6d76dce18c5c824fc8d06f4eef33d6df0c21ec7799510cde0f159a2", size = 4574214, upload-time = "2026-03-05T20:06:34.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/20/5369c3ce21588c708bcbe517a8fbe1a8dfdb5dfd5137e14790b1da71612c/ruff-0.15.5-py3-none-linux_armv6l.whl", hash = "sha256:4ae44c42281f42e3b06b988e442d344a5b9b72450ff3c892e30d11b29a96a57c", size = 10478185, upload-time = "2026-03-05T20:06:29.093Z" }, + { url = "https://files.pythonhosted.org/packages/44/ed/e81dd668547da281e5dce710cf0bc60193f8d3d43833e8241d006720e42b/ruff-0.15.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6edd3792d408ebcf61adabc01822da687579a1a023f297618ac27a5b51ef0080", size = 10859201, upload-time = "2026-03-05T20:06:32.632Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8f/533075f00aaf19b07c5cd6aa6e5d89424b06b3b3f4583bfa9c640a079059/ruff-0.15.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:89f463f7c8205a9f8dea9d658d59eff49db05f88f89cc3047fb1a02d9f344010", size = 10184752, upload-time = "2026-03-05T20:06:40.312Z" }, + { url = "https://files.pythonhosted.org/packages/66/0e/ba49e2c3fa0395b3152bad634c7432f7edfc509c133b8f4529053ff024fb/ruff-0.15.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba786a8295c6574c1116704cf0b9e6563de3432ac888d8f83685654fe528fd65", size = 10534857, upload-time = "2026-03-05T20:06:19.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/71/39234440f27a226475a0659561adb0d784b4d247dfe7f43ffc12dd02e288/ruff-0.15.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd4b801e57955fe9f02b31d20375ab3a5c4415f2e5105b79fb94cf2642c91440", size = 10309120, upload-time = "2026-03-05T20:06:00.435Z" }, + { url = "https://files.pythonhosted.org/packages/f5/87/4140aa86a93df032156982b726f4952aaec4a883bb98cb6ef73c347da253/ruff-0.15.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391f7c73388f3d8c11b794dbbc2959a5b5afe66642c142a6effa90b45f6f5204", size = 11047428, upload-time = "2026-03-05T20:05:51.867Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/4953e7e3287676f78fbe85e3a0ca414c5ca81237b7575bdadc00229ac240/ruff-0.15.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dc18f30302e379fe1e998548b0f5e9f4dff907f52f73ad6da419ea9c19d66c8", size = 11914251, upload-time = "2026-03-05T20:06:22.887Z" }, + { url = "https://files.pythonhosted.org/packages/77/46/0f7c865c10cf896ccf5a939c3e84e1cfaeed608ff5249584799a74d33835/ruff-0.15.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc6e7f90087e2d27f98dc34ed1b3ab7c8f0d273cc5431415454e22c0bd2a681", size = 11333801, upload-time = "2026-03-05T20:05:57.168Z" }, + { url = "https://files.pythonhosted.org/packages/d3/01/a10fe54b653061585e655f5286c2662ebddb68831ed3eaebfb0eb08c0a16/ruff-0.15.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1cb7169f53c1ddb06e71a9aebd7e98fc0fea936b39afb36d8e86d36ecc2636a", size = 11206821, upload-time = "2026-03-05T20:06:03.441Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/2132ceaf20c5e8699aa83da2706ecb5c5dcdf78b453f77edca7fb70f8a93/ruff-0.15.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9b037924500a31ee17389b5c8c4d88874cc6ea8e42f12e9c61a3d754ff72f1ca", size = 11133326, upload-time = "2026-03-05T20:06:25.655Z" }, + { url = "https://files.pythonhosted.org/packages/72/cb/2e5259a7eb2a0f87c08c0fe5bf5825a1e4b90883a52685524596bfc93072/ruff-0.15.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:65bb414e5b4eadd95a8c1e4804f6772bbe8995889f203a01f77ddf2d790929dd", size = 10510820, upload-time = "2026-03-05T20:06:37.79Z" }, + { url = "https://files.pythonhosted.org/packages/ff/20/b67ce78f9e6c59ffbdb5b4503d0090e749b5f2d31b599b554698a80d861c/ruff-0.15.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d20aa469ae3b57033519c559e9bc9cd9e782842e39be05b50e852c7c981fa01d", size = 10302395, upload-time = "2026-03-05T20:05:54.504Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e5/719f1acccd31b720d477751558ed74e9c88134adcc377e5e886af89d3072/ruff-0.15.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:15388dd28c9161cdb8eda68993533acc870aa4e646a0a277aa166de9ad5a8752", size = 10754069, upload-time = "2026-03-05T20:06:06.422Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/d1db14469e32d98f3ca27079dbd30b7b44dbb5317d06ab36718dee3baf03/ruff-0.15.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b30da330cbd03bed0c21420b6b953158f60c74c54c5f4c1dabbdf3a57bf355d2", size = 11304315, upload-time = "2026-03-05T20:06:10.867Z" }, + { url = "https://files.pythonhosted.org/packages/28/3a/950367aee7c69027f4f422059227b290ed780366b6aecee5de5039d50fa8/ruff-0.15.5-py3-none-win32.whl", hash = "sha256:732e5ee1f98ba5b3679029989a06ca39a950cced52143a0ea82a2102cb592b74", size = 10551676, upload-time = "2026-03-05T20:06:13.705Z" }, + { url = "https://files.pythonhosted.org/packages/b8/00/bf077a505b4e649bdd3c47ff8ec967735ce2544c8e4a43aba42ee9bf935d/ruff-0.15.5-py3-none-win_amd64.whl", hash = "sha256:821d41c5fa9e19117616c35eaa3f4b75046ec76c65e7ae20a333e9a8696bc7fe", size = 11678972, upload-time = "2026-03-05T20:06:45.379Z" }, + { url = "https://files.pythonhosted.org/packages/fe/4e/cd76eca6db6115604b7626668e891c9dd03330384082e33662fb0f113614/ruff-0.15.5-py3-none-win_arm64.whl", hash = "sha256:b498d1c60d2fe5c10c45ec3f698901065772730b411f164ae270bb6bfcc4740b", size = 10965572, upload-time = "2026-03-05T20:06:16.984Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] From 80c246a0270b0e65159a95ecaceef694380f2382 Mon Sep 17 00:00:00 2001 From: xzascc Date: Wed, 18 Mar 2026 23:14:49 -0400 Subject: [PATCH 2/6] chore: add development test scripts Add quick_test.sh and full_test.sh for running validation checks Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- scripts/full_test.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ scripts/quick_test.sh | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100755 scripts/full_test.sh create mode 100755 scripts/quick_test.sh diff --git a/scripts/full_test.sh b/scripts/full_test.sh new file mode 100755 index 0000000..9940abf --- /dev/null +++ b/scripts/full_test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Full test script - runs all models for complete validation +# Usage: ./scripts/full_test.sh + +set -e + +echo "=== Full Test (All Models) ===" +echo "" + +# All models to test (comma-separated) +export MODELS="${MODELS:-Qwen/Qwen2.5-0.5B,Qwen/Qwen2.5-1.5B,Qwen/Qwen2.5-3B,Qwen/Qwen2.5-7B}" + +echo "Models: $MODELS" +echo "" + +# Install dependencies +echo "[1/4] Installing dependencies..." +uv sync + +# Lint check +echo "[2/4] Running lint check..." +uv run ruff check src/ tests/ + +# Format check +echo "[3/4] Running format check..." +uv run ruff format --check src/ tests/ + +# Type check +echo "[4/4] Running type check..." +uv run mypy src/ + +# Run tests for each model +echo "[5/5] Running tests for all models..." +IFS=',' read -ra MODEL_ARRAY <<< "$MODELS" +for model in "${MODEL_ARRAY[@]}"; do + echo "" + echo "--- Testing with model: $model ---" + export MODEL_NAME="$model" + uv run pytest -v +done + +echo "" +echo "=== Full Test Complete ===" diff --git a/scripts/quick_test.sh b/scripts/quick_test.sh new file mode 100755 index 0000000..2a13cf0 --- /dev/null +++ b/scripts/quick_test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Quick validation script - uses small model for fast iteration +# Usage: ./scripts/quick_test.sh + +set -e + +echo "=== Quick Validation (Small Model) ===" +echo "" + +# Small model for quick testing +export MODEL_NAME="${MODEL_NAME:-Qwen/Qwen2.5-0.5B}" + +echo "Model: $MODEL_NAME" +echo "" + +# Install dependencies +echo "[1/4] Installing dependencies..." +uv sync + +# Lint check +echo "[2/4] Running lint check..." +uv run ruff check src/ tests/ + +# Format check +echo "[3/4] Running format check..." +uv run ruff format --check src/ tests/ + +# Type check +echo "[4/4] Running type check..." +uv run mypy src/ + +# Run tests +echo "[5/5] Running tests..." +uv run pytest -v + +echo "" +echo "=== Quick Validation Complete ===" From 391b00d4004913dabb19df9fd9f2bd294c502d1b Mon Sep 17 00:00:00 2001 From: xzascc Date: Wed, 18 Mar 2026 23:15:00 -0400 Subject: [PATCH 3/6] docs: add experiment draft for token probing Add draft notes for token probe experiment requirements and questions Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .sisyphus/drafts/token-probe-experiment.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .sisyphus/drafts/token-probe-experiment.md diff --git a/.sisyphus/drafts/token-probe-experiment.md b/.sisyphus/drafts/token-probe-experiment.md new file mode 100644 index 0000000..154d468 --- /dev/null +++ b/.sisyphus/drafts/token-probe-experiment.md @@ -0,0 +1,22 @@ +# Draft: Token Probe Experiment + +## Requirements (confirmed) +- Create a new git branch for experiments +- Two Python files for two tasks: + 1. Find top 50 pos/neg tokens per concept using discriminative method, decode and display + 2. Train linear probe on tokens closest to pos/neg centers, test separability across 10 layers + +## Open Questions +- **Model**: Which model to probe? +- **Concepts**: What specific concepts to analyze? +- **Data source**: Where does the token/concept data come from? +- **Discriminative method**: Which specific method? (mass-mean, contrast-cons, etc.) +- **Distance metric**: How to measure "closest to center"? (cosine, euclidean) +- **Token selection criteria**: Top-K based on what score? + +## Technical Decisions +- (pending user input) + +## Scope Boundaries +- INCLUDE: Two Python scripts, new branch +- EXCLUDE: (pending) From 27bd96f8fb46650d28094cfc8a6f210ab194ca70 Mon Sep 17 00:00:00 2001 From: xzAscC Date: Wed, 18 Mar 2026 23:21:27 -0400 Subject: [PATCH 4/6] Update TEMPLATE.md Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TEMPLATE.md b/TEMPLATE.md index 8290d4d..9267f49 100644 --- a/TEMPLATE.md +++ b/TEMPLATE.md @@ -35,10 +35,10 @@ Replace `ai_research_template` with your project name: ```bash # Update all references -find . -type f \( -name "*.py" -o -name "*.toml" \) -exec sed -i 's/ai_research_template/YOUR_PROJECT_NAME/g' {} + +find . -type f \( -name "*.py" -o -name "*.toml" \) -exec sed -i 's/probing_reflection/YOUR_PROJECT_NAME/g' {} + # Rename the source directory -mv src/ai_research_template src/YOUR_PROJECT_NAME +mv src/probing_reflection src/YOUR_PROJECT_NAME ``` ### 4. Verify Everything Works From 631bb6ab076e146aa5d6434dfec77e33d67d386f Mon Sep 17 00:00:00 2001 From: xzAscC Date: Wed, 18 Mar 2026 23:21:40 -0400 Subject: [PATCH 5/6] Update scripts/quick_test.sh Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- scripts/quick_test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/quick_test.sh b/scripts/quick_test.sh index 2a13cf0..9a8eb71 100755 --- a/scripts/quick_test.sh +++ b/scripts/quick_test.sh @@ -14,19 +14,19 @@ echo "Model: $MODEL_NAME" echo "" # Install dependencies -echo "[1/4] Installing dependencies..." +echo "[1/5] Installing dependencies..." uv sync # Lint check -echo "[2/4] Running lint check..." +echo "[2/5] Running lint check..." uv run ruff check src/ tests/ # Format check -echo "[3/4] Running format check..." +echo "[3/5] Running format check..." uv run ruff format --check src/ tests/ # Type check -echo "[4/4] Running type check..." +echo "[4/5] Running type check..." uv run mypy src/ # Run tests From 44579c34708b5d2e2c6d74115fec5f863b696ba2 Mon Sep 17 00:00:00 2001 From: xzascc Date: Wed, 18 Mar 2026 23:34:21 -0400 Subject: [PATCH 6/6] fix: address cubic review comments on PR #1 Fixes 8 issues identified by automated code review: - types.py: Use MappingProxyType for immutable metadata in frozen dataclass - AGENTS.md: Replace TypeScript terminology (unknown) with Python equivalents - AGENTS.md: Replace Throw with Raise, errors with exceptions - TEMPLATE.md: Correct package name from ai_research_template to probing_reflection - TEMPLATE.md: Fix file reference from AGENTS.md to pyproject.toml - SPEC.md: Move .sisyphus/ to top-level (not nested under docs/) - scripts/*.sh: Fix step counters from /4 to /5 --- AGENTS.md | 4 ++-- SPEC.md | 3 ++- TEMPLATE.md | 5 +++-- scripts/full_test.sh | 8 ++++---- src/probing_reflection/types.py | 3 ++- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 78f50b3..d3352ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,9 +53,9 @@ Before opening a PR, ALL of these must pass: - Use ESM-style imports (from x import y) - Group imports: stdlib → third-party → local - Use type hints on ALL function parameters and returns -- Never use `Any` — use `unknown` patterns or proper types +- Never use `Any` — use `object`, `Protocol`, or generic `TypeVar` patterns instead - Fail fast on invalid input -- Throw typed/domain-specific errors +- Raise typed/domain-specific exceptions - Preserve original error as `cause` when wrapping ## 5) When Writing Tests diff --git a/SPEC.md b/SPEC.md index 2674b02..8e8c708 100644 --- a/SPEC.md +++ b/SPEC.md @@ -20,9 +20,10 @@ ProbingReflection investigates self-reflection in Large Language Models through - `tests/` — Test files mirroring source structure - `docs/` — Documentation including: - `docs/design-docs/` — Design and experiment documentation + - `docs/QUALITY_SCORE.md` — Quality tracking metrics +- `.sisyphus/` — Work management including: - `.sisyphus/plans/` — Execution plans for tasks - `.sisyphus/roadmap.md` — Project roadmap - - `docs/QUALITY_SCORE.md` — Quality tracking metrics ## 4) Three-Tier Boundaries diff --git a/TEMPLATE.md b/TEMPLATE.md index 9267f49..8bb121d 100644 --- a/TEMPLATE.md +++ b/TEMPLATE.md @@ -31,7 +31,7 @@ uv sync ### 3. Rename the Package -Replace `ai_research_template` with your project name: +Replace `probing_reflection` with your project name: ```bash # Update all references @@ -68,7 +68,8 @@ uv run pytest | `pyproject.toml` | Project name, description, dependencies | | `README.md` | Project description, remove template-specific content | | `ARCHITECTURE.md` | Your system architecture | -| `AGENTS.md` | Package references (line 35: `packages = ["YOUR_PROJECT_NAME"]`) | +| `AGENTS.md` | Package references | +| `pyproject.toml` | packages = ["YOUR_PROJECT_NAME"] on line 35 | | `.sisyphus/roadmap.md` | Your project roadmap | | `.github/` | Update issue templates if needed | diff --git a/scripts/full_test.sh b/scripts/full_test.sh index 9940abf..5460d5a 100755 --- a/scripts/full_test.sh +++ b/scripts/full_test.sh @@ -14,19 +14,19 @@ echo "Models: $MODELS" echo "" # Install dependencies -echo "[1/4] Installing dependencies..." +echo "[1/5] Installing dependencies..." uv sync # Lint check -echo "[2/4] Running lint check..." +echo "[2/5] Running lint check..." uv run ruff check src/ tests/ # Format check -echo "[3/4] Running format check..." +echo "[3/5] Running format check..." uv run ruff format --check src/ tests/ # Type check -echo "[4/4] Running type check..." +echo "[4/5] Running type check..." uv run mypy src/ # Run tests for each model diff --git a/src/probing_reflection/types.py b/src/probing_reflection/types.py index 68e3190..465a181 100644 --- a/src/probing_reflection/types.py +++ b/src/probing_reflection/types.py @@ -8,6 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from types import MappingProxyType from typing import TypedDict @@ -39,7 +40,7 @@ class ReflectionResult: sample_id: str reflection_score: float - metadata: dict[str, str] = field(default_factory=dict) + metadata: MappingProxyType[str, str] = field(default_factory=lambda: MappingProxyType({})) class ContrastivePair(TypedDict):