From 335de445f399172b360d07ecdc57cc6faa9f2210 Mon Sep 17 00:00:00 2001 From: Aparna Pradhan Date: Mon, 28 Sep 2026 12:17:06 +0530 Subject: [PATCH 1/2] fix(db): admit IDEMPOTENCY_CONFLICT audit events; self-sufficient stripe fixture --- .../012_audit_event_type_idempotency.sql | 11 +++++++++ .../test_stripe_idempotency_restart.py | 23 ++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 database/migrations/012_audit_event_type_idempotency.sql diff --git a/database/migrations/012_audit_event_type_idempotency.sql b/database/migrations/012_audit_event_type_idempotency.sql new file mode 100644 index 0000000..c4dc7bf --- /dev/null +++ b/database/migrations/012_audit_event_type_idempotency.sql @@ -0,0 +1,11 @@ +-- 012_audit_event_type_idempotency.sql: admit IDEMPOTENCY_CONFLICT writes. +-- apps/api/webhooks.py persists event_type='IDEMPOTENCY_CONFLICT' on webhook +-- idempotency conflicts, but 003's chk_audit_event_type allowlist omits it, +-- so live conflicts crash with CheckViolation. This upgrade is idempotent: +-- on fresh DBs 003 creates the old constraint and this file widens it; on +-- already-migrated DBs 003 is skipped by version tracking and this file +-- performs the same widening. No other constraint is touched. +DO $$ BEGIN + ALTER TABLE audit_logs DROP CONSTRAINT IF EXISTS chk_audit_event_type; + ALTER TABLE audit_logs ADD CONSTRAINT chk_audit_event_type CHECK (event_type IN ('pipeline_started','pipeline_completed','pipeline_failed','assertion_created','assertion_validated','action_proposed','action_approved','action_rejected','commentary_submitted','commentary_reviewed','user_login','export_downloaded','config_changed','anomaly_detected','data_quality_alert','row_inserted','row_updated','row_deleted','rls_violation_attempted','admin_cross_tenant_access','IDEMPOTENCY_CONFLICT')); +END $$; diff --git a/tests/integration/test_stripe_idempotency_restart.py b/tests/integration/test_stripe_idempotency_restart.py index 0f02684..8dc1480 100644 --- a/tests/integration/test_stripe_idempotency_restart.py +++ b/tests/integration/test_stripe_idempotency_restart.py @@ -32,6 +32,8 @@ from sqlalchemy import create_engine, text from sqlalchemy.pool import NullPool +from tests.integration.test_tenant_isolation import _apply_migrations + logger = logging.getLogger(__name__) DEFAULT_DSN = "postgresql+psycopg://finsight:finsight@localhost:5432/finsight" @@ -68,6 +70,7 @@ def engine() -> Generator[sqlalchemy.Engine, None, None]: eng = create_engine(DSN, isolation_level="AUTOCOMMIT", poolclass=NullPool) if not _db_reachable(eng): pytest.skip("integration database not reachable") + _apply_migrations() # base schema (audit_logs etc.); idempotent, shared with tenant suite with eng.connect() as conn: conn.execute( text( @@ -116,13 +119,17 @@ def _ingest( """ incoming = _payload_hash(payload) with engine.connect() as conn: - existing = conn.execute( - text( - f"SELECT payload_hash, result_net FROM {PROBE_TABLE} " - "WHERE idempotency_key = :key" - ), - {"key": idempotency_key}, - ).mappings().first() + existing = ( + conn.execute( + text( + f"SELECT payload_hash, result_net FROM {PROBE_TABLE} " + "WHERE idempotency_key = :key" + ), + {"key": idempotency_key}, + ) + .mappings() + .first() + ) if existing is not None: if existing["payload_hash"] != incoming: conn.execute( @@ -130,7 +137,7 @@ def _ingest( "INSERT INTO audit_logs " "(id, tenant_id, period, event_type, event_data) " "VALUES (:id, :tenant_id, :period, :event_type, " - ":event_data::jsonb)" + "CAST(:event_data AS JSONB))" ), { "id": f"stripe_conflict_{uuid4().hex[:8]}", From 0218cbe395dc5966c7dd396d160bbd48831736d3 Mon Sep 17 00:00:00 2001 From: Aparna Pradhan Date: Mon, 28 Sep 2026 12:17:18 +0530 Subject: [PATCH 2/2] chore: remove dead finance-llm stack and its dedicated tests (quarantined, unreachable) --- finance/llm/__init__.py | 10 + finance/llm/client.py | 69 ---- finance/llm/model_router.py | 88 ----- finance/llm/response_validator.py | 33 -- finance/llm/structured_generation.py | 76 ---- finance/llm/telemetry.py | 72 ---- tests/llm/test_providers.py | 93 ----- tests/unit/test_finance/test_llm_runtime.py | 400 -------------------- 8 files changed, 10 insertions(+), 831 deletions(-) delete mode 100644 finance/llm/client.py delete mode 100644 finance/llm/model_router.py delete mode 100644 finance/llm/response_validator.py delete mode 100644 finance/llm/structured_generation.py delete mode 100644 finance/llm/telemetry.py delete mode 100644 tests/llm/test_providers.py delete mode 100644 tests/unit/test_finance/test_llm_runtime.py diff --git a/finance/llm/__init__.py b/finance/llm/__init__.py index e69de29..4d7f6f6 100644 --- a/finance/llm/__init__.py +++ b/finance/llm/__init__.py @@ -0,0 +1,10 @@ +"""Quarantined: legacy ``finance.llm`` provider stack (REMOVED). + +The provider modules (client, model_router, structured_generation, +response_validator, telemetry) were unreachable from all executable paths — +imported only by each other and dead-code tests — and have been deleted. + +Live LLM access goes through ``shared.utils.llm_client`` (used by +``agents/`` and ``apps/``). Do NOT resurrect provider logic in this package; +extend the shared client instead. +""" diff --git a/finance/llm/client.py b/finance/llm/client.py deleted file mode 100644 index 00b8e35..0000000 --- a/finance/llm/client.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -from pydantic import BaseModel - -from finance.llm.model_router import ModelRouter -from finance.llm.structured_generation import StructuredGeneration -from finance.llm.telemetry import Telemetry - - -class LLMClient: - def __init__( - self, - router: ModelRouter | None = None, - telemetry: Telemetry | None = None, - ): - self._router = router or ModelRouter() - self._telemetry = telemetry or Telemetry() - - def generate( - self, - system_prompt: str, - user_prompt: str, - response_model: type[BaseModel], - ) -> BaseModel: - last_error: Exception | None = None - while True: - provider = self._router.select() - if provider is None: - raise last_error or RuntimeError("No providers available") - - gen = StructuredGeneration( - api_key=provider.api_key, - base_url=provider.base_url, - model=provider.model, - ) - - import time - start = time.monotonic() - try: - result = gen.generate( - system_prompt=system_prompt, - user_prompt=user_prompt, - response_model=response_model, - ) - latency_ms = int((time.monotonic() - start) * 1000) - self._telemetry.record( - provider=provider.name, - model=provider.model, - prompt_name=response_model.__name__, - prompt_tokens=0, - completion_tokens=0, - latency_ms=latency_ms, - success=True, - ) - self._router.record_success(provider.name) - return result - except Exception as e: - latency_ms = int((time.monotonic() - start) * 1000) - self._telemetry.record( - provider=provider.name, - model=provider.model, - prompt_name=response_model.__name__, - prompt_tokens=0, - completion_tokens=0, - latency_ms=latency_ms, - success=False, - ) - self._router.record_failure(provider.name) - last_error = e diff --git a/finance/llm/model_router.py b/finance/llm/model_router.py deleted file mode 100644 index 6919847..0000000 --- a/finance/llm/model_router.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import os -import time -from typing import Any, ClassVar - -from pydantic import BaseModel - - -class ProviderConfig(BaseModel): - name: str - model: str - base_url: str - api_key_env: str - - @property - def api_key(self) -> str: - return os.environ.get(self.api_key_env, "") - - @property - def is_configured(self) -> bool: - return bool(self.api_key) and bool(self.base_url) - - -class ModelRouter: - _PROVIDER_DEFS: ClassVar[list[dict[str, Any]]] = [ - { - "name": "groq", - "api_key_env": "GROQ_API_KEY", - "base_url_env": "GROQ_BASE_URL", - "model_env": "GROQ_CHAT_MODEL", - "default_base_url": "https://api.groq.com/openai/v1", - "default_model": "llama-3.3-70b-versatile", - }, - { - "name": "openrouter", - "api_key_env": "OPENROUTER_API_KEY", - "base_url_env": "OPENROUTER_BASE_URL", - "model_env": "OPENROUTER_CHAT_MODEL", - "default_base_url": "https://openrouter.ai/api/v1", - "default_model": "nvidia/nemotron-3-ultra-550b-a55b:free", - }, - { - "name": "poolside", - "api_key_env": "POOLSIDE_API_KEY", - "base_url_env": "POOLSIDE_BASE_URL", - "model_env": "POOLSIDE_CHAT_MODEL", - "default_base_url": "https://inference.poolside.ai/v1", - "default_model": "poolside/laguna-m.1", - }, - ] - - def __init__(self, cooldown_minutes: int = 5): - self.cooldown_minutes = cooldown_minutes - self._failed_at: dict[str, float] = {} - self._providers: list[ProviderConfig] = [] - for pdef in self._PROVIDER_DEFS: - base_url = os.environ.get(pdef["base_url_env"], pdef["default_base_url"]) - model = os.environ.get(pdef["model_env"], pdef["default_model"]) - cfg = ProviderConfig( - name=pdef["name"], - model=model, - base_url=base_url, - api_key_env=pdef["api_key_env"], - ) - if cfg.is_configured: - self._providers.append(cfg) - - def list_providers(self) -> list[ProviderConfig]: - return list(self._providers) - - def select(self, tasks: list[str] | None = None) -> ProviderConfig | None: - now = time.time() - for p in self._providers: - failed_at = self._failed_at.get(p.name) - if failed_at is not None: - elapsed = now - failed_at - if elapsed < self.cooldown_minutes * 60: - continue - del self._failed_at[p.name] - return p - return None - - def record_failure(self, name: str) -> None: - self._failed_at[name] = time.time() - - def record_success(self, name: str) -> None: - self._failed_at.pop(name, None) diff --git a/finance/llm/response_validator.py b/finance/llm/response_validator.py deleted file mode 100644 index 7046e65..0000000 --- a/finance/llm/response_validator.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from pydantic import BaseModel - - -class ValidationResult(BaseModel): - is_valid: bool - errors: list[str] - - -class ResponseValidator: - def validate( - self, - output: BaseModel, - expected_schema: type[BaseModel], - ) -> ValidationResult: - if isinstance(output, expected_schema): - return ValidationResult(is_valid=True, errors=[]) - return ValidationResult(is_valid=False, errors=["Output does not match expected schema"]) - - def validate_evidence( - self, - claim: str, - evidence_ids: list[str], - min_evidence: int = 1, - ) -> ValidationResult: - errors: list[str] = [] - if len(evidence_ids) < min_evidence: - errors.append( - f"Claim requires at least {min_evidence} evidence source(s), " - f"got {len(evidence_ids)}" - ) - return ValidationResult(is_valid=len(errors) == 0, errors=errors) diff --git a/finance/llm/structured_generation.py b/finance/llm/structured_generation.py deleted file mode 100644 index c3d99ad..0000000 --- a/finance/llm/structured_generation.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import json -import time -from typing import Any - -import httpx -from pydantic import BaseModel - - -class StructuredGeneration: - def __init__(self, api_key: str, base_url: str, model: str = "default"): - self.api_key = api_key - self.base_url = base_url.rstrip("/") - self.model = model - - def generate( - self, - system_prompt: str, - user_prompt: str, - response_model: type[BaseModel], - max_retries: int = 3, - ) -> BaseModel: - last_error: Exception | None = None - for attempt in range(max_retries): - try: - return self._call(system_prompt, user_prompt, response_model) - except Exception as e: - last_error = e - if attempt < max_retries - 1: - time.sleep(2 ** attempt) - raise last_error or RuntimeError("Generation failed after retries") - - def _call( - self, - system_prompt: str, - user_prompt: str, - response_model: type[BaseModel], - ) -> BaseModel: - with httpx.Client(timeout=60.0) as client: - resp = client.post( - f"{self.base_url}/chat/completions", - headers={ - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - }, - json={ - "model": self.model, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - }, - ) - resp.raise_for_status() - body = resp.json() - - content = body["choices"][0]["message"]["content"] - parsed = self._extract_json(content) - if parsed is None: - raise ValueError(f"LLM returned invalid JSON: {content[:200]}") - return response_model.model_validate(parsed) - - @staticmethod - def _extract_json(text: str) -> dict[str, Any] | None: - text = text.strip() - if text.startswith("```"): - lines = text.splitlines() - start = 1 if lines[0].startswith("```") else 0 - end = -1 if lines[-1].strip().startswith("```") else len(lines) - text = "\n".join(lines[start:end]).strip() - try: - parsed: Any = json.loads(text) - return parsed if isinstance(parsed, dict) else None - except json.JSONDecodeError: - return None diff --git a/finance/llm/telemetry.py b/finance/llm/telemetry.py deleted file mode 100644 index 3659a3e..0000000 --- a/finance/llm/telemetry.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Any - - -class Telemetry: - def __init__(self) -> None: - self._records: list[dict[str, Any]] = [] - - def record( - self, - provider: str, - model: str, - prompt_name: str, - prompt_tokens: int, - completion_tokens: int, - latency_ms: int, - success: bool, - ) -> None: - self._records.append({ - "provider": provider, - "model": model, - "prompt_name": prompt_name, - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "latency_ms": latency_ms, - "success": success, - "timestamp": datetime.now(), - }) - - def summary(self) -> dict[str, Any]: - total_calls = len(self._records) - if total_calls == 0: - return { - "total_calls": 0, - "total_tokens": 0, - "success_rate": 0.0, - "avg_latency_ms": 0.0, - "by_provider": {}, - } - - total_tokens = sum(r["prompt_tokens"] + r["completion_tokens"] for r in self._records) - successes = sum(1 for r in self._records if r["success"]) - avg_latency = sum(r["latency_ms"] for r in self._records) / total_calls - - by_provider: dict[str, dict[str, Any]] = {} - for r in self._records: - p = r["provider"] - if p not in by_provider: - by_provider[p] = { - "calls": 0, - "tokens": 0, - "avg_latency_ms": 0.0, - "success_rate": 0.0, - } - by_provider[p]["calls"] += 1 - by_provider[p]["tokens"] += r["prompt_tokens"] + r["completion_tokens"] - - for p, stats in by_provider.items(): - prov_records = [r for r in self._records if r["provider"] == p] - stats["avg_latency_ms"] = sum(r["latency_ms"] for r in prov_records) / len(prov_records) - prov_successes = sum(1 for r in prov_records if r["success"]) - stats["success_rate"] = prov_successes / len(prov_records) - - return { - "total_calls": total_calls, - "total_tokens": total_tokens, - "success_rate": successes / total_calls, - "avg_latency_ms": avg_latency, - "by_provider": by_provider, - } diff --git a/tests/llm/test_providers.py b/tests/llm/test_providers.py deleted file mode 100644 index f373258..0000000 --- a/tests/llm/test_providers.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Real LLM provider integration tests (pytest -m llm).""" - -import pytest -from dotenv import load_dotenv -from pydantic import BaseModel - -from finance.llm.client import LLMClient -from finance.llm.model_router import ModelRouter - - -class VarianceOutput(BaseModel): - explanation: str - root_causes: list[str] - confidence: str - - -load_dotenv() - - -@pytest.fixture -def client() -> LLMClient: - return LLMClient() - - -@pytest.fixture -def router() -> ModelRouter: - return ModelRouter() - - -@pytest.mark.llm -@pytest.mark.parametrize("provider_name", ["groq", "openrouter", "poolside"]) -def test_each_provider_individually(provider_name: str, router: ModelRouter) -> None: - router._providers = [p for p in router._providers if p.name == provider_name] - client = LLMClient(router=router) - result = client.generate( - system_prompt=( - 'Return ONLY valid JSON: {"explanation": str, "root_causes": [str], ' - '"confidence": str}. No markdown.' - ), - user_prompt="Revenue: actual 10.5M, budget 9.3M, variance +1.2M (12.9% favorable).", - response_model=VarianceOutput, - ) - assert isinstance(result, VarianceOutput) - assert result.explanation - assert len(result.root_causes) >= 1 - assert result.confidence.lower() in ("low", "medium", "high") - - -@pytest.mark.llm -def test_llm_fallback_chain(router: ModelRouter) -> None: - class SimpleOutput(BaseModel): - explanation: str - confidence: str - - router.record_failure("groq") - router.record_failure("openrouter") - client = LLMClient(router=router) - result = client.generate( - system_prompt=( - 'Return ONLY valid JSON: {"explanation": str, "confidence": str}. No markdown.' - ), - user_prompt="Revenue up 12.9% favorable.", - response_model=SimpleOutput, - ) - assert isinstance(result, SimpleOutput) - assert result.explanation - assert result.confidence.lower() in ("low", "medium", "high") - - -@pytest.mark.llm -def test_router_all_healthy(router: ModelRouter) -> None: - names = [p.name for p in router.list_providers()] - assert "groq" in names - assert "openrouter" in names - assert "poolside" in names - - -@pytest.mark.llm -def test_llm_telemetry(client: LLMClient) -> None: - class SimpleOutput(BaseModel): - explanation: str - - client.generate( - system_prompt='Return ONLY valid JSON: {"explanation": str}.', - user_prompt="Say hi.", - response_model=SimpleOutput, - ) - stats = client._telemetry.summary() - assert stats["total_calls"] >= 1 - for _provider, data in stats["by_provider"].items(): - assert data["calls"] >= 0 - assert "avg_latency_ms" in data - assert "success_rate" in data diff --git a/tests/unit/test_finance/test_llm_runtime.py b/tests/unit/test_finance/test_llm_runtime.py deleted file mode 100644 index 6c20a06..0000000 --- a/tests/unit/test_finance/test_llm_runtime.py +++ /dev/null @@ -1,400 +0,0 @@ -"""TDD tests for Phase 6 — LLM Runtime. - -Multi-provider support with automatic fallback (Groq → OpenRouter → Poolside). -All three use OpenAI-compatible APIs. Tests mock at the HTTP level. -""" -from __future__ import annotations - -import os -from collections.abc import Generator - -import httpx -import pytest -from pydantic import BaseModel -from pytest_httpx import HTTPXMock - -# ── Fixtures ───────────────────────────────────────────────────────────────── - - -@pytest.fixture(autouse=True) -def _set_env() -> Generator[None, None, None]: - """Set env vars for all tests so ModelRouter finds providers.""" - old = {} - for k in ("GROQ_API_KEY", "GROQ_BASE_URL", "GROQ_CHAT_MODEL", - "OPENROUTER_API_KEY", "OPENROUTER_BASE_URL", "OPENROUTER_CHAT_MODEL", - "POOLSIDE_API_KEY", "POOLSIDE_BASE_URL", "POOLSIDE_CHAT_MODEL"): - old[k] = os.environ.get(k) - os.environ["GROQ_API_KEY"] = "gsk-test-key" - os.environ["GROQ_BASE_URL"] = "https://api.groq.com/openai/v1" - os.environ["GROQ_CHAT_MODEL"] = "llama-3.3-70b-versatile" - os.environ["OPENROUTER_API_KEY"] = "sk-or-test-key" - os.environ["OPENROUTER_BASE_URL"] = "https://openrouter.ai/api/v1" - os.environ["OPENROUTER_CHAT_MODEL"] = "meta-llama/llama-3.1-8b-instruct:free" - os.environ["POOLSIDE_API_KEY"] = "sky-test-key" - os.environ["POOLSIDE_BASE_URL"] = "https://inference.poolside.ai/v1" - os.environ["POOLSIDE_CHAT_MODEL"] = "poolside/laguna-m.1" - yield - for k, v in old.items(): - if v is None: - os.environ.pop(k, None) - else: - os.environ[k] = v - - -# ── Model Router ───────────────────────────────────────────────────────────── - - -class TestModelRouter: - def test_router_has_configured_providers(self) -> None: - from finance.llm.model_router import ModelRouter - router = ModelRouter() - providers = router.list_providers() - assert len(providers) > 0 - - def test_router_selects_preferred_provider(self) -> None: - from finance.llm.model_router import ModelRouter - router = ModelRouter() - provider = router.select(tasks=["analysis"]) - assert provider is not None - assert hasattr(provider, "name") - - def test_router_falls_back_on_provider_failure(self) -> None: - from finance.llm.model_router import ModelRouter - router = ModelRouter() - provider = router.select(tasks=["analysis"]) - assert provider is not None - router.record_failure(provider.name) - fallback = router.select(tasks=["analysis"]) - assert fallback is not None - assert fallback.name != provider.name - - def test_router_returns_none_when_all_fail(self) -> None: - from finance.llm.model_router import ModelRouter - router = ModelRouter() - for _ in range(6): - p = router.select(tasks=["analysis"]) - if p is None: - break - router.record_failure(p.name) - last = router.select(tasks=["analysis"]) - assert last is None - - def test_router_recovers_after_cooldown(self) -> None: - from finance.llm.model_router import ModelRouter - router = ModelRouter(cooldown_minutes=0) - p1 = router.select(tasks=["analysis"]) - assert p1 is not None - router.record_failure(p1.name) - recovered = router.select(tasks=["analysis"]) - assert recovered is not None - - def test_model_router_config(self) -> None: - from finance.llm.model_router import ProviderConfig - cfg = ProviderConfig( - name="test", - model="test-model", - base_url="https://test.example.com/v1", - api_key_env="TEST_KEY", - ) - assert cfg.name == "test" - assert cfg.model == "test-model" - - -# ── Structured Generation ─────────────────────────────────────────────────── - - -class TestStructuredGeneration: - def test_generate_returns_typed_output(self, httpx_mock: HTTPXMock) -> None: - from finance.llm.structured_generation import StructuredGeneration - - class TestOutput(BaseModel): - summary: str - confidence: str - - httpx_mock.add_response( - url="https://api.test.com/v1/chat/completions", - method="POST", - json={ - "choices": [{ - "message": { - "content": '{"summary": "Revenue up 10%", "confidence": "high"}', - }, - }], - "usage": {"prompt_tokens": 50, "completion_tokens": 20}, - }, - ) - - gen = StructuredGeneration(api_key="test", base_url="https://api.test.com/v1") - result = gen.generate( - system_prompt="Analyze this variance.", - user_prompt="Revenue is up 10%.", - response_model=TestOutput, - ) - assert isinstance(result, TestOutput) - assert result.summary == "Revenue up 10%" - assert result.confidence == "high" - - def test_generate_retries_on_failure(self, httpx_mock: HTTPXMock) -> None: - from finance.llm.structured_generation import StructuredGeneration - - class TestOutput(BaseModel): - result: str - - httpx_mock.add_response( - url="https://api.test.com/v1/chat/completions", - method="POST", - status_code=500, - json={"error": "Internal error"}, - ) - httpx_mock.add_response( - url="https://api.test.com/v1/chat/completions", - method="POST", - json={ - "choices": [{"message": {"content": '{"result": "success"}'}}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5}, - }, - ) - - gen = StructuredGeneration(api_key="test", base_url="https://api.test.com/v1") - result = gen.generate( - system_prompt="Test.", - user_prompt="Test.", - response_model=TestOutput, - max_retries=2, - ) - assert isinstance(result, TestOutput) - assert result.result == "success" - - def test_generate_raises_on_invalid_json(self, httpx_mock: HTTPXMock) -> None: - from finance.llm.structured_generation import StructuredGeneration - - class TestOutput(BaseModel): - value: str - - httpx_mock.add_response( - url="https://api.test.com/v1/chat/completions", - method="POST", - json={ - "choices": [{"message": {"content": "not valid json"}}], - }, - ) - - gen = StructuredGeneration(api_key="test", base_url="https://api.test.com/v1") - with pytest.raises(Exception, match="JSON|valid"): - gen.generate( - system_prompt="Test.", - user_prompt="Test.", - response_model=TestOutput, - max_retries=1, - ) - - def test_generate_exhausts_retries(self, httpx_mock: HTTPXMock) -> None: - from finance.llm.structured_generation import StructuredGeneration - - class TestOutput(BaseModel): - value: str - - httpx_mock.add_response( - url="https://api.test.com/v1/chat/completions", - method="POST", - status_code=500, - json={"error": "Error"}, - is_reusable=True, - ) - - gen = StructuredGeneration(api_key="test", base_url="https://api.test.com/v1") - with pytest.raises(httpx.HTTPStatusError): - gen.generate( - system_prompt="Test.", - user_prompt="Test.", - response_model=TestOutput, - max_retries=2, - ) - - -# ── Response Validator ─────────────────────────────────────────────────────── - - -class TestResponseValidator: - def test_validate_valid_output(self) -> None: - from finance.llm.response_validator import ResponseValidator - - class TestOutput(BaseModel): - summary: str - confidence: str - - validator = ResponseValidator() - result = validator.validate( - output=TestOutput(summary="Good quarter.", confidence="high"), - expected_schema=TestOutput, - ) - assert result.is_valid is True - assert len(result.errors) == 0 - - def test_validate_missing_field(self) -> None: - from finance.llm.response_validator import ResponseValidator - - class TestOutput(BaseModel): - summary: str - confidence: str - evidence_ids: list[str] - - validator = ResponseValidator() - result = validator.validate( - output=TestOutput(summary="Test", confidence="high", evidence_ids=[]), - expected_schema=TestOutput, - ) - assert result.is_valid is True - - def test_validate_evidence_requirement(self) -> None: - from finance.llm.response_validator import ResponseValidator - - validator = ResponseValidator() - result = validator.validate_evidence( - claim="Revenue increased 10%", - evidence_ids=[], - min_evidence=1, - ) - assert result.is_valid is False - assert "evidence" in str(result.errors).lower() - - def test_validate_evidence_passes_with_sources(self) -> None: - from finance.llm.response_validator import ResponseValidator - - validator = ResponseValidator() - result = validator.validate_evidence( - claim="Revenue increased 10% due to volume.", - evidence_ids=["var_4010", "kpi_revenue_growth"], - min_evidence=1, - ) - assert result.is_valid is True - - -# ── Telemetry ──────────────────────────────────────────────────────────────── - - -class TestTelemetry: - def test_record_completion(self) -> None: - from finance.llm.telemetry import Telemetry - t = Telemetry() - t.record( - provider="groq", - model="llama-3.3-70b", - prompt_name="variance_analysis", - prompt_tokens=150, - completion_tokens=45, - latency_ms=1200, - success=True, - ) - stats = t.summary() - assert stats["total_calls"] == 1 - assert stats["total_tokens"] == 195 - - def test_multiple_records(self) -> None: - from finance.llm.telemetry import Telemetry - t = Telemetry() - for _ in range(3): - t.record( - provider="groq", - model="llama-3.3-70b", - prompt_name="test", - prompt_tokens=100, - completion_tokens=20, - latency_ms=500, - success=True, - ) - t.record( - provider="openrouter", - model="llama-3.1-8b", - prompt_name="test", - prompt_tokens=50, - completion_tokens=10, - latency_ms=800, - success=False, - ) - stats = t.summary() - assert stats["total_calls"] == 4 - assert stats["total_tokens"] == 420 - assert stats["success_rate"] == 0.75 - - def test_per_provider_breakdown(self) -> None: - from finance.llm.telemetry import Telemetry - - t = Telemetry() - t.record( - provider="groq", model="m1", prompt_name="t", prompt_tokens=100, - completion_tokens=20, latency_ms=500, success=True, - ) - t.record( - provider="poolside", model="m2", prompt_name="t", prompt_tokens=200, - completion_tokens=40, latency_ms=1000, success=True, - ) - stats = t.summary() - assert "groq" in stats["by_provider"] - assert "poolside" in stats["by_provider"] - assert stats["by_provider"]["groq"]["calls"] == 1 - assert stats["by_provider"]["poolside"]["calls"] == 1 - - -# ── Unified Client ─────────────────────────────────────────────────────────── - - -class TestLLMClient: - def test_client_generate_returns_typed_output(self, httpx_mock: HTTPXMock) -> None: - from pydantic import BaseModel - - from finance.llm.client import LLMClient - - class TestOutput(BaseModel): - result: str - - httpx_mock.add_response( - url="https://api.groq.com/openai/v1/chat/completions", - method="POST", - json={ - "choices": [{"message": {"content": '{"result": "success"}'}}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5}, - }, - ) - - client = LLMClient() - result = client.generate( - system_prompt="Test.", - user_prompt="Test.", - response_model=TestOutput, - ) - assert isinstance(result, TestOutput) - assert result.result == "success" - - def test_client_falls_back_on_provider_error(self, httpx_mock: HTTPXMock) -> None: - from pydantic import BaseModel - - from finance.llm.client import LLMClient - - class TestOutput(BaseModel): - result: str - - httpx_mock.add_response( - url="https://api.groq.com/openai/v1/chat/completions", - method="POST", - status_code=500, - json={"error": "Server error"}, - is_reusable=True, - ) - httpx_mock.add_response( - url="https://openrouter.ai/api/v1/chat/completions", - method="POST", - json={ - "choices": [{"message": {"content": '{"result": "fallback_worked"}'}}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5}, - }, - ) - - client = LLMClient() - result = client.generate( - system_prompt="Test.", - user_prompt="Test.", - response_model=TestOutput, - ) - assert isinstance(result, TestOutput) - assert result.result == "fallback_worked"