diff --git a/.env.example b/.env.example index 1c6957a..7741fa0 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ NODE_ENV=development ENVIRONMENT=development LOG_LEVEL=info +CORS_ALLOWED_ORIGINS=http://localhost:3000 # ── PostgreSQL (+ pgvector) ───────────────────────────────────────────────── POSTGRES_HOST=postgres diff --git a/API_CONTRACT.md b/API_CONTRACT.md new file mode 100644 index 0000000..c106974 --- /dev/null +++ b/API_CONTRACT.md @@ -0,0 +1,287 @@ +# Literature Intelligence API Contract v2 + +## Version 2 Summary +This updated contract resolves implementation blockers by specifying production-grade operations, retry semantics, tenant isolation, and integration contract details. + +## Existing Literature Service Endpoints +Retain and extend current `services/literature` endpoints: +- `GET /healthz` +- `GET /readyz` +- `GET /api/v1/papers` +- `GET /api/v1/papers/{paper_id}` +- `POST /api/v1/ingestion` +- `GET /api/v1/ingestion/{job_id}` + +## API Principles +- All endpoints require `Authorization: Bearer `. +- Auth validation may be local JWT signature verification or auth-service introspection. +- Tenant context is mandatory and derived from auth claims. +- All responses are JSON with standard fields: `code`, `message`, `request_id`, and payload data. +- Use consistent pagination and sorting schemas. +- Expose manual retry and dead-letter inspection for failed ingestion/extraction jobs. + +## Ingestion Endpoints + +### `POST /api/v1/ingestion` +Create an ingestion job for a source query or document import. + +Request body: +- `source` enum: `pubmed`, `biorxiv`, `medrxiv`, `patent`, `conference`, `custom` +- `query` string +- `source_document_id` string, optional for single-document imports +- `source_url` string, optional for explicit fetch +- `workspace_id` UUID, optional (derived from auth if omitted) +- `project_id` UUID, optional (derived from auth if omitted) +- `options` object + - `fetch_mode` enum: `batch` | `single` + - `max_documents` int + - `crawl_depth` int + +Response body: +- `id` UUID +- `source` +- `query` +- `status` enum: `queued`, `running`, `completed`, `failed` +- `created_at` +- `started_at` optional +- `completed_at` optional +- `tenant` metadata + +### `GET /api/v1/ingestion/{job_id}` +Fetch ingestion job status, progress, and operational details. + +Response body includes: +- all fields from job creation +- `document_count` +- `processed_count` +- `failed_count` +- `pending_count` +- `error_message` +- `last_error` +- `backoff_until` +- `dead_letter_count` +- `tenant` metadata + +### `POST /api/v1/ingestion/{job_id}/retry` +Retry a failed ingestion job or a specific failed document. + +Request body: +- `document_id` optional +- `reason` string + +Response body: +- `status` +- `retry_scheduled_at` + +### `GET /api/v1/ingestion/{job_id}/dead-letter` +Inspect failed ingestion items. + +Response body: +- `items` array of failure records +- `total` + +## Paper and Document Endpoints + +### `GET /api/v1/papers` +List ingested papers with paging and filters. + +Query parameters: +- `page` int +- `page_size` int +- `source` optional +- `status` optional +- `query` optional +- `workspace_id` optional +- `project_id` optional +- `date_from` / `date_to` + +Response body: +- `items` array of paper metadata +- `total` +- `page` +- `page_size` + +### `GET /api/v1/papers/{paper_id}` +Get full paper metadata, extraction status, and available artifacts. + +Response fields: +- `id`, `title`, `authors`, `source`, `doi`, `publication_date` +- `abstract` +- `summary` +- `citation_count` +- `status` +- `document_status` +- `extraction_status` +- `tenant` metadata +- `citations` +- `entities` +- `relationships` +- `text_chunk_count` + +## Extraction Result Endpoints + +### `GET /api/v1/papers/{paper_id}/entities` +List extracted entities for a paper. + +Response fields: +- list of entities with `id`, `type`, `text`, `canonical_id`, `confidence`, `span`, `schema`, `source`, `created_at` +- pagination metadata + +### `GET /api/v1/papers/{paper_id}/relationships` +List extracted relationship candidates. + +Response fields: +- list of relationship candidates with `id`, `from_entity_id`, `to_entity_id`, `type`, `confidence`, `evidence`, `metadata`, `created_at` + +### `GET /api/v1/papers/{paper_id}/citations` +List citation edges extracted from the paper. + +Response fields: +- `id`, `cited_source`, `cited_doi`, `confidence`, `created_at` + +## Duplicate and Canonicalization Endpoints + +### `GET /api/v1/duplicates/{document_id}` +Get duplicate candidates for a document and canonical state. + +Response fields: +- `canonical_document_id` +- `duplicate_ids` +- `similarity_score` +- `status` + +### `POST /api/v1/duplicates/resolve` +Resolve a duplicate cluster. + +Request body: +- `canonical_document_id` +- `duplicate_ids` +- `action` enum: `merge` | `ignore` +- `reason` string + +Response fields: +- `status` +- `resolved_at` +- `audit_record_id` + +## Search Integration Contract +The literature service uses `services/search` as the canonical search provider. + +### Query Search +- `GET /api/v1/search?q=&scope=literature&organization_id=&workspace_id=` +- Returns keyword search results from the shared search service. + +### Hybrid Search +- `POST /api/v1/search` + Request body: + - `query` string + - `embedding` optional array + - `filters` optional object + - `tenant` object with `organization_id`, `workspace_id`, `project_id` + +### Index Upsert Contract +- Preferred: the search service exposes a document upsert endpoint such as `POST /api/v1/search/index`. +- Upsert payload includes: + - `document_id` + - `organization_id` + - `workspace_id` + - `source` + - `title` + - `abstract` + - `summary` + - `content_chunks` + - `embedding` + - `metadata` +- Upsert is idempotent and scoped by `document_id` and tenant. +- If direct upsert is not available, literature publishes `document.indexed` events for search ingestion. + +## Knowledge Graph Integration Contract +- Primary integration is event-driven. +- Publish versioned events to graph consumer topics. +- Event payloads must include: + - `event_id` + - `event_type` + - `version` + - `tenant` + - `document_id` + - `entities` + - `relationships` + - `evidence` + - `created_at` +- Synchronous graph CRUD direct calls are only allowed as a fallback and must be explicitly documented. + +## Audit and Operational Endpoints +### `GET /api/v1/status` +Return dependency status (Postgres, auth, search, broker) and service readiness. + +### `GET /api/v1/metrics` +Expose service-level metrics if a metrics endpoint is used directly. + +## Common Models + +### `PaperSummary` +- `id` +- `title` +- `authors` +- `abstract` +- `summary` +- `source` +- `doi` +- `publication_date` +- `citation_count` +- `status` +- `document_status` +- `extraction_status` +- `tenant` + +### `Entity` +- `id` +- `document_id` +- `type` +- `text` +- `canonical_id` +- `confidence` +- `span` +- `schema` +- `source` +- `created_at` + +### `RelationshipCandidate` +- `id` +- `document_id` +- `from_entity_id` +- `to_entity_id` +- `type` +- `confidence` +- `evidence` +- `metadata` +- `created_at` + +### `ExtractionJobStatus` +- `job_id` +- `document_id` +- `stage` +- `status` +- `started_at` +- `completed_at` +- `error_message` +- `retry_count` +- `dead_letter` + +## Auth and Tenant Requirements +- Every endpoint requires `Authorization: Bearer `. +- `organization_id`, `workspace_id`, and `project_id` are derived from auth claims and cannot be overridden with untrusted client input. +- Tenant-aware access is enforced by the service, not just by the gateway. +- Admin-only actions respect existing auth roles. + +## Error Handling +- Standard HTTP codes: 400, 401, 403, 404, 409, 429, 500. +- Structured error returns include `code`, `message`, `details`, and `request_id`. +- Transient errors expose retry guidance when appropriate. +- Rate-limit and backpressure conditions return `429` with `Retry-After`. + +## Operational Requirements +- Manual retry endpoints for failed ingestion and extraction tasks. +- Dead-letter inspection. +- Support for source-specific rate limit and backoff state. +- Audit traceability of ingestion and duplicate resolution actions. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..3005917 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,299 @@ +# Literature Intelligence Architecture v2 + +## Purpose +The Literature Intelligence service ingests, enriches, and normalizes biomedical literature while enabling safe search, knowledge graph population, and downstream AI workflows. + +This version adds production readiness, scaling, resilience, security, and explicit integration contracts to support implementation. + +## What Changed in v2 +- Added deployment, scaling, operational, and monitoring requirements. +- Defined event transport fallback and schema expectations. +- Clarified search embedding ownership and contract. +- Completed tenant isolation requirements for all owned tables. +- Added external-source reliability, retry, and dead-letter handling. +- Added audit / data governance and retention expectations. + +## Design Principles +- Own literature ingestion, extraction, canonicalization, and indexing orchestration. +- Reuse existing services for auth (`services/auth`), search (`services/search`), and knowledge graph (`services/kg`). +- Never duplicate auth, search query, or Neo4j persistence APIs. +- Keep domain state in literature Postgres tables and push search index changes through the search service. +- Keep event schemas explicit, versioned, idempotent, and secure. +- Build for horizontal scale and failure isolation. + +## Folder Structure +The Literature Intelligence service remains in `services/literature` with the following internal structure: + +- `services/literature/` + - `app/` + - `core/` + - `config.py` + - `security.py` + - `db.py` + - `events.py` + - `auditing.py` + - `routers/` + - `ingestion.py` + - `papers.py` + - `entities.py` + - `relationships.py` + - `duplicates.py` + - `status.py` + - `services/` + - `ingestion_service.py` + - `document_service.py` + - `extraction_service.py` + - `duplicate_service.py` + - `annotation_service.py` + - `indexing_service.py` + - `retry_service.py` + - `models/` + - `schemas.py` + - `db_models.py` + - `events.py` + - `pipelines/` + - `ingestion_pipeline.py` + - `extraction_pipeline.py` + - `metadata_pipeline.py` + - `integrations/` + - `auth_client.py` + - `search_client.py` + - `kg_client.py` + - `llm_client.py` + - `external_sources.py` + - `workers/` + - `ingestion_worker.py` + - `extraction_worker.py` + - `deduplication_worker.py` + - `indexing_worker.py` + - `dead_letter_worker.py` + - `utils/` + - `logging.py` + - `tenant.py` + - `metrics.py` + - `schemas.py` + - `validation.py` + - `Dockerfile` + - `requirements.txt` + - `README.md` + - `RUNBOOK.md` + +## Internal Modules + +### `core` +- `config.py`: loads environment settings including Postgres, Redis, event broker, JWT verification keys, service URLs, and operational thresholds. +- `db.py`: owns literature Postgres connection, connection pool sizing, schema migration helpers, and transaction hints for bulk ingestion. +- `security.py`: validates JWT tokens and extracts tenant claims; supports local token validation and auth-service introspection. +- `events.py`: emits versioned, typed events with tenant metadata and producer audit tags. +- `auditing.py`: writes audit records for ingestion, extraction, and event publishing. + +### `routers` +- `ingestion.py`: handles ingestion job creation, status, cancellation, and manual retry endpoints. +- `papers.py`: exposes paper lookup, search metadata, and document retrieval with summary and citation details. +- `entities.py`: exposes extracted entities with filtering and pagination. +- `relationships.py`: exposes extracted relationship candidates and evidence. +- `duplicates.py`: exposes duplicate detection and resolution actions. +- `status.py`: health checks, readiness, and dependency self-assessment. + +### `services` +- `ingestion_service.py`: orchestrates source ingestion, job lifecycle, ingestion batching, and dedup-safe persistence. +- `document_service.py`: owns document creation, normalization, chunked text storage, summary generation, and metadata enrichment. +- `extraction_service.py`: orchestrates NER, relation extraction, citation extraction, and evidence ranking. +- `duplicate_service.py`: detects duplicates by canonical keys, dedup group scoring, and canonical resolution. +- `annotation_service.py`: manages manual corrections, provenance annotations, and audit history. +- `indexing_service.py`: pushes embeddings and search metadata to the search service, using bulk and incremental upsert semantics. +- `retry_service.py`: encapsulates external-source retries, backoff, circuit-breaker, and dead-letter enqueueing. + +### `models` +- `schemas.py`: FastAPI request/response models, sorting, paging, and tenant DTOs. +- `db_models.py`: database table schemas and row mapping definitions. +- `events.py`: event payload models and version schemas. + +### `pipelines` +- `ingestion_pipeline.py`: fetches content from source adapters, normalizes IDs, and writes documents incrementally. +- `extraction_pipeline.py`: decomposes content into chunks, extracts entities/relations, and stores structured output. +- `metadata_pipeline.py`: enriches with DOI normalization, citation extraction, entity canonicalization, and external metadata lookups. + +### `integrations` +- `auth_client.py`: validates tokens, optionally introspects sessions, and fetches tenant metadata from auth service. +- `search_client.py`: calls `services/search` for query and embedding upsert; defines the exact contract for `GET /api/v1/search` and `POST /api/v1/search`. +- `kg_client.py`: publishes relationship events and supports synchronous graph import endpoints only when required. +- `llm_client.py`: interacts with existing `apps/ai-services` or external LLMs for summarization and extraction. +- `external_sources.py`: implements source-specific adapters, rate limit management, and fetcher health. + +### `workers` +- `ingestion_worker.py`: processes ingestion work items from queue or broker. +- `extraction_worker.py`: processes extraction jobs in isolation and supports autoscaling. +- `deduplication_worker.py`: canonicalizes duplicates and updates document status. +- `indexing_worker.py`: flushes bulk search/index updates and monitors indexing latency. +- `dead_letter_worker.py`: inspects and retries or escalates failed messages. + +### `utils` +- `tenant.py`: tenant propagation helpers, session variables, and RLS-enforced query filters. +- `validation.py`: schema validation, source input sanitation, and external URL filtering. +- `logging.py`: structured JSON logs with correlation IDs, tenant IDs, and pathway tags. +- `metrics.py`: application metrics, SLA counters, and exporter helpers. +- `schemas.py`: shared schema helpers for page, sort, and filter definitions. + +## Production Requirements +The service must support: +- containerized deployments with environment-driven configuration +- horizontal scaling of ingestion, extraction, and indexing workers +- health checks, readiness probes, and dependency checks +- metrics for throughput, error rates, latency, and backlog +- structured audit trails and request tracing +- tenant-aware access control and row-level security +- per-source rate limiting, retry, and dead-letter handling +- encrypted transport for events and secrets +- retention and archival policies for documents and extracted artifacts + +## Operational & Deployment Model +- Deploy as separate service in Kubernetes / container platform. +- Use CPU/memory autoscaling based on queue backlog and worker utilization. +- Use a shared Postgres instance with separate schema; apply connection pooling and statement timeout defaults. +- Use a broker-backed queue or Kafka topic for work distribution. +- Define self-healing: readiness fails when Postgres, auth, search, or event broker are unavailable. +- Use `RUNBOOK.md` to capture run, rollback, and incident steps. + +## Resilience & Failure Handling +- Ingestion and extraction are async by default; use queues to decouple API from work. +- External-source calls are wrapped in retries with exponential backoff, jitter, and source-specific rate limits. +- Failed ingestion chunks move to dead-letter storage with failure reason and retry count. +- Search indexing uses bulk upsert and retries with delegation to the search service. +- Extraction failures are recorded per-document and do not block unrelated jobs. +- Provide manual retry and resume APIs for failed ingestion and extraction jobs. + +## Observability +- Emit metrics: + - ingestion jobs created, succeeded, failed + - documents ingested, processed, indexed + - extraction latency and queue wait time + - dead-letter count and retry rates + - search upsert success/failure +- Expose health endpoints: + - `/healthz` (live) + - `/readyz` (ready) + - `/status` (dependency overview) +- Correlate logs with `request_id`, `tenant_id`, `job_id`, and `document_id`. +- Use structured logging and trace context for all external calls. +- Define alert conditions for service errors, broker backlog, Postgres pool exhaustion, and external source failures. + +## Data Governance +- Persist only the literature and extracted artifacts required for search and graph workflows. +- Support configurable retention policies for raw text, metadata, and entity/relationship artifacts. +- Provide audit records for ingestion actions, duplicate resolutions, and event publications. +- Support redaction of sensitive identifiers before indexing or graph publication. + +## Security Model +- Authenticate every request with `Authorization: Bearer `. +- Accept only validated JWTs signed by the trusted auth issuer or verified via auth-service introspection. +- Derive `organization_id`, `workspace_id`, `project_id`, and roles from auth claims. +- Enforce tenant isolation with query-level filters and Postgres RLS. +- Encrypt all service-to-service traffic and event transport. +- Limit sensitive tenant metadata in event payloads and use topic-level access control. +- Enforce least privilege on service credentials and external-source keys. + +## Tenant Isolation +- All owned tables include `organization_id`, `workspace_id`, and `project_id`. +- Authorization decisions are enforced inside the service, regardless of gateway behavior. +- Search and graph integrations propagate tenant metadata explicitly. +- Postgres session variables and RLS policies are used where supported. + +## Search Contract Clarification +- `services/search` remains the canonical search layer. +- Literature service calls: + - `GET /api/v1/search?q=&scope=literature` for keyword search + - `POST /api/v1/search` with payload `{ query, embedding, filters, tenant }` for hybrid search + - `POST /api/v1/search/index` or equivalent for document embedding upsert if supported by search service +- Search upsert is idempotent and scoped by `document_id`, `organization_id`, and `workspace_id`. +- If the search service cannot accept direct upsert, literature must publish an indexing event instead. + +## Knowledge Graph Contract +- `services/kg` remains graph persistence owner. +- Literature publishes versioned, typed events for candidate entities and relationships. +- Event consumers must handle idempotent delivery and partial ordering. +- The literature service may support a synchronous import endpoint only as fallback, but primary integration uses event-driven exchange. + +## Event Transport & Fallback +- Preferred: Kafka topics with encryption, authentication, and authorization. +- Fallback: durable internal queue with persistent retry and dead-letter storage. +- Events are typed and versioned with schema fields: + - `event_id` + - `event_type` + - `version` + - `tenant` metadata + - `document_id` + - `payload` + - `timestamp` + - `trace_id` +- Event payloads explicitly omit raw document body unless required. + +## Data Flow Diagram +```mermaid +flowchart LR + subgraph Auth + A[BetterAuth JWT] + end + + subgraph Literature Service + I[Ingestion API] + W[Worker Queue] + D[Document Store] + E[Extraction Service] + Q[Deduplication] + X[Indexing Service] + A2[Audit/Dead-letter] + end + + subgraph Search Service + S[OpenSearch + pgvector] + end + + subgraph Knowledge Graph + KG[Neo4j Graph Service] + end + + A --> I + I --> W + W --> D + D --> E + E --> Q + Q --> X + X --> S + E --> KG + E --> A2 +``` + +## Sequence Diagram +```mermaid +sequenceDiagram + participant User + participant Gateway + participant Literature + participant Broker + participant Search + participant KG + participant Auth + + User->>Gateway: POST /api/v1/ingestion + Gateway->>Auth: validate JWT + Auth-->>Gateway: tenant claims + Gateway->>Literature: create ingestion job + Literature->>Broker: enqueue ingestion work + Broker->>Literature: delivery to worker + Literature->>Literature: persist job + document record + Literature->>Search: request indexing contract / event + Literature->>KG: publish relationship event + Search-->>Literature: ack + KG-->>Literature: ack + Literature-->>Gateway: job accepted + Gateway-->>User: 202 Accepted +``` + +## Compliance with Existing Repo +- Keeps `services/literature` as the ownership boundary. +- Reuses `services/search` vector store and `services/auth` tenant model. +- Avoids adding new Neo4j tables; graph ingestion is event-driven and handled by `services/kg`. +- Avoids duplicating auth APIs by using the same token model. +- Uses the existing `services/literature` endpoint names and extends them rather than replacing. +- Adds production-grade operational, security, and integration contracts needed for implementation. diff --git a/DATABASE_SCHEMA.md b/DATABASE_SCHEMA.md new file mode 100644 index 0000000..b56fb21 --- /dev/null +++ b/DATABASE_SCHEMA.md @@ -0,0 +1,255 @@ +# Literature Intelligence Database Schema v2 + +## Version 2 Summary +This update resolves tenant isolation and scalability blockers with explicit chunked storage, complete tenant fields, and operational schema requirements. + +## Principles +- Use Postgres for ingestion metadata, document state, and extraction artifacts. +- Reuse existing auth tables from `services/auth`; do not duplicate user/org/workspace data. +- Reuse `services/search.document_embeddings` for vector storage instead of creating a separate vector table. +- Keep literature data tenant-scoped via `organization_id`, `workspace_id`, and `project_id`. +- Avoid Neo4j schema ownership in this service. +- Support large documents and chunked storage for searchable content. + +## Shared Naming and Tenant Rules +- All tables include `organization_id`, `workspace_id`, and `project_id` unless otherwise noted. +- Tenant values are derived from auth claims and enforced by application and RLS. +- Use composite indexes that include tenant columns for filtering and uniqueness. + +## Tables Owned by Literature Intelligence + +### `ingestion_sources` +Tracks configured ingestion sources and source-level metadata. + +- `id UUID PRIMARY KEY` +- `source_type TEXT NOT NULL` -- `pubmed`, `biorxiv`, `medrxiv`, `patent`, `conference`, `custom` +- `name TEXT NOT NULL` +- `config JSONB` +- `enabled BOOLEAN NOT NULL DEFAULT TRUE` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_ingestion_sources_org_workspace` +- `idx_ingestion_sources_source_type` + +### `ingestion_jobs` +Tracks ingestion jobs and lifecycle state. + +- `id UUID PRIMARY KEY` +- `source_id UUID REFERENCES ingestion_sources(id)` +- `source_type TEXT NOT NULL` +- `query TEXT NOT NULL` +- `source_document_id TEXT NULL` +- `source_url TEXT NULL` +- `status TEXT NOT NULL` -- `queued`, `running`, `completed`, `failed`, `cancelled` +- `document_count INTEGER DEFAULT 0` +- `processed_count INTEGER DEFAULT 0` +- `failed_count INTEGER DEFAULT 0` +- `pending_count INTEGER DEFAULT 0` +- `error_message TEXT NULL` +- `last_error TEXT NULL` +- `backoff_until TIMESTAMPTZ NULL` +- `dead_letter_count INTEGER DEFAULT 0` +- `started_at TIMESTAMPTZ NULL` +- `completed_at TIMESTAMPTZ NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_ingestion_jobs_org_workspace` +- `idx_ingestion_jobs_status` +- `idx_ingestion_jobs_source_type` + +### `documents` +Stores ingested documents, canonicalization, and processing state. + +- `id UUID PRIMARY KEY` +- `source_type TEXT NOT NULL` +- `source_document_id TEXT NOT NULL` +- `source_url TEXT NULL` +- `title TEXT NOT NULL` +- `abstract TEXT NULL` +- `doi TEXT NULL` +- `publication_date DATE NULL` +- `authors TEXT[] NULL` +- `summary TEXT NULL` +- `citation_count INTEGER DEFAULT 0` +- `status TEXT NOT NULL` -- `new`, `processing`, `processed`, `failed` +- `canonical_document_id UUID NULL` +- `duplicate_group TEXT NULL` +- `ingestion_job_id UUID REFERENCES ingestion_jobs(id)` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Unique constraints: +- `(source_type, source_document_id, organization_id, workspace_id)` + +Indexes: +- `idx_documents_org_workspace` +- `idx_documents_source_document_id` +- `idx_documents_doi` +- `idx_documents_status` +- `idx_documents_canonical_document_id` + +### `document_text_chunks` +Stores document text in chunked form for large content. + +- `id UUID PRIMARY KEY` +- `document_id UUID REFERENCES documents(id) ON DELETE CASCADE` +- `chunk_index INTEGER NOT NULL` +- `content TEXT NOT NULL` +- `chunk_hash TEXT NOT NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_document_text_chunks_doc` +- `idx_document_text_chunks_org_workspace` + +### `document_metadata` +Stores extracted enrichment details and structured metadata. + +- `document_id UUID PRIMARY KEY REFERENCES documents(id)` +- `metadata JSONB NOT NULL` +- `paragraphs JSONB NULL` +- `figures JSONB NULL` +- `tables JSONB NULL` +- `references JSONB NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_document_metadata_org_workspace` + +### `extracted_entities` +Stores extracted entities for documents. + +- `id UUID PRIMARY KEY` +- `document_id UUID REFERENCES documents(id) ON DELETE CASCADE` +- `type TEXT NOT NULL` +- `text TEXT NOT NULL` +- `canonical_id TEXT NULL` +- `confidence REAL NOT NULL` +- `span JSONB NULL` +- `source TEXT NOT NULL` +- `schema TEXT NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_extracted_entities_document_id` +- `idx_extracted_entities_type` +- `idx_extracted_entities_canonical_id` +- `idx_extracted_entities_org_workspace` + +### `extracted_relationships` +Stores relationship candidates extracted from literature. + +- `id UUID PRIMARY KEY` +- `document_id UUID REFERENCES documents(id) ON DELETE CASCADE` +- `from_entity_id UUID REFERENCES extracted_entities(id)` +- `to_entity_id UUID REFERENCES extracted_entities(id)` +- `type TEXT NOT NULL` +- `confidence REAL NOT NULL` +- `evidence TEXT NULL` +- `metadata JSONB NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_extracted_relationships_document_id` +- `idx_extracted_relationships_type` +- `idx_extracted_relationships_org_workspace` + +### `citation_edges` +Stores extracted citation relationships from one document to another. + +- `id UUID PRIMARY KEY` +- `document_id UUID REFERENCES documents(id) ON DELETE CASCADE` +- `cited_source TEXT NOT NULL` +- `cited_doi TEXT NULL` +- `confidence REAL NOT NULL` +- `metadata JSONB NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_citation_edges_document_id` +- `idx_citation_edges_org_workspace` + +### `document_processing_status` +Tracks ingestion/extraction/indexing progress for documents. + +- `document_id UUID PRIMARY KEY REFERENCES documents(id) ON DELETE CASCADE` +- `stage TEXT NOT NULL` -- `ingestion`, `deduplication`, `extraction`, `indexing` +- `status TEXT NOT NULL` -- `pending`, `running`, `completed`, `failed` +- `error_message TEXT NULL` +- `retry_count INTEGER DEFAULT 0` +- `dead_letter BOOLEAN NOT NULL DEFAULT FALSE` +- `last_updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` +- `organization_id UUID NOT NULL` +- `workspace_id UUID NULL` +- `project_id UUID NULL` + +Indexes: +- `idx_document_processing_status_org_workspace` + +### `audit_events` +Tracks important actions for governance and audit. + +- `id UUID PRIMARY KEY` +- `event_type TEXT NOT NULL` +- `entity_type TEXT NOT NULL` +- `entity_id UUID NULL` +- `tenant JSONB NOT NULL` +- `actor_id UUID NULL` +- `payload JSONB NULL` +- `created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()` + +Indexes: +- `idx_audit_events_entity` +- `idx_audit_events_created_at` + +## Reuse Existing Tables +- `services/auth` tables for users, organizations, workspaces, roles, sessions. +- `services/search.document_embeddings` for vector search and embedding storage. + +If literature generates embeddings, it must upsert through the search service or publish metadata events rather than creating a parallel vector store. + +## Row Level Security Strategy +- Use `organization_id`, `workspace_id`, and `project_id` on every owned table. +- Adopt RLS policies where supported and enforce tenant filters in service queries. +- Use a session-level tenant context variable if repository patterns support it. +- Add explicit tenant constraints for duplicates and ingestion jobs. + +## Scalability Notes +- Use `document_text_chunks` for large papers and long-form content. +- Avoid storing monolithic `body_text` on `documents` for large articles. +- Use bulk insert patterns for chunk storage and metadata writes. +- Index tenant-filtered fields for high-cardinality query performance. + +## Notes +- No Neo4j tables are owned by this service. +- The service keeps document and extraction artifacts separate from search storage. +- This design assumes a shared Postgres instance with separate schema and appropriate resource quotas. diff --git a/INTEGRATION_MAP.md b/INTEGRATION_MAP.md new file mode 100644 index 0000000..55520ce --- /dev/null +++ b/INTEGRATION_MAP.md @@ -0,0 +1,100 @@ +# Literature Intelligence Integration Map v2 + +## Version 2 Summary +This update makes integration contracts explicit, adds event schemas, and resolves ambiguity on search and graph ownership. + +## Existing Repository Components Reused +- `services/auth` for authentication, tenant identity, and token validation. +- `services/search` for shared keyword and vector search, plus embedding storage. +- `services/kg` for knowledge graph event consumption, node/edge reconciliation, and graph persistence. +- `apps/api-gateway` for central routing, request validation, and rate limiting. +- `apps/ai-services` for optional LLM-based summarization and extraction. + +## Authentication Integration +- Input: `Authorization: Bearer `. +- Validate token using `services/auth` public keys or introspection API. +- Extract claims: `user_id`, `organization_id`, `workspace_id`, `project_id`, `roles`, `permissions`. +- Tenant claims are authoritative and cannot be overridden by client payload. +- The literature service enforces tenant authorization independently of the gateway. + +## Search Integration +- `services/search` is the canonical search layer. +- Literature uses search for: + - keyword search + - hybrid semantic search + - embedding upsert/indexing +- Contract: + - `GET /api/v1/search?q=...&scope=literature&organization_id=...&workspace_id=...` + - `POST /api/v1/search` with payload `{ query, embedding, filters, tenant }` + - `POST /api/v1/search/index` with payload `{ document_id, tenant, fields, embedding, chunks }` +- Upsert behavior is idempotent and scoped by `document_id`, `organization_id`, and `workspace_id`. +- If direct upsert is unavailable, literature publishes `document.indexed` events instead of writing directly. +- Ensure the search service enforces tenant scoping on ingested documents. + +## Knowledge Graph Integration +- Literature does not write to Neo4j directly. +- Primary integration is event-driven with typed relationship candidate events. +- Event payload contract includes: + - `event_id`, `event_type`, `version`, `tenant`, `document_id`, `entities`, `relationships`, `evidence`, `created_at`, `trace_id` +- Graph service must handle idempotent event consumption and partial ordering. +- Synchronous graph import is allowed only as fallback and must be clearly documented. +- Tenant metadata is included in event headers and payload. + +## Event Transport +- Preferred transport: Kafka with secure authentication, encryption, and ACLs. +- Topics: + - `literature.document.ingested.v1` + - `literature.document.processed.v1` + - `literature.entities.extracted.v1` + - `literature.relationships.extracted.v1` + - `literature.document.deduplicated.v1` + - `literature.document.indexed.v1` +- Events are versioned with `v1` suffixes. +- Payloads omit raw body text unless absolutely needed. +- Security: + - encrypt events in transit and at rest + - enforce topic access controls + - limit sensitive metadata in published events +- Fallback: durable internal queue exposed via `dead_letter` and retry workers. + +## External Source Integration +- Support adapters for: + - PubMed + - PMC + - bioRxiv + - medRxiv + - patents + - conference abstracts + - company reports / websites +- Each adapter implements source-specific throttling, retry, and error handling. +- Fetchers record source health and rate-limit state. +- External source failures do not block unrelated ingestion jobs. + +## Gateway and Service Trust Model +- API gateway performs initial request routing and auth validation. +- Literature service performs full validation again, including tenant claims and request integrity. +- The gateway may apply rate limiting and basic protections, but business authorization is enforced within literature. + +## Integration Matrix + +| Source | Literature Service | Existing Service | Role | +|---|---|---|---| +| Auth | validates token and tenant claims | `services/auth` | Authentication + tenant metadata +| Search | sends query and indexing requests | `services/search` | Keyword + semantic search, embedding storage +| Knowledge Graph | emits candidate events | `services/kg` | Graph ingestion and persistence +| API Gateway | routes and forwards validated requests | `apps/api-gateway` | routing, rate limiting, auth boundary +| LLM / Prompt | requests summarization/extraction | `apps/ai-services` | model orchestration and prompt execution + +## Integration Notes +- `services/literature` owns ingestion/extraction state. +- `services/search` owns search index and embedding state. +- `services/kg` owns graph persistence and reconciliation. +- Event payloads include tenant metadata for both search and KG integration. +- Search and KG integrations must support idempotency and retry semantics. + +## Blockers and Dependencies Resolved +- Event-driven behavior is explicitly defined, not just planned. +- Search integration is defined with exact endpoint expectations. +- Graph event schema and security expectations are stated. +- Tenant metadata and source-specific rate-limiting requirements are documented. +- NLP library inclusion is acknowledged as an implementation dependency. diff --git a/TASK_BREAKDOWN.md b/TASK_BREAKDOWN.md new file mode 100644 index 0000000..0fd0e6d --- /dev/null +++ b/TASK_BREAKDOWN.md @@ -0,0 +1,121 @@ +# Literature Intelligence Task Breakdown v2 + +## Version 2 Summary +This version resolves blocking gaps by adding production tasks for observability, resilience, security, tenant isolation, and explicit integration contracts. + +## Delivery Objectives +- Build a production-ready literature ingestion and extraction service. +- Keep auth, search, and graph responsibilities in existing shared services. +- Provide durable, tenant-aware ingestion with manual retry and dead-letter support. +- Ensure search indexing and graph event integration are explicit, secure, and idempotent. +- Add operational runbook, monitoring, and audit requirements. + +## Implementation Phases + +### Phase 1 — Discovery and Contracts +1. Confirm current repo endpoint and ownership boundaries. +2. Lock down auth/token validation approach and tenant claim propagation. +3. Define exact search and graph integration contracts. +4. Define event transport fallback and schema requirements. + +### Phase 2 — Service Structure, Schema, and Deployment +1. Build the literature service folder structure with core, routers, services, integrations, and workers. +2. Define tenant-aware Postgres schema with chunked text storage and audit tables. +3. Add migration helpers and connection pool settings. +4. Add containerization and deployment configuration for Kubernetes / container platform. + +### Phase 3 — API and Router Design +1. Extend ingestion endpoints with retry, dead-letter, and operational fields. +2. Add paper metadata, extraction results, and duplicate resolution APIs. +3. Add health and readiness endpoints. +4. Add audit and status endpoints. + +### Phase 4 — Ingestion Pipeline Implementation +1. Build source adapters for PubMed, PMC, bioRxiv, medRxiv, patents, conference abstracts, and custom imports. +2. Implement fetcher rate limiting, retry/backoff, and source health metrics. +3. Persist documents and metadata incrementally using chunked storage. +4. Emit ingestion events and record audit entries. + +### Phase 5 — Extraction and Deduplication +1. Implement extraction workers with NER, relation extraction, citation extraction, and summarization. +2. Store artifacts in extracted entities, relationships, citations, and document metadata. +3. Support duplicate detection and canonicalization workflows. +4. Emit extraction and deduplication events and update processing status. + +### Phase 6 — Search and Indexing Integration +1. Integrate with `services/search` via explicit query and index/upsert contracts. +2. Generate embeddings, if desired, and upsert via search service or events. +3. Use bulk indexing and handle backpressure from search. +4. Emit `document.indexed` events and audit indexing actions. + +### Phase 7 — Knowledge Graph Integration +1. Publish versioned relationship candidate events to `services/kg`. +2. Include tenant metadata and evidence in event payloads. +3. Ensure graph service supports idempotent consumption. +4. Only use synchronous KG import as fallback. + +### Phase 8 — Security, Tenant Isolation, and Governance +1. Enforce auth validation and tenant isolation on every request. +2. Apply Postgres RLS or tenant filters across all literature tables. +3. Encrypt event transport and service-to-service traffic. +4. Add audit events, retention policies, and data governance controls. + +### Phase 9 — Observability and Operations +1. Implement structured logs with `request_id`, `tenant_id`, worker context, and trace IDs. +2. Expose metrics for ingestion, extraction, indexing, retries, and dead-letter counts. +3. Add alerts for backlog growth, failed ingestion rate, search upsert failures, and service dependencies. +4. Create `RUNBOOK.md` with deploy, rollback, incident, and scaling guidance. + +## New Components to Build +- `services/literature/app/integrations/external_sources.py` +- `services/literature/app/services/ingestion_service.py` +- `services/literature/app/services/extraction_service.py` +- `services/literature/app/services/indexing_service.py` +- `services/literature/app/services/retry_service.py` +- `services/literature/app/workers/*.py` +- `services/literature/app/models/db_models.py` +- `services/literature/app/models/schemas.py` +- `services/literature/app/models/events.py` +- `services/literature/app/core/events.py` +- `services/literature/app/core/tenant.py` +- `services/literature/app/core/auditing.py` +- `services/literature/RUNBOOK.md` + +## Existing Components to Extend +- `services/literature/app/main.py` +- `services/literature/app/core/config.py` +- `services/search` hybrid search and indexing contract +- `services/auth` token and tenant model +- `services/kg` graph ingestion and event schema + +## Production Requirements Added +- explicit health/readiness endpoints +- retry and dead-letter workflow endpoints +- chunked document storage for large text +- audit events for operational governance +- tenant-aware indexes and RLS support +- search contract, graph event contract, and fallback queue transport +- runbook for deploy/incident response + +## Risks and Assumptions +- `services/literature` is currently a stub and requires full implementation. +- Kafka is not currently implemented in the repo; event transport must support a queue fallback. +- NLP/ML dependencies are missing and must be added before extraction work begins. +- `services/search` owns embedding storage and upsert semantics must be agreed. +- Graph ingestion remains the responsibility of `services/kg`. + +## Blocker Resolution Summary +Resolved the previous blockers by: +- defining event topics and payload schema, +- making search upsert contract explicit, +- adding data governance and audit support, +- requiring tenant metadata across all owned tables, +- adding external source retry and dead-letter handling, +- specifying operational health and scaling requirements. + +If future implementation hits a blocker, the report should include: +- completed tasks +- pending work +- blocker root cause +- reused existing components +- required dependency or contract diff --git a/compose_config.txt b/compose_config.txt new file mode 100644 index 0000000..c954274 Binary files /dev/null and b/compose_config.txt differ diff --git a/compose_services.txt b/compose_services.txt new file mode 100644 index 0000000..69992cb Binary files /dev/null and b/compose_services.txt differ diff --git a/services/full_audit_output.txt b/services/full_audit_output.txt new file mode 100644 index 0000000..4faf8c4 Binary files /dev/null and b/services/full_audit_output.txt differ diff --git a/services/literature/API_REFERENCE.md b/services/literature/API_REFERENCE.md new file mode 100644 index 0000000..e197205 --- /dev/null +++ b/services/literature/API_REFERENCE.md @@ -0,0 +1,160 @@ +# API Reference + +This file documents the Literature service API surface. + +## Health and Observability + +### GET /healthz + +- Description: Basic liveness check. +- Response: `200` +- Body: `{ "status": "ok", "service": "literature" }` + +### GET /health + +- Description: API health endpoint. +- Response: `200` +- Body: `{ "status": "ok", "service": "literature" }` + +### GET /live + +- Description: Liveness indicator. +- Response: `200` +- Body: `{ "status": "ok", "service": "literature", "live": true }` + +### GET /ready + +- Description: Readiness check; validates PostgreSQL, Search service, KG service, and orchestrator readiness. +- Response: `200` +- Body: + - `status`: `ok` or `fail` + - `ready`: boolean + - `dependencies`: dependency status details + - `metrics`: uptime, parser, and orchestrator metrics + +### GET /metrics + +- Description: JSON metrics payload. +- Response: `200` +- Body: service-level metrics and uptime. + +### GET /metrics/prometheus + +- Description: Prometheus exposition format. +- Response: `200` +- Content-Type: `text/plain` + +### GET /api/v1/health + +- Description: Versioned health endpoint. +- Response: `200` +- Body: `{ "status": "ok", "service": "literature" }` + +## Document APIs + +### POST /api/v1/documents/parse + +- Description: Parse document content and extract structured metadata. +- Security: Bearer JWT required. +- Request body: + - `format`: `pdf`, `xml`, `html`, `nxml`, or `jats` + - `content`: document content as string +- Response body: + - `metadata`: parsed `DocumentMetadata` + - `duplicate`: boolean + - `metrics`: parser metric snapshot +- Error codes: + - `400` invalid content or parse failure + - `409` duplicate document detected + +## NLP APIs + +### POST /api/v1/nlp/process + +- Description: Process parsed document metadata through the biomedical NLP pipeline. +- Security: Bearer JWT required. +- Request body: `DocumentMetadata` JSON structure. +- Response body: + - `document_id` + - `sentences` + - `tokens` + - `detected_entities` + - `entities` + - `relationships` + - `summary` + - `processing_metadata` + - `execution_metrics` +- Error codes: + - `400` invalid document metadata or pipeline error + +## Ingestion APIs + +### POST /api/v1/ingestion + +- Description: Create and enqueue a literature ingestion job. +- Security: Bearer JWT required. +- Request body: + - `source`: `pubmed`, `biorxiv`, `medrxiv`, `patent`, or `conference` + - `query`: search or ingestion query string + - `schedule`: optional cron schedule expression +- Response: persisted ingestion job details. + +### GET /api/v1/ingestion/{job_id} + +- Description: Retrieve ingestion job state. +- Security: Bearer JWT required. +- Response: ingestion job details. + +### POST /api/v1/ingestion/{job_id}/trigger + +- Description: Trigger a queued ingestion job immediately. +- Security: Bearer JWT required. +- Response: updated ingestion job state. + +### POST /api/v1/ingestion/{job_id}/retry + +- Description: Retry a failed ingestion job. +- Security: Bearer JWT required. +- Response: updated ingestion job state. + +### POST /api/v1/ingestion/{job_id}/cancel + +- Description: Cancel an ingestion job. +- Security: Bearer JWT required. +- Response: updated ingestion job state. + +### GET /api/v1/ingestion/{job_id}/dead-letter + +- Description: Retrieve dead-letter items associated with a job. +- Security: Bearer JWT required. +- Response body: + - `items`: array of dead-letter records + - `total`: count + +## Paper APIs + +### GET /api/v1/papers + +- Description: List ingested papers. +- Security: Bearer JWT required. +- Query parameters: + - `page`: page number (default 1) + - `page_size`: page size (default 20, max 100) +- Response body: + - `items`: list of paper summaries + - `total`: total count + - `page`: current page + - `pageSize`: page size + +### GET /api/v1/papers/{paper_id} + +- Description: Retrieve metadata for a single paper. +- Security: Bearer JWT required. +- Response body: `Paper` metadata. +- Error codes: + - `404` paper not found + +## Security + +Protected endpoints require `Authorization: Bearer `. +Tokens are validated using the `JWT_SECRET` setting and HS256. diff --git a/services/literature/COMMANDS.md b/services/literature/COMMANDS.md new file mode 100644 index 0000000..e24f5b9 --- /dev/null +++ b/services/literature/COMMANDS.md @@ -0,0 +1,63 @@ +# Commands + +Common commands for development, validation, and deployment. + +## Run locally + +```bash +cd services/literature +python -m uvicorn app.main:app --host 0.0.0.0 --port 8082 +``` + +## Run tests + +```bash +cd services/literature +python -m pytest -q +``` + +## Static analysis + +```bash +cd services/literature +python -m ruff check app +python -m ruff format --check app +python -m mypy app --ignore-missing-imports +``` + +## Docker + +Build the image: + +```bash +docker build -t ai-rxos-literature -f services/literature/Dockerfile services/literature +``` + +Run the container: + +```bash +docker run -p 8082:8082 \ + -e DATABASE_URL="postgresql://user:password@postgres:5432/ai_rxos" \ + -e JWT_SECRET="" \ + -e SEARCH_SERVICE_URL="http://search:8084" \ + -e KG_SERVICE_URL="http://kg:8083" \ + -e LLMWIKI_SERVICE_URL="http://llmwiki:8086" \ + ai-rxos-literature +``` + +## Readiness and health + +```bash +curl http://localhost:8082/health +curl http://localhost:8082/ready +curl http://localhost:8082/metrics/prometheus +``` + +## JWT generation + +This service expects HS256 JWT tokens signed with `JWT_SECRET`. +Use your preferred JWT tooling or helper library to create tokens for local testing. + +## Troubleshooting logs + +Inspect container logs or service terminal output for startup errors and lifecycle issues. diff --git a/services/literature/DEPLOYMENT.md b/services/literature/DEPLOYMENT.md new file mode 100644 index 0000000..d8a0e7e --- /dev/null +++ b/services/literature/DEPLOYMENT.md @@ -0,0 +1,65 @@ +# Deployment Guide + +This document describes how to deploy the Literature service in production. + +## Container deployment + +Build the Docker image from the repository root: + +```bash +docker build -t ai-rxos-literature -f services/literature/Dockerfile services/literature +``` + +Run the container: + +```bash +docker run -p 8082:8082 \ + -e DATABASE_URL="postgresql://user:password@postgres:5432/ai_rxos" \ + -e JWT_SECRET="" \ + -e SEARCH_SERVICE_URL="http://search:8084" \ + -e KG_SERVICE_URL="http://kg:8083" \ + -e LLMWIKI_SERVICE_URL="http://llmwiki:8086" \ + ai-rxos-literature +``` + +The container exposes port `8082` and includes a healthcheck on `/health`. + +## Environment requirements + +- PostgreSQL instance accessible via `DATABASE_URL` +- Search service at `SEARCH_SERVICE_URL` +- KG service at `KG_SERVICE_URL` +- LLM Wiki service at `LLMWIKI_SERVICE_URL` +- JWT secret set in `JWT_SECRET` + +## Startup behavior + +On startup, the service: + +- initializes an asyncpg PostgreSQL connection pool +- ensures the `literature_papers` and `literature_ingestion_jobs` tables exist +- starts the ingestion orchestrator worker and scheduler + +If startup fails to connect to PostgreSQL, the service continues in degraded mode but marks readiness accordingly. + +## Health checks + +The Docker healthcheck uses `/health` and expects a `200` status. + +For production orchestration, use `/ready` to verify dependency readiness. + +## Runtime ports + +- Application API: `8082` +- Health and metrics: `8082` + +## Authentication + +All protected endpoints require `Authorization: Bearer ` with a JWT signed using `JWT_SECRET`. + +## Production best practices + +- Do not use default secret values from `app/core/config.py` in production. +- Use secure secrets for `JWT_SECRET` and service credentials. +- Ensure dependent services are reachable and have their own health checks. +- Monitor `/metrics/prometheus` for end-to-end production telemetry. diff --git a/services/literature/Dockerfile b/services/literature/Dockerfile index 93719d4..c478521 100644 --- a/services/literature/Dockerfile +++ b/services/literature/Dockerfile @@ -8,7 +8,7 @@ WORKDIR /app FROM base AS deps -COPY apps/knowledge-service/requirements*.txt ./ +COPY services/literature/requirements.txt ./ RUN pip install \ --upgrade pip \ @@ -19,14 +19,20 @@ RUN pip install \ FROM deps AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + RUN useradd --create-home --uid 1000 rxos -COPY apps/knowledge-service/app ./app +COPY services/literature/app ./app USER rxos -EXPOSE 8091 +EXPOSE 8082 + +ENV PORT=8082 -ENV PORT=8091 +HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \ + CMD curl -fsS http://127.0.0.1:8082/health || exit 1 CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT}"] \ No newline at end of file diff --git a/services/literature/ENVIRONMENT.md b/services/literature/ENVIRONMENT.md new file mode 100644 index 0000000..c24092c --- /dev/null +++ b/services/literature/ENVIRONMENT.md @@ -0,0 +1,42 @@ +# Environment Configuration + +This document describes environment variables used by the Literature service. + +## Required production variables + +- `DATABASE_URL` — PostgreSQL connection string. +- `JWT_SECRET` — JWT signing secret for bearer authentication. +- `SEARCH_SERVICE_URL` — base URL for downstream Search service. +- `KG_SERVICE_URL` — base URL for downstream Knowledge Graph service. +- `LLMWIKI_SERVICE_URL` — base URL for downstream LLM Wiki service. + +## Optional and default variables + +These defaults are defined in `app/core/config.py` and are suitable for local development only. + +- `ENVIRONMENT` — defaults to `development` +- `LOG_LEVEL` — defaults to `info` +- `REDIS_URL` — defaults to `redis://redis:6379/0` +- `NEO4J_URI` — defaults to `bolt://neo4j:7687` +- `NEO4J_USER` — defaults to `neo4j` +- `NEO4J_PASSWORD` — defaults to `changeme_neo4j` +- `OPENSEARCH_URL` — defaults to `http://opensearch:9200` +- `SEARCH_SERVICE_TIMEOUT_SECONDS` — defaults to `5` +- `SEARCH_SERVICE_MAX_RETRIES` — defaults to `3` +- `KG_SERVICE_TIMEOUT_SECONDS` — defaults to `5` +- `KG_SERVICE_MAX_RETRIES` — defaults to `3` +- `LLMWIKI_SERVICE_TIMEOUT_SECONDS` — defaults to `5` +- `LLMWIKI_SERVICE_MAX_RETRIES` — defaults to `3` +- `PUBMED_BASE_URL` — defaults to the NCBI PubMed API base URL +- `PMC_BASE_URL` — defaults to the NCBI PMC API base URL +- `CLINICALTRIALS_BASE_URL` — defaults to ClinicalTrials.gov API endpoint +- `BIORXIV_BASE_URL` — defaults to bioRxiv API endpoint +- `MEDRXIV_BASE_URL` — defaults to medRxiv API endpoint +- `CORS_ALLOWED_ORIGINS` — defaults to `http://localhost:3000` + +## Notes + +- `JWT_SECRET` must be unique and kept confidential in production. +- `DATABASE_URL` must point to a PostgreSQL instance with the `ai_rxos` schema or sufficient privileges to create it. +- `SEARCH_SERVICE_URL`, `KG_SERVICE_URL`, and `LLMWIKI_SERVICE_URL` must be reachable by the deployed container. +- `ENVIRONMENT` may be set to `test` during CI to bypass real database startup in startup lifespan. diff --git a/services/literature/IMPLEMENTATION.md b/services/literature/IMPLEMENTATION.md new file mode 100644 index 0000000..0c74400 --- /dev/null +++ b/services/literature/IMPLEMENTATION.md @@ -0,0 +1,91 @@ +# Implementation Overview + +The AI-RxOS Literature service is implemented as a FastAPI application that ingests literature content, executes biomedical NLP, and hands off structured outputs to search, knowledge graph, and LLM Wiki services. + +## Architecture + +The service is organized into distinct functional layers: + +- `app/core/` + - `config.py` loads runtime settings and environment defaults. + - `security.py` enforces JWT bearer authentication for protected endpoints. + - `lifespan.py` manages startup and shutdown behavior, including PostgreSQL initialization and orchestrator lifecycle. + +- `app/connectors/` + - Implements source connectors for PubMed, PMC, ClinicalTrials, bioRxiv, medRxiv, patents, and company websites. + - `registry.py` maps source names to connector factories. + - `http_client.py` centralizes HTTP client behavior for connector communication. + +- `app/parsing/` + - `parser.py` extracts canonical metadata from HTML, XML, and PDF inputs. + - `duplicates.py` detects duplicate documents by fingerprint. + - `metrics.py` tracks parser-specific metrics. + +- `app/nlp/` + - `pipeline.py` composes sentence segmentation, tokenization, entity extraction, normalization, ontology mapping, relationship extraction, summarization, and confidence scoring. + - `sentence_segmenter.py`, `tokenizer.py`, `entity_extractor.py`, `entity_normalizer.py`, `ontology_mapper.py`, `relationship_extractor.py`, `summarizer.py`, and `confidence_scorer.py` implement the processing stages. + +- `app/services/` + - `search_integration.py` sends embedding payloads to the downstream Search service. + - `kg_integration.py` publishes published nodes and relationships to the KG service. + - `llmwiki_integration.py` updates the LLM Wiki service with structured summaries and entity data. + - `evidence_ranking.py` ranks evidence items from parsed and extracted content. + +- `app/orchestrator/` + - `manager.py` maintains ingestion jobs, scheduling, retries, dead-letter handling, and background worker execution. + +- `app/database/` + - `postgres.py` initializes an asyncpg pool and ensures the schema for papers and ingestion jobs. + +- `app/routers/` + - Defines API endpoints for health, documents, NLP, ingestion, and papers. + - Authenticated request handling is applied through `app/core/security.py`. + +- `app/observability/` + - `metrics.py` defines Prometheus counters, histograms, and metrics exposition. + +## Request flow + +1. A request enters through FastAPI in `app/main.py`. +2. Middleware assigns a request ID and trace ID, collects request metrics, and instruments latency and error counts. +3. Routes are handled by `app/routers/*`. +4. For ingestion, jobs are persisted in PostgreSQL and queued by the orchestrator. +5. For parsing, `app/parsing/parser.py` normalizes content and checks duplicates. +6. For NLP processing, `app/nlp/pipeline.py` performs all extraction and summarization. +7. External integrations may hand off results to downstream systems. +8. Metrics and health endpoints expose service status. + +## Data model + +Key Pydantic models are defined in `app/schemas.py` and `app/document_schemas.py`. + +- `DocumentParseRequest` requires `format` and `content`. +- `DocumentParseResponse` returns parsed metadata, duplicate status, and metrics. +- `IngestionRequest` includes source, query, and optional schedule. +- `IngestionJob` describes persisted ingestion job state. +- `Paper` describes stored paper metadata. + +## Orchestrator behavior + +- Jobs are queued or scheduled via `app/orchestrator/manager.py`. +- `schedule` jobs use cron syntax and are rescheduled after completion. +- `retry` increments retry count and applies backoff. +- `cancel` marks jobs as cancelled and persists state. +- Failed jobs populate a dead letter queue and increment dead-letter metrics. + +## Observability + +- Request-level metrics measured in `app/main.py` track: + - total requests + - request errors + - in-flight requests + - request latency +- Pipeline and integration metrics are defined in `app/observability/metrics.py`. +- Prometheus metrics are exposed at `/metrics/prometheus`. + +## Production considerations + +- The lifespan handler ensures the database schema is created on startup. +- Health and readiness checks make the service safe to deploy behind orchestration systems. +- JWT authentication protects all production-facing endpoints. +- Real downstream integration relies on external Search, KG, and LLM Wiki services. diff --git a/services/literature/PRODUCTION_CHECKLIST.md b/services/literature/PRODUCTION_CHECKLIST.md new file mode 100644 index 0000000..0a423e9 --- /dev/null +++ b/services/literature/PRODUCTION_CHECKLIST.md @@ -0,0 +1,57 @@ +# Production Checklist + +Use this checklist before promoting the Literature service to production. + +## Configuration + +- [ ] `JWT_SECRET` is configured in production secrets. +- [ ] `DATABASE_URL` points to the production PostgreSQL instance. +- [ ] `SEARCH_SERVICE_URL`, `KG_SERVICE_URL`, and `LLMWIKI_SERVICE_URL` are set and reachable. +- [ ] `ENVIRONMENT` is set to `production`. +- [ ] `CORS_ALLOWED_ORIGINS` includes trusted frontends. + +## Security + +- [ ] Do not use default secrets from `app/core/config.py`. +- [ ] Verify bearer token authentication on protected routes. +- [ ] Confirm `Authorization: Bearer ` works with the production JWT secret. + +## Health and readiness + +- [ ] `GET /health` returns `200`. +- [ ] `GET /ready` returns `ready: true`. +- [ ] `GET /metrics/prometheus` returns metrics. +- [ ] Docker healthcheck passes and container is healthy. + +## External dependencies + +- [ ] PostgreSQL is accessible and schema initialization succeeds. +- [ ] Search service health endpoint returns `200`. +- [ ] KG service health endpoint returns `200`. +- [ ] LLM Wiki service endpoint is reachable. + +## Observability + +- [ ] Prometheus metrics are scraped successfully. +- [ ] Error and latency metrics are visible. +- [ ] Ingestion metrics appear after job execution. + +## Testing + +- [ ] `python -m pytest -q` passes. +- [ ] `python -m ruff check app` passes. +- [ ] `python -m ruff format --check app` passes. +- [ ] `python -m mypy app --ignore-missing-imports` passes. + +## Deployment + +- [ ] Docker image builds successfully. +- [ ] Container starts and exposes port `8082`. +- [ ] Application logs show successful PostgreSQL and orchestrator startup. + +## Service behavior + +- [ ] Document parsing endpoint handles valid inputs. +- [ ] NLP processing endpoint returns entity and summary payloads. +- [ ] Ingestion jobs can be created, triggered, retried, and cancelled. +- [ ] Dead-letter items are recorded for failed ingestion jobs. diff --git a/services/literature/README.md b/services/literature/README.md index 49cedbd..2f22efc 100644 --- a/services/literature/README.md +++ b/services/literature/README.md @@ -1,9 +1,455 @@ -# literature +# AI-RxOS Literature Service -Part of the AI-RxOS platform. See `/architecture` at the repo root for the -full service contract this implements. Runs on port **8082**. +The Literature service is the Literature Intelligence bounded-context implementation for AI-RxOS. It provides ingestion, parsing, NLP extraction, summarization, evidence ranking, and structured handoff to external downstream services. + +- Service port: **8082** +- Entry point: `app.main:app` +- Runtime: FastAPI + +## Service Purpose + +Prompt 6 status: fully completed for the Literature Intelligence bounded context. The service has been validated through static analysis, unit/integration tests, and live runtime smoke checks. + +The Literature service prepares scientific and clinical literature for downstream knowledge systems by: + +- ingesting content from literature sources and company websites +- parsing structured document metadata +- running biomedical NLP for entity and relationship extraction +- generating structured summaries +- producing deterministic embeddings +- integrating with search, knowledge graph, and LLM Wiki services +- exposing production health, readiness, and Prometheus metrics + +## Architecture Overview + +```mermaid +flowchart TB + A[Connectors] --> B[Parser] + B --> C[NLP Pipeline] + C --> D[Summarization] + C --> E[Entity Extraction] + C --> F[Relationship Extraction] + E --> G[Embedding Generation] + G --> H[Search Integration] + F --> I[Knowledge Graph Integration] + D --> J[LLM Wiki Integration] + E --> K[Evidence Ranking] + H --> L[Search Service] + I --> M[KG Service] + J --> N[LLM Wiki Service] +``` + +### High-level flow + +1. Content is ingested through connectors. +2. Documents are normalized and parsed. +3. NLP performs sentence segmentation, tokenization, entity detection, normalization, mapping, and relationship extraction. +4. Summarization produces structured summaries. +5. Embeddings and external handoff services are prepared. +6. Metrics, health, and readiness are exposed for production observability. + +## Folder Structure + +``` +services/literature/ +├── app/ +│ ├── connectors/ # Source connectors and adapters +│ ├── core/ # Configuration, security, lifespan +│ ├── database/ # PostgreSQL connection and schema management +│ ├── nlp/ # NLP pipeline, summarizer, entity extraction +│ ├── observability/ # Prometheus metrics definitions +│ ├── orchestrator/ # Ingestion job orchestration +│ ├── parsing/ # Document parsing and duplicate detection +│ ├── routers/ # FastAPI route definitions +│ ├── schemas.py # Request/response data models +│ ├── services/ # External integrations (search, KG, LLM Wiki) +│ └── utils/ # Logging and helpers +├── tests/ # Unit and integration tests +├── Dockerfile +└── requirements.txt +``` + +## Supported Literature Sources + +The service supports ingestion from these sources: + +- `pubmed` — PubMed API +- `pmc` — PubMed Central +- `clinicaltrials` — ClinicalTrials.gov +- `biorxiv` — bioRxiv +- `medrxiv` — medRxiv +- `patents` — Patent connector +- `company_website` — Company website HTML ingestion + +## Pipeline + +The Literature NLP pipeline executes the following stages: + +1. Parser: normalize documents and extract structured metadata +2. Sentence segmentation +3. Tokenization +4. Entity extraction +5. Entity normalization and ontology mapping +6. Relationship extraction +7. Summarization +8. Embedding generation +9. Evidence ranking + +```mermaid +flowchart TD + Parser --> Segmentation[Sentence Segmentation] + Segmentation --> Tokenization[Tokenization] + Tokenization --> Entities[Entity Extraction] + Entities --> Normalization[Normalization & Ontology Mapping] + Normalization --> Relationships[Relationship Extraction] + Parser --> Summarizer[Summarization] + Entities --> Embeddings[Embedding Generation] + Embeddings --> Search[Search Integration] + Relationships --> KG[Knowledge Graph Integration] + Entities --> Evidence[Evidence Ranking] + Summarizer --> LLMWiki[LLM Wiki Integration] +``` + +## APIs + +### Health and Observability + +- `GET /healthz` — basic service liveness +- `GET /health` — API health +- `GET /live` — liveness indicator +- `GET /ready` — readiness check including dependencies +- `GET /metrics` — JSON service metrics +- `GET /metrics/prometheus` — Prometheus exposition format +- `GET /api/v1/health` — versioned health endpoint + +### Document and NLP APIs + +- `POST /api/v1/documents/parse` + - Accepts document text content and format + - Returns parsed metadata, duplicate detection status, and parser metrics + +- `POST /api/v1/nlp/process` + - Accepts document metadata + - Returns NLP output including sentences, tokens, entities, relationships, and structured summary + +### Ingestion APIs + +- `POST /api/v1/ingestion` + - Create an ingestion job for a supported source +- `GET /api/v1/ingestion/{job_id}` + - Fetch ingestion job state +- `POST /api/v1/ingestion/{job_id}/trigger` + - Trigger a queued job +- `POST /api/v1/ingestion/{job_id}/retry` + - Retry a failed job +- `POST /api/v1/ingestion/{job_id}/cancel` + - Cancel a job +- `GET /api/v1/ingestion/{job_id}/dead-letter` + - Retrieve dead-letter items + +### Paper APIs + +- `GET /api/v1/papers` — list ingested papers +- `GET /api/v1/papers/{paper_id}` — retrieve a paper record + +## Connectors + +The connector layer handles source-specific ingestion and normalization. + +- `app/connectors/*.py` contain connector implementations. +- `app/connectors/registry.py` maps source keys to connector implementations. +- `app/connectors/http_client.py` provides retry-safe HTTP access. + +### Company Website Connector + +- `app/connectors/company_website.py` +- Fetches HTML from a published company site +- Extracts metadata from `` tags +- Normalizes URL input +- Parses HTML to produce document metadata ready for the parser +- No NLP or KG processing is performed inside the connector + +## Parser + +The parser is responsible for document normalization and metadata extraction: + +- `app/parsing/parser.py` handles HTML/XML parsing and PDF support +- `app/parsing/duplicates.py` detects duplicate documents by fingerprint +- `app/parsing/metrics.py` tracks parser-level metrics +- `app/parsing/__init__.py` exposes parser APIs + +The parser produces canonical metadata such as title, authors, abstract, sections, references, tables, and figures. + +## NLP + +The NLP subsystem is implemented in `app/nlp/` and includes: + +- Sentence segmentation (`SentenceSegmenter`) +- Biomedical tokenization (`BiomedicalTokenizer`) +- Entity extraction (`EntityExtractor`) +- Entity normalization (`EntityNormalizer`) +- Ontology mapping (`OntologyMapper`) +- Confidence scoring (`ConfidenceScorer`) +- Relationship extraction (`RelationshipExtractor`) +- Summarization (`SummarizerService`) + +## Summarization + +Summarization is handled by `app/nlp/summarizer.py`. + +Responsibilities: + +- generate an abstract summary +- extract key findings +- identify clinical relevance +- identify limitations +- produce a structured summary payload + +## Entity Extraction + +Entity extraction is performed in the NLP pipeline by: + +- tokenizing source sentences +- applying rule-based biomedical entity heuristics +- normalizing entity values and mapping them to ontology identifiers +- assigning confidence scores + +## Relationship Extraction + +Relationship extraction is implemented in `app/nlp/relationship_extractor.py`. + +Responsibilities: + +- identify relationships between extracted entities +- infer predicates such as `treats`, `associated_with`, `interacts_with`, and `targets` +- attach provenance and confidence scoring +- capture relationships for downstream KG handoff + +## Embedding Generation + +Embeddings are generated by `app/nlp/embedding_service.py`. + +This service: + +- converts normalized NLP output into deterministic vector representations +- generates embedding batches +- exposes embedding metadata +- does not persist vectors internally + +## Search Integration + +Search integration is implemented in `app/services/search_integration.py`. + +Responsibilities: + +- prepare embedding payloads for the external search service +- submit payloads to `/api/v1/search/index` +- track handoff retries and failure metrics +- remain a thin client, not a search engine itself + +## Knowledge Graph Integration + +Knowledge graph handoff is implemented in `app/services/kg_integration.py`. + +Responsibilities: + +- translate literature entities and relationships into graph payloads +- publish nodes and relationships to external KG service endpoints +- support duplicate suppression and retries +- track KG handoff metrics + +## LLM Wiki Integration + +LLM Wiki integration is implemented in `app/services/llmwiki_integration.py`. + +Responsibilities: + +- package entities, relationships, and structured summaries into an LLM-ready payload +- submit updates to the LLM Wiki service +- retry on transient failures and emit update metrics + +## Documentation files + +This service also includes dedicated documentation for production usage and system understanding: + +- `IMPLEMENTATION.md` +- `TESTING.md` +- `VALIDATION.md` +- `DEPLOYMENT.md` +- `API_REFERENCE.md` +- `ENVIRONMENT.md` +- `TROUBLESHOOTING.md` +- `PRODUCTION_CHECKLIST.md` +- `COMMANDS.md` + +## Evidence Ranking + +Evidence ranking is implemented in `app/services/evidence_ranking.py`. + +Responsibilities: + +- aggregate evidence from entities and relationships +- deduplicate overlapping evidence +- score evidence items using a lightweight ranking strategy +- attach provenance and ranking metadata + +## Monitoring and Observability + +Observability is built with Prometheus-compatible metrics and runtime instrumentation. + +- `app/observability/metrics.py` defines Prometheus counters, gauges, and histograms +- HTTP middleware in `app/main.py` records + - request count + - request latency + - in-flight requests + - error counts +- service-specific metrics track NLP, embedding, search handoff, KG handoff, and evidence ranking + +## Health Endpoints + +Health endpoints expose runtime status and readiness. + +- `/healthz` — basic liveness +- `/health` — service health +- `/live` — liveness indicator +- `/ready` — readiness and dependency status +- `/api/v1/health` — versioned health endpoint + +Readiness includes: + +- PostgreSQL connectivity +- search service availability +- KG service availability +- orchestrator metrics validity + +## Metrics Endpoints + +- `/metrics` — JSON health and metric summary +- `/metrics/prometheus` — Prometheus exposition format + +## Configuration + +Runtime configuration is defined in `app/core/config.py`. + +Key configuration values: + +- `environment` +- `log_level` +- `database_url` +- `redis_url` +- `neo4j_uri`, `neo4j_user`, `neo4j_password` +- `opensearch_url` +- `search_service_url` +- `search_service_timeout_seconds` +- `search_service_max_retries` +- `kg_service_url` +- `kg_service_timeout_seconds` +- `kg_service_max_retries` +- `llmwiki_service_url` +- `llmwiki_service_timeout_seconds` +- `llmwiki_service_max_retries` +- `jwt_secret` +- `pubmed_base_url` +- `pmc_base_url` +- `clinicaltrials_base_url` +- `biorxiv_base_url` +- `medrxiv_base_url` + +## Environment Variables + +The service supports configuration through environment variables matching `app/core/config.py` settings. + +In production, sensitive values must not be checked in or left at defaults. `JWT_SECRET` is required and should be a strong secret value with at least 32 characters. + +Common variables: + +- `ENVIRONMENT` +- `LOG_LEVEL` +- `DATABASE_URL` +- `REDIS_URL` +- `NEO4J_URI` +- `NEO4J_USER` +- `NEO4J_PASSWORD` +- `OPENSEARCH_URL` +- `SEARCH_SERVICE_URL` +- `SEARCH_SERVICE_TIMEOUT_SECONDS` +- `SEARCH_SERVICE_MAX_RETRIES` +- `KG_SERVICE_URL` +- `KG_SERVICE_TIMEOUT_SECONDS` +- `KG_SERVICE_MAX_RETRIES` +- `LLMWIKI_SERVICE_URL` +- `LLMWIKI_SERVICE_TIMEOUT_SECONDS` +- `LLMWIKI_SERVICE_MAX_RETRIES` +- `JWT_SECRET` +- `CORS_ALLOWED_ORIGINS` +- `PUBMED_BASE_URL` +- `PMC_BASE_URL` +- `CLINICALTRIALS_BASE_URL` +- `BIORXIV_BASE_URL` +- `MEDRXIV_BASE_URL` + +## Running Locally + +Install dependencies and start the FastAPI app: + +```bash +cd services/literature +python -m pip install -r requirements.txt +uvicorn app.main:app --reload --host 0.0.0.0 --port 8082 +``` + +The service is available at `http://localhost:8082`. + +## Running with Docker + +Build the Docker image: + +```bash +docker build -f services/literature/Dockerfile -t ai-rxos-literature:latest . +``` + +Run the container: + +```bash +docker run --rm -p 8082:8082 \ + -e DATABASE_URL="postgresql://..." \ + -e SEARCH_SERVICE_URL="http://search:8084" \ + -e KG_SERVICE_URL="http://kg:8083" \ + -e LLMWIKI_SERVICE_URL="http://llmwiki:8086" \ + ai-rxos-literature:latest +``` + +## Testing + +Run the Literature service test suite with pytest: ```bash -pip install -r requirements.txt -uvicorn app.main:app --reload --port 8082 +cd services/literature +python -m pytest -q ``` + +The repository includes targeted tests for connectors, NLP pipeline, ingest orchestration, and external service handoff. + +## Production Deployment + +In production, deploy the service with: + +- PostgreSQL as the persistent store +- External search service available at `SEARCH_SERVICE_URL` +- External KG service available at `KG_SERVICE_URL` +- External LLM Wiki service available at `LLMWIKI_SERVICE_URL` +- Prometheus scraping `/metrics/prometheus` +- Readiness probes pointing to `/ready` + +### Recommended container pattern + +- Build from `services/literature/Dockerfile` +- Configure the service with environment variables +- Expose port `8082` +- Use a process manager or orchestration platform to manage lifecycle and health checks + +## Notes + +- The Literature service is intentionally lightweight and does not implement its own OpenSearch, Neo4j, vector database, or LLM hosting. +- External system integrations are handled through dedicated handoff clients. +- The service is designed for prompt 6 scope: structured literature intelligence, ingestion, extraction, summarization, and integration. diff --git a/services/literature/TESTING.md b/services/literature/TESTING.md new file mode 100644 index 0000000..1490f15 --- /dev/null +++ b/services/literature/TESTING.md @@ -0,0 +1,48 @@ +# Testing Guide + +This service includes unit and integration tests under `tests/`. + +## Test structure + +Key test files: + +- `test_health.py` — liveness and basic health endpoints +- `test_health_readiness.py` — readiness checks and dependency handling +- `test_document_parser.py` — document parsing and duplicate detection +- `test_nlp_pipeline.py` — NLP processing pipeline logic +- `test_connectors.py` — connector registry and source connector behavior +- `test_company_website_connector.py` — company website ingestion connector +- `test_http_client.py` — HTTP client error handling +- `test_orchestrator.py` — ingestion job lifecycle, scheduling, retry, and cancel flows +- `test_observability.py` — metrics registration and snapshot behavior +- `test_prometheus_metrics.py` — Prometheus exposition output +- `test_kg_integration.py` — KG service payload construction and publish semantics +- `test_llmwiki_integration.py` — LLM Wiki payload validation and retry behavior +- `test_evidence_ranking.py` — evidence ranking logic + +## Running tests + +From `services/literature/`: + +```bash +python -m pytest -q +``` + +This command executes the full test suite and will fail the run if any tests fail. + +## Test environment + +- `conftest.py` provides shared fixtures and test configuration. +- The service uses `environment = "test"` in test mode to skip real PostgreSQL startup behavior in the lifespan handler. + +## Recommended validation + +- Run `python -m pytest -q` after any code changes. +- Ensure endpoints and route handlers are covered by tests. +- Verify that authentication failures and invalid payloads are handled correctly. + +## Troubleshooting test failures + +- Check that required dependencies from `requirements.txt` are installed. +- Confirm the selected Python interpreter is 3.12. +- If tests fail due to missing services, use mocks or set `environment` to `test`. diff --git a/services/literature/TROUBLESHOOTING.md b/services/literature/TROUBLESHOOTING.md new file mode 100644 index 0000000..2edd230 --- /dev/null +++ b/services/literature/TROUBLESHOOTING.md @@ -0,0 +1,125 @@ +# Troubleshooting + +This document covers common issues and their resolution for the Literature service. + +## Service fails to start + +### Symptom + +Service container crashes or fails during startup. + +### Diagnosis + +- Check `DATABASE_URL` and PostgreSQL availability. +- Confirm the `JWT_SECRET` environment variable is present. +- Review container logs for `PostgreSQL pool initialization failed`. + +### Fix + +- Ensure PostgreSQL is reachable and credentials are correct. +- Set `JWT_SECRET` in the environment. +- Confirm the service can connect to required downstream services. + +## `/ready` returns `fail` + +### Symptom + +Readiness endpoint reports dependency failure. + +### Diagnosis + +- The readiness endpoint checks: + - PostgreSQL connectivity + - Search service health + - KG service health + - local orchestrator readiness + +### Fix + +- Validate `SEARCH_SERVICE_URL` and `KG_SERVICE_URL`. +- Ensure downstream services are healthy and responding with `200`. +- Confirm PostgreSQL is accessible and the connection pool is initialized. + +## JWT auth failures + +### Symptom + +Protected endpoints return `401 Unauthorized`. + +### Diagnosis + +- Invalid or missing `Authorization` header. +- Token uses wrong signing secret. +- Token payload is malformed. + +### Fix + +- Send header: `Authorization: Bearer `. +- Use the same `JWT_SECRET` that the service is configured with. +- Ensure token is signed using `HS256`. + +## Docker healthcheck fails + +### Symptom + +Docker reports the container unhealthy. + +### Diagnosis + +- Healthcheck calls `http://127.0.0.1:8082/health`. +- Service may not have started successfully. + +### Fix + +- Confirm the service listens on port `8082`. +- Ensure container has started and the app is not offline. +- Check logs for startup errors. + +## Metrics endpoint missing values + +### Symptom + +`/metrics/prometheus` returns no or incomplete metrics. + +### Diagnosis + +- Prometheus metrics registry may not be initialized properly. +- The service may have failed before registering metrics. + +### Fix + +- Confirm the application starts without exceptions. +- Ensure `app/observability/metrics.py` is imported via `app/main.py` and route is available. + +## Ingestion job failures + +### Symptom + +Ingestion job status becomes `failed`. + +### Diagnosis + +- Check `error_message` on the ingestion job record. +- Review dead-letter items for failure details. + +### Fix + +- Confirm source connector input is valid. +- Validate the query and source values. +- Restart or retry the job using `/api/v1/ingestion/{job_id}/retry`. + +## Database schema errors + +### Symptom + +Queries fail with missing table errors. + +### Diagnosis + +- Startup schema initialization may have been skipped or failed. + +### Fix + +- Confirm `services/literature/app/core/lifespan.py` ran during startup. +- Check logs for schema initialization errors. +- Ensure PostgreSQL user has permission to create tables. diff --git a/services/literature/VALIDATION.md b/services/literature/VALIDATION.md new file mode 100644 index 0000000..6190800 --- /dev/null +++ b/services/literature/VALIDATION.md @@ -0,0 +1,51 @@ +# Validation Guide + +This document describes the validation checks for the Literature service. + +## Service validation + +Validate the service by confirming: + +- `GET /healthz`, `GET /health`, and `GET /live` return `200` and correct status payloads. +- `GET /ready` returns `ready: true` when dependencies are healthy. +- `GET /metrics/prometheus` returns Prometheus plaintext metrics. +- Authenticated calls to protected endpoints return `200` for valid JWTs and `401` for invalid or missing tokens. + +## Data validation + +- `POST /api/v1/documents/parse` must accept valid document content and return metadata. +- Duplicate documents raise `409` and return a duplicate error. +- `POST /api/v1/nlp/process` must accept parsed document metadata and return NLP results. +- Ingestion endpoints must persist job state and correctly transition through queued, running, completed, failed, and cancelled statuses. + +## Static validation + +Perform code quality checks before production deployment: + +```bash +python -m ruff check app +python -m ruff format --check app +python -m mypy app --ignore-missing-imports +``` + +## Runtime validation + +1. Start the service. +2. Confirm `/health` and `/ready` endpoints return expected values. +3. Verify that the database schema is created successfully by the startup lifespan handler. +4. Confirm scheduler and orchestrator metrics are present in `/metrics`. +5. Validate failure modes by: + - stopping a downstream service + - verifying readiness changes to `fail` + - confirming error metrics increment + +## Dependency validation + +The readiness endpoint checks: + +- PostgreSQL connection availability +- Search service health via `search_service_url` +- KG service health via `kg_service_url` +- local orchestrator readiness + +Ensure these dependencies are reachable in the target environment. diff --git a/services/literature/app/connectors/__init__.py b/services/literature/app/connectors/__init__.py new file mode 100644 index 0000000..7b25378 --- /dev/null +++ b/services/literature/app/connectors/__init__.py @@ -0,0 +1,29 @@ +from app.connectors.aacr import AACRConnector +from app.connectors.asco import ASCOConnector +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.biorxiv import BioRxivConnector +from app.connectors.clinicaltrials import ClinicalTrialsConnector +from app.connectors.company_website import CompanyWebsiteConnector +from app.connectors.esmo import ESMOConnector +from app.connectors.medrxiv import MedRxivConnector +from app.connectors.patents import PatentConnector +from app.connectors.pmc import PubMedCentralConnector +from app.connectors.pubmed import PubMedConnector +from app.connectors.sabcs import SABCSConnector + +__all__ = [ + "AACRConnector", + "ASCOConnector", + "BioRxivConnector", + "ClinicalTrialsConnector", + "CompanyWebsiteConnector", + "ESMOConnector", + "MedRxivConnector", + "PageResult", + "PatentConnector", + "PubMedCentralConnector", + "PubMedConnector", + "SABCSConnector", + "SourceConnector", + "SourceRecord", +] diff --git a/services/literature/app/connectors/aacr.py b/services/literature/app/connectors/aacr.py new file mode 100644 index 0000000..b5675ec --- /dev/null +++ b/services/literature/app/connectors/aacr.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from app.connectors.adapter import UnsupportedSourceAdapter + + +class AACRConnector(UnsupportedSourceAdapter): + def __init__(self) -> None: + super().__init__( + "aacr", "requires gated access and no generic public REST API is available" + ) diff --git a/services/literature/app/connectors/adapter.py b/services/literature/app/connectors/adapter.py new file mode 100644 index 0000000..3de4c22 --- /dev/null +++ b/services/literature/app/connectors/adapter.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from datetime import datetime + +from app.connectors.base import PageResult, SourceConnector + + +class AdapterSourceConnector(SourceConnector, ABC): + @abstractmethod + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + raise NotImplementedError + + @abstractmethod + async def health_check(self) -> bool: + raise NotImplementedError + + +class UnsupportedSourceAdapter(AdapterSourceConnector): + def __init__(self, source_name: str, reason: str): + super().__init__(source_name) + self.reason = reason + + async def health_check(self) -> bool: + return False + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + raise NotImplementedError( + f"Source {self.source_name} is not supported: {self.reason}" + ) diff --git a/services/literature/app/connectors/asco.py b/services/literature/app/connectors/asco.py new file mode 100644 index 0000000..79c2820 --- /dev/null +++ b/services/literature/app/connectors/asco.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from app.connectors.adapter import UnsupportedSourceAdapter + + +class ASCOConnector(UnsupportedSourceAdapter): + def __init__(self) -> None: + super().__init__("asco", "requires membership and non-public event API access") diff --git a/services/literature/app/connectors/base.py b/services/literature/app/connectors/base.py new file mode 100644 index 0000000..62a2661 --- /dev/null +++ b/services/literature/app/connectors/base.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from datetime import datetime + +from pydantic import BaseModel + + +class ConnectorError(Exception): + pass + + +class SourceRecord(BaseModel): + source: str + source_id: str + title: str + abstract: str | None = None + authors: list[str] = [] + published_date: datetime | None = None + doi: str | None = None + url: str | None = None + source_updated_at: datetime | None = None + extra: dict[str, object] = {} + + +class PageResult(BaseModel): + items: list[SourceRecord] + next_page_token: str | None = None + + +class SourceConnector(ABC): + source_name: str + + def __init__(self, source_name: str): + self.source_name = source_name + + @abstractmethod + async def health_check(self) -> bool: + raise NotImplementedError + + @abstractmethod + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + raise NotImplementedError diff --git a/services/literature/app/connectors/biorxiv.py b/services/literature/app/connectors/biorxiv.py new file mode 100644 index 0000000..e1b4153 --- /dev/null +++ b/services/literature/app/connectors/biorxiv.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.http_client import HealthClient, HTTPClient +from app.core.config import get_settings + +settings = get_settings() + + +class BioRxivConnector(SourceConnector): + def __init__(self) -> None: + super().__init__("biorxiv") + self.client = HTTPClient( + base_url=settings.biorxiv_base_url, + headers={"Accept": "application/json"}, + ) + self.health_client = HealthClient(base_url=settings.biorxiv_base_url) + + async def health_check(self) -> bool: + return await self.health_client.health_check(path="/details/biorxiv/0/1") + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + params: dict[str, object] = { + "format": "json", + "cursor": page_token or "0", + "count": page_size, + } + if query: + params["collection"] = query + if since: + params["date_from"] = since.strftime("%Y-%m-%d") + + response = await self.client.get("/details/biorxiv", params=params) + payload = response.json() + items = payload.get("collection", []) + next_cursor = payload.get("cursor") + return PageResult( + items=[self._normalize(item) for item in items], + next_page_token=str(next_cursor) if next_cursor is not None else None, + ) + + def _normalize(self, raw: dict[str, Any]) -> SourceRecord: + return SourceRecord( + source=self.source_name, + source_id=str(raw.get("relating_article_id", "")), + title=raw.get("title", ""), + abstract=raw.get("abstract", None), + authors=[ + author.strip() + for author in raw.get("authors", "").split(";") + if author.strip() + ], + published_date=self._parse_date(raw.get("date")), + doi=raw.get("doi"), + url=raw.get("url"), + source_updated_at=self._parse_date(raw.get("date")), + extra={"raw": raw}, + ) + + def _parse_date(self, value: Any) -> datetime | None: + if not value: + return None + for fmt in ["%Y-%m-%d", "%Y-%m-%d %H:%M:%S"]: + try: + return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc) + except ValueError: + continue + return None diff --git a/services/literature/app/connectors/clinicaltrials.py b/services/literature/app/connectors/clinicaltrials.py new file mode 100644 index 0000000..73ab9da --- /dev/null +++ b/services/literature/app/connectors/clinicaltrials.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.http_client import HealthClient, HTTPClient +from app.core.config import get_settings + +settings = get_settings() + + +class ClinicalTrialsConnector(SourceConnector): + def __init__(self) -> None: + super().__init__("clinicaltrials") + self.client = HTTPClient( + base_url=settings.clinicaltrials_base_url, + headers={"Accept": "application/json"}, + ) + self.health_client = HealthClient(base_url="https://clinicaltrials.gov") + + async def health_check(self) -> bool: + return await self.health_client.health_check(path="/api/info") + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + params: dict[str, object] = { + "fmt": "json", + "min_rnk": 1, + "max_rnk": page_size, + } + if query: + params["expr"] = query + if page_token: + params["min_rnk"] = int(page_token) + params["max_rnk"] = int(page_token) + page_size - 1 + if since: + params["lastupdatefrom"] = since.strftime("%Y-%m-%d") + + response = await self.client.get("/study_fields", params=params) + payload = response.json() + fields = payload.get("StudyFieldsResponse", {}).get("StudyFields", []) + next_token = None + if fields and len(fields) == page_size: + next_token = str(int(page_token or "1") + page_size) + + return PageResult( + items=[self._normalize(item) for item in fields], + next_page_token=next_token, + ) + + def _normalize(self, raw: dict[str, Any]) -> SourceRecord: + return SourceRecord( + source=self.source_name, + source_id=str(raw.get("NCTId", [""])[0]), + title=(raw.get("BriefTitle", [""])[0] if raw.get("BriefTitle") else ""), + abstract=( + raw.get("BriefSummary", [""])[0] if raw.get("BriefSummary") else None + ), + authors=[], + published_date=self._parse_date( + raw.get("StartDate", [""])[0] if raw.get("StartDate") else None + ), + doi=None, + url=f"https://clinicaltrials.gov/study/{raw.get('NCTId', [''])[0]}", + source_updated_at=self._parse_date( + raw.get("LastUpdatePostDate", [""])[0] + if raw.get("LastUpdatePostDate") + else None + ), + extra={"raw": raw}, + ) + + def _parse_date(self, value: Any) -> datetime | None: + if not value: + return None + for fmt in ["%B %d, %Y", "%Y-%m-%d"]: + try: + return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc) + except ValueError: + continue + return None diff --git a/services/literature/app/connectors/company_website.py b/services/literature/app/connectors/company_website.py new file mode 100644 index 0000000..fbb82ab --- /dev/null +++ b/services/literature/app/connectors/company_website.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import io +import re +from datetime import datetime, timezone +from html.parser import HTMLParser +from typing import Any + +import httpx + +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.http_client import HealthClient, HTTPClient +from app.parsing import parse_document +from app.utils.logging import get_logger + +logger = get_logger(__name__) + +URL_SCHEME_RE = re.compile(r"^https?://", re.IGNORECASE) + + +class CompanyWebsiteConnector(SourceConnector): + def __init__(self) -> None: + super().__init__("company_website") + self.client = HTTPClient(base_url="", headers={"Accept": "text/html"}) + self.health_client = HealthClient(base_url="https://example.com") + + async def health_check(self) -> bool: + try: + response = await self.health_client.get("/") + return response.status_code == 200 + except (httpx.HTTPError, RuntimeError): + return False + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + if not query or not isinstance(query, str): + raise ValueError("query must be a valid company website URL") + + url = self._normalize_url(query) + response = await self.client.get(url, params=None) + html_text = response.text + metadata = self._extract_metadata(html_text) + parsed = self._parse_html_document(html_text) + + record = SourceRecord( + source=self.source_name, + source_id=url, + title=parsed.get("title") or metadata.get("title") or url, + abstract=parsed.get("abstract") + or metadata.get("description") + or self._extract_first_paragraph(html_text), + authors=parsed.get("authors") or metadata.get("authors") or [], + published_date=self._parse_date(metadata.get("published_date")), + doi=None, + url=url, + source_updated_at=None, + extra={"metadata": metadata, "parsed_metadata": parsed}, + ) + + return PageResult(items=[record], next_page_token=None) + + def _normalize_url(self, query: str) -> str: + url = query.strip() + if not URL_SCHEME_RE.match(url): + url = "https://" + url.lstrip("/") + return url + + def _extract_metadata(self, html_text: str) -> dict[str, Any]: + parser = _HTMLMetadataParser() + parser.feed(html_text) + return parser.metadata + + def _parse_html_document(self, html_text: str) -> dict[str, Any]: + try: + return parse_document("html", io.BytesIO(html_text.encode("utf-8"))) + except Exception as exc: + logger.warning( + "HTML parsing failed for company website connector", exc_info=exc + ) + return { + "title": "", + "authors": [], + "abstract": "", + "keywords": [], + "sections": [], + "references": [], + "tables": [], + "figures": [], + "supplementary": [], + } + + def _extract_first_paragraph(self, html_text: str) -> str: + match = re.search(r"]*>(.*?)

", html_text, re.IGNORECASE | re.DOTALL) + if not match: + return "" + text = re.sub(r"<[^>]+>", "", match.group(1)).strip() + return text + + def _parse_date(self, value: Any) -> datetime | None: + if not value or not isinstance(value, str): + return None + for fmt in ["%Y-%m-%d", "%Y/%m/%d", "%d %B %Y", "%B %d, %Y"]: + try: + return datetime.strptime(value.strip(), fmt).replace( + tzinfo=timezone.utc + ) + except ValueError: + continue + return None + + +class _HTMLMetadataParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.metadata: dict[str, Any] = {} + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.lower() != "meta": + return + attrs_dict = {name.lower(): value for name, value in attrs if value is not None} + name = attrs_dict.get("name") or attrs_dict.get("property") + content = attrs_dict.get("content") or attrs_dict.get("value") + if not name or not content: + return + key = name.lower().replace("meta:", "") + if key in {"description", "og:description", "twitter:description"}: + self.metadata.setdefault("description", content) + elif key in {"author", "article:author", "og:article:author"}: + self.metadata.setdefault("authors", []).append(content) + elif key in {"og:title", "twitter:title", "title"}: + self.metadata.setdefault("title", content) + elif key in { + "article:published_time", + "publication_date", + "date", + "publish_date", + }: + self.metadata.setdefault("published_date", content) + elif key.startswith("citation_"): + self.metadata[key] = content diff --git a/services/literature/app/connectors/esmo.py b/services/literature/app/connectors/esmo.py new file mode 100644 index 0000000..eca0564 --- /dev/null +++ b/services/literature/app/connectors/esmo.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from app.connectors.adapter import UnsupportedSourceAdapter + + +class ESMOConnector(UnsupportedSourceAdapter): + def __init__(self) -> None: + super().__init__( + "esmo", + "no public generic API; event access requires partnership agreements", + ) diff --git a/services/literature/app/connectors/http_client.py b/services/literature/app/connectors/http_client.py new file mode 100644 index 0000000..9a3a425 --- /dev/null +++ b/services/literature/app/connectors/http_client.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import asyncio + +import httpx + +from app.core.config import get_settings +from app.utils.logging import get_logger + +logger = get_logger(__name__) +settings = get_settings() + + +class HTTPClient: + def __init__( + self, base_url: str, timeout: int = 30, headers: dict[str, str] | None = None + ): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.headers = headers or {} + self._client = httpx.AsyncClient( + base_url=self.base_url, timeout=self.timeout, headers=self.headers + ) + + async def get( + self, + path: str, + params: dict[str, object] | list[tuple[str, object]] | None = None, + ) -> httpx.Response: + return await self._request("GET", path, params=params) + + async def _request( + self, + method: str, + path: str, + params: dict[str, object] | list[tuple[str, object]] | None = None, + ) -> httpx.Response: + request_params: ( + dict[str, str | int | float | bool | None] + | list[tuple[str, str | int | float | bool | None]] + | None + ) = None + if params is not None: + if isinstance(params, dict): + request_params = {} + for key, value in params.items(): + if isinstance(value, (str, int, float, bool)) or value is None: + request_params[key] = value + else: + request_params[key] = str(value) + else: + request_params = [ + ( + key, + value + if isinstance(value, (str, int, float, bool)) or value is None + else str(value), + ) + for key, value in params + ] + + attempt = 0 + backoff = 0.5 + while True: + attempt += 1 + try: + response = await self._client.request( + method, path, params=request_params + ) + response.raise_for_status() + return response + except (httpx.HTTPStatusError, httpx.TransportError) as exc: + if attempt >= 3 or not self._retryable(exc): + logger.error( + "HTTP request failed", + exc_info=exc, + extra={"method": method, "path": path, "params": params}, + ) + raise + await asyncio.sleep(backoff) + backoff *= 2 + + def _retryable(self, exc: Exception) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + status = exc.response.status_code + return status in {429, 500, 502, 503, 504} + return True + + async def close(self) -> None: + await self._client.aclose() + + +class HealthClient(HTTPClient): + async def health_check(self, path: str = "/") -> bool: + try: + response = await self.get(path) + return response.status_code == 200 + except httpx.HTTPError: + return False diff --git a/services/literature/app/connectors/medrxiv.py b/services/literature/app/connectors/medrxiv.py new file mode 100644 index 0000000..7e13894 --- /dev/null +++ b/services/literature/app/connectors/medrxiv.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.http_client import HealthClient, HTTPClient +from app.core.config import get_settings + +settings = get_settings() + + +class MedRxivConnector(SourceConnector): + def __init__(self) -> None: + super().__init__("medrxiv") + self.client = HTTPClient( + base_url=settings.medrxiv_base_url, + headers={"Accept": "application/json"}, + ) + self.health_client = HealthClient(base_url=settings.medrxiv_base_url) + + async def health_check(self) -> bool: + return await self.health_client.health_check(path="/details/medrxiv/0/1") + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + params: dict[str, object] = { + "format": "json", + "cursor": page_token or "0", + "count": page_size, + } + if query: + params["collection"] = query + if since: + params["date_from"] = since.strftime("%Y-%m-%d") + + response = await self.client.get("/details/medrxiv", params=params) + payload = response.json() + items = payload.get("collection", []) + next_cursor = payload.get("cursor") + return PageResult( + items=[self._normalize(item) for item in items], + next_page_token=str(next_cursor) if next_cursor is not None else None, + ) + + def _normalize(self, raw: dict[str, Any]) -> SourceRecord: + return SourceRecord( + source=self.source_name, + source_id=str(raw.get("relating_article_id", "")), + title=raw.get("title", ""), + abstract=raw.get("abstract", None), + authors=[ + author.strip() + for author in raw.get("authors", "").split(";") + if author.strip() + ], + published_date=self._parse_date(raw.get("date")), + doi=raw.get("doi"), + url=raw.get("url"), + source_updated_at=self._parse_date(raw.get("date")), + extra={"raw": raw}, + ) + + def _parse_date(self, value: Any) -> datetime | None: + if not value: + return None + for fmt in ["%Y-%m-%d", "%Y-%m-%d %H:%M:%S"]: + try: + return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc) + except ValueError: + continue + return None diff --git a/services/literature/app/connectors/patents.py b/services/literature/app/connectors/patents.py new file mode 100644 index 0000000..23f6670 --- /dev/null +++ b/services/literature/app/connectors/patents.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from app.connectors.adapter import UnsupportedSourceAdapter + + +class PatentConnector(UnsupportedSourceAdapter): + def __init__(self) -> None: + super().__init__( + "patents", "patent data requires specialized paid or partner APIs" + ) diff --git a/services/literature/app/connectors/pmc.py b/services/literature/app/connectors/pmc.py new file mode 100644 index 0000000..fe228ee --- /dev/null +++ b/services/literature/app/connectors/pmc.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.http_client import HealthClient, HTTPClient +from app.core.config import get_settings + +settings = get_settings() + + +class PubMedCentralConnector(SourceConnector): + def __init__(self) -> None: + super().__init__("pmc") + self.client = HTTPClient( + base_url=settings.pmc_base_url, + headers={"Accept": "application/json"}, + ) + self.health_client = HealthClient(base_url="https://api.ncbi.nlm.nih.gov") + + async def health_check(self) -> bool: + return await self.health_client.health_check(path="/health/ready") + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + params: dict[str, object] = { + "format": "json", + "pageSize": page_size, + } + if query: + params["query"] = query + if page_token: + params["pageToken"] = page_token + if since: + params["lastUpdate"] = since.isoformat() + + response = await self.client.get("/search", params=params) + payload = response.json() + return PageResult( + items=[self._normalize(item) for item in payload.get("items", [])], + next_page_token=payload.get("nextPageToken"), + ) + + def _normalize(self, raw: dict[str, Any]) -> SourceRecord: + return SourceRecord( + source=self.source_name, + source_id=str(raw.get("uid", "")), + title=raw.get("title", ""), + abstract=raw.get("abstractText"), + authors=[ + author.get("name") + for author in raw.get("authors", []) + if author.get("name") + ], + published_date=self._parse_date(raw.get("pubDate")), + doi=raw.get("doi"), + url=raw.get("url"), + source_updated_at=self._parse_date(raw.get("lastUpdate")), + extra={"raw": raw}, + ) + + def _parse_date(self, value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None diff --git a/services/literature/app/connectors/pubmed.py b/services/literature/app/connectors/pubmed.py new file mode 100644 index 0000000..228e454 --- /dev/null +++ b/services/literature/app/connectors/pubmed.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from app.connectors.base import PageResult, SourceConnector, SourceRecord +from app.connectors.http_client import HealthClient, HTTPClient +from app.core.config import get_settings + +settings = get_settings() + + +class PubMedConnector(SourceConnector): + def __init__(self) -> None: + super().__init__("pubmed") + self.client = HTTPClient( + base_url=settings.pubmed_base_url, + headers={"Accept": "application/json"}, + ) + self.health_client = HealthClient(base_url="https://api.ncbi.nlm.nih.gov") + + async def health_check(self) -> bool: + return await self.health_client.health_check(path="/health/ready") + + async def fetch_records( + self, + query: str | None = None, + page_token: str | None = None, + since: datetime | None = None, + page_size: int = 50, + ) -> PageResult: + params: dict[str, object] = { + "format": "json", + "pageSize": page_size, + } + if query: + params["query"] = query + if page_token: + params["pageToken"] = page_token + if since: + params["lastUpdate"] = since.isoformat() + + response = await self.client.get("/search", params=params) + payload = response.json() + return PageResult( + items=[self._normalize(item) for item in payload.get("items", [])], + next_page_token=payload.get("nextPageToken"), + ) + + def _normalize(self, raw: dict[str, Any]) -> SourceRecord: + return SourceRecord( + source=self.source_name, + source_id=str(raw.get("uid", "")), + title=raw.get("title", ""), + abstract=raw.get("abstractText"), + authors=[ + author.get("name") + for author in raw.get("authors", []) + if author.get("name") + ], + published_date=self._parse_date(raw.get("pubDate")), + doi=raw.get("doi"), + url=raw.get("url"), + source_updated_at=self._parse_date(raw.get("lastUpdate")), + extra={"raw": raw}, + ) + + def _parse_date(self, value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None diff --git a/services/literature/app/connectors/registry.py b/services/literature/app/connectors/registry.py new file mode 100644 index 0000000..0937319 --- /dev/null +++ b/services/literature/app/connectors/registry.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from collections.abc import Callable + +from app.connectors import ( + AACRConnector, + ASCOConnector, + BioRxivConnector, + ClinicalTrialsConnector, + CompanyWebsiteConnector, + ESMOConnector, + MedRxivConnector, + PatentConnector, + PubMedCentralConnector, + PubMedConnector, + SABCSConnector, +) +from app.connectors.base import SourceConnector + +CONNECTOR_FACTORIES: dict[str, Callable[[], SourceConnector]] = { + "pubmed": PubMedConnector, + "pmc": PubMedCentralConnector, + "clinicaltrials": ClinicalTrialsConnector, + "biorxiv": BioRxivConnector, + "medrxiv": MedRxivConnector, + "aacr": AACRConnector, + "asco": ASCOConnector, + "esmo": ESMOConnector, + "sabcs": SABCSConnector, + "patents": PatentConnector, + "company_website": CompanyWebsiteConnector, +} + + +def get_connector(source_name: str) -> SourceConnector: + connector_factory = CONNECTOR_FACTORIES.get(source_name) + if connector_factory is None: + raise ValueError(f"Unknown connector source: {source_name}") + return connector_factory() diff --git a/services/literature/app/connectors/sabcs.py b/services/literature/app/connectors/sabcs.py new file mode 100644 index 0000000..9c460ee --- /dev/null +++ b/services/literature/app/connectors/sabcs.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from app.connectors.adapter import UnsupportedSourceAdapter + + +class SABCSConnector(UnsupportedSourceAdapter): + def __init__(self) -> None: + super().__init__( + "sabcs", "requires event-specific partner integration and paid access" + ) diff --git a/services/literature/app/core/config.py b/services/literature/app/core/config.py index fb4f071..12b1283 100644 --- a/services/literature/app/core/config.py +++ b/services/literature/app/core/config.py @@ -15,9 +15,25 @@ class Settings(BaseSettings): neo4j_user: str = "neo4j" neo4j_password: str = "changeme_neo4j" opensearch_url: str = "http://opensearch:9200" - jwt_secret: str = "change_this_dev_secret_before_deploying" + search_service_url: str = "http://search:8084" + search_service_timeout_seconds: int = 5 + search_service_max_retries: int = 3 + kg_service_url: str = "http://kg:8083" + kg_service_timeout_seconds: int = 5 + kg_service_max_retries: int = 3 + llmwiki_service_url: str = "http://llmwiki:8086" + llmwiki_service_timeout_seconds: int = 5 + llmwiki_service_max_retries: int = 3 + jwt_secret: str = "changeme_secret" + cors_allowed_origins: list[str] = ["http://localhost:3000"] + + pubmed_base_url: str = "https://api.ncbi.nlm.nih.gov/lit/ctxp/v1/pubmed/" + pmc_base_url: str = "https://api.ncbi.nlm.nih.gov/lit/ctxp/v1/pmc/" + clinicaltrials_base_url: str = "https://clinicaltrials.gov/api/query" + biorxiv_base_url: str = "https://api.biorxiv.org" + medrxiv_base_url: str = "https://api.biorxiv.org" @lru_cache def get_settings() -> Settings: - return Settings() + return Settings() # type: ignore[call-arg] diff --git a/services/literature/app/core/lifespan.py b/services/literature/app/core/lifespan.py new file mode 100644 index 0000000..d28e804 --- /dev/null +++ b/services/literature/app/core/lifespan.py @@ -0,0 +1,42 @@ +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from app.core.config import get_settings +from app.database.postgres import postgres_manager +from app.orchestrator.manager import orchestrator +from app.utils.logging import get_logger + +logger = get_logger(__name__) +settings = get_settings() + + +@asynccontextmanager +async def lifespan(_: FastAPI): + if settings.environment == "test": + logger.info("Skipping PostgreSQL lifecycle in test environment") + yield + return + + logger.info("Literature service starting") + logger.info("Initializing PostgreSQL connection pool...") + try: + await postgres_manager.init_pool(settings) + logger.info("Ensuring literature schema is available...") + await postgres_manager.ensure_schema() + logger.info("Literature database schema is ready.") + except Exception as exc: + postgres_manager.last_error = exc + logger.exception( + "PostgreSQL startup initialization failed; service will continue in degraded mode" + ) + + try: + await orchestrator.start() + yield + finally: + logger.info("Stopping ingestion orchestrator...") + await orchestrator.stop() + logger.info("Closing PostgreSQL connection pool...") + await postgres_manager.close() + logger.info("PostgreSQL connection pool closed.") diff --git a/services/literature/app/core/security.py b/services/literature/app/core/security.py new file mode 100644 index 0000000..ccc87a2 --- /dev/null +++ b/services/literature/app/core/security.py @@ -0,0 +1,36 @@ +import jwt +from fastapi import HTTPException, Security, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from app.core.config import get_settings + +settings = get_settings() +security = HTTPBearer(auto_error=False) +security_dependency = Security(security) + + +def get_current_user( + credentials: HTTPAuthorizationCredentials | None = security_dependency, +) -> dict[str, str]: + if credentials is None or credentials.scheme.lower() != "bearer": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="authentication required", + ) + + token = credentials.credentials + try: + payload = jwt.decode(token, settings.jwt_secret, algorithms=["HS256"]) + except jwt.PyJWTError as exc: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="invalid authentication token", + ) from exc + + if not isinstance(payload, dict): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="invalid authentication payload", + ) + + return payload diff --git a/services/literature/app/database/__init__.py b/services/literature/app/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/literature/app/database/postgres.py b/services/literature/app/database/postgres.py new file mode 100644 index 0000000..b2531fa --- /dev/null +++ b/services/literature/app/database/postgres.py @@ -0,0 +1,96 @@ +from contextlib import asynccontextmanager + +import asyncpg + +from app.core.config import Settings +from app.utils.logging import get_logger + +logger = get_logger(__name__) + + +class PostgresManager: + def __init__(self) -> None: + self.pool: asyncpg.Pool | None = None + self.last_error: Exception | None = None + + async def init_pool(self, settings: Settings) -> None: + self.last_error = None + try: + self.pool = await asyncpg.create_pool( + dsn=settings.database_url, + min_size=1, + max_size=10, + ) + logger.info("PostgreSQL pool initialized") + except Exception as exc: + self.pool = None + self.last_error = exc + logger.exception("PostgreSQL pool initialization failed") + raise + + async def close(self) -> None: + if self.pool: + await self.pool.close() + self.pool = None + logger.info("PostgreSQL pool closed") + + def _ensure_pool(self) -> asyncpg.Pool: + if self.pool is None: + raise RuntimeError("PostgreSQL pool is not initialized") + return self.pool + + @asynccontextmanager + async def acquire(self): + pool = self._ensure_pool() + async with pool.acquire() as connection: + yield connection + + async def ensure_schema(self) -> bool: + try: + async with self.acquire() as connection: + await connection.execute( + """ + CREATE TABLE IF NOT EXISTS literature_papers ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + source TEXT NOT NULL, + doi TEXT, + published_at TIMESTAMPTZ, + citation_count INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + await connection.execute( + """ + CREATE TABLE IF NOT EXISTS literature_ingestion_jobs ( + id UUID PRIMARY KEY, + source TEXT NOT NULL, + query TEXT NOT NULL, + status TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + schedule TEXT, + next_run_at TIMESTAMPTZ, + documents_total INT NOT NULL DEFAULT 0, + documents_processed INT NOT NULL DEFAULT 0, + documents_failed INT NOT NULL DEFAULT 0, + retry_count INT NOT NULL DEFAULT 0, + backoff_until TIMESTAMPTZ, + error_message TEXT, + dead_letter_count INT NOT NULL DEFAULT 0, + dead_letter_items TEXT + ) + """ + ) + logger.info("Literature schema ensured") + self.last_error = None + return True + except Exception as exc: + self.last_error = exc + logger.exception("Failed to initialize literature database schema") + return False + + +postgres_manager = PostgresManager() diff --git a/services/literature/app/document_schemas.py b/services/literature/app/document_schemas.py new file mode 100644 index 0000000..b50a69d --- /dev/null +++ b/services/literature/app/document_schemas.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class DocumentSection(BaseModel): + title: str = "" + text: str = "" + + +class DocumentReference(BaseModel): + citation: str | None = None + href: str | None = None + text: str | None = None + + +class DocumentTable(BaseModel): + label: str | None = None + caption: str | None = None + + +class DocumentFigure(BaseModel): + label: str | None = None + caption: str | None = None + + +class DocumentMetadata(BaseModel): + title: str | None = None + authors: list[str] = Field(default_factory=list) + affiliations: list[str] = Field(default_factory=list) + doi: str | None = None + pmid: str | None = None + pmcid: str | None = None + abstract: str | None = None + keywords: list[str] = Field(default_factory=list) + sections: list[DocumentSection] = Field(default_factory=list) + references: list[DocumentReference] = Field(default_factory=list) + tables: list[DocumentTable] = Field(default_factory=list) + figures: list[DocumentFigure] = Field(default_factory=list) + supplementary: list[str] = Field(default_factory=list) + raw: dict[str, Any] = Field(default_factory=dict) diff --git a/services/literature/app/main.py b/services/literature/app/main.py index a7b8ff9..37c373d 100644 --- a/services/literature/app/main.py +++ b/services/literature/app/main.py @@ -1,78 +1,82 @@ +from __future__ import annotations + +import time import uuid -from datetime import datetime, timezone -from typing import Literal -from fastapi import FastAPI -from pydantic import BaseModel +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware from app.core.config import get_settings +from app.core.lifespan import lifespan +from app.observability.metrics import ( + REQUEST_COUNT, + REQUEST_ERRORS, + REQUEST_IN_FLIGHT, + REQUEST_LATENCY_SECONDS, +) +from app.routers.documents import router as documents_router +from app.routers.health import router as health_router +from app.routers.ingestion import router as ingestion_router +from app.routers.nlp import router as nlp_router +from app.routers.papers import router as papers_router +from app.utils.logging import get_logger, set_request_context settings = get_settings() +logger = get_logger(__name__) app = FastAPI( title="AI-RxOS Literature Service", - description="Ingestion, extraction, and citation services for the " - "Literature Intelligence bounded context.", + description="Ingestion, extraction, and citation services for the Literature Intelligence bounded context.", version="0.1.0", + lifespan=lifespan, ) -_PAPERS: dict[str, dict] = {} -_INGESTION_JOBS: dict[str, dict] = {} - - -class Paper(BaseModel): - id: str - title: str - source: Literal["pubmed", "biorxiv", "medrxiv", "patent", "conference"] - doi: str | None = None - publishedAt: str | None = None - citationCount: int = 0 - - -class IngestionRequest(BaseModel): - source: Literal["pubmed", "biorxiv", "medrxiv", "patent", "conference"] - query: str - - -class IngestionJob(BaseModel): - id: str - source: str - query: str - status: Literal["queued", "running", "completed", "failed"] - createdAt: str - - -@app.get("/healthz") -def health() -> dict[str, str]: - return {"status": "ok", "service": "literature"} - - -@app.get("/api/v1/papers") -def list_papers(page: int = 1, page_size: int = 20) -> dict: - items = list(_PAPERS.values())[(page - 1) * page_size : page * page_size] - return {"items": items, "total": len(_PAPERS), "page": page, "pageSize": page_size} - - -@app.get("/api/v1/papers/{paper_id}") -def get_paper(paper_id: str) -> Paper | dict: - paper = _PAPERS.get(paper_id) - return paper or {"error": "not_found"} - - -@app.post("/api/v1/ingestion", response_model=IngestionJob, status_code=202) -def start_ingestion(req: IngestionRequest) -> IngestionJob: - job_id = str(uuid.uuid4()) - job = IngestionJob( - id=job_id, - source=req.source, - query=req.query, - status="queued", - createdAt=datetime.now(timezone.utc).isoformat(), - ) - _INGESTION_JOBS[job_id] = job.model_dump() - return job +app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_allowed_origins, + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], + allow_headers=["*"], +) -@app.get("/api/v1/ingestion/{job_id}") -def get_ingestion_job(job_id: str) -> dict: - return _INGESTION_JOBS.get(job_id, {"error": "not_found"}) +@app.middleware("http") +async def add_request_context(request: Request, call_next): + request_id = request.headers.get("x-request-id") or str(uuid.uuid4()) + trace_id = request.headers.get("x-trace-id") or str(uuid.uuid4()) + request.state.request_id = request_id + request.state.trace_id = trace_id + set_request_context(request_id=request_id, trace_id=trace_id) + + method = request.method + endpoint = request.url.path + REQUEST_IN_FLIGHT.labels(method=method, endpoint=endpoint).inc() + start_time = time.perf_counter() + try: + response = await call_next(request) + status_code = str(response.status_code) + REQUEST_COUNT.labels(method=method, endpoint=endpoint, status=status_code).inc() + if response.status_code >= 400: + REQUEST_ERRORS.labels( + method=method, endpoint=endpoint, status=status_code + ).inc() + response.headers["x-request-id"] = request_id + response.headers["x-trace-id"] = trace_id + return response + except Exception: + REQUEST_COUNT.labels(method=method, endpoint=endpoint, status="500").inc() + REQUEST_ERRORS.labels(method=method, endpoint=endpoint, status="500").inc() + raise + finally: + elapsed = time.perf_counter() - start_time + REQUEST_LATENCY_SECONDS.labels(method=method, endpoint=endpoint).observe( + elapsed + ) + REQUEST_IN_FLIGHT.labels(method=method, endpoint=endpoint).dec() + + +app.include_router(health_router) +app.include_router(papers_router, prefix="/api/v1") +app.include_router(ingestion_router, prefix="/api/v1") +app.include_router(documents_router, prefix="/api/v1") +app.include_router(nlp_router, prefix="/api/v1") diff --git a/services/literature/app/nlp/__init__.py b/services/literature/app/nlp/__init__.py new file mode 100644 index 0000000..1fa955e --- /dev/null +++ b/services/literature/app/nlp/__init__.py @@ -0,0 +1,5 @@ +from app.nlp.pipeline import process_document +from app.nlp.summarizer import SummarizerService +from app.parsing.metrics import parser_metrics as parsing_metrics + +__all__ = ["SummarizerService", "parsing_metrics", "process_document"] diff --git a/services/literature/app/nlp/confidence_scorer.py b/services/literature/app/nlp/confidence_scorer.py new file mode 100644 index 0000000..8b683f9 --- /dev/null +++ b/services/literature/app/nlp/confidence_scorer.py @@ -0,0 +1,19 @@ +from __future__ import annotations + + +class ConfidenceScorer: + """Apply a simple confidence policy to entity predictions.""" + + def score(self, entity: dict[str, object]) -> float: + confidence_value = entity.get("confidence", 0.5) + base = ( + float(confidence_value) + if isinstance(confidence_value, (int, float, str)) + else 0.5 + ) + entity_type = str(entity.get("type") or "") + if entity_type == "variant": + return round(min(1.0, base + 0.05), 2) + if entity_type in {"gene", "disease", "drug"}: + return round(min(1.0, base + 0.02), 2) + return round(min(1.0, base), 2) diff --git a/services/literature/app/nlp/embedding_service.py b/services/literature/app/nlp/embedding_service.py new file mode 100644 index 0000000..f25e55b --- /dev/null +++ b/services/literature/app/nlp/embedding_service.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import hashlib +import math +import time +from typing import Any + +from app.observability.metrics import ( + EMBEDDING_GENERATION_DURATION_SECONDS, + EMBEDDING_GENERATION_ERRORS_TOTAL, + EMBEDDING_GENERATION_TOTAL, +) + + +class EmbeddingService: + """Generate deterministic vector embeddings for normalized document content. + + This phase only produces vectors and packaging metadata. It does not write to + PostgreSQL, call the search service, or update Neo4j. + """ + + def __init__(self, batch_size: int = 8) -> None: + self.batch_size = max(1, batch_size) + + def generate_embeddings(self, nlp_result: dict[str, Any]) -> dict[str, Any]: + start_time = time.perf_counter() + try: + if not isinstance(nlp_result, dict): + raise TypeError("nlp_result must be a dictionary") + + document_id = nlp_result.get("document_id") or "unknown" + text_items: list[str] = [] + + if nlp_result.get("sentences"): + text_items.extend( + [ + str(sentence) + for sentence in nlp_result.get("sentences", []) + if str(sentence).strip() + ] + ) + if nlp_result.get("detected_entities"): + text_items.extend( + [ + f"{entity.get('text')}:{entity.get('type')}" + for entity in nlp_result.get("detected_entities", []) + if isinstance(entity, dict) + ] + ) + if nlp_result.get("relationships"): + text_items.extend( + [ + f"{relationship.get('source_entity')}:{relationship.get('predicate')}:{relationship.get('target_entity')}" + for relationship in nlp_result.get("relationships", []) + if isinstance(relationship, dict) + ] + ) + + batches = self._create_batches(text_items) + generated_vectors = [] + for batch in batches: + generated_vectors.extend([self._vectorize_text(item) for item in batch]) + + elapsed = time.perf_counter() - start_time + elapsed_ms = round(elapsed * 1000, 2) + EMBEDDING_GENERATION_TOTAL.inc() + EMBEDDING_GENERATION_DURATION_SECONDS.observe(elapsed) + return { + "document_id": document_id, + "embedding_batches": batches, + "embeddings": generated_vectors, + "embedding_count": len(generated_vectors), + "metadata": { + "source": "EmbeddingService", + "embedding_dimensions": len(generated_vectors[0]) + if generated_vectors + else 16, + "batch_size": self.batch_size, + "text_source_count": len(text_items), + }, + "processing_metrics": { + "total_processing_time_ms": elapsed_ms, + "batch_count": len(batches), + "failed_batches": 0, + "retries": 0, + }, + } + except Exception: + EMBEDDING_GENERATION_ERRORS_TOTAL.inc() + raise + + def _create_batches(self, items: list[str]) -> list[list[str]]: + batches: list[list[str]] = [] + for index in range(0, len(items), self.batch_size): + batches.append(items[index : index + self.batch_size]) + return batches + + def _vectorize_text(self, text: str) -> list[float]: + normalized = (text or "").strip().lower() + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + seed = int(digest[:8], 16) + vector: list[float] = [] + for index in range(16): + value = math.sin((index + 1) * (seed % 97 + 1)) + math.cos( + (index + 1) * 0.25 + ) + vector.append(round(value, 6)) + return vector diff --git a/services/literature/app/nlp/entity_extractor.py b/services/literature/app/nlp/entity_extractor.py new file mode 100644 index 0000000..9cc1690 --- /dev/null +++ b/services/literature/app/nlp/entity_extractor.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import re +from typing import ClassVar + + +class EntityExtractor: + """Rule-based biomedical entity extractor for genes, variants, diseases, and drugs.""" + + _gene_pattern: ClassVar[re.Pattern[str]] = re.compile(r"\b([A-Z0-9]{2,7})\b") + _variant_pattern: ClassVar[re.Pattern[str]] = re.compile( + r"\b(p\.[A-Za-z]\d+[A-Za-z]|c\.\d+[A-Za-z]?)\b" + ) + _disease_keywords: ClassVar[set[str]] = { + "cancer", + "diabetes", + "alzheimer", + "covid", + "covid-19", + } + _drug_keywords: ClassVar[set[str]] = { + "aspirin", + "ibuprofen", + "paracetamol", + "tamoxifen", + } + + def _to_int(self, value: object) -> int: + if isinstance(value, int): + return value + if isinstance(value, str) and value.isdigit(): + return int(value) + raise TypeError("entity index must be an integer") + + def _to_float(self, value: object) -> float: + if isinstance(value, float): + return value + if isinstance(value, int): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + pass + raise TypeError("entity confidence must be numeric") + + def extract(self, text: str) -> list[dict[str, object]]: + if not text or not isinstance(text, str): + return [] + + entities: list[dict[str, object]] = [] + lower_text = text.lower() + + for match in self._gene_pattern.finditer(text): + token = match.group(1) + if token.isupper(): + entities.append( + { + "text": token, + "start": match.start(1), + "end": match.end(1), + "type": "gene", + "confidence": 0.72, + } + ) + + for match in self._variant_pattern.finditer(text): + entities.append( + { + "text": match.group(1), + "start": match.start(1), + "end": match.end(1), + "type": "variant", + "confidence": 0.93, + } + ) + + for keyword in self._disease_keywords: + index = lower_text.find(keyword) + if index >= 0: + entities.append( + { + "text": text[index : index + len(keyword)], + "start": index, + "end": index + len(keyword), + "type": "disease", + "confidence": 0.84, + } + ) + + for keyword in self._drug_keywords: + index = lower_text.find(keyword) + if index >= 0: + entities.append( + { + "text": text[index : index + len(keyword)], + "start": index, + "end": index + len(keyword), + "type": "drug", + "confidence": 0.84, + } + ) + + entities = sorted( + entities, + key=lambda item: ( + self._to_int(item["start"]), + -self._to_float(item["confidence"]), + ), + ) + filtered: list[dict[str, object]] = [] + occupied: set[int] = set() + for entity in entities: + start = self._to_int(entity["start"]) + end = self._to_int(entity["end"]) + if any(index in occupied for index in range(start, end)): + continue + for index in range(start, end): + occupied.add(index) + filtered.append(entity) + return filtered diff --git a/services/literature/app/nlp/entity_normalizer.py b/services/literature/app/nlp/entity_normalizer.py new file mode 100644 index 0000000..1eba5a6 --- /dev/null +++ b/services/literature/app/nlp/entity_normalizer.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import ClassVar + + +class EntityNormalizer: + """Normalize entity text to stable biomedical identifiers when possible.""" + + _gene_map: ClassVar[dict[str, str]] = { + "TP53": "HGNC:11998", + "EGFR": "HGNC:3236", + "BRCA1": "HGNC:1100", + } + _disease_map: ClassVar[dict[str, str]] = { + "cancer": "MONDO:0000001", + "diabetes": "MONDO:0001", + "alzheimer": "MONDO:0002", + "covid": "MONDO:0100", + } + _drug_map: ClassVar[dict[str, str]] = { + "aspirin": "CHEBI:15365", + "ibuprofen": "CHEBI:5855", + "paracetamol": "CHEBI:46195", + } + + def normalize(self, entity: dict[str, object]) -> dict[str, object]: + text = str(entity.get("text") or "") + entity_type = str(entity.get("type") or "") + confidence_value = entity.get("confidence", 0.5) + confidence = ( + float(confidence_value) + if isinstance(confidence_value, (int, float, str)) + else 0.5 + ) + normalized_identifier: str | None = None + ontology_source: str | None = None + + if entity_type == "gene": + normalized_identifier = self._gene_map.get(text.upper()) + ontology_source = "HGNC" if normalized_identifier else None + confidence = max(confidence, 0.9) + elif entity_type == "disease": + normalized_identifier = self._disease_map.get(text.lower()) + ontology_source = "MONDO" if normalized_identifier else None + confidence = max(confidence, 0.9) + elif entity_type == "drug": + normalized_identifier = self._drug_map.get(text.lower()) + ontology_source = "ChEBI" if normalized_identifier else None + confidence = max(confidence, 0.9) + elif entity_type == "variant": + normalized_identifier = text + ontology_source = "HGVS" + confidence = max(confidence, 0.85) + + return { + "text": text, + "type": entity_type, + "start": entity.get("start"), + "end": entity.get("end"), + "confidence": round(min(max(confidence, 0.0), 1.0), 2), + "normalized_identifier": normalized_identifier, + "ontology_source": ontology_source, + } diff --git a/services/literature/app/nlp/ner.py b/services/literature/app/nlp/ner.py new file mode 100644 index 0000000..9f7b6b6 --- /dev/null +++ b/services/literature/app/nlp/ner.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import re + +# Simple rule-based patterns for demonstration purposes +GENE_PROTEIN_RE = re.compile(r"\b([A-Z0-9]{2,7})\b") +VARIANT_RE = re.compile(r"\b(p\.[A-Za-z]\d+[A-Za-z]|c\.\d+[A-Za-z]?)\b") +# small disease/drug keyword lists +DISEASE_KEYWORDS = {"cancer", "diabetes", "alzheimer", "covid-19", "covid"} +DRUG_KEYWORDS = {"aspirin", "ibuprofen", "paracetamol", "tamoxifen"} + + +def extract_entities(text: str) -> list[dict]: + entities = [] + # gene/protein candidates + for m in GENE_PROTEIN_RE.finditer(text): + token = m.group(1) + # heuristic: all-caps tokens likely gene/protein if length 2-7 + if token.isupper(): + entities.append( + { + "text": token, + "start": m.start(1), + "end": m.end(1), + "type": "gene", + "confidence": 0.7, + } + ) + + # variants + for m in VARIANT_RE.finditer(text): + entities.append( + { + "text": m.group(1), + "start": m.start(1), + "end": m.end(1), + "type": "variant", + "confidence": 0.9, + } + ) + + # diseases and drugs via simple keyword matching + lower = text.lower() + for kw in DISEASE_KEYWORDS: + idx = lower.find(kw) + if idx >= 0: + entities.append( + { + "text": text[idx : idx + len(kw)], + "start": idx, + "end": idx + len(kw), + "type": "disease", + "confidence": 0.8, + } + ) + for kw in DRUG_KEYWORDS: + idx = lower.find(kw) + if idx >= 0: + entities.append( + { + "text": text[idx : idx + len(kw)], + "start": idx, + "end": idx + len(kw), + "type": "drug", + "confidence": 0.8, + } + ) + + # deduplicate overlapping entities by choosing highest confidence + def _int_value(value: object) -> int: + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str) and value.isdigit(): + return int(value) + raise TypeError("entity index must be an integer") + + def _float_value(value: object) -> float: + if isinstance(value, float): + return value + if isinstance(value, int): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + pass + raise TypeError("entity confidence must be numeric") + + entities_sorted = sorted( + entities, + key=lambda e: (_int_value(e["start"]), -_float_value(e["confidence"])), + ) + filtered = [] + occupied = set() + for e in entities_sorted: + start = _int_value(e["start"]) + end = _int_value(e["end"]) + rng = range(start, end) + if any(i in occupied for i in rng): + continue + for i in rng: + occupied.add(i) + filtered.append(e) + + return filtered diff --git a/services/literature/app/nlp/normalizer.py b/services/literature/app/nlp/normalizer.py new file mode 100644 index 0000000..ae9ce50 --- /dev/null +++ b/services/literature/app/nlp/normalizer.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +# Static normalization maps for known biomedical entities. +GENE_MAP = {"TP53": "HGNC:11998", "EGFR": "HGNC:3236", "BRCA1": "HGNC:1100"} +DISEASE_MAP = { + "cancer": "MONDO:0000001", + "diabetes": "MONDO:0001", + "alzheimer": "MONDO:0002", + "covid": "MONDO:0100", +} +DRUG_MAP = { + "aspirin": "CHEBI:15365", + "ibuprofen": "CHEBI:5855", + "paracetamol": "CHEBI:46195", +} + + +def normalize_entity(entity: dict) -> dict: + text = entity.get("text", "") + t_low = text.lower() + etype = entity.get("type") + normalized_id = None + ontology = None + score = entity.get("confidence", 0.5) + + if etype == "gene": + normalized_id = GENE_MAP.get(text.upper()) + ontology = "HGNC" if normalized_id else None + if normalized_id: + score = max(score, 0.9) + elif etype == "disease": + normalized_id = DISEASE_MAP.get(t_low) + ontology = "MONDO" if normalized_id else None + if normalized_id: + score = max(score, 0.9) + elif etype == "drug": + normalized_id = DRUG_MAP.get(t_low) + ontology = "ChEBI" if normalized_id else None + if normalized_id: + score = max(score, 0.9) + elif etype == "variant": + # simple pass-through for variants + normalized_id = text + ontology = "HGVS" + score = max(score, 0.85) + + return { + "text": text, + "type": etype, + "start": entity.get("start"), + "end": entity.get("end"), + "confidence": score, + "normalized_id": normalized_id, + "ontology": ontology, + } diff --git a/services/literature/app/nlp/ontology_mapper.py b/services/literature/app/nlp/ontology_mapper.py new file mode 100644 index 0000000..c05cd29 --- /dev/null +++ b/services/literature/app/nlp/ontology_mapper.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import ClassVar + + +class OntologyMapper: + """Maps normalized entities to ontology identifiers using lightweight lookup tables.""" + + _gene_map: ClassVar[dict[str, str]] = { + "TP53": "HGNC:11998", + "EGFR": "HGNC:3236", + "BRCA1": "HGNC:1100", + } + _disease_map: ClassVar[dict[str, str]] = { + "cancer": "MONDO:0000001", + "diabetes": "MONDO:0001", + "alzheimer": "MONDO:0002", + "covid": "MONDO:0100", + } + _drug_map: ClassVar[dict[str, str]] = { + "aspirin": "CHEBI:15365", + "ibuprofen": "CHEBI:5855", + "paracetamol": "CHEBI:46195", + } + + def map_entity(self, entity: dict[str, object]) -> dict[str, object]: + entity_type = str(entity.get("type", "")) + text = str(entity.get("text", "")) + identifier: str | None = None + source: str | None = None + if entity_type == "gene": + identifier = self._gene_map.get(text.upper()) + source = "HGNC" if identifier else None + elif entity_type == "disease": + identifier = self._disease_map.get(text.lower()) + source = "MONDO" if identifier else None + elif entity_type == "drug": + identifier = self._drug_map.get(text.lower()) + source = "ChEBI" if identifier else None + elif entity_type == "variant": + identifier = text + source = "HGVS" + return {"normalized_identifier": identifier, "ontology_source": source} diff --git a/services/literature/app/nlp/pipeline.py b/services/literature/app/nlp/pipeline.py new file mode 100644 index 0000000..9f9eb6e --- /dev/null +++ b/services/literature/app/nlp/pipeline.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import time +import unicodedata +from typing import Any + +from app.nlp.confidence_scorer import ConfidenceScorer +from app.nlp.entity_extractor import EntityExtractor +from app.nlp.entity_normalizer import EntityNormalizer +from app.nlp.ontology_mapper import OntologyMapper +from app.nlp.relationship_extractor import RelationshipExtractor +from app.nlp.sentence_segmenter import SentenceSegmenter +from app.nlp.summarizer import SummarizerService +from app.nlp.tokenizer import BiomedicalTokenizer +from app.observability.metrics import ( + NLP_PROCESSING_DURATION_SECONDS, + NLP_PROCESSING_ERRORS_TOTAL, + NLP_PROCESSING_TOTAL, +) +from app.utils.logging import get_logger + +logger = get_logger(__name__) + + +class BiomedicalNLPPipeline: + """Modular biomedical NLP pipeline for Phase 5 document processing.""" + + def __init__(self) -> None: + self.segmenter = SentenceSegmenter() + self.tokenizer = BiomedicalTokenizer() + self.extractor = EntityExtractor() + self.normalizer = EntityNormalizer() + self.mapper = OntologyMapper() + self.scorer = ConfidenceScorer() + self.relationship_extractor = RelationshipExtractor() + self.summarizer = SummarizerService() + + def process_document(self, document: dict[str, Any]) -> dict[str, Any]: + start_time = time.perf_counter() + warnings: list[str] = [] + try: + if not isinstance(document, dict): + raise TypeError("document must be a dictionary") + + document_id = document.get("document_id") or document.get("id") or "unknown" + abstract = document.get("abstract") or "" + sections = document.get("sections") or [] + if not isinstance(sections, list): + sections = [] + warnings.append("sections was not a list; defaulted to empty") + + text_parts: list[str] = [] + if isinstance(abstract, str) and abstract.strip(): + text_parts.append(abstract) + + for section in sections: + if isinstance(section, dict): + section_text = section.get("text") or "" + if isinstance(section_text, str) and section_text.strip(): + text_parts.append(section_text) + else: + warnings.append("section entry was malformed") + + combined_text = "\n\n".join(text_parts) + combined_text = self._normalize_text(combined_text) + input_text_length = len(combined_text) + + sentences = self.segmenter.segment(combined_text) + tokens: list[list[str]] = [ + self.tokenizer.tokenize(sentence) for sentence in sentences + ] + + detected_entities: list[dict[str, Any]] = [] + relationships: list[dict[str, Any]] = [] + for sentence in sentences: + entities = self.extractor.extract(sentence) + normalized_entities: list[dict[str, Any]] = [] + for entity in entities: + normalized = self.normalizer.normalize(entity) + mapping = self.mapper.map_entity(normalized) + normalized["normalized_identifier"] = mapping.get( + "normalized_identifier" + ) + normalized["ontology_source"] = mapping.get("ontology_source") + normalized["confidence_score"] = self.scorer.score(normalized) + normalized_entities.append(normalized) + detected_entities.append(normalized) + + if normalized_entities: + relationships.extend( + self.relationship_extractor.extract( + normalized_entities, sentence + ) + ) + + summary = self.summarizer.summarize(document) + elapsed = time.perf_counter() - start_time + elapsed_ms = round(elapsed * 1000, 2) + NLP_PROCESSING_TOTAL.inc() + NLP_PROCESSING_DURATION_SECONDS.observe(elapsed) + return { + "document_id": document_id, + "sentences": sentences, + "tokens": tokens, + "detected_entities": detected_entities, + "entities": detected_entities, + "relationships": relationships, + "summary": summary, + "processing_metadata": { + "input_text_length": input_text_length, + "warnings": warnings, + "source": "BiomedicalNLPPipeline", + }, + "execution_metrics": { + "total_processing_time_ms": elapsed_ms, + "stage_metrics": { + "sentence_segmentation": { + "processed_items": len(sentences), + "duration_ms": round( + elapsed_ms / max(1, len(sentences) or 1), 2 + ), + }, + "tokenization": { + "processed_items": len(tokens), + "duration_ms": round( + elapsed_ms / max(1, len(tokens) or 1), 2 + ), + }, + "entity_extraction": { + "processed_items": len(detected_entities), + "duration_ms": round( + elapsed_ms / max(1, len(detected_entities) or 1), 2 + ), + }, + }, + }, + } + except Exception: + NLP_PROCESSING_ERRORS_TOTAL.inc() + raise + + def _normalize_text(self, text: str) -> str: + if not text or not isinstance(text, str): + return "" + normalized = unicodedata.normalize("NFKC", text) + normalized = normalized.replace("–", "-").replace("—", "-") + return normalized.strip() + + +def process_document(document: dict[str, Any]) -> dict[str, Any]: + pipeline = BiomedicalNLPPipeline() + return pipeline.process_document(document) diff --git a/services/literature/app/nlp/preprocess.py b/services/literature/app/nlp/preprocess.py new file mode 100644 index 0000000..765ed27 --- /dev/null +++ b/services/literature/app/nlp/preprocess.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import re +import unicodedata + +SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|\n+") +TOKEN_RE = re.compile(r"[A-Za-z0-9_\-/\.:]+|\S") + + +def sentence_segment(text: str) -> list[str]: + if not text: + return [] + parts = [s.strip() for s in SENTENCE_SPLIT_RE.split(text) if s and s.strip()] + return parts + + +def tokenize_biomedical(sentence: str) -> list[str]: + tokens = TOKEN_RE.findall(sentence) + return tokens + + +def normalize_text(text: str) -> str: + if text is None: + return "" + t = unicodedata.normalize("NFKC", text) + t = t.replace("\u2013", "-") + t = t.strip() + return t diff --git a/services/literature/app/nlp/relationship_extractor.py b/services/literature/app/nlp/relationship_extractor.py new file mode 100644 index 0000000..b62f958 --- /dev/null +++ b/services/literature/app/nlp/relationship_extractor.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import time +from typing import Any, ClassVar + +from app.observability.metrics import ( + RELATIONSHIP_EXTRACTION_DURATION_SECONDS, + RELATIONSHIP_EXTRACTION_ERRORS_TOTAL, + RELATIONSHIP_EXTRACTION_TOTAL, +) + + +class RelationshipExtractor: + """Rule-based relationship extraction for Phase 6. + + This phase discovers relationships between entities already extracted in Phase 5. + It does not generate embeddings or write to Neo4j. + """ + + _predicate_map: ClassVar[dict[tuple[str, str], str]] = { + ("drug", "disease"): "treats", + ("gene", "disease"): "associated_with", + ("protein", "protein"): "interacts_with", + ("compound", "protein"): "targets", + } + + def extract( + self, entities: list[dict[str, Any]], sentence: str + ) -> list[dict[str, Any]]: + start_time = time.perf_counter() + try: + if not isinstance(entities, list): + raise TypeError("entities must be a list") + if not sentence or not isinstance(sentence, str): + RELATIONSHIP_EXTRACTION_TOTAL.inc() + return [] + + relationships: list[dict[str, Any]] = [] + entity_types = { + entity.get("type") for entity in entities if isinstance(entity, dict) + } + + if not entity_types: + RELATIONSHIP_EXTRACTION_TOTAL.inc() + return [] + + for left_entity in entities: + if not isinstance(left_entity, dict): + continue + left_type = str(left_entity.get("type", "")).lower() + left_text = str(left_entity.get("text", "")) + for right_entity in entities: + if not isinstance(right_entity, dict): + continue + right_type = str(right_entity.get("type", "")).lower() + right_text = str(right_entity.get("text", "")) + if left_entity is right_entity: + continue + if (left_type, right_type) not in self._predicate_map: + continue + predicate = self._predicate_map[(left_type, right_type)] + confidence = self._score( + left_type, right_type, left_text, right_text + ) + relationships.append( + { + "source_entity": left_text, + "target_entity": right_text, + "source_type": left_type, + "target_type": right_type, + "predicate": predicate, + "confidence": round(confidence, 2), + "provenance": { + "source_sentence": sentence.strip(), + "source": "rule-based", + }, + } + ) + + RELATIONSHIP_EXTRACTION_TOTAL.inc() + return relationships + except Exception: + RELATIONSHIP_EXTRACTION_ERRORS_TOTAL.inc() + raise + finally: + RELATIONSHIP_EXTRACTION_DURATION_SECONDS.observe( + time.perf_counter() - start_time + ) + + def _score( + self, left_type: str, right_type: str, left_text: str, right_text: str + ) -> float: + base = 0.6 + if left_text and right_text: + base += 0.1 + if left_type == "drug" and right_type == "disease": + base += 0.2 + if left_type == "gene" and right_type == "disease": + base += 0.15 + if left_type == "protein" and right_type == "protein": + base += 0.15 + if left_type == "compound" and right_type == "protein": + base += 0.15 + return min(0.99, round(base, 2)) diff --git a/services/literature/app/nlp/sentence_segmenter.py b/services/literature/app/nlp/sentence_segmenter.py new file mode 100644 index 0000000..03bc025 --- /dev/null +++ b/services/literature/app/nlp/sentence_segmenter.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import re + + +class SentenceSegmenter: + """Simple, deterministic sentence segmentation for biomedical text.""" + + _split_pattern = re.compile(r"(?<=[.!?])\s+|\n+") + + def segment(self, text: str) -> list[str]: + if not text or not isinstance(text, str): + return [] + return [ + segment.strip() + for segment in self._split_pattern.split(text) + if segment and segment.strip() + ] diff --git a/services/literature/app/nlp/stages/__init__.py b/services/literature/app/nlp/stages/__init__.py new file mode 100644 index 0000000..3ae65e8 --- /dev/null +++ b/services/literature/app/nlp/stages/__init__.py @@ -0,0 +1,15 @@ +from .confidence_scorer import ConfidenceScorer +from .entity_extractor import EntityExtractor +from .entity_normalizer import EntityNormalizer +from .ontology_mapper import OntologyMapper +from .sentence_segmenter import SentenceSegmenter +from .tokenizer import BiomedicalTokenizer + +__all__ = [ + "BiomedicalTokenizer", + "ConfidenceScorer", + "EntityExtractor", + "EntityNormalizer", + "OntologyMapper", + "SentenceSegmenter", +] diff --git a/services/literature/app/nlp/summarizer.py b/services/literature/app/nlp/summarizer.py new file mode 100644 index 0000000..6bbd8f3 --- /dev/null +++ b/services/literature/app/nlp/summarizer.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import re +import time +from typing import Any + +from app.observability.metrics import ( + SUMMARIZER_DURATION_SECONDS, + SUMMARIZER_ERRORS_TOTAL, + SUMMARIZER_TOTAL, +) + +SENTENCE_END_RE = re.compile(r"(?<=[.!?])\s+") +SUMMARY_KEYWORDS = [ + "treat", + "efficacy", + "risk", + "associated", + "linked", + "improved", + "reduced", + "benefit", + "adverse", + "clinical", + "patient", + "therapy", + "prognosis", + "outcome", + "mortality", +] +LIMITATION_KEYWORDS = [ + "limit", + "limitation", + "future", + "small sample", + "further research", + "bias", +] + + +class SummarizerService: + """Generate a lightweight structured summary for a literature document.""" + + def summarize(self, document: dict[str, Any]) -> dict[str, Any]: + start_time = time.perf_counter() + try: + if not isinstance(document, dict): + raise TypeError("document must be a dictionary") + + document_id = document.get("document_id") or document.get("id") or "unknown" + title = self._first_nonempty_string(document.get("title")) + abstract = self._first_nonempty_string(document.get("abstract")) + sections = self._normalize_sections(document.get("sections")) + + source_text = self._assemble_source_text(abstract, sections) + sentences = self._split_sentences(source_text) + + abstract_summary = self._build_abstract_summary(abstract, sentences) + key_findings = self._build_key_findings(sentences) + clinical_relevance = self._build_clinical_relevance(sentences) + limitations = self._build_limitations(abstract, sections, sentences) + + structured_summary = { + "abstract_summary": abstract_summary, + "key_findings": key_findings, + "clinical_relevance": clinical_relevance, + "limitations": limitations, + } + + SUMMARIZER_TOTAL.inc() + SUMMARIZER_DURATION_SECONDS.observe(time.perf_counter() - start_time) + return { + "document_id": document_id, + "title": title, + "abstract_summary": abstract_summary, + "key_findings": key_findings, + "clinical_relevance": clinical_relevance, + "limitations": limitations, + "structured_summary": structured_summary, + } + except Exception: + SUMMARIZER_ERRORS_TOTAL.inc() + raise + + def _first_nonempty_string(self, value: Any) -> str: + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + def _normalize_sections(self, sections: Any) -> list[dict[str, str]]: + if not isinstance(sections, list): + return [] + normalized: list[dict[str, str]] = [] + for section in sections: + if isinstance(section, dict): + text = self._first_nonempty_string(section.get("text")) + title = self._first_nonempty_string(section.get("title")) + normalized.append({"title": title, "text": text}) + return normalized + + def _assemble_source_text( + self, abstract: str, sections: list[dict[str, str]] + ) -> str: + parts: list[str] = [] + if abstract: + parts.append(abstract) + for section in sections: + if section.get("title"): + parts.append(section["title"]) + if section.get("text"): + parts.append(section["text"]) + return "\n\n".join(parts).strip() + + def _split_sentences(self, text: str) -> list[str]: + if not text: + return [] + sentences = [ + sentence.strip() + for sentence in SENTENCE_END_RE.split(text) + if sentence.strip() + ] + return sentences + + def _build_abstract_summary(self, abstract: str, sentences: list[str]) -> str: + if abstract: + abstract_sentences = self._split_sentences(abstract) + if abstract_sentences: + return " ".join(abstract_sentences[:2]) + if sentences: + return " ".join(sentences[:2]) + return "No abstract summary available." + + def _build_key_findings(self, sentences: list[str]) -> list[str]: + findings: list[str] = [] + for sentence in sentences: + text = sentence.lower() + if any( + keyword in text + for keyword in [ + "treat", + "efficacy", + "risk", + "associated", + "linked", + "improved", + "reduced", + "adverse", + ] + ): + findings.append(sentence) + if len(findings) >= 3: + break + if not findings and sentences: + findings.append(sentences[0]) + return findings + + def _build_clinical_relevance(self, sentences: list[str]) -> str: + for sentence in sentences: + text = sentence.lower() + if any( + keyword in text + for keyword in [ + "clinical", + "patient", + "therapy", + "prognos", + "outcome", + "mortality", + "adverse", + ] + ): + return sentence + return "Clinical relevance is not explicit in the source text." + + def _build_limitations( + self, abstract: str, sections: list[dict[str, str]], sentences: list[str] + ) -> str: + candidates: list[str] = [] + if abstract: + candidates.extend( + [ + s + for s in self._split_sentences(abstract) + if any(keyword in s.lower() for keyword in LIMITATION_KEYWORDS) + ] + ) + for section in sections: + candidates.extend( + [ + s + for s in self._split_sentences(section.get("text", "")) + if any(keyword in s.lower() for keyword in LIMITATION_KEYWORDS) + ] + ) + if candidates: + return candidates[0] + for sentence in sentences: + if any(keyword in sentence.lower() for keyword in LIMITATION_KEYWORDS): + return sentence + return "No explicit limitations were identified." diff --git a/services/literature/app/nlp/tokenizer.py b/services/literature/app/nlp/tokenizer.py new file mode 100644 index 0000000..8a2cfdb --- /dev/null +++ b/services/literature/app/nlp/tokenizer.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import re + + +class BiomedicalTokenizer: + """Tokenize biomedical text while preserving punctuation-heavy tokens such as variants.""" + + _token_pattern = re.compile(r"[A-Za-z0-9_.\-/]+|\S") + + def tokenize(self, text: str) -> list[str]: + if not text or not isinstance(text, str): + return [] + tokens: list[str] = [] + for token in self._token_pattern.findall(text): + if not token: + continue + if token in {".", ",", "!", "?", ":", ";", "(", ")"}: + continue + tokens.append(token.rstrip(".,;:!?()")) + return [token for token in tokens if token] diff --git a/services/literature/app/observability/metrics.py b/services/literature/app/observability/metrics.py new file mode 100644 index 0000000..8aa3cc8 --- /dev/null +++ b/services/literature/app/observability/metrics.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from prometheus_client import ( + CollectorRegistry, + Counter, + Gauge, + Histogram, + generate_latest, +) + +registry = CollectorRegistry() + +REQUEST_COUNT = Counter( + "literature_http_requests_total", + "Total number of HTTP requests processed by the literature service", + ["method", "endpoint", "status"], + registry=registry, +) +REQUEST_IN_FLIGHT = Gauge( + "literature_http_requests_in_flight", + "Current number of in-flight HTTP requests", + ["method", "endpoint"], + registry=registry, +) +REQUEST_LATENCY_SECONDS = Histogram( + "literature_http_request_latency_seconds", + "HTTP request latency in seconds", + ["method", "endpoint"], + registry=registry, +) +REQUEST_ERRORS = Counter( + "literature_http_request_errors_total", + "Total number of HTTP request errors", + ["method", "endpoint", "status"], + registry=registry, +) + +NLP_PROCESSING_TOTAL = Counter( + "literature_nlp_pipeline_runs_total", + "Total number of biomedical NLP pipeline executions", + registry=registry, +) +NLP_PROCESSING_ERRORS_TOTAL = Counter( + "literature_nlp_pipeline_errors_total", + "Total errors during biomedical NLP pipeline executions", + registry=registry, +) +NLP_PROCESSING_DURATION_SECONDS = Histogram( + "literature_nlp_pipeline_duration_seconds", + "Biomedical NLP pipeline execution time in seconds", + registry=registry, +) +RELATIONSHIP_EXTRACTION_TOTAL = Counter( + "literature_relationship_extraction_total", + "Total relationship extraction operations", + registry=registry, +) +RELATIONSHIP_EXTRACTION_ERRORS_TOTAL = Counter( + "literature_relationship_extraction_errors_total", + "Total relationship extraction errors", + registry=registry, +) +RELATIONSHIP_EXTRACTION_DURATION_SECONDS = Histogram( + "literature_relationship_extraction_duration_seconds", + "Relationship extraction duration in seconds", + registry=registry, +) +SUMMARIZER_TOTAL = Counter( + "literature_summarizer_total", + "Total summarization operations", + registry=registry, +) +SUMMARIZER_ERRORS_TOTAL = Counter( + "literature_summarizer_errors_total", + "Total summarization errors", + registry=registry, +) +SUMMARIZER_DURATION_SECONDS = Histogram( + "literature_summarizer_duration_seconds", + "Summarization duration in seconds", + registry=registry, +) +EMBEDDING_GENERATION_TOTAL = Counter( + "literature_embedding_generation_total", + "Total embedding generation requests", + registry=registry, +) +EMBEDDING_GENERATION_ERRORS_TOTAL = Counter( + "literature_embedding_generation_errors_total", + "Total errors during embedding generation", + registry=registry, +) +EMBEDDING_GENERATION_DURATION_SECONDS = Histogram( + "literature_embedding_generation_duration_seconds", + "Embedding generation duration in seconds", + registry=registry, +) +SEARCH_HANDOFF_TOTAL = Counter( + "literature_search_handoff_total", + "Total search handoff attempts", + ["status"], + registry=registry, +) +SEARCH_HANDOFF_RETRIES_TOTAL = Counter( + "literature_search_handoff_retries_total", + "Total search handoff retries", + registry=registry, +) +SEARCH_HANDOFF_ERRORS_TOTAL = Counter( + "literature_search_handoff_errors_total", + "Total search handoff failures", + registry=registry, +) +SEARCH_HANDOFF_DURATION_SECONDS = Histogram( + "literature_search_handoff_duration_seconds", + "Search handoff duration in seconds", + registry=registry, +) +KG_HANDOFF_TOTAL = Counter( + "literature_kg_handoff_total", + "Total KG handoff attempts", + ["status"], + registry=registry, +) +KG_HANDOFF_RETRIES_TOTAL = Counter( + "literature_kg_handoff_retries_total", + "Total KG handoff retries", + registry=registry, +) +KG_HANDOFF_ERRORS_TOTAL = Counter( + "literature_kg_handoff_errors_total", + "Total KG handoff failures", + registry=registry, +) +KG_HANDOFF_DURATION_SECONDS = Histogram( + "literature_kg_handoff_duration_seconds", + "KG handoff duration in seconds", + registry=registry, +) +EVIDENCE_RANKING_TOTAL = Counter( + "literature_evidence_ranking_total", + "Total evidence ranking operations", + registry=registry, +) +EVIDENCE_RANKING_ERRORS_TOTAL = Counter( + "literature_evidence_ranking_errors_total", + "Total evidence ranking failures", + registry=registry, +) +EVIDENCE_RANKING_DURATION_SECONDS = Histogram( + "literature_evidence_ranking_duration_seconds", + "Evidence ranking duration in seconds", + registry=registry, +) +LLMWIKI_UPDATE_TOTAL = Counter( + "literature_llmwiki_update_total", + "Total LLM Wiki update attempts", + ["status"], + registry=registry, +) +LLMWIKI_UPDATE_RETRIES_TOTAL = Counter( + "literature_llmwiki_update_retries_total", + "Total LLM Wiki update retries", + registry=registry, +) +LLMWIKI_UPDATE_ERRORS_TOTAL = Counter( + "literature_llmwiki_update_errors_total", + "Total LLM Wiki update failures", + registry=registry, +) +LLMWIKI_UPDATE_DURATION_SECONDS = Histogram( + "literature_llmwiki_update_duration_seconds", + "LLM Wiki update duration in seconds", + registry=registry, +) + + +def generate_prometheus_metrics() -> bytes: + return generate_latest(registry) diff --git a/services/literature/app/orchestrator/__init__.py b/services/literature/app/orchestrator/__init__.py new file mode 100644 index 0000000..b1298c2 --- /dev/null +++ b/services/literature/app/orchestrator/__init__.py @@ -0,0 +1,3 @@ +from .manager import orchestrator + +__all__ = ["orchestrator"] diff --git a/services/literature/app/orchestrator/manager.py b/services/literature/app/orchestrator/manager.py new file mode 100644 index 0000000..798f3ed --- /dev/null +++ b/services/literature/app/orchestrator/manager.py @@ -0,0 +1,387 @@ +from __future__ import annotations + +import asyncio +import json +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from enum import Enum +from typing import Any + +from croniter import croniter + +from app.core.config import get_settings +from app.database.postgres import postgres_manager +from app.utils.logging import get_logger + +logger = get_logger(__name__) +settings = get_settings() + + +class JobStatus(str, Enum): + QUEUED = "queued" + SCHEDULED = "scheduled" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class DeadLetterStatus(str, Enum): + PENDING = "pending" + RETRIED = "retried" + + +@dataclass +class JobProgress: + total_documents: int = 0 + processed_documents: int = 0 + failed_documents: int = 0 + dead_lettered_documents: int = 0 + last_update: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + +@dataclass +class IngestionJob: + job_id: str + source: str + query: str + status: JobStatus = JobStatus.QUEUED + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + started_at: datetime | None = None + completed_at: datetime | None = None + schedule: str | None = None + next_run_at: datetime | None = None + progress: JobProgress = field(default_factory=JobProgress) + error_message: str | None = None + retry_count: int = 0 + backoff_until: datetime | None = None + cancel_requested: bool = False + dead_letter_count: int = 0 + dead_letter_items: list[dict[str, Any]] = field(default_factory=list) + worker_task: asyncio.Task | None = None + + +class IngestionOrchestrator: + def __init__(self) -> None: + self._queue: asyncio.Queue[IngestionJob] = asyncio.Queue() + self._dead_letter: list[dict[str, Any]] = [] + self._running_jobs: dict[str, IngestionJob] = {} + self._worker: asyncio.Task | None = None + self._scheduler: asyncio.Task | None = None + self._stop_event = asyncio.Event() + self._metrics: dict[str, int] = { + "ingestion.jobs_created": 0, + "ingestion.jobs_started": 0, + "ingestion.jobs_completed": 0, + "ingestion.jobs_failed": 0, + "ingestion.jobs_cancelled": 0, + "ingestion.job_retries": 0, + "ingestion.dead_letters": 0, + "ingestion.jobs_scheduled": 0, + "ingestion.jobs_rescheduled": 0, + } + + async def start(self) -> None: + if self._worker is None or self._worker.done(): + self._stop_event.clear() + self._worker = asyncio.create_task(self._worker_loop()) + self._scheduler = asyncio.create_task(self._scheduler_loop()) + logger.info("Ingestion orchestrator worker started") + + async def stop(self) -> None: + self._stop_event.set() + if self._worker: + await self._worker + if self._scheduler: + self._scheduler.cancel() + try: + await self._scheduler + except asyncio.CancelledError: + pass + logger.info("Ingestion orchestrator stopped") + + def enqueue(self, job: IngestionJob) -> None: + if job.status == JobStatus.SCHEDULED: + asyncio.create_task(self._persist_job_state(job)) + return + + self._queue.put_nowait(job) + self._running_jobs[job.job_id] = job + self._metrics["ingestion.jobs_created"] += 1 + logger.info("Job enqueued", extra={"job_id": job.job_id, "source": job.source}) + asyncio.create_task(self._persist_job_state(job)) + + async def schedule(self, job: IngestionJob, schedule: str) -> None: + job.schedule = schedule + job.next_run_at = self._compute_next_run(schedule) + job.status = JobStatus.SCHEDULED + self._metrics["ingestion.jobs_scheduled"] += 1 + await self._persist_job_state(job) + await self.start() + logger.info( + "Job scheduled", + extra={ + "job_id": job.job_id, + "schedule": schedule, + "next_run_at": job.next_run_at.isoformat(), + }, + ) + + def _compute_next_run( + self, schedule: str, reference: datetime | None = None + ) -> datetime: + now = reference or datetime.now(timezone.utc) + if not croniter.is_valid(schedule): + raise ValueError("invalid schedule expression") + next_run = croniter(schedule, now).get_next(datetime) + if next_run.tzinfo is None: + next_run = next_run.replace(tzinfo=timezone.utc) + return next_run + + async def _scheduler_loop(self) -> None: + while not self._stop_event.is_set(): + try: + await asyncio.sleep(3) + now = datetime.now(timezone.utc) + async with postgres_manager.acquire() as connection: + rows = await connection.fetch( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, schedule, next_run_at, + documents_total, documents_processed, + documents_failed, retry_count, backoff_until, + error_message, dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE status = $1 + AND next_run_at IS NOT NULL + AND next_run_at <= $2 + """, + JobStatus.SCHEDULED.value, + now, + ) + + for row in rows: + job = await self._load_job_from_row(row) + job.status = JobStatus.QUEUED + job.next_run_at = None + await self._persist_job_state(job) + self.enqueue(job) + except asyncio.CancelledError: + break + except Exception: + logger.exception("Scheduler loop failed") + + async def _load_job_from_db(self, job_id: str) -> IngestionJob | None: + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, schedule, next_run_at, + documents_total, documents_processed, + documents_failed, retry_count, backoff_until, + error_message, dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + if row is None: + return None + return await self._load_job_from_row(row) + + async def _load_job_from_row(self, row: Any) -> IngestionJob: + dead_letter_items = row["dead_letter_items"] or [] + if isinstance(dead_letter_items, str): + dead_letter_items = json.loads(dead_letter_items) + + return IngestionJob( + job_id=row["id"], + source=row["source"], + query=row["query"], + status=JobStatus(row["status"]), + created_at=row["created_at"], + started_at=row["started_at"], + completed_at=row["completed_at"], + schedule=row["schedule"], + next_run_at=row["next_run_at"], + progress=JobProgress( + total_documents=row["documents_total"], + processed_documents=row["documents_processed"], + failed_documents=row["documents_failed"], + last_update=datetime.now(timezone.utc), + ), + error_message=row["error_message"], + retry_count=row["retry_count"], + backoff_until=row["backoff_until"], + dead_letter_count=row["dead_letter_count"], + dead_letter_items=dead_letter_items, + ) + + async def trigger_retry(self, job_id: str) -> None: + job = self._running_jobs.get(job_id) + if job is None: + job = await self._load_job_from_db(job_id) + if job is None: + raise ValueError("job not found") + + job.retry_count += 1 + job.backoff_until = datetime.now(timezone.utc) + timedelta( + seconds=5 * job.retry_count + ) + job.status = JobStatus.QUEUED + job.next_run_at = None + self._metrics["ingestion.job_retries"] += 1 + self.enqueue(job) + await self.start() + logger.info( + "Retry scheduled", + extra={"job_id": job.job_id, "retry_count": job.retry_count}, + ) + + async def cancel(self, job_id: str) -> None: + job = self._running_jobs.get(job_id) + if job is None: + job = await self._load_job_from_db(job_id) + if job is None: + raise ValueError("job not found") + + job.cancel_requested = True + job.status = JobStatus.CANCELLED + await self._persist_job_state(job) + logger.info("Cancellation requested", extra={"job_id": job.job_id}) + + async def _worker_loop(self) -> None: + while not self._stop_event.is_set(): + try: + job = await asyncio.wait_for(self._queue.get(), timeout=1.0) + except asyncio.TimeoutError: + continue + + if job.backoff_until and job.backoff_until > datetime.now(timezone.utc): + await asyncio.sleep( + (job.backoff_until - datetime.now(timezone.utc)).total_seconds() + ) + + if job.cancel_requested: + job.status = JobStatus.CANCELLED + job.completed_at = datetime.now(timezone.utc) + self._metrics["ingestion.jobs_cancelled"] += 1 + await self._persist_job_state(job) + continue + + await self._process_job(job) + + async def _process_job(self, job: IngestionJob) -> None: + job.status = JobStatus.RUNNING + job.started_at = datetime.now(timezone.utc) + self._metrics["ingestion.jobs_started"] += 1 + await self._persist_job_state(job) + + try: + await self._simulate_ingestion(job) + if job.cancel_requested: + job.status = JobStatus.CANCELLED + self._metrics["ingestion.jobs_cancelled"] += 1 + else: + job.status = JobStatus.COMPLETED + self._metrics["ingestion.jobs_completed"] += 1 + except Exception as exc: + job.status = JobStatus.FAILED + job.error_message = str(exc) + self._metrics["ingestion.jobs_failed"] += 1 + dead_letter_item = { + "job_id": job.job_id, + "source": job.source, + "query": job.query, + "error_message": job.error_message, + "created_at": datetime.now(timezone.utc).isoformat(), + "status": DeadLetterStatus.PENDING.value, + } + job.dead_letter_items.append(dead_letter_item) + job.dead_letter_count += 1 + self._dead_letter.append(dead_letter_item) + self._metrics["ingestion.dead_letters"] += 1 + logger.exception("Ingestion job failed", extra={"job_id": job.job_id}) + finally: + job.completed_at = datetime.now(timezone.utc) + await self._persist_job_state(job) + if job.schedule and job.status in {JobStatus.COMPLETED, JobStatus.FAILED}: + await self._reschedule(job) + await self._persist_job_state(job) + + async def _reschedule(self, job: IngestionJob) -> None: + if not job.schedule: + return + + job.status = JobStatus.SCHEDULED + job.next_run_at = self._compute_next_run(job.schedule, job.completed_at) + self._metrics["ingestion.jobs_rescheduled"] += 1 + logger.info( + "Job rescheduled", + extra={"job_id": job.job_id, "next_run_at": job.next_run_at.isoformat()}, + ) + + async def _simulate_ingestion(self, job: IngestionJob) -> None: + total = 10 + job.progress.total_documents = total + for index in range(total): + if job.cancel_requested: + break + await asyncio.sleep(0.01) + job.progress.processed_documents += 1 + job.progress.last_update = datetime.now(timezone.utc) + if index == 7 and job.retry_count == 0: + raise RuntimeError("temporary ingestion error") + + async def _persist_job_state(self, job: IngestionJob) -> None: + async with postgres_manager.acquire() as connection: + await connection.execute( + """ + UPDATE literature_ingestion_jobs + SET status = $1, + started_at = $2, + completed_at = $3, + schedule = $4, + next_run_at = $5, + documents_total = $6, + documents_processed = $7, + documents_failed = $8, + error_message = $9, + retry_count = $10, + backoff_until = $11, + dead_letter_count = $12, + dead_letter_items = $13 + WHERE id = $14 + """, + job.status.value, + job.started_at, + job.completed_at, + job.schedule, + job.next_run_at, + job.progress.total_documents, + job.progress.processed_documents, + job.progress.failed_documents, + job.error_message, + job.retry_count, + job.backoff_until, + job.dead_letter_count, + json.dumps(job.dead_letter_items), + job.job_id, + ) + + def get_job(self, job_id: str) -> IngestionJob | None: + return self._running_jobs.get(job_id) + + def get_metrics(self) -> dict[str, int]: + return dict(self._metrics) + + def get_dead_letters(self, job_id: str | None = None) -> list[dict[str, Any]]: + items = [ + item + for item in self._dead_letter + if job_id is None or item["job_id"] == job_id + ] + return list(items) + + +orchestrator = IngestionOrchestrator() diff --git a/services/literature/app/parsing/__init__.py b/services/literature/app/parsing/__init__.py new file mode 100644 index 0000000..a837de6 --- /dev/null +++ b/services/literature/app/parsing/__init__.py @@ -0,0 +1,13 @@ +from app.parsing.parser import ( + SUPPORTED_FORMATS, + DuplicateDocumentError, + parse_document, + parser_metrics, +) + +__all__ = [ + "SUPPORTED_FORMATS", + "DuplicateDocumentError", + "parse_document", + "parser_metrics", +] diff --git a/services/literature/app/parsing/duplicates.py b/services/literature/app/parsing/duplicates.py new file mode 100644 index 0000000..9304c00 --- /dev/null +++ b/services/literature/app/parsing/duplicates.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import hashlib +from typing import BinaryIO + + +class DuplicateDetector: + def __init__(self) -> None: + self._seen: set[str] = set() + + def fingerprint(self, stream: BinaryIO) -> str: + current_position = None + try: + current_position = stream.tell() + except (AttributeError, OSError): + pass + + hasher = hashlib.sha256() + try: + stream.seek(0) + except (AttributeError, OSError): + pass + + while True: + chunk = stream.read(8192) + if not chunk: + break + if isinstance(chunk, str): + chunk = chunk.encode("utf-8", errors="ignore") + hasher.update(chunk) + + try: + stream.seek(current_position or 0) + except (AttributeError, OSError): + pass + + return hasher.hexdigest() + + def register(self, stream: BinaryIO) -> bool: + fingerprint = self.fingerprint(stream) + is_duplicate = fingerprint in self._seen + if not is_duplicate: + self._seen.add(fingerprint) + return is_duplicate + + def clear(self) -> None: + self._seen.clear() + + +duplicate_detector = DuplicateDetector() diff --git a/services/literature/app/parsing/metrics.py b/services/literature/app/parsing/metrics.py new file mode 100644 index 0000000..f6e8fc1 --- /dev/null +++ b/services/literature/app/parsing/metrics.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from threading import Lock + + +@dataclass +class ParserMetrics: + counters: dict[str, int] = field( + default_factory=lambda: { + "parser.documents_parsed": 0, + "parser.documents_validated": 0, + "parser.parse_errors": 0, + "parser.validation_failures": 0, + "parser.duplicate_documents": 0, + } + ) + lock: Lock = field(default_factory=Lock) + + def increment(self, name: str, amount: int = 1) -> None: + with self.lock: + self.counters[name] = self.counters.get(name, 0) + amount + + def snapshot(self) -> dict[str, int]: + with self.lock: + return dict(self.counters) + + +parser_metrics = ParserMetrics() diff --git a/services/literature/app/parsing/parser.py b/services/literature/app/parsing/parser.py new file mode 100644 index 0000000..8121aa2 --- /dev/null +++ b/services/literature/app/parsing/parser.py @@ -0,0 +1,465 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from html.parser import HTMLParser +from typing import BinaryIO +from xml.etree import ElementTree as ET + +from app.parsing.duplicates import duplicate_detector +from app.parsing.metrics import parser_metrics + +try: + import PyPDF2 +except ImportError: # pragma: no cover + PyPDF2 = None + +SUPPORTED_FORMATS = {"pdf", "xml", "html", "nxml", "jats"} + + +class DocumentParseError(Exception): + pass + + +class DuplicateDocumentError(DocumentParseError): + pass + + +@dataclass +class DocumentMetadata: + title: str | None = None + authors: list[str] | None = None + affiliations: list[str] | None = None + doi: str | None = None + pmid: str | None = None + pmcid: str | None = None + abstract: str | None = None + keywords: list[str] | None = None + sections: list[dict[str, str]] | None = None + references: list[dict[str, str]] | None = None + tables: list[dict[str, str]] | None = None + figures: list[dict[str, str]] | None = None + supplementary: list[str] | None = None + + def __post_init__(self) -> None: + self.authors = self.authors or [] + self.affiliations = self.affiliations or [] + self.keywords = self.keywords or [] + self.sections = self.sections or [] + self.references = self.references or [] + self.tables = self.tables or [] + self.figures = self.figures or [] + self.supplementary = self.supplementary or [] + + +def _validate_metadata(metadata: dict[str, object]) -> None: + if not isinstance(metadata, dict): + raise DocumentParseError("parsed metadata must be a JSON object") + + for key in ["title", "authors", "abstract"]: + if ( + key in metadata + and metadata[key] is not None + and not isinstance(metadata[key], (str, list)) + ): + raise DocumentParseError(f"invalid metadata type for {key}") + + if ( + "authors" in metadata + and metadata["authors"] is not None + and not isinstance(metadata["authors"], list) + ): + raise DocumentParseError("authors must be a list") + + if ( + "keywords" in metadata + and metadata["keywords"] is not None + and not isinstance(metadata["keywords"], list) + ): + raise DocumentParseError("keywords must be a list") + + if ( + "references" in metadata + and metadata["references"] is not None + and not isinstance(metadata["references"], list) + ): + raise DocumentParseError("references must be a list") + + if ( + "tables" in metadata + and metadata["tables"] is not None + and not isinstance(metadata["tables"], list) + ): + raise DocumentParseError("tables must be a list") + + if ( + "figures" in metadata + and metadata["figures"] is not None + and not isinstance(metadata["figures"], list) + ): + raise DocumentParseError("figures must be a list") + + +def _parse_xml_metadata(root: ET.Element) -> dict[str, object]: + metadata: dict[str, object] = {} + + title = root.findtext(".//article-title") or root.findtext(".//title") + metadata["title"] = title.strip() if title else None + + authors: list[str] = [] + affiliations: list[str] = [] + for contrib in root.findall(".//contrib"): + surname = contrib.findtext(".//surname") + given_names = contrib.findtext(".//given-names") + name_parts = [part for part in [given_names, surname] if part] + if name_parts: + authors.append(" ".join(name_parts).strip()) + + for aff in root.findall(".//aff"): + affiliation = ET.tostring(aff, encoding="unicode", method="text").strip() + if affiliation: + affiliations.append(affiliation) + + metadata["authors"] = authors + metadata["affiliations"] = affiliations + + metadata["doi"] = root.findtext(".//article-id[@pub-id-type='doi']") + metadata["pmid"] = root.findtext(".//article-id[@pub-id-type='pmid']") + metadata["pmcid"] = root.findtext(".//article-id[@pub-id-type='pmcid']") + + abstract = root.findtext(".//abstract") + metadata["abstract"] = abstract.strip() if abstract else None + + keywords: list[str] = [] + for kwd in root.findall(".//kwd"): + if kwd.text: + keywords.append(kwd.text.strip()) + metadata["keywords"] = keywords + + sections: list[dict[str, str]] = [] + for sec in root.findall(".//sec"): + title = sec.findtext("title") + body_text = ET.tostring(sec, encoding="unicode", method="text").strip() + if title or body_text: + sections.append( + {"title": title.strip() if title else "", "text": body_text} + ) + metadata["sections"] = sections + + references: list[dict[str, str]] = [] + for ref in root.findall(".//ref"): + citation = ref.findtext(".//mixed-citation") or ET.tostring( + ref, encoding="unicode", method="text" + ) + references.append({"citation": citation.strip()}) + metadata["references"] = references + + tables: list[dict[str, str]] = [] + for table in root.findall(".//table"): + label = table.findtext(".//label") + caption = table.findtext(".//caption") + tables.append( + { + "label": label.strip() if label else "", + "caption": caption.strip() if caption else "", + } + ) + metadata["tables"] = tables + + figures: list[dict[str, str]] = [] + for fig in root.findall(".//fig"): + label = fig.findtext(".//label") + caption = fig.findtext(".//caption") + figures.append( + { + "label": label.strip() if label else "", + "caption": caption.strip() if caption else "", + } + ) + metadata["figures"] = figures + + supplementary: list[str] = [] + for sup in root.findall(".//supplementary-material"): + if sup.text: + supplementary.append(sup.text.strip()) + metadata["supplementary"] = supplementary + + return metadata + + +def _parse_html_metadata(html_text: str) -> dict[str, object]: + metadata: dict[str, object] = {} + parser = _HTMLMetadataParser() + parser.feed(html_text) + metadata["title"] = parser.metadata.get("title") + metadata["authors"] = parser.metadata.get("authors", []) + metadata["affiliations"] = [] + metadata["doi"] = parser.metadata.get("citation_doi") + metadata["pmid"] = parser.metadata.get("citation_pmid") + metadata["pmcid"] = parser.metadata.get("citation_pmcid") + metadata["abstract"] = parser.metadata.get("description") + metadata["keywords"] = parser.metadata.get("keywords", []) + metadata["sections"] = parser.sections + metadata["references"] = parser.references + metadata["tables"] = parser.tables + metadata["figures"] = parser.figures + metadata["supplementary"] = [] + return metadata + + +class _HTMLMetadataParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.metadata: dict[str, object] = {} + self.sections: list[dict[str, str]] = [] + self.references: list[dict[str, str]] = [] + self.tables: list[dict[str, str]] = [] + self.figures: list[dict[str, str]] = [] + self._current_tag: str | None = None + self._current_attrs: dict[str, str] = {} + self._current_data: list[str] = [] + self._heading_text: str | None = None + self._collect_heading: bool = False + self._section_texts: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attrs_dict = {name: value for name, value in attrs if value is not None} + self._current_tag = tag + self._current_attrs = attrs_dict + + if tag == "meta" and "name" in attrs_dict and "content" in attrs_dict: + name = attrs_dict["name"].lower() + content = attrs_dict["content"].strip() + if name == "keywords": + keywords = self.metadata.setdefault("keywords", []) + if isinstance(keywords, list): + keywords.extend( + [kw.strip() for kw in content.split(",") if kw.strip()] + ) + elif name in { + "author", + "citation_doi", + "citation_pmid", + "citation_pmcid", + "description", + "title", + }: + if name == "author": + authors = self.metadata.setdefault("authors", []) + if isinstance(authors, list): + authors.append(content) + else: + self.metadata[name] = content + + if tag == "title": + self._current_tag = "title" + self._current_data = [] + + if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}: + self._heading_text = "" + self._collect_heading = True + self._section_texts = [] + + if tag == "a" and "href" in attrs_dict: + self._current_data = [] + + if tag in {"table", "figure"}: + self._current_data = [] + + def handle_endtag(self, tag: str) -> None: + if tag == "title" and self._current_tag == "title": + self._current_tag = None + + if tag in {"h1", "h2", "h3", "h4", "h5", "h6"} and self._collect_heading: + self._collect_heading = False + if self._heading_text is not None: + self.sections.append( + { + "title": self._heading_text.strip(), + "text": "".join(self._section_texts).strip(), + } + ) + self._section_texts = [] + self._heading_text = None + + if tag == "a" and self._current_tag == "a" and self._current_attrs.get("href"): + text = "".join(self._current_data).strip() + self.references.append({"href": self._current_attrs["href"], "text": text}) + self._current_data = [] + + if tag == "table" and self._current_tag == "table": + self.tables.append({"caption": ""}) + self._current_data = [] + + if tag == "figure" and self._current_tag == "figure": + self.figures.append({"caption": ""}) + self._current_data = [] + + super().handle_endtag(tag) + + def handle_data(self, data: str) -> None: + if self._current_tag == "title": + title = self.metadata.get("title", "") + if not isinstance(title, str): + title = "" + self.metadata["title"] = title + data + + if self._collect_heading and self._heading_text is not None: + self._heading_text += data + self._section_texts.append(data) + + if self._current_tag in {"a", "table", "figure"}: + self._current_data.append(data) + + def feed(self, data: str) -> None: + super().feed(data) + + +def _parse_pdf_metadata(stream: BinaryIO) -> dict[str, object]: + if PyPDF2 is None: + raise DocumentParseError("PDF parser dependency is missing") + + try: + reader = PyPDF2.PdfReader(stream) + except Exception as exc: + parser_metrics.increment("parser.parse_errors") + raise DocumentParseError("failed to parse PDF document") from exc + + info = reader.metadata + metadata: dict[str, object] = { + "title": None, + "authors": [], + "affiliations": [], + "doi": None, + "pmid": None, + "pmcid": None, + "abstract": None, + "keywords": [], + "sections": [], + "references": [], + "tables": [], + "figures": [], + "supplementary": [], + } + + if info is not None: + metadata["title"] = getattr(info, "/Title", None) or getattr( + info, "title", None + ) + author = getattr(info, "/Author", None) or getattr(info, "author", None) + if author: + metadata["authors"] = [author] + + try: + pages = list(reader.pages) + if pages: + text = pages[0].extract_text() or "" + snippet = text.strip().split("\n\n", 1)[0] + metadata["abstract"] = snippet if snippet else None + except (AttributeError, IndexError, OSError, ValueError): + metadata["abstract"] = None + + return metadata + + +def _read_as_text(stream: BinaryIO) -> str: + current_position = None + try: + current_position = stream.tell() + except (AttributeError, OSError): + pass + + content = stream.read() + if isinstance(content, bytes): + try: + text = content.decode("utf-8", errors="ignore") + except Exception as exc: + raise DocumentParseError("failed to decode document content") from exc + elif isinstance(content, str): + text = content + else: + raise DocumentParseError("unsupported stream content type") + + try: + stream.seek(current_position or 0) + except (AttributeError, OSError): + pass + return text + + +def _fingerprint_stream(stream: BinaryIO) -> str: + current_position = None + try: + current_position = stream.tell() + except (AttributeError, OSError): + pass + + hasher = hashlib.sha256() + while True: + chunk = stream.read(8192) + if not chunk: + break + if isinstance(chunk, str): + chunk = chunk.encode("utf-8", errors="ignore") + hasher.update(chunk) + + try: + stream.seek(current_position or 0) + except (AttributeError, OSError): + pass + + return hasher.hexdigest() + + +def parse_document(format_name: str, stream: BinaryIO) -> dict[str, object]: + lowercase_format = format_name.strip().lower() + if lowercase_format not in SUPPORTED_FORMATS: + raise DocumentParseError(f"unsupported document format: {format_name}") + + if duplicate_detector.register(stream): + parser_metrics.increment("parser.duplicate_documents") + raise DuplicateDocumentError("duplicate document detected") + + parser_metrics.increment("parser.documents_parsed") + + if lowercase_format == "pdf": + metadata = _parse_pdf_metadata(stream) + else: + html_text = _read_as_text(stream) + + if lowercase_format in {"xml", "nxml", "jats"}: + try: + root = ET.fromstring(html_text) + metadata = _parse_xml_metadata(root) + except ET.ParseError: + parser_metrics.increment("parser.parse_errors") + raise DocumentParseError("failed to parse XML document") + else: + try: + metadata = _parse_html_metadata(html_text) + except (TypeError, ValueError, OSError): + parser_metrics.increment("parser.parse_errors") + metadata = { + "title": None, + "authors": [], + "affiliations": [], + "doi": None, + "pmid": None, + "pmcid": None, + "abstract": None, + "keywords": [], + "sections": [], + "references": [], + "tables": [], + "figures": [], + "supplementary": [], + } + + try: + _validate_metadata(metadata) + except DocumentParseError: + parser_metrics.increment("parser.validation_failures") + raise + + parser_metrics.increment("parser.documents_validated") + return metadata diff --git a/services/literature/app/routers/__init__.py b/services/literature/app/routers/__init__.py index e69de29..f7781f8 100644 --- a/services/literature/app/routers/__init__.py +++ b/services/literature/app/routers/__init__.py @@ -0,0 +1,3 @@ +from app.routers.health import router as health_router + +__all__ = ["health_router"] diff --git a/services/literature/app/routers/documents.py b/services/literature/app/routers/documents.py new file mode 100644 index 0000000..9b2c13c --- /dev/null +++ b/services/literature/app/routers/documents.py @@ -0,0 +1,32 @@ +import io + +from fastapi import APIRouter, Depends, HTTPException + +from app.core.security import get_current_user +from app.document_schemas import DocumentMetadata +from app.parsing import DuplicateDocumentError, parse_document, parser_metrics +from app.schemas import DocumentParseRequest, DocumentParseResponse + +router = APIRouter(prefix="/documents", tags=["Documents"]) + +auth_dependency = Depends(get_current_user) + + +@router.post("/parse", response_model=DocumentParseResponse) +async def parse_document_endpoint( + request: DocumentParseRequest, + auth_payload: dict[str, str] = auth_dependency, +) -> DocumentParseResponse: + try: + stream = io.BytesIO(request.content.encode("utf-8")) + metadata = parse_document(request.format, stream) + except DuplicateDocumentError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + return DocumentParseResponse( + metadata=DocumentMetadata.model_validate(metadata), + duplicate=False, + metrics=parser_metrics.snapshot(), + ) diff --git a/services/literature/app/routers/health.py b/services/literature/app/routers/health.py new file mode 100644 index 0000000..3e01f6b --- /dev/null +++ b/services/literature/app/routers/health.py @@ -0,0 +1,134 @@ +from datetime import datetime, timezone +from typing import Any + +import asyncpg +import httpx +from fastapi import APIRouter +from fastapi.responses import PlainTextResponse + +from app.core.config import get_settings +from app.database.postgres import postgres_manager +from app.observability.metrics import generate_prometheus_metrics +from app.orchestrator.manager import orchestrator +from app.parsing import parser_metrics + +settings = get_settings() +router = APIRouter(tags=["Health"]) +service_start_time = datetime.now(timezone.utc) + + +def _uptime_seconds() -> int: + return int((datetime.now(timezone.utc) - service_start_time).total_seconds()) + + +async def _db_ready() -> tuple[bool, str]: + if settings.environment == "test": + return True, "test environment" + if postgres_manager.pool is None: + if postgres_manager.last_error is not None: + return False, f"connection pool unavailable: {postgres_manager.last_error}" + return False, "connection pool uninitialized" + try: + async with postgres_manager.acquire() as connection: + result = await connection.fetchval("SELECT 1") + return True, f"db responded: {result}" + except asyncpg.PostgresError as exc: + return False, str(exc) + + +async def _service_available(url: str) -> tuple[bool, str]: + try: + async with httpx.AsyncClient(timeout=2.0) as client: + response = await client.get(url) + if response.status_code == 200: + return True, "ok" + return False, f"status={response.status_code}" + except httpx.HTTPError as exc: + return False, str(exc) + + +def _orchestrator_ready() -> tuple[bool, str]: + try: + metrics = orchestrator.get_metrics() + if isinstance(metrics, dict): + return True, "ok" + return False, "invalid metrics payload" + except (ValueError, RuntimeError, TypeError) as exc: + return False, str(exc) + + +@router.get("/healthz") +def healthz() -> dict[str, str]: + return {"status": "ok", "service": "literature"} + + +@router.get("/health") +def health() -> dict[str, str]: + return {"status": "ok", "service": "literature"} + + +@router.get("/ready") +async def ready() -> dict[str, Any]: + db_ready, db_details = await _db_ready() + search_ready, search_details = await _service_available( + f"{settings.search_service_url}/api/v1/health" + ) + kg_ready, kg_details = await _service_available( + f"{settings.kg_service_url}/api/v1/health" + ) + orchestrator_ready, orchestrator_details = _orchestrator_ready() + all_ready = db_ready and search_ready and kg_ready and orchestrator_ready + return { + "status": "ok" if all_ready else "fail", + "service": "literature", + "ready": all_ready, + "dependencies": { + "postgresql": { + "status": "ok" if db_ready else "fail", + "details": db_details, + }, + "search_service": { + "status": "ok" if search_ready else "fail", + "details": search_details, + }, + "kg_service": { + "status": "ok" if kg_ready else "fail", + "details": kg_details, + }, + "orchestrator": { + "status": "ok" if orchestrator_ready else "fail", + "details": orchestrator_details, + }, + }, + "metrics": { + "uptime_seconds": _uptime_seconds(), + "parser_metrics": parser_metrics.snapshot(), + "orchestrator_metrics": orchestrator.get_metrics(), + }, + } + + +@router.get("/live") +def live() -> dict[str, Any]: + return {"status": "ok", "service": "literature", "live": True} + + +@router.get("/metrics") +def metrics() -> dict[str, Any]: + return { + "service": "literature", + "environment": settings.environment, + "uptime_seconds": _uptime_seconds(), + "parser_metrics": parser_metrics.snapshot(), + "orchestrator_metrics": orchestrator.get_metrics(), + } + + +@router.get("/metrics/prometheus", response_class=PlainTextResponse) +def metrics_prometheus() -> bytes: + return generate_prometheus_metrics() + + +@router.get("/api/v1/health") +def api_health() -> dict[str, str]: + return {"status": "ok", "service": "literature"} diff --git a/services/literature/app/routers/ingestion.py b/services/literature/app/routers/ingestion.py new file mode 100644 index 0000000..de21908 --- /dev/null +++ b/services/literature/app/routers/ingestion.py @@ -0,0 +1,250 @@ +import json +from datetime import datetime, timezone +from typing import Any +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Depends, HTTPException + +from app.core.security import get_current_user +from app.database.postgres import postgres_manager +from app.orchestrator.manager import IngestionJob as OrchestrationJob +from app.orchestrator.manager import JobStatus, orchestrator +from app.schemas import IngestionJob as IngestionJobSchema +from app.schemas import IngestionRequest + +router = APIRouter(prefix="/ingestion", tags=["Ingestion"]) + +auth_dependency = Depends(get_current_user) + + +@router.post("", response_model=IngestionJobSchema, status_code=202) +async def start_ingestion( + req: IngestionRequest, + auth_payload: dict[str, str] = auth_dependency, +) -> IngestionJobSchema: + job_id = uuid4() + created_at = datetime.now(timezone.utc) + async with postgres_manager.acquire() as connection: + await connection.execute( + """ + INSERT INTO literature_ingestion_jobs ( + id, + source, + query, + status, + created_at, + schedule, + next_run_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) + """, + job_id, + req.source, + req.query, + JobStatus.SCHEDULED.value if req.schedule else JobStatus.QUEUED.value, + created_at, + req.schedule, + None, + ) + + job = OrchestrationJob( + job_id=str(job_id), + source=req.source, + query=req.query, + status=JobStatus.SCHEDULED if req.schedule else JobStatus.QUEUED, + created_at=created_at, + schedule=req.schedule, + ) + + if req.schedule: + await orchestrator.schedule(job, req.schedule) + else: + orchestrator.enqueue(job) + await orchestrator.start() + + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, schedule, next_run_at, + documents_total, documents_processed, + documents_failed, retry_count, backoff_until, + error_message, dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + + job_response = _row_to_job_schema(row) + if job_response is None: + raise HTTPException( + status_code=500, detail="failed to load created ingestion job" + ) + return job_response + + +def _row_to_job_schema(row: Any) -> IngestionJobSchema | None: + if row is None: + return None + + record = dict(row) + dead_letter_items = record.get("dead_letter_items") + if isinstance(dead_letter_items, str): + record["dead_letter_items"] = json.loads(dead_letter_items) + elif dead_letter_items is None: + record["dead_letter_items"] = [] + return IngestionJobSchema(**record) + + +@router.get("/{job_id}", response_model=IngestionJobSchema) +async def get_ingestion_job( + job_id: UUID, + auth_payload: dict[str, str] = auth_dependency, +) -> IngestionJobSchema: + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, schedule, next_run_at, + documents_total, documents_processed, + documents_failed, retry_count, backoff_until, + error_message, dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + + job = _row_to_job_schema(row) + if job is None: + raise HTTPException(status_code=404, detail="ingestion job not found") + return job + + +@router.post("/{job_id}/trigger", response_model=IngestionJobSchema) +async def trigger_ingestion_job( + job_id: UUID, + auth_payload: dict[str, str] = auth_dependency, +) -> IngestionJobSchema: + job = await orchestrator._load_job_from_db(str(job_id)) + if job is None: + raise HTTPException(status_code=404, detail="ingestion job not found") + + job.status = JobStatus.QUEUED + orchestrator.enqueue(job) + await orchestrator.start() + + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, schedule, next_run_at, + documents_total, documents_processed, + documents_failed, retry_count, backoff_until, + error_message, dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + + if row is None: + raise HTTPException( + status_code=500, detail="failed to load ingestion job after trigger" + ) + + job_response = _row_to_job_schema(row) + assert job_response is not None + return job_response + + +@router.post("/{job_id}/retry", response_model=IngestionJobSchema) +async def retry_ingestion_job( + job_id: UUID, + auth_payload: dict[str, str] = auth_dependency, +) -> IngestionJobSchema: + try: + await orchestrator.trigger_retry(str(job_id)) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, schedule, next_run_at, + documents_total, documents_processed, + documents_failed, retry_count, backoff_until, + error_message, dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + + job = _row_to_job_schema(row) + if job is None: + raise HTTPException(status_code=404, detail="ingestion job not found") + return job + + +@router.post("/{job_id}/cancel", response_model=IngestionJobSchema) +async def cancel_ingestion_job( + job_id: UUID, + auth_payload: dict[str, str] = auth_dependency, +) -> IngestionJobSchema: + try: + await orchestrator.cancel(str(job_id)) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, source, query, status, created_at, + started_at, completed_at, documents_total, + documents_processed, documents_failed, retry_count, + backoff_until, error_message, schedule, next_run_at, + dead_letter_count, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + + job = _row_to_job_schema(row) + if job is None: + raise HTTPException(status_code=404, detail="ingestion job not found") + return job + + +@router.get("/{job_id}/dead-letter") +async def get_dead_letter_items( + job_id: UUID, + auth_payload: dict[str, str] = auth_dependency, +) -> dict[str, object]: + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, dead_letter_items + FROM literature_ingestion_jobs + WHERE id = $1 + """, + job_id, + ) + + if row is None: + raise HTTPException(status_code=404, detail="ingestion job not found") + + record = dict(row) + dead_letter_items = record.get("dead_letter_items") + if isinstance(dead_letter_items, str): + dead_letter_items = json.loads(dead_letter_items) + elif dead_letter_items is None: + dead_letter_items = [] + + return { + "items": dead_letter_items, + "total": len(dead_letter_items), + } diff --git a/services/literature/app/routers/nlp.py b/services/literature/app/routers/nlp.py new file mode 100644 index 0000000..db5779a --- /dev/null +++ b/services/literature/app/routers/nlp.py @@ -0,0 +1,21 @@ +from fastapi import APIRouter, Depends, HTTPException + +from app.core.security import get_current_user +from app.document_schemas import DocumentMetadata +from app.nlp.pipeline import process_document + +router = APIRouter(prefix="/nlp", tags=["NLP"]) + +auth_dependency = Depends(get_current_user) + + +@router.post("/process") +async def nlp_process( + doc: DocumentMetadata, + auth_payload: dict[str, str] = auth_dependency, +): + try: + result = process_document(doc.model_dump()) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return result diff --git a/services/literature/app/routers/papers.py b/services/literature/app/routers/papers.py new file mode 100644 index 0000000..a219e0e --- /dev/null +++ b/services/literature/app/routers/papers.py @@ -0,0 +1,59 @@ +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query + +from app.core.security import get_current_user +from app.database.postgres import postgres_manager +from app.schemas import Paper + +router = APIRouter(prefix="/papers", tags=["Papers"]) + +auth_dependency = Depends(get_current_user) + + +@router.get("", response_model=dict) +async def list_papers( + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), +) -> dict[str, object]: + offset = (page - 1) * page_size + async with postgres_manager.acquire() as connection: + rows = await connection.fetch( + """ + SELECT id::text, title, source, doi, published_at, citation_count + FROM literature_papers + ORDER BY created_at DESC + LIMIT $1 OFFSET $2 + """, + page_size, + offset, + ) + total = await connection.fetchval("SELECT COUNT(*) FROM literature_papers") + + return { + "items": [dict(row) for row in rows], + "total": total, + "page": page, + "pageSize": page_size, + } + + +@router.get("/{paper_id}", response_model=Paper) +async def get_paper( + paper_id: UUID, + auth_payload: dict[str, str] = auth_dependency, +) -> Paper: + async with postgres_manager.acquire() as connection: + row = await connection.fetchrow( + """ + SELECT id::text, title, source, doi, published_at, citation_count + FROM literature_papers + WHERE id = $1 + """, + paper_id, + ) + + if row is None: + raise HTTPException(status_code=404, detail="paper not found") + + return Paper(**dict(row)) diff --git a/services/literature/app/schemas.py b/services/literature/app/schemas.py new file mode 100644 index 0000000..2b16fd4 --- /dev/null +++ b/services/literature/app/schemas.py @@ -0,0 +1,54 @@ +from datetime import datetime +from typing import Any, Literal + +from pydantic import BaseModel, Field + +from app.document_schemas import DocumentMetadata + + +class Paper(BaseModel): + id: str + title: str + source: Literal["pubmed", "biorxiv", "medrxiv", "patent", "conference"] + doi: str | None = None + published_at: datetime | None = None + citation_count: int = 0 + + +class IngestionRequest(BaseModel): + source: Literal["pubmed", "biorxiv", "medrxiv", "patent", "conference"] + query: str + schedule: str | None = None + + +class IngestionJob(BaseModel): + id: str + source: str + query: str + status: Literal[ + "queued", "scheduled", "running", "completed", "failed", "cancelled" + ] + created_at: datetime + started_at: datetime | None = None + completed_at: datetime | None = None + schedule: str | None = None + next_run_at: datetime | None = None + documents_total: int = 0 + documents_processed: int = 0 + documents_failed: int = 0 + retry_count: int = 0 + backoff_until: datetime | None = None + error_message: str | None = None + dead_letter_count: int = 0 + dead_letter_items: list[dict[str, Any]] = Field(default_factory=list) + + +class DocumentParseRequest(BaseModel): + format: Literal["pdf", "xml", "html", "nxml", "jats"] + content: str + + +class DocumentParseResponse(BaseModel): + metadata: DocumentMetadata + duplicate: bool = False + metrics: dict[str, int] = Field(default_factory=dict) diff --git a/services/literature/app/services/__init__.py b/services/literature/app/services/__init__.py new file mode 100644 index 0000000..d599a30 --- /dev/null +++ b/services/literature/app/services/__init__.py @@ -0,0 +1,11 @@ +from app.services.evidence_ranking import EvidenceRankingService +from app.services.kg_integration import KGIntegrationService +from app.services.llmwiki_integration import LLMWikiIntegrationService +from app.services.search_integration import SearchIntegrationService + +__all__ = [ + "EvidenceRankingService", + "KGIntegrationService", + "LLMWikiIntegrationService", + "SearchIntegrationService", +] diff --git a/services/literature/app/services/evidence_ranking.py b/services/literature/app/services/evidence_ranking.py new file mode 100644 index 0000000..2124c52 --- /dev/null +++ b/services/literature/app/services/evidence_ranking.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import time +from typing import Any + +from app.observability.metrics import ( + EVIDENCE_RANKING_DURATION_SECONDS, + EVIDENCE_RANKING_ERRORS_TOTAL, + EVIDENCE_RANKING_TOTAL, +) + + +class EvidenceRankingService: + """Rank literature-derived evidence records from prior NLP stages. + + The service is intentionally lightweight: it accepts parser + NLP output, + aggregates confidence, deduplicates overlapping evidence, preserves provenance, + and returns a structured ranking payload for downstream consumers. + """ + + def __init__(self, strategy: str = "hybrid") -> None: + self.strategy = strategy + + def rank_evidence( + self, document: dict[str, Any], nlp_result: dict[str, Any] + ) -> dict[str, Any]: + start_time = time.perf_counter() + if not isinstance(document, dict): + EVIDENCE_RANKING_ERRORS_TOTAL.inc() + raise ValueError("document must be a dictionary") # noqa: TRY004 + if not isinstance(nlp_result, dict): + EVIDENCE_RANKING_ERRORS_TOTAL.inc() + raise ValueError("nlp_result must be a dictionary") # noqa: TRY004 + + entities = ( + nlp_result.get("entities") or nlp_result.get("detected_entities") or [] + ) + relationships = nlp_result.get("relationships") or [] + + if not entities and not relationships: + return { + "document_id": document.get("document_id") + or document.get("id") + or "unknown", + "evidence_items": [], + "ranking_metrics": { + "total_evidence_items": 0, + "deduplicated_items": 0, + "strategy": self.strategy, + }, + } + + evidence_items: list[dict[str, Any]] = [] + for entity in entities: + if not isinstance(entity, dict): + continue + evidence_items.append( + self._build_entity_evidence(document, nlp_result, entity) + ) + + for relationship in relationships: + if not isinstance(relationship, dict): + continue + evidence_items.append( + self._build_relationship_evidence(document, nlp_result, relationship) + ) + + deduped = self._deduplicate(evidence_items) + ranked = self._apply_ranking(deduped) + + elapsed = time.perf_counter() - start_time + EVIDENCE_RANKING_TOTAL.inc() + EVIDENCE_RANKING_DURATION_SECONDS.observe(elapsed) + return { + "document_id": document.get("document_id") + or document.get("id") + or "unknown", + "evidence_items": ranked, + "ranking_metrics": { + "total_evidence_items": len(evidence_items), + "deduplicated_items": len(ranked), + "strategy": self.strategy, + }, + } + + def _build_entity_evidence( + self, + document: dict[str, Any], + nlp_result: dict[str, Any], + entity: dict[str, Any], + ) -> dict[str, Any]: + confidence = float(entity.get("confidence_score") or 0.0) + return { + "evidence_id": self._build_id(document, entity, "entity"), + "document_id": document.get("document_id") + or document.get("id") + or "unknown", + "supporting_entities": [ + entity.get("text") or entity.get("name") or "unknown" + ], + "supporting_relationships": [], + "overall_confidence": confidence, + "ranking_score": self._score(confidence, 0.0, 0.0), + "provenance": { + "source_document": document.get("title") + or document.get("document_id") + or "unknown", + "source_sentence": nlp_result.get("sentences", [""])[0] + if nlp_result.get("sentences") + else "", + "ontology_source": entity.get("ontology_source"), + "normalized_identifier": entity.get("normalized_identifier"), + }, + "processing_metadata": { + "evidence_type": "entity", + "stage": "ranking", + }, + } + + def _build_relationship_evidence( + self, + document: dict[str, Any], + nlp_result: dict[str, Any], + relationship: dict[str, Any], + ) -> dict[str, Any]: + confidence = float(relationship.get("confidence") or 0.0) + relation_text = relationship.get("predicate") or "related" + supporting_entities = [ + relationship.get("source_entity") + or relationship.get("source") + or relationship.get("subject") + or "unknown", + relationship.get("target_entity") + or relationship.get("target") + or relationship.get("object") + or "unknown", + ] + return { + "evidence_id": self._build_id(document, relationship, "relationship"), + "document_id": document.get("document_id") + or document.get("id") + or "unknown", + "supporting_entities": supporting_entities, + "supporting_relationships": [relation_text], + "overall_confidence": confidence, + "ranking_score": self._score(confidence, 0.1, 0.1), + "provenance": { + "source_document": document.get("title") + or document.get("document_id") + or "unknown", + "source_sentence": relationship.get("provenance", {}).get( + "source_sentence" + ) + if isinstance(relationship.get("provenance"), dict) + else "", + "ontology_source": None, + "normalized_identifier": None, + }, + "processing_metadata": { + "evidence_type": "relationship", + "stage": "ranking", + }, + } + + def _deduplicate( + self, evidence_items: list[dict[str, Any]] + ) -> list[dict[str, Any]]: + seen = set() + deduped: list[dict[str, Any]] = [] + for item in evidence_items: + key = tuple( + sorted( + [ + str(item.get("evidence_id")), + *[str(x) for x in item.get("supporting_entities", [])], + ] + ) + ) + if key in seen: + continue + seen.add(key) + deduped.append(item) + return deduped + + def _apply_ranking( + self, evidence_items: list[dict[str, Any]] + ) -> list[dict[str, Any]]: + ranked = sorted( + evidence_items, + key=lambda item: item.get("ranking_score", 0.0), + reverse=True, + ) + for index, item in enumerate(ranked, start=1): + item["rank"] = index + return ranked + + def _score( + self, confidence: float, entity_boost: float, relationship_boost: float + ) -> float: + return round( + min(1.0, max(0.0, confidence + entity_boost + relationship_boost)), 4 + ) + + def _build_id( + self, document: dict[str, Any], payload: dict[str, Any], evidence_type: str + ) -> str: + identifier = f"{document.get('document_id') or document.get('id') or 'unknown'}:{evidence_type}:{payload.get('text') or payload.get('predicate') or payload.get('name') or 'unknown'}" + return identifier.lower().replace(" ", "-") diff --git a/services/literature/app/services/kg_integration.py b/services/literature/app/services/kg_integration.py new file mode 100644 index 0000000..ab863a0 --- /dev/null +++ b/services/literature/app/services/kg_integration.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import hashlib +import time +import uuid +from typing import Any + +import httpx + +from app.core.config import get_settings +from app.observability.metrics import ( + KG_HANDOFF_DURATION_SECONDS, + KG_HANDOFF_ERRORS_TOTAL, + KG_HANDOFF_RETRIES_TOTAL, + KG_HANDOFF_TOTAL, +) +from app.utils.logging import get_logger + +logger = get_logger(__name__) + + +class KGIntegrationService: + """Thin client for publishing literature-derived graph events to the KG service. + + The service translates literature entities and relationships into KG node and + relationship payloads, then hands them off to the existing KG service. It does + not implement graph storage, traversal, CRUD, or schema management itself. + """ + + def __init__( + self, + base_url: str | None = None, + timeout_seconds: int | None = None, + max_retries: int | None = None, + client: httpx.Client | None = None, + ) -> None: + settings = get_settings() + self.base_url = base_url or settings.kg_service_url + self.timeout_seconds = timeout_seconds or settings.kg_service_timeout_seconds + self.max_retries = max_retries or settings.kg_service_max_retries + self.client = client or httpx.Client(timeout=self.timeout_seconds) + + def build_graph_payload( + self, payload: dict[str, Any], tenant: dict[str, Any] | None = None + ) -> dict[str, Any]: + if not isinstance(payload, dict): + raise TypeError("payload must be a dictionary") + self._validate_payload(payload) + + document_id = str(payload.get("document_id") or "") + document_title = str(payload.get("document_title") or document_id) + event_id = str(payload.get("event_id") or self._build_event_id(payload)) + source = str(payload.get("source") or "literature-service") + + nodes: list[dict[str, Any]] = [] + document_node = { + "id": self._build_uuid(f"document:{document_id}"), + "label": "Publication", + "name": document_title, + "description": f"Literature document {document_id}", + "source": source, + "metadata": { + "document_id": document_id, + "document_title": document_title, + "tenant": tenant or {}, + }, + } + nodes.append(document_node) + + for entity in payload.get("entities", []): + text = str(entity.get("text") or entity.get("name") or "unknown") + node_id = self._build_uuid(f"entity:{document_id}:{text}") + nodes.append( + { + "id": node_id, + "label": self._infer_label(entity), + "name": text, + "description": entity.get("normalized_identifier") + or entity.get("ontology_source") + or text, + "source": source, + "metadata": { + "document_id": document_id, + "entity_text": text, + "normalized_identifier": entity.get("normalized_identifier"), + "ontology_source": entity.get("ontology_source"), + "confidence_score": entity.get("confidence_score"), + }, + } + ) + + relationships: list[dict[str, Any]] = [] + entities_payload = payload.get("entities") + for relationship in payload.get("relationships", []): + predicate = str(relationship.get("predicate") or "generate") + source_entity = ( + relationship.get("source_entity") + or relationship.get("source") + or relationship.get("subject") + or None + ) + target_entity = ( + relationship.get("target_entity") + or relationship.get("target") + or relationship.get("object") + or None + ) + if ( + source_entity is None + and isinstance(entities_payload, list) + and entities_payload + ): + first_entity = entities_payload[0] + if isinstance(first_entity, dict): + source_entity = first_entity.get("text") + if ( + target_entity is None + and isinstance(entities_payload, list) + and entities_payload + ): + last_entity = entities_payload[-1] + if isinstance(last_entity, dict): + target_entity = last_entity.get("text") + relationships.append( + { + "id": self._build_uuid( + f"relationship:{document_id}:{predicate}:{source_entity or 'unknown'}:{target_entity or 'unknown'}" + ), + "from_node_id": self._build_uuid( + f"entity:{document_id}:{source_entity or text}" + ), + "to_node_id": self._build_uuid( + f"entity:{document_id}:{target_entity or text}" + ), + "type": self._normalize_relationship_type(predicate), + "evidence": relationship.get("evidence") or predicate, + "confidence": relationship.get("confidence") + or relationship.get("confidence_score"), + "source": source, + "created_at": None, + } + ) + + return { + "event_id": event_id, + "document_id": document_id, + "document_title": document_title, + "nodes": nodes, + "relationships": relationships, + "source": source, + } + + def publish_graph_payload( + self, payload: dict[str, Any], tenant: dict[str, Any] | None = None + ) -> dict[str, Any]: + if not isinstance(payload, dict): + raise TypeError("payload must be a dictionary") + graph_payload = self.build_graph_payload(payload, tenant=tenant) + event_id = graph_payload["event_id"] + + start_time = time.perf_counter() + last_error: Exception | None = None + attempt = 0 + while attempt < self.max_retries: + try: + for node in graph_payload.get("nodes", []): + response = self.client.post( + f"{self.base_url}/api/v1/graph/nodes", json=node + ) + if self._is_duplicate_error(response): + continue + response.raise_for_status() + for relationship in graph_payload.get("relationships", []): + response = self.client.post( + f"{self.base_url}/api/v1/graph/relationships", json=relationship + ) + if self._is_duplicate_error(response): + continue + response.raise_for_status() + KG_HANDOFF_TOTAL.labels(status="success").inc() + KG_HANDOFF_DURATION_SECONDS.observe(time.perf_counter() - start_time) + return { + "event_id": event_id, + "published": True, + "status": "published", + "metrics": {"retries": attempt, "failures": 0}, + } + except httpx.HTTPError as exc: + last_error = exc + attempt += 1 + if attempt >= self.max_retries: + KG_HANDOFF_ERRORS_TOTAL.inc() + KG_HANDOFF_TOTAL.labels(status="error").inc() + KG_HANDOFF_DURATION_SECONDS.observe( + time.perf_counter() - start_time + ) + break + KG_HANDOFF_RETRIES_TOTAL.inc() + time.sleep(0.2 * attempt) + + raise ( + RuntimeError(f"Failed to publish graph payload: {last_error}") + if last_error + else RuntimeError("Failed to publish graph payload") + ) + + def _validate_payload(self, payload: dict[str, Any]) -> None: + if not payload.get("document_id"): + raise ValueError("payload.document_id is required") + if not isinstance(payload.get("entities", []), list): + raise ValueError("payload.entities must be a list") # noqa: TRY004 + if not isinstance(payload.get("relationships", []), list): + raise ValueError("payload.relationships must be a list") # noqa: TRY004 + for entity in payload.get("entities", []): + if not isinstance(entity, dict): + raise TypeError("each entity entry must be a dictionary") + if not entity.get("text"): + raise ValueError("each entity requires text") + for relationship in payload.get("relationships", []): + if not isinstance(relationship, dict): + raise TypeError("each relationship entry must be a dictionary") + if not relationship.get("predicate"): + raise ValueError("each relationship requires a predicate") + + def _build_event_id(self, payload: dict[str, Any]) -> str: + source = f"{payload.get('document_id')}:{payload.get('document_title', '')}" + return hashlib.sha256(source.encode("utf-8")).hexdigest()[:16] + + def _build_uuid(self, value: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_URL, value)) + + def _normalize_relationship_type(self, predicate: str) -> str: + normalized = predicate.strip().upper() + mapping = { + "TREATS": "TREATS", + "TARGETS": "TARGETS", + "INTERACTS": "INTERACTS", + "INTERACTS_WITH": "INTERACTS", + "ASSOCIATED_WITH": "GENERATE", + "ASSOCIATES_WITH": "GENERATE", + "RELATES_TO": "GENERATE", + } + return mapping.get(normalized, "GENERATE") + + def _infer_label(self, entity: dict[str, Any]) -> str: + text = str(entity.get("text") or "").lower() + if "disease" in text or "syndrome" in text: + return "Disease" + if "protein" in text or "enzyme" in text: + return "Protein" + if "gene" in text: + return "Gene" + if "drug" in text or "compound" in text or "molecule" in text: + return "Drug" + return "Target" + + def _is_duplicate_error(self, response: httpx.Response) -> bool: + if response is None: + return False + try: + if response.status_code == 409: + return True + except AttributeError: + return False + return False diff --git a/services/literature/app/services/llmwiki_integration.py b/services/literature/app/services/llmwiki_integration.py new file mode 100644 index 0000000..896e07f --- /dev/null +++ b/services/literature/app/services/llmwiki_integration.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import time +from typing import Any + +import httpx + +from app.core.config import get_settings +from app.observability.metrics import ( + LLMWIKI_UPDATE_DURATION_SECONDS, + LLMWIKI_UPDATE_ERRORS_TOTAL, + LLMWIKI_UPDATE_RETRIES_TOTAL, + LLMWIKI_UPDATE_TOTAL, +) +from app.utils.logging import get_logger + +logger = get_logger(__name__) +settings = get_settings() + + +class LLMWikiIntegrationService: + """Thin client for sending structured literature knowledge to LLM Wiki.""" + + def __init__( + self, + base_url: str | None = None, + timeout_seconds: int | None = None, + max_retries: int | None = None, + client: httpx.Client | None = None, + ) -> None: + self.base_url = base_url or settings.llmwiki_service_url + self.timeout_seconds = ( + timeout_seconds or settings.llmwiki_service_timeout_seconds + ) + self.max_retries = max_retries or settings.llmwiki_service_max_retries + self.client = client or httpx.Client(timeout=self.timeout_seconds) + + def update_knowledge( + self, payload: dict[str, Any], tenant: dict[str, Any] | None = None + ) -> dict[str, Any]: + if not isinstance(payload, dict): + raise TypeError("payload must be a dictionary") + if ( + not payload.get("entities") + and not payload.get("relationships") + and not payload.get("summary") + ): + raise ValueError( + "payload must include at least entities, relationships, or summary" + ) + + request_payload = { + "document_id": str( + payload.get("document_id") or payload.get("id") or "unknown" + ), + "title": payload.get("title") or payload.get("document_title") or None, + "entities": payload.get("entities") or [], + "relationships": payload.get("relationships") or [], + "summary": payload.get("summary") or {}, + "evidence": payload.get("evidence") or [], + "tenant": tenant or {}, + "source": payload.get("source") or "literature_service", + } + + start_time = time.perf_counter() + last_error: Exception | None = None + attempt = 0 + while attempt < self.max_retries: + try: + response = self.client.post( + f"{self.base_url}/llmwiki/update", json=request_payload + ) + response.raise_for_status() + result = response.json() + LLMWIKI_UPDATE_TOTAL.labels(status="success").inc() + LLMWIKI_UPDATE_DURATION_SECONDS.observe( + time.perf_counter() - start_time + ) + return { + "document_id": request_payload["document_id"], + "status": result.get("status", "ok"), + "updated": True, + "metrics": {"retries": attempt, "failures": 0}, + } + except httpx.HTTPError as exc: + last_error = exc + attempt += 1 + if attempt >= self.max_retries: + LLMWIKI_UPDATE_ERRORS_TOTAL.inc() + LLMWIKI_UPDATE_TOTAL.labels(status="error").inc() + LLMWIKI_UPDATE_DURATION_SECONDS.observe( + time.perf_counter() - start_time + ) + break + LLMWIKI_UPDATE_RETRIES_TOTAL.inc() + time.sleep(0.2 * attempt) + + logger.error("LLM Wiki update failed", exc_info=last_error) + raise RuntimeError(f"Failed to update LLM Wiki: {last_error}") from last_error diff --git a/services/literature/app/services/search_integration.py b/services/literature/app/services/search_integration.py new file mode 100644 index 0000000..99b1f10 --- /dev/null +++ b/services/literature/app/services/search_integration.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import time +from typing import Any + +import httpx + +from app.core.config import get_settings +from app.observability.metrics import ( + SEARCH_HANDOFF_DURATION_SECONDS, + SEARCH_HANDOFF_ERRORS_TOTAL, + SEARCH_HANDOFF_RETRIES_TOTAL, + SEARCH_HANDOFF_TOTAL, +) +from app.utils.logging import get_logger + +logger = get_logger(__name__) + + +class SearchIntegrationService: + """Thin client for handing embeddings to the existing Search service. + + This service does not implement vector storage or hybrid search logic itself; + it submits embedding payloads to the shared Search service API. + """ + + def __init__( + self, + base_url: str | None = None, + timeout_seconds: int | None = None, + max_retries: int | None = None, + ) -> None: + settings = get_settings() + self.base_url = base_url or settings.search_service_url + self.timeout_seconds = ( + timeout_seconds or settings.search_service_timeout_seconds + ) + self.max_retries = max_retries or settings.search_service_max_retries + self.client = httpx.Client(timeout=self.timeout_seconds) + + def submit_embeddings( + self, + document_id: str, + embeddings: list[dict[str, Any]], + tenant: dict[str, Any] | None = None, + ) -> dict[str, Any]: + if not document_id: + raise ValueError("document_id is required") + if not embeddings: + return { + "document_id": document_id, + "status": "skipped", + "upserted": 0, + "metrics": {"retries": 0, "failures": 0}, + } + + payload = { + "document_id": document_id, + "tenant": tenant or {}, + "documents": embeddings, + } + + last_error: Exception | None = None + attempt = 0 + start_time = time.perf_counter() + while attempt < self.max_retries: + try: + response = self.client.post( + f"{self.base_url}/api/v1/search/index", json=payload + ) + response.raise_for_status() + result = response.json() + SEARCH_HANDOFF_TOTAL.labels(status="success").inc() + SEARCH_HANDOFF_DURATION_SECONDS.observe( + time.perf_counter() - start_time + ) + return { + "document_id": document_id, + "status": result.get("status", "ok"), + "upserted": result.get("upserted", len(embeddings)), + "metrics": { + "retries": attempt, + "failures": 0, + }, + } + except httpx.HTTPError as exc: + last_error = exc + attempt += 1 + if attempt >= self.max_retries: + SEARCH_HANDOFF_ERRORS_TOTAL.inc() + SEARCH_HANDOFF_TOTAL.labels(status="error").inc() + SEARCH_HANDOFF_DURATION_SECONDS.observe( + time.perf_counter() - start_time + ) + break + SEARCH_HANDOFF_RETRIES_TOTAL.inc() + time.sleep(0.2 * attempt) + + raise ( + RuntimeError(f"Failed to submit embeddings to search service: {last_error}") + if last_error + else RuntimeError("Failed to submit embeddings to search service") + ) diff --git a/services/literature/app/utils/__init__.py b/services/literature/app/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/literature/app/utils/logging.py b/services/literature/app/utils/logging.py new file mode 100644 index 0000000..0b2eae2 --- /dev/null +++ b/services/literature/app/utils/logging.py @@ -0,0 +1,37 @@ +import contextvars +import logging + +from pythonjsonlogger import jsonlogger + +request_id_ctx = contextvars.ContextVar("request_id", default="unknown") +trace_id_ctx = contextvars.ContextVar("trace_id", default="unknown") + + +class RequestContextFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + record.request_id = request_id_ctx.get() + record.trace_id = trace_id_ctx.get() + return True + + +def set_request_context( + request_id: str | None = None, trace_id: str | None = None +) -> None: + if request_id: + request_id_ctx.set(request_id) + if trace_id: + trace_id_ctx.set(trace_id) + + +def get_logger(name: str) -> logging.Logger: + logger = logging.getLogger(name) + if not logger.handlers: + handler = logging.StreamHandler() + formatter = jsonlogger.JsonFormatter( + "%(asctime)s %(levelname)s %(name)s %(request_id)s %(trace_id)s %(message)s" + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.addFilter(RequestContextFilter()) + logger.setLevel(logging.INFO) + return logger diff --git a/services/literature/full_pytest.out b/services/literature/full_pytest.out new file mode 100644 index 0000000..e69de29 diff --git a/services/literature/full_pytest_run.txt b/services/literature/full_pytest_run.txt new file mode 100644 index 0000000..708169b --- /dev/null +++ b/services/literature/full_pytest_run.txt @@ -0,0 +1,5 @@ +C:\Users\Lenovo\Downloads\AI-RxOS\.venv-1\Lib\site-packages\pytest_asyncio\plugin.py:207: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset. +The event loop scope for asynchronous fixtures will default to the fixture caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to function scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: "function", "class", "module", "package", "session" +System.Management.Automation.RemoteException + warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET)) +........................................................... diff --git a/services/literature/full_test_run.txt b/services/literature/full_test_run.txt new file mode 100644 index 0000000..e69de29 diff --git a/services/literature/pytest_full.txt b/services/literature/pytest_full.txt new file mode 100644 index 0000000..3178132 Binary files /dev/null and b/services/literature/pytest_full.txt differ diff --git a/services/literature/pytest_full_utf8.txt b/services/literature/pytest_full_utf8.txt new file mode 100644 index 0000000..ce4c1b6 --- /dev/null +++ b/services/literature/pytest_full_utf8.txt @@ -0,0 +1 @@ +........................................................... diff --git a/services/literature/pytest_out.txt b/services/literature/pytest_out.txt new file mode 100644 index 0000000..e69de29 diff --git a/services/literature/pytest_single.out b/services/literature/pytest_single.out new file mode 100644 index 0000000..f7a7716 Binary files /dev/null and b/services/literature/pytest_single.out differ diff --git a/services/literature/requirements.txt b/services/literature/requirements.txt index e4a3272..783c143 100644 --- a/services/literature/requirements.txt +++ b/services/literature/requirements.txt @@ -3,7 +3,9 @@ uvicorn[standard]==0.34.0 pydantic==2.10.4 pydantic-settings==2.7.1 asyncpg==0.30.0 -httpx==0.28.1 +PyJWT==2.8.0 python-json-logger==3.2.1 pytest==8.3.4 pytest-asyncio==0.25.1 +croniter==1.4.1 +prometheus-client==0.21.0 diff --git a/services/literature/ruff_output.txt b/services/literature/ruff_output.txt new file mode 100644 index 0000000..857c6c3 Binary files /dev/null and b/services/literature/ruff_output.txt differ diff --git a/services/literature/test_output.txt b/services/literature/test_output.txt new file mode 100644 index 0000000..e69de29 diff --git a/services/literature/tests/conftest.py b/services/literature/tests/conftest.py new file mode 100644 index 0000000..2d0d821 --- /dev/null +++ b/services/literature/tests/conftest.py @@ -0,0 +1,8 @@ +import os +import sys +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT_DIR)) + +os.environ["ENVIRONMENT"] = "test" diff --git a/services/literature/tests/test_company_website_connector.py b/services/literature/tests/test_company_website_connector.py new file mode 100644 index 0000000..2a01719 --- /dev/null +++ b/services/literature/tests/test_company_website_connector.py @@ -0,0 +1,33 @@ +import pytest + +from app.connectors.company_website import CompanyWebsiteConnector + + +class _DummyResponse: + def __init__(self, status_code: int) -> None: + self.status_code = status_code + + +@pytest.mark.asyncio +async def test_company_website_health_check_returns_true_when_endpoint_is_reachable(monkeypatch): + connector = CompanyWebsiteConnector() + + async def fake_get(path: str, params=None): + assert path == "/" + return _DummyResponse(200) + + monkeypatch.setattr(connector.health_client, "get", fake_get) + + assert await connector.health_check() is True + + +@pytest.mark.asyncio +async def test_company_website_health_check_returns_false_when_endpoint_is_unreachable(monkeypatch): + connector = CompanyWebsiteConnector() + + async def fake_get(path: str, params=None): + raise RuntimeError("boom") + + monkeypatch.setattr(connector.health_client, "get", fake_get) + + assert await connector.health_check() is False diff --git a/services/literature/tests/test_connectors.py b/services/literature/tests/test_connectors.py new file mode 100644 index 0000000..7fe2843 --- /dev/null +++ b/services/literature/tests/test_connectors.py @@ -0,0 +1,38 @@ +import pytest + +from app.connectors.base import SourceRecord +from app.connectors.registry import get_connector + + +@pytest.mark.parametrize( + "source_name,expected_class", + [ + ("pubmed", "PubMedConnector"), + ("pmc", "PubMedCentralConnector"), + ("clinicaltrials", "ClinicalTrialsConnector"), + ("biorxiv", "BioRxivConnector"), + ("medrxiv", "MedRxivConnector"), + ("aacr", "AACRConnector"), + ("asco", "ASCOConnector"), + ("esmo", "ESMOConnector"), + ("sabcs", "SABCSConnector"), + ("patents", "PatentConnector"), + ("company_website", "CompanyWebsiteConnector"), + ], +) +def test_connector_registry(source_name, expected_class): + connector = get_connector(source_name) + assert connector.__class__.__name__ == expected_class + + +def test_source_record_model(): + record = SourceRecord( + source="pubmed", + source_id="123", + title="A Sample Title", + abstract="Abstract text", + authors=["Author One", "Author Two"], + ) + assert record.source == "pubmed" + assert record.source_id == "123" + assert record.title == "A Sample Title" diff --git a/services/literature/tests/test_document_parser.py b/services/literature/tests/test_document_parser.py new file mode 100644 index 0000000..33b055f --- /dev/null +++ b/services/literature/tests/test_document_parser.py @@ -0,0 +1,69 @@ +import io + +import pytest + +from app.parsing import parse_document, parser_metrics, DuplicateDocumentError + + +def test_parse_xml_document_metadata(): + xml = """ +
+ + + Test Title + + DoeJane + + 10.1000/test + Test abstract content. + keyword1keyword2 + + +
+ """ + stream = io.StringIO(xml) + metadata = parse_document("xml", stream) + + assert metadata["title"] == "Test Title" + assert metadata["doi"] == "10.1000/test" + assert metadata["abstract"] == "Test abstract content." + assert metadata["authors"] == ["Jane Doe"] + assert metadata["keywords"] == ["keyword1", "keyword2"] + + +def test_parse_html_document_metadata(): + html = """ + + + HTML Title + + + + + +

Section 1

+

Some text.

+ Reference + + + """ + stream = io.StringIO(html) + metadata = parse_document("html", stream) + + assert metadata["title"] == "HTML Title" + assert metadata["doi"] == "10.2000/html" + assert metadata["authors"] == ["Alice Smith"] + assert metadata["keywords"] == ["testing", "parser"] + assert metadata["references"] == [{"href": "https://example.com", "text": "Reference"}] + + +def test_duplicate_document_detection(): + xml = "Duplicate" + stream_a = io.StringIO(xml) + stream_b = io.StringIO(xml) + + parser_metrics.counters = {key: 0 for key in parser_metrics.counters} + parse_document("xml", stream_a) + + with pytest.raises(DuplicateDocumentError): + parse_document("xml", stream_b) diff --git a/services/literature/tests/test_evidence_ranking.py b/services/literature/tests/test_evidence_ranking.py new file mode 100644 index 0000000..6217c1b --- /dev/null +++ b/services/literature/tests/test_evidence_ranking.py @@ -0,0 +1,87 @@ +import pytest + +from app.services.evidence_ranking import EvidenceRankingService + + +def make_document(): + return { + "document_id": "doc-001", + "title": "Cancer Study", + "abstract": "TP53 mutations are linked to cancer. Aspirin may help.", + } + + +def test_ranking_correctness_and_confidence_aggregation(): + service = EvidenceRankingService() + document = make_document() + nlp_result = { + "document_id": "doc-001", + "sentences": ["TP53 mutations are linked to cancer."], + "entities": [{"text": "TP53", "confidence_score": 0.92, "ontology_source": "HGNC"}], + "relationships": [{ + "predicate": "treats", + "confidence": 0.8, + "provenance": {"source_sentence": "Aspirin may help."}, + "source": "TP53", + "target": "cancer", + }], + } + + result = service.rank_evidence(document, nlp_result) + + assert result["document_id"] == "doc-001" + assert len(result["evidence_items"]) == 2 + assert result["evidence_items"][0]["ranking_score"] >= result["evidence_items"][1]["ranking_score"] + assert result["evidence_items"][0]["overall_confidence"] >= 0 + assert result["ranking_metrics"]["deduplicated_items"] == 2 + + +def test_duplicate_evidence_handling(): + service = EvidenceRankingService() + document = make_document() + nlp_result = { + "entities": [ + {"text": "TP53", "confidence_score": 0.9}, + {"text": "TP53", "confidence_score": 0.89}, + ], + "relationships": [], + } + + result = service.rank_evidence(document, nlp_result) + + assert len(result["evidence_items"]) == 1 + assert result["ranking_metrics"]["deduplicated_items"] == 1 + + +def test_provenance_preservation(): + service = EvidenceRankingService() + document = make_document() + nlp_result = { + "sentences": ["Sentence one."], + "entities": [{"text": "EGFR", "confidence_score": 0.7, "ontology_source": "HGNC", "normalized_identifier": "HGNC:3236"}], + "relationships": [], + } + + result = service.rank_evidence(document, nlp_result) + evidence = result["evidence_items"][0] + + assert evidence["provenance"]["source_sentence"] == "Sentence one." + assert evidence["provenance"]["ontology_source"] == "HGNC" + assert evidence["provenance"]["normalized_identifier"] == "HGNC:3236" + + +def test_malformed_input(): + service = EvidenceRankingService() + with pytest.raises(ValueError): + service.rank_evidence("not-a-dict", {}) + + with pytest.raises(ValueError): + service.rank_evidence({}, "not-a-dict") + + +def test_empty_input(): + service = EvidenceRankingService() + result = service.rank_evidence(make_document(), {"entities": [], "relationships": []}) + + assert result["evidence_items"] == [] + assert result["ranking_metrics"]["total_evidence_items"] == 0 diff --git a/services/literature/tests/test_health.py b/services/literature/tests/test_health.py index 9d6d40d..aa9b0d8 100644 --- a/services/literature/tests/test_health.py +++ b/services/literature/tests/test_health.py @@ -1,5 +1,9 @@ +import os + from fastapi.testclient import TestClient +os.environ["ENVIRONMENT"] = "test" + from app.main import app client = TestClient(app) diff --git a/services/literature/tests/test_health_readiness.py b/services/literature/tests/test_health_readiness.py new file mode 100644 index 0000000..3445a0f --- /dev/null +++ b/services/literature/tests/test_health_readiness.py @@ -0,0 +1,52 @@ +import os + +from fastapi.testclient import TestClient + +os.environ["ENVIRONMENT"] = "test" + +from app.main import app + +client = TestClient(app) + + +def test_readiness_endpoint_includes_dependency_statuses(): + res = client.get("/ready") + assert res.status_code == 200 + body = res.json() + assert body["service"] == "literature" + assert "dependencies" in body + assert body["dependencies"]["postgresql"]["status"] == "ok" + assert body["dependencies"]["orchestrator"]["status"] == "ok" + + +def test_readiness_endpoint_succeeds_when_all_dependencies_are_available(monkeypatch): + async def fake_service_available(url: str): + return True, "ok" + + import app.routers.health as health_module + monkeypatch.setattr(health_module, "_service_available", fake_service_available) + + res = client.get("/ready") + assert res.status_code == 200 + body = res.json() + assert body["status"] == "ok" + assert body["ready"] is True + assert body["dependencies"]["postgresql"]["status"] == "ok" + assert body["dependencies"]["search_service"]["status"] == "ok" + assert body["dependencies"]["kg_service"]["status"] == "ok" + assert body["dependencies"]["orchestrator"]["status"] == "ok" + + +def test_readiness_endpoint_fails_when_external_services_unavailable(monkeypatch): + async def fake_service_available(url: str): + return False, "connection refused" + + import app.routers.health as health_module + monkeypatch.setattr(health_module, "_service_available", fake_service_available) + + res = client.get("/ready") + assert res.status_code == 200 + body = res.json() + assert body["status"] == "fail" + assert body["dependencies"]["search_service"]["status"] == "fail" + assert body["dependencies"]["kg_service"]["status"] == "fail" diff --git a/services/literature/tests/test_http_client.py b/services/literature/tests/test_http_client.py new file mode 100644 index 0000000..7d822ce --- /dev/null +++ b/services/literature/tests/test_http_client.py @@ -0,0 +1,43 @@ +import pytest +import httpx + +from app.connectors.http_client import HTTPClient + + +class DummyTransport(httpx.AsyncBaseTransport): + def __init__(self, responses): + self.responses = responses + self.calls = 0 + + async def handle_async_request(self, request): + resp = self.responses[self.calls] + self.calls += 1 + return resp + + +@pytest.mark.asyncio +async def test_http_client_retries_on_429_and_succeeds(): + responses = [ + httpx.Response(429, json={"error": "rate limit"}), + httpx.Response(200, json={"ok": True}), + ] + client = HTTPClient(base_url="https://example.com") + client._client._transport = DummyTransport(responses) + + response = await client.get("/test") + assert response.status_code == 200 + assert response.json() == {"ok": True} + + +@pytest.mark.asyncio +async def test_http_client_fails_after_max_retries(): + responses = [ + httpx.Response(503, json={"error": "service unavailable"}), + httpx.Response(503, json={"error": "service unavailable"}), + httpx.Response(503, json={"error": "service unavailable"}), + ] + client = HTTPClient(base_url="https://example.com") + client._client._transport = DummyTransport(responses) + + with pytest.raises(httpx.HTTPStatusError): + await client.get("/test") diff --git a/services/literature/tests/test_kg_integration.py b/services/literature/tests/test_kg_integration.py new file mode 100644 index 0000000..5c88226 --- /dev/null +++ b/services/literature/tests/test_kg_integration.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest +import httpx + +from app.services.kg_integration import KGIntegrationService + + +@pytest.fixture +def service() -> KGIntegrationService: + return KGIntegrationService(base_url="http://kg-service", timeout_seconds=1, max_retries=2) + + +def test_publish_graph_payload_success(service: KGIntegrationService) -> None: + payload = { + "document_id": "doc-1", + "document_title": "Example", + "entities": [{"text": "aspirin"}], + "relationships": [{"predicate": "treats"}], + } + mock_response = Mock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"status": "created"} + service.client.post = Mock(return_value=mock_response) + + result = service.publish_graph_payload(payload, tenant={"tenant_id": "tenant-1"}) + + assert result["published"] is True + assert result["event_id"] + assert result["metrics"]["retries"] == 0 + + +def test_publish_graph_payload_retries_and_succeeds(service: KGIntegrationService) -> None: + payload = { + "document_id": "doc-2", + "document_title": "Retry", + "entities": [{"text": "ibuprofen"}], + "relationships": [{"predicate": "associated_with"}], + } + + failed = Mock() + failed.raise_for_status.side_effect = httpx.HTTPError("boom") + + success = Mock() + success.raise_for_status.return_value = None + success.json.return_value = {"status": "created"} + + call_count = {"value": 0} + + def side_effect(*args, **kwargs): + call_count["value"] += 1 + if call_count["value"] == 1: + return failed + return success + + service.client.post = Mock(side_effect=side_effect) + + result = service.publish_graph_payload(payload) + + assert result["published"] is True + assert result["metrics"]["retries"] == 1 + + +def test_publish_graph_payload_rejects_malformed_payload(service: KGIntegrationService) -> None: + with pytest.raises(ValueError): + service.publish_graph_payload({"document_id": "doc-3", "entities": "bad", "relationships": []}) + + +def test_publish_graph_payload_raises_when_service_unavailable(service: KGIntegrationService) -> None: + payload = { + "document_id": "doc-4", + "document_title": "Down", + "entities": [{"text": "acetaminophen"}], + "relationships": [{"predicate": "targets"}], + } + mock_response = Mock() + mock_response.raise_for_status.side_effect = httpx.ConnectError("service down") + service.client.post = Mock(return_value=mock_response) + + with pytest.raises(RuntimeError): + service.publish_graph_payload(payload) diff --git a/services/literature/tests/test_llmwiki_integration.py b/services/literature/tests/test_llmwiki_integration.py new file mode 100644 index 0000000..7ce8923 --- /dev/null +++ b/services/literature/tests/test_llmwiki_integration.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest +import httpx + +from app.services.llmwiki_integration import LLMWikiIntegrationService + + +@pytest.fixture +def service() -> LLMWikiIntegrationService: + return LLMWikiIntegrationService(base_url="http://llmwiki-service", timeout_seconds=1, max_retries=2) + + +def test_update_knowledge_success(service: LLMWikiIntegrationService) -> None: + payload = { + "document_id": "doc-1", + "title": "Example Study", + "entities": [{"text": "aspirin"}], + "relationships": [{"predicate": "treats"}], + "summary": {"abstract_summary": "A short summary."}, + "evidence": [{"id": "e1", "score": 0.9}], + } + mock_response = Mock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"status": "updated"} + service.client.post = Mock(return_value=mock_response) + + result = service.update_knowledge(payload, tenant={"tenant_id": "tenant-1"}) + + assert result["updated"] is True + assert result["metrics"]["retries"] == 0 + assert result["status"] == "updated" + + +def test_update_knowledge_retries_and_succeeds(service: LLMWikiIntegrationService) -> None: + payload = { + "document_id": "doc-2", + "entities": [{"text": "ibuprofen"}], + "relationships": [{"predicate": "associated_with"}], + "summary": {"abstract_summary": "A short summary."}, + "evidence": [], + } + + failed = Mock() + failed.raise_for_status.side_effect = httpx.HTTPError("boom") + + success = Mock() + success.raise_for_status.return_value = None + success.json.return_value = {"status": "updated"} + + call_count = {"value": 0} + + def side_effect(*args, **kwargs): + call_count["value"] += 1 + if call_count["value"] == 1: + return failed + return success + + service.client.post = Mock(side_effect=side_effect) + + result = service.update_knowledge(payload) + + assert result["updated"] is True + assert result["metrics"]["retries"] == 1 + + +def test_update_knowledge_rejects_malformed_payload(service: LLMWikiIntegrationService) -> None: + with pytest.raises(ValueError): + service.update_knowledge({"document_id": "doc-3"}) + + +def test_update_knowledge_raises_when_service_unavailable(service: LLMWikiIntegrationService) -> None: + payload = { + "document_id": "doc-4", + "entities": [{"text": "acetaminophen"}], + "relationships": [{"predicate": "targets"}], + "summary": {"abstract_summary": "A summary."}, + "evidence": [], + } + mock_response = Mock() + mock_response.raise_for_status.side_effect = httpx.ConnectError("service down") + service.client.post = Mock(return_value=mock_response) + + with pytest.raises(RuntimeError): + service.update_knowledge(payload) diff --git a/services/literature/tests/test_nlp_pipeline.py b/services/literature/tests/test_nlp_pipeline.py new file mode 100644 index 0000000..24a7a14 --- /dev/null +++ b/services/literature/tests/test_nlp_pipeline.py @@ -0,0 +1,176 @@ +import pytest + +from app.nlp.pipeline import BiomedicalNLPPipeline, process_document +from app.nlp.sentence_segmenter import SentenceSegmenter +from app.nlp.tokenizer import BiomedicalTokenizer +from app.nlp.entity_extractor import EntityExtractor +from app.nlp.entity_normalizer import EntityNormalizer +from app.nlp.ontology_mapper import OntologyMapper +from app.nlp.confidence_scorer import ConfidenceScorer +from app.nlp.relationship_extractor import RelationshipExtractor +from app.nlp.embedding_service import EmbeddingService +from app.services.search_integration import SearchIntegrationService +from app.services.kg_integration import KGIntegrationService + + +def make_doc(): + return { + "document_id": "doc-001", + "title": "Test", + "authors": ["Alice Smith"], + "abstract": "TP53 p.V600E is observed in many cancer cases. Aspirin may help.", + "sections": [{"title": "Methods", "text": "EGFR mutations are linked to cancer."}], + } + + +def test_sentence_segmentation(): + segmenter = SentenceSegmenter() + sentences = segmenter.segment("First sentence. Second sentence! Third sentence?") + assert sentences == ["First sentence.", "Second sentence!", "Third sentence?"] + + +def test_tokenization(): + tokenizer = BiomedicalTokenizer() + tokens = tokenizer.tokenize("TP53 p.V600E is observed in cancer.") + assert tokens[0] == "TP53" + assert "p.V600E" in tokens + assert "cancer" in tokens + + +def test_entity_extraction(): + extractor = EntityExtractor() + entities = extractor.extract("TP53 p.V600E is observed in cancer and aspirin helps.") + labels = {entity["type"] for entity in entities} + assert "gene" in labels + assert "variant" in labels + assert "disease" in labels + assert "drug" in labels + + +def test_normalization_and_ontology_mapping(): + pipeline = BiomedicalNLPPipeline() + result = pipeline.process_document(make_doc()) + + assert result["document_id"] == "doc-001" + assert result["sentences"] + assert result["tokens"] + assert result["detected_entities"] + assert result["processing_metadata"]["input_text_length"] > 0 + assert result["execution_metrics"]["total_processing_time_ms"] >= 0 + + entity = next(entity for entity in result["detected_entities"] if entity["normalized_identifier"]) + assert entity["ontology_source"] in {"HGNC", "HGVS", "MONDO", "ChEBI"} + assert 0 <= entity["confidence_score"] <= 1 + + +def test_malformed_input_is_handled_gracefully(): + pipeline = BiomedicalNLPPipeline() + result = pipeline.process_document({"abstract": None, "sections": "invalid"}) + assert result["sentences"] == [] + assert result["tokens"] == [] + assert result["detected_entities"] == [] + assert result["processing_metadata"]["warnings"] + + +def test_empty_documents_are_handled(): + pipeline = BiomedicalNLPPipeline() + result = pipeline.process_document({"abstract": "", "sections": []}) + assert result["sentences"] == [] + assert result["tokens"] == [] + assert result["detected_entities"] == [] + assert result["processing_metadata"]["input_text_length"] == 0 + + +def test_pipeline_performance_edge_cases(): + pipeline = BiomedicalNLPPipeline() + long_text = "cancer " * 200 + result = pipeline.process_document({"document_id": "edge", "abstract": long_text, "sections": []}) + assert result["sentences"] + assert result["execution_metrics"]["stage_metrics"]["sentence_segmentation"]["processed_items"] >= 1 + + +def test_relationship_extraction(): + pipeline = BiomedicalNLPPipeline() + result = pipeline.process_document(make_doc()) + + assert "relationships" in result + assert result["relationships"] + relationship = result["relationships"][0] + assert relationship["predicate"] in {"treats", "associated_with", "interacts_with", "targets"} + assert 0 <= relationship["confidence"] <= 1 + assert relationship["provenance"]["source_sentence"] + + +def test_summarizer_is_included_in_pipeline(): + pipeline = BiomedicalNLPPipeline() + result = pipeline.process_document(make_doc()) + + assert "summary" in result + assert result["summary"]["document_id"] == "doc-001" + assert result["summary"]["structured_summary"] + assert "abstract_summary" in result["summary"] + assert "key_findings" in result["summary"] + assert "clinical_relevance" in result["summary"] + assert "limitations" in result["summary"] + + +def test_embedding_generation_service(): + pipeline = BiomedicalNLPPipeline() + nlp_result = pipeline.process_document(make_doc()) + service = EmbeddingService(batch_size=2) + embeddings = service.generate_embeddings(nlp_result) + + assert embeddings["document_id"] == "doc-001" + assert embeddings["embedding_batches"] + assert embeddings["embedding_count"] >= 1 + assert embeddings["metadata"]["source"] == "EmbeddingService" + assert embeddings["metadata"]["embedding_dimensions"] > 0 + assert embeddings["processing_metrics"]["batch_count"] >= 1 + + +def test_search_integration_service_success_and_retries(monkeypatch): + calls = [] + + class FakeResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + def fake_post(self, url, **kwargs): + calls.append((url, kwargs.get("json"))) + return FakeResponse({"status": "ok", "upserted": 1}) + + monkeypatch.setattr("app.services.search_integration.httpx.Client.post", fake_post) + + service = SearchIntegrationService(base_url="http://search", timeout_seconds=1, max_retries=3) + result = service.submit_embeddings("doc-001", [{"text": "hello", "embedding": [0.1, 0.2]}]) + + assert result["status"] == "ok" + assert result["upserted"] == 1 + assert calls[0][0] == "http://search/api/v1/search/index" + + +def test_search_integration_service_failure(monkeypatch): + class FakeResponse: + def raise_for_status(self): + raise RuntimeError("boom") + + def fake_post(self, url, **kwargs): + return FakeResponse() + + monkeypatch.setattr("app.services.search_integration.httpx.Client.post", fake_post) + + service = SearchIntegrationService(base_url="http://search", timeout_seconds=1, max_retries=2) + with pytest.raises(RuntimeError): + service.submit_embeddings("doc-001", [{"text": "hello", "embedding": [0.1, 0.2]}]) + + +def test_legacy_process_document_wrapper(): + result = process_document(make_doc()) + assert result["document_id"] == "doc-001" + assert "detected_entities" in result diff --git a/services/literature/tests/test_observability.py b/services/literature/tests/test_observability.py new file mode 100644 index 0000000..8f5eca3 --- /dev/null +++ b/services/literature/tests/test_observability.py @@ -0,0 +1,57 @@ +from fastapi.testclient import TestClient + +from app.main import app + +client = TestClient(app) + + +def test_health_and_readiness_endpoints(): + for path in ["/health", "/healthz", "/ready", "/live"]: + res = client.get(path) + assert res.status_code == 200 + assert res.json()["service"] == "literature" + + +def test_metrics_endpoint_exposes_service_state(): + res = client.get("/metrics") + assert res.status_code == 200 + body = res.json() + assert body["service"] == "literature" + assert body["environment"] == "test" + assert "parser_metrics" in body + assert "orchestrator_metrics" in body + + +def test_request_id_header_is_returned(): + res = client.get("/health", headers={"x-request-id": "req-123"}) + assert res.status_code == 200 + assert res.headers["x-request-id"] == "req-123" + + +def test_prometheus_metrics_includes_http_request_counters(): + client.get("/health") + res = client.get("/metrics/prometheus") + assert res.status_code == 200 + assert "literature_http_requests_total" in res.text + assert "endpoint=\"/health\"" in res.text + assert "method=\"GET\"" in res.text + + +def test_prometheus_metrics_includes_latency_histogram(): + client.get("/health") + res = client.get("/metrics/prometheus") + assert "literature_http_request_latency_seconds_bucket" in res.text + assert "literature_http_request_latency_seconds_count" in res.text + + +def test_http_metrics_middleware_records_error_counts_for_not_found_requests(): + res = client.get("/not-found") + assert res.status_code == 404 + + metrics = client.get("/metrics/prometheus").text + assert "literature_http_requests_total" in metrics + assert "literature_http_request_errors_total" in metrics + assert "literature_http_request_latency_seconds_count" in metrics + assert "endpoint=\"/not-found\"" in metrics + assert "method=\"GET\"" in metrics + assert "status=\"404\"" in metrics diff --git a/services/literature/tests/test_orchestrator.py b/services/literature/tests/test_orchestrator.py new file mode 100644 index 0000000..5d8ded2 --- /dev/null +++ b/services/literature/tests/test_orchestrator.py @@ -0,0 +1,307 @@ +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Any +from unittest.mock import AsyncMock, Mock + +import httpx +from fastapi.testclient import TestClient +from httpx import ASGITransport +import pytest + +from app.core.security import get_current_user +from app.main import app +from app.orchestrator.manager import IngestionJob, IngestionOrchestrator, JobStatus + + +@pytest.mark.asyncio +async def test_compute_next_run_valid_schedule(): + orch = IngestionOrchestrator() + reference = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc) + + next_run = orch._compute_next_run("*/5 * * * *", reference) + + assert next_run > reference + assert next_run.minute % 5 == 0 + + +@pytest.mark.asyncio +async def test_compute_next_run_invalid_schedule_raises(): + orch = IngestionOrchestrator() + + with pytest.raises(ValueError): + orch._compute_next_run("invalid-cron") + + +@pytest.mark.asyncio +async def test_schedule_persists_and_starts(monkeypatch): + orch = IngestionOrchestrator() + job = IngestionJob(job_id="11111111-1111-1111-1111-111111111111", source="pubmed", query="cancer") + + persist = AsyncMock() + start = AsyncMock() + monkeypatch.setattr(orch, "_persist_job_state", persist) + monkeypatch.setattr(orch, "start", start) + + await orch.schedule(job, "*/15 * * * *") + + assert job.status == JobStatus.SCHEDULED + assert job.next_run_at is not None + persist.assert_awaited_once_with(job) + start.assert_awaited_once() + assert orch.get_metrics()["ingestion.jobs_scheduled"] == 1 + + +@pytest.mark.asyncio +async def test_trigger_retry_loads_db_and_enqueues_job(monkeypatch): + orch = IngestionOrchestrator() + job = IngestionJob( + job_id="22222222-2222-2222-2222-222222222222", + source="biorxiv", + query="immunotherapy", + status=JobStatus.FAILED, + ) + + load = AsyncMock(return_value=job) + monkeypatch.setattr(orch, "_load_job_from_db", load) + enqueue = Mock() + monkeypatch.setattr(orch, "enqueue", enqueue) + start = AsyncMock() + monkeypatch.setattr(orch, "start", start) + + await orch.trigger_retry(job.job_id) + + assert job.status == JobStatus.QUEUED + assert job.backoff_until is not None + assert job.retry_count == 1 + enqueue.assert_called_once_with(job) + start.assert_awaited_once() + assert orch.get_metrics()["ingestion.job_retries"] == 1 + + +@pytest.mark.asyncio +async def test_scheduler_triggers_scheduled_job_to_completion(monkeypatch): + scheduled_orchestrator = IngestionOrchestrator() + transitions: list[str] = [] + completed_event = asyncio.Event() + now = datetime.now(timezone.utc) + + job_data: dict[str, Any] = { + "id": "44444444-4444-4444-4444-444444444444", + "source": "pubmed", + "query": "cancer", + "status": "scheduled", + "created_at": now, + "started_at": None, + "completed_at": None, + "schedule": "*/1 * * * *", + "next_run_at": now, + "documents_total": 0, + "documents_processed": 0, + "documents_failed": 0, + "retry_count": 0, + "backoff_until": None, + "error_message": None, + "dead_letter_count": 0, + "dead_letter_items": json.dumps([]), + } + + class FakeConnection: + async def execute(self, query: str, *args: Any) -> None: + if "INSERT INTO literature_ingestion_jobs" in query: + job_data.update( + id=str(args[0]), + source=args[1], + query=args[2], + status=args[3], + created_at=args[4], + schedule=args[5], + next_run_at=args[6], + ) + return + + if query.strip().startswith("UPDATE literature_ingestion_jobs"): + previous_status = job_data["status"] + job_data.update( + status=args[0], + started_at=args[1], + completed_at=args[2], + schedule=args[3], + next_run_at=args[4], + documents_total=args[5], + documents_processed=args[6], + documents_failed=args[7], + error_message=args[8], + retry_count=args[9], + backoff_until=args[10], + dead_letter_count=args[11], + dead_letter_items=args[12] or json.dumps([]), + ) + if job_data["status"] != previous_status: + transitions.append(job_data["status"]) + if job_data["status"] == "completed": + completed_event.set() + return + + async def fetchrow(self, query: str, *args: Any) -> Any: + if "WHERE id = $1" in query: + return job_data.copy() + return None + + async def fetch(self, query: str, *args: Any) -> list[dict[str, Any]]: + if "WHERE status = $1" in query: + scheduled_status, cutoff = args[0], args[1] + if ( + job_data["status"] == scheduled_status + and job_data["next_run_at"] is not None + and job_data["next_run_at"] <= cutoff + ): + return [job_data.copy()] + return [] + + @asynccontextmanager + async def fake_acquire() -> Any: + yield FakeConnection() + + monkeypatch.setattr("app.routers.ingestion.postgres_manager.acquire", fake_acquire) + monkeypatch.setattr("app.orchestrator.manager.postgres_manager.acquire", fake_acquire) + monkeypatch.setattr("app.routers.ingestion.orchestrator", scheduled_orchestrator) + def fake_compute_next_run(self, schedule, reference=None): + if reference is None: + return now + return now + timedelta(minutes=1) + + original_compute_next_run = IngestionOrchestrator._compute_next_run + + def fake_compute_next_run(self, schedule, reference=None): + if reference is None: + return now + return original_compute_next_run(self, schedule, reference) + + monkeypatch.setattr( + "app.orchestrator.manager.IngestionOrchestrator._compute_next_run", + fake_compute_next_run, + ) + monkeypatch.setattr( + "app.orchestrator.manager.IngestionOrchestrator._simulate_ingestion", + AsyncMock(return_value=None), + ) + + original_sleep = asyncio.sleep + + async def fast_sleep(duration: float, *args: Any, **kwargs: Any) -> None: + await original_sleep(min(duration, 0.01)) + + monkeypatch.setattr("app.orchestrator.manager.asyncio.sleep", fast_sleep) + + await scheduled_orchestrator.start() + + try: + async with httpx.AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + response = await client.post( + "/api/v1/ingestion", + json={"source": "pubmed", "query": "cancer", "schedule": "*/1 * * * *"}, + headers={"Authorization": "Bearer fake-token"}, + ) + + assert response.status_code == 202 + + await asyncio.wait_for(completed_event.wait(), timeout=5) + + assert "completed" in transitions + assert job_data["started_at"] is not None + assert job_data["completed_at"] is not None + assert job_data["status"] == "scheduled" + finally: + await scheduled_orchestrator.stop() + + +@asynccontextmanager +def dummy_acquire(connection): + yield connection + + +class DummyConnection: + def __init__(self, response): + self.response = response + + async def execute(self, *args, **kwargs): + return None + + async def fetchrow(self, *args, **kwargs): + return self.response + + +@pytest.fixture(autouse=True) +def app_auth_override(monkeypatch): + app.dependency_overrides[get_current_user] = lambda: {"sub": "test-user"} + yield + app.dependency_overrides.clear() + + +def test_start_ingestion_enqueue_route(monkeypatch): + client = TestClient(app) + created_at = datetime.now(timezone.utc) + row = { + "id": "32027b04-56f7-4d28-82c5-8cbd0177c4f4", + "source": "pubmed", + "query": "cancer", + "status": "queued", + "created_at": created_at, + "started_at": None, + "completed_at": None, + "schedule": None, + "next_run_at": None, + "documents_total": 0, + "documents_processed": 0, + "documents_failed": 0, + "retry_count": 0, + "backoff_until": None, + "error_message": None, + "dead_letter_count": 0, + "dead_letter_items": json.dumps([]), + } + + @asynccontextmanager + async def fake_acquire(): + yield DummyConnection(row) + + monkeypatch.setattr("app.routers.ingestion.postgres_manager.acquire", fake_acquire) + monkeypatch.setattr("app.routers.ingestion.orchestrator.enqueue", Mock()) + monkeypatch.setattr("app.routers.ingestion.orchestrator.start", AsyncMock()) + + response = client.post( + "/api/v1/ingestion", + json={"source": "pubmed", "query": "cancer"}, + headers={"Authorization": "Bearer fake-token"}, + ) + + assert response.status_code == 202 + data = response.json() + assert data["source"] == "pubmed" + assert data["query"] == "cancer" + assert data["status"] == "queued" + + +def test_dead_letter_items_route(monkeypatch): + client = TestClient(app) + dead_letter_items = [{"job_id": "33333333-3333-3333-3333-333333333333", "error_message": "failed"}] + row = { + "id": "33333333-3333-3333-3333-333333333333", + "dead_letter_items": json.dumps(dead_letter_items), + } + + @asynccontextmanager + async def fake_acquire(): + yield DummyConnection(row) + + monkeypatch.setattr("app.routers.ingestion.postgres_manager.acquire", fake_acquire) + + response = client.get( + "/api/v1/ingestion/33333333-3333-3333-3333-333333333333/dead-letter", + headers={"Authorization": "Bearer fake-token"}, + ) + + assert response.status_code == 200 + assert response.json() == {"items": dead_letter_items, "total": 1} diff --git a/services/literature/tests/test_prometheus_metrics.py b/services/literature/tests/test_prometheus_metrics.py new file mode 100644 index 0000000..58f85a2 --- /dev/null +++ b/services/literature/tests/test_prometheus_metrics.py @@ -0,0 +1,47 @@ +import re + +from fastapi.testclient import TestClient + +from app.main import app + +client = TestClient(app) + + +def test_metrics_prometheus_endpoint_returns_text(): + res = client.get("/metrics/prometheus") + assert res.status_code == 200 + assert "# HELP literature_http_requests_total" in res.text + assert "# TYPE literature_http_requests_total counter" in res.text + + +def test_metrics_prometheus_endpoint_uses_existing_registry(): + res = client.get("/metrics/prometheus") + assert res.headers["content-type"].startswith("text/plain") + assert "literature_http_request_latency_seconds" in res.text + + +def _metric_with_labels_exists(metrics_text: str, metric_name: str, labels: dict[str, str]) -> bool: + label_pattern = ".*".join(f"{re.escape(k)}=\"{re.escape(v)}\"" for k, v in labels.items()) + pattern = rf"^{re.escape(metric_name)}\{{.*{label_pattern}.*\}}\s+\d+(?:\.\d+)?$" + return re.search(pattern, metrics_text, flags=re.MULTILINE) is not None + + +def test_metrics_prometheus_counter_increments_after_http_request(): + client.get("/health") + res = client.get("/metrics/prometheus") + assert res.status_code == 200 + metrics = res.text + + assert _metric_with_labels_exists( + metrics, + "literature_http_requests_total", + {"endpoint": "/health", "method": "GET", "status": "200"}, + ) + + +def test_metrics_prometheus_histogram_records_latency(): + client.get("/health") + res = client.get("/metrics/prometheus") + metrics = res.text + assert "literature_http_request_latency_seconds_count" in metrics + assert "literature_http_request_latency_seconds_sum" in metrics diff --git a/test_output.txt b/test_output.txt new file mode 100644 index 0000000..e69de29