diff --git a/crates/tinyinference/src/cache/mod.rs b/crates/tinyinference/src/cache/mod.rs index b6d0637..0149959 100644 --- a/crates/tinyinference/src/cache/mod.rs +++ b/crates/tinyinference/src/cache/mod.rs @@ -297,5 +297,32 @@ impl CacheLayoutEvent { } } +impl CachePolicy { + /// Creates a policy with response caching enabled and no expiry. + pub fn enabled() -> Self { + Self { + response_cache_enabled: true, + ..Self::default() + } + } + + /// Returns the configured entry TTL. + pub fn ttl(&self) -> Option { + self.ttl_ms.map(std::time::Duration::from_millis) + } + + /// Sets the entry TTL. + pub fn with_ttl(mut self, ttl: std::time::Duration) -> Self { + self.ttl_ms = Some(ttl.as_millis() as u64); + self + } + + /// Sets the cache-key namespace. + pub fn with_namespace(mut self, namespace: impl Into) -> Self { + self.namespace = Some(namespace.into()); + self + } +} + #[cfg(test)] mod test; diff --git a/crates/tinyinference/src/cache/types.rs b/crates/tinyinference/src/cache/types.rs index a7fb1fa..1587bf9 100644 --- a/crates/tinyinference/src/cache/types.rs +++ b/crates/tinyinference/src/cache/types.rs @@ -130,4 +130,10 @@ pub struct CachePolicy { /// When `true`, middleware must preserve the order and content of cacheable /// prefix segments. Violations are reported as [`CacheLayoutEvent`]s. pub protect_prompt_prefix: bool, + /// Entry time-to-live in milliseconds; `None` means no expiry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl_ms: Option, + /// Optional cache-key namespace. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, } diff --git a/crates/tinyinference/src/embeddings/mod.rs b/crates/tinyinference/src/embeddings/mod.rs index 45b8028..8f570f1 100644 --- a/crates/tinyinference/src/embeddings/mod.rs +++ b/crates/tinyinference/src/embeddings/mod.rs @@ -294,6 +294,7 @@ mod voyage; pub use noop::NoopEmbeddingModel; pub use ollama::{ DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL, DEFAULT_OLLAMA_URL, OllamaEmbeddingModel, + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, }; pub use openai::OpenAiEmbeddingModel; pub use rate_limit::{DEFAULT_REQUESTS_PER_MINUTE, acquire, rate_limit, set_rate_limit}; diff --git a/crates/tinyinference/src/embeddings/ollama.rs b/crates/tinyinference/src/embeddings/ollama.rs index 7fd38be..ef5d415 100644 --- a/crates/tinyinference/src/embeddings/ollama.rs +++ b/crates/tinyinference/src/embeddings/ollama.rs @@ -2,6 +2,10 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; use super::EmbeddingModel; use crate::{Error, Result}; @@ -12,6 +16,8 @@ pub const DEFAULT_OLLAMA_URL: &str = "http://localhost:11434"; pub const DEFAULT_OLLAMA_MODEL: &str = "bge-m3"; /// Default output dimensionality used when zero is requested. pub const DEFAULT_OLLAMA_DIMENSIONS: usize = 1024; +/// Context and batch window recommended for long-document embedding models. +pub const RECOMMENDED_OLLAMA_CONTEXT_TOKENS: u32 = 8192; /// Client for Ollama's native `/api/embed` endpoint. #[derive(Debug)] @@ -19,7 +25,8 @@ pub struct OllamaEmbeddingModel { client: reqwest::Client, base_url: String, model: String, - dimensions: usize, + dimensions: Arc, + options: Option, } impl OllamaEmbeddingModel { @@ -33,14 +40,47 @@ impl OllamaEmbeddingModel { client: reqwest::Client::new(), base_url: normalize_base_url(base_url)?, model: normalize_model(model)?, - dimensions: if dimensions == 0 { + dimensions: Arc::new(AtomicUsize::new(if dimensions == 0 { DEFAULT_OLLAMA_DIMENSIONS } else { dimensions - }, + })), + options: None, }) } + fn try_new_unresolved(base_url: &str, model: &str) -> Result { + Ok(Self { + client: reqwest::Client::new(), + base_url: normalize_base_url(base_url)?, + model: normalize_model(model)?, + dimensions: Arc::new(AtomicUsize::new(0)), + options: None, + }) + } + + /// Embeds inputs while learning the installed model's vector width. + pub async fn embed_discovering_dimensions( + base_url: &str, + model: &str, + client: reqwest::Client, + texts: &[String], + num_ctx: u32, + num_batch: u32, + ) -> Result<(usize, Vec>)> { + if !texts.iter().any(|text| !text.trim().is_empty()) { + return Err(Error::Validation( + "dynamic embedding dimension discovery requires at least one nonblank input" + .to_string(), + )); + } + let adapter = Self::try_new_unresolved(base_url, model)? + .with_client(client) + .with_context_options(num_ctx, num_batch); + let vectors = adapter.embed(texts).await?; + Ok((adapter.dimensions(), vectors)) + } + /// Creates an Ollama model, panicking for an invalid configuration. /// /// # Panics @@ -57,6 +97,15 @@ impl OllamaEmbeddingModel { self } + /// Requests an explicit context and batch window from Ollama. + pub fn with_context_options(mut self, num_ctx: u32, num_batch: u32) -> Self { + self.options = Some(OllamaOptions { + num_ctx: num_ctx.max(1), + num_batch: num_batch.max(1), + }); + self + } + /// Returns the normalized API base URL. pub fn base_url(&self) -> &str { &self.base_url @@ -78,6 +127,7 @@ impl OllamaEmbeddingModel { .json(&OllamaRequest { model: self.model.clone(), input, + options: self.options, }) .send() .await @@ -126,10 +176,24 @@ impl OllamaEmbeddingModel { } fn validate_dimensions(&self, index: usize, vector: &[f32]) -> Result<()> { - if vector.len() != self.dimensions { + if vector.is_empty() { + return Err(Error::Embedding(format!( + "ollama embed returned an empty vector at index {index}" + ))); + } + let expected = match self.dimensions.compare_exchange( + 0, + vector.len(), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => vector.len(), + Err(expected) => expected, + }; + if vector.len() != expected { return Err(Error::Embedding(format!( "ollama embed dimension mismatch at index {index}: expected {}, got {}", - self.dimensions, + expected, vector.len() ))); } @@ -151,6 +215,14 @@ impl Default for OllamaEmbeddingModel { struct OllamaRequest { model: String, input: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + options: Option, +} + +#[derive(Clone, Copy, Debug, Serialize)] +struct OllamaOptions { + num_ctx: u32, + num_batch: u32, } #[derive(Deserialize)] @@ -253,7 +325,7 @@ impl EmbeddingModel for OllamaEmbeddingModel { } fn dimensions(&self) -> usize { - self.dimensions + self.dimensions.load(Ordering::Acquire) } async fn embed(&self, texts: &[String]) -> Result>> { @@ -263,13 +335,13 @@ impl EmbeddingModel for OllamaEmbeddingModel { let live = texts .iter() .enumerate() - .filter(|(_, text)| !text.trim().is_empty()) - .map(|(index, text)| (index, text.clone())) + .filter_map(|(index, text)| { + let text = text.trim(); + (!text.is_empty()).then(|| (index, text.to_owned())) + }) .collect::>(); if live.is_empty() { - return Err(Error::Validation( - "Ollama embedding batches must not contain blank inputs".into(), - )); + return Ok(vec![Vec::new(); texts.len()]); } if live.len() != texts.len() { return Err(Error::Validation( @@ -335,10 +407,10 @@ mod tests { } #[tokio::test] - async fn blank_inputs_are_rejected_without_network() { + async fn blank_inputs_are_position_safe_without_network() { let model = OllamaEmbeddingModel::default(); - let error = model.embed(&[" ".into(), "\n".into()]).await.unwrap_err(); - assert!(matches!(error, Error::Validation(_))); + let vectors = model.embed(&[" ".into(), "\n".into()]).await.unwrap(); + assert_eq!(vectors, vec![Vec::::new(), Vec::new()]); } #[test] diff --git a/crates/tinyinference/src/message/mod.rs b/crates/tinyinference/src/message/mod.rs index e0fadcd..d57d904 100644 --- a/crates/tinyinference/src/message/mod.rs +++ b/crates/tinyinference/src/message/mod.rs @@ -129,6 +129,8 @@ impl Message { Message::Tool(ToolMessage { tool_call_id: tool_call_id.into(), content: vec![ContentBlock::Text(content.into())], + trusted_verbatim: false, + artifact: None, }) } @@ -142,6 +144,14 @@ impl Message { } } + /// Returns an out-of-band tool artifact when this is a tool message. + pub fn artifact(&self) -> Option<&serde_json::Value> { + match self { + Message::Tool(message) => message.artifact.as_ref(), + _ => None, + } + } + /// Returns the total number of Unicode scalar values across all text content /// blocks, without allocating the concatenated string. /// @@ -162,9 +172,8 @@ impl Message { .sum() } - /// Approximate character weight of the message across *all* content blocks - /// (text, JSON, images, reasoning, provider extensions), for token - /// estimation and context-window gating. + /// Approximate character weight of provider-visible content and structural + /// tool-call payloads, for token estimation and context-window gating. /// /// Distinct from [`char_len`](Self::char_len), which counts only visible /// text: a transcript dominated by images, large tool-result JSON, or model @@ -178,11 +187,31 @@ impl Message { Message::Assistant(m) => &m.content, Message::Tool(m) => &m.content, }; - content + let content_weight: usize = content .iter() .map(ContentBlock::estimated_char_weight) - .sum() + .sum(); + let structural_weight = match self { + Message::Assistant(message) => tool_calls_char_weight(&message.tool_calls), + Message::Tool(message) => message.tool_call_id.chars().count(), + _ => 0, + }; + content_weight + structural_weight + } +} + +fn tool_calls_char_weight(tool_calls: &[crate::tool::ToolCall]) -> usize { + if tool_calls.is_empty() { + return 0; } + serde_json::to_string(tool_calls) + .map(|rendered| rendered.chars().count()) + .unwrap_or_else(|_| { + tool_calls + .iter() + .map(|call| call.name.chars().count() + call.arguments.to_string().chars().count()) + .sum() + }) } #[cfg(test)] diff --git a/crates/tinyinference/src/message/types.rs b/crates/tinyinference/src/message/types.rs index 9bc488b..b444ef8 100644 --- a/crates/tinyinference/src/message/types.rs +++ b/crates/tinyinference/src/message/types.rs @@ -99,6 +99,16 @@ pub struct ToolMessage { pub tool_call_id: String, /// Ordered content blocks. pub content: Vec, + /// Whether a consuming runtime must preserve the content byte-for-byte. + #[serde(default, skip_serializing_if = "is_false")] + pub trusted_verbatim: bool, + /// Host-side structured payload that is never sent to the provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact: Option, +} + +fn is_false(value: &bool) -> bool { + !*value } /// A structured conversation message. diff --git a/crates/tinyinference/src/model/mod.rs b/crates/tinyinference/src/model/mod.rs index 4572243..b4928ae 100644 --- a/crates/tinyinference/src/model/mod.rs +++ b/crates/tinyinference/src/model/mod.rs @@ -47,6 +47,7 @@ enum ContextPatternMatch { /// substrings such as `gpt-4.1` and `gpt-4-turbo` must stay before broader /// patterns such as `gpt-4` that would otherwise shadow them. const MODEL_CONTEXT_PATTERNS: &[(&str, ContextPatternMatch, u64)] = &[ + ("gpt-5", ContextPatternMatch::Substring, 400_000), ("claude-haiku-4.5", ContextPatternMatch::Substring, 200_000), ("claude-haiku-4", ContextPatternMatch::Substring, 200_000), ("claude-haiku", ContextPatternMatch::Substring, 200_000), @@ -161,6 +162,7 @@ impl ModelProfile { && (!set.native_structured_output || self.native_structured_output) && (!set.json_schema || self.json_schema) && (!set.reasoning || self.reasoning) + && (!set.reasoning_effort || self.reasoning_effort) && (!set.image_in || self.modalities.image_in) && (!set.image_out || self.modalities.image_out) && (!set.audio_in || self.modalities.audio_in) @@ -215,6 +217,7 @@ impl ModelProfile { native_structured_output: true, json_schema: true, reasoning: true, + reasoning_effort: true, ..Self::default() } } @@ -356,6 +359,17 @@ impl ModelRequest { self } + /// Sets provider-neutral reasoning configuration. + pub fn with_reasoning(mut self, reasoning: ReasoningConfig) -> Self { + self.reasoning = Some(reasoning); + self + } + + /// Sets only the requested reasoning effort. + pub fn with_reasoning_effort(self, effort: ReasoningEffort) -> Self { + self.with_reasoning(ReasoningConfig::effort(effort)) + } + /// Returns the ids of cacheable segments in declaration order, describing /// the stable prompt prefix middleware should preserve. pub fn cacheable_prefix_ids(&self) -> Vec { @@ -381,6 +395,8 @@ impl ModelResponse { finish_reason: None, raw: None, resolved_model: None, + continue_turn: None, + served_from_cache: false, } } @@ -609,6 +625,8 @@ impl StreamAccumulator { finish_reason: None, raw: None, resolved_model: None, + continue_turn: None, + served_from_cache: false, }) } } diff --git a/crates/tinyinference/src/model/types.rs b/crates/tinyinference/src/model/types.rs index c268472..f6388b9 100644 --- a/crates/tinyinference/src/model/types.rs +++ b/crates/tinyinference/src/model/types.rs @@ -71,6 +71,65 @@ pub enum ResponseFormat { }, } +/// Provider-neutral reasoning effort for one model call. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + /// Smallest available effort. + Minimal, + /// Below-default effort. + Low, + /// Provider default effort. + #[default] + Medium, + /// Above-default effort. + High, + /// Explicitly disable reasoning. + None, +} + +impl ReasoningEffort { + /// Returns the common provider wire token. + pub fn as_str(self) -> &'static str { + match self { + Self::Minimal => "minimal", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + Self::None => "none", + } + } +} + +/// Provider-neutral reasoning configuration. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReasoningConfig { + /// Requested reasoning effort. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effort: Option, + /// Explicit thinking-token budget. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub budget_tokens: Option, + /// Requested reasoning-summary verbosity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +impl ReasoningConfig { + /// Creates a config containing only `effort`. + pub fn effort(effort: ReasoningEffort) -> Self { + Self { + effort: Some(effort), + ..Self::default() + } + } + + /// Returns whether no reasoning option is set. + pub fn is_empty(&self) -> bool { + self.effort.is_none() && self.budget_tokens.is_none() && self.summary.is_none() + } +} + /// Lifecycle status of a model, used by [`ModelProfile`]. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -172,6 +231,9 @@ pub struct ModelProfile { /// Emits reasoning/thinking output. #[serde(default)] pub reasoning: bool, + /// Accepts a configurable reasoning effort. + #[serde(default)] + pub reasoning_effort: bool, /// Maximum input (context) tokens, when known. #[serde(default, skip_serializing_if = "Option::is_none")] pub max_input_tokens: Option, @@ -210,6 +272,9 @@ pub struct CapabilitySet { /// Requires reasoning output. #[serde(default)] pub reasoning: bool, + /// Requires configurable reasoning effort. + #[serde(default)] + pub reasoning_effort: bool, /// Requires image input (vision). #[serde(default)] pub image_in: bool, @@ -372,6 +437,9 @@ pub struct ModelRequest { /// Optional provider continuation/response id for stateful follow-ups. #[serde(default, skip_serializing_if = "Option::is_none")] pub continuation_id: Option, + /// Provider-neutral reasoning configuration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning: Option, } /// A provider-neutral chat model response. @@ -391,6 +459,12 @@ pub struct ModelResponse { /// Durable model-selection metadata attached by a consuming runtime. #[serde(default, skip_serializing_if = "Option::is_none")] pub resolved_model: Option, + /// Runtime nudge indicating that this response does not end the turn. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub continue_turn: Option, + /// Whether a consuming runtime served this response from local cache. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub served_from_cache: bool, } /// An incremental streamed chunk of a model response. @@ -433,6 +507,9 @@ pub struct ProviderError { /// Whether retrying the same request may succeed. #[serde(default)] pub retryable: bool, + /// Parsed provider Retry-After delay in milliseconds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_after_ms: Option, /// Raw provider payload, when available. #[serde(default, skip_serializing_if = "Option::is_none")] pub raw: Option, @@ -503,6 +580,14 @@ pub trait ChatModel: Send + Sync { None } + /// Returns a stable, credential-safe identity for response-cache scoping. + /// + /// The default declines identity. Implementations must never include raw + /// credentials in the returned value. + fn cache_identity(&self) -> Option { + None + } + /// Invokes the model and returns a complete response. async fn invoke(&self, state: &State, request: ModelRequest) -> Result; diff --git a/crates/tinyinference/src/providers/mock.rs b/crates/tinyinference/src/providers/mock.rs index b47a584..b674059 100644 --- a/crates/tinyinference/src/providers/mock.rs +++ b/crates/tinyinference/src/providers/mock.rs @@ -246,6 +246,8 @@ impl ChatModel for MockModel { finish_reason: Some("tool_calls".to_string()), raw: None, resolved_model: None, + continue_turn: None, + served_from_cache: false, } } @@ -346,6 +348,8 @@ impl MockModel { finish_reason: Some("stop".to_string()), raw: None, resolved_model: None, + continue_turn: None, + served_from_cache: false, } } } diff --git a/crates/tinyinference/src/providers/openai/convert.rs b/crates/tinyinference/src/providers/openai/convert.rs index 04d5268..b6b8798 100644 --- a/crates/tinyinference/src/providers/openai/convert.rs +++ b/crates/tinyinference/src/providers/openai/convert.rs @@ -324,6 +324,8 @@ pub(super) fn parse_chat_response( finish_reason: choice.finish_reason, raw: Some(value), resolved_model: None, + continue_turn: None, + served_from_cache: false, }) } @@ -332,9 +334,20 @@ pub(super) fn parse_chat_response( /// the slot's position so delta ids and the final call id always agree. pub(super) fn tool_call_id(slot: usize, id: &str) -> String { if id.is_empty() { - format!("tool-{slot}") + static NEXT_SYNTHETIC_ID: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(1); + let sequence = NEXT_SYNTHETIC_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + format!("tacall-{sequence}-{slot}") } else { - id.to_string() + id.chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + character + } else { + '-' + } + }) + .collect() } } @@ -494,13 +507,18 @@ pub(super) fn convert_usage(wire: UsageWire) -> Usage { total_tokens, cache_read_tokens: wire .prompt_tokens_details + .as_ref() .map(|d| d.cached_tokens) .unwrap_or(0), + cache_creation_tokens: wire + .prompt_tokens_details + .as_ref() + .map(|details| details.cache_write_tokens) + .unwrap_or(0), reasoning_tokens: wire .completion_tokens_details .map(|d| d.reasoning_tokens) .unwrap_or(0), - ..Usage::default() } } diff --git a/crates/tinyinference/src/providers/openai/local.rs b/crates/tinyinference/src/providers/openai/local.rs new file mode 100644 index 0000000..2b31fcb --- /dev/null +++ b/crates/tinyinference/src/providers/openai/local.rs @@ -0,0 +1,514 @@ +//! Local OpenAI-compatible runtimes: identification, capability **probing**, +//! and the native-API escape hatches the OpenAI wire format cannot express. +//! +//! # Why this module exists +//! +//! A local runtime is not "hosted OpenAI at a different URL". It differs in +//! three ways the transport used to paper over with hard-coded guesses: +//! +//! 1. **Its context window is tiny and not derivable from the model id.** +//! `derive_profile` filled `max_input_tokens` from the generic hint table, +//! which matches bare substrings — so `llama3.2:3b` on Ollama claimed +//! 128 000 tokens while Ollama's real default `num_ctx` is **2048**, roughly +//! a 60× overstatement. Compaction fires at `window * threshold`, so it never +//! fired and the server silently truncated the front of the prompt. +//! LangChain refuses to guess here (ChatOllama ships no profile at all and +//! its summarization middleware hard-fails asking for absolute counts), and +//! an invented window is strictly worse than the `None` this crate already +//! supports. See [`LocalProbe::max_input_tokens`]. +//! 2. **Whether it accepts native `tools` is a property of the loaded model, +//! not of "being local".** The transport hard-disabled native tools for every +//! local runtime unconditionally, which forced the prompt-guided branch — +//! injecting the protocol block *plus* every tool's JSON Schema into the +//! system prompt, against that real 2048-token window, which then truncated +//! from the front and dropped the very prompt carrying the protocol. +//! Ollama reports this directly in `/api/show`'s `capabilities` array. +//! 3. **Some knobs have no OpenAI-wire spelling at all.** `num_ctx` and +//! `keep_alive` are `/api/chat` fields; `POST /v1/chat/completions` drops +//! them on the floor. See [`LocalRuntimeKind::native_root`]. +//! +//! Probing is **opt-in** and never runs during construction: it costs a network +//! round trip, and a constructor that blocks on one is unusable in the contexts +//! this crate is embedded in. + +use std::time::Duration; + +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::Error; + +/// A local OpenAI-compatible model server. +/// +/// The single place that answers "is this endpoint a local runtime, and which +/// one?". Adding a runtime is one variant plus its arms here — not a condition +/// to keep in sync across the transport. +/// +/// Before this existed only Ollama and LM Studio were recognised; +/// llama.cpp-server and vLLM fell through to the hosted `Compatible` path and +/// got Bearer auth, `tool_calling: true`, `image_in: true`, no `/v1` +/// normalisation, and none of the request-shape degrade knobs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LocalRuntimeKind { + /// Ollama. Serves an OpenAI-compatible surface under `/v1` **and** its own + /// native API under `/api` — the only place `num_ctx` and `keep_alive` are + /// readable. + Ollama, + /// LM Studio. OpenAI-compatible under `/v1`, with a richer model listing + /// under `/api/v0/models` (context length, load state, quantisation). + LmStudio, + /// `llama-server` from llama.cpp. OpenAI-compatible only. + LlamaCpp, + /// vLLM's OpenAI-compatible server. + Vllm, +} + +impl LocalRuntimeKind { + /// Stable identifier used in provider ids, log lines, and errors. + pub fn as_str(self) -> &'static str { + match self { + LocalRuntimeKind::Ollama => "ollama", + LocalRuntimeKind::LmStudio => "lm_studio", + LocalRuntimeKind::LlamaCpp => "llama_cpp", + LocalRuntimeKind::Vllm => "vllm", + } + } + + /// The server root assumed when a spec carries a blank `base_url`. + pub fn default_root(self) -> &'static str { + match self { + LocalRuntimeKind::Ollama => "http://localhost:11434", + LocalRuntimeKind::LmStudio => "http://localhost:1234", + LocalRuntimeKind::LlamaCpp => "http://localhost:8080", + LocalRuntimeKind::Vllm => "http://localhost:8000", + } + } + + /// Strips the OpenAI-compatibility suffix off `base_url`, yielding the + /// server root the runtime's **native** API hangs off. + /// + /// `http://localhost:11434/v1` → `http://localhost:11434`, so `/api/show`, + /// `/api/chat` and `/api/v0/models` can be reached. Idempotent for a base + /// that already is the root. + pub fn native_root(self, base_url: &str) -> String { + base_url + .trim_end_matches('/') + .trim_end_matches("/v1") + .trim_end_matches('/') + .to_string() + } + + /// Whether this runtime speaks a native (non-OpenAI) API this crate knows + /// how to use. Only Ollama does today. + pub fn has_native_api(self) -> bool { + matches!(self, LocalRuntimeKind::Ollama) + } +} + +/// What a probe of a live local server learned about the loaded model. +/// +/// Every field is [`Option`] on purpose: a runtime that does not report a fact +/// leaves it `None` and the caller keeps whatever it already had, rather than +/// having a guess written over it. This is the shape LangChain's +/// `libs/model-profiles` keys on (`max_input_tokens` is a first-class key +/// there) — with the pointed difference that no `ollama` profile file exists in +/// that repo, which is the evidence that a static catalogue is the wrong answer +/// for local models and runtime probing is the state of the art. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LocalProbe { + /// The model's real context window in tokens, when the server reports one. + /// + /// For Ollama this is the `*.context_length` entry of `/api/show`'s + /// `model_info` — the **architecture's** trained window. Note the runtime + /// still loads with `num_ctx` (default 2048) unless told otherwise, so a + /// caller that wants the full window must also request it; see + /// [`OpenAiModel::with_local_num_ctx`][wlnc]. + /// + /// [wlnc]: super::OpenAiModel::with_local_num_ctx + pub max_input_tokens: Option, + /// Whether the loaded model advertises native tool calling. + pub tool_calling: Option, + /// Whether the loaded model advertises image input. + pub vision: Option, + /// Whether the loaded model advertises a reasoning/thinking channel. + pub reasoning: Option, + /// The `num_ctx` the runtime says it actually loaded the model with, when + /// it reports one. This — not [`Self::max_input_tokens`] — is the number + /// that bounds a live request. + pub loaded_num_ctx: Option, +} + +impl LocalProbe { + /// Whether the probe learned anything at all. + pub fn is_empty(&self) -> bool { + *self == LocalProbe::default() + } + + /// The context window to advertise: the loaded `num_ctx` when known (it is + /// the real ceiling for a live request), else the architecture window, else + /// `None`. + /// + /// Deliberately **not** "the bigger of the two". Overstating the window is + /// the LOCAL-1 defect: compaction is gated on it, so a window larger than + /// the server will honour means compaction never fires and the server + /// truncates the prompt from the front instead — losing the system prompt + /// silently. + pub fn effective_context_window(&self) -> Option { + self.loaded_num_ctx.or(self.max_input_tokens) + } +} + +// --------------------------------------------------------------------------- +// Ollama `/api/show` +// --------------------------------------------------------------------------- + +/// The subset of Ollama's `POST /api/show` body this crate reads. +#[derive(Debug, Default, Deserialize)] +struct OllamaShowResponse { + /// Architecture metadata. Keys are namespaced by architecture + /// (`llama.context_length`, `qwen3.context_length`, …), so the reader scans + /// for a `*.context_length` suffix rather than guessing the prefix. + #[serde(default)] + model_info: serde_json::Map, + /// Capability tags: `completion`, `tools`, `vision`, `thinking`, `insert`. + #[serde(default)] + capabilities: Vec, +} + +/// Extracts the architecture context length from an Ollama `model_info` map. +/// +/// Keys are `{architecture}.context_length`, so match on the suffix. Returns +/// the smallest candidate when several match, staying conservative for the same +/// reason [`LocalProbe::effective_context_window`] does. +pub(super) fn context_length_from_model_info( + model_info: &serde_json::Map, +) -> Option { + model_info + .iter() + .filter(|(key, _)| key.ends_with(".context_length") || key.as_str() == "context_length") + .filter_map(|(_, value)| value.as_u64()) + .filter(|value| *value > 0) + .min() +} + +/// Turns an Ollama `/api/show` body into a [`LocalProbe`]. +/// +/// Pure, so the whole mapping is unit-testable without a live Ollama. +pub(super) fn probe_from_ollama_show(body: &Value) -> LocalProbe { + let parsed: OllamaShowResponse = + serde_json::from_value(body.clone()).unwrap_or_else(|_| OllamaShowResponse::default()); + let has = |tag: &str| { + parsed + .capabilities + .iter() + .any(|c| c.eq_ignore_ascii_case(tag)) + }; + // An empty `capabilities` array means "this server did not tell us", + // not "this model can do nothing" — leave those `None` so the caller keeps + // whatever it already had. + let capabilities_reported = !parsed.capabilities.is_empty(); + LocalProbe { + max_input_tokens: context_length_from_model_info(&parsed.model_info), + tool_calling: capabilities_reported.then(|| has("tools")), + vision: capabilities_reported.then(|| has("vision")), + reasoning: capabilities_reported.then(|| has("thinking")), + loaded_num_ctx: None, + } +} + +// --------------------------------------------------------------------------- +// LM Studio `/api/v0/models` +// --------------------------------------------------------------------------- + +/// One entry of LM Studio's richer `GET /api/v0/models` listing. +#[derive(Debug, Deserialize)] +struct LmStudioModel { + #[serde(default)] + id: String, + /// The model's context length. LM Studio reports the trained window here. + #[serde(default)] + max_context_length: Option, + /// The context the model is currently **loaded** with, when loaded. + #[serde(default)] + loaded_context_length: Option, + /// `llm`, `vlm` (vision), or `embeddings`. + #[serde(default)] + r#type: Option, +} + +#[derive(Debug, Default, Deserialize)] +struct LmStudioModelList { + #[serde(default)] + data: Vec, +} + +/// Turns an LM Studio `/api/v0/models` body into a [`LocalProbe`] for `model`. +/// +/// Pure, so the mapping is unit-testable without a live LM Studio. +pub(super) fn probe_from_lm_studio_models(body: &Value, model: &str) -> LocalProbe { + let parsed: LmStudioModelList = + serde_json::from_value(body.clone()).unwrap_or_else(|_| LmStudioModelList::default()); + let Some(entry) = parsed.data.iter().find(|m| m.id == model) else { + return LocalProbe::default(); + }; + LocalProbe { + max_input_tokens: entry.max_context_length.filter(|v| *v > 0), + // LM Studio does not report tool support in this listing; leave it + // untouched rather than inventing an answer. + tool_calling: None, + vision: entry + .r#type + .as_deref() + .map(|t| t.eq_ignore_ascii_case("vlm")), + reasoning: None, + loaded_num_ctx: entry.loaded_context_length.filter(|v| *v > 0), + } +} + +// --------------------------------------------------------------------------- +// Request bodies for the native escape hatches +// --------------------------------------------------------------------------- + +/// The `POST /api/show` body: which model to describe. +pub(super) fn ollama_show_body(model: &str) -> Value { + json!({ "model": model }) +} + +/// The `POST /api/chat` preflight body that loads `model` with explicit +/// `options` and residency. +/// +/// # Why a preflight rather than a request field +/// +/// `num_ctx` and `keep_alive` are **`/api/chat` fields**. The chat adapter +/// speaks `POST {base_url}/chat/completions`, and Ollama's OpenAI-compatibility +/// layer does not read them — so +/// [`with_default_provider_options`][super::OpenAiModel::with_default_provider_options] +/// documenting `{"options": {"num_ctx": 8192}}` as "the local escape hatch" +/// was, on this path, a field that went nowhere. (The crate's tests asserted +/// only that the request JSON *contained* it, never that a server honoured it, +/// which is exactly why that went unnoticed.) +/// +/// An `/api/chat` call with an empty `messages` array is Ollama's documented +/// **load** request: it loads the model with the given `options` and holds it +/// resident for `keep_alive`. Issuing it once before the first real turn gets +/// `num_ctx` where the OpenAI wire cannot, and doubles as the warm-up that +/// keeps Ollama from unloading after its 5-minute default and charging the next +/// turn a cold multi-gigabyte load inside the 600 s unary deadline. +/// +/// **Caveat, stated plainly:** this configures the *loaded runner*. Ollama +/// reuses an already-loaded runner for a subsequent `/v1` request that does not +/// demand conflicting options, which is the case here — but it is a property of +/// the server's runner reuse, not a guarantee of the OpenAI wire format. A full +/// native `/api/chat` chat adapter remains the complete fix and is called out as +/// a follow-up. +pub(super) fn ollama_load_body( + model: &str, + options: Option<&Value>, + keep_alive: Option<&str>, +) -> Value { + let mut body = json!({ "model": model, "messages": [] }); + if let Some(options) = options.filter(|o| o.is_object()) { + body["options"] = options.clone(); + } + if let Some(keep_alive) = keep_alive { + body["keep_alive"] = json!(keep_alive); + } + body +} + +/// Extracts an `options` object out of merged provider options, if present. +/// +/// The escape hatch's documented shape is `{"options": {"num_ctx": 8192}}`, so +/// this is what the preflight forwards natively. +pub(super) fn local_options_object(provider_options: &Value) -> Option<&Value> { + provider_options.get("options").filter(|v| v.is_object()) +} + +// --------------------------------------------------------------------------- +// Error classification +// --------------------------------------------------------------------------- + +/// Returns `true` when a provider failure says the model's Jinja chat template +/// rejected the message list. +/// +/// This is distinct from a rejected model id, sampling parameter, or +/// credential. The markers are emitted by local OpenAI-compatible runtimes +/// such as LM Studio, llama.cpp, and Ollama while rendering model-owned chat +/// templates. Matching is deliberately narrow and case-insensitive so hosts +/// can present accurate remediation without misclassifying ordinary 400s. +pub fn is_chat_template_rejection_message(body: &str) -> bool { + const PHRASES: &[&str] = &[ + "no user query found in messages", + "unable to generate parser for this template", + "automatic parser generation failed", + "jinja exception", + ]; + + let lower = body.to_ascii_lowercase(); + PHRASES.iter().any(|phrase| lower.contains(phrase)) +} + +/// Rewrites a local runtime's opaque 404 into a message naming the fix. +/// +/// The embeddings adapter has done this for a while — "Run `ollama pull +/// {model}` or choose an installed embedding model" — while the chat path +/// surfaced whatever the server said, typically a bare +/// `{"error":"model 'x' not found"}`. Returns `None` when the failure is not a +/// missing-model 404, so the original message survives untouched. +pub(super) fn missing_model_remediation( + kind: LocalRuntimeKind, + status: u16, + body: &str, + model: &str, + base_url: &str, +) -> Option { + if status != 404 { + return None; + } + let lower = body.to_ascii_lowercase(); + if !(lower.contains("model") + && (lower.contains("not found") || lower.contains("does not exist"))) + { + return None; + } + Some(match kind { + LocalRuntimeKind::Ollama => format!( + "Ollama model `{model}` is not installed at {base_url}. \ + Run `ollama pull {model}`, or call `list_models()` to see what is installed" + ), + LocalRuntimeKind::LmStudio => format!( + "LM Studio at {base_url} is not serving a model called `{model}`. \ + Load it in LM Studio, or call `list_models()` to see what is loaded — \ + the id is whichever GGUF the server has open, so there is no default to guess" + ), + _ => format!( + "{} at {base_url} is not serving a model called `{model}`. \ + Call `list_models()` to see what is available", + kind.as_str() + ), + }) +} + +/// Recognises "the prompt did not fit in this model's context window" from a +/// provider error. +/// +/// Hosted providers raise an explicit 400 for this; local servers usually +/// truncate the front of the prompt silently instead, which is why this must be +/// paired with a *real* context window from [`LocalProbe`] rather than relied on +/// alone. When it does fire, the classification is stable so a caller can act on +/// it (compact and retry) instead of string-matching a provider message. +/// +/// Surfaced as a [`ProviderError::code`][pe] of +/// [`CONTEXT_OVERFLOW_CODE`], because a typed +/// `TinyAgentsError::ContextOverflow` variant would have to be added in +/// `src/error.rs` — outside this module's ownership. Promoting the code to a +/// typed variant is a follow-up. +/// +/// [pe]: crate::model::ProviderError::code +pub(super) fn is_context_overflow(status: u16, message: &str) -> bool { + if !matches!(status, 400 | 413 | 422 | 500) { + return false; + } + let lower = message.to_ascii_lowercase(); + const PHRASES: [&str; 7] = [ + "context length", + "context window", + "maximum context", + "too many tokens", + "reduce the length of the messages", + "prompt is too long", + "exceeds the maximum", + ]; + PHRASES.iter().any(|phrase| lower.contains(phrase)) +} + +/// The [`ProviderError::code`][pe] stamped on a recognised context overflow. +/// +/// [pe]: crate::model::ProviderError::code +pub const CONTEXT_OVERFLOW_CODE: &str = "context_overflow"; + +/// Recognises "this endpoint rejects the `tools` parameter" from a 400 body. +/// +/// Drives the [`Degrade::native_tools`][d] latch, which is the auto-degrade half +/// of C11: a local server that cannot do native tools tells us so once, and +/// every later call goes straight to the prompt-guided branch — instead of the +/// old behaviour, which assumed *every* local server was in that state forever. +/// +/// [d]: super::transport::Degrade +#[cfg(test)] +pub(super) fn mentions_tools_unsupported(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + if !(lower.contains("tool") || lower.contains("function")) { + return false; + } + // `tool_choice` rejections are a *different* degrade with its own latch; + // matching them here would flip the wrong knob. + if lower.contains("tool_choice") && !lower.contains("tools") { + return false; + } + const PHRASES: [&str; 8] = [ + "does not support tools", + "does not support function", + "not supported", + "unsupported", + "unknown parameter", + "unrecognized", + "invalid parameter", + "no tool support", + ]; + PHRASES.iter().any(|phrase| lower.contains(phrase)) +} + +/// Deadline for a probe request. Probing is a convenience, never the point of +/// the call, so it fails fast rather than blocking a turn behind a wedged +/// server — the same failure the `list_models` deadline was added for. +pub(super) const PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Maps a probe transport failure onto the crate error with a grep-friendly +/// message naming the endpoint. +pub(super) fn probe_error(endpoint: &str, detail: impl std::fmt::Display) -> Error { + Error::Model(format!( + "[openai] local probe of {endpoint} failed: {detail}" + )) +} + +/// Normalizes a local runtime root to its OpenAI-compatible `/v1` base URL. +pub(super) fn normalize_local_v1_base_url( + raw: String, + default_root: &str, +) -> crate::Result { + let trimmed = raw.trim().trim_end_matches('/'); + let root = if trimmed.is_empty() { + default_root.to_owned() + } else if trimmed.contains("://") { + trimmed.to_owned() + } else { + format!("http://{trimmed}") + }; + let mut url = reqwest::Url::parse(&root).map_err(|error| { + Error::Validation(format!("invalid local runtime URL `{root}`: {error}")) + })?; + if !matches!(url.scheme(), "http" | "https") { + return Err(Error::Validation(format!( + "local runtime URL must use http or https, got `{}`", + url.scheme() + ))); + } + let mut segments: Vec<&str> = url + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + if segments.ends_with(&["chat", "completions"]) { + segments.truncate(segments.len() - 2); + } else if segments.last() == Some(&"models") { + segments.pop(); + } + if segments.last() != Some(&"v1") { + segments.push("v1"); + } + url.set_path(&format!("/{}", segments.join("/"))); + url.set_query(None); + url.set_fragment(None); + Ok(url.to_string().trim_end_matches('/').to_owned()) +} diff --git a/crates/tinyinference/src/providers/openai/local_test.rs b/crates/tinyinference/src/providers/openai/local_test.rs new file mode 100644 index 0000000..19c0afa --- /dev/null +++ b/crates/tinyinference/src/providers/openai/local_test.rs @@ -0,0 +1,270 @@ +//! Unit tests for local-runtime identification, probing, and classification. + +use super::*; +use serde_json::json; + +const LMSTUDIO_CHAT_TEMPLATE_REJECTION: &str = "lmstudio returned: Engine protocol predict \ + request returned 400: {\"error\":{\"code\":400,\"message\":\"Unable to generate parser \ + for this template. Automatic parser generation failed: While executing CallExpression at \ + line 79, column 24 in source: {{- raise_exception('No user query found in messages.') }}. \ + Error: Jinja Exception: No user query found in messages.\"}}"; + +#[test] +fn chat_template_rejections_are_classified_inside_runtime_wrappers() { + let aggregate = + format!("The model may not be available. Attempts: {LMSTUDIO_CHAT_TEMPLATE_REJECTION}"); + assert!(is_chat_template_rejection_message(&aggregate)); +} + +#[test] +fn chat_template_rejection_detection_is_case_insensitive() { + assert!(is_chat_template_rejection_message( + "Error: JINJA EXCEPTION: No User Query Found In Messages." + )); +} + +#[test] +fn unrelated_provider_rejections_are_not_chat_template_failures() { + for body in [ + "openai API error (400): invalid temperature: only 1 is allowed for this model", + "The model `gpt-5.5` does not exist or you do not have access to it.", + "lmstudio returned: model 'qwen3.5-9b' does not support tools", + "openrouter API error (429): rate limited", + "Failed to render the prompt template file on disk", + ] { + assert!(!is_chat_template_rejection_message(body), "{body:?}"); + } +} + +#[test] +fn native_root_strips_the_openai_compat_suffix() { + let kind = LocalRuntimeKind::Ollama; + assert_eq!( + kind.native_root("http://localhost:11434/v1"), + "http://localhost:11434" + ); + assert_eq!( + kind.native_root("http://localhost:11434/v1/"), + "http://localhost:11434" + ); + // Idempotent for a base that is already the root. + assert_eq!( + kind.native_root("http://localhost:11434"), + "http://localhost:11434" + ); +} + +#[test] +fn ollama_show_reports_the_real_window_not_the_model_id_guess() { + // `llama3.2:3b` matches the generic hint table's `("llama3", Substring, + // 128_000)` entry. The server says 8192. The probe must report the server. + let body = json!({ + "model_info": { + "general.architecture": "llama", + "llama.context_length": 8192, + "llama.embedding_length": 3072 + }, + "capabilities": ["completion", "tools"] + }); + let probe = probe_from_ollama_show(&body); + assert_eq!(probe.max_input_tokens, Some(8192)); + assert_eq!(probe.tool_calling, Some(true)); + assert_eq!(probe.vision, Some(false)); + assert_eq!(probe.reasoning, Some(false)); +} + +#[test] +fn ollama_show_reads_vision_and_thinking_capabilities() { + let body = json!({ + "model_info": { "qwen3.context_length": 40960 }, + "capabilities": ["completion", "tools", "vision", "thinking"] + }); + let probe = probe_from_ollama_show(&body); + assert_eq!(probe.max_input_tokens, Some(40960)); + assert_eq!(probe.tool_calling, Some(true)); + assert_eq!(probe.vision, Some(true)); + assert_eq!(probe.reasoning, Some(true)); +} + +#[test] +fn an_absent_capabilities_array_leaves_every_capability_unknown() { + // "the server did not tell us" must not be read as "the model cannot". + let body = json!({ "model_info": { "llama.context_length": 4096 } }); + let probe = probe_from_ollama_show(&body); + assert_eq!(probe.max_input_tokens, Some(4096)); + assert_eq!(probe.tool_calling, None); + assert_eq!(probe.vision, None); + assert_eq!(probe.reasoning, None); +} + +#[test] +fn a_body_without_model_info_probes_to_nothing_rather_than_a_guess() { + assert!(probe_from_ollama_show(&json!({})).is_empty()); + assert!(probe_from_ollama_show(&json!({ "error": "model not found" })).is_empty()); +} + +#[test] +fn context_length_scan_is_architecture_agnostic_and_conservative() { + let info = json!({ "gemma3.context_length": 8192, "clip.context_length": 77 }) + .as_object() + .cloned() + .unwrap(); + // Multimodal models carry a second, tiny window for the projector. Taking + // the max would overstate; take the min. + assert_eq!(context_length_from_model_info(&info), Some(77)); + + let zeroed = json!({ "llama.context_length": 0 }) + .as_object() + .cloned() + .unwrap(); + assert_eq!(context_length_from_model_info(&zeroed), None); +} + +#[test] +fn lm_studio_listing_reports_loaded_and_trained_windows() { + let body = json!({ + "data": [ + { "id": "other-model", "max_context_length": 999999 }, + { + "id": "qwen3-4b", + "type": "llm", + "max_context_length": 40960, + "loaded_context_length": 4096 + } + ] + }); + let probe = probe_from_lm_studio_models(&body, "qwen3-4b"); + assert_eq!(probe.max_input_tokens, Some(40960)); + assert_eq!(probe.loaded_num_ctx, Some(4096)); + assert_eq!(probe.vision, Some(false)); + // The loaded window is the ceiling a live request actually has. + assert_eq!(probe.effective_context_window(), Some(4096)); +} + +#[test] +fn lm_studio_vision_models_are_detected_by_type() { + let body = json!({ "data": [{ "id": "llava", "type": "vlm", "max_context_length": 4096 }] }); + assert_eq!( + probe_from_lm_studio_models(&body, "llava").vision, + Some(true) + ); +} + +#[test] +fn an_unlisted_lm_studio_model_probes_to_nothing() { + let body = json!({ "data": [{ "id": "a", "max_context_length": 4096 }] }); + assert!(probe_from_lm_studio_models(&body, "b").is_empty()); +} + +#[test] +fn effective_window_prefers_the_loaded_ctx_over_the_trained_window() { + let probe = LocalProbe { + max_input_tokens: Some(131_072), + loaded_num_ctx: Some(2048), + ..LocalProbe::default() + }; + // Overstating is the LOCAL-1 defect: compaction is gated on this number, so + // a window bigger than the server honours means it never fires. + assert_eq!(probe.effective_context_window(), Some(2048)); +} + +#[test] +fn load_body_carries_num_ctx_and_keep_alive_natively() { + let options = json!({ "num_ctx": 8192, "num_batch": 512 }); + let body = ollama_load_body("llama3.2", Some(&options), Some("30m")); + assert_eq!(body["model"], json!("llama3.2")); + assert_eq!(body["messages"], json!([])); + // The whole point: `num_ctx` reaches a field Ollama actually reads. + assert_eq!(body["options"]["num_ctx"], json!(8192)); + assert_eq!(body["options"]["num_batch"], json!(512)); + assert_eq!(body["keep_alive"], json!("30m")); +} + +#[test] +fn load_body_omits_absent_and_malformed_options() { + let body = ollama_load_body("m", None, None); + assert!(body.get("options").is_none()); + assert!(body.get("keep_alive").is_none()); + + let scalar = json!("not an object"); + let body = ollama_load_body("m", Some(&scalar), None); + assert!(body.get("options").is_none()); +} + +#[test] +fn options_are_lifted_out_of_the_documented_provider_options_shape() { + let provider_options = json!({ "options": { "num_ctx": 8192 }, "keep_alive": "5m" }); + assert_eq!( + local_options_object(&provider_options), + Some(&json!({ "num_ctx": 8192 })) + ); + assert_eq!(local_options_object(&json!({})), None); + // A non-object `options` is caller error, not something to forward. + assert_eq!(local_options_object(&json!({ "options": 7 })), None); +} + +#[test] +fn missing_model_404_gains_an_actionable_remediation() { + let message = missing_model_remediation( + LocalRuntimeKind::Ollama, + 404, + r#"{"error":"model 'llama3.2' not found"}"#, + "llama3.2", + "http://localhost:11434/v1", + ) + .expect("a missing-model 404 is rewritten"); + assert!(message.contains("ollama pull llama3.2"), "{message}"); + + let lm = missing_model_remediation( + LocalRuntimeKind::LmStudio, + 404, + "model does not exist", + "qwen3-4b", + "http://localhost:1234/v1", + ) + .expect("LM Studio gets its own wording"); + assert!(lm.contains("list_models()"), "{lm}"); +} + +#[test] +fn unrelated_failures_keep_their_original_message() { + assert!(missing_model_remediation(LocalRuntimeKind::Ollama, 500, "boom", "m", "u").is_none()); + assert!( + missing_model_remediation(LocalRuntimeKind::Ollama, 404, "route not found", "m", "u") + .is_none() + ); + assert!( + missing_model_remediation(LocalRuntimeKind::Ollama, 401, "model not found", "m", "u") + .is_none() + ); +} + +#[test] +fn context_overflow_is_recognised_across_provider_phrasings() { + assert!(is_context_overflow( + 400, + "This model's maximum context length is 4096 tokens, however you requested 5000" + )); + assert!(is_context_overflow(413, "prompt is too long")); + assert!(is_context_overflow( + 400, + "Please reduce the length of the messages" + )); + // Not an overflow. + assert!(!is_context_overflow(400, "invalid api key")); + assert!(!is_context_overflow(401, "maximum context length exceeded")); +} + +#[test] +fn tools_rejections_are_told_apart_from_tool_choice_rejections() { + assert!(mentions_tools_unsupported( + "registry.ollama.ai/library/gemma3 does not support tools" + )); + assert!(mentions_tools_unsupported("unknown parameter: 'tools'")); + // `tool_choice` has its own latch; flipping the tools latch for it would + // disable native tools on a server that supports them fine. + assert!(!mentions_tools_unsupported( + "invalid parameter: tool_choice must be a string" + )); + assert!(!mentions_tools_unsupported("rate limited")); +} diff --git a/crates/tinyinference/src/providers/openai/mod.rs b/crates/tinyinference/src/providers/openai/mod.rs index 5bfdd3c..283e697 100644 --- a/crates/tinyinference/src/providers/openai/mod.rs +++ b/crates/tinyinference/src/providers/openai/mod.rs @@ -75,16 +75,21 @@ const DEFAULT_CONNECT_TIMEOUT_SECS: u64 = 30; const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 600; mod convert; +mod local; mod prompt_tools; mod reasoning_tags; mod responses; mod sse; mod transport; +pub use local::{ + CONTEXT_OVERFLOW_CODE, LocalProbe, LocalRuntimeKind, is_chat_template_rejection_message, +}; pub use reasoning_tags::ReasoningTagExtraction; pub use transport::{AuthStyle, OpenAiModel}; use convert::*; +use local::*; use reasoning_tags::*; use sse::*; #[cfg(test)] @@ -93,5 +98,7 @@ use transport::{ merge_provider_options, merge_system_into_user, request_timeout, }; +#[cfg(test)] +mod local_test; #[cfg(test)] mod test; diff --git a/crates/tinyinference/src/providers/openai/responses.rs b/crates/tinyinference/src/providers/openai/responses.rs index ed39cab..782c394 100644 --- a/crates/tinyinference/src/providers/openai/responses.rs +++ b/crates/tinyinference/src/providers/openai/responses.rs @@ -7,25 +7,51 @@ //! OpenAI Codex OAuth path requires (paired with `with_extra_query_param` + //! `with_user_agent`). //! -//! This first port is **text-in / text-out**: system messages fold into -//! `instructions`, user/assistant/tool turns become `input` items, and the -//! terminal `output_text` (or the first `output_text` content part) becomes the -//! assistant reply. Native tool calls over `/responses` and true SSE streaming -//! are follow-ups; the harness embeds tool specs in the prompt for this path -//! (its [`profile`](super::OpenAiModel) advertises the caller's chosen -//! `tool_calling`). +//! System messages fold into `instructions`, user/assistant/tool turns become +//! `input` items, and the terminal `output_text` (or the first `output_text` +//! content part) becomes the assistant reply. +//! +//! # What this path now carries +//! +//! The request used to be `{model, input, instructions, stream, store, +//! max_output_tokens}` and **silently dropped everything else** a caller set — +//! `tools`, `tool_choice`, `response_format`, `temperature`, `top_p`, `seed`, +//! `stop_sequences`, `continuation_id`, and `provider_options`. That last one +//! made `reasoning: {effort, summary}` unreachable on the only wire format in +//! this crate that supports it. All of them are on the wire now, and the +//! response side reads reasoning items, their `encrypted_content`, and the +//! cache/reasoning usage breakdowns that were previously ignored (so every +//! cached token on this path was billed at the full input rate). +//! +//! # Remaining gaps +//! +//! Tool *declarations* are sent, but a model that calls one comes back as a +//! `function_call` output item this port does not yet decode into +//! [`ToolCall`](crate::tool::ToolCall)s — and tool *results* are +//! rendered as `user` turns carrying an explicit `[tool_result id=…]` prefix +//! rather than native `function_call_output` items. That preserves the causal +//! link the previous fold-into-assistant behaviour erased, but structural +//! tool support and true SSE streaming remain follow-ups. use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::message::{AssistantMessage, ContentBlock, Message}; -use crate::model::{ModelResponse, ToolChoice}; -use crate::tool::{ToolCall, ToolSchema}; +use crate::model::ModelResponse; use crate::usage::Usage; -use crate::{Error, Result}; /// The `/v1/responses` request body. -#[derive(Debug, Serialize)] +/// +/// # What used to be missing +/// +/// This struct carried only `{model, input, instructions, stream, store, +/// max_output_tokens}`. Everything else a caller set was **silently dropped**: +/// `tools`, `tool_choice`, `response_format`, `temperature`, `top_p`, +/// `stop_sequences`, `seed`, `previous_response_id`, and — most pointedly — +/// `provider_options`, which meant `reasoning: {effort, summary}` was +/// unreachable on the one wire format that supports it. A request that looked +/// fully configured produced an unconfigured call. +#[derive(Debug, Default, Serialize)] pub(super) struct ResponsesRequest { pub(super) model: String, pub(super) input: Vec, @@ -39,52 +65,128 @@ pub(super) struct ResponsesRequest { /// `max_tokens`. Omitted for the Codex OAuth backend, which rejects it. #[serde(skip_serializing_if = "Option::is_none")] pub(super) max_output_tokens: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] + /// Tool declarations. The Responses API flattens the function schema onto + /// the tool object rather than nesting it under `function` as Chat + /// Completions does. + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(super) tools: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub(super) tool_choice: Option, + /// Structured output. The Responses API nests it under `text.format`, not + /// the Chat Completions `response_format`. + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) text: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) seed: Option, + /// Stop sequences. Serialized only when non-empty. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) stop: Vec, + /// The stateful-continuation handle. [`ModelRequest::continuation_id`] + /// existed with a builder and **no reader anywhere in the crate**, so this + /// was never sent and stateful follow-ups silently restarted. #[serde(skip_serializing_if = "Option::is_none")] pub(super) previous_response_id: Option, + /// `reasoning: { effort, summary }`, lowered from the provider-neutral + /// [`ReasoningConfig`][rc]. + /// + /// [rc]: crate::model::ReasoningConfig #[serde(skip_serializing_if = "Option::is_none")] - pub(super) text: Option, - #[serde(flatten, skip_serializing_if = "serde_json::Map::is_empty")] + pub(super) reasoning: Option, + /// Which extra payloads to return. + /// + /// Load-bearing for reasoning replay: with `store: false` the server keeps + /// no state, so reasoning items may be dropped between turns **unless** they + /// carry `encrypted_content` — which only arrives when + /// `include: ["reasoning.encrypted_content"]` is requested. Asking for + /// reasoning and not asking for this is asking for reasoning that cannot be + /// replayed. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) include: Vec, + /// Provider-specific passthrough merged onto the body. Keys here win, and + /// this is the escape hatch for anything the typed fields above cannot say. + #[serde(flatten)] pub(super) extra: serde_json::Map, } +/// The `include` entry that makes reasoning replayable under `store: false`. +pub(super) const INCLUDE_ENCRYPTED_REASONING: &str = "reasoning.encrypted_content"; + +/// Lowers a provider-neutral [`ReasoningConfig`][rc] onto the Responses +/// `reasoning` object. +/// +/// Returns `None` when the config asks for nothing, so an empty config never +/// adds a field. `budget_tokens` has no Responses spelling and is dropped here +/// deliberately — it is Anthropic's knob, and inventing an OpenAI field for it +/// would be worse than ignoring it. +/// +/// [rc]: crate::model::ReasoningConfig +pub(super) fn translate_reasoning(config: &crate::model::ReasoningConfig) -> Option { + if config.is_empty() { + return None; + } + let mut object = serde_json::Map::new(); + if let Some(effort) = config.effort { + object.insert("effort".to_string(), Value::String(effort.as_str().into())); + } + if let Some(summary) = &config.summary { + object.insert("summary".to_string(), Value::String(summary.clone())); + } + (!object.is_empty()).then_some(Value::Object(object)) +} + +/// Translates a tool schema onto the Responses API's flattened tool shape. +pub(super) fn translate_tool(schema: &crate::tool::ToolSchema) -> Value { + serde_json::json!({ + "type": "function", + "name": schema.name, + "description": schema.description, + "parameters": schema.parameters, + }) +} + +/// Translates a [`ResponseFormat`][rf] onto the Responses API's `text.format` +/// nesting (Chat Completions' `response_format` has no counterpart here). +/// +/// [rf]: crate::model::ResponseFormat +pub(super) fn translate_text_format( + format: &crate::model::ResponseFormat, + strict: bool, +) -> Option { + use crate::model::ResponseFormat; + let inner = match format { + ResponseFormat::Text => return None, + ResponseFormat::JsonObject => serde_json::json!({ "type": "json_object" }), + ResponseFormat::JsonSchema { name, schema } | ResponseFormat::Auto { name, schema } => { + serde_json::json!({ + "type": "json_schema", + "name": name, + "schema": schema, + "strict": strict, + }) + } + }; + Some(serde_json::json!({ "format": inner })) +} + #[derive(Debug, Serialize)] pub(super) struct ResponsesInput { - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) role: Option, - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub(super) kind: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] + pub(super) role: String, pub(super) content: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) call_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) arguments: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) output: Option, } #[derive(Debug, Serialize)] pub(super) struct ResponsesContentPart { #[serde(rename = "type")] pub(super) kind: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) text: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(super) image_url: Option, + pub(super) text: String, } #[derive(Debug, Deserialize)] pub(super) struct ResponsesResponse { - #[serde(default)] - pub(super) status: Option, - #[serde(default)] - pub(super) incomplete_details: Option, #[serde(default)] pub(super) output: Vec, #[serde(default)] @@ -93,24 +195,24 @@ pub(super) struct ResponsesResponse { pub(super) usage: Option, } -#[derive(Debug, Deserialize)] -pub(super) struct ResponsesIncompleteDetails { - #[serde(default)] - pub(super) reason: Option, -} - -#[derive(Debug, Deserialize)] +#[derive(Debug, Default, Deserialize)] pub(super) struct ResponsesOutput { + /// Item kind: `message`, `reasoning`, `function_call`, … #[serde(rename = "type", default)] pub(super) kind: Option, #[serde(default)] pub(super) content: Vec, + /// Reasoning summary parts, on a `reasoning` item. #[serde(default)] - pub(super) call_id: Option, - #[serde(default)] - pub(super) name: Option, + pub(super) summary: Vec, + /// The opaque reasoning payload that survives `store: false`. + /// + /// Only present when the request asked for + /// [`INCLUDE_ENCRYPTED_REASONING`]. Preserved so a caller can replay + /// reasoning across turns; without it the server drops reasoning between + /// stateless turns. #[serde(default)] - pub(super) arguments: Option, + pub(super) encrypted_content: Option, } #[derive(Debug, Deserialize)] @@ -120,13 +222,44 @@ pub(super) struct ResponsesContent { pub(super) text: Option, } -/// Responses-API usage block (`input_tokens` / `output_tokens`). -#[derive(Debug, Deserialize)] +/// Responses-API usage block. +/// +/// The details sub-objects used to be absent from this struct entirely, so +/// **every cached token on this path was billed at the full input rate** and +/// reasoning tokens were invisible. +#[derive(Debug, Default, Deserialize)] pub(super) struct ResponsesUsage { #[serde(default)] pub(super) input_tokens: Option, #[serde(default)] pub(super) output_tokens: Option, + #[serde(default)] + pub(super) input_tokens_details: Option, + #[serde(default)] + pub(super) output_tokens_details: Option, +} + +/// `usage.input_tokens_details` — the cache breakdown of the input total. +#[derive(Debug, Default, Deserialize)] +pub(super) struct ResponsesInputTokenDetails { + #[serde(default)] + pub(super) cached_tokens: Option, + /// Cache **writes**, under either spelling gateways use. + #[serde(default)] + pub(super) cache_write_tokens: Option, + #[serde(default)] + pub(super) cache_creation_tokens: Option, +} + +/// `usage.output_tokens_details` — where OpenAI reports reasoning tokens. +/// +/// Note that **Anthropic has no equivalent field**: its thinking tokens are +/// billed inside `output_tokens`, so a zero here is not evidence that no +/// reasoning happened on an Anthropic-shaped gateway. +#[derive(Debug, Default, Deserialize)] +pub(super) struct ResponsesOutputTokenDetails { + #[serde(default)] + pub(super) reasoning_tokens: Option, } /// Concatenates the visible text of a message's content blocks. @@ -138,84 +271,22 @@ fn message_text(content: &[ContentBlock]) -> String { .join("") } -fn message_output_text(content: &[ContentBlock]) -> Result { - let mut output = String::new(); - for block in content { - match block { - ContentBlock::Text(text) => output.push_str(text), - ContentBlock::Json(value) => output.push_str(&value.to_string()), - ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } => {} - ContentBlock::Image(_) | ContentBlock::ProviderExtension(_) => { - return Err(Error::Validation( - "tool output cannot be represented by the Responses API".into(), - )); - } - } - } - Ok(output) -} - -fn input_parts(message: &Message) -> Result> { - let role = normalize_role(message); - let content = match message { - Message::System(message) => &message.content, - Message::User(message) => &message.content, - Message::Assistant(message) => &message.content, - Message::Tool(message) => &message.content, - }; - let mut parts = Vec::new(); - for block in content { - match block { - ContentBlock::Text(text) if !text.trim().is_empty() => { - parts.push(ResponsesContentPart { - kind: if role == "assistant" { - "output_text".into() - } else { - "input_text".into() - }, - text: Some(text.clone()), - image_url: None, - }); - } - ContentBlock::Json(value) => parts.push(ResponsesContentPart { - kind: if role == "assistant" { - "output_text".into() - } else { - "input_text".into() - }, - text: Some(value.to_string()), - image_url: None, - }), - ContentBlock::Image(image) if matches!(message, Message::User(_)) => { - parts.push(ResponsesContentPart { - kind: "input_image".into(), - text: None, - image_url: Some(image.url.clone()), - }); - } - ContentBlock::Image(_) => { - return Err(Error::Validation( - "Responses API images are supported only in user messages".into(), - )); - } - ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } => {} - ContentBlock::ProviderExtension(_) => { - return Err(Error::Validation( - "provider extension content cannot be represented by the Responses API".into(), - )); - } - ContentBlock::Text(_) => {} - } - } - Ok(parts) -} - -/// Normalizes a message role for the Responses API: assistant + tool turns fold -/// into `assistant` (which the API keys to `output_text`), everything else to -/// `user` (`input_text`). Mirrors the host `normalize_responses_role`. +/// Normalizes a message role for the Responses API. +/// +/// Assistant turns key to `output_text`; everything else to `input_text`. +/// +/// **Tool results no longer fold into `assistant`.** They used to, which erased +/// tool identity entirely: a tool result became an anonymous assistant utterance +/// with no `tool_call_id`, so the model saw an assistant asserting a fact rather +/// than the answer to a call it made. They are now rendered as `user` turns +/// carrying an explicit `[tool_result …]` prefix (see +/// [`build_responses_input`]), which preserves the causal link on a wire format +/// this text-in/text-out port cannot express structurally. A true +/// `function_call_output` item is the complete fix and rides with native tool +/// support on this path. fn normalize_role(message: &Message) -> &'static str { match message { - Message::Assistant(_) | Message::Tool(_) => "assistant", + Message::Assistant(_) => "assistant", _ => "user", } } @@ -225,14 +296,12 @@ fn normalize_role(message: &Message) -> &'static str { /// the content-part `kind` tracks the *normalized* role (`output_text` for /// assistant/tool, `input_text` otherwise) — the API rejects `input_text` on an /// assistant item. -pub(super) fn build_responses_input( - messages: &[Message], -) -> Result<(Option, Vec)> { +pub(super) fn build_responses_input(messages: &[Message]) -> (Option, Vec) { let mut instructions_parts = Vec::new(); let mut input = Vec::new(); for message in messages { - match message { + let text = match message { Message::System(m) => { let t = message_text(&m.content); if !t.trim().is_empty() { @@ -240,144 +309,44 @@ pub(super) fn build_responses_input( } continue; } - Message::Tool(tool) => { - input.push(ResponsesInput { - role: None, - kind: Some("function_call_output".into()), - content: Vec::new(), - call_id: Some(tool.tool_call_id.clone()), - name: None, - arguments: None, - output: Some(message_output_text(&tool.content)?), - }); - continue; - } - Message::Assistant(assistant) => { - let content = input_parts(message)?; - if !content.is_empty() { - input.push(ResponsesInput { - role: Some("assistant".into()), - kind: None, - content, - call_id: None, - name: None, - arguments: None, - output: None, - }); - } - for call in &assistant.tool_calls { - input.push(ResponsesInput { - role: None, - kind: Some("function_call".into()), - content: Vec::new(), - call_id: Some(call.id.clone()), - name: Some(call.name.clone()), - arguments: Some(serde_json::to_string(&call.arguments)?), - output: None, - }); + Message::User(m) => message_text(&m.content), + Message::Assistant(m) => message_text(&m.content), + // Keep the call id visible so the model can tell *which* call this + // answers. Folding it into an anonymous assistant turn lost that. + Message::Tool(m) => { + let body = message_text(&m.content); + if body.trim().is_empty() { + String::new() + } else { + format!("[tool_result id={} ]\n{body}", m.tool_call_id) } - continue; } - Message::User(_) => {} - } - let content = input_parts(message)?; - if content.is_empty() { + }; + if text.trim().is_empty() { continue; } let role = normalize_role(message); input.push(ResponsesInput { - role: Some(role.to_string()), - kind: None, - content, - call_id: None, - name: None, - arguments: None, - output: None, + role: role.to_string(), + content: vec![ResponsesContentPart { + kind: if role == "assistant" { + "output_text".to_string() + } else { + "input_text".to_string() + }, + text, + }], }); } let instructions = (!instructions_parts.is_empty()).then(|| instructions_parts.join("\n\n")); - Ok((instructions, input)) -} - -pub(super) fn responses_text_format( - format: Option<&crate::model::ResponseFormat>, -) -> Option { - use crate::model::ResponseFormat; - - format.map(|format| match format { - ResponseFormat::Text => serde_json::json!({"format": {"type": "text"}}), - ResponseFormat::JsonObject => { - serde_json::json!({"format": {"type": "json_object"}}) - } - ResponseFormat::JsonSchema { name, schema } | ResponseFormat::Auto { name, schema } => { - serde_json::json!({ - "format": { - "type": "json_schema", - "name": name, - "schema": schema, - "strict": true - } - }) - } - }) -} - -pub(super) fn responses_extra_options(options: &Value) -> Result> { - if options.is_null() { - return Ok(serde_json::Map::new()); - } - let object = options.as_object().ok_or_else(|| { - Error::Validation("provider_options for Responses must be a JSON object".into()) - })?; - const RESERVED: &[&str] = &[ - "model", - "input", - "instructions", - "stream", - "store", - "max_output_tokens", - "tools", - "tool_choice", - "previous_response_id", - "text", - ]; - Ok(object - .iter() - .filter(|(key, _)| !RESERVED.contains(&key.as_str())) - .map(|(key, value)| (key.clone(), value.clone())) - .collect()) -} - -pub(super) fn responses_tools(tools: &[ToolSchema]) -> Vec { - tools - .iter() - .map(|tool| { - serde_json::json!({ - "type": "function", - "name": tool.name, - "description": tool.description, - "parameters": tool.parameters, - }) - }) - .collect() -} - -pub(super) fn responses_tool_choice(choice: &ToolChoice, has_tools: bool) -> Option { - if !has_tools { - return None; - } - Some(match choice { - ToolChoice::Auto => Value::String("auto".into()), - ToolChoice::None => Value::String("none".into()), - ToolChoice::Required => Value::String("required".into()), - ToolChoice::Tool(name) => serde_json::json!({"type": "function", "name": name}), - }) + (instructions, input) } /// Extracts the assistant text from a Responses body: the convenience /// `output_text` field first, else the first `output_text` content part. pub(super) fn extract_responses_text(response: &ResponsesResponse) -> Option { + // `output_text` is the whole answer when the server supplies it. if let Some(text) = response .output_text .as_deref() @@ -387,6 +356,11 @@ pub(super) fn extract_responses_text(response: &ResponsesResponse) -> Option Option Result { - let parsed: ResponsesResponse = serde_json::from_value(value.clone())?; - let text = extract_responses_text(&parsed).unwrap_or_default(); - let tool_calls = parsed +/// Collects the reasoning text a Responses body carries. +/// +/// Reads `reasoning` items' `summary` parts, then any `content` parts on the +/// same item. Returns `None` when there is none. The parser used to read +/// **no** reasoning at all from this path. +pub(super) fn extract_responses_reasoning(response: &ResponsesResponse) -> Option { + let mut text = String::new(); + for item in &response.output { + if item.kind.as_deref() != Some("reasoning") { + continue; + } + for part in item.summary.iter().chain(item.content.iter()) { + if let Some(fragment) = part + .text + .as_deref() + .map(str::trim) + .filter(|t| !t.is_empty()) + { + if !text.is_empty() { + text.push('\n'); + } + text.push_str(fragment); + } + } + } + (!text.is_empty()).then_some(text) +} + +/// The opaque reasoning payload to replay on the next turn, when the request +/// asked for [`INCLUDE_ENCRYPTED_REASONING`] and the server supplied it. +pub(super) fn extract_encrypted_reasoning(response: &ResponsesResponse) -> Option { + response .output .iter() - .filter(|item| item.kind.as_deref() == Some("function_call")) - .map(|item| { - let id = item.call_id.clone().ok_or_else(|| { - Error::Serialization(serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Responses function_call missing call_id", - ))) - })?; - let name = item.name.clone().ok_or_else(|| { - Error::Serialization(serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Responses function_call missing name", - ))) - })?; - let raw = item.arguments.clone().unwrap_or_else(|| "{}".into()); - Ok(match serde_json::from_str(&raw) { - Ok(arguments) => ToolCall::new(id, name, arguments), - Err(error) => ToolCall::invalid(id, name, raw, error.to_string()), - }) + .find_map(|item| item.encrypted_content.clone()) + .filter(|value| !value.is_empty()) +} + +/// Maps a Responses `usage` block onto the neutral [`Usage`], including the +/// cache and reasoning breakdowns. +pub(super) fn convert_responses_usage(wire: &ResponsesUsage) -> Usage { + let input_details = wire.input_tokens_details.as_ref(); + let cache_read_tokens = input_details.and_then(|d| d.cached_tokens).unwrap_or(0); + let cache_creation_tokens = input_details + .map(|d| { + d.cache_write_tokens + .unwrap_or(0) + .max(d.cache_creation_tokens.unwrap_or(0)) }) - .collect::>>()?; - let usage = parsed.usage.as_ref().map(|u| Usage { - input_tokens: u.input_tokens.unwrap_or(0), - output_tokens: u.output_tokens.unwrap_or(0), - total_tokens: u.input_tokens.unwrap_or(0) + u.output_tokens.unwrap_or(0), - ..Usage::default() - }); - Ok(ModelResponse { + .unwrap_or(0); + let input_tokens = wire.input_tokens.unwrap_or(0); + let output_tokens = wire.output_tokens.unwrap_or(0); + Usage { + input_tokens, + output_tokens, + total_tokens: input_tokens + output_tokens, + cache_read_tokens, + cache_creation_tokens, + reasoning_tokens: wire + .output_tokens_details + .as_ref() + .and_then(|d| d.reasoning_tokens) + .unwrap_or(0), + } +} + +/// Parses a raw `/v1/responses` JSON body into a [`ModelResponse`]. +/// +/// Reasoning items surface as a leading +/// [`ContentBlock::Thinking`] block, consistent with the Chat Completions path; +/// the encrypted payload, when present, rides on the block's `signature` so it +/// can be replayed on a later turn. +pub(super) fn parse_responses_response(value: Value) -> ModelResponse { + let parsed: ResponsesResponse = + serde_json::from_value(value.clone()).unwrap_or_else(|_| ResponsesResponse { + output: Vec::new(), + output_text: None, + usage: None, + }); + let text = extract_responses_text(&parsed).unwrap_or_default(); + let usage = parsed.usage.as_ref().map(convert_responses_usage); + + let mut content = Vec::new(); + if let Some(reasoning) = extract_responses_reasoning(&parsed) { + content.push(ContentBlock::Thinking { + text: reasoning, + signature: extract_encrypted_reasoning(&parsed), + }); + } + content.push(ContentBlock::Text(text)); + + ModelResponse { message: AssistantMessage { id: None, - content: vec![ContentBlock::Text(text)], - tool_calls, + content, + tool_calls: Vec::new(), usage, }, usage, - finish_reason: Some(if parsed.status.as_deref() == Some("incomplete") { - match parsed - .incomplete_details - .as_ref() - .and_then(|details| details.reason.as_deref()) - { - Some("max_output_tokens") => "length".to_string(), - Some(reason) => reason.to_string(), - None => "incomplete".to_string(), - } - } else if parsed - .output - .iter() - .any(|item| item.kind.as_deref() == Some("function_call")) - { - "tool_calls".to_string() - } else { - "stop".to_string() - }), + finish_reason: Some("stop".to_string()), raw: Some(value), resolved_model: None, - }) + continue_turn: None, + served_from_cache: false, + } } #[cfg(test)] @@ -483,23 +498,21 @@ mod tests { Message::assistant("hello"), Message::user(" "), // empty → skipped ]; - let (instructions, input) = build_responses_input(&messages).unwrap(); + let (instructions, input) = build_responses_input(&messages); assert_eq!(instructions.as_deref(), Some("be terse\n\nand correct")); assert_eq!(input.len(), 2); - assert_eq!(input[0].role.as_deref(), Some("user")); + assert_eq!(input[0].role, "user"); assert_eq!(input[0].content[0].kind, "input_text"); - assert_eq!(input[0].content[0].text.as_deref(), Some("hi")); + assert_eq!(input[0].content[0].text, "hi"); // Assistant items must use `output_text`, not `input_text`. - assert_eq!(input[1].role.as_deref(), Some("assistant")); + assert_eq!(input[1].role, "assistant"); assert_eq!(input[1].content[0].kind, "output_text"); - assert_eq!(input[1].content[0].text.as_deref(), Some("hello")); + assert_eq!(input[1].content[0].text, "hello"); } #[test] fn extract_text_prefers_output_text_then_scans_content() { let with_convenience = ResponsesResponse { - status: Some("completed".into()), - incomplete_details: None, output: Vec::new(), output_text: Some(" final ".to_string()), usage: None, @@ -510,10 +523,7 @@ mod tests { ); let via_content = ResponsesResponse { - status: Some("completed".into()), - incomplete_details: None, output: vec![ResponsesOutput { - kind: Some("message".into()), content: vec![ ResponsesContent { kind: Some("reasoning".into()), @@ -524,9 +534,7 @@ mod tests { text: Some("answer".into()), }, ], - call_id: None, - name: None, - arguments: None, + ..ResponsesOutput::default() }], output_text: None, usage: None, @@ -537,8 +545,6 @@ mod tests { ); let empty = ResponsesResponse { - status: Some("completed".into()), - incomplete_details: None, output: Vec::new(), output_text: None, usage: None, @@ -552,7 +558,7 @@ mod tests { "output_text": "the answer", "usage": { "input_tokens": 12, "output_tokens": 5 } }); - let resp = parse_responses_response(body).unwrap(); + let resp = parse_responses_response(body); assert_eq!(resp.text(), "the answer"); assert_eq!(resp.finish_reason.as_deref(), Some("stop")); let usage = resp.usage.expect("usage mapped"); @@ -563,97 +569,7 @@ mod tests { #[test] fn parse_tolerates_a_body_without_output() { - let resp = parse_responses_response(json!({ "id": "resp_1" })).unwrap(); + let resp = parse_responses_response(json!({ "id": "resp_1" })); assert_eq!(resp.text(), ""); } - - #[test] - fn parse_rejects_incompatible_output_shape() { - assert!(parse_responses_response(json!({"output": "not-an-array"})).is_err()); - } - - #[test] - fn build_input_preserves_user_images() { - use crate::message::{ImageRef, UserMessage}; - - let message = Message::User(UserMessage { - content: vec![ - ContentBlock::Text("inspect".into()), - ContentBlock::Image(ImageRef { - url: "https://example.test/image.png".into(), - mime_type: Some("image/png".into()), - }), - ], - }); - let (_, input) = build_responses_input(&[message]).unwrap(); - assert_eq!(input[0].content.len(), 2); - assert_eq!(input[0].content[1].kind, "input_image"); - assert_eq!( - input[0].content[1].image_url.as_deref(), - Some("https://example.test/image.png") - ); - } - - #[test] - fn responses_request_preserves_tools_and_choice() { - let tools = vec![ToolSchema::new( - "lookup", - "Look up a value", - json!({"type": "object"}), - )]; - assert_eq!(responses_tools(&tools)[0]["name"], "lookup"); - assert_eq!( - responses_tool_choice(&ToolChoice::Tool("lookup".into()), true).unwrap()["name"], - "lookup" - ); - } - - #[test] - fn build_input_correlates_function_calls_and_outputs() { - let messages = vec![ - Message::Assistant(AssistantMessage { - id: None, - content: Vec::new(), - tool_calls: vec![ToolCall::new("call-1", "lookup", json!({"query": "rust"}))], - usage: None, - }), - Message::Tool(crate::message::ToolMessage { - tool_call_id: "call-1".into(), - content: vec![ContentBlock::Json(json!({"answer": 42}))], - }), - ]; - let (_, input) = build_responses_input(&messages).unwrap(); - assert_eq!(input.len(), 2); - assert_eq!(input[0].kind.as_deref(), Some("function_call")); - assert_eq!(input[0].call_id.as_deref(), Some("call-1")); - assert_eq!(input[0].name.as_deref(), Some("lookup")); - assert_eq!(input[0].arguments.as_deref(), Some("{\"query\":\"rust\"}")); - assert_eq!(input[1].kind.as_deref(), Some("function_call_output")); - assert_eq!(input[1].call_id.as_deref(), Some("call-1")); - assert_eq!(input[1].output.as_deref(), Some("{\"answer\":42}")); - } - - #[test] - fn structured_formats_map_to_responses_text_configuration() { - let schema = json!({"type": "object"}); - let format = responses_text_format(Some(&crate::model::ResponseFormat::json_schema( - "answer", - schema.clone(), - ))) - .unwrap(); - assert_eq!(format["format"]["type"], "json_schema"); - assert_eq!(format["format"]["name"], "answer"); - assert_eq!(format["format"]["schema"], schema); - } - - #[test] - fn incomplete_response_preserves_length_finish_reason() { - let response = parse_responses_response(json!({ - "status": "incomplete", - "incomplete_details": {"reason": "max_output_tokens"}, - "output_text": "partial" - })) - .unwrap(); - assert_eq!(response.finish_reason.as_deref(), Some("length")); - } } diff --git a/crates/tinyinference/src/providers/openai/sse.rs b/crates/tinyinference/src/providers/openai/sse.rs index 6069292..b4bff20 100644 --- a/crates/tinyinference/src/providers/openai/sse.rs +++ b/crates/tinyinference/src/providers/openai/sse.rs @@ -138,6 +138,9 @@ impl OpenAiStreamAcc { } if let Some(args) = function.arguments.filter(|a| !a.is_empty()) { slot.args.push_str(&args); + if slot.id.is_empty() { + slot.id = tool_call_id(idx, ""); + } let call_id = tool_call_id(idx, &slot.id); pending.push_back(ModelStreamItem::ToolCallDelta(ToolDelta { call_id, @@ -274,6 +277,8 @@ impl OpenAiStreamAcc { finish_reason: self.finish_reason, raw: None, resolved_model: None, + continue_turn: None, + served_from_cache: false, } } } diff --git a/crates/tinyinference/src/providers/openai/test.rs b/crates/tinyinference/src/providers/openai/test.rs index 44678a3..921efeb 100644 --- a/crates/tinyinference/src/providers/openai/test.rs +++ b/crates/tinyinference/src/providers/openai/test.rs @@ -230,6 +230,8 @@ fn translates_structured_tool_result_content() { let request = ModelRequest::new(vec![Message::Tool(crate::message::ToolMessage { tool_call_id: "call-1".into(), content: vec![ContentBlock::Json(json!({"temperature": 21}))], + trusted_verbatim: false, + artifact: None, })]); let value = serde_json::to_value(model().translate_request(&request).unwrap()).unwrap(); assert_eq!( @@ -388,10 +390,11 @@ fn parses_id_less_tool_call_with_synthesized_fallback_id() { let response = parse_response(body).unwrap(); let calls = response.tool_calls(); assert_eq!(calls.len(), 2); - assert_eq!(calls[0].id, "tool-0"); + assert!(calls[0].id.starts_with("tacall-")); assert_eq!(calls[0].name, "ping"); assert_eq!(calls[0].arguments, json!({})); - assert_eq!(calls[1].id, "tool-1"); + assert!(calls[1].id.starts_with("tacall-")); + assert_ne!(calls[0].id, calls[1].id); assert_eq!(calls[1].name, "pong"); assert_eq!(calls[1].arguments, json!({ "n": 1 })); } @@ -666,6 +669,7 @@ fn provider_failed_stream_item_finishes_as_provider_error() { code: Some("rate_limit".to_string()), message: "too many requests".to_string(), retryable: true, + retry_after_ms: None, raw: None, })); @@ -2138,6 +2142,7 @@ fn degrade_for_400_targets_only_the_shape_the_request_used() { Some(Degrade { named_tool_choice: true, json_object: false, + ..Degrade::default() }) ); @@ -2152,6 +2157,7 @@ fn degrade_for_400_targets_only_the_shape_the_request_used() { Some(Degrade { named_tool_choice: false, json_object: true, + ..Degrade::default() }) ); } @@ -2186,6 +2192,7 @@ fn degrade_for_400_ignores_unrelated_or_already_degraded_failures() { Degrade { named_tool_choice: true, json_object: false, + ..Degrade::default() }, ), None @@ -2205,11 +2212,13 @@ fn degrade_for_400_unions_with_existing_baseline_degrade() { Degrade { named_tool_choice: true, json_object: false, + ..Degrade::default() }, ), Some(Degrade { named_tool_choice: true, json_object: true, + ..Degrade::default() }) ); } diff --git a/crates/tinyinference/src/providers/openai/transport.rs b/crates/tinyinference/src/providers/openai/transport.rs index b0012b7..492b55d 100644 --- a/crates/tinyinference/src/providers/openai/transport.rs +++ b/crates/tinyinference/src/providers/openai/transport.rs @@ -6,6 +6,7 @@ use super::responses; use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; /// How the provider expects the API credential to be sent on each request. /// @@ -106,6 +107,10 @@ pub struct OpenAiModel { /// inline `` reasoning, and unconditional extraction would silently /// strip legitimate content that mentions a literal `` tag. reasoning_tags_overridden: bool, + local_runtime: Option, + keep_alive: Option, + json_schema_strict: AtomicBool, + native_tools_on_wire: AtomicBool, } impl std::fmt::Debug for OpenAiModel { @@ -278,7 +283,12 @@ pub(super) fn merge_system_into_user(messages: &[Message]) -> Vec { /// reject `max_tokens` and require `max_completion_tokens` instead. pub(super) fn is_reasoning_model(model: &str) -> bool { let lower = model.to_ascii_lowercase(); - lower.starts_with("o1") || lower.starts_with("o3") || lower.starts_with("o4") + lower.starts_with("o1") + || lower.starts_with("o3") + || lower.starts_with("o4") + || lower.starts_with("gpt-5") + || lower.starts_with("gpt5") + || lower.contains("/gpt-5") } /// Derives a static [`ModelProfile`] for an OpenAI(-compatible) model id. @@ -311,6 +321,7 @@ pub(super) fn derive_profile(provider: &str, model: &str) -> ModelProfile { native_structured_output: native_structured, json_schema: true, reasoning, + reasoning_effort: reasoning, max_input_tokens: crate::model::context_window_for_model_id(model), ..ModelProfile::default() } @@ -349,6 +360,10 @@ impl OpenAiModel { // `` and must not strip literal mentions of the tag. reasoning_tags: Some(ReasoningTagExtraction::default()), reasoning_tags_overridden: false, + local_runtime: None, + keep_alive: None, + json_schema_strict: AtomicBool::new(true), + native_tools_on_wire: AtomicBool::new(true), } } @@ -603,6 +618,16 @@ impl OpenAiModel { "provider spec base_url must not be empty".to_string(), )); } + let kind = match spec.kind { + crate::providers::ProviderKind::Ollama => Some(LocalRuntimeKind::Ollama), + crate::providers::ProviderKind::LmStudio => Some(LocalRuntimeKind::LmStudio), + crate::providers::ProviderKind::LlamaCpp => Some(LocalRuntimeKind::LlamaCpp), + crate::providers::ProviderKind::Vllm => Some(LocalRuntimeKind::Vllm), + _ => None, + }; + if let Some(kind) = kind { + return Self::local_runtime(kind, &spec.provider, spec.base_url, api_key, spec.model); + } Ok(Self::compatible_provider( spec.provider, api_key, @@ -776,7 +801,177 @@ impl OpenAiModel { /// A local Ollama server (`http://localhost:11434/v1`), default model /// `llama3.2`. Ollama ignores the API key, so a placeholder is used. pub fn ollama() -> Self { - Self::compatible_provider("ollama", "ollama", "http://localhost:11434/v1", "llama3.2") + Self::ollama_at("http://localhost:11434", "llama3.2") + .expect("the built-in Ollama URL is valid") + } + + /// Constructs an Ollama model at a custom server root. + pub fn ollama_at(base_url: impl Into, model: impl Into) -> Result { + Self::local_runtime(LocalRuntimeKind::Ollama, "ollama", base_url, "", model) + } + + /// Constructs a llama.cpp model at a custom server root. + pub fn llama_cpp(base_url: impl Into, model: impl Into) -> Result { + Self::local_runtime(LocalRuntimeKind::LlamaCpp, "llama_cpp", base_url, "", model) + } + + /// Constructs a vLLM model at a custom server root. + pub fn vllm( + base_url: impl Into, + api_key: impl Into, + model: impl Into, + ) -> Result { + Self::local_runtime(LocalRuntimeKind::Vllm, "vllm", base_url, api_key, model) + } + + fn local_runtime( + kind: LocalRuntimeKind, + provider: &str, + base_url: impl Into, + api_key: impl Into, + model: impl Into, + ) -> Result { + let base_url = normalize_local_v1_base_url(base_url.into(), kind.default_root())?; + let mut output = Self::compatible_provider(provider, api_key, base_url, model) + .with_auth_style(AuthStyle::None) + .with_vision(false) + .with_named_tool_choice(false) + .with_json_object_format(false); + output.local_runtime = Some(kind); + output.json_schema_strict.store(false, Ordering::Relaxed); + output.profile.max_input_tokens = None; + Ok(output) + } + + /// Sets native Ollama context options and the advertised context window. + pub fn with_local_num_ctx(mut self, num_ctx: u32) -> Self { + self.default_provider_options = merge_provider_options( + &self.default_provider_options, + &json!({"options": {"num_ctx": num_ctx}}), + ); + self.profile.max_input_tokens = Some(u64::from(num_ctx)); + self + } + + /// Sets the native local-runtime model residency hint. + pub fn with_keep_alive(mut self, keep_alive: impl Into) -> Self { + self.keep_alive = Some(keep_alive.into()); + self + } + + /// Returns the configured local runtime kind. + pub fn local_runtime_kind(&self) -> Option { + self.local_runtime + } + + /// Probes a local runtime for its loaded model profile. + pub async fn probe_local_profile(&self) -> Result { + let Some(kind) = self.local_runtime else { + return Err(Error::Validation(format!( + "probe_local_profile is only meaningful for a local runtime; `{}` at {} is not one", + self.provider, self.base_url + ))); + }; + let root = kind.native_root(&self.base_url); + let (endpoint, builder) = match kind { + LocalRuntimeKind::Ollama => { + let endpoint = format!("{root}/api/show"); + let builder = self + .authorized(self.client.post(&endpoint)) + .json(&ollama_show_body(&self.model)); + (endpoint, builder) + } + LocalRuntimeKind::LmStudio => { + let endpoint = format!("{root}/api/v0/models"); + let builder = self.authorized(self.client.get(&endpoint)); + (endpoint, builder) + } + LocalRuntimeKind::LlamaCpp | LocalRuntimeKind::Vllm => return Ok(LocalProbe::default()), + }; + let response = builder + .timeout(PROBE_TIMEOUT) + .send() + .await + .map_err(|error| probe_error(&endpoint, error))?; + if !response.status().is_success() { + return Ok(LocalProbe::default()); + } + let body: Value = response + .json() + .await + .map_err(|error| probe_error(&endpoint, error))?; + Ok(match kind { + LocalRuntimeKind::Ollama => probe_from_ollama_show(&body), + LocalRuntimeKind::LmStudio => probe_from_lm_studio_models(&body, &self.model), + _ => LocalProbe::default(), + }) + } + + /// Applies a probed local profile. + pub fn apply_local_probe(mut self, probe: &LocalProbe) -> Self { + if let Some(window) = probe.effective_context_window() { + self.profile.max_input_tokens = Some(window); + } + if let Some(value) = probe.tool_calling { + self.profile.tool_calling = value; + self.profile.parallel_tool_calls = value; + self.profile.streaming_tool_chunks = value; + } + if let Some(value) = probe.vision { + self.profile.modalities.image_in = value; + } + if let Some(value) = probe.reasoning { + self.profile.reasoning = value; + } + self + } + + /// Probes and applies local capabilities. + pub async fn probed(self) -> Result { + let probe = self.probe_local_profile().await?; + Ok(self.apply_local_probe(&probe)) + } + + /// Validates that the configured model is advertised by the provider. + pub async fn validate_model(&self) -> Result<()> { + let listed = self.list_models().await?; + if listed.is_empty() || listed.iter().any(|entry| entry.id == self.model) { + return Ok(()); + } + let mut available: Vec<&str> = listed.iter().map(|entry| entry.id.as_str()).collect(); + available.sort_unstable(); + let remediation = if self.local_runtime == Some(LocalRuntimeKind::Ollama) { + format!(" Run `ollama pull {}` to install it.", self.model) + } else { + String::new() + }; + Err(Error::Validation(format!( + "{} at {} does not serve model `{}`.{} Available: {}", + self.provider, + self.base_url, + self.model, + remediation, + available.join(", ") + ))) + } + + /// Loads an Ollama model through its native warm-up API. + pub async fn warm_up(&self) -> Result<()> { + let Some(kind) = self.local_runtime.filter(|kind| kind.has_native_api()) else { + return Ok(()); + }; + let url = format!("{}/api/chat", kind.native_root(&self.base_url)); + let body = ollama_load_body( + &self.model, + local_options_object(&self.default_provider_options), + self.keep_alive.as_deref(), + ); + self.authorized(self.client.post(&url)) + .json(&body) + .send() + .await + .map_err(|error| Error::Model(format!("openai warm-up of {url} failed: {error}")))?; + Ok(()) } /// Returns the default model id this instance will request. @@ -801,6 +996,8 @@ impl OpenAiModel { Degrade { named_tool_choice: !self.named_tool_choice_supported, json_object: !self.json_object_format_supported, + json_schema_strict: !self.json_schema_strict.load(Ordering::Relaxed), + native_tools: !self.native_tools_on_wire.load(Ordering::Relaxed), } } @@ -830,7 +1027,7 @@ impl OpenAiModel { // handed tools gets the tool protocol embedded in its system prompt and no // native `tools` on the wire (many local runtimes 400 on `tools`). The // model's `` blocks are parsed back in [`Self::invoke`]/stream. - let prompt_guided_tools = !self.profile.tool_calling + let prompt_guided_tools = (!self.profile.tool_calling || degrade.native_tools) && !request.tools.is_empty() && request.tool_choice != ToolChoice::None; let prompt_tool_schemas = match &request.tool_choice { @@ -905,7 +1102,7 @@ impl OpenAiModel { Some(translate_tool_choice(&request.tool_choice)) }; - let response_format = request.response_format.as_ref().and_then(|format| { + let mut response_format = request.response_format.as_ref().and_then(|format| { if degrade.json_object && matches!(format, ResponseFormat::JsonObject) { // The endpoint rejects `{"type":"json_object"}`; use a permissive // `json_schema` that still guarantees a JSON object. @@ -914,6 +1111,15 @@ impl OpenAiModel { translate_response_format(format) } }); + if let Some(format) = response_format.as_mut() + && let Some(json_schema) = format.get_mut("json_schema") + { + let strict = !degrade.json_schema_strict && !degrade.json_object; + json_schema["strict"] = json!(strict); + if strict && let Some(schema) = json_schema.get_mut("schema") { + make_schema_strict(schema); + } + } let model = request.model.clone().unwrap_or_else(|| self.model.clone()); // The o-series reasoning models reject `max_tokens` and require @@ -932,9 +1138,22 @@ impl OpenAiModel { self.temperature_override, &self.temperature_unsupported, ); + let merged_provider_options = + merge_provider_options(&self.default_provider_options, &request.provider_options); + let reasoning_effort = if merged_provider_options + .get("reasoning_effort") + .is_some_and(|value| !value.is_null()) + { + None + } else { + request + .reasoning + .as_ref() + .and_then(|reasoning| reasoning.effort) + }; Ok(ChatCompletionRequest { - model, + model: model.clone(), messages, tools, tool_choice, @@ -943,14 +1162,12 @@ impl OpenAiModel { top_p: request.top_p, max_tokens, max_completion_tokens, + reasoning_effort, stop: request.stop_sequences.clone(), seed: request.seed, stream: false, stream_options: None, - extra: provider_extra_options(&merge_provider_options( - &self.default_provider_options, - &request.provider_options, - ))?, + extra: provider_extra_options(&merged_provider_options)?, }) } @@ -994,7 +1211,7 @@ impl OpenAiModel { request: &ModelRequest, ) -> Result { let model = request.model.clone().unwrap_or_else(|| self.model.clone()); - let (instructions, input) = responses::build_responses_input(&request.messages)?; + let (instructions, input) = responses::build_responses_input(&request.messages); let max_output_tokens = if self.responses_omit_max_output_tokens { None } else { @@ -1002,21 +1219,57 @@ impl OpenAiModel { }; let provider_options = merge_provider_options(&self.default_provider_options, &request.provider_options); + let extra = provider_extra_options(&provider_options)?; + let reasoning = if extra.contains_key("reasoning") { + None + } else { + request + .reasoning + .as_ref() + .and_then(responses::translate_reasoning) + }; + let include = if reasoning.is_some() { + vec![responses::INCLUDE_ENCRYPTED_REASONING.to_string()] + } else { + Vec::new() + }; + let tools: Vec = if self.native_tools_on_wire.load(Ordering::Relaxed) { + request + .tools + .iter() + .map(responses::translate_tool) + .collect() + } else { + Vec::new() + }; + let tool_choice = (!tools.is_empty()).then(|| translate_tool_choice(&request.tool_choice)); + let strict = self.json_schema_strict.load(Ordering::Relaxed); Ok(responses::ResponsesRequest { - model, + model: model.clone(), input, instructions, stream: None, store: Some(false), max_output_tokens, - tools: responses::responses_tools(&request.tools), - tool_choice: responses::responses_tool_choice( - &request.tool_choice, - !request.tools.is_empty(), - ), + tools, + tool_choice, previous_response_id: request.continuation_id.clone(), - text: responses::responses_text_format(request.response_format.as_ref()), - extra: responses::responses_extra_options(&provider_options)?, + text: request + .response_format + .as_ref() + .and_then(|format| responses::translate_text_format(format, strict)), + temperature: effective_temperature( + &model, + request.temperature, + self.temperature_override, + &self.temperature_unsupported, + ), + top_p: request.top_p, + seed: request.seed, + stop: request.stop_sequences.clone(), + reasoning, + include, + extra, }) } @@ -1049,7 +1302,7 @@ impl OpenAiModel { .await .map_err(|e| Error::Model(format!("openai responses body read failed: {e}")))?; let value: Value = serde_json::from_str(&text)?; - responses::parse_responses_response(value) + Ok(responses::parse_responses_response(value)) } /// Shared `POST {responses_url}` with auth, query params, and timeout, mapped @@ -1159,6 +1412,12 @@ impl OpenAiModel { Ok(response) => Ok(response), Err(Error::Provider(err)) if err.status == Some(400) => { if let Some(degrade) = degrade_for_400(&err.message, request, baseline) { + if degrade.native_tools { + self.native_tools_on_wire.store(false, Ordering::Relaxed); + } + if degrade.json_schema_strict { + self.json_schema_strict.store(false, Ordering::Relaxed); + } let retry = self.build_chat_body(request, degrade, streaming)?; self.post_json(&retry, request.timeout_ms, streaming, what) .await @@ -1189,6 +1448,7 @@ impl OpenAiModel { message, retryable, raw, + retry_after_ms: None, } } @@ -1210,6 +1470,17 @@ impl OpenAiModel { .and_then(|error| error.get("code").or_else(|| error.get("type"))) .and_then(Value::as_str) .map(str::to_string); + let code = if is_context_overflow(status, &message) { + Some(CONTEXT_OVERFLOW_CODE.to_string()) + } else { + code + }; + let message = self + .local_runtime + .and_then(|kind| { + missing_model_remediation(kind, status, &message, &self.model, &self.base_url) + }) + .unwrap_or(message); self.provider_error(message, Some(status), code, raw) } } @@ -1229,6 +1500,27 @@ pub(super) struct Degrade { /// Degrade `response_format:{"type":"json_object"}` to a permissive /// `json_schema`. pub json_object: bool, + pub json_schema_strict: bool, + pub native_tools: bool, +} + +fn make_schema_strict(schema: &mut Value) { + match schema { + Value::Object(object) => { + if object.get("type").and_then(Value::as_str) == Some("object") { + object.insert("additionalProperties".to_string(), Value::Bool(false)); + if let Some(properties) = object.get("properties").and_then(Value::as_object) { + let required = properties.keys().cloned().map(Value::String).collect(); + object.insert("required".to_string(), Value::Array(required)); + } + } + for value in object.values_mut() { + make_schema_strict(value); + } + } + Value::Array(values) => values.iter_mut().for_each(make_schema_strict), + _ => {} + } } /// Computes the additional degradation to apply after an HTTP 400, or `None` @@ -1261,6 +1553,21 @@ pub(super) fn degrade_for_400( { degrade.json_object = true; } + if !already.json_schema_strict + && lower.contains("strict") + && matches!( + request.response_format, + Some(ResponseFormat::JsonSchema { .. } | ResponseFormat::Auto { .. }) + ) + { + degrade.json_schema_strict = true; + } + if !already.native_tools + && !request.tools.is_empty() + && (lower.contains("does not support tools") || lower.contains("tools unsupported")) + { + degrade.native_tools = true; + } (degrade != already).then_some(degrade) } diff --git a/crates/tinyinference/src/providers/openai/types.rs b/crates/tinyinference/src/providers/openai/types.rs index f59a068..9205241 100644 --- a/crates/tinyinference/src/providers/openai/types.rs +++ b/crates/tinyinference/src/providers/openai/types.rs @@ -44,6 +44,9 @@ pub struct ChatCompletionRequest { /// reject `max_tokens`. Omitted when unset. #[serde(skip_serializing_if = "Option::is_none")] pub max_completion_tokens: Option, + /// Reasoning effort for OpenAI reasoning models. + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, /// Stop sequences that terminate generation. Omitted when empty. #[serde(skip_serializing_if = "Vec::is_empty")] pub stop: Vec, @@ -379,6 +382,9 @@ pub struct PromptTokensDetailsWire { /// Input tokens served from OpenAI's prompt cache. #[serde(default)] pub cached_tokens: u64, + /// Input tokens written to a provider prompt cache. + #[serde(default, alias = "cache_creation_tokens")] + pub cache_write_tokens: u64, } /// The `completion_tokens_details` breakdown of a [`UsageWire`]. diff --git a/crates/tinyinference/src/providers/types.rs b/crates/tinyinference/src/providers/types.rs index 8b5d53c..4bcd65e 100644 --- a/crates/tinyinference/src/providers/types.rs +++ b/crates/tinyinference/src/providers/types.rs @@ -28,6 +28,12 @@ pub enum ProviderKind { Anthropic, /// Local Ollama server exposing `/v1/chat/completions`. Ollama, + /// Local LM Studio server exposing `/v1/chat/completions`. + LmStudio, + /// Local llama.cpp server exposing `/v1/chat/completions`. + LlamaCpp, + /// Local vLLM server exposing `/v1/chat/completions`. + Vllm, /// DeepSeek OpenAI-compatible endpoint. DeepSeek, /// Groq OpenAI-compatible endpoint. @@ -51,6 +57,9 @@ impl ProviderKind { ProviderKind::OpenAi => "openai", ProviderKind::Anthropic => "anthropic", ProviderKind::Ollama => "ollama", + ProviderKind::LmStudio => "lmstudio", + ProviderKind::LlamaCpp => "llama_cpp", + ProviderKind::Vllm => "vllm", ProviderKind::DeepSeek => "deepseek", ProviderKind::Groq => "groq", ProviderKind::Xai => "xai", @@ -73,6 +82,11 @@ impl ProviderKind { "openai" => Some(ProviderKind::OpenAi), "anthropic" => Some(ProviderKind::Anthropic), "ollama" => Some(ProviderKind::Ollama), + "lmstudio" | "lm_studio" | "lm-studio" => Some(ProviderKind::LmStudio), + "llamacpp" | "llama_cpp" | "llama-cpp" | "llamaserver" => { + Some(ProviderKind::LlamaCpp) + } + "vllm" => Some(ProviderKind::Vllm), "deepseek" => Some(ProviderKind::DeepSeek), "groq" => Some(ProviderKind::Groq), "xai" => Some(ProviderKind::Xai), @@ -142,6 +156,9 @@ impl ProviderSpec { ProviderKind::Ollama => { Self::new(kind, "llama3.2", "http://localhost:11434/v1", None, false) } + ProviderKind::LmStudio => Self::new(kind, "", "http://localhost:1234/v1", None, false), + ProviderKind::LlamaCpp => Self::new(kind, "", "http://localhost:8080/v1", None, false), + ProviderKind::Vllm => Self::new(kind, "", "http://localhost:8000/v1", None, false), ProviderKind::DeepSeek => Self::new( kind, "deepseek-chat", diff --git a/crates/tinyinference/src/tool.rs b/crates/tinyinference/src/tool.rs index 246ea8c..f4f6e43 100644 --- a/crates/tinyinference/src/tool.rs +++ b/crates/tinyinference/src/tool.rs @@ -158,34 +158,40 @@ fn validate_schema_value(schema: &Value, value: &Value, path: &str) -> crate::Re validate_type_spec(type_spec, value, path)?; } if let Some(required) = schema.get("required").and_then(Value::as_array) { - let object = value.as_object().ok_or_else(|| { - crate::Error::Validation(format!("{path} must be an object with declared fields")) - })?; - for field in required.iter().filter_map(Value::as_str) { - if !object.contains_key(field) { - return Err(crate::Error::Validation(format!( - "{path}.{field} is required" - ))); + if let Some(object) = value.as_object() { + for field in required.iter().filter_map(Value::as_str) { + if !object.contains_key(field) { + return Err(crate::Error::Validation(format!( + "{path}.{field} is required" + ))); + } } + } else if schema.get("type").is_none() { + return Err(crate::Error::Validation(format!( + "{path} must be an object with declared fields" + ))); } } if let Some(properties) = schema.get("properties").and_then(Value::as_object) { - let object = value.as_object().ok_or_else(|| { - crate::Error::Validation(format!("{path} must be an object with declared fields")) - })?; - if schema.get("additionalProperties").and_then(Value::as_bool) == Some(false) { - for field in object.keys() { - if !properties.contains_key(field) { - return Err(crate::Error::Validation(format!( - "{path}.{field} is not allowed" - ))); + if let Some(object) = value.as_object() { + if schema.get("additionalProperties").and_then(Value::as_bool) == Some(false) { + for field in object.keys() { + if !properties.contains_key(field) { + return Err(crate::Error::Validation(format!( + "{path}.{field} is not allowed" + ))); + } } } - } - for (field, field_schema) in properties { - if let Some(field_value) = object.get(field) { - validate_schema_value(field_schema, field_value, &format!("{path}.{field}"))?; + for (field, field_schema) in properties { + if let Some(field_value) = object.get(field) { + validate_schema_value(field_schema, field_value, &format!("{path}.{field}"))?; + } } + } else if schema.get("type").is_none() { + return Err(crate::Error::Validation(format!( + "{path} must be an object with declared fields" + ))); } } if let Some(items_schema) = schema.get("items")