From 75b17f89b86aeb5b1645d99473b9eb98cd1c1080 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 19:57:14 +0300 Subject: [PATCH] Carry runtime model selection metadata Co-authored-by: Medulla --- crates/tinyinference/src/model/mod.rs | 20 +++++++ crates/tinyinference/src/model/test.rs | 18 ++++++- crates/tinyinference/src/model/types.rs | 53 +++++++++++++++++++ crates/tinyinference/src/providers/mock.rs | 2 + .../src/providers/openai/convert.rs | 1 + .../src/providers/openai/responses.rs | 1 + .../tinyinference/src/providers/openai/sse.rs | 1 + 7 files changed, 95 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference/src/model/mod.rs b/crates/tinyinference/src/model/mod.rs index 953526d..4572243 100644 --- a/crates/tinyinference/src/model/mod.rs +++ b/crates/tinyinference/src/model/mod.rs @@ -253,6 +253,18 @@ impl ModelRequest { self } + /// Adds an uninterpreted runtime model-selection hint. + pub fn with_model_hint(mut self, hint: ModelHint) -> Self { + self.model_hints.push(hint); + self + } + + /// Sets whether a consuming runtime may reuse its previous model. + pub fn with_reuse_previous_model(mut self, reuse: bool) -> Self { + self.reuse_previous_model = reuse; + self + } + /// Sets the sampling temperature. pub fn with_temperature(mut self, temperature: f64) -> Self { self.temperature = Some(temperature); @@ -368,6 +380,7 @@ impl ModelResponse { usage: None, finish_reason: None, raw: None, + resolved_model: None, } } @@ -384,6 +397,12 @@ impl ModelResponse { self } + /// Attaches durable selection metadata supplied by a consuming runtime. + pub fn with_resolved_model(mut self, resolved: ResolvedModel) -> Self { + self.resolved_model = Some(resolved); + self + } + /// Returns the tool calls requested by the model, if any. pub fn tool_calls(&self) -> &[ToolCall] { &self.message.tool_calls @@ -589,6 +608,7 @@ impl StreamAccumulator { usage: self.usage, finish_reason: None, raw: None, + resolved_model: None, }) } } diff --git a/crates/tinyinference/src/model/test.rs b/crates/tinyinference/src/model/test.rs index 4abb11f..9ad83da 100644 --- a/crates/tinyinference/src/model/test.rs +++ b/crates/tinyinference/src/model/test.rs @@ -12,6 +12,12 @@ use serde_json::json; fn request_builder_sets_fields() { let req = ModelRequest::new(vec![Message::user("hi")]) .with_model("gpt") + .with_model_hint(ModelHint { + model: "fast".into(), + priority: 10, + reason: Some("latency".into()), + }) + .with_reuse_previous_model(true) .with_temperature(0.5) .with_top_p(0.9) .with_max_tokens(128) @@ -29,6 +35,8 @@ fn request_builder_sets_fields() { assert_eq!(req.timeout_ms, Some(1000)); assert_eq!(req.tool_choice, ToolChoice::Required); assert_eq!(req.tags, vec!["t".to_string()]); + assert_eq!(req.model_hints[0].model, "fast"); + assert!(req.reuse_previous_model); } #[test] @@ -252,10 +260,18 @@ fn model_request_capability_and_provider_option_builders() { #[test] fn response_helpers() { - let resp = ModelResponse::assistant("hi").with_finish_reason("stop"); + let resolved = ResolvedModel { + name: "fast".into(), + requested: Some("fast".into()), + source: ModelResolutionSource::Hint, + }; + let resp = ModelResponse::assistant("hi") + .with_finish_reason("stop") + .with_resolved_model(resolved.clone()); assert_eq!(resp.text(), "hi"); assert!(resp.tool_calls().is_empty()); assert_eq!(resp.finish_reason.as_deref(), Some("stop")); + assert_eq!(resp.resolved_model, Some(resolved)); } #[test] diff --git a/crates/tinyinference/src/model/types.rs b/crates/tinyinference/src/model/types.rs index c3b69c2..c268472 100644 --- a/crates/tinyinference/src/model/types.rs +++ b/crates/tinyinference/src/model/types.rs @@ -260,6 +260,50 @@ pub struct PromptSegment { pub cacheable: bool, } +/// Runtime-supplied model candidate metadata. +/// +/// TinyInference carries this serializable value without registering, ranking, +/// or resolving models; consuming runtimes own those policies. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ModelHint { + /// Runtime registry name or provider model id. + pub model: String, + /// Higher values indicate stronger runtime preference. + #[serde(default)] + pub priority: i32, + /// Optional runtime explanation for observability. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// Runtime-owned source that selected a model. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ModelResolutionSource { + /// Explicit request-level override. + RequestOverride, + /// Reused from durable runtime state. + StateReuse, + /// Chosen from runtime hints. + Hint, + /// Default declared by an agent. + AgentDefault, + /// Default declared by a consuming registry. + RegistryDefault, +} + +/// Durable metadata describing a runtime-selected model. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedModel { + /// Runtime registry name or provider model id. + pub name: String, + /// Originally requested name, when different. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested: Option, + /// Runtime selection source. + pub source: ModelResolutionSource, +} + /// A provider-neutral chat model request. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ModelRequest { @@ -277,6 +321,12 @@ pub struct ModelRequest { /// Model id or registry alias override. #[serde(default, skip_serializing_if = "Option::is_none")] pub model: Option, + /// Ordered runtime model-selection hints carried without interpretation. + #[serde(default)] + pub model_hints: Vec, + /// Whether a consuming runtime may reuse its prior selected model. + #[serde(default)] + pub reuse_previous_model: bool, /// Sampling temperature. #[serde(default, skip_serializing_if = "Option::is_none")] pub temperature: Option, @@ -338,6 +388,9 @@ pub struct ModelResponse { /// Raw provider metadata preserved for callers who need it. #[serde(default, skip_serializing_if = "Option::is_none")] pub raw: Option, + /// Durable model-selection metadata attached by a consuming runtime. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolved_model: Option, } /// An incremental streamed chunk of a model response. diff --git a/crates/tinyinference/src/providers/mock.rs b/crates/tinyinference/src/providers/mock.rs index 38e9380..b47a584 100644 --- a/crates/tinyinference/src/providers/mock.rs +++ b/crates/tinyinference/src/providers/mock.rs @@ -245,6 +245,7 @@ impl ChatModel for MockModel { usage: Some(usage), finish_reason: Some("tool_calls".to_string()), raw: None, + resolved_model: None, } } @@ -344,6 +345,7 @@ impl MockModel { usage: Some(Usage::new(10, output_tokens)), finish_reason: Some("stop".to_string()), raw: None, + resolved_model: None, } } } diff --git a/crates/tinyinference/src/providers/openai/convert.rs b/crates/tinyinference/src/providers/openai/convert.rs index 844093a..04d5268 100644 --- a/crates/tinyinference/src/providers/openai/convert.rs +++ b/crates/tinyinference/src/providers/openai/convert.rs @@ -323,6 +323,7 @@ pub(super) fn parse_chat_response( usage, finish_reason: choice.finish_reason, raw: Some(value), + resolved_model: None, }) } diff --git a/crates/tinyinference/src/providers/openai/responses.rs b/crates/tinyinference/src/providers/openai/responses.rs index 4efaed8..ed39cab 100644 --- a/crates/tinyinference/src/providers/openai/responses.rs +++ b/crates/tinyinference/src/providers/openai/responses.rs @@ -464,6 +464,7 @@ pub(super) fn parse_responses_response(value: Value) -> Result { "stop".to_string() }), raw: Some(value), + resolved_model: None, }) } diff --git a/crates/tinyinference/src/providers/openai/sse.rs b/crates/tinyinference/src/providers/openai/sse.rs index 921601c..6069292 100644 --- a/crates/tinyinference/src/providers/openai/sse.rs +++ b/crates/tinyinference/src/providers/openai/sse.rs @@ -273,6 +273,7 @@ impl OpenAiStreamAcc { usage: self.usage, finish_reason: self.finish_reason, raw: None, + resolved_model: None, } } }