From 54969398c56f3e7411c8081fee01de3cda2a42b1 Mon Sep 17 00:00:00 2001 From: yh928 Date: Thu, 3 Sep 2026 07:39:47 +0900 Subject: [PATCH] feat(composio): tell the model what a toolkit's actions hand back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `toolkit_description` answers "what can this service do" — an input-side question, and so is everything else the model reads before calling: the tool catalogue, the parameter schema. Nothing tells it what comes *back*. So a list action returns records keyed by id, the model has no statement that the id is the handle for the detail it actually wanted, and it re-issues the same list call. Observed live against Gmail: a sub-agent searched with GMAIL_LIST_THREADS, got snippets rather than bodies, and reported that mail which does exist could not be found — the thread ids it needed were in the result it already had. `toolkit_result_notes` is the output-side counterpart, sitting beside `toolkit_description` because they answer the two halves of the same question. Two entries to start: gmail and slack, the toolkits whose response shape this crate's curated catalogues let us state rather than guess. Deliberately narrow: - Only action slugs `GMAIL_CURATED` / `SLACK_CURATED` actually carry. A note naming a renamed or dropped slug is worse than no note — it sends the model after a tool that is not in its list — so a test checks each toolkit's prose against its OWN catalogue. Pooling them would let a Gmail note name a Slack-only action and pass, which is the likeliest editing mistake. - No field-by-field record shapes. Composio dispatch prefers the backend's rendered `markdownFormatted` body and falls back to the JSON envelope only when that is absent, so a note reciting JSON keys is true on one of two renderings. An earlier revision made exactly that mistake. - `None` for anything unestablished. A guess about a response is worse than silence here, because the model acts on it. Tests live in a sibling `descriptions_tests.rs`, matching `mod_tests.rs`, so the crate's `expect_used` / `unwrap_used` lints stay on for real code. 307 tests pass; fmt and clippy clean. --- .../src/composio/catalogs/descriptions.rs | 58 +++++++++++++++++- .../composio/catalogs/descriptions_tests.rs | 61 +++++++++++++++++++ .../src/composio/catalogs/mod.rs | 2 +- crates/tinymemory-bus/src/composio/mod.rs | 3 +- 4 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 crates/tinymemory-bus/src/composio/catalogs/descriptions_tests.rs diff --git a/crates/tinymemory-bus/src/composio/catalogs/descriptions.rs b/crates/tinymemory-bus/src/composio/catalogs/descriptions.rs index 5f5fd856..9bda6ff2 100644 --- a/crates/tinymemory-bus/src/composio/catalogs/descriptions.rs +++ b/crates/tinymemory-bus/src/composio/catalogs/descriptions.rs @@ -1,4 +1,5 @@ -//! Human-readable capability summaries for Composio toolkit slugs. +//! Human-readable capability summaries for Composio toolkit slugs, plus what +//! the toolkit's actions hand back. /// Human-readable capability summary for a Composio toolkit slug. /// @@ -63,3 +64,58 @@ pub fn toolkit_description(slug: &str) -> &'static str { _ => "Interact with this connected service via its available actions", } } + +/// What a toolkit's actions hand back, and which field feeds which follow-up +/// action. `None` for a toolkit we have not established this for. +/// +/// [`toolkit_description`] answers "what can this service do", which is an +/// **input**-side question — and so is everything else the model reads before +/// calling: the tool catalogue, the parameter schema. Nothing tells it what +/// comes back. So a list action returns records keyed by id, the model has no +/// statement that the id is the handle for the detail it actually wanted, and it +/// re-issues the same list call. That was observed live against Gmail. +/// +/// The rule for adding an entry: name only action slugs this crate's curated +/// catalogues carry, and say only what a caller has established by observing +/// those actions. A toolkit nobody has checked gets no entry — a guess about a +/// response is worse here than silence, because the model will act on it. +/// +/// **Do not describe field-by-field record shapes here.** A note may say what a +/// result *contains* and what to do with it, not how it is serialized. Composio +/// dispatch prefers the backend's rendered `markdownFormatted` body and falls +/// back to the JSON envelope only when that is absent, so a note reciting JSON +/// keys is true on one of two renderings. An earlier revision of this text made +/// exactly that mistake and told the model every Gmail read action answers with +/// a markdown body, when only `GMAIL_FETCH_EMAILS` carries one. +pub fn toolkit_result_notes(slug: &str) -> Option<&'static str> { + match slug { + // Slugs: `gmail::GMAIL_CURATED`. + // + // The thread/message distinction is the whole point of this entry. Live, + // a sub-agent searched with GMAIL_LIST_THREADS, got no message body back, + // and reported that mail which does exist could not be found. + "gmail" => Some( + "GMAIL_LIST_THREADS answers with thread ids, a one-line snippet, and a message \ + count — never a message body, so a thread whose snippet looks right still has \ + to be read. Pass a thread id to GMAIL_FETCH_MESSAGE_BY_THREAD_ID, or a message \ + id to GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID, to get the body; GMAIL_FETCH_EMAILS \ + carries one already. Bodies are the backend's rendered text, not the raw \ + message, and attachments arrive as a filename and type that GMAIL_GET_ATTACHMENT \ + fetches. Repeating a search returns the same snippets, so read the thread \ + instead of searching again.", + ), + // Slugs: `messaging::SLACK_CURATED`. + "slack" => Some( + "SLACK_LIST_CONVERSATIONS answers with a channel id per channel, and that id is \ + the channel argument SLACK_FETCH_CONVERSATION_HISTORY and the post actions take. \ + History entries identify their author by Slack user id, not display name, so \ + resolve it with SLACK_FIND_USERS before quoting a name, and identify themselves \ + by a ts timestamp, which is what threads and reactions key on.", + ), + _ => None, + } +} + +#[cfg(test)] +#[path = "descriptions_tests.rs"] +mod tests; diff --git a/crates/tinymemory-bus/src/composio/catalogs/descriptions_tests.rs b/crates/tinymemory-bus/src/composio/catalogs/descriptions_tests.rs new file mode 100644 index 00000000..78c9bbb7 --- /dev/null +++ b/crates/tinymemory-bus/src/composio/catalogs/descriptions_tests.rs @@ -0,0 +1,61 @@ +//! Tests for the surrounding module. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use super::*; + +/// Every action slug these notes tell the model to call must be one the +/// toolkit actually exposes. A note naming a slug that was renamed or +/// dropped from the curated list is worse than no note: it sends the model +/// after a tool that is not in its list. +#[test] +fn result_notes_only_name_curated_action_slugs() { + // Each toolkit is checked against its OWN catalogue. Pooling them would + // let a Gmail note name a Slack-only action and still pass, which is the + // mistake most likely to be made when editing prose that mentions both. + let gmail: Vec<&str> = crate::composio::catalogs::gmail::GMAIL_CURATED + .iter() + .map(|tool| tool.slug) + .collect(); + let slack: Vec<&str> = crate::composio::catalogs::messaging::SLACK_CURATED + .iter() + .map(|tool| tool.slug) + .collect(); + + for (slug, curated) in [("gmail", &gmail), ("slack", &slack)] { + let notes = toolkit_result_notes(slug).expect("both toolkits have notes"); + for word in notes.split(|c: char| !(c.is_ascii_uppercase() || c == '_')) { + // An all-caps underscored token in this prose is an action slug. + if word.len() > 6 && word.contains('_') { + assert!( + curated.contains(&word), + "{slug} notes name `{word}`, which is not one of {slug}'s curated actions" + ); + } + } + } +} + +/// A toolkit nobody has established a result shape for gets no entry — a +/// guess about a response is worse here than silence. +#[test] +fn result_notes_absent_for_unestablished_toolkits() { + assert!(toolkit_result_notes("notion").is_none()); + assert!(toolkit_result_notes("definitely_not_a_toolkit").is_none()); +} + +/// The failure this entry exists for: a sub-agent searched threads, got +/// snippets rather than bodies, and reported that mail which does exist +/// could not be found. The note has to name both halves — that a thread +/// listing has no body, and which action produces one. +#[test] +fn gmail_notes_separate_finding_a_thread_from_reading_it() { + let notes = toolkit_result_notes("gmail").expect("gmail has notes"); + assert!( + notes.contains("GMAIL_LIST_THREADS") && notes.contains("never a message body"), + "must say a thread listing carries no body: {notes}" + ); + assert!( + notes.contains("GMAIL_FETCH_MESSAGE_BY_THREAD_ID"), + "must name the action that reads the thread: {notes}" + ); +} diff --git a/crates/tinymemory-bus/src/composio/catalogs/mod.rs b/crates/tinymemory-bus/src/composio/catalogs/mod.rs index 2010317b..92f79692 100644 --- a/crates/tinymemory-bus/src/composio/catalogs/mod.rs +++ b/crates/tinymemory-bus/src/composio/catalogs/mod.rs @@ -52,7 +52,7 @@ use super::scopes::{ classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope, UserScopePref, }; -pub use descriptions::toolkit_description; +pub use descriptions::{toolkit_description, toolkit_result_notes}; /// Every toolkit the capability surface reports on, in display order. pub const CAPABILITY_TOOLKITS: &[&str] = &[ diff --git a/crates/tinymemory-bus/src/composio/mod.rs b/crates/tinymemory-bus/src/composio/mod.rs index 84602f92..f3dfe448 100644 --- a/crates/tinymemory-bus/src/composio/mod.rs +++ b/crates/tinymemory-bus/src/composio/mod.rs @@ -76,5 +76,6 @@ pub use tasks::{GithubFetchMode, NormalizedTask, TaskContainer, TaskFetchFilter, pub use catalogs::{ catalog_for_toolkit, curated_scope_for, has_native_provider, is_action_visible_with_pref, native_provider_sync_interval_secs, parse_sync_interval_override, sync_interval_env_var, - toolkit_description, toolkit_has_scope, CAPABILITY_TOOLKITS, NATIVE_PROVIDERS, + toolkit_description, toolkit_has_scope, toolkit_result_notes, CAPABILITY_TOOLKITS, + NATIVE_PROVIDERS, };