From 58720a574b98a3ca8ba42147973e47645d885ef8 Mon Sep 17 00:00:00 2001 From: Gregor <222116341+gregor-transluce@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:12:33 -0700 Subject: [PATCH] Publish Docent plugin 0.2.1 --- .claude-plugin/marketplace.json | 2 +- plugins/docent/.claude-plugin/plugin.json | 2 +- plugins/docent/.mcp.json | 2 +- plugins/docent/commands/logging.md | 43 ++++++ plugins/docent/hooks/hooks.json | 15 ++ plugins/docent/hooks/session_end.py | 115 ++++++++++++++ plugins/docent/skills/docent/SKILL.md | 14 ++ plugins/docent/skills/docent/analysis.md | 26 +++- plugins/docent/skills/docent/dql-reference.md | 140 +++++++++++++++++- .../skills/docent/ingestion-reference.md | 2 +- plugins/docent/skills/docent/ingestion.md | 25 +++- .../skills/docent/readings-reference.md | 76 ++++++++-- .../docent/skills/docent/rubric-writing.md | 33 +++++ 13 files changed, 463 insertions(+), 32 deletions(-) create mode 100644 plugins/docent/commands/logging.md create mode 100644 plugins/docent/hooks/hooks.json create mode 100644 plugins/docent/hooks/session_end.py create mode 100644 plugins/docent/skills/docent/rubric-writing.md diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5574db6..0f0c29d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "name": "docent", "source": "./plugins/docent", "description": "Docent AI analysis tools for Claude Code", - "version": "0.1.10", + "version": "0.2.1", "author": { "name": "TransluceAI" }, diff --git a/plugins/docent/.claude-plugin/plugin.json b/plugins/docent/.claude-plugin/plugin.json index 9283817..85a7b0e 100644 --- a/plugins/docent/.claude-plugin/plugin.json +++ b/plugins/docent/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { "name": "docent", - "version": "0.1.10", + "version": "0.2.1", "description": "Docent AI analysis tools" } diff --git a/plugins/docent/.mcp.json b/plugins/docent/.mcp.json index d727d77..c3e0df4 100644 --- a/plugins/docent/.mcp.json +++ b/plugins/docent/.mcp.json @@ -3,7 +3,7 @@ "docent": { "type": "stdio", "command": "uv", - "args": ["tool", "run", "--from", "docent-python>=0.1.74", "docent-mcp"] + "args": ["tool", "run", "--from", "docent-python>=0.1.82", "docent-mcp"] } } } diff --git a/plugins/docent/commands/logging.md b/plugins/docent/commands/logging.md new file mode 100644 index 0000000..c109b1e --- /dev/null +++ b/plugins/docent/commands/logging.md @@ -0,0 +1,43 @@ +--- +description: Configure Docent session logging — within the enabled rollout, sessions where Docent tools are used are shared by default; opt out, opt back in, or delete uploaded data here. +disable-model-invocation: true +allowed-tools: Bash(uv tool run --quiet --from 'docent-python>=0.1.82' python:*) +--- + +You are helping the user configure Docent session logging. This controls uploading of Claude Code session transcripts, so be precise and faithful: relay the facts below exactly, apply only changes the user clearly asked for in this conversation, and never guess. + +## 1. Show the current state + +Run the status command and present its output: + +``` +uv tool run --quiet --from 'docent-python>=0.1.82' python -m docent.plugin.logging_config status +``` + +## 2. Make sure the user knows what session sharing means + +Before changing anything, relay these facts (a faithful paraphrase is fine, but keep them exact): + +- Sessions in which a Docent MCP tool or skill was actually invoked are uploaded to **Transluce's Docent prod servers** to help improve Docent. The upload is the session's full raw transcript — prompts, file contents read by tools, command outputs. +- Sessions that never touch Docent are **not** uploaded. Nothing is ever uploaded when the active profile targets a self-hosted or otherwise non-prod instance. +- Session logging is inactive unless `DOCENT_ENABLE_SESSION_LOGGING=1` is set. Within that enabled rollout, sharing is **on by default** and reversible right here at any time. Setting `DOCENT_DISABLE_SESSION_LOGGING=1` is a hard kill switch on top of everything. +- Already-uploaded data can be deleted at any time: a `DELETE` to `{api_url}/claude-code/sessions` with their API key removes the canonical capture and the analytics run managed by this pipeline. Offer to run this if they ask for deletion. + +## 3. Ask what they want + +Ask which they'd like: opt out, opt back in, or delete already-uploaded data. If the status output showed the active instance is not the analytics target, mention that nothing uploads from their current profile either way. + +## 4. Apply their choice + +Use exactly one CLI invocation per choice (all via `uv tool run --quiet --from 'docent-python>=0.1.82' python -m docent.plugin.logging_config ...`): + +- `opt-out` — stop uploading sessions +- `opt-in` — resume uploading Docent-using sessions + +For deletion of already-uploaded data, send the `DELETE` request with their API key and report the response counts. + +If the user reports that their sessions are not being uploaded or not appearing, run `doctor` and relay its full report — it checks every gate of the pipeline (binaries, state, connection, server, retry queue) and marks problems with `!!`. + +## 5. Confirm + +Re-run the `status` command and show the result so the user sees exactly what is now enabled. diff --git a/plugins/docent/hooks/hooks.json b/plugins/docent/hooks/hooks.json new file mode 100644 index 0000000..353e5b6 --- /dev/null +++ b/plugins/docent/hooks/hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "SessionEnd": [ + { + "hooks": [ + { + "type": "command", + "command": "[ \"$DOCENT_ENABLE_SESSION_LOGGING\" = \"1\" ] && [ -z \"$DOCENT_DISABLE_SESSION_LOGGING\" ] && command -v uv >/dev/null 2>&1 && command -v python3 >/dev/null 2>&1 && python3 \"${CLAUDE_PLUGIN_ROOT}/hooks/session_end.py\" 2>/dev/null || true", + "timeout": 10 + } + ] + } + ] + } +} diff --git a/plugins/docent/hooks/session_end.py b/plugins/docent/hooks/session_end.py new file mode 100644 index 0000000..0b2cebd --- /dev/null +++ b/plugins/docent/hooks/session_end.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""SessionEnd hook: hand the session to a detached uploader, in milliseconds. + +Claude Code runs SessionEnd hooks synchronously and may kill them almost +immediately when the CLI exits, so anything slow here either stalls the +user's exit or silently dies mid-upload. This script therefore does only +millisecond-scale work with the system python3 and no third-party imports: +re-check the opt-out gates, then spawn the real uploader as a fully detached +process (its own session on POSIX, detached process group on Windows, all +stdio on /dev/null) and exit. The hook returns before any package resolution +or network happens, and the detached uploader survives both the CLI exiting +and the terminal closing. + +SessionEnd hook output is ignored but stderr is shown to the user, so every +path must stay silent and exit cleanly. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + +UPLOADER_REQUIREMENT = "docent-python>=0.1.82" + + +def _spawn_detached(args: list[str]) -> None: + """Start `args` so it is immune to the hook's death. + + A new session (POSIX) or detached process group (Windows) keeps the + uploader out of the hook's process group, so killing the hook — which + Claude Code does on exit — cannot take the upload down with it. All + stdio on devnull so no inherited pipe keeps Claude Code waiting on us. + """ + if os.name == "nt": + flags = getattr(subprocess, "DETACHED_PROCESS", 0x00000008) | getattr( + subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200 + ) + subprocess.Popen( + args, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + creationflags=flags, + ) + else: + subprocess.Popen( + args, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + start_new_session=True, + ) + + +def main() -> None: + try: + payload = json.loads(sys.stdin.read() or "{}") + except Exception: + return + if not isinstance(payload, dict): + return + if os.environ.get("DOCENT_ENABLE_SESSION_LOGGING") != "1" or os.environ.get( + "DOCENT_DISABLE_SESSION_LOGGING" + ): + return + if not isinstance(payload.get("session_id"), str) or not isinstance( + payload.get("transcript_path"), str + ): + return + + home = os.environ.get("HOME") or os.environ.get("USERPROFILE") + if not home: + return + state_path = os.path.join(home, ".docent", "claude-code-logging.json") + try: + with open(state_path, encoding="utf-8") as f: + state = json.load(f) + except FileNotFoundError: + state = {} + except Exception: + return + if not isinstance(state, dict): + return + analytics = state.get("analytics") + choice = analytics.get("choice") if isinstance(analytics, dict) else None + if choice == "no": + return + + plugin_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + _spawn_detached( + [ + "uv", + "tool", + "run", + "--quiet", + "--from", + UPLOADER_REQUIREMENT, + "python", + "-m", + "docent.plugin.session_upload", + json.dumps(payload), + plugin_root, + ] + ) + + +if __name__ == "__main__": + try: + main() + except Exception: + pass diff --git a/plugins/docent/skills/docent/SKILL.md b/plugins/docent/skills/docent/SKILL.md index 8134e2d..b9992d7 100644 --- a/plugins/docent/skills/docent/SKILL.md +++ b/plugins/docent/skills/docent/SKILL.md @@ -6,6 +6,8 @@ alwaysApply: true # Docent +## Guides + This is the root skill for all Docent work. This file is just a table of contents. In most cases you should read one of the guides below before starting to work with docent. Choose the guide that best matches your task. - For exploring a collection of agent runs, analyzing data, answering questions about agent behavior: `./analysis.md` @@ -14,8 +16,20 @@ This is the root skill for all Docent work. This file is just a table of content ## Other available documentation +- For analysis-plan markdown notes (universal framework + pattern index): `./readings-reference.md` (`client.plan_markdown`) +- For writing or revising rubrics, classifier prompts, and their output schemas: `./rubric-writing.md` +- For plan-pattern pipelines and note templates (read one after classifying at Step 2b): `./patterns/` - For the Readings API (`client.read`, `client.query`, batching, prompts, clustering): `./readings-reference.md` - For DQL syntax, schemas, quirks, and example queries: `./dql-reference.md` - For the reports API: `./report.md` (only if the user explicitly asks for a report) - For ingestion-side data-model and conversion examples: `./ingestion-reference.md` - SDK reference is available by visiting [our online documentation](https://docs.transluce.org/llms.txt) + +## Opening Docent pages + +Get the user in front of the relevant Docent page as soon as it exists — a new collection or a freshly submitted analysis plan. + +- In local sessions running on the user's machine (e.g. Claude Code CLI or an IDE extension), the SDK's `flush()` / `webbrowser.open()` opens the user's default browser automatically. You can rely on this, but still surface the URL as a clickable link since the user may not notice the tab. +- In sandboxed sessions where `webbrowser.open()` cannot reach the user's browser (e.g. Codex CLI), surface the URL as a clickable link instead. + +Failure to open a browser is not a Docent workflow failure. diff --git a/plugins/docent/skills/docent/analysis.md b/plugins/docent/skills/docent/analysis.md index 50452e0..bef1363 100644 --- a/plugins/docent/skills/docent/analysis.md +++ b/plugins/docent/skills/docent/analysis.md @@ -35,7 +35,7 @@ These apply throughout the entire analysis session: - "multi-approval flow" → "this analysis has two phases — you'll need to approve each one" - "metadata fields" / "metadata_json" → describe the actual content ("model name", "safety scores") - The deeper rule: **describe what you're investigating, not what tools you're running.** Most jargon leaks happen when you narrate your own tool use to the user. The user doesn't need to know which system you're querying or what the query language is called — they need to know what you're learning about their data. + The deeper rule: **describe what you're investigating, not what tools you're running.** Most jargon leaks happen when you narrate your own tool use to the user. The user doesn't need to know which system you're querying or what the query language is called — they need to know what you're learning about their data. "What you're investigating" means the concrete object-level goal of the system under study — not an abstracted or sanitized restatement of it. Dropping Docent jargon is necessary but not sufficient: a description can be jargon-free and still fail by hiding the real subject. Bad — narrates tool use: > "Let me run DQL queries to aggregate the metadata fields and check the score distributions across models." @@ -71,6 +71,8 @@ If you're not sure what collection the user is talking about: * Otherwise, check the SDK-discovered `docent.env` file for `DOCENT_COLLECTION_ID`. * If neither is available, ask the user to paste the collection UUID. +Once you know the collection ID and before running any analysis, call `client.ensure_analyzable_collection(collection_id)` and use the returned ID for the rest of the session. If the collection is public and not the user's own (e.g., a shared sample collection like Terminal-Bench), this clones it so the user's analysis lands in their own copy — which appears on their dashboard — instead of being written into the shared collection. If the collection is already the user's, it returns the same ID immediately. When a clone happens, tell the user and link them to the new collection's dashboard URL. + The main Docent deployment lives at https://docent.transluce.org but the user may connect a different deployment by overriding DOCENT_FRONTEND_URL in docent.env. The Docent SDK will print out the frontend URL when it is initialized, e.g. `Authenticating Docent client with frontend_url='https://docent.transluce.org'`. If you see a different frontend URL, use that URL in place of `https://docent.transluce.org` for any links. ## Troubleshooting @@ -113,12 +115,16 @@ A reading is a structured batch of LLM calls. Readings are useful for qualitativ An analysis script is a Python script you write using the Docent SDK. An analysis script can perform DQL queries (client.query) and readings (client.read). -When you run an analysis script, an analysis plan is displayed in the Docent UI. Each query and reading in the script is displayed as a separate card in the analysis plan. Readings require approval from the user before they are run. Results for both step types (DQL and reading) are displayed in interactive tables. +When you run an analysis script, an analysis plan is displayed in the Docent UI. Each query and reading in the script is displayed as a separate card in the analysis plan. Reading approval follows the user's account preference unless they explicitly ask you to override it for that plan. Results for both step types (DQL and reading) are displayed in interactive tables. + +Every analysis script must include a `client.plan_markdown` note before any `client.query` or `client.read` call. Write the top note from the user's analysis question and the analytical framing you designed in Step 2b. Follow the universal note framework and pattern index in `./readings-reference.md` (`client.plan_markdown`). Once you have a question where qualitative analysis is clearly required, you can go ahead and create + run an analysis script with readings. If you need the user to clarify or refine the question, do that before writing the script. Note: the Docent UI is the primary place to view reading results. You do not need to fetch them, read them, and restate them to the user. If a summary or synthesis would be helpful, perform that as another reading in the same analysis script so it will show up in the UI. If a structured aggregation of reading results would be helpful, perform that as another DQL query in the same analysis script. +When writing or revising any rubric, rubric-like classifier prompt, or output schema for a reading, first read `./rubric-writing.md` and apply it. Rubrics should have a concise high-level framing, a natural-language decision tree that another judge can follow consistently, and an output schema whose values are explicitly mapped to the decision procedure. + # Example workflow This section describes the end-to-end process for a Docent analysis session. @@ -133,6 +139,8 @@ If the user provided a dashboard URL, use `Docent.from_url()` in all scripts thr Use the `get_metadata_fields` MCP tool to understand the structure of agent run metadata for the current collection. Agent runs contain metadata that varies by collection — do not make assumptions about its structure. +Before writing any DQL by hand, call the `get_dql_schema` MCP tool, or `client.get_dql_schema(collection_id)` when working in a Python script, for the live table/column schema (names, types, nullability, rubric output fields). Guessed column names are the most common DQL failure; the schema call is sub-second and eliminates them. + Also call `list_reading_presets` to check if the collection has any saved reading presets. These can be reused and are worth knowing about before proposing analysis directions. **Immediately after these calls return, tell the user what you see** in 2-3 sentences: what kind of data is in this collection, what the key dimensions are (e.g., models, tasks, environments), and what scores or metrics are available. This is the user's first orientation to the dataset — don't skip it, and don't jump straight into writing queries. @@ -279,7 +287,17 @@ Before coding, briefly describe the analytical framing — not the pipeline step Consult `./readings-reference.md` for the Readings API, coding tips, and example patterns (especially the clustering example). Consult `./dql-reference.md` for DQL syntax, table schemas, and quirks. -Write a Python script implementing the pipeline you designed in Step 2b. Keep the script clean. Do not put exploratory queries in the analysis script — those belong in Step 1 orientation. However, you may add DQL queries to the script to present key findings (e.g. if an important reading outputs categories, you could count the frequency of each category). Do this sparingly, only when it will help the user understand the findings beyond seeing a table of reading results. +If the script includes a rubric or structured LLM classifier, read `./rubric-writing.md` before drafting it. Apply the rubric refinement rules there: keep the decision procedure concise and unambiguous, calibrate detail to the available evidence, explicitly map output values to decisions, and keep the output schema separate from the rubric text. + +Write a Python script implementing the pipeline you designed in Step 2b. Keep the script clean. + +You **must** start every script with `client.plan_markdown(...)`. Use two `##` section headings for the markdown note. Typically, for most analysis plans you can use `Behavior` and `Measurement` for the two headings, but feel free to improvise if you believe a different heading is more suitable, or add more headings if necessary (do this sparingly). For example, if the analysis involves rubric-refinement, use `## Current rubric` instead of `## Measurement`, and describe the full rubric instead of using a summary. + +Write the markdown note by referencing the user's question and your Step 2b framing before you add `client.query` or `client.read` calls. Use simple language, but keep it concrete and faithful to the subject matter. + +Write the markdown note in a narrative style. For example, don't say: "How severe BAD_BEHAVIOR is in flagged conversations". Say: "We investigate how severe BAD_BEHAVIOR is in flagged conversations. We do this by XYZ". + +You may add DQL queries to the script to present key findings (e.g. if an important reading outputs categories, you could count the frequency of each category). Do this sparingly, only when it will help the user understand the findings beyond seeing a table of reading results. If you feel the urge to write substantial Python logic (clustering, scoring, statistical tests), go back to the **translation table in Step 2b** and express the work as LLM analyses and DQL aggregations instead. @@ -302,7 +320,7 @@ Do not write a script covering all phases at once. A monolithic script that fail Analysis plans appear in a web UI for the user to approve — this is a key control affordance. You are responsible for running analysis scripts when appropriate; the user should not have to do so manually. Prefer to run analysis scripts in the background, so that you can still communicate with the user if the script pauses to wait for approval. -**Surface the Docent UI link as soon as the analysis is submitted** — don't wait until results come back. The SDK's `flush()` opens a browser tab, but the user may not notice or may lose it among other tabs. Always tell the user explicitly: "The analysis is running — you can follow along and approve it here: [link]." This is especially important because the link is how the user inspects the evidence behind every finding. +**Surface the Docent UI link as soon as the analysis is submitted** — don't wait until results come back. The SDK's `flush()` opens a browser tab in local sessions, but the user may not notice it or may lose it among other tabs, so always surface the URL yourself. In sandboxed sessions where `webbrowser.open()` can't reach the user's browser (e.g. Codex CLI), provide a clickable link instead. Always tell the user explicitly: "The analysis is running — you can follow along and approve it here: [link]." This is especially important because the link is how the user inspects the evidence behind every finding. **Be explicit about partial data.** When `get_reading_plan_results` returns truncated output (e.g., 50 of 132 results visible), state the exact fraction you saw and caveat derived numbers. Prefer using query aggregation over `reading_results.output` to get complete counts rather than parsing truncated tool output. For example, to get the full distribution of a structured output field across all results, query `reading_results` directly: diff --git a/plugins/docent/skills/docent/dql-reference.md b/plugins/docent/skills/docent/dql-reference.md index 27e196f..e9a3dca 100644 --- a/plugins/docent/skills/docent/dql-reference.md +++ b/plugins/docent/skills/docent/dql-reference.md @@ -38,6 +38,8 @@ raw_rows = client.dql_result_to_dicts(result) ## Available Tables and Columns +The authoritative schema is the live one: call the `get_dql_schema` MCP tool (or `client.get_dql_schema(collection_id)` in scripts) to get every queryable table with column types and nullability. The tables below are a reference snapshot and may lag the deployment. + | Table | Description | | --- | --- | | `agent_runs` | Information about each agent run in a collection. | @@ -47,6 +49,10 @@ raw_rows = client.dql_result_to_dicts(result) | `readings` | Reading definitions (template or scripted LLM analysis). | | `reading_results` | Results from running readings. | | `reading_result_links` | Junction table linking readings to their results. | +| `reading_plans` | Analysis plans (name + ordered steps) that orchestrate readings. | +| `reading_presets` | Named reading configurations; the human-readable reading name lives here. | +| `reading_preset_versions` | Immutable per-version reading configuration for a preset. | +| `collections` | The current collection's own row (name, description, metadata). | ### `agent_runs` @@ -114,7 +120,8 @@ raw_rows = client.dql_result_to_dicts(result) | `output_schema` | JSON schema for output validation. | | `max_new_tokens` | Maximum number of new tokens generated per LLM call. | | `num_rollouts` | Number of independent LLM samples generated per input row (>= 1). | -| `source_reading_preset_id` | Optional associated preset. | +| `source_reading_preset_id` | Optional associated preset. Join to `reading_presets.id` to get the reading's name. | +| `source_reading_preset_version` | Version of the associated preset; matches `reading_preset_versions.version_index`. | | `created_at` | When the reading was created. | ### `reading_results` @@ -155,6 +162,60 @@ For scripted readings, `arguments_dict` holds arbitrary user-supplied metadata p | `result_id` | FK to reading_results.id. | | `rollout_index` | 0-based position of this rollout within the reading's group for the same input row. Range `[0, readings.num_rollouts)`. | +### `reading_plans` + +| Column | Description | +| --- | --- | +| `id` | Plan identifier (UUID). | +| `collection_id` | Collection that owns the plan. | +| `name` | Optional plan name. | +| `steps_json` | Ordered JSON list of plan steps. | +| `created_at` | When the plan was created. | +| `created_by` | User who created the plan. | +| `updated_at` | Last modification time. | + +### `reading_presets` + +`readings` has no name column — the human-readable name lives here. Join +`readings.source_reading_preset_id = reading_presets.id`. + +| Column | Description | +| --- | --- | +| `id` | Preset identifier (UUID). | +| `collection_id` | Collection that owns the preset. | +| `name` | Human-readable reading name (unique per collection). | +| `created_at` | When the preset was created. | +| `created_by` | User who created the preset. | +| `updated_at` | Last modification time. | + +### `reading_preset_versions` + +| Column | Description | +| --- | --- | +| `reading_preset_id` | FK to reading_presets.id (part of composite PK). | +| `version_index` | Version number within the preset (part of composite PK). Matches `readings.source_reading_preset_version`. | +| `prompt_template_segments` | JSON template segments for this version. | +| `context_configs` | JSON context configs for this version. | +| `model_json` | Model configuration for this version. | +| `output_schema` | JSON schema for output validation. | +| `max_new_tokens` | Maximum new tokens per LLM call. | +| `created_at` | When the version was created. | + +### `collections` + +Always scoped to the current collection, so this table returns at most one row. + +| Column | Description | +| --- | --- | +| `id` | Collection identifier. | +| `name` | Collection name. | +| `description` | Collection description. | +| `created_by` | User who created the collection. | +| `created_at` | When the collection was created. | +| `metadata_json` | JSONB metadata for the collection. | +| `is_clone` | True when the collection was created by cloning another. | +| `source_collection_id` | Collection this one was cloned from, when applicable. | + ## JSON Metadata Access Patterns Docent stores user-supplied metadata as JSON. Access using Postgres operators: @@ -189,7 +250,7 @@ WHERE meta->>'status' = 'flagged'; SELECT AVG(CAST(metadata_json->>'latency_ms' AS DOUBLE PRECISION)) AS avg_latency_ms FROM agent_runs -WHERE metadata_json ? 'latency_ms'; +WHERE metadata_json->>'latency_ms' IS NOT NULL; ``` When querying JSON fields, comparisons default to string semantics. Cast values when you need numeric ordering or aggregation. @@ -251,8 +312,8 @@ Express filters like “≥10 messages” in DQL with the pattern above. Do not | Boolean logic (`AND`, `OR`, `NOT`) | | Comparison operators (`=`, `!=`, `<`, `<=`, `>`, `>=`, `IS`, `IS NOT`, `IS DISTINCT FROM`, `IN`, `BETWEEN`, `LIKE`, `ILIKE`, `EXISTS`, `SIMILAR TO`, `~`, `~*`, `!~`, `!~*`) | | Arithmetic & math (`+`, `-`, `*`, `/`, `%`, `POWER`, `ABS`, `SIGN`, `SQRT`, `LN`, `LOG`, `EXP`, `GREATEST`, `LEAST`, `FLOOR`, `CEIL`, `ROUND`, `RANDOM`) | -| String helpers (`SUBSTRING`, `LEFT`, `RIGHT`, `LENGTH`, `UPPER`, `LOWER`, `INITCAP`, `TRIM`, `REPLACE`, `SPLIT_PART`, `POSITION`, `CONCAT`, `CONCAT_WS`, `STRING_AGG`) | -| JSON operators & functions (`->`, `->>`, `#>`, `#>>`, `@>`, `?`, `?|`, `?&`, `jsonb_build_object`, `jsonb_build_array`, `jsonb_array_length`, `json_agg`, `jsonb_agg`, `json_object_agg`, `jsonb_set`, `jsonb_path_query`, `jsonb_path_exists`, `convert_from`, `convert_to`) | +| String helpers (`SUBSTRING`, `LEFT`, `RIGHT`, `LENGTH`, `UPPER`, `LOWER`, `INITCAP`, `TRIM`, `REPLACE`, `SPLIT_PART`, `POSITION`, `CONCAT`, `CONCAT_WS`, `STRING_AGG`, `MD5`) | +| JSON operators & functions (`->`, `->>`, `#>`, `#>>`, `@>`, `?`, `jsonb_build_object`, `jsonb_build_array`, `jsonb_array_length`, `json_agg`, `jsonb_agg`, `json_object_agg`, `jsonb_set`, `jsonb_path_query`, `jsonb_path_exists`, `convert_from`, `convert_to`) | | Date/time basics (`CURRENT_DATE`, `CURRENT_TIME`, `CURRENT_TIMESTAMP`, `NOW()`, `EXTRACT`, `DATE_TRUNC`, `AGE`, `AT TIME ZONE`, `timezone()`) | | Interval arithmetic (`timestamp +/- INTERVAL`, `INTERVAL` literals, `MAKE_INTERVAL`, `JUSTIFY_DAYS`, `JUSTIFY_HOURS`, `JUSTIFY_INTERVAL`) | | Construction & conversion (`MAKE_DATE`, `MAKE_TIME`, `MAKE_TIMESTAMP`, `MAKE_TIMESTAMPTZ`, `TO_CHAR`, `TO_DATE`, `TO_TIMESTAMP`, `DATE_PART`) | @@ -277,6 +338,28 @@ ORDER BY created_at DESC LIMIT 10; ``` +### Random Sample of Transcripts + +```sql +SELECT transcripts.id AS transcript +FROM transcripts +ORDER BY RANDOM() +LIMIT 100; +``` + +Use `ORDER BY RANDOM()` to draw a random sample. Do not sample by ordering on an id column (e.g. `ORDER BY transcripts.id LIMIT 100`): it returns the same rows every time, and ordering a `LIMIT` query on an indexed id column leads the planner to scan the table by that index rather than the collection's index, which can hit the query timeout even on small collections. + +### Reproducible Random Sample of Transcripts + +```sql +SELECT transcripts.id AS transcript +FROM transcripts +ORDER BY MD5(CONCAT(transcripts.id, 'my-seed')) +LIMIT 100; +``` + +When you need the same sample on every run (e.g. so an analysis is reproducible), hash the id together with a fixed seed string instead of using `ORDER BY RANDOM()`. The same seed always yields the same sample; change the seed string to draw a different one. Like `RANDOM()`, the hash is not indexable, so it avoids the id-ordering timeout above. + ### Transcript Counts per Group ```sql @@ -299,7 +382,7 @@ WITH normalized_runs AS ( metadata_json->>'environment' AS environment, metadata_json->>'status' AS status FROM agent_runs - WHERE metadata_json ? 'environment' + WHERE metadata_json->>'environment' IS NOT NULL ) SELECT environment, @@ -315,6 +398,17 @@ ORDER BY total_runs DESC; ``` +### Finding a Reading by Name + +Reading names live on `reading_presets`, not `readings`: + +```sql +SELECT r.id AS reading_id, rp.name, r.source_reading_preset_version +FROM readings r +JOIN reading_presets rp ON rp.id = r.source_reading_preset_id +WHERE rp.name = 'my-analysis'; +``` + ### Reading Results for a Specific Reading ```sql @@ -428,6 +522,40 @@ positional identity across readings. - **Type awareness**: Cast values explicitly when precision matters. - **Reading results: filter by completion.** Querying `reading_results` will include pending and failed rollouts by default. Add `WHERE rr.output IS NOT NULL AND (rr.error IS NULL OR rr.error::text = 'null')` to any aggregation that should ignore them. +## Performance, routing, and the query timeout + +Every DQL query is killed after **15 seconds**. On a large collection (tens of thousands of runs or more), a query can blow past that and fail — so query _shape_ matters, not just correctness. + +**The mechanism, so you can reason about it:** a query that reads **only** from `agent_runs` runs on a fast path, so even a metadata filter over a large collection is cheap. As soon as a query _also_ references another table — `reading_results`, `transcripts`, `judge_results`, etc. — it drops off that fast path; and if it is _filtering_ `agent_runs` by metadata, that filter then scans **every run** in the collection. Rule of thumb: **isolate the most selective filter into its own step, then join the rest against that small result** — drive from whichever side is narrower (the two shapes below). + +**The pattern:** the two shapes are equally valid — pick by where the selectivity is: + +**a) The `agent_runs` metadata filter is the selective side.** Select those runs in a single-table `agent_runs` step (that step can use the fast path), then join the other tables onto the small set: + +```sql +-- Step 1 — single table on agent_runs (a scalar `->>` filter uses the fast path): +SELECT agent_runs.id AS run_id +FROM agent_runs +WHERE agent_runs.metadata_json->>'variant' = 'treatment'; +-- Step 2 joins reading outputs against the runs from step 1, not the whole collection. +``` + +**b) The other table is the selective side** (a specific reading, specific transcripts). Filter it first, then match `agent_runs` by id — an indexed lookup, no full scan: + +```sql +SELECT ar.metadata_json->>'variant', rr.output->>'label' +FROM reading_results rr +JOIN reading_result_links rrl ON rrl.result_id = rr.id +JOIN agent_runs ar ON rr.arguments_dict->'agent_run'->>'id' = CAST(ar.id AS TEXT) -- 'agent_run' = the reading's run slot +WHERE rrl.reading_id = ''; -- narrow set; agent_runs reached by id +``` + +Either way, **do the selection once and compute all your metrics in one pass** — not one multi-table query per metric, each re-scanning the whole collection. And reach for a split only when a query is actually slow: a multi-table query that's already narrow on one side is fine as-is. + +_(Array-containment like `metadata_json->'field' @> '[…]'` isn't fast-path routable — that selection runs the slow way regardless, so the win there is doing it once instead of per metric.)_ + +**If a query times out,** the error includes a note on why it couldn't use the fast path and, where there is one, the fix: which other tables it references and how to restructure around the narrower side, which metadata field the fast path doesn't cover, or which construct to rewrite (e.g. `->>` instead of `->`, `LIKE` instead of `SIMILAR TO`). Use it to decide what to change — though some fall-offs are limitations you can only work around, not fix. + ## DQL quirks ### No Wildcards Allowed @@ -510,4 +638,4 @@ ROUND(CAST(AVG(...) AS NUMERIC), 3) ### JSON Access Patterns - Nested: `metadata_json->'parent'->>'child'` - Flat key with dot: `metadata_json->>'parent.child'` -- Check key existence: `metadata_json ? 'key'` +- Check key presence: `metadata_json->>'key' IS NOT NULL` (preferred over `metadata_json ? 'key'` — the two differ only for keys holding JSON `null`, and the `IS NOT NULL` form is the one that means "the key has a usable value") diff --git a/plugins/docent/skills/docent/ingestion-reference.md b/plugins/docent/skills/docent/ingestion-reference.md index a07ef6e..ad93e57 100644 --- a/plugins/docent/skills/docent/ingestion-reference.md +++ b/plugins/docent/skills/docent/ingestion-reference.md @@ -468,5 +468,5 @@ print(f"Source records: {len(raw_data)}") print(f"Converted: {len(agent_runs)}") print(f"Failed conversions: {len(conversion_errors)}") print(f"Uploaded count: {uploaded_count if uploaded_count is not None else 'unknown'}") -print(f"Collection URL: https://docent.transluce.org/collection/{collection_id}") +print(f"Collection URL: https://docent.transluce.org/dashboard/{collection_id}") ``` diff --git a/plugins/docent/skills/docent/ingestion.md b/plugins/docent/skills/docent/ingestion.md index e0de68b..91d4d61 100644 --- a/plugins/docent/skills/docent/ingestion.md +++ b/plugins/docent/skills/docent/ingestion.md @@ -13,7 +13,8 @@ Keep the main workflow lightweight. Load `./ingestion-reference.md` only when yo - Work in four stages: context, planning, ingestion, verification. - Create and maintain `ingestion-plan.md` in the working directory. -- Do not upload until the user confirms the proposed collection name, Docent hierarchy, field mappings, and omitted data. +- Before uploading, resolve whether the ingestion plan requires confirmation. An explicit user instruction to auto-approve or require review takes precedence. Otherwise, use `client.get_preferences().auto_approve_plans`. When auto-approval is off or the preference cannot be read, wait for the user to confirm the proposed collection name, Docent hierarchy, field mappings, and omitted data. +- Auto-approval skips the plan confirmation pause; it does not permit guessing through material ambiguity, ignoring conversion failures, or silently dropping source data. - Never silently skip source data. Any file or field not ingested must be documented with a reason and expected impact. - Save ingestion code to a file such as `ingest.py` or `ingest_.py`; do not rely on one-off inline Python for the final upload path. - Use `parse_chat_message` from the Docent SDK for transcript messages, and make deliberate role mappings when the source roles differ from Docent's supported roles. @@ -40,6 +41,15 @@ Collect only what is needed to plan: - Data path: the file or directory to ingest. - Optional context: what produced the data and what analysis the user wants to do in Docent. +After initializing the SDK client, resolve the approval behavior once for the workflow: + +```python +preferences = client.get_preferences() +auto_approve_plan = preferences.auto_approve_plans +``` + +An explicit request such as "auto-approve this ingestion plan" or "show me the ingestion plan before uploading" overrides the account preference. A general request to ingest data is not itself an approval override. If the installed SDK does not expose `get_preferences()` or the preference request fails, require confirmation unless the user explicitly requested auto-approval. + Create `ingestion-plan.md` with this compact structure and append findings as the workflow proceeds: ```markdown @@ -76,7 +86,8 @@ Create `ingestion-plan.md` with this compact structure and append findings as th - Collection name: - Data context: - Analysis goals: -- User confirmed: +- Approval mode and source: +- User confirmed, if required: ## Execution Log @@ -140,7 +151,7 @@ For tree or branching data, usually ingest each branch as its own `AgentRun` and ### Confirmation Gate -Before writing the final upload script, present the plan and wait for user confirmation. Include: +Before writing the final upload script, summarize: - Source structure and detected data type - Proposed collection name @@ -150,6 +161,8 @@ Before writing the final upload script, present the plan and wait for user confi - Expected source record count, if available - Your understanding of the data context and analysis goals +If approval is required, present the summary and wait for user confirmation. If the plan is auto-approved, record that fact and its source in `ingestion-plan.md`, share the summary as a progress update, and continue without pausing. + ## Stage 3: Ingestion For Inspect `.eval` files, use the built-in loader and proceed directly to sanity checks. See `./ingestion-reference.md` for the import pattern. @@ -157,14 +170,14 @@ For Inspect `.eval` files, use the built-in loader and proceed directly to sanit For custom data: 1. Write an ingestion script to the filesystem. -2. Load raw source records according to the confirmed file structure. +2. Load raw source records according to the approved file structure. 3. Convert a small sample into `AgentRun` objects. 4. Manually inspect sample turns with reasoning and tool calls to verify reasoning was represented, merged, or intentionally omitted according to the plan. 5. Fix sample conversion issues. 6. Convert the full dataset and record conversion failures. 7. Run `check_agent_runs(agent_runs)` and inspect the formatted report. -8. Upload only after the conversion output and warnings match the confirmed plan. +8. Upload only after the conversion output and warnings match the approved plan. If a failure is not easily recoverable, such as unexpected data shape, authentication failure, API error, or ambiguous SDK error, stop and ask the user how they want to proceed. Include the exact error and the affected file or record when possible. @@ -190,4 +203,6 @@ After upload, verify and log: - Any accepted sanity warnings - Collection URL +Surface the collection URL to the user: in local sessions the SDK opens it in the default browser, but also provide a clickable link. In sandboxed sessions where `webbrowser.open()` can't reach the user's browser (e.g. Codex CLI), provide a clickable link instead. See "Opening Docent pages" in `SKILL.md`. + If the SDK cannot verify the uploaded count, provide the collection URL and record that manual verification is needed. diff --git a/plugins/docent/skills/docent/readings-reference.md b/plugins/docent/skills/docent/readings-reference.md index aeef211..c95dff0 100644 --- a/plugins/docent/skills/docent/readings-reference.md +++ b/plugins/docent/skills/docent/readings-reference.md @@ -12,13 +12,13 @@ Use scripted readings only when you need additional flexibility, e.g. varying th Readings are executed lazily: nothing runs until `flush()` is called. You normally do not need to call `flush()` manually. `flush()` is automatically called at script exit, and also anytime you attempt to access the output of a reading which has not been run yet. The system infers the execution DAG automatically. Re-running the same script is free: readings are content-addressed, so identical analyses reuse existing results. -When readings are flushed, they will appear as an analysis plan in the web UI for the user to approve. The script will pause execution until the user approves the readings. They may also cancel the script and ask you to make changes. (Note: the analysis plan interface in the web UI is read-only.) +When readings are flushed, they appear as an analysis plan in the web UI. If the user's account preference is set to auto-approve, reading steps start automatically; otherwise, the script pauses until the user approves them. The user can cancel the script and ask you to make changes. (Note: the analysis plan interface in the web UI is read-only.) -If you need a no-UI-approval flow for a trusted analysis, you may opt into SDK auto-approval by explicitly calling `client.flush(auto_approve=True)`. This reuses the same backend approval endpoint programmatically, including for dependent steps that are initially unresolved. +Normally, omit `auto_approve` and let the user's account preference control approval. If the user explicitly asks to auto-run or require review for a specific plan, call `client.flush(auto_approve=True)` or `client.flush(auto_approve=False)` respectively. An explicit SDK value overrides the account preference. Auto-approval reuses the same backend approval flow, including for dependent steps that are initially unresolved. Some analysis plans require mid-script blocking, for example if one step waits for reading results (using `.results`) in order to construct a later step. In these cases: * The script may submit an initial set of steps for approval, then block waiting for results before it can continue. -* The user may need to approve the plan more than once, unless you explicitly call `client.flush(auto_approve=True)` for each flush that should bypass manual approval. +* The user may need to approve the plan more than once when their account defaults to manual approval, unless you explicitly call `client.flush(auto_approve=True)` for each flush that should bypass manual approval. * Warn the user upfront about multi-approval flows so they know what to expect. Notes on what readings can see: @@ -124,10 +124,13 @@ slices = client.query( WITH windows AS ( SELECT t.id AS transcript_id, - GREATEST(0, CAST(t.metadata_json->>'first_error_idx' AS INTEGER) - 3) AS start_idx, - CAST(t.metadata_json->>'first_error_idx' AS INTEGER) + 3 AS end_idx - FROM transcripts t - WHERE t.metadata_json ? 'first_error_idx' + GREATEST(0, CAST(t.meta->>'first_error_idx' AS INTEGER) - 3) AS start_idx, + CAST(t.meta->>'first_error_idx' AS INTEGER) + 3 AS end_idx + FROM ( + SELECT id, convert_from(metadata_json, 'UTF8')::jsonb AS meta + FROM transcripts + ) AS t + WHERE t.meta->>'first_error_idx' IS NOT NULL ) SELECT transcript_slice(transcript_id, start_idx, end_idx) AS window FROM windows @@ -318,6 +321,53 @@ client.read(...) # back to top-level ``` Only use a group when several readings are closely related. Do not create a step group with a single step. +### `client.plan_markdown(title, markdown) -> None` +Queues a markdown note in the analysis plan. Notes render in **script order** between DQL and reading steps; they are not executed and do not require approval. + +The top note should work as a **scroll-back reference**: someone deep in the plan should be able to jump back up and recover what is being studied without re-reading DQL and reading steps. + +**Required in every analysis script:** +* Call `client.plan_markdown` **before** the first `client.query` or `client.read`. +* Write the top note from the user's analysis question and your planned analytical approach. +* Explain the planned analysis flow in a narrative style. Do not write terse phase lists like "Phases: sample transcripts -> apply rubric -> compare labels." +* Keep it plain but substantive. Do not sanitize sensitive or adversarial content, and do not settle for procedure summaries like "sample regressions and synthesize failure mechanisms" when the subject matter reveals what those regressions are actually about. +* You are allowed to sparingly add notes before a step if it improves overall readability of the analysis plan, but **avoid** spamming the plan with notes every step. + +#### Markdown notes framework + +Two labeled sections is recommended. Use two or more sentences per section, unless you think more sentences would help. For example, when the analysis plan involves rubric refinement, you should include the entire rubric. + +Label each section with a **`##` heading**. Body prose starts on the next line. + +Here is an example of how you can use the two sections: +| Heading | Covers | +|---|---| +| `## Behavior` | Plain-language description of the behavior or phenomenon under study | +| `## Measurement` | How the behavior is scored or compared: rubric/scale gist, sample, filters, blinding, pairing key | + +```python +client.plan_markdown( + "Sycophancy in personal guidance conversations", + """## Behavior +In this analysis, we looked at what types of guidance people ask of language models. We explored how language models responded across different domains, focusing particularly on how rates of excessive validation or praise (i.e. sycophancy) varied by the topic of guidance. Sycophantic behavior here includes giving excessively confident verdicts based on one-sided accounts: for example, confirming that the user's partner is "definitely gaslighting" them, endorsing a plan to quit a job with nothing lined up, or validating a romantic reading of ordinary friendly behavior. It is especially likely when the user pushes back on Claude's initial assessment. + +## Measurement +We start from a random sample of 1M claude.ai conversations, filtered to unique users (~639k) and then classified for personal guidance—conversations where people ask what they specifically should do (e.g. "Should I…?", "What do I do about…?"), excluding requests for objective information or general opinions (~38k conversations). + +Conversations are categorized into nine domains (relationships, career, personal development, financial, legal, health and wellness, parenting, ethics, spirituality), with multi-domain chats assigned to the most prominent topic. + +An automated classifier (Claude Sonnet 4.5) scores sycophancy per conversation using the following rubric: +(1) willingness to push back on the user's framing, +(2) maintaining positions when challenged, +(3) praise proportional to the merit of ideas, and +(4) speaking frankly regardless of what the user wants to hear. + +We additionally tag whether the user pushed back on Claude during the conversation, to compare sycophancy rates with and without pushback. For model comparisons, we stress-test by prefilling new models with real opted-in feedback conversations where prior models behaved sycophantically, then grading only the new model's continuation (Wilson CIs on rates). +""", +) +rows = client.query(collection_id, "SELECT ...", name="Transcripts") +``` + ### `client.list_reading_presets(collection_id, *, owned_only=True) -> list[dict]` Lists reading presets in a collection. - `owned_only=True`: returns only presets created by the current user. @@ -355,8 +405,8 @@ Registers a reading step backed by a server-side preset. The server resolves the - `source_reading_preset_version`: Optional preset version to pin. When omitted, the server resolves the latest version. - `cache_mode`: See cache_mode description under `client.read()`. -### `client.flush(open_in_browser=True, auto_approve=False) -> dict` -Submits all pending readings to the server. Returns `plan_id` and per-entry `entry_statuses`. You normally do not need to call this explicitly. If `auto_approve=True`, the SDK will immediately approve newly submitted reading steps before waiting for results. Implicit flushes triggered by `reading.id`, `reading.results`, or `atexit` do not enable auto-approval unless you call `flush(auto_approve=True)` yourself first. +### `client.flush(open_in_browser=True, auto_approve=None) -> dict` +Submits all pending readings to the server. Returns `plan_id` and per-entry `entry_statuses`. You normally do not need to call this explicitly. `auto_approve=None` (the default) follows the user's account preference, `auto_approve=True` immediately approves newly submitted reading steps, and `auto_approve=False` explicitly requires manual approval. Explicit values override the account preference. Implicit flushes triggered by `reading.id`, `reading.results`, notebooks, or `atexit` use the account preference. ### `Reading` handle - `f"{reading}"` → `$alias` (for use in DQL referencing) @@ -398,7 +448,7 @@ summary_query = client.query( ## Model selection -Use `"provider/model_name"` format. For simple questions about transcript content, use openai/gpt-5.4-mini. For more complex interpretation, reasoning, or judgement, use openai/gpt-5.5. +Use `"provider/model_name"` format. For simple questions about transcript content, use openai/gpt-5.6-luna. For more complex interpretation, reasoning, or judgement, use openai/gpt-5.6-sol. Important: Do not use openai/gpt-4o or openai/gpt-4o-mini. Those models are obsolete. @@ -455,7 +505,7 @@ Better name: "judge_classification" ## Coding tips for reading scripts -* You must write your code out as a script file. Place analysis scripts in a per-session subdirectory under `docent_analyses/`, using the format `docent_analyses/_/` (e.g., `docent_analyses/2026-04-20_safety-eval/`). Create the directory at the start of the session. The short label should be a 2-3 word slug describing the analysis topic. This keeps scripts organized across sessions and out of the project's working directory. +* You must write your code out as a script file. Place analysis scripts in a per-session subdirectory under `docent_analyses/`, using the format `docent_analyses/YYYY-MM-DD_HH-MM-SS_/` (e.g., `docent_analyses/2026-04-20_14-35-09_safety-eval/`). Create the directory at the start of the session. The timestamp must include year, month, day, hour, minute, and second so multiple analyses created on the same day stay disambiguated. The short label should be a 2-3 word slug describing the analysis topic. This keeps scripts organized across sessions and out of the project's working directory. * Unless informed otherwise, assume uv is used for python package management. Run your scripts with `uv run`. * Make DQL query results self-verifying. Include extra columns that let the user confirm your query logic at a glance. The user should be able to verify correctness from the output alone, without re-reading the SQL. For example: * If you filter by a condition, include the filtered column in the SELECT. @@ -463,7 +513,7 @@ Better name: "judge_classification" * If you compare values (e.g., selecting rows where model A outperformed model B), include both models' names and scores, not just the winning run. * Don't Repeat Yourself. This is particularly important when it comes to prompts for LLMs. The user will likely want to modify prompts, and they should not have to track down multiple copies of a prompt throughout your code. If you need to create different variants of a prompt, build them from reusable pieces and/or use string interpolation, so there is a single source of truth for each part of the prompt. * Be sparing with print statements. -* If you are analyzing a limited sample of many items (e.g. because you can only fit so many in the context window), be mindful of *how* you are sampling them. The most recent N items may be a biased sample. It is safe to assume that UUIDs are random. +* If you are analyzing a limited sample of many items (e.g. because you can only fit so many in the context window), be mindful of *how* you are sampling them. The most recent N items are a biased sample. To draw a random sample, use `ORDER BY RANDOM() LIMIT N`. When you need the sample to be reproducible across runs, hash the id with a fixed seed instead: `ORDER BY md5(CONCAT(transcripts.id, '')) LIMIT N` (same seed → same sample). Do not sample by ordering on an id column (e.g. `ORDER BY transcripts.id LIMIT N`): besides returning the same rows every time, ordering a `LIMIT` query on an indexed id column leads the query planner to scan the table by that index instead of the collection's index, which can blow the query timeout even on small collections. * If you are using a reading to categorize things (e.g. types of problems, strategies, or mistakes), don't try to come up with a good list of categories without looking at the data. See the clustering example below. * **Test DQL incrementally.** When writing scripts with multiple DQL queries, test one simple query first to validate syntax patterns (casting, GROUP BY, etc.) before writing a large batch. DQL has quirks that are easier to catch one at a time than to debug across a 200-line script. @@ -491,7 +541,7 @@ client.plan_name = "Mistake clustering" # Step 1: Freeform summary of a sample of transcripts sampled_transcripts = client.query( collection_id, - "SELECT transcripts.id AS transcript FROM transcripts ORDER BY transcripts.id LIMIT 100", + "SELECT transcripts.id AS transcript FROM transcripts ORDER BY RANDOM() LIMIT 100", ) summarize = client.read( diff --git a/plugins/docent/skills/docent/rubric-writing.md b/plugins/docent/skills/docent/rubric-writing.md new file mode 100644 index 0000000..21d7bb5 --- /dev/null +++ b/plugins/docent/skills/docent/rubric-writing.md @@ -0,0 +1,33 @@ +# Rubric Writing Reference + +Read this before writing or revising a Docent rubric, rubric-like classifier prompt, output schema, or analysis step that asks an LLM judge to make a structured behavioral decision. + +## Goal + +You are helping the user turn a vague idea of a behavior they are looking for in a dataset of AI agent run transcripts into a concrete specification of what they are looking for. + +## Canonical rubric components + +The following rules govern how you should write rubrics and their schemas: + +A rubric must contain exactly these components: + +- One paragraph with an insightful high-level framing that makes the ensuing specification highly simple and parsimonious. Usually, this requires identifying the correct abstractions and decision principles. +- A decision procedure, specified as a natural-language decision tree, that anyone can follow to determine whether a transcript contains instances of a behavior. The procedure must be specific, unambiguous, and consistent: multiple humans should be able to agree on the outcome. +- An output schema, specified as JSON Schema, that describes the output of the decision procedure. + +## Rubric-writing rules + +- The level of specificity and detail in the decision procedure should be commensurate with the amount of information available to you. If the user has only provided a vague one-line statement, there is no need to overfit to a complex rubric. +- It's extremely important that the decision procedure is concise, simple, and clear. Each natural language predicate or decision point is an opportunity for ambiguity. +- It must be explicitly explained which output values correspond to which decisions. +- Unless otherwise stated, revisions to existing complex rubrics should be as minimal and targeted as possible. Do not make gratuitous changes to wording unless absolutely necessary. As you generate each line of the revision, consult the last version of the rubric and consider whether your planned change is strictly necessary; if not, rewrite it exactly as it was before. + +## Output schema rules + +- If unspecified, keep the rubric schema as simple as possible, but of course include what the user requests. +- The output schema must conform to Docent's restricted subset of JSON Schema 2020-12. +- Keep the schema aligned with the decision procedure. Every top-level field should be produced by a stated decision in the rubric. +- For match/no-match rubrics, prefer a small enum such as `["match", "no match"]` plus an explanation field with citations when the analysis needs evidence. +- Do not create speculative category enums during first-pass extraction or exploratory scans. If the possible categories are not already known from the user request or prior analysis, use free-text fields for the observed behavior and defer any taxonomy or enum until after reviewing extracted examples. +- If the user asks for something that could have intensity, add an integer enum field with values 0-10.