diff --git a/docs/index.md b/docs/index.md index e681df42f73..899e8d0aa03 100644 --- a/docs/index.md +++ b/docs/index.md @@ -207,3 +207,27 @@ Watch for project deploy to finish + +## Agent detection + +The CLI reads the `NETLIFY_AGENT` environment variable to learn which AI agent or tool is running it. Agents, MCP servers, and wrappers that invoke the CLI should set it to their name, optionally followed by `@` and a version: + +```bash +NETLIFY_AGENT=claude-code@2.1.0 netlify deploy +``` + +Recognized values are `claude`, `codex`, `copilot`, `gemini`, `cursor`, `opencode`, `kiro`, `cline`, `amp`, `warp`, `claudeai`, and `chatgpt`, plus the aliases `claude-code`, `claude-ai`, `github-copilot`, `github-copilot-cli`, `github-copilot-vscode-agent`, `cursor-cli`, `gemini-cli`, `kiro-cli`, and `warp-oz`. Matching ignores case and treats `_` as `-`. Any other value is recorded as `other`. Characters other than letters, digits, `_`, `.`, and `-` are removed, and values are truncated to 64 characters. + +The CLI also recognizes markers that agent products set on their own, such as `AI_AGENT`, `CODEX_CI`, and `GEMINI_CLI`. `NETLIFY_AGENT` takes precedence over all of them, even when its value isn't recognized. + +### Telemetry + +When telemetry is enabled, each CLI telemetry event includes the detected agent: + +- `agent`: the recognized name, or `other` +- `agent_source`: the name of the environment variable that identified the agent +- `agent_version`: the version, when the agent provides one +- `agent_markers`: every detected agent name, when markers from more than one agent are present +- `agent_other_value`: the sanitized value, when `agent` is `other` + +No agent fields are sent when no agent is detected. Telemetry isn't sent in CI, or at all after you run `netlify --telemetry-disable`. diff --git a/src/utils/agent-detection.ts b/src/utils/agent-detection.ts index c585d0f5dfd..1e94020994b 100644 --- a/src/utils/agent-detection.ts +++ b/src/utils/agent-detection.ts @@ -28,7 +28,7 @@ export type DrivingAgent = { } const ANNOUNCED_NAME_TABLE = new Map([ - ...CANONICAL_AGENT_NAMES.map((name) => [name, name] as const), + ...CANONICAL_AGENT_NAMES.filter((name) => name !== 'other').map((name) => [name, name] as const), ['claude-code', 'claude'], ['claude-ai', 'claudeai'], ['github-copilot', 'copilot'], diff --git a/src/utils/telemetry/telemetry.ts b/src/utils/telemetry/telemetry.ts index 6c4fba7caf5..550828517bc 100644 --- a/src/utils/telemetry/telemetry.ts +++ b/src/utils/telemetry/telemetry.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'url' import { getGlobalConfigStore } from '@netlify/dev-utils' import { isCI } from 'ci-info' +import { getDrivingAgent } from '../agent-detection.js' import execa from '../execa.js' import { isTelemetryDisabled, cliVersion } from './utils.js' @@ -45,6 +46,20 @@ const eventConfig = { ], } +// Every key is always present so a caller's payload can never supply its own agent attribution; +// undefined values are dropped when the event is serialized. +const getAgentProperties = () => { + const agent = getDrivingAgent() + + return { + agent: agent?.name, + agent_source: agent?.source, + agent_version: agent?.version, + agent_markers: agent?.markers, + agent_other_value: agent?.otherValue, + } +} + /** * Tracks a custom event with the provided payload */ @@ -82,7 +97,7 @@ export async function track( anonymousId: cliId, duration, status, - properties: { ...properties, nodejsVersion, cliVersion }, + properties: { ...properties, nodejsVersion, cliVersion, ...getAgentProperties() }, } return send('track', defaultData) diff --git a/tests/unit/utils/agent-detection.test.ts b/tests/unit/utils/agent-detection.test.ts index 6b3ed84c52f..bcb117faee2 100644 --- a/tests/unit/utils/agent-detection.test.ts +++ b/tests/unit/utils/agent-detection.test.ts @@ -271,6 +271,16 @@ test.each(['constructor', '__proto__', 'toString'])('%s does not resolve via the }) }) +test.each(['NETLIFY_AGENT', 'AI_AGENT'])('%s=other keeps the announced value in otherValue', (source) => { + expect(getDrivingAgent({ [source]: 'other' })).toEqual({ name: 'other', source, otherValue: 'other' }) + expect(getDrivingAgent({ [source]: 'Other@1.0' })).toEqual({ + name: 'other', + source, + version: '1.0', + otherValue: 'Other', + }) +}) + test('NETLIFY_AGENT=constructor_1-0_agent does not resolve constructor via the split-at-last-underscore path', () => { expect(getDrivingAgent({ NETLIFY_AGENT: 'constructor_1-0_agent' })).toEqual({ name: 'other', diff --git a/tests/unit/utils/telemetry/telemetry.test.ts b/tests/unit/utils/telemetry/telemetry.test.ts new file mode 100644 index 00000000000..090fdb8b5e1 --- /dev/null +++ b/tests/unit/utils/telemetry/telemetry.test.ts @@ -0,0 +1,124 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' + +import { track } from '../../../../src/utils/telemetry/telemetry.js' +import { cliVersion } from '../../../../src/utils/telemetry/utils.js' +import execa from '../../../../src/utils/execa.js' + +vi.mock('ci-info', () => ({ isCI: false })) + +vi.mock('@netlify/dev-utils', async (importOriginal) => ({ + ...(await importOriginal()), + getGlobalConfigStore: vi.fn(() => + Promise.resolve({ get: (key: string) => (key === 'telemetryDisabled' ? false : 'test-user-1') }), + ), +})) + +vi.mock('../../../../src/utils/execa.js', () => ({ default: vi.fn(() => ({ unref: vi.fn() })) })) + +const AGENT_ENV_KEYS = [ + 'NETLIFY_AGENT', + 'CODEX_CI', + 'CODEX_VERSION', + 'GEMINI_CLI', + 'COPILOT_CLI', + 'COPILOT_AGENT_SESSION_ID', + 'OPENCODE', + 'OPENCODE_TERMINAL', + 'AGENT_DISPLAY_OUT', + 'AGENT_CONTEXT_OUT', + 'OZ_RUN_ID', + 'WARP_RUN_ID', + 'AI_AGENT', + 'COPILOT_AGENT', + 'CURSOR_AGENT', + 'CLINE_ACTIVE', + 'AGENT', + 'CLAUDE_CODE_CHILD_SESSION', +] + +const getTrackedProperties = (): Record => { + const { calls } = vi.mocked(execa).mock + const [, [, optionsJson]] = calls[calls.length - 1] as [string, string[]] + return (JSON.parse(optionsJson) as { data: { properties: Record } }).data.properties +} + +const getTrackedAgentProperties = () => + Object.fromEntries( + Object.entries(getTrackedProperties()).filter(([key]) => key === 'agent' || key.startsWith('agent_')), + ) + +beforeEach(() => { + vi.clearAllMocks() + AGENT_ENV_KEYS.forEach((key) => vi.stubEnv(key, undefined)) +}) + +afterEach(() => { + vi.unstubAllEnvs() +}) + +describe('track', () => { + test('adds the driving agent alongside the existing properties', async () => { + vi.stubEnv('AI_AGENT', 'claude-code') + + await track('command', { command: 'status' }) + + expect(getTrackedProperties()).toMatchObject({ command: 'status', cliVersion }) + expect(getTrackedAgentProperties()).toEqual({ agent: 'claude', agent_source: 'AI_AGENT' }) + }) + + test('adds no agent properties when no agent is detected', async () => { + await track('command', { command: 'status' }) + + expect(getTrackedAgentProperties()).toEqual({}) + }) + + test('drops agent properties supplied by the caller', async () => { + await track('command', { + command: 'status', + agent: 'spoofed', + agent_source: 'spoofed', + agent_version: 'spoofed', + agent_markers: ['spoofed'], + agent_other_value: 'spoofed', + }) + + expect(getTrackedAgentProperties()).toEqual({}) + }) + + test('adds the agent version when the agent announces one', async () => { + vi.stubEnv('AI_AGENT', 'claude-code_2-1-263_agent') + + await track('command', { command: 'status' }) + + expect(getTrackedAgentProperties()).toEqual({ + agent: 'claude', + agent_source: 'AI_AGENT', + agent_version: '2.1.263', + }) + }) + + test('lists every matched agent when agents are nested', async () => { + vi.stubEnv('AI_AGENT', 'claude-code') + vi.stubEnv('CODEX_CI', '1') + + await track('command', { command: 'status' }) + + expect(getTrackedAgentProperties()).toEqual({ + agent: 'codex', + agent_source: 'CODEX_CI', + agent_markers: ['codex', 'claude'], + }) + }) + + test('reports an unknown AI_AGENT value as other with its sanitized value', async () => { + vi.stubEnv('AI_AGENT', 'some-new-tool') + + await track('command', { command: 'status' }) + + expect(getTrackedAgentProperties()).toEqual({ + agent: 'other', + agent_source: 'AI_AGENT', + agent_other_value: 'some-new-tool', + }) + }) +})