From 109ca7649312ec7e1ac74c6fedf2205194039996 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Thu, 25 Jun 2026 13:10:10 +0800 Subject: [PATCH 1/5] fix: apply saved todo config on next organize --- src/viewer/index.html | 13 ++++++++----- src/viewer/parts/app/05-i18n.js | 4 ++-- src/viewer/parts/app/08-state-expert.js | 2 +- src/viewer/parts/app/60-actions-todo.js | 7 +++++-- test/viewer-session-id.test.ts | 18 ++++++++++++++++-- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/viewer/index.html b/src/viewer/index.html index 7bb7ce8f..e88ca85e 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -4330,7 +4330,7 @@

AI Todo

'settings.apiKeyLabel': 'API key:', 'settings.save': 'Save config', 'settings.saving': 'Saving...', - 'settings.savedRestart': 'Config saved. Restart the service to apply it.', + 'settings.savedReady': 'Config saved. The next organize run will use it now.', 'settings.saveFailed': 'Config save failed', 'act.status.updateFailed': 'Todo status update failed', 'obs.type.file_read': 'Read file', @@ -4558,7 +4558,7 @@

AI Todo

'settings.apiKeyLabel': 'API key:', 'settings.save': '保存配置', 'settings.saving': '保存中...', - 'settings.savedRestart': '配置已保存,重启后生效。', + 'settings.savedReady': '配置已保存,下次整理会立即使用新配置。', 'settings.saveFailed': '配置保存失败', 'act.status.updateFailed': '待办状态更新失败', 'obs.type.file_read': '读取文件', @@ -4767,7 +4767,7 @@

AI Todo

audit: { loaded: false, entries: [], opFilter: '' }, activity: { loaded: false, observations: [], sessions: [], typeFilter: '', loadingPhase: '', warnings: [] }, lessons: { loaded: false, items: [], search: '', skillSearch: '', skillRootFilter: 'all', mode: 'explicit', projects: [] }, - actions: { loaded: false, items: [], reviewItems: [], frontier: [], statusFilter: '', search: '', doneExpanded: false, earlierOpenExpanded: false, olderBacklogExpanded: false, extractStatus: '', extractMessage: '', extractInFlight: false, cardRefreshInFlight: {}, cardRefreshNotice: '', stale: false, config: null, configSaving: false, configDraft: {} }, + actions: { loaded: false, items: [], reviewItems: [], frontier: [], statusFilter: '', search: '', doneExpanded: false, earlierOpenExpanded: false, olderBacklogExpanded: false, extractStatus: '', extractMessage: '', extractInFlight: false, cardRefreshInFlight: {}, cardRefreshNotice: '', stale: false, config: null, configSaving: false, configDraft: {}, forceNextExtract: false }, inbox: { loaded: false, items: [], awaitingItems: [], answeredItems: [], dismissedItems: [], replyingId: null, pendingById: {}, briefingExpanded: false, answeredExpanded: false }, crystals: { loaded: false, items: [], search: '', lessonMap: {} }, profile: { loaded: false, projects: [], selectedProject: '', data: null }, @@ -9064,7 +9064,8 @@

AI Todo

apiPost('config/todo-extractor', body).then(function(res) { if (res && res.config) state.actions.config = res; state.actions.configDraft = {}; - state.actions.extractMessage = t('settings.savedRestart'); + state.actions.forceNextExtract = true; + state.actions.extractMessage = t('settings.savedReady'); }).catch(function() { state.actions.extractStatus = 'error'; state.actions.extractMessage = t('settings.saveFailed'); @@ -9138,6 +9139,8 @@

AI Todo

function startTodoExtraction(force) { if (state.actions.extractInFlight) return; + var shouldForce = force === true || state.actions.forceNextExtract === true; + state.actions.forceNextExtract = false; state.actions.extractInFlight = true; state.actions.extractStatus = 'running'; state.actions.extractMessage = t('act.extract.starting'); @@ -9166,7 +9169,7 @@

AI Todo

// don't hard-code maxSessions/maxObservationsPerSession here or the // settings would never take effect on this primary extraction path. apiPost('todo-extract/generate', { - force: force === true + force: shouldForce }).then(function(result) { var delta = todoExtractionDelta(result); if (!result || result.success !== true) { diff --git a/src/viewer/parts/app/05-i18n.js b/src/viewer/parts/app/05-i18n.js index fa99271e..94f729a2 100644 --- a/src/viewer/parts/app/05-i18n.js +++ b/src/viewer/parts/app/05-i18n.js @@ -129,7 +129,7 @@ 'settings.apiKeyLabel': 'API key:', 'settings.save': 'Save config', 'settings.saving': 'Saving...', - 'settings.savedRestart': 'Config saved. Restart the service to apply it.', + 'settings.savedReady': 'Config saved. The next organize run will use it now.', 'settings.saveFailed': 'Config save failed', 'act.status.updateFailed': 'Todo status update failed', 'obs.type.file_read': 'Read file', @@ -357,7 +357,7 @@ 'settings.apiKeyLabel': 'API key:', 'settings.save': '保存配置', 'settings.saving': '保存中...', - 'settings.savedRestart': '配置已保存,重启后生效。', + 'settings.savedReady': '配置已保存,下次整理会立即使用新配置。', 'settings.saveFailed': '配置保存失败', 'act.status.updateFailed': '待办状态更新失败', 'obs.type.file_read': '读取文件', diff --git a/src/viewer/parts/app/08-state-expert.js b/src/viewer/parts/app/08-state-expert.js index 4e4dd4b4..bbb009d9 100644 --- a/src/viewer/parts/app/08-state-expert.js +++ b/src/viewer/parts/app/08-state-expert.js @@ -72,7 +72,7 @@ audit: { loaded: false, entries: [], opFilter: '' }, activity: { loaded: false, observations: [], sessions: [], typeFilter: '', loadingPhase: '', warnings: [] }, lessons: { loaded: false, items: [], search: '', skillSearch: '', skillRootFilter: 'all', mode: 'explicit', projects: [] }, - actions: { loaded: false, items: [], reviewItems: [], frontier: [], statusFilter: '', search: '', doneExpanded: false, earlierOpenExpanded: false, olderBacklogExpanded: false, extractStatus: '', extractMessage: '', extractInFlight: false, cardRefreshInFlight: {}, cardRefreshNotice: '', stale: false, config: null, configSaving: false, configDraft: {} }, + actions: { loaded: false, items: [], reviewItems: [], frontier: [], statusFilter: '', search: '', doneExpanded: false, earlierOpenExpanded: false, olderBacklogExpanded: false, extractStatus: '', extractMessage: '', extractInFlight: false, cardRefreshInFlight: {}, cardRefreshNotice: '', stale: false, config: null, configSaving: false, configDraft: {}, forceNextExtract: false }, inbox: { loaded: false, items: [], awaitingItems: [], answeredItems: [], dismissedItems: [], replyingId: null, pendingById: {}, briefingExpanded: false, answeredExpanded: false }, crystals: { loaded: false, items: [], search: '', lessonMap: {} }, profile: { loaded: false, projects: [], selectedProject: '', data: null }, diff --git a/src/viewer/parts/app/60-actions-todo.js b/src/viewer/parts/app/60-actions-todo.js index b37cc17c..366539ba 100644 --- a/src/viewer/parts/app/60-actions-todo.js +++ b/src/viewer/parts/app/60-actions-todo.js @@ -223,7 +223,8 @@ apiPost('config/todo-extractor', body).then(function(res) { if (res && res.config) state.actions.config = res; state.actions.configDraft = {}; - state.actions.extractMessage = t('settings.savedRestart'); + state.actions.forceNextExtract = true; + state.actions.extractMessage = t('settings.savedReady'); }).catch(function() { state.actions.extractStatus = 'error'; state.actions.extractMessage = t('settings.saveFailed'); @@ -297,6 +298,8 @@ function startTodoExtraction(force) { if (state.actions.extractInFlight) return; + var shouldForce = force === true || state.actions.forceNextExtract === true; + state.actions.forceNextExtract = false; state.actions.extractInFlight = true; state.actions.extractStatus = 'running'; state.actions.extractMessage = t('act.extract.starting'); @@ -325,7 +328,7 @@ // don't hard-code maxSessions/maxObservationsPerSession here or the // settings would never take effect on this primary extraction path. apiPost('todo-extract/generate', { - force: force === true + force: shouldForce }).then(function(result) { var delta = todoExtractionDelta(result); if (!result || result.success !== true) { diff --git a/test/viewer-session-id.test.ts b/test/viewer-session-id.test.ts index 374741ca..dea530a6 100644 --- a/test/viewer-session-id.test.ts +++ b/test/viewer-session-id.test.ts @@ -1267,12 +1267,20 @@ describe("viewer session rendering", () => { it("renders and saves todo extractor config from the global settings panel", async () => { const { sandbox, getElement, dispatchDocumentClick } = loadViewerSandbox(); const posts: any[] = []; + const extractPosts: any[] = []; sandbox.fetch = async (input: unknown, init?: { body?: string }) => { const url = String(input); if (url.includes("config/todo-extractor") && init?.body) { posts.push(JSON.parse(init.body)); return { ok: true, json: async () => ({ success: true, envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: true } }) }; } + if (url.includes("todo-extract/generate") && init?.body) { + extractPosts.push(JSON.parse(init.body)); + return { ok: true, json: async () => ({ success: true, engine: "langextract", directCreated: 0, reviewCreated: 0, hiddenHistory: 0, discarded: 0 }) }; + } + if (url.includes("actions")) return { ok: true, json: async () => ({ actions: [] }) }; + if (url.includes("frontier")) return { ok: true, json: async () => ({ frontier: [] }) }; + if (url.includes("inbox")) return { ok: true, json: async () => ({ items: [] }) }; return { ok: true, json: async () => ({ success: true, envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: false } }) }; }; sandbox.state.activeTab = "actions"; @@ -1301,10 +1309,16 @@ describe("viewer session rendering", () => { target.getAttribute = (name: string) => name === "data-action" ? "save-todo-config" : null; target.closest = (selector: string) => selector === "[data-action]" ? target : null; dispatchDocumentClick(target); - await waitFor(() => sandbox.state.actions.extractMessage === "Config saved. Restart the service to apply it."); + await waitFor(() => sandbox.state.actions.extractMessage === "Config saved. The next organize run will use it now."); expect(posts[0]).toMatchObject({ LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS: "120000", LANGEXTRACT_API_KEY: "secret" }); - expect(sandbox.state.actions.extractMessage).toBe("Config saved. Restart the service to apply it."); + expect(sandbox.state.actions.extractMessage).toBe("Config saved. The next organize run will use it now."); + expect(sandbox.state.actions.forceNextExtract).toBe(true); + + sandbox.startTodoExtraction(false); + await waitFor(() => extractPosts.length === 1); + expect(extractPosts[0]).toMatchObject({ force: true }); + expect(sandbox.state.actions.forceNextExtract).toBe(false); }); it("keeps unsaved todo extractor config while the settings panel rerenders", () => { From e78b84e976f1beb7b2a37397932ae6a36ff8318b Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Thu, 25 Jun 2026 13:36:18 +0800 Subject: [PATCH 2/5] fix: keep todo LLM extraction retryable --- package.json | 2 + src/config.ts | 17 +++++++ src/functions/todo-extract.ts | 23 +++++++--- src/triggers/api.ts | 2 +- src/viewer/index.html | 9 +++- src/viewer/parts/app/05-i18n.js | 8 +++- src/viewer/parts/app/60-actions-todo.js | 1 + src/viewer/server.ts | 2 +- test/package-scripts.test.ts | 15 +++++++ test/review-action.test.ts | 3 +- test/todo-extract.test.ts | 60 ++++++++++++++++++++++++- test/todo-extractor-config.test.ts | 9 ++++ test/viewer-session-id.test.ts | 8 ++-- 13 files changed, 140 insertions(+), 19 deletions(-) create mode 100644 test/package-scripts.test.ts diff --git a/package.json b/package.json index bd57dbc9..931f6b9e 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "scripts": { "build": "tsdown && node scripts/build-viewer-html.mjs --check && (cp iii-config.yaml dist/ 2>/dev/null || true) && (cp iii-config.docker.yaml dist/ 2>/dev/null || true) && (cp docker-compose.yml dist/ 2>/dev/null || true) && (cp .env.example dist/ 2>/dev/null || true) && mkdir -p dist/viewer/agent-avatars dist/viewer/demo dist/functions && cp src/viewer/index.html dist/viewer/ && cp src/viewer/favicon.svg dist/viewer/ && cp src/viewer/agent-avatars/* dist/viewer/agent-avatars/ && cp src/viewer/demo/* dist/viewer/demo/ && cp src/functions/todo-extract-langextract.py dist/functions/ && cp src/functions/todo-update-llm.py dist/functions/", "dev": "tsx src/index.ts", + "postinstall": "python3 -m pip install --user -r requirements-langextract.txt || python3 -m pip install -r requirements-langextract.txt || echo \"[agentmemory] LangExtract Python deps were not installed; install requirements-langextract.txt manually or set LANGEXTRACT_PYTHON.\"", "viewer:build": "node scripts/build-viewer-html.mjs", "viewer:watch": "node scripts/build-viewer-html.mjs --watch", "start": "node dist/cli.mjs", @@ -68,6 +69,7 @@ "iii-config.docker.yaml", "docker-compose.yml", ".env.example", + "requirements-langextract.txt", "LICENSE", "README.md", "AGENTS.md" diff --git a/src/config.ts b/src/config.ts index 3688b666..eb228071 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,4 @@ +import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; @@ -91,6 +92,7 @@ function maskSecret(value: string | undefined): string { export function getTodoExtractorUserConfig(): Record { const env = getMergedEnv(); + const runtime = detectLangExtractRuntime(env); return { AGENTMEMORY_TODO_EXTRACTOR: env["AGENTMEMORY_TODO_EXTRACTOR"] || "auto", LANGEXTRACT_PYTHON: env["LANGEXTRACT_PYTHON"] || "python3", @@ -106,6 +108,21 @@ export function getTodoExtractorUserConfig(): Record { env["AGENTMEMORY_TODO_EXTRACT_MAX_INTERACTIONS_PER_SESSION"] || String(DEFAULT_TODO_EXTRACT_MAX_INTERACTIONS), LANGEXTRACT_API_KEY_CONFIGURED: hasRealValue(env["LANGEXTRACT_API_KEY"]), LANGEXTRACT_API_KEY_MASKED: maskSecret(env["LANGEXTRACT_API_KEY"]), + LANGEXTRACT_RUNTIME_READY: runtime.ready, + LANGEXTRACT_RUNTIME_ERROR: runtime.error, + }; +} + +function detectLangExtractRuntime(env: Record): { ready: boolean; error: string } { + const python = env["LANGEXTRACT_PYTHON"] || "python3"; + const result = spawnSync(python, ["-c", "import langextract"], { + encoding: "utf8", + timeout: 3000, + }); + if (result.status === 0) return { ready: true, error: "" }; + return { + ready: false, + error: String(result.stderr || result.error?.message || "langextract unavailable").replace(/\s+/g, " ").trim(), }; } diff --git a/src/functions/todo-extract.ts b/src/functions/todo-extract.ts index d0726472..be5d95c3 100644 --- a/src/functions/todo-extract.ts +++ b/src/functions/todo-extract.ts @@ -947,7 +947,14 @@ function parseCheckpoint(cursor: string | undefined): Record { if (!cursor) return {}; try { const parsed = JSON.parse(cursor) as unknown; - return parsed && typeof parsed === "object" ? parsed as Record : {}; + if (!parsed || typeof parsed !== "object") return {}; + const raw = parsed as Record; + if (raw.__engine !== "langextract") return {}; + const out: Record = {}; + for (const [key, value] of Object.entries(raw)) { + if (key !== "__engine" && typeof value === "string") out[key] = value; + } + return out; } catch { return {}; } @@ -2001,14 +2008,16 @@ export async function generateTodosFromSessions( existing.add(titleKey); seenTitles.push(titleKey); } - processed[session.id] = key; + if (engine === "langextract" && !fallbackReason) processed[session.id] = key; } - await kv.set(KV.scanCheckpoints, checkpointId, { - sourceId: checkpointId, - cursor: JSON.stringify(processed), - lastSuccessAt: now, - }); + if (Object.keys(processed).length) { + await kv.set(KV.scanCheckpoints, checkpointId, { + sourceId: checkpointId, + cursor: JSON.stringify({ __engine: "langextract", ...processed }), + lastSuccessAt: now, + }); + } return { success: true, diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 5f683d6a..891f1d96 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -522,7 +522,7 @@ export function registerApiTriggers( success: true, envPath: getUserEnvPath(), config: getTodoExtractorUserConfig(), - restartRequired: true, + restartRequired: false, }, }; }, diff --git a/src/viewer/index.html b/src/viewer/index.html index e88ca85e..740a9627 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -4319,7 +4319,7 @@

AI Todo

'act.empty.title': 'No todos yet', 'act.empty.lead': 'This is where todos, blocked items, and completed work extracted from your sessions will appear.', 'settings.title': 'Settings', - 'settings.subtitle': 'Local configuration is written to the user config file and takes effect after restarting the service.', + 'settings.subtitle': 'Local configuration is written to the user config file and applies on the next organize run.', 'settings.close': 'Close', 'settings.language': 'UI language', 'settings.extractor': 'LLM extraction config', @@ -4328,6 +4328,8 @@

AI Todo

'settings.apiKeyKeep': 'Enter a new API key to replace it, or leave blank to keep the current key', 'settings.apiKeyMissing': 'Not configured', 'settings.apiKeyLabel': 'API key:', + 'settings.runtimeReady': 'LLM runtime ready', + 'settings.runtimeMissing': 'LLM runtime missing', 'settings.save': 'Save config', 'settings.saving': 'Saving...', 'settings.savedReady': 'Config saved. The next organize run will use it now.', @@ -4547,7 +4549,7 @@

AI Todo

'act.empty.title': '还没有待办', 'act.empty.lead': '这里会放从会话里整理出的待办、卡住事项和已完成事项。', 'settings.title': '设置', - 'settings.subtitle': '本机配置会写入用户配置文件,重启服务后生效。', + 'settings.subtitle': '本机配置会写入用户配置文件,下次整理会直接使用。', 'settings.close': '关闭', 'settings.language': '界面语言', 'settings.extractor': '大模型抽取配置', @@ -4556,6 +4558,8 @@

AI Todo

'settings.apiKeyKeep': '输入新 API key 覆盖,留空保持不变', 'settings.apiKeyMissing': '未配置', 'settings.apiKeyLabel': 'API key:', + 'settings.runtimeReady': '大模型运行环境已就绪', + 'settings.runtimeMissing': '大模型运行环境缺失', 'settings.save': '保存配置', 'settings.saving': '保存中...', 'settings.savedReady': '配置已保存,下次整理会立即使用新配置。', @@ -9011,6 +9015,7 @@

AI Todo

html += '
'; if (maskedKey) html += '
' + esc(t('settings.apiKeyLabel')) + ' ' + esc(maskedKey) + '
'; html += '
'; + html += '
' + esc(cfg.LANGEXTRACT_RUNTIME_READY ? t('settings.runtimeReady') : t('settings.runtimeMissing')) + (cfg.LANGEXTRACT_RUNTIME_ERROR ? ': ' + esc(cfg.LANGEXTRACT_RUNTIME_ERROR) : '') + '
'; html += '
'; html += ''; if (state.actions.config && state.actions.config.envPath) html += '' + esc(state.actions.config.envPath) + ''; diff --git a/src/viewer/parts/app/05-i18n.js b/src/viewer/parts/app/05-i18n.js index 94f729a2..b10586b7 100644 --- a/src/viewer/parts/app/05-i18n.js +++ b/src/viewer/parts/app/05-i18n.js @@ -118,7 +118,7 @@ 'act.empty.title': 'No todos yet', 'act.empty.lead': 'This is where todos, blocked items, and completed work extracted from your sessions will appear.', 'settings.title': 'Settings', - 'settings.subtitle': 'Local configuration is written to the user config file and takes effect after restarting the service.', + 'settings.subtitle': 'Local configuration is written to the user config file and applies on the next organize run.', 'settings.close': 'Close', 'settings.language': 'UI language', 'settings.extractor': 'LLM extraction config', @@ -127,6 +127,8 @@ 'settings.apiKeyKeep': 'Enter a new API key to replace it, or leave blank to keep the current key', 'settings.apiKeyMissing': 'Not configured', 'settings.apiKeyLabel': 'API key:', + 'settings.runtimeReady': 'LLM runtime ready', + 'settings.runtimeMissing': 'LLM runtime missing', 'settings.save': 'Save config', 'settings.saving': 'Saving...', 'settings.savedReady': 'Config saved. The next organize run will use it now.', @@ -346,7 +348,7 @@ 'act.empty.title': '还没有待办', 'act.empty.lead': '这里会放从会话里整理出的待办、卡住事项和已完成事项。', 'settings.title': '设置', - 'settings.subtitle': '本机配置会写入用户配置文件,重启服务后生效。', + 'settings.subtitle': '本机配置会写入用户配置文件,下次整理会直接使用。', 'settings.close': '关闭', 'settings.language': '界面语言', 'settings.extractor': '大模型抽取配置', @@ -355,6 +357,8 @@ 'settings.apiKeyKeep': '输入新 API key 覆盖,留空保持不变', 'settings.apiKeyMissing': '未配置', 'settings.apiKeyLabel': 'API key:', + 'settings.runtimeReady': '大模型运行环境已就绪', + 'settings.runtimeMissing': '大模型运行环境缺失', 'settings.save': '保存配置', 'settings.saving': '保存中...', 'settings.savedReady': '配置已保存,下次整理会立即使用新配置。', diff --git a/src/viewer/parts/app/60-actions-todo.js b/src/viewer/parts/app/60-actions-todo.js index 366539ba..b0d26afd 100644 --- a/src/viewer/parts/app/60-actions-todo.js +++ b/src/viewer/parts/app/60-actions-todo.js @@ -170,6 +170,7 @@ html += '
'; if (maskedKey) html += '
' + esc(t('settings.apiKeyLabel')) + ' ' + esc(maskedKey) + '
'; html += '
'; + html += '
' + esc(cfg.LANGEXTRACT_RUNTIME_READY ? t('settings.runtimeReady') : t('settings.runtimeMissing')) + (cfg.LANGEXTRACT_RUNTIME_ERROR ? ': ' + esc(cfg.LANGEXTRACT_RUNTIME_ERROR) : '') + '
'; html += '
'; html += ''; if (state.actions.config && state.actions.config.envPath) html += '' + esc(state.actions.config.envPath) + ''; diff --git a/src/viewer/server.ts b/src/viewer/server.ts index 76a493a3..f12dbd42 100644 --- a/src/viewer/server.ts +++ b/src/viewer/server.ts @@ -1560,7 +1560,7 @@ export function startViewerServer( success: true, envPath: getUserEnvPath(), config: getTodoExtractorUserConfig(), - restartRequired: true, + restartRequired: false, }, req); return; } diff --git a/test/package-scripts.test.ts b/test/package-scripts.test.ts new file mode 100644 index 00000000..9dfedb7e --- /dev/null +++ b/test/package-scripts.test.ts @@ -0,0 +1,15 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOT = join(import.meta.dirname, ".."); + +describe("package scripts", () => { + it("installs LangExtract Python dependencies during npm install", () => { + const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf-8")) as { + scripts?: Record; + }; + + expect(pkg.scripts?.postinstall).toContain("requirements-langextract.txt"); + }); +}); diff --git a/test/review-action.test.ts b/test/review-action.test.ts index 1ed5972c..e6965dd8 100644 --- a/test/review-action.test.ts +++ b/test/review-action.test.ts @@ -159,7 +159,7 @@ describe("review action candidates", () => { const oldKey = process.env.LANGEXTRACT_API_KEY; process.env.LANGEXTRACT_MODEL = "deepseek/deepseek-v4-flash"; process.env.LANGEXTRACT_API_KEY = "secret"; - const response = await sdk.trigger("api::todo-extractor-config", req()) as { status_code: number; body: { success: boolean; config: Record; envPath: string } }; + const response = await sdk.trigger("api::todo-extractor-config", req()) as { status_code: number; body: { success: boolean; config: Record; envPath: string; restartRequired: boolean } }; if (oldModel === undefined) delete process.env.LANGEXTRACT_MODEL; else process.env.LANGEXTRACT_MODEL = oldModel; if (oldKey === undefined) delete process.env.LANGEXTRACT_API_KEY; @@ -171,6 +171,7 @@ describe("review action candidates", () => { expect(response.body.config.LANGEXTRACT_MODEL).toBe("deepseek/deepseek-v4-flash"); expect(response.body.config.LANGEXTRACT_API_KEY).toBeUndefined(); expect(response.body.config.LANGEXTRACT_API_KEY_CONFIGURED).toBe(true); + expect(response.body.restartRequired).toBe(false); }); it("rejects invalid todo extraction limits through the API", async () => { diff --git a/test/todo-extract.test.ts b/test/todo-extract.test.ts index f083defb..d7bd96c6 100644 --- a/test/todo-extract.test.ts +++ b/test/todo-extract.test.ts @@ -24,7 +24,7 @@ vi.mock("../src/config.js", () => ({ })); import { cleanPollutedTodoCards, updateChangedTodoCards, cleanTodoTitle, generateTodosFromSessions, refreshTodoAction, validateTodoEvidence, runLangExtractSidecar, type ExtractedTodo } from "../src/functions/todo-extract.js"; -import type { Action, CompressedObservation, ReviewQueueItem, Session } from "../src/types.js"; +import type { Action, CompressedObservation, ReviewQueueItem, ScanCheckpoint, Session } from "../src/types.js"; import { KV } from "../src/state/schema.js"; import { mockKV } from "./helpers/mocks.js"; @@ -178,17 +178,73 @@ describe("todo extraction", () => { }); it("uses scan checkpoints to skip unchanged sessions", async () => { + process.env.AGENTMEMORY_TODO_EXTRACTOR = "langextract"; await kv.set(KV.sessions, "ses_1", session()); await kv.set(KV.observations("ses_1"), "obs_1", obs()); - await generateTodosFromSessions(kv as never, { force: true, scanSources: false }); + await generateTodosFromSessions(kv as never, { + force: true, + scanSources: false, + runLangExtractSidecar: async () => [{ + title: "修复 CI 失败并重新跑测试", + description: "下一步请修复 CI 失败,并重新跑测试。", + confidence: 0.95, + timeBucket: "current", + typeBucket: "follow_up", + sourceSessionId: "ses_1", + evidence: { sourceObservationId: "obs_1", quote: "下一步请修复 CI 失败,并重新跑测试。" }, + dedupeKey: "fix-ci", + }], + }); const second = await generateTodosFromSessions(kv as never, { scanSources: false }); + delete process.env.AGENTMEMORY_TODO_EXTRACTOR; expect(second.scannedObservations).toBe(0); expect(second.directCreated).toBe(0); expect(await kv.list(KV.actions)).toHaveLength(1); }); + it("does not write scan checkpoints when LangExtract falls back to rules", async () => { + process.env.AGENTMEMORY_TODO_EXTRACTOR = "langextract"; + await kv.set(KV.sessions, "ses_1", session()); + await kv.set(KV.observations("ses_1"), "obs_1", obs()); + + const first = await generateTodosFromSessions(kv as never, { + force: true, + scanSources: false, + runLangExtractSidecar: async () => { + throw new Error("langextract unavailable: No module named 'langextract'"); + }, + }); + expect(await kv.get(KV.scanCheckpoints, "todo-extract:all")).toBeNull(); + const second = await generateTodosFromSessions(kv as never, { + scanSources: false, + runLangExtractSidecar: async () => [], + }); + delete process.env.AGENTMEMORY_TODO_EXTRACTOR; + + expect(first).toMatchObject({ engine: "rules", llmFallback: true }); + expect(second.scannedObservations).toBe(1); + }); + + it("ignores legacy untagged scan checkpoints before LangExtract runs", async () => { + process.env.AGENTMEMORY_TODO_EXTRACTOR = "langextract"; + await kv.set(KV.sessions, "ses_1", session()); + await kv.set(KV.observations("ses_1"), "obs_1", obs()); + await kv.set(KV.scanCheckpoints, "todo-extract:all", { + sourceId: "todo-extract:all", + cursor: JSON.stringify({ ses_1: "2026-06-17T09:00:00.000Z:1" }), + }); + + const result = await generateTodosFromSessions(kv as never, { + scanSources: false, + runLangExtractSidecar: async () => [], + }); + delete process.env.AGENTMEMORY_TODO_EXTRACTOR; + + expect(result.scannedObservations).toBe(1); + }); + it("discards medium-confidence rule todos instead of sending them to review", async () => { await kv.set(KV.sessions, "ses_1", session()); await kv.set(KV.observations("ses_1"), "obs_1", obs({ narrative: "下一步请修复 CI 失败,并重新跑测试。" })); diff --git a/test/todo-extractor-config.test.ts b/test/todo-extractor-config.test.ts index 16444373..1ef6400a 100644 --- a/test/todo-extractor-config.test.ts +++ b/test/todo-extractor-config.test.ts @@ -71,6 +71,15 @@ describe("todo extractor user config", () => { expect(cfg.LANGEXTRACT_BASE_URL).toBe("https://api.novita.ai/openai/v1"); }); + it("exposes whether the LangExtract runtime has the Python dependency", async () => { + const { getTodoExtractorUserConfig } = await freshConfig(); + + const cfg = getTodoExtractorUserConfig(); + + expect(cfg.LANGEXTRACT_RUNTIME_READY).toBeTypeOf("boolean"); + expect(cfg.LANGEXTRACT_RUNTIME_ERROR).toBeTypeOf("string"); + }); + it("round-trips the LLM extract timeout: defaults when unset, persists + reads back when set", async () => { const { getTodoExtractorUserConfig, getUserEnvPath, writeUserEnv } = await freshConfig(); diff --git a/test/viewer-session-id.test.ts b/test/viewer-session-id.test.ts index dea530a6..f86e48f1 100644 --- a/test/viewer-session-id.test.ts +++ b/test/viewer-session-id.test.ts @@ -1272,7 +1272,7 @@ describe("viewer session rendering", () => { const url = String(input); if (url.includes("config/todo-extractor") && init?.body) { posts.push(JSON.parse(init.body)); - return { ok: true, json: async () => ({ success: true, envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: true } }) }; + return { ok: true, json: async () => ({ success: true, envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: true, LANGEXTRACT_RUNTIME_READY: true, LANGEXTRACT_RUNTIME_ERROR: "" } }) }; } if (url.includes("todo-extract/generate") && init?.body) { extractPosts.push(JSON.parse(init.body)); @@ -1281,7 +1281,7 @@ describe("viewer session rendering", () => { if (url.includes("actions")) return { ok: true, json: async () => ({ actions: [] }) }; if (url.includes("frontier")) return { ok: true, json: async () => ({ frontier: [] }) }; if (url.includes("inbox")) return { ok: true, json: async () => ({ items: [] }) }; - return { ok: true, json: async () => ({ success: true, envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: false } }) }; + return { ok: true, json: async () => ({ success: true, envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: false, LANGEXTRACT_RUNTIME_READY: false, LANGEXTRACT_RUNTIME_ERROR: "No module named langextract" } }) }; }; sandbox.state.activeTab = "actions"; sandbox.state.actions = { @@ -1291,7 +1291,7 @@ describe("viewer session rendering", () => { statusFilter: "", search: "", reviewItems: [], - config: { envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: false } }, + config: { envPath: "/tmp/.env", config: { LANGEXTRACT_MODEL: "deepseek/deepseek-v4-flash", LANGEXTRACT_API_KEY_CONFIGURED: false, LANGEXTRACT_RUNTIME_READY: false, LANGEXTRACT_RUNTIME_ERROR: "No module named langextract" } }, }; sandbox.state.inbox = { loaded: true, items: [] }; sandbox.renderActions(); @@ -1300,6 +1300,8 @@ describe("viewer session rendering", () => { sandbox.state.settings.open = true; sandbox.renderSettingsPanel(); expect(getElement("settings-panel").innerHTML).toContain("LLM extraction config"); + expect(getElement("settings-panel").innerHTML).toContain("LLM runtime missing"); + expect(getElement("settings-panel").innerHTML).not.toContain("restart"); expect(getElement("settings-panel").innerHTML).not.toContain("pa/gpt-5.5"); getElement("todo-config-LANGEXTRACT_MODEL").value = "deepseek/deepseek-v4-flash"; From 25467e8721dfcd6313fe4696907cee0183f99b4c Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Thu, 25 Jun 2026 13:46:17 +0800 Subject: [PATCH 3/5] fix: install LangExtract in managed venv --- .gitignore | 1 + README.md | 2 +- package.json | 3 ++- scripts/install-langextract.mjs | 32 ++++++++++++++++++++++++++++++ src/config.ts | 22 +++++++++++++++++--- src/functions/todo-extract.ts | 5 +++-- test/package-scripts.test.ts | 5 ++++- test/todo-extractor-config.test.ts | 20 ++++++++++++++++++- 8 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 scripts/install-langextract.mjs diff --git a/.gitignore b/.gitignore index a90b79aa..e06b9974 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +.agentmemory-python/ dist/ artifacts/ *.tsbuildinfo diff --git a/README.md b/README.md index d24ca529..4253c999 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Then refresh the viewer: the dashboard fills with browsable sessions, memory, an - **Codex sessions (local):** the daemon scans your Codex session directories (`~/.codex/sessions` and `~/.codex/archived_sessions`) on startup and on an interval. Toggle with `AGENTMEMORY_CODEX_AUTOSCAN=false`; tune the cadence with `AGENTMEMORY_CODEX_SCAN_INTERVAL_MS` (default 5 min). You can also import a transcript on demand from a source checkout with `node dist/cli.mjs import-jsonl ` (installed package binary: `agentmemory-lab import-jsonl `). - **Browser AI conversations:** load the browser extension under [`browser-extension/`](browser-extension/); it captures supported AI sites and posts them to the local daemon, which extracts todos into the same queue. -- **LangExtract extraction (optional):** install Python deps with `python3 -m pip install -r requirements-langextract.txt` (includes SOCKS proxy support). Configure it during first-run setup (`node dist/cli.mjs --reset` from source, or `agentmemory-lab --reset` when installed), from the viewer Settings panel, or in `~/.agentmemory/.env`: `AGENTMEMORY_TODO_EXTRACTOR=langextract`, `LANGEXTRACT_PYTHON=/path/to/python`, `LANGEXTRACT_PROVIDER=openai`, `LANGEXTRACT_MODEL=deepseek/deepseek-v4-flash`, `LANGEXTRACT_BASE_URL=https://api.novita.ai/openai/v1`, `LANGEXTRACT_API_KEY=`, and optionally `AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS=120000`. Trigger it from the To-Do tab or with `POST /agentmemory/todo-extract/generate`. +- **LangExtract extraction (optional):** `npm install` creates a project-local `.agentmemory-python` environment and installs the Python deps from `requirements-langextract.txt` (includes SOCKS proxy support). Configure it during first-run setup (`node dist/cli.mjs --reset` from source, or `agentmemory-lab --reset` when installed), from the viewer Settings panel, or in `~/.agentmemory/.env`: `AGENTMEMORY_TODO_EXTRACTOR=langextract`, `LANGEXTRACT_PROVIDER=openai`, `LANGEXTRACT_MODEL=deepseek/deepseek-v4-flash`, `LANGEXTRACT_BASE_URL=https://api.novita.ai/openai/v1`, `LANGEXTRACT_API_KEY=`, and optionally `AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS=120000`. Set `LANGEXTRACT_PYTHON=/path/to/python` only when you want to override the managed environment. Trigger it from the To-Do tab or with `POST /agentmemory/todo-extract/generate`. - **Isolated data roots:** set `AGENTMEMORY_HOME=/path/to/data` to keep AI Todo runtime files (`.env`, preferences, pidfiles, snapshots) separate from your normal `~/.agentmemory`. This does not change the Codex session source; local Codex history is still read from `~/.codex/sessions` and `~/.codex/archived_sessions`. - **First-run setup:** the first-run CLI model choice configures To-Do extraction — it seeds the `LANGEXTRACT_*` settings above for the model you pick (or keeps the rules extractor if you skip). The legacy memory compression/consolidation/embeddings provider is an advanced `.env`-only setting. diff --git a/package.json b/package.json index 931f6b9e..7220bc47 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "scripts": { "build": "tsdown && node scripts/build-viewer-html.mjs --check && (cp iii-config.yaml dist/ 2>/dev/null || true) && (cp iii-config.docker.yaml dist/ 2>/dev/null || true) && (cp docker-compose.yml dist/ 2>/dev/null || true) && (cp .env.example dist/ 2>/dev/null || true) && mkdir -p dist/viewer/agent-avatars dist/viewer/demo dist/functions && cp src/viewer/index.html dist/viewer/ && cp src/viewer/favicon.svg dist/viewer/ && cp src/viewer/agent-avatars/* dist/viewer/agent-avatars/ && cp src/viewer/demo/* dist/viewer/demo/ && cp src/functions/todo-extract-langextract.py dist/functions/ && cp src/functions/todo-update-llm.py dist/functions/", "dev": "tsx src/index.ts", - "postinstall": "python3 -m pip install --user -r requirements-langextract.txt || python3 -m pip install -r requirements-langextract.txt || echo \"[agentmemory] LangExtract Python deps were not installed; install requirements-langextract.txt manually or set LANGEXTRACT_PYTHON.\"", + "postinstall": "node scripts/install-langextract.mjs", "viewer:build": "node scripts/build-viewer-html.mjs", "viewer:watch": "node scripts/build-viewer-html.mjs --watch", "start": "node dist/cli.mjs", @@ -65,6 +65,7 @@ "files": [ "dist/", "plugin/", + "scripts/install-langextract.mjs", "iii-config.yaml", "iii-config.docker.yaml", "docker-compose.yml", diff --git a/scripts/install-langextract.mjs b/scripts/install-langextract.mjs new file mode 100644 index 00000000..e9e9e45a --- /dev/null +++ b/scripts/install-langextract.mjs @@ -0,0 +1,32 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const root = process.cwd(); +const requirements = join(root, "requirements-langextract.txt"); +const venv = join(root, ".agentmemory-python"); +const python = process.platform === "win32" ? join(venv, "Scripts", "python.exe") : join(venv, "bin", "python"); + +function run(command, args) { + return spawnSync(command, args, { stdio: "inherit" }); +} + +if (!existsSync(requirements)) { + process.stderr.write("[agentmemory] requirements-langextract.txt not found; skipping LangExtract setup.\n"); + process.exit(0); +} + +if (!existsSync(python)) { + const created = run("python3", ["-m", "venv", venv]); + if (created.status !== 0) { + process.stderr.write("[agentmemory] Could not create .agentmemory-python venv; set LANGEXTRACT_PYTHON manually.\n"); + process.exit(0); + } +} + +const installed = run(python, ["-m", "pip", "install", "-r", requirements]); +if (installed.status !== 0) { + process.stderr.write("[agentmemory] LangExtract Python deps were not installed; set LANGEXTRACT_PYTHON manually.\n"); + process.exit(0); +} + diff --git a/src/config.ts b/src/config.ts index eb228071..6189ab1a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,8 @@ import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { homedir } from "node:os"; +import { fileURLToPath } from "node:url"; import type { AgentMemoryConfig, ProviderConfig, @@ -95,7 +96,7 @@ export function getTodoExtractorUserConfig(): Record { const runtime = detectLangExtractRuntime(env); return { AGENTMEMORY_TODO_EXTRACTOR: env["AGENTMEMORY_TODO_EXTRACTOR"] || "auto", - LANGEXTRACT_PYTHON: env["LANGEXTRACT_PYTHON"] || "python3", + LANGEXTRACT_PYTHON: resolveLangExtractPython(env), LANGEXTRACT_MODEL: normalizeTodoExtractorModel(env["LANGEXTRACT_MODEL"]), LANGEXTRACT_PROVIDER: normalizeTodoExtractorProvider(env["LANGEXTRACT_PROVIDER"]), LANGEXTRACT_BASE_URL: env["LANGEXTRACT_BASE_URL"] || DEFAULT_LANGEXTRACT_BASE_URL, @@ -114,7 +115,7 @@ export function getTodoExtractorUserConfig(): Record { } function detectLangExtractRuntime(env: Record): { ready: boolean; error: string } { - const python = env["LANGEXTRACT_PYTHON"] || "python3"; + const python = resolveLangExtractPython(env); const result = spawnSync(python, ["-c", "import langextract"], { encoding: "utf8", timeout: 3000, @@ -126,6 +127,21 @@ function detectLangExtractRuntime(env: Record): { ready: boolean }; } +export function resolveLangExtractPython(env?: Record): string { + const source = env ?? getMergedEnv(); + const configured = source["LANGEXTRACT_PYTHON"]?.trim(); + if (configured && configured !== "python3") return configured; + const moduleRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const candidates = [process.cwd(), moduleRoot].map((root) => + process.platform === "win32" + ? join(root, ".agentmemory-python", "Scripts", "python.exe") + : join(root, ".agentmemory-python", "bin", "python"), + ); + const local = candidates.find((candidate) => existsSync(candidate)); + if (local) return local; + return configured || "python3"; +} + export function normalizeTodoExtractorModel(value: string | undefined): string { const model = value?.trim(); return model && !LEGACY_LANGEXTRACT_MODELS.has(model) ? model : DEFAULT_LANGEXTRACT_MODEL; diff --git a/src/functions/todo-extract.ts b/src/functions/todo-extract.ts index be5d95c3..0401dc7a 100644 --- a/src/functions/todo-extract.ts +++ b/src/functions/todo-extract.ts @@ -15,6 +15,7 @@ import { getEnvVar, normalizeTodoExtractorModel, normalizeTodoExtractorProvider, + resolveLangExtractPython, } from "../config.js"; import { scanCodexSource } from "./source-scan-codex.js"; import { @@ -821,7 +822,7 @@ export async function runLangExtractSidecar( ): Promise { const script = sidecarPath(); if (!script) throw new Error("langextract sidecar not found"); - const python = getEnvVar("LANGEXTRACT_PYTHON") || "python3"; + const python = resolveLangExtractPython(); const env = { ...process.env }; for (const key of SIDE_CAR_ENV_KEYS) { const value = getEnvVar(key); @@ -1268,7 +1269,7 @@ async function runCleanupSidecar( ): Promise { const script = sidecarPath(CLEANUP_SIDE_CAR); if (!script) throw new Error("cleanup sidecar not found"); - const python = getEnvVar("LANGEXTRACT_PYTHON") || "python3"; + const python = resolveLangExtractPython(); const env = { ...process.env }; for (const key of SIDE_CAR_ENV_KEYS) { const value = getEnvVar(key); diff --git a/test/package-scripts.test.ts b/test/package-scripts.test.ts index 9dfedb7e..c4d7bc22 100644 --- a/test/package-scripts.test.ts +++ b/test/package-scripts.test.ts @@ -7,9 +7,12 @@ const ROOT = join(import.meta.dirname, ".."); describe("package scripts", () => { it("installs LangExtract Python dependencies during npm install", () => { const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf-8")) as { + files?: string[]; scripts?: Record; }; - expect(pkg.scripts?.postinstall).toContain("requirements-langextract.txt"); + expect(pkg.scripts?.postinstall).toBe("node scripts/install-langextract.mjs"); + expect(pkg.files).toContain("scripts/install-langextract.mjs"); + expect(readFileSync(join(ROOT, "scripts/install-langextract.mjs"), "utf-8")).toContain(".agentmemory-python"); }); }); diff --git a/test/todo-extractor-config.test.ts b/test/todo-extractor-config.test.ts index 1ef6400a..61553b1d 100644 --- a/test/todo-extractor-config.test.ts +++ b/test/todo-extractor-config.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -80,6 +80,24 @@ describe("todo extractor user config", () => { expect(cfg.LANGEXTRACT_RUNTIME_ERROR).toBeTypeOf("string"); }); + it("defaults LangExtract Python to the project-managed venv when it exists", async () => { + const originalCwd = process.cwd(); + const projectRoot = mkdtempSync(join(tmpdir(), "agentmemory-langextract-project-")); + const venvPython = join(projectRoot, ".agentmemory-python", "bin", "python"); + mkdirSync(join(projectRoot, ".agentmemory-python", "bin"), { recursive: true }); + writeFileSync(venvPython, "#!/bin/sh\n"); + const expectedPython = join(realpathSync(projectRoot), ".agentmemory-python", "bin", "python"); + process.chdir(projectRoot); + try { + const { resolveLangExtractPython } = await freshConfig(); + expect(resolveLangExtractPython({})).toBe(expectedPython); + expect(resolveLangExtractPython({ LANGEXTRACT_PYTHON: "__custom_python__" })).toBe("__custom_python__"); + } finally { + process.chdir(originalCwd); + rmSync(projectRoot, { recursive: true, force: true }); + } + }); + it("round-trips the LLM extract timeout: defaults when unset, persists + reads back when set", async () => { const { getTodoExtractorUserConfig, getUserEnvPath, writeUserEnv } = await freshConfig(); From 4c9fcaec8fb79497de25aa1d259b9a62520aa9dd Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Thu, 25 Jun 2026 14:04:27 +0800 Subject: [PATCH 4/5] fix: keep long todo extraction status accurate --- src/config.ts | 6 +++++- src/viewer/server.ts | 12 +++++++++++- test/todo-extractor-config.test.ts | 8 ++++++++ test/viewer-security.test.ts | 27 +++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 6189ab1a..ac3622ad 100644 --- a/src/config.ts +++ b/src/config.ts @@ -116,7 +116,7 @@ export function getTodoExtractorUserConfig(): Record { function detectLangExtractRuntime(env: Record): { ready: boolean; error: string } { const python = resolveLangExtractPython(env); - const result = spawnSync(python, ["-c", "import langextract"], { + const result = spawnSync(python, ["-c", detectLangExtractRuntimeProbe()], { encoding: "utf8", timeout: 3000, }); @@ -127,6 +127,10 @@ function detectLangExtractRuntime(env: Record): { ready: boolean }; } +export function detectLangExtractRuntimeProbe(): string { + return "import importlib.util; raise SystemExit(0 if importlib.util.find_spec('langextract') else 1)"; +} + export function resolveLangExtractPython(env?: Record): string { const source = env ?? getMergedEnv(); const configured = source["LANGEXTRACT_PYTHON"]?.trim(); diff --git a/src/viewer/server.ts b/src/viewer/server.ts index f12dbd42..075ac7c8 100644 --- a/src/viewer/server.ts +++ b/src/viewer/server.ts @@ -14,6 +14,7 @@ import type { Action, CompressedObservation, Memory, ReviewQueueItem, Session } import { KV, fingerprintId } from "../state/schema.js"; import { generateTodosFromSessions, refreshTodoAction, updateChangedTodoCards } from "../functions/todo-extract.js"; import { + DEFAULT_TODO_EXTRACT_MAX_SESSIONS, detectEmbeddingProvider, detectLlmProviderKind, getTodoExtractorUserConfig, @@ -648,7 +649,7 @@ async function readTodoExtractStatus(kv: ViewerKv): Promise { if (!parsed || typeof parsed !== "object" || !parsed.status) return { status: "idle" }; if (parsed.status !== "running") return parsed; const startedAt = parsed.startedAt ? Date.parse(parsed.startedAt) : NaN; - const maxAgeMs = proxyTimeoutMsForPath("/agentmemory/todo-extract/generate") + 30_000; + const maxAgeMs = todoExtractRunningMaxAgeMs(); if (!Number.isFinite(startedAt) || Date.now() - startedAt <= maxAgeMs) return parsed; return { status: "error", @@ -661,6 +662,15 @@ async function readTodoExtractStatus(kv: ViewerKv): Promise { } } +function todoExtractRunningMaxAgeMs(): number { + const perSessionMs = proxyTimeoutMsForPath("/agentmemory/todo-extract/generate"); + const rawMaxSessions = Number(process.env.AGENTMEMORY_TODO_EXTRACT_MAX_SESSIONS); + const maxSessions = Number.isFinite(rawMaxSessions) && rawMaxSessions > 0 + ? Math.min(100, Math.floor(rawMaxSessions)) + : DEFAULT_TODO_EXTRACT_MAX_SESSIONS; + return perSessionMs * maxSessions + 60_000; +} + async function writeTodoExtractStatus(kv: ViewerKv, status: TodoExtractStatus): Promise { await kv.set(KV.scanCheckpoints, TODO_EXTRACT_STATUS_ID, { sourceId: TODO_EXTRACT_STATUS_ID, diff --git a/test/todo-extractor-config.test.ts b/test/todo-extractor-config.test.ts index 61553b1d..e0a3cccb 100644 --- a/test/todo-extractor-config.test.ts +++ b/test/todo-extractor-config.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -80,6 +81,13 @@ describe("todo extractor user config", () => { expect(cfg.LANGEXTRACT_RUNTIME_ERROR).toBeTypeOf("string"); }); + it("checks LangExtract runtime readiness without importing the heavy package", async () => { + const { detectLangExtractRuntimeProbe } = await freshConfig(); + expect(detectLangExtractRuntimeProbe()).toContain("find_spec('langextract')"); + expect(detectLangExtractRuntimeProbe()).not.toContain("import langextract"); + expect(spawnSync("python3", ["-c", detectLangExtractRuntimeProbe()], { encoding: "utf8" }).status).toBeTypeOf("number"); + }); + it("defaults LangExtract Python to the project-managed venv when it exists", async () => { const originalCwd = process.cwd(); const projectRoot = mkdtempSync(join(tmpdir(), "agentmemory-langextract-project-")); diff --git a/test/viewer-security.test.ts b/test/viewer-security.test.ts index adbb37e9..a881a4fd 100644 --- a/test/viewer-security.test.ts +++ b/test/viewer-security.test.ts @@ -340,6 +340,33 @@ describe("viewer request handler DNS rebinding defence (e2e)", () => { } }); + it("keeps todo extraction running across one proxy timeout while the backend may still be processing more sessions", async () => { + const previousTimeout = process.env.AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS; + process.env.AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS = "1000"; + try { + const startedAt = new Date(Date.now() - 30 * 1000).toISOString(); + const { port } = await spinUpViewer({ + [KV.scanCheckpoints]: { + "todo-extract:status": { + sourceId: "todo-extract:status", + cursor: JSON.stringify({ status: "running", startedAt }), + }, + }, + }); + const actions = await request(port, `localhost:${port}`, "/agentmemory/actions"); + const actionsBody = JSON.parse(actions.body); + + expect(actions.status).toBe(200); + expect(actionsBody.todoExtract).toMatchObject({ status: "running", startedAt }); + } finally { + if (previousTimeout === undefined) { + delete process.env.AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS; + } else { + process.env.AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS = previousTimeout; + } + } + }); + it("generates todos through the viewer fallback endpoint when REST proxy misses", async () => { const previousExtractor = process.env.AGENTMEMORY_TODO_EXTRACTOR; const previousDirectConfidence = process.env.AGENTMEMORY_TODO_DIRECT_CONFIDENCE; From d80910474bacb5ce1e7e5d602d6ec608054fe4bc Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Thu, 25 Jun 2026 14:08:48 +0800 Subject: [PATCH 5/5] fix: label partial LLM extraction accurately --- src/viewer/index.html | 6 +++++- src/viewer/parts/app/05-i18n.js | 2 ++ src/viewer/parts/app/60-actions-todo.js | 4 +++- test/viewer-session-id.test.ts | 24 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/viewer/index.html b/src/viewer/index.html index 740a9627..332078dd 100644 --- a/src/viewer/index.html +++ b/src/viewer/index.html @@ -4275,6 +4275,7 @@

AI Todo

'act.extract.run': 'Organize with LLM', 'act.extract.running': 'Organizing...', 'act.extract.done': 'Organized', + 'act.extract.partial': 'Partially organized', 'act.extract.rules': 'LLM unavailable', 'act.extract.error': 'Organize failed', 'act.extract.failedExisting': 'Extraction failed; showing existing todos', @@ -4505,6 +4506,7 @@

AI Todo

'act.extract.run': '用大模型整理', 'act.extract.running': '整理中...', 'act.extract.done': '已整理', + 'act.extract.partial': '部分大模型整理', 'act.extract.rules': '未走大模型', 'act.extract.error': '整理失败', 'act.extract.failedExisting': '抽取失败,已显示现有待办', @@ -8933,6 +8935,7 @@

AI Todo

if (st.status === 'done' && st.summary) { state.actions.extractStatus = 'done'; state.actions.extractFallback = !todoExtractionUsedLlm(st.summary); + state.actions.extractPartial = st.summary && st.summary.engine === 'mixed'; state.actions.extractMessage = todoExtractionSummary(st.summary); } else if (st.status === 'error') { state.actions.extractStatus = 'error'; @@ -9184,6 +9187,7 @@

AI Todo

} state.actions.extractStatus = 'done'; state.actions.extractFallback = !todoExtractionUsedLlm(result); + state.actions.extractPartial = result && result.engine === 'mixed'; state.actions.extractMessage = todoExtractionSummary(result); if (actionsScrolledAway()) { state.actions.stale = delta > 0; @@ -9937,7 +9941,7 @@

AI Todo

var extractTitle = state.actions.extractMessage || t('act.extract.title'); var extractLabel = t('act.extract.run'); if (state.actions.extractInFlight) extractLabel = t('act.extract.running'); - else if (state.actions.extractStatus === 'done') extractLabel = state.actions.extractFallback ? t('act.extract.rules') : t('act.extract.done'); + else if (state.actions.extractStatus === 'done') extractLabel = state.actions.extractPartial ? t('act.extract.partial') : (state.actions.extractFallback ? t('act.extract.rules') : t('act.extract.done')); else if (state.actions.extractStatus === 'error') extractLabel = t('act.extract.error'); html += ''; var cleanupLabel = t('act.cleanup.run'); diff --git a/src/viewer/parts/app/05-i18n.js b/src/viewer/parts/app/05-i18n.js index b10586b7..90de04eb 100644 --- a/src/viewer/parts/app/05-i18n.js +++ b/src/viewer/parts/app/05-i18n.js @@ -74,6 +74,7 @@ 'act.extract.run': 'Organize with LLM', 'act.extract.running': 'Organizing...', 'act.extract.done': 'Organized', + 'act.extract.partial': 'Partially organized', 'act.extract.rules': 'LLM unavailable', 'act.extract.error': 'Organize failed', 'act.extract.failedExisting': 'Extraction failed; showing existing todos', @@ -304,6 +305,7 @@ 'act.extract.run': '用大模型整理', 'act.extract.running': '整理中...', 'act.extract.done': '已整理', + 'act.extract.partial': '部分大模型整理', 'act.extract.rules': '未走大模型', 'act.extract.error': '整理失败', 'act.extract.failedExisting': '抽取失败,已显示现有待办', diff --git a/src/viewer/parts/app/60-actions-todo.js b/src/viewer/parts/app/60-actions-todo.js index b0d26afd..c374f8a6 100644 --- a/src/viewer/parts/app/60-actions-todo.js +++ b/src/viewer/parts/app/60-actions-todo.js @@ -88,6 +88,7 @@ if (st.status === 'done' && st.summary) { state.actions.extractStatus = 'done'; state.actions.extractFallback = !todoExtractionUsedLlm(st.summary); + state.actions.extractPartial = st.summary && st.summary.engine === 'mixed'; state.actions.extractMessage = todoExtractionSummary(st.summary); } else if (st.status === 'error') { state.actions.extractStatus = 'error'; @@ -339,6 +340,7 @@ } state.actions.extractStatus = 'done'; state.actions.extractFallback = !todoExtractionUsedLlm(result); + state.actions.extractPartial = result && result.engine === 'mixed'; state.actions.extractMessage = todoExtractionSummary(result); if (actionsScrolledAway()) { state.actions.stale = delta > 0; @@ -1092,7 +1094,7 @@ var extractTitle = state.actions.extractMessage || t('act.extract.title'); var extractLabel = t('act.extract.run'); if (state.actions.extractInFlight) extractLabel = t('act.extract.running'); - else if (state.actions.extractStatus === 'done') extractLabel = state.actions.extractFallback ? t('act.extract.rules') : t('act.extract.done'); + else if (state.actions.extractStatus === 'done') extractLabel = state.actions.extractPartial ? t('act.extract.partial') : (state.actions.extractFallback ? t('act.extract.rules') : t('act.extract.done')); else if (state.actions.extractStatus === 'error') extractLabel = t('act.extract.error'); html += ''; var cleanupLabel = t('act.cleanup.run'); diff --git a/test/viewer-session-id.test.ts b/test/viewer-session-id.test.ts index f86e48f1..740d9609 100644 --- a/test/viewer-session-id.test.ts +++ b/test/viewer-session-id.test.ts @@ -1399,6 +1399,30 @@ describe("viewer session rendering", () => { expect(html).toContain("missing key"); }); + it("labels mixed extraction as partial instead of saying the LLM was unavailable", () => { + const { sandbox, getElement } = loadViewerSandbox(); + sandbox.state.activeTab = "actions"; + sandbox.state.actions = { + loaded: true, + items: [], + frontier: [], + statusFilter: "", + search: "", + reviewItems: [], + extractStatus: "done", + extractFallback: true, + extractPartial: true, + extractMessage: "Partial LLM extraction complete · reason: sidecar timed out", + }; + sandbox.state.inbox = { loaded: true, items: [] }; + sandbox.renderActions(); + + const html = getElement("view-actions").innerHTML; + expect(html).toContain("Partially organized"); + expect(html).not.toContain("LLM unavailable"); + expect(html).toContain("sidecar timed out"); + }); + it("filters actions from Todo and Done metric cards", () => { const { sandbox, getElement, dispatchDocumentClick } = loadViewerSandbox(); sandbox.state.activeTab = "actions";