Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
node_modules/
.agentmemory-python/
dist/
artifacts/
*.tsbuildinfo
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Then refresh the viewer: the dashboard fills with browsable sessions, memory, an

- **Codex sessions (local):** the daemon scans your Codex session directories (`~/.codex/sessions` and `~/.codex/archived_sessions`) on startup and on an interval. Toggle with `AGENTMEMORY_CODEX_AUTOSCAN=false`; tune the cadence with `AGENTMEMORY_CODEX_SCAN_INTERVAL_MS` (default 5 min). You can also import a transcript on demand from a source checkout with `node dist/cli.mjs import-jsonl <path>` (installed package binary: `agentmemory-lab import-jsonl <path>`).
- **Browser AI conversations:** load the browser extension under [`browser-extension/`](browser-extension/); it captures supported AI sites and posts them to the local daemon, which extracts todos into the same queue.
- **LangExtract extraction (optional):** install Python deps with `python3 -m pip install -r requirements-langextract.txt` (includes SOCKS proxy support). Configure it during first-run setup (`node dist/cli.mjs --reset` from source, or `agentmemory-lab --reset` when installed), from the viewer Settings panel, or in `~/.agentmemory/.env`: `AGENTMEMORY_TODO_EXTRACTOR=langextract`, `LANGEXTRACT_PYTHON=/path/to/python`, `LANGEXTRACT_PROVIDER=openai`, `LANGEXTRACT_MODEL=deepseek/deepseek-v4-flash`, `LANGEXTRACT_BASE_URL=https://api.novita.ai/openai/v1`, `LANGEXTRACT_API_KEY=<runtime secret>`, and optionally `AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS=120000`. Trigger it from the To-Do tab or with `POST /agentmemory/todo-extract/generate`.
- **LangExtract extraction (optional):** `npm install` creates a project-local `.agentmemory-python` environment and installs the Python deps from `requirements-langextract.txt` (includes SOCKS proxy support). Configure it during first-run setup (`node dist/cli.mjs --reset` from source, or `agentmemory-lab --reset` when installed), from the viewer Settings panel, or in `~/.agentmemory/.env`: `AGENTMEMORY_TODO_EXTRACTOR=langextract`, `LANGEXTRACT_PROVIDER=openai`, `LANGEXTRACT_MODEL=deepseek/deepseek-v4-flash`, `LANGEXTRACT_BASE_URL=https://api.novita.ai/openai/v1`, `LANGEXTRACT_API_KEY=<runtime secret>`, and optionally `AGENTMEMORY_TODO_EXTRACT_TIMEOUT_MS=120000`. Set `LANGEXTRACT_PYTHON=/path/to/python` only when you want to override the managed environment. Trigger it from the To-Do tab or with `POST /agentmemory/todo-extract/generate`.
- **Isolated data roots:** set `AGENTMEMORY_HOME=/path/to/data` to keep AI Todo runtime files (`.env`, preferences, pidfiles, snapshots) separate from your normal `~/.agentmemory`. This does not change the Codex session source; local Codex history is still read from `~/.codex/sessions` and `~/.codex/archived_sessions`.
- **First-run setup:** the first-run CLI model choice configures To-Do extraction — it seeds the `LANGEXTRACT_*` settings above for the model you pick (or keeps the rules extractor if you skip). The legacy memory compression/consolidation/embeddings provider is an advanced `.env`-only setting.

Expand Down
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"scripts": {
"build": "tsdown && node scripts/build-viewer-html.mjs --check && (cp iii-config.yaml dist/ 2>/dev/null || true) && (cp iii-config.docker.yaml dist/ 2>/dev/null || true) && (cp docker-compose.yml dist/ 2>/dev/null || true) && (cp .env.example dist/ 2>/dev/null || true) && mkdir -p dist/viewer/agent-avatars dist/viewer/demo dist/functions && cp src/viewer/index.html dist/viewer/ && cp src/viewer/favicon.svg dist/viewer/ && cp src/viewer/agent-avatars/* dist/viewer/agent-avatars/ && cp src/viewer/demo/* dist/viewer/demo/ && cp src/functions/todo-extract-langextract.py dist/functions/ && cp src/functions/todo-update-llm.py dist/functions/",
"dev": "tsx src/index.ts",
"postinstall": "node scripts/install-langextract.mjs",
"viewer:build": "node scripts/build-viewer-html.mjs",
"viewer:watch": "node scripts/build-viewer-html.mjs --watch",
"start": "node dist/cli.mjs",
Expand Down Expand Up @@ -64,10 +65,12 @@
"files": [
"dist/",
"plugin/",
"scripts/install-langextract.mjs",
"iii-config.yaml",
"iii-config.docker.yaml",
"docker-compose.yml",
".env.example",
"requirements-langextract.txt",
"LICENSE",
"README.md",
"AGENTS.md"
Expand Down
32 changes: 32 additions & 0 deletions scripts/install-langextract.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { existsSync } from "node:fs";
import { join } from "node:path";
import { spawnSync } from "node:child_process";

const root = process.cwd();
const requirements = join(root, "requirements-langextract.txt");
const venv = join(root, ".agentmemory-python");
const python = process.platform === "win32" ? join(venv, "Scripts", "python.exe") : join(venv, "bin", "python");

function run(command, args) {
return spawnSync(command, args, { stdio: "inherit" });
}

if (!existsSync(requirements)) {
process.stderr.write("[agentmemory] requirements-langextract.txt not found; skipping LangExtract setup.\n");
process.exit(0);
}

if (!existsSync(python)) {
const created = run("python3", ["-m", "venv", venv]);
if (created.status !== 0) {
process.stderr.write("[agentmemory] Could not create .agentmemory-python venv; set LANGEXTRACT_PYTHON manually.\n");
process.exit(0);
}
}

const installed = run(python, ["-m", "pip", "install", "-r", requirements]);
if (installed.status !== 0) {
process.stderr.write("[agentmemory] LangExtract Python deps were not installed; set LANGEXTRACT_PYTHON manually.\n");
process.exit(0);
}

41 changes: 39 additions & 2 deletions src/config.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { dirname, join, resolve } from "node:path";
import { homedir } from "node:os";
import { fileURLToPath } from "node:url";
import type {
AgentMemoryConfig,
ProviderConfig,
Expand Down Expand Up @@ -91,9 +93,10 @@ function maskSecret(value: string | undefined): string {

export function getTodoExtractorUserConfig(): Record<string, string | boolean> {
const env = getMergedEnv();
const runtime = detectLangExtractRuntime(env);
return {
AGENTMEMORY_TODO_EXTRACTOR: env["AGENTMEMORY_TODO_EXTRACTOR"] || "auto",
LANGEXTRACT_PYTHON: env["LANGEXTRACT_PYTHON"] || "python3",
LANGEXTRACT_PYTHON: resolveLangExtractPython(env),
LANGEXTRACT_MODEL: normalizeTodoExtractorModel(env["LANGEXTRACT_MODEL"]),
LANGEXTRACT_PROVIDER: normalizeTodoExtractorProvider(env["LANGEXTRACT_PROVIDER"]),
LANGEXTRACT_BASE_URL: env["LANGEXTRACT_BASE_URL"] || DEFAULT_LANGEXTRACT_BASE_URL,
Expand All @@ -106,9 +109,43 @@ export function getTodoExtractorUserConfig(): Record<string, string | boolean> {
env["AGENTMEMORY_TODO_EXTRACT_MAX_INTERACTIONS_PER_SESSION"] || String(DEFAULT_TODO_EXTRACT_MAX_INTERACTIONS),
LANGEXTRACT_API_KEY_CONFIGURED: hasRealValue(env["LANGEXTRACT_API_KEY"]),
LANGEXTRACT_API_KEY_MASKED: maskSecret(env["LANGEXTRACT_API_KEY"]),
LANGEXTRACT_RUNTIME_READY: runtime.ready,
LANGEXTRACT_RUNTIME_ERROR: runtime.error,
};
}

function detectLangExtractRuntime(env: Record<string, string>): { ready: boolean; error: string } {
const python = resolveLangExtractPython(env);
const result = spawnSync(python, ["-c", detectLangExtractRuntimeProbe()], {
encoding: "utf8",
timeout: 3000,
});
if (result.status === 0) return { ready: true, error: "" };
return {
ready: false,
error: String(result.stderr || result.error?.message || "langextract unavailable").replace(/\s+/g, " ").trim(),
};
}

export function detectLangExtractRuntimeProbe(): string {
return "import importlib.util; raise SystemExit(0 if importlib.util.find_spec('langextract') else 1)";
}

export function resolveLangExtractPython(env?: Record<string, string>): string {
const source = env ?? getMergedEnv();
const configured = source["LANGEXTRACT_PYTHON"]?.trim();
if (configured && configured !== "python3") return configured;
const moduleRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const candidates = [process.cwd(), moduleRoot].map((root) =>
process.platform === "win32"
? join(root, ".agentmemory-python", "Scripts", "python.exe")
: join(root, ".agentmemory-python", "bin", "python"),
);
const local = candidates.find((candidate) => existsSync(candidate));
if (local) return local;
return configured || "python3";
}

export function normalizeTodoExtractorModel(value: string | undefined): string {
const model = value?.trim();
return model && !LEGACY_LANGEXTRACT_MODELS.has(model) ? model : DEFAULT_LANGEXTRACT_MODEL;
Expand Down
28 changes: 19 additions & 9 deletions src/functions/todo-extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
getEnvVar,
normalizeTodoExtractorModel,
normalizeTodoExtractorProvider,
resolveLangExtractPython,
} from "../config.js";
import { scanCodexSource } from "./source-scan-codex.js";
import {
Expand Down Expand Up @@ -821,7 +822,7 @@ export async function runLangExtractSidecar(
): Promise<ExtractedTodo[]> {
const script = sidecarPath();
if (!script) throw new Error("langextract sidecar not found");
const python = getEnvVar("LANGEXTRACT_PYTHON") || "python3";
const python = resolveLangExtractPython();
const env = { ...process.env };
for (const key of SIDE_CAR_ENV_KEYS) {
const value = getEnvVar(key);
Expand Down Expand Up @@ -947,7 +948,14 @@ function parseCheckpoint(cursor: string | undefined): Record<string, string> {
if (!cursor) return {};
try {
const parsed = JSON.parse(cursor) as unknown;
return parsed && typeof parsed === "object" ? parsed as Record<string, string> : {};
if (!parsed || typeof parsed !== "object") return {};
const raw = parsed as Record<string, unknown>;
if (raw.__engine !== "langextract") return {};
const out: Record<string, string> = {};
for (const [key, value] of Object.entries(raw)) {
if (key !== "__engine" && typeof value === "string") out[key] = value;
}
return out;
} catch {
return {};
}
Expand Down Expand Up @@ -1261,7 +1269,7 @@ async function runCleanupSidecar(
): Promise<LlmCleanupItem[]> {
const script = sidecarPath(CLEANUP_SIDE_CAR);
if (!script) throw new Error("cleanup sidecar not found");
const python = getEnvVar("LANGEXTRACT_PYTHON") || "python3";
const python = resolveLangExtractPython();
const env = { ...process.env };
for (const key of SIDE_CAR_ENV_KEYS) {
const value = getEnvVar(key);
Expand Down Expand Up @@ -2001,14 +2009,16 @@ export async function generateTodosFromSessions(
existing.add(titleKey);
seenTitles.push(titleKey);
}
processed[session.id] = key;
if (engine === "langextract" && !fallbackReason) processed[session.id] = key;
}

await kv.set(KV.scanCheckpoints, checkpointId, {
sourceId: checkpointId,
cursor: JSON.stringify(processed),
lastSuccessAt: now,
});
if (Object.keys(processed).length) {
await kv.set(KV.scanCheckpoints, checkpointId, {
sourceId: checkpointId,
cursor: JSON.stringify({ __engine: "langextract", ...processed }),
lastSuccessAt: now,
});
}

return {
success: true,
Expand Down
2 changes: 1 addition & 1 deletion src/triggers/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ export function registerApiTriggers(
success: true,
envPath: getUserEnvPath(),
config: getTodoExtractorUserConfig(),
restartRequired: true,
restartRequired: false,
},
};
},
Expand Down
Loading
Loading