Skip to content

feat(toolkit): Simplify deepnote toolkit kernel management, to not use deepnote environments #376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tkislan wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

feat(toolkit): Simplify deepnote toolkit kernel management, to not use deepnote environments #376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
eeaadd3
feat(toolkit): Simplify deepnote toolkit kernel management, to not us…
tkislan Apr 1, 2026
0015264
refactor(deepnote): Enhance deepnote server management and cleanup logic
tkislan Apr 2, 2026
7740937
fix(deepnote): Improve controller management and cleanup logic
tkislan Apr 2, 2026
33f05b8
Merge origin/main into tk/simplify-deepnote-toolkit-kernel
tkislan Aug 24, 2026
3dd23fe
fix(python-api): don't hang forever waiting for the Jupyter API hands…
tkislan Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 71 additions & 42 deletions src/kernels/deepnote/deepnoteServerStarter.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import * as fs from 'fs-extra';
import { inject, injectable, named } from 'inversify';
import * as os from 'os';
import { CancellationToken, l10n, Uri } from 'vscode';
import { CancellationToken, CancellationTokenSource, l10n, Uri } from 'vscode';

import { startServer, stopServer } from '@deepnote/runtime-core';

Expand All @@ -20,13 +20,15 @@ import { IAsyncDisposableRegistry, IDisposable, IOutputChannel } from '../../pla
import { sleep } from '../../platform/common/utils/async';
import { generateUuid } from '../../platform/common/uuid';
import { DeepnoteServerStartupError } from '../../platform/errors/deepnoteKernelErrors';
import { getCachedEnvironment } from '../../platform/interpreter/helpers';
import { IInstaller, InstallerResponse, Product } from '../../platform/interpreter/installer/types';
import { logger } from '../../platform/logging';
import { IUserpodApiEndpoints } from '../../platform/notebooks/deepnote/types';
import { PythonEnvironment } from '../../platform/pythonEnvironments/info';
import * as path from '../../platform/vscode-path/path';
import { DeepnoteAgentSkillsManager } from './deepnoteAgentSkillsManager.node';
import { applyIntegrationEndpointEnv } from './deepnoteIntegrationEndpointEnv';
import { DeepnoteServerInfo, IDeepnoteServerStarter, IDeepnoteToolkitInstaller } from './types';
import { DeepnoteServerInfo, IDeepnoteServerStarter } from './types';

const MAX_OUTPUT_TRACKING_LENGTH = 5000;
const SERVER_STARTUP_TIMEOUT_MS = 120_000;
Expand All @@ -49,7 +51,7 @@ type PendingOperation =
};

interface ProjectContext {
environmentId: string;
interpreterId: string;
serverInfo: DeepnoteServerInfo | null;
}

Expand All @@ -72,7 +74,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension

constructor(
@inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory,
@inject(IDeepnoteToolkitInstaller) private readonly toolkitInstaller: IDeepnoteToolkitInstaller,
@inject(IInstaller) private readonly installer: IInstaller,
@inject(DeepnoteAgentSkillsManager) private readonly agentSkillsManager: DeepnoteAgentSkillsManager,
@inject(IOutputChannel) @named(STANDARD_OUTPUT_CHANNEL) private readonly outputChannel: IOutputChannel,
@inject(IAsyncDisposableRegistry) asyncRegistry: IAsyncDisposableRegistry,
Expand All @@ -98,14 +100,11 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
*/
public async startServer(
interpreter: PythonEnvironment,
venvPath: Uri,
managedVenv: boolean,
additionalPackages: string[],
environmentId: string,
deepnoteFileUri: Uri,
token?: CancellationToken
): Promise<DeepnoteServerInfo> {
const fileKey = deepnoteFileUri.fsPath;
const interpreterId = interpreter.id;

let pendingOp = this.pendingOperations.get(fileKey);
if (pendingOp) {
Expand All @@ -119,12 +118,12 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension

let existingContext = this.projectContexts.get(fileKey);
if (existingContext != null) {
const { environmentId: existingEnvironmentId, serverInfo: existingServerInfo } = existingContext;
const { interpreterId: existingInterpreterId, serverInfo: existingServerInfo } = existingContext;

if (existingEnvironmentId === environmentId) {
if (existingInterpreterId === interpreterId) {
if (existingServerInfo != null && (await this.isServerRunning(existingServerInfo))) {
logger.info(
`Deepnote server already running at ${existingServerInfo.url} for ${fileKey} (environmentId ${environmentId})`
`Deepnote server already running at ${existingServerInfo.url} for ${fileKey} (interpreter ${interpreterId})`
);
return existingServerInfo;
}
Expand All @@ -136,14 +135,15 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
}
} else {
logger.info(
`Stopping existing server for ${fileKey} with environmentId ${existingEnvironmentId} to start new one with environmentId ${environmentId}...`
`Stopping existing server for ${fileKey} with interpreter ${existingInterpreterId} to start new one with interpreter ${interpreterId}...`
);
await this.stopServerForEnvironment(existingContext, deepnoteFileUri, token);
existingContext.environmentId = environmentId;
existingContext = { interpreterId, serverInfo: null };
this.projectContexts.set(fileKey, existingContext);
}
} else {
const newContext: ProjectContext = {
environmentId,
interpreterId,
serverInfo: null
};

Expand All @@ -153,16 +153,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension

const operation = {
type: 'start' as const,
promise: this.startServerForEnvironment(
existingContext,
interpreter,
venvPath,
managedVenv,
additionalPackages,
environmentId,
deepnoteFileUri,
token
)
promise: this.startServerForEnvironment(existingContext, interpreter, deepnoteFileUri, token)
};
this.pendingOperations.set(fileKey, operation);

Expand Down Expand Up @@ -223,7 +214,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
* Core server start using @deepnote/runtime-core's `startServer`.
*
* Extension-specific layers:
* - Toolkit/venv installation (before start)
* - Toolkit check/install via IInstaller (before start)
* - Integration endpoint env var injection (via ServerOptions.env) — these point the toolkit at the
* extension's loopback `userpod-api` endpoint, which is how it fetches SQL credentials at kernel init
* - Lock file creation (after start, using returned PID)
Expand All @@ -232,34 +223,49 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
private async startServerForEnvironment(
projectContext: ProjectContext,
interpreter: PythonEnvironment,
venvPath: Uri,
managedVenv: boolean,
additionalPackages: string[],
environmentId: string,
deepnoteFileUri: Uri,
token?: CancellationToken
): Promise<DeepnoteServerInfo> {
const fileKey = deepnoteFileUri.fsPath;
const interpreterId = interpreter.id;

Cancellation.throwIfCanceled(token);

logger.info(`Ensuring deepnote-toolkit is installed in venv for environment ${environmentId}...`);
const { pythonInterpreter: venvInterpreter } = await this.toolkitInstaller.ensureVenvAndToolkit(
interpreter,
venvPath,
managedVenv,
token
);
// Check if deepnote-toolkit is installed, and install if needed
logger.info(`Checking deepnote-toolkit installation for interpreter ${interpreterId}...`);
const isInstalled = await this.installer.isInstalled(Product.deepnoteToolkit, interpreter);

this.agentSkillsManager.ensureSkillsUpdated(environmentId, venvInterpreter);
if (!isInstalled) {
logger.info(`deepnote-toolkit not installed, installing via IInstaller...`);
const cts = new CancellationTokenSource();
let cancellationListener: IDisposable | undefined;

Cancellation.throwIfCanceled(token);
try {
if (token) {
cancellationListener = token.onCancellationRequested(() => cts.cancel());
}

const result = await this.installer.install(Product.deepnoteToolkit, interpreter, cts);

if (result === InstallerResponse.Cancelled) {
throw new Error('deepnote-toolkit installation was cancelled by the user');
} else if (result !== InstallerResponse.Installed) {
throw new Error('Failed to install deepnote-toolkit. Check the Output panel for details.');
}
} finally {
cancellationListener?.dispose();
cts.dispose();
}
}

await this.toolkitInstaller.installAdditionalPackages(venvPath, additionalPackages, token);
this.agentSkillsManager.ensureSkillsUpdated(interpreterId, interpreter);

Cancellation.throwIfCanceled(token);

logger.info(`Starting deepnote-toolkit server for ${fileKey} (environmentId ${environmentId})`);
// Derive the environment path from the interpreter
const envPath = this.deriveEnvPath(interpreter);

logger.info(`Starting deepnote-toolkit server for ${fileKey} (interpreter ${interpreterId})`);
this.outputChannel.appendLine(l10n.t('Starting Deepnote server...'));

const extraEnv: Record<string, string> = {};
Expand All @@ -276,7 +282,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
let serverInfo: DeepnoteServerInfo | undefined;
try {
serverInfo = await startServer({
pythonEnv: venvPath.fsPath,
pythonEnv: envPath,
workingDirectory: path.dirname(deepnoteFileUri.fsPath),
startupTimeoutMs: SERVER_STARTUP_TIMEOUT_MS,
env: extraEnv
Expand All @@ -287,7 +293,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension

throw new DeepnoteServerStartupError(
interpreter.uri.fsPath,
serverInfo?.jupyterPort ?? 0,
0,
'unknown',
capturedOutput?.stdout || '',
capturedOutput?.stderr || '',
Expand All @@ -314,6 +320,29 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
return serverInfo;
}

/**
* Derive the environment path from a Python interpreter.
* Uses the cached environment info, or falls back to navigating up from the executable.
*/
private deriveEnvPath(interpreter: PythonEnvironment): string {
const cachedEnv = getCachedEnvironment(interpreter);
// eslint-disable-next-line local-rules/dont-use-fspath
const folderPath = cachedEnv?.environment?.folderUri?.fsPath;

if (folderPath) {
return folderPath;
}

const sysPrefix = cachedEnv?.executable?.sysPrefix;

if (sysPrefix) {
return sysPrefix;
}

// Fallback: go up from bin/python (or Scripts/python.exe on Windows)
return path.dirname(path.dirname(interpreter.uri.fsPath));
}

/**
* Stop the server using @deepnote/runtime-core's `stopServer` (SIGTERM -> wait -> SIGKILL).
*/
Expand Down
Loading
Loading