From aea946452bad3ac77cb9a368cf4ac723b14d170f Mon Sep 17 00:00:00 2001 From: moonrailgun Date: Sat, 29 Aug 2026 01:06:21 +0800 Subject: [PATCH] feat(timeline): support video source ranges in agent tools --- .../lib/ai/agent/tools/timeline-tools.test.ts | 106 ++++++++++++++++++ .../src/lib/ai/agent/tools/timeline-tools.ts | 93 ++++++++++++++- apps/web/src/lib/timeline/element-utils.ts | 8 +- 3 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 apps/web/src/lib/ai/agent/tools/timeline-tools.test.ts diff --git a/apps/web/src/lib/ai/agent/tools/timeline-tools.test.ts b/apps/web/src/lib/ai/agent/tools/timeline-tools.test.ts new file mode 100644 index 0000000..46eeecd --- /dev/null +++ b/apps/web/src/lib/ai/agent/tools/timeline-tools.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { MediaAsset } from "@/types/assets"; +import type { CreateTimelineElement, VideoTrack } from "@/types/timeline"; + +const videoAsset = { + id: "video-media", + name: "Video", + type: "video", + file: new File([], "video.mp4", { type: "video/mp4" }), + duration: 12, +} satisfies MediaAsset; + +let tracks: VideoTrack[] = []; +const insertElement = ({ element }: { element: CreateTimelineElement }) => { + if (element.type !== "video") return; + tracks = [ + { + id: "video-track", + name: "Video", + type: "video", + isMain: true, + muted: false, + hidden: false, + elements: [{ ...element, id: "video-element" }], + }, + ]; +}; +const getTracks = () => tracks; +const getTotalDuration = mock(() => 0); +const editor = { + media: { getAssets: () => [videoAsset] }, + timeline: { insertElement, getTracks, getTotalDuration }, +}; + +mock.module("@/core", () => ({ + EditorCore: { getInstance: () => editor }, +})); + +const { addVideoToTimelineTool, getTimelineStateTool } = await import( + "./timeline-tools" +); + +describe("timeline video tools", () => { + beforeEach(() => { + tracks = []; + getTotalDuration.mockReset(); + getTotalDuration.mockReturnValue(0); + }); + + test("adds and reports a source subclip", async () => { + const addResult = await addVideoToTimelineTool.execute({ + mediaId: videoAsset.id, + startTime: 4, + trimStart: 2, + sourceOutPoint: 7, + }); + + expect(addResult.success).toBe(true); + getTotalDuration.mockReturnValue(9); + + const stateResult = await getTimelineStateTool.execute({}); + const tracks = stateResult.data?.tracks as Array<{ + elements: Array>; + }>; + + expect(tracks[0].elements[0]).toMatchObject({ + startTime: 4, + duration: 5, + trimStart: 2, + trimEnd: 5, + sourceOutPoint: 7, + }); + }); + + test("derives trimEnd when duration selects the first source segment", async () => { + const addResult = await addVideoToTimelineTool.execute({ + mediaId: videoAsset.id, + duration: 5, + }); + + expect(addResult.success).toBe(true); + + const stateResult = await getTimelineStateTool.execute({}); + const stateTracks = stateResult.data?.tracks as Array<{ + elements: Array>; + }>; + + expect(stateTracks[0].elements[0]).toMatchObject({ + duration: 5, + trimStart: 0, + trimEnd: 7, + sourceOutPoint: 5, + }); + }); + + test("rejects a source range outside the video", async () => { + const result = await addVideoToTimelineTool.execute({ + mediaId: videoAsset.id, + trimStart: 10, + sourceOutPoint: 13, + }); + + expect(result.success).toBe(false); + expect(tracks).toEqual([]); + }); +}); diff --git a/apps/web/src/lib/ai/agent/tools/timeline-tools.ts b/apps/web/src/lib/ai/agent/tools/timeline-tools.ts index 9579f53..7d56f24 100644 --- a/apps/web/src/lib/ai/agent/tools/timeline-tools.ts +++ b/apps/web/src/lib/ai/agent/tools/timeline-tools.ts @@ -7,6 +7,9 @@ import { } from "@/lib/timeline/element-utils"; import type { AgentTool } from "./types"; +const isFiniteNumber = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value); + export const getTimelineStateTool: AgentTool = { name: "get_timeline_state", description: @@ -34,6 +37,13 @@ export const getTimelineStateTool: AgentTool = { duration: element.duration, trimStart: element.trimStart, trimEnd: element.trimEnd, + ...(element.type === "video" + ? { + sourceOutPoint: + element.trimStart + + element.duration * (element.playbackRate ?? 1), + } + : {}), ...("content" in element ? { content: element.content } : {}), ...("mediaId" in element ? { mediaId: element.mediaId } : {}), })), @@ -65,7 +75,17 @@ export const addVideoToTimelineTool: AgentTool = { duration: { type: "number", description: - "Duration in seconds. For videos, defaults to the media's original duration. For images, defaults to 5 seconds.", + "Duration in seconds. For trimmed videos, defaults to the selected source range. For other videos, defaults to the original duration. For images, defaults to 5 seconds.", + }, + trimStart: { + type: "number", + description: + "Source video in-point in seconds. Only supported for videos.", + }, + sourceOutPoint: { + type: "number", + description: + "Source video out-point in seconds. Only supported for videos. When omitted, duration determines the out-point.", }, }, required: ["mediaId"], @@ -90,10 +110,77 @@ export const addVideoToTimelineTool: AgentTool = { }; } - const duration = (args.duration as number) ?? asset.duration ?? 5; + const hasSourceRange = + args.trimStart !== undefined || args.sourceOutPoint !== undefined; + if (!isVideo && hasSourceRange) { + return { + success: false, + message: "trimStart and sourceOutPoint are only supported for videos", + }; + } + + let duration = (args.duration as number) ?? asset.duration ?? 5; + let trimStart = 0; + let trimEnd = 0; + + if (isVideo && (hasSourceRange || args.duration !== undefined)) { + const sourceDuration = asset.duration; + const requestedTrimStart = args.trimStart ?? 0; + const requestedSourceOutPoint = args.sourceOutPoint; + if ( + !isFiniteNumber(sourceDuration) || + sourceDuration <= 0 || + !isFiniteNumber(requestedTrimStart) || + requestedTrimStart < 0 || + (args.duration !== undefined && + (!isFiniteNumber(args.duration) || args.duration <= 0)) || + (requestedSourceOutPoint !== undefined && + !isFiniteNumber(requestedSourceOutPoint)) + ) { + return { + success: false, + message: `Video '${asset.name}' has an invalid source range`, + }; + } + + trimStart = requestedTrimStart; + const sourceOutPoint = + requestedSourceOutPoint ?? + (args.duration !== undefined + ? trimStart + args.duration + : sourceDuration); + if (sourceOutPoint <= trimStart || sourceOutPoint > sourceDuration) { + return { + success: false, + message: `Source range must be within 0-${sourceDuration}s and end after trimStart`, + }; + } + + const sourceClipDuration = sourceOutPoint - trimStart; + if ( + args.duration !== undefined && + requestedSourceOutPoint !== undefined && + Math.abs(args.duration - sourceClipDuration) > 0.000001 + ) { + return { + success: false, + message: "duration must equal sourceOutPoint - trimStart", + }; + } + + duration = sourceClipDuration; + trimEnd = sourceDuration - sourceOutPoint; + } const element = isVideo - ? buildVideoElement({ mediaId, name: asset.name, duration, startTime }) + ? buildVideoElement({ + mediaId, + name: asset.name, + duration, + startTime, + trimStart, + trimEnd, + }) : buildImageElement({ mediaId, name: asset.name, duration, startTime }); editor.timeline.insertElement({ diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index 55cdca6..92f25a8 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -174,11 +174,15 @@ export function buildVideoElement({ name, duration, startTime, + trimStart = 0, + trimEnd = 0, }: { mediaId: string; name: string; duration: number; startTime: number; + trimStart?: number; + trimEnd?: number; }): CreateVideoElement { return { type: "video", @@ -186,8 +190,8 @@ export function buildVideoElement({ name, duration, startTime, - trimStart: 0, - trimEnd: 0, + trimStart, + trimEnd, muted: false, hidden: false, transform: { scale: 1, position: { x: 0, y: 0 }, rotate: 0 },