Hermes/infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): 添加视频节点支持功能

Browse Source 
- 在后端添加 AI 视频相关接口处理函数
- 扩展前端画布组件支持视频节点类型
- 实现视频上传、生成和播放功能
- 更新画布数据结构文档以包含视频节点
- 添加视频节点的操作工具栏和配置面板
- 集成视频存储和检索服务
- 优化视频节点的渲染和交互体验
This commit is contained in:
HouYunFei
2026-05-23 16:53:21 +08:00
parent 0cc27b5a4c
commit 44c5825cb0
23 changed files with 306 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/src/app/(user)/canvas/[id]/canvas-client-page.tsx
+88 -14
View File
@@ -3,11 +3,13 @@
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react";
import { useParams, useRouter } from "next/navigation";
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload } from "lucide-react";
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
import { requestVideoGeneration } from "@/services/api/video";
import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage";
import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
import { nanoid } from "nanoid";
import { getDataUrlByteSize, readImageMeta } from "@/lib/image-utils";
import { canvasThemes, type CanvasBackgroundMode } from "@/lib/canvas-theme";
@@ -135,7 +137,7 @@ function CanvasRefreshShell() {
);
}
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config) => void; onClose: () => void }) {
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video) => void; onClose: () => void }) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
return (
<div
@@ -156,6 +158,7 @@ function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: Pending
<div className="grid gap-1">
<ConnectionCreateOption icon={<List className="size-5" />} title="文本生成" description="脚本、广告词、品牌文案" onClick={() => onCreate(CanvasNodeType.Text)} />
<ConnectionCreateOption icon={<ImageIcon className="size-5" />} title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} />
<ConnectionCreateOption icon={<Video className="size-5" />} title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} />
<ConnectionCreateOption icon={<Settings2 className="size-5" />} title="配置节点" description="模型、尺寸、数量和输入顺序" onClick={() => onCreate(CanvasNodeType.Config)} />
</div>
</div>
@@ -463,7 +466,7 @@ function InfiniteCanvasPage() {
);
const createConnectedNode = useCallback(
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config, pending: PendingConnectionCreate) => {
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video, pending: PendingConnectionCreate) => {
const metadata = type === CanvasNodeType.Config ? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: 3 } : undefined;
const newNode = createCanvasNode(type, pending.position, metadata);
const connection = normalizeConnection(pending.connection.nodeId, newNode.id, [...nodesRef.current, newNode], pending.connection.handleType);
@@ -1069,6 +1072,27 @@ function InfiniteCanvasPage() {
setDialogNodeId(id);
}, []);
const createVideoFileNode = useCallback(async (file: File, position: Position) => {
const video = await uploadMediaFile(file, "video");
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
setNodes((prev) => [
...prev,
{
id,
type: CanvasNodeType.Video,
title: file.name,
position: { x: position.x - spec.width / 2, y: position.y - spec.height / 2 },
width: spec.width,
height: spec.height,
metadata: videoMetadata(video),
},
]);
setSelectedNodeIds(new Set([id]));
setSelectedConnectionId(null);
setDialogNodeId(id);
}, []);
const createTextNodeFromClipboard = useCallback(
(text: string) => {
const trimmed = text.trim();
@@ -1282,10 +1306,10 @@ function InfiniteCanvasPage() {
}, []);
const downloadNodeImage = useCallback((node: CanvasNodeData) => {
if (node.type !== CanvasNodeType.Image || !node.metadata?.content) return;
if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video) || !node.metadata?.content) return;
const link = document.createElement("a");
link.href = node.metadata.content;
link.download = `canvas-image-${node.id}.${imageExtension(node.metadata.content)}`;
link.download = `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : imageExtension(node.metadata.content)}`;
link.click();
}, []);
@@ -1298,6 +1322,12 @@ function InfiniteCanvasPage() {
message.success("已加入我的素材");
return;
}
if (node.type === CanvasNodeType.Video) {
if (!node.metadata?.content) return message.error("没有可保存的视频");
addAsset({ kind: "video", title: node.metadata?.prompt?.slice(0, 24) || "画布视频", coverUrl: "", tags: [], source: "Canvas", data: { url: node.metadata.content, storageKey: node.metadata.storageKey, width: node.width, height: node.height, bytes: node.metadata.bytes || 0, mimeType: node.metadata.mimeType || "video/mp4" }, metadata: { source: "canvas", nodeId: node.id, prompt: node.metadata?.prompt } });
message.success("已加入我的素材");
return;
}
if (!node.metadata?.content) return message.error("没有可保存的图片");
const dataUrl = node.metadata.storageKey ? "" : node.metadata.content;
addAsset({
@@ -1408,9 +1438,19 @@ function InfiniteCanvasPage() {
async (event: ReactChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
const target = uploadTargetRef.current;
if (!file || !file.type.startsWith("image/")) return;
if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/"))) return;
if (target?.nodeId) {
if (file.type.startsWith("video/")) {
const video = await uploadMediaFile(file, "video");
setNodes((prev) => prev.map((node) => (node.id === target.nodeId ? { ...node, type: CanvasNodeType.Video, title: file.name, width: NODE_DEFAULT_SIZE[CanvasNodeType.Video].width, height: NODE_DEFAULT_SIZE[CanvasNodeType.Video].height, metadata: { ...node.metadata, ...videoMetadata(video), errorDetails: undefined } } : node)));
setSelectedNodeIds(new Set([target.nodeId]));
setSelectedConnectionId(null);
setDialogNodeId(target.nodeId);
uploadTargetRef.current = null;
event.target.value = "";
return;
}
const image = await uploadImage(file);
const size = fitImageNodeSize(image.width, image.height);
setNodes((prev) =>
@@ -1449,25 +1489,25 @@ function InfiniteCanvasPage() {
setDialogNodeId(target.nodeId);
} else {
const position = target?.position || screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
void createImageFileNode(file, position);
void (file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position));
}
uploadTargetRef.current = null;
event.target.value = "";
},
[createImageFileNode, screenToCanvas, size.height, size.width],
[createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width],
);
const handleDrop = useCallback(
(event: ReactDragEvent<HTMLDivElement>) => {
event.preventDefault();
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/"));
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/"));
if (!file) return;
const pos = screenToCanvas(event.clientX, event.clientY);
void createImageFileNode(file, pos);
void (file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos));
},
[createImageFileNode, screenToCanvas],
[createImageFileNode, createVideoFileNode, screenToCanvas],
);
const pasteAssistantImage = useCallback(
@@ -1681,6 +1721,28 @@ function InfiniteCanvasPage() {
return;
}
if (mode === "video") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const isEmptyVideoNode = sourceNode?.type === CanvasNodeType.Video && !sourceNode.metadata?.content;
const videoId = isEmptyVideoNode ? nodeId : nanoid();
const parent = sourceNode?.position || { x: 0, y: 0 };
const videoNode: CanvasNodeData = {
id: videoId,
type: CanvasNodeType.Video,
title: effectivePrompt.slice(0, 32) || "Generated Video",
position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y },
width: isEmptyVideoNode ? sourceNode.width : spec.width,
height: isEmptyVideoNode ? sourceNode.height : spec.height,
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size },
};
pendingChildIds = [videoId];
setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode]));
if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]);
const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, effectivePrompt), "video");
setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size } } : node)));
return;
}
let streamed = "";
const isConfigNode = sourceNode?.type === CanvasNodeType.Config;
const textCount = isConfigNode ? getGenerationCount(generationConfig.count) : 1;
@@ -1906,12 +1968,18 @@ function InfiniteCanvasPage() {
(payload: InsertAssetPayload) => {
if (payload.kind === "text") {
insertAssistantText(payload.content);
} else if (payload.kind === "video") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const center = screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
setNodes((prev) => [...prev, { id, type: CanvasNodeType.Video, title: payload.title, position: { x: center.x - spec.width / 2, y: center.y - spec.height / 2 }, width: spec.width, height: spec.height, metadata: { content: payload.url, storageKey: payload.storageKey, status: NODE_STATUS_SUCCESS } }]);
setSelectedNodeIds(new Set([id]));
} else {
insertAssistantImage({ id: `asset-${Date.now()}`, prompt: payload.title, dataUrl: payload.dataUrl, storageKey: payload.storageKey });
}
setAssetPickerOpen(false);
},
[insertAssistantImage, insertAssistantText],
[insertAssistantImage, insertAssistantText, screenToCanvas, size.height, size.width],
);
if (!projectLoaded) return <CanvasRefreshShell />;
@@ -2100,6 +2168,7 @@ function InfiniteCanvasPage() {
canRedo={historyState.canRedo}
backgroundMode={backgroundMode}
onAddImage={() => createNode(CanvasNodeType.Image)}
onAddVideo={() => createNode(CanvasNodeType.Video)}
onAddText={() => createNode(CanvasNodeType.Text)}
onAddConfig={() => createNode(CanvasNodeType.Config)}
onUndo={undoCanvas}
@@ -2138,7 +2207,7 @@ function InfiniteCanvasPage() {
/>
) : null}
<input ref={imageInputRef} type="file" accept="image/*" className="hidden" onChange={handleImageInputChange} />
<input ref={imageInputRef} type="file" accept="image/*,video/*" className="hidden" onChange={handleImageInputChange} />
<CanvasNodeInfoModal node={infoNode} open={Boolean(infoNode)} onClose={() => setInfoNodeId(null)} />
@@ -2406,6 +2475,10 @@ function imageMetadata(image: UploadedImage): CanvasNodeMetadata {
return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType };
}
function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
return { content: video.url, storageKey: video.storageKey, status: "success", bytes: video.bytes, mimeType: video.mimeType || "video/mp4" };
}
function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata {
return {
generationType: type,
@@ -2437,6 +2510,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
return Promise.all(
nodes.map(async (node) => {
const content = node.metadata?.content;
if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
if (node.type !== CanvasNodeType.Image || !content) return node;
if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } };
if (!content.startsWith("data:image/")) return node;
@@ -2502,7 +2576,7 @@ function getInputSummary(inputs: NodeGenerationInput[]) {
}
function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefined, mode: CanvasNodeGenerationMode): AiConfig {
const defaultModel = mode === "image" ? config.imageModel : config.textModel;
const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : config.textModel;
return {
...config,
model: node?.metadata?.model || defaultModel || config.model || defaultConfig.model,