Hermes/infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): 添加视频节点支持功能

Browse Source 
- 在后端添加 AI 视频相关接口处理函数
- 扩展前端画布组件支持视频节点类型
- 实现视频上传、生成和播放功能
- 更新画布数据结构文档以包含视频节点
- 添加视频节点的操作工具栏和配置面板
- 集成视频存储和检索服务
- 优化视频节点的渲染和交互体验
This commit is contained in:
HouYunFei
2026-05-23 16:53:21 +08:00
parent 0cc27b5a4c
commit 44c5825cb0
23 changed files with 306 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/src/app/(user)/assets/page.tsx
+16 -11
View File
@@ -26,6 +26,7 @@ const kindOptions = [
{ label: "全部", value: "all" },
{ label: "文本", value: "text" },
{ label: "图片", value: "image" },
{ label: "视频", value: "video" },
];
export default function AssetsPage() {
@@ -52,7 +53,7 @@ export default function AssetsPage() {
const title = Form.useWatch("title", form) || "";
const tags = Form.useWatch("tags", form) || [];
const content = Form.useWatch("content", form) || "";
const validAssets = useMemo(() => assets.filter((asset) => asset.kind === "text" || asset.kind === "image"), [assets]);
const validAssets = useMemo(() => assets.filter((asset) => asset.kind === "text" || asset.kind === "image" || asset.kind === "video"), [assets]);
const filteredAssets = useMemo(() => {
const query = keyword.trim().toLowerCase();
@@ -145,9 +146,9 @@ export default function AssetsPage() {
};
const downloadImage = (asset: Asset) => {
if (asset.kind !== "image") return;
if (asset.kind !== "image" && asset.kind !== "video") return;
const link = document.createElement("a");
link.href = asset.data.dataUrl;
link.href = asset.kind === "video" ? asset.data.url : asset.data.dataUrl;
link.download = `${asset.title || "asset"}.${asset.data.mimeType.split("/")[1] || "png"}`;
link.click();
};
@@ -385,7 +386,7 @@ function AssetCard({ asset, onOpen, onEdit, onCopy, onDownload, onDelete }: { as
{asset.source || "未标注来源"}
</Typography.Text>
</div>
<Tag className="m-0 shrink-0 text-[11px]">{asset.kind === "image" ? "图片" : "文本"}</Tag>
<Tag className="m-0 shrink-0 text-[11px]">{asset.kind === "image" ? "图片" : asset.kind === "video" ? "视频" : "文本"}</Tag>
</div>
<Typography.Paragraph type="secondary" ellipsis={{ rows: 3 }} className="!mb-0 !mt-2 !text-xs !leading-5">
{summary}
@@ -404,15 +405,17 @@ function AssetCard({ asset, onOpen, onEdit, onCopy, onDownload, onDelete }: { as
<Button size="small" onClick={onOpen}>
查看
</Button>
<Button size="small" icon={<PencilLine className="size-3.5" />} onClick={onEdit}>
编辑
</Button>
{asset.kind !== "video" ? (
<Button size="small" icon={<PencilLine className="size-3.5" />} onClick={onEdit}>
编辑
</Button>
) : null}
{asset.kind === "text" ? (
<Button size="small" icon={<Copy className="size-3.5" />} onClick={() => void onCopy(asset)}>
复制
</Button>
) : null}
{asset.kind === "image" ? (
{asset.kind === "image" || asset.kind === "video" ? (
<Button size="small" icon={<Download className="size-3.5" />} onClick={() => onDownload(asset)}>
下载
</Button>
@@ -441,7 +444,7 @@ function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: Asset | nu
{asset.title}
</Typography.Title>
<Space size={[4, 4]} wrap>
<Tag>{asset.kind === "image" ? "图片" : "文本"}</Tag>
<Tag>{asset.kind === "image" ? "图片" : asset.kind === "video" ? "视频" : "文本"}</Tag>
{(asset.tags || []).map((tag) => (
<Tag key={tag}>{tag}</Tag>
))}
@@ -453,6 +456,8 @@ function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: Asset | nu
</Typography.Text>
{asset.kind === "text" ? (
<Typography.Paragraph className="mt-2 whitespace-pre-wrap">{asset.data.content}</Typography.Paragraph>
) : asset.kind === "video" ? (
<video src={asset.data.url} controls className="mt-2 aspect-video w-full rounded-lg bg-black" />
) : (
<Typography.Text className="mt-2 block">
{asset.data.width}x{asset.data.height} · {formatBytes(asset.data.bytes)} · {asset.data.mimeType}
@@ -471,9 +476,9 @@ function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: Asset | nu
复制文本
</Button>
) : null}
{asset.kind === "image" ? (
{asset.kind === "image" || asset.kind === "video" ? (
<Button type="primary" icon={<Download className="size-4" />} onClick={() => onDownload(asset)}>
下载图片
{asset.kind === "video" ? "下载视频" : "下载图片"}
</Button>
) : null}
</Space>
web/src/app/(user)/canvas/[id]/canvas-client-page.tsx
+88 -14
View File
@@ -3,11 +3,13 @@
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react";
import { useParams, useRouter } from "next/navigation";
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload } from "lucide-react";
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
import { requestVideoGeneration } from "@/services/api/video";
import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage";
import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
import { nanoid } from "nanoid";
import { getDataUrlByteSize, readImageMeta } from "@/lib/image-utils";
import { canvasThemes, type CanvasBackgroundMode } from "@/lib/canvas-theme";
@@ -135,7 +137,7 @@ function CanvasRefreshShell() {
);
}
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config) => void; onClose: () => void }) {
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video) => void; onClose: () => void }) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
return (
<div
@@ -156,6 +158,7 @@ function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: Pending
<div className="grid gap-1">
<ConnectionCreateOption icon={<List className="size-5" />} title="文本生成" description="脚本、广告词、品牌文案" onClick={() => onCreate(CanvasNodeType.Text)} />
<ConnectionCreateOption icon={<ImageIcon className="size-5" />} title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} />
<ConnectionCreateOption icon={<Video className="size-5" />} title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} />
<ConnectionCreateOption icon={<Settings2 className="size-5" />} title="配置节点" description="模型、尺寸、数量和输入顺序" onClick={() => onCreate(CanvasNodeType.Config)} />
</div>
</div>
@@ -463,7 +466,7 @@ function InfiniteCanvasPage() {
);
const createConnectedNode = useCallback(
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config, pending: PendingConnectionCreate) => {
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video, pending: PendingConnectionCreate) => {
const metadata = type === CanvasNodeType.Config ? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: 3 } : undefined;
const newNode = createCanvasNode(type, pending.position, metadata);
const connection = normalizeConnection(pending.connection.nodeId, newNode.id, [...nodesRef.current, newNode], pending.connection.handleType);
@@ -1069,6 +1072,27 @@ function InfiniteCanvasPage() {
setDialogNodeId(id);
}, []);
const createVideoFileNode = useCallback(async (file: File, position: Position) => {
const video = await uploadMediaFile(file, "video");
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
setNodes((prev) => [
...prev,
{
id,
type: CanvasNodeType.Video,
title: file.name,
position: { x: position.x - spec.width / 2, y: position.y - spec.height / 2 },
width: spec.width,
height: spec.height,
metadata: videoMetadata(video),
},
]);
setSelectedNodeIds(new Set([id]));
setSelectedConnectionId(null);
setDialogNodeId(id);
}, []);
const createTextNodeFromClipboard = useCallback(
(text: string) => {
const trimmed = text.trim();
@@ -1282,10 +1306,10 @@ function InfiniteCanvasPage() {
}, []);
const downloadNodeImage = useCallback((node: CanvasNodeData) => {
if (node.type !== CanvasNodeType.Image || !node.metadata?.content) return;
if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video) || !node.metadata?.content) return;
const link = document.createElement("a");
link.href = node.metadata.content;
link.download = `canvas-image-${node.id}.${imageExtension(node.metadata.content)}`;
link.download = `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : imageExtension(node.metadata.content)}`;
link.click();
}, []);
@@ -1298,6 +1322,12 @@ function InfiniteCanvasPage() {
message.success("已加入我的素材");
return;
}
if (node.type === CanvasNodeType.Video) {
if (!node.metadata?.content) return message.error("没有可保存的视频");
addAsset({ kind: "video", title: node.metadata?.prompt?.slice(0, 24) || "画布视频", coverUrl: "", tags: [], source: "Canvas", data: { url: node.metadata.content, storageKey: node.metadata.storageKey, width: node.width, height: node.height, bytes: node.metadata.bytes || 0, mimeType: node.metadata.mimeType || "video/mp4" }, metadata: { source: "canvas", nodeId: node.id, prompt: node.metadata?.prompt } });
message.success("已加入我的素材");
return;
}
if (!node.metadata?.content) return message.error("没有可保存的图片");
const dataUrl = node.metadata.storageKey ? "" : node.metadata.content;
addAsset({
@@ -1408,9 +1438,19 @@ function InfiniteCanvasPage() {
async (event: ReactChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
const target = uploadTargetRef.current;
if (!file || !file.type.startsWith("image/")) return;
if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/"))) return;
if (target?.nodeId) {
if (file.type.startsWith("video/")) {
const video = await uploadMediaFile(file, "video");
setNodes((prev) => prev.map((node) => (node.id === target.nodeId ? { ...node, type: CanvasNodeType.Video, title: file.name, width: NODE_DEFAULT_SIZE[CanvasNodeType.Video].width, height: NODE_DEFAULT_SIZE[CanvasNodeType.Video].height, metadata: { ...node.metadata, ...videoMetadata(video), errorDetails: undefined } } : node)));
setSelectedNodeIds(new Set([target.nodeId]));
setSelectedConnectionId(null);
setDialogNodeId(target.nodeId);
uploadTargetRef.current = null;
event.target.value = "";
return;
}
const image = await uploadImage(file);
const size = fitImageNodeSize(image.width, image.height);
setNodes((prev) =>
@@ -1449,25 +1489,25 @@ function InfiniteCanvasPage() {
setDialogNodeId(target.nodeId);
} else {
const position = target?.position || screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
void createImageFileNode(file, position);
void (file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position));
}
uploadTargetRef.current = null;
event.target.value = "";
},
[createImageFileNode, screenToCanvas, size.height, size.width],
[createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width],
);
const handleDrop = useCallback(
(event: ReactDragEvent<HTMLDivElement>) => {
event.preventDefault();
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/"));
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/"));
if (!file) return;
const pos = screenToCanvas(event.clientX, event.clientY);
void createImageFileNode(file, pos);
void (file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos));
},
[createImageFileNode, screenToCanvas],
[createImageFileNode, createVideoFileNode, screenToCanvas],
);
const pasteAssistantImage = useCallback(
@@ -1681,6 +1721,28 @@ function InfiniteCanvasPage() {
return;
}
if (mode === "video") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const isEmptyVideoNode = sourceNode?.type === CanvasNodeType.Video && !sourceNode.metadata?.content;
const videoId = isEmptyVideoNode ? nodeId : nanoid();
const parent = sourceNode?.position || { x: 0, y: 0 };
const videoNode: CanvasNodeData = {
id: videoId,
type: CanvasNodeType.Video,
title: effectivePrompt.slice(0, 32) || "Generated Video",
position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y },
width: isEmptyVideoNode ? sourceNode.width : spec.width,
height: isEmptyVideoNode ? sourceNode.height : spec.height,
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size },
};
pendingChildIds = [videoId];
setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode]));
if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]);
const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, effectivePrompt), "video");
setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size } } : node)));
return;
}
let streamed = "";
const isConfigNode = sourceNode?.type === CanvasNodeType.Config;
const textCount = isConfigNode ? getGenerationCount(generationConfig.count) : 1;
@@ -1906,12 +1968,18 @@ function InfiniteCanvasPage() {
(payload: InsertAssetPayload) => {
if (payload.kind === "text") {
insertAssistantText(payload.content);
} else if (payload.kind === "video") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
const center = screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
setNodes((prev) => [...prev, { id, type: CanvasNodeType.Video, title: payload.title, position: { x: center.x - spec.width / 2, y: center.y - spec.height / 2 }, width: spec.width, height: spec.height, metadata: { content: payload.url, storageKey: payload.storageKey, status: NODE_STATUS_SUCCESS } }]);
setSelectedNodeIds(new Set([id]));
} else {
insertAssistantImage({ id: `asset-${Date.now()}`, prompt: payload.title, dataUrl: payload.dataUrl, storageKey: payload.storageKey });
}
setAssetPickerOpen(false);
},
[insertAssistantImage, insertAssistantText],
[insertAssistantImage, insertAssistantText, screenToCanvas, size.height, size.width],
);
if (!projectLoaded) return <CanvasRefreshShell />;
@@ -2100,6 +2168,7 @@ function InfiniteCanvasPage() {
canRedo={historyState.canRedo}
backgroundMode={backgroundMode}
onAddImage={() => createNode(CanvasNodeType.Image)}
onAddVideo={() => createNode(CanvasNodeType.Video)}
onAddText={() => createNode(CanvasNodeType.Text)}
onAddConfig={() => createNode(CanvasNodeType.Config)}
onUndo={undoCanvas}
@@ -2138,7 +2207,7 @@ function InfiniteCanvasPage() {
/>
) : null}
<input ref={imageInputRef} type="file" accept="image/*" className="hidden" onChange={handleImageInputChange} />
<input ref={imageInputRef} type="file" accept="image/*,video/*" className="hidden" onChange={handleImageInputChange} />
<CanvasNodeInfoModal node={infoNode} open={Boolean(infoNode)} onClose={() => setInfoNodeId(null)} />
@@ -2406,6 +2475,10 @@ function imageMetadata(image: UploadedImage): CanvasNodeMetadata {
return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType };
}
function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
return { content: video.url, storageKey: video.storageKey, status: "success", bytes: video.bytes, mimeType: video.mimeType || "video/mp4" };
}
function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata {
return {
generationType: type,
@@ -2437,6 +2510,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
return Promise.all(
nodes.map(async (node) => {
const content = node.metadata?.content;
if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
if (node.type !== CanvasNodeType.Image || !content) return node;
if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } };
if (!content.startsWith("data:image/")) return node;
@@ -2502,7 +2576,7 @@ function getInputSummary(inputs: NodeGenerationInput[]) {
}
function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefined, mode: CanvasNodeGenerationMode): AiConfig {
const defaultModel = mode === "image" ? config.imageModel : config.textModel;
const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : config.textModel;
return {
...config,
model: node?.metadata?.model || defaultModel || config.model || defaultConfig.model,
web/src/app/(user)/canvas/components/asset-picker-modal.tsx
+5 -4
View File
@@ -12,7 +12,7 @@ import { fetchAssetLibrary, type AssetLibraryItem } from "@/services/api/assets"
export type AssetPickerTab = "my-assets" | "library";
export type InsertAssetPayload = { kind: "text"; content: string; title: string } | { kind: "image"; dataUrl: string; title: string; storageKey?: string };
export type InsertAssetPayload = { kind: "text"; content: string; title: string } | { kind: "image"; dataUrl: string; title: string; storageKey?: string } | { kind: "video"; url: string; title: string; storageKey?: string };
type Props = {
open: boolean;
@@ -48,6 +48,7 @@ const kindOptions = [
{ label: "全部", value: "all" },
{ label: "文本", value: "text" },
{ label: "图片", value: "image" },
{ label: "视频", value: "video" },
];
function LibraryTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) => void }) {
@@ -157,7 +158,7 @@ function PickerCard({ title, kind, cover, loading, onClick }: { title: string; k
<div className="p-2.5">
<div className="flex items-center justify-between gap-2">
<span className="line-clamp-1 text-xs font-medium text-stone-800 dark:text-stone-200">{title}</span>
<Tag className="m-0 shrink-0 text-[10px]">{kind === "image" ? "图片" : "文本"}</Tag>
<Tag className="m-0 shrink-0 text-[10px]">{kind === "image" ? "图片" : kind === "video" ? "视频" : "文本"}</Tag>
</div>
</div>
{loading && (
@@ -190,7 +191,7 @@ function MyAssetsTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) =>
const filtered = useMemo(() => {
const query = keyword.trim().toLowerCase();
return assets
.filter((a) => a.kind === "text" || a.kind === "image")
.filter((a) => a.kind === "text" || a.kind === "image" || a.kind === "video")
.filter((a) => kindFilter === "all" || a.kind === kindFilter)
.filter((a) => !query || [a.title, ...(a.tags || [])].join(" ").toLowerCase().includes(query));
}, [assets, keyword, kindFilter]);
@@ -206,7 +207,7 @@ function MyAssetsTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) =>
if (asset.kind === "text") {
onInsert({ kind: "text", content: asset.data.content, title: asset.title });
} else {
onInsert({ kind: "image", dataUrl: asset.data.dataUrl, storageKey: asset.data.storageKey, title: asset.title });
onInsert(asset.kind === "video" ? { kind: "video", url: asset.data.url, storageKey: asset.data.storageKey, title: asset.title } : { kind: "image", dataUrl: asset.data.dataUrl, storageKey: asset.data.storageKey, title: asset.title });
}
};
web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx
+12 -3
View File
@@ -2,7 +2,7 @@
import type { CSSProperties } from "react";
import { useState } from "react";
import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play } from "lucide-react";
import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play, Video } from "lucide-react";
import { App, Button, Empty, Input, InputNumber, Modal, Segmented } from "antd";
import { ModelPicker } from "@/components/model-picker";
@@ -91,6 +91,15 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
</span>
),
},
{
value: "video",
label: (
<span className="inline-flex items-center gap-1">
<Video className="size-3.5" />
视频
</span>
),
},
]}
/>
</div>
@@ -107,7 +116,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
<div className="mb-2 grid min-w-0 cursor-default grid-cols-[minmax(0,1fr)_92px_64px] items-center gap-2" onMouseDown={(event) => event.stopPropagation()}>
<ModelPicker className="canvas-compact-control h-10" config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} fullWidth />
{mode === "image" ? <CanvasSizePicker className="h-10 min-w-0" value={node.metadata?.size || globalConfig.size || defaultConfig.size} onChange={(value) => onConfigChange(node.id, { size: value })} /> : null}
{mode === "image" || mode === "video" ? <CanvasSizePicker className="h-10 min-w-0" value={node.metadata?.size || globalConfig.size || defaultConfig.size} onChange={(value) => onConfigChange(node.id, { size: value })} /> : null}
<InputNumber min={1} max={15} className="canvas-compact-control canvas-control-number h-10 !w-full" value={count} onChange={(value) => onConfigChange(node.id, { count: Number(value) || 1 })} />
</div>
@@ -294,7 +303,7 @@ function InputChip({ label, value, style }: { label: string; value: string; styl
}
function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: CanvasGenerationMode): AiConfig {
const defaultModel = mode === "image" ? globalConfig.imageModel : globalConfig.textModel;
const defaultModel = mode === "image" ? globalConfig.imageModel : mode === "video" ? globalConfig.videoModel : globalConfig.textModel;
return {
...globalConfig,
model: node.metadata?.model || defaultModel || globalConfig.model || defaultConfig.model,
web/src/app/(user)/canvas/components/canvas-node-generation.ts
+1 -1
View File
@@ -11,7 +11,7 @@ export type NodeGenerationContext = {
export type NodeGenerationInput = {
nodeId: string;
type: "text" | "image";
type: "text" | "image" | "video";
title: string;
text?: string;
image?: ReferenceImage;
web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx
+9 -6
View File
@@ -2,7 +2,7 @@
import { useEffect, useMemo, useState, type ReactNode } from "react";
import { Modal, Segmented, Tooltip } from "antd";
import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload } from "lucide-react";
import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react";
import { canvasThemes } from "@/lib/canvas-theme";
import { formatBytes, getDataUrlByteSize } from "@/lib/image-utils";
@@ -57,12 +57,14 @@ export function CanvasNodeHoverToolbar({
const left = viewport.x + (node.position.x + node.width / 2) * viewport.k;
const top = viewport.y + node.position.y * viewport.k - 14;
const isImage = node.type === CanvasNodeType.Image;
const isVideo = node.type === CanvasNodeType.Video;
const hasImage = isImage && Boolean(node.metadata?.content);
const hasVideo = isVideo && Boolean(node.metadata?.content);
const isText = node.type === CanvasNodeType.Text;
const isConfig = node.type === CanvasNodeType.Config;
const canOpenDialog = isText || hasImage;
const canOpenDialog = isText || hasImage || isVideo;
const canRetry = node.metadata?.status === "error";
const hasSpecificTools = canRetry || isText || isImage || isConfig;
const hasSpecificTools = canRetry || isText || isImage || isVideo || isConfig;
return (
<div
@@ -77,8 +79,8 @@ export function CanvasNodeHoverToolbar({
<ToolbarAction title="移除节点" label="删除" icon={<Trash2 className="size-4" />} onClick={() => onDelete(node)} danger />
{hasSpecificTools ? <ToolbarDivider /> : null}
{canRetry ? <ToolbarAction title="重新生成" label="重试" icon={<RefreshCw className="size-4" />} onClick={() => onRetry(node)} /> : null}
{hasImage || isText ? <ToolbarAction title="加入我的素材" label="存素材" icon={<FolderPlus className="size-4" />} onClick={() => onSaveAsset(node)} /> : null}
{hasImage ? <IconAction title="下载图片" icon={<Download className="size-5" />} onClick={() => onDownload(node)} /> : null}
{hasImage || hasVideo || isText ? <ToolbarAction title="加入我的素材" label="存素材" icon={<FolderPlus className="size-4" />} onClick={() => onSaveAsset(node)} /> : null}
{hasImage || hasVideo ? <IconAction title={hasVideo ? "下载视频" : "下载图片"} icon={<Download className="size-5" />} onClick={() => onDownload(node)} /> : null}
{canOpenDialog ? <ToolbarAction title="编辑" label="编辑" icon={<MessageSquare className="size-4" />} onClick={() => onToggleDialog(node)} /> : null}
{isText ? <ToolbarAction title="编辑文本" label="编辑文字" icon={<Pencil className="size-4" />} onClick={() => onEditText(node)} /> : null}
{isText ? <ToolbarAction title="用文本生图" label="生图" icon={<ImageIcon className="size-4" />} onClick={() => onGenerateImage(node)} /> : null}
@@ -86,6 +88,7 @@ export function CanvasNodeHoverToolbar({
{isText ? <ToolbarAction title="减小字号" label="缩小" icon={<Minus className="size-4" />} onClick={() => onDecreaseFont(node)} /> : null}
{isText ? <ToolbarAction title="增大字号" label="放大" icon={<Plus className="size-4" />} onClick={() => onIncreaseFont(node)} /> : null}
{isImage ? <ToolbarAction title={hasImage ? "替换图片" : "上传图片"} label={hasImage ? "替换图片" : "上传图片"} icon={<Upload className="size-4" />} onClick={() => onUpload(node)} /> : null}
{isVideo ? <ToolbarAction title={hasVideo ? "替换视频" : "上传视频"} label={hasVideo ? "替换视频" : "上传视频"} icon={<Video className="size-4" />} onClick={() => onUpload(node)} /> : null}
{hasImage ? (
<ToolbarAction
title={node.metadata?.freeResize ? "切换为等比缩放" : "切换为自由比例"}
@@ -148,7 +151,7 @@ export function CanvasNodeInfoModal({ node, open, onClose }: { node: CanvasNodeD
{view === "info" ? (
<div className="thin-scrollbar h-full space-y-3 overflow-auto pr-1">
<InfoRow label="ID" value={node.id} />
<InfoRow label="类型" value={node.type === CanvasNodeType.Text ? "文本" : node.type === CanvasNodeType.Image ? "图片" : "生成配置"} />
<InfoRow label="类型" value={node.type === CanvasNodeType.Text ? "文本" : node.type === CanvasNodeType.Image ? "图片" : node.type === CanvasNodeType.Video ? "视频" : "生成配置"} />
<InfoRow label="尺寸" value={`${Math.round(node.width)} x ${Math.round(node.height)}`} />
<InfoRow label="位置" value={`${Math.round(node.position.x)}, ${Math.round(node.position.y)}`} />
<InfoRow label="状态" value={node.metadata?.status || "idle"} />
web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx
+3 -3
View File
@@ -68,7 +68,7 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi
}}
className="thin-scrollbar h-24 w-full resize-none rounded-xl border px-3 py-2 text-sm leading-5 outline-none"
style={{ background: theme.node.fill, borderColor: theme.node.stroke, color: theme.node.text }}
placeholder={mode === "image" ? (hasImageContent ? "请输入你想要把这张图修改成什么" : "描述要生成的图片内容") : hasTextContent ? "请输入你想要将本段文本修改成什么" : "请输入你想要生成的文本内容"}
placeholder={mode === "video" ? "描述要生成的视频内容" : mode === "image" ? (hasImageContent ? "请输入你想要把这张图修改成什么" : "描述要生成的图片内容") : hasTextContent ? "请输入你想要将本段文本修改成什么" : "请输入你想要生成的文本内容"}
/>
<div className="mt-2 flex min-w-0 items-center justify-between gap-2">
@@ -105,11 +105,11 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi
}
function defaultMode(type: CanvasNodeData["type"]): CanvasNodeGenerationMode {
return type === CanvasNodeType.Text ? "text" : "image";
return type === CanvasNodeType.Text ? "text" : type === CanvasNodeType.Video ? "video" : "image";
}
function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: CanvasNodeGenerationMode): AiConfig {
const defaultModel = mode === "image" ? globalConfig.imageModel : globalConfig.textModel;
const defaultModel = mode === "image" ? globalConfig.imageModel : mode === "video" ? globalConfig.videoModel : globalConfig.textModel;
return {
...globalConfig,
model: node.metadata?.model || defaultModel || globalConfig.model || defaultConfig.model,
web/src/app/(user)/canvas/components/canvas-node.tsx
+17 -4
View File
@@ -2,7 +2,7 @@
import React, { useCallback, useEffect, useRef, useState } from "react";
import type { ReactNode } from "react";
import { ChevronRight, Image as ImageIcon, RefreshCw, Star } from "lucide-react";
import { ChevronRight, Image as ImageIcon, RefreshCw, Star, Video } from "lucide-react";
import { canvasThemes } from "@/lib/canvas-theme";
import { useThemeStore } from "@/stores/use-theme-store";
@@ -95,6 +95,7 @@ export const CanvasNode = React.memo(function CanvasNode({
const [hovered, setHovered] = useState(false);
const [isEditingContent, setIsEditingContent] = useState(false);
const hasImageContent = data.type === CanvasNodeType.Image && Boolean(data.metadata?.content);
const hasVideoContent = data.type === CanvasNodeType.Video && Boolean(data.metadata?.content);
const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1;
const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId);
const isActive = isConnectionTarget || isSelected || isFocusRelated;
@@ -246,7 +247,7 @@ export const CanvasNode = React.memo(function CanvasNode({
<div
className="relative h-full w-full overflow-visible rounded-3xl border-2"
style={{
background: hasImageContent ? "transparent" : theme.node.fill,
background: hasImageContent || hasVideoContent ? "transparent" : theme.node.fill,
borderColor: hasImageContent ? imageBorderColor : isActive ? selectionBlue : isRelated ? theme.node.muted : theme.node.stroke,
boxShadow: isActive ? `0 0 0 1px ${selectionBlue}55` : isRelated && !isBatchChild ? `0 0 0 1px ${theme.node.muted}55, 0 18px 48px rgba(0,0,0,.14)` : undefined,
}}
@@ -266,7 +267,7 @@ export const CanvasNode = React.memo(function CanvasNode({
className={`relative flex h-full w-full items-center justify-center rounded-[inherit] ${isBatchRoot ? "overflow-visible" : "overflow-hidden"}`}
style={
{
background: hasImageContent ? "transparent" : theme.node.fill,
background: hasImageContent || hasVideoContent ? "transparent" : theme.node.fill,
"--batch-from-x": `${batchMotion?.x || 0}px`,
"--batch-from-y": `${batchMotion?.y || 0}px`,
"--batch-from-rotate": `${6 + (batchMotion?.index || 0) * 4}deg`,
@@ -295,7 +296,7 @@ export const CanvasNode = React.memo(function CanvasNode({
/>
</div>
{!hasImageContent ? <div className="pointer-events-none absolute inset-x-0 bottom-0 h-12" style={{ background: `linear-gradient(to top, ${theme.canvas.background}66, transparent)` }} /> : null}
{!hasImageContent && !hasVideoContent ? <div className="pointer-events-none absolute inset-x-0 bottom-0 h-12" style={{ background: `linear-gradient(to top, ${theme.canvas.background}66, transparent)` }} /> : null}
<ResizeHandle corner="top-left" onMouseDown={handleResizeMouseDown} />
<ResizeHandle corner="top-right" onMouseDown={handleResizeMouseDown} />
@@ -325,6 +326,7 @@ const nodeContentRenderers = {
[CanvasNodeType.Text]: TextContent,
[CanvasNodeType.Image]: ImageNodeContent,
[CanvasNodeType.Config]: EmptyImageContent,
[CanvasNodeType.Video]: VideoNodeContent,
} satisfies Record<CanvasNodeType, (props: NodeContentRendererProps) => ReactNode>;
function LoadingContent({ theme }: Pick<NodeContentRendererProps, "theme">) {
@@ -454,6 +456,17 @@ function EmptyImageContent({ theme, isBatchRoot, batchCount, batchExpanded, batc
return content;
}
function VideoNodeContent({ node, theme }: NodeContentRendererProps) {
if (!node.metadata?.content)
return (
<div className="flex h-full w-full flex-col items-center justify-center gap-3" style={{ color: theme.node.placeholder }}>
<Video className="size-7 opacity-35" />
<span className="text-sm">视频节点</span>
</div>
);
return <video src={node.metadata.content} controls className="h-full w-full rounded-[18px] bg-black object-contain" data-canvas-no-zoom />;
}
function ImageContent({
node,
isBatchRoot,
web/src/app/(user)/canvas/components/canvas-toolbar.tsx
+7 -1
View File
@@ -1,7 +1,7 @@
import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactNode, RefObject } from "react";
import { useRef, useState } from "react";
import { Button, Segmented } from "antd";
import { CircleDot, Eraser, FolderOpen, Grid2x2, Hand, Image as ImageIcon, Library, Moon, Palette, Redo2, Settings2, Square, Sun, Trash2, Type, Undo2, Upload } from "lucide-react";
import { CircleDot, Eraser, FolderOpen, Grid2x2, Hand, Image as ImageIcon, Library, Moon, Palette, Redo2, Settings2, Square, Sun, Trash2, Type, Undo2, Upload, Video } from "lucide-react";
import { canvasThemes, type CanvasBackgroundMode, type CanvasColorTheme, type CanvasTheme } from "@/lib/canvas-theme";
import { useThemeStore } from "@/stores/use-theme-store";
@@ -13,6 +13,7 @@ export function CanvasToolbar({
canRedo,
backgroundMode,
onAddImage,
onAddVideo,
onAddText,
onAddConfig,
onUndo,
@@ -30,6 +31,7 @@ export function CanvasToolbar({
canRedo: boolean;
backgroundMode: CanvasBackgroundMode;
onAddImage: () => void;
onAddVideo: () => void;
onAddText: () => void;
onAddConfig: () => void;
onUndo: () => void;
@@ -75,6 +77,9 @@ export function CanvasToolbar({
<ToolbarButton id="tool-image" label="图片" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddImage}>
<ImageIcon className="size-4.5" />
</ToolbarButton>
<ToolbarButton id="tool-video" label="视频" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddVideo}>
<Video className="size-4.5" />
</ToolbarButton>
<ToolbarButton id="tool-config" label="生成配置" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddConfig}>
<Settings2 className="size-4.5" />
</ToolbarButton>
@@ -262,6 +267,7 @@ function toolLabel(id: string) {
if (id === "tool-redo") return "重做";
if (id === "tool-text") return "文本";
if (id === "tool-image") return "图片";
if (id === "tool-video") return "视频";
if (id === "tool-config") return "生成配置";
if (id === "tool-upload") return "上传图片";
if (id === "tool-library") return "素材库";
web/src/app/(user)/canvas/constants.ts
+5
View File
@@ -12,6 +12,7 @@ export const NODE_DEFAULT_SIZE = {
[CanvasNodeType.Image]: { width: 340, height: 240, title: "New Generation" },
[CanvasNodeType.Text]: { width: 340, height: 240, title: "Note" },
[CanvasNodeType.Config]: { width: 340, height: 240, title: "生成配置" },
[CanvasNodeType.Video]: { width: 420, height: 236, title: "Video" },
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
export const NODE_SPECS = {
@@ -27,6 +28,10 @@ export const NODE_SPECS = {
...NODE_DEFAULT_SIZE[CanvasNodeType.Config],
metadata: { content: "", status: "idle", generationMode: "image" },
},
[CanvasNodeType.Video]: {
...NODE_DEFAULT_SIZE[CanvasNodeType.Video],
metadata: { content: "", status: "idle" },
},
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
export function getNodeSpec(type: CanvasNodeType) {
web/src/app/(user)/canvas/types.ts
+2 -1
View File
@@ -13,10 +13,11 @@ export enum CanvasNodeType {
Image = "image",
Text = "text",
Config = "config",
Video = "video",
}
export type CanvasNodeStatus = "idle" | "success" | "loading" | "error";
export type CanvasGenerationMode = "text" | "image";
export type CanvasGenerationMode = "text" | "image" | "video";
export type CanvasImageGenerationType = "generation" | "edit";
export type CanvasNodeMetadata = {
web/src/components/model-picker.tsx
+1
View File
@@ -92,5 +92,6 @@ function resolveModelIcon(model: string) {
if (name.includes("claude") || name.includes("anthropic")) return "/icons/claude.svg";
if (name.includes("gemini") || name.includes("google")) return "/icons/gemini.svg";
if (name.includes("gpt") || name.includes("openai")) return "/icons/openai.svg";
if (name.includes("grok") || name.includes("grok")) return "/icons/grok.svg";
return "";
}
web/src/services/api/video.ts
+25
View File
@@ -0,0 +1,25 @@
import axios from "axios";
import { buildApiUrl, type AiConfig } from "@/stores/use-config-store";
type VideoResponse = { id: string; status?: string; error?: { message?: string } };
function aiApiUrl(config: AiConfig, path: string) {
return config.channelMode === "remote" ? `/api/v1${path}` : buildApiUrl(config.baseUrl, path);
}
function aiHeaders(config: AiConfig) {
return config.channelMode === "remote" ? undefined : { Authorization: `Bearer ${config.apiKey}` };
}
export async function requestVideoGeneration(config: AiConfig, prompt: string) {
const created = await axios.post<VideoResponse>(aiApiUrl(config, "/videos"), { model: config.model, prompt, size: config.size || undefined }, { headers: { ...(aiHeaders(config) || {}), "Content-Type": "application/json" } });
for (;;) {
const video = await axios.get<VideoResponse>(aiApiUrl(config, `/videos/${created.data.id}`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model: config.model } : undefined });
if (video.data.status === "completed") break;
if (video.data.status === "failed" || video.data.status === "cancelled") throw new Error(video.data.error?.message || "视频生成失败");
await new Promise((resolve) => setTimeout(resolve, 2500));
}
const content = await axios.get<Blob>(aiApiUrl(config, `/videos/${created.data.id}/content`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model: config.model } : undefined, responseType: "blob" });
return content.data;
}
web/src/services/file-storage.ts
+45
View File
@@ -0,0 +1,45 @@
"use client";
import localforage from "localforage";
import { nanoid } from "nanoid";
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string };
const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" });
const objectUrls = new Map<string, string>();
export async function uploadMediaFile(input: string | Blob, prefix = "file"): Promise<UploadedFile> {
const blob = typeof input === "string" ? await (await fetch(input)).blob() : input;
const storageKey = `${prefix}:${nanoid()}`;
await store.setItem(storageKey, blob);
const url = URL.createObjectURL(blob);
objectUrls.set(storageKey, url);
return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream" };
}
export async function resolveMediaUrl(storageKey?: string, fallback = "") {
if (!storageKey) return fallback;
const cached = objectUrls.get(storageKey);
if (cached) return cached;
const blob = await store.getItem<Blob>(storageKey);
if (!blob) return fallback;
const url = URL.createObjectURL(blob);
objectUrls.set(storageKey, url);
return url;
}
export async function cleanupUnusedMedia(usedData: unknown) {
const usedKeys = collectMediaStorageKeys(usedData);
const unused: string[] = [];
await store.iterate((_value, key) => {
if (!usedKeys.has(key)) unused.push(key);
});
await Promise.all(unused.map((key) => store.removeItem(key)));
}
export function collectMediaStorageKeys(value: unknown, keys = new Set<string>()) {
if (!value || typeof value !== "object") return keys;
if ("storageKey" in value && typeof value.storageKey === "string" && value.storageKey.includes(":")) keys.add(value.storageKey);
Object.values(value).forEach((item) => (Array.isArray(item) ? item.forEach((child) => collectMediaStorageKeys(child, keys)) : collectMediaStorageKeys(item, keys)));
return keys;
}
web/src/stores/use-asset-store.ts
+6 -2
View File
@@ -6,11 +6,13 @@ import { persist, type PersistStorage, type StorageValue } from "zustand/middlew
import { nanoid } from "nanoid";
import { localForageStorage } from "@/lib/localforage-storage";
import { cleanupUnusedImages, resolveImageUrl, uploadImage } from "@/services/image-storage";
import { cleanupUnusedMedia, resolveMediaUrl } from "@/services/file-storage";
export type AssetKind = "text" | "image";
export type AssetKind = "text" | "image" | "video";
export type TextAsset = AssetBase<"text"> & { data: { content: string } };
export type ImageAsset = AssetBase<"image"> & { data: { dataUrl: string; storageKey?: string; width: number; height: number; bytes: number; mimeType: string } };
export type Asset = TextAsset | ImageAsset;
export type VideoAsset = AssetBase<"video"> & { data: { url: string; storageKey?: string; width: number; height: number; bytes: number; mimeType: string } };
export type Asset = TextAsset | ImageAsset | VideoAsset;
type AssetBase<T extends AssetKind> = {
id: string;
@@ -42,6 +44,7 @@ const assetStorage: PersistStorage<AssetStore> = {
const parsed = JSON.parse(value) as StorageValue<AssetStore>;
parsed.state.assets = await Promise.all(
parsed.state.assets.map(async (asset) => {
if (asset.kind === "video" && asset.data.storageKey) return { ...asset, data: { ...asset.data, url: await resolveMediaUrl(asset.data.storageKey, asset.data.url) } };
if (asset.kind !== "image") return asset;
if (asset.data.storageKey)
return {
@@ -84,6 +87,7 @@ export const useAssetStore = create<AssetStore>()(
window.setTimeout(async () => {
const { useCanvasStore } = await import("@/app/(user)/canvas/stores/use-canvas-store");
await cleanupUnusedImages({ assets: get().assets, projects: useCanvasStore.getState().projects, extra });
await cleanupUnusedMedia({ assets: get().assets, projects: useCanvasStore.getState().projects, extra });
}, 0);
},
}),
web/src/stores/use-config-store.ts
+4 -1
View File
@@ -13,6 +13,7 @@ export type AiConfig = {
apiKey: string;
model: string;
imageModel: string;
videoModel: string;
textModel: string;
systemPrompt: string;
models: string[];
@@ -29,6 +30,7 @@ export const defaultConfig: AiConfig = {
apiKey: "",
model: "gpt-image-2",
imageModel: "gpt-image-2",
videoModel: "sora-2",
textModel: "gpt-5.5",
systemPrompt: "",
models: [],
@@ -61,6 +63,7 @@ function resolveEffectiveConfig(config: AiConfig, modelChannel: AdminPublicSetti
models,
model: models.includes(config.model) ? config.model : modelChannel.defaultModel,
imageModel: models.includes(config.imageModel) ? config.imageModel : modelChannel.defaultImageModel || modelChannel.defaultModel,
videoModel: models.includes(config.videoModel) ? config.videoModel : "sora-2",
textModel: models.includes(config.textModel) ? config.textModel : modelChannel.defaultTextModel || modelChannel.defaultModel,
systemPrompt: modelChannel.systemPrompt,
};
@@ -104,7 +107,7 @@ export const useConfigStore = create<ConfigStore>()(
partialize: (state) => ({ config: state.config }),
merge: (persisted, current) => {
const config = { ...defaultConfig, ...((persisted as Partial<ConfigStore>).config || {}) };
return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, textModel: config.textModel || config.model } };
return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "sora-2", textModel: config.textModel || config.model } };
},
},
),