diff --git a/docs/canvas-data-structure.md b/docs/canvas-data-structure.md index dd34b5c..678858f 100644 --- a/docs/canvas-data-structure.md +++ b/docs/canvas-data-structure.md @@ -9,8 +9,9 @@ - 画布项目 JSON：`localForage`，数据库名 `infinite-canvas`，storeName `app_state`，key 为 `infinite-canvas:canvas_store`。 - 我的素材 JSON：`localForage`，数据库名 `infinite-canvas`，storeName `app_state`，key 为 `infinite-canvas:asset_store`。 - 图片 Blob：单独存到 `localForage` 实例，数据库名 `infinite-canvas`，storeName `image_files`。 +- 视频等媒体 Blob：单独存到 `localForage` 实例，数据库名 `infinite-canvas`，storeName `media_files`。 -画布 JSON 不直接长期保存大体积 base64 图片。图片节点、助手图片和素材图片只保存展示 URL、`storageKey` 和图片元信息，真实图片 Blob 通过 `storageKey` 读取。 +画布 JSON 不直接长期保存大体积 base64 图片或视频。图片节点、视频节点、助手图片和素材媒体只保存展示 URL、`storageKey` 和元信息，真实 Blob 通过 `storageKey` 读取。 ## 画布项目结构 @@ -50,7 +51,7 @@ type CanvasProject = { ```ts type CanvasNodeData = { id: string; - type: "image" | "text" | "config"; + type: "image" | "text" | "config" | "video"; title: string; position: { x: number; y: number }; width: number; @@ -62,7 +63,7 @@ type CanvasNodeData = { 通用字段： - `id`：节点 ID。 -- `type`：节点类型，当前有图片、文本、生成配置三类。 +- `type`：节点类型，当前有图片、文本、生成配置、视频四类。 - `title`：节点标题。 - `position`：画布世界坐标，不是屏幕坐标。 - `width` / `height`：画布世界坐标下的节点尺寸。 @@ -77,7 +78,7 @@ type CanvasNodeMetadata = { status?: "idle" | "success" | "loading" | "error"; errorDetails?: string; fontSize?: number; - generationMode?: "text" | "image"; + generationMode?: "text" | "image" | "video"; model?: string; size?: string; count?: number; @@ -99,8 +100,9 @@ type CanvasNodeMetadata = { 不同节点的使用方式： - 图片节点：`content` 是当前可展示的图片 URL，通常是 `blob:` URL；`storageKey` 指向本地图片 Blob；`naturalWidth/naturalHeight/bytes/mimeType` 保存原图信息。 +- 视频节点：`content` 是当前可播放的视频 URL，通常是 `blob:` URL；`storageKey` 指向本地视频 Blob；`bytes/mimeType` 保存文件信息。 - 文本节点：`content` 保存文本内容；`fontSize` 保存字体大小；`prompt/status/errorDetails` 保存生成状态。 -- 生成配置节点：`generationMode/model/size/count/inputOrder` 保存生成配置；上游输入通过 `connections` 计算。 +- 生成配置节点：`generationMode/model/size/count/inputOrder` 保存生成配置；`generationMode` 可选择文本、图片或视频；上游输入通过 `connections` 计算。 - 图片组节点：根节点用 `isBatchRoot/batchChildIds/primaryImageId/imageBatchExpanded` 记录批量生成结果；子图节点用 `batchRootId` 指回根节点。 ## 连线结构 diff --git a/docs/canvas-node-manual.md b/docs/canvas-node-manual.md index 090dc49..845b9a6 100644 --- a/docs/canvas-node-manual.md +++ b/docs/canvas-node-manual.md @@ -39,6 +39,14 @@ - 每个生成出来的图片节点都会在自身 `metadata` 下记录提示词、生成类型、模型、尺寸、质量、数量和参考图引用，方便后续重试。 - 如果重试时参考图已经丢失或无法恢复，系统会直接提示，避免误发空的图生图请求。 +### 视频节点 + +- 工具栏可以新建视频节点，也可以拖入或上传本地视频文件。 +- 视频节点使用原生播放器展示内容，可在节点内直接播放、暂停和拖动进度。 +- 空视频节点下方对话框可输入提示词生成视频，结果会回填到当前节点。 +- 从文本、图片或配置节点创建视频生成时，会在右侧生成新的视频节点并自动连接。 +- 视频生成接口使用 OpenAI 兼容的 `POST /v1/videos`、`GET /v1/videos/{id}` 和 `GET /v1/videos/{id}/content`。 + ### 推荐流程 1. 新建文本节点，写入图片创作想法。 diff --git a/docs/pending-test.md b/docs/pending-test.md index 9876946..465b2cf 100644 --- a/docs/pending-test.md +++ b/docs/pending-test.md @@ -1,5 +1,6 @@ # 待测试 +- 画布新增视频节点：工具栏可新建视频节点，上传/拖拽本地视频后可在节点内播放、下载、保存到我的素材；节点下方对话框和生成配置节点支持视频生成，前端会调用 OpenAI 兼容 `videos` 接口并把生成结果保存为本地视频 Blob；后端远程渠道新增 `/api/v1/videos`、`/api/v1/videos/:id`、`/api/v1/videos/:id/content` 代理，需要确认本地渠道和远程渠道都能生成、轮询和下载视频。 - 模型选择器改为共享的 shadcn 风格下拉，按模型名称显示 OpenAI、Claude、Gemini 图标，并把画布节点下方、右侧助手和配置弹窗统一到同一套组件；右侧助手的模型和模式按钮会随工具栏宽度自动在文字态和图标态之间切换，需要确认生图页、画布助手、节点配置和配置弹窗中的选中态和下拉列表显示正常。 - 画布图像设置里的尺寸输入去掉了原生数字步进箭头，需要确认宽高仍可直接输入、禁用态正常、设置值回写正常。 - 后台提示词定时同步默认改为开启且默认每 5 分钟执行一次，需要确认新环境未保存过设置时开关默认打开、Cron 默认值正确，用户手动关闭或修改 Cron 后能保持配置。 diff --git a/docs/todo.md b/docs/todo.md index 7e444ef..6a1ee7b 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -14,7 +14,6 @@ - 日志管理：新增 `credit_logs` 记录用户算力点变更流水，新增 `api_logs` 记录第三方大模型 API 调用情况。 - 对话工具调用：在助手对话中增加类似工具调用的形式，展示模型调用、生成图片、插入画布等步骤。 - 画布助手对话框优化：把当前对话面板调整得更简洁，减少干扰，突出输入、引用和结果。 -- 考虑引入视频生成能力 ## P1 算力点和商业化 diff --git a/handler/ai.go b/handler/ai.go index f76053c..9e5050f 100644 --- a/handler/ai.go +++ b/handler/ai.go @@ -25,6 +25,38 @@ func AIChatCompletions(w http.ResponseWriter, r *http.Request) { proxyAIRequest(w, r, "/chat/completions") } +func AIVideos(w http.ResponseWriter, r *http.Request) { + proxyAIRequest(w, r, "/videos") +} + +func AIVideo(w http.ResponseWriter, r *http.Request, id string) { + proxyAIGetRequest(w, r, "/videos/"+id) +} + +func AIVideoContent(w http.ResponseWriter, r *http.Request, id string) { + proxyAIGetRequest(w, r, "/videos/"+id+"/content") +} + +func proxyAIGetRequest(w http.ResponseWriter, r *http.Request, path string) { + modelName := r.URL.Query().Get("model") + if strings.TrimSpace(modelName) == "" { + modelName = "sora-2" + } + channel, err := service.SelectModelChannel(modelName) + if err != nil { + log.Printf("AI proxy select channel failed: model=%s err=%v", modelName, err) + Fail(w, "AI 接口请求失败") + return + } + request, err := http.NewRequest(http.MethodGet, service.BuildModelChannelURL(channel, path), nil) + if err != nil { + Fail(w, "AI 接口请求失败") + return + } + request.Header.Set("Authorization", "Bearer "+channel.APIKey) + copyAIResponse(w, request) +} + func proxyAIRequest(w http.ResponseWriter, r *http.Request, path string) { body, contentType, modelName, err := readAIRequest(r) if err != nil { @@ -48,6 +80,10 @@ func proxyAIRequest(w http.ResponseWriter, r *http.Request, path string) { if contentType != "" { request.Header.Set("Content-Type", contentType) } + copyAIResponse(w, request) +} + +func copyAIResponse(w http.ResponseWriter, request *http.Request) { response, err := http.DefaultClient.Do(request) if err != nil { log.Printf("AI proxy request failed: url=%s err=%v", request.URL.String(), err) diff --git a/router/router.go b/router/router.go index 59e5a75..b4f6c6a 100644 --- a/router/router.go +++ b/router/router.go @@ -23,6 +23,13 @@ func New() *gin.Engine { api.POST("/v1/images/generations", gin.WrapF(handler.AIImagesGenerations)) api.POST("/v1/images/edits", gin.WrapF(handler.AIImagesEdits)) api.POST("/v1/chat/completions", gin.WrapF(handler.AIChatCompletions)) + api.POST("/v1/videos", gin.WrapF(handler.AIVideos)) + api.GET("/v1/videos/:id", func(c *gin.Context) { + handler.AIVideo(c.Writer, c.Request, c.Param("id")) + }) + api.GET("/v1/videos/:id/content", func(c *gin.Context) { + handler.AIVideoContent(c.Writer, c.Request, c.Param("id")) + }) api.GET("/prompts", middleware.OptionalAuth, gin.WrapF(handler.Prompts)) api.GET("/assets", middleware.OptionalAuth, gin.WrapF(handler.Assets)) api.POST("/admin/login", gin.WrapF(handler.AdminLogin)) diff --git a/web/public/icons/grok.svg b/web/public/icons/grok.svg new file mode 100644 index 0000000..efb1a61 --- /dev/null +++ b/web/public/icons/grok.svg @@ -0,0 +1 @@ +Grok \ No newline at end of file diff --git a/web/src/app/(user)/assets/page.tsx b/web/src/app/(user)/assets/page.tsx index c8ddc93..d92d37d 100644 --- a/web/src/app/(user)/assets/page.tsx +++ b/web/src/app/(user)/assets/page.tsx @@ -26,6 +26,7 @@ const kindOptions = [ { label: "全部", value: "all" }, { label: "文本", value: "text" }, { label: "图片", value: "image" }, + { label: "视频", value: "video" }, ]; export default function AssetsPage() { @@ -52,7 +53,7 @@ export default function AssetsPage() { const title = Form.useWatch("title", form) || ""; const tags = Form.useWatch("tags", form) || []; const content = Form.useWatch("content", form) || ""; - const validAssets = useMemo(() => assets.filter((asset) => asset.kind === "text" || asset.kind === "image"), [assets]); + const validAssets = useMemo(() => assets.filter((asset) => asset.kind === "text" || asset.kind === "image" || asset.kind === "video"), [assets]); const filteredAssets = useMemo(() => { const query = keyword.trim().toLowerCase(); @@ -145,9 +146,9 @@ export default function AssetsPage() { }; const downloadImage = (asset: Asset) => { - if (asset.kind !== "image") return; + if (asset.kind !== "image" && asset.kind !== "video") return; const link = document.createElement("a"); - link.href = asset.data.dataUrl; + link.href = asset.kind === "video" ? asset.data.url : asset.data.dataUrl; link.download = `${asset.title || "asset"}.${asset.data.mimeType.split("/")[1] || "png"}`; link.click(); }; @@ -385,7 +386,7 @@ function AssetCard({ asset, onOpen, onEdit, onCopy, onDownload, onDelete }: { as {asset.source || "未标注来源"} - {asset.kind === "image" ? "图片" : "文本"} + {asset.kind === "image" ? "图片" : asset.kind === "video" ? "视频" : "文本"} {summary} @@ -404,15 +405,17 @@ function AssetCard({ asset, onOpen, onEdit, onCopy, onDownload, onDelete }: { as - + {asset.kind !== "video" ? ( + + ) : null} {asset.kind === "text" ? ( ) : null} - {asset.kind === "image" ? ( + {asset.kind === "image" || asset.kind === "video" ? ( @@ -441,7 +444,7 @@ function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: Asset | nu {asset.title} - {asset.kind === "image" ? "图片" : "文本"} + {asset.kind === "image" ? "图片" : asset.kind === "video" ? "视频" : "文本"} {(asset.tags || []).map((tag) => ( {tag} ))} @@ -453,6 +456,8 @@ function AssetDrawer({ asset, onClose, onCopy, onDownload }: { asset: Asset | nu {asset.kind === "text" ? ( {asset.data.content} + ) : asset.kind === "video" ? ( + diff --git a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx index 0e8dd24..5edd9ed 100644 --- a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx +++ b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx @@ -3,11 +3,13 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react"; import { useParams, useRouter } from "next/navigation"; -import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload } from "lucide-react"; +import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react"; import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image"; +import { requestVideoGeneration } from "@/services/api/video"; import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store"; import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage"; +import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage"; import { nanoid } from "nanoid"; import { getDataUrlByteSize, readImageMeta } from "@/lib/image-utils"; import { canvasThemes, type CanvasBackgroundMode } from "@/lib/canvas-theme"; @@ -135,7 +137,7 @@ function CanvasRefreshShell() { ); } -function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config) => void; onClose: () => void }) { +function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video) => void; onClose: () => void }) { const theme = canvasThemes[useThemeStore((state) => state.theme)]; return (
} title="文本生成" description="脚本、广告词、品牌文案" onClick={() => onCreate(CanvasNodeType.Text)} /> } title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} /> + } title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} /> } title="配置节点" description="模型、尺寸、数量和输入顺序" onClick={() => onCreate(CanvasNodeType.Config)} />
@@ -463,7 +466,7 @@ function InfiniteCanvasPage() { ); const createConnectedNode = useCallback( - (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config, pending: PendingConnectionCreate) => { + (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video, pending: PendingConnectionCreate) => { const metadata = type === CanvasNodeType.Config ? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: 3 } : undefined; const newNode = createCanvasNode(type, pending.position, metadata); const connection = normalizeConnection(pending.connection.nodeId, newNode.id, [...nodesRef.current, newNode], pending.connection.handleType); @@ -1069,6 +1072,27 @@ function InfiniteCanvasPage() { setDialogNodeId(id); }, []); + const createVideoFileNode = useCallback(async (file: File, position: Position) => { + const video = await uploadMediaFile(file, "video"); + const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video]; + const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + setNodes((prev) => [ + ...prev, + { + id, + type: CanvasNodeType.Video, + title: file.name, + position: { x: position.x - spec.width / 2, y: position.y - spec.height / 2 }, + width: spec.width, + height: spec.height, + metadata: videoMetadata(video), + }, + ]); + setSelectedNodeIds(new Set([id])); + setSelectedConnectionId(null); + setDialogNodeId(id); + }, []); + const createTextNodeFromClipboard = useCallback( (text: string) => { const trimmed = text.trim(); @@ -1282,10 +1306,10 @@ function InfiniteCanvasPage() { }, []); const downloadNodeImage = useCallback((node: CanvasNodeData) => { - if (node.type !== CanvasNodeType.Image || !node.metadata?.content) return; + if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video) || !node.metadata?.content) return; const link = document.createElement("a"); link.href = node.metadata.content; - link.download = `canvas-image-${node.id}.${imageExtension(node.metadata.content)}`; + link.download = `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : imageExtension(node.metadata.content)}`; link.click(); }, []); @@ -1298,6 +1322,12 @@ function InfiniteCanvasPage() { message.success("已加入我的素材"); return; } + if (node.type === CanvasNodeType.Video) { + if (!node.metadata?.content) return message.error("没有可保存的视频"); + addAsset({ kind: "video", title: node.metadata?.prompt?.slice(0, 24) || "画布视频", coverUrl: "", tags: [], source: "Canvas", data: { url: node.metadata.content, storageKey: node.metadata.storageKey, width: node.width, height: node.height, bytes: node.metadata.bytes || 0, mimeType: node.metadata.mimeType || "video/mp4" }, metadata: { source: "canvas", nodeId: node.id, prompt: node.metadata?.prompt } }); + message.success("已加入我的素材"); + return; + } if (!node.metadata?.content) return message.error("没有可保存的图片"); const dataUrl = node.metadata.storageKey ? "" : node.metadata.content; addAsset({ @@ -1408,9 +1438,19 @@ function InfiniteCanvasPage() { async (event: ReactChangeEvent) => { const file = event.target.files?.[0]; const target = uploadTargetRef.current; - if (!file || !file.type.startsWith("image/")) return; + if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/"))) return; if (target?.nodeId) { + if (file.type.startsWith("video/")) { + const video = await uploadMediaFile(file, "video"); + setNodes((prev) => prev.map((node) => (node.id === target.nodeId ? { ...node, type: CanvasNodeType.Video, title: file.name, width: NODE_DEFAULT_SIZE[CanvasNodeType.Video].width, height: NODE_DEFAULT_SIZE[CanvasNodeType.Video].height, metadata: { ...node.metadata, ...videoMetadata(video), errorDetails: undefined } } : node))); + setSelectedNodeIds(new Set([target.nodeId])); + setSelectedConnectionId(null); + setDialogNodeId(target.nodeId); + uploadTargetRef.current = null; + event.target.value = ""; + return; + } const image = await uploadImage(file); const size = fitImageNodeSize(image.width, image.height); setNodes((prev) => @@ -1449,25 +1489,25 @@ function InfiniteCanvasPage() { setDialogNodeId(target.nodeId); } else { const position = target?.position || screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2); - void createImageFileNode(file, position); + void (file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position)); } uploadTargetRef.current = null; event.target.value = ""; }, - [createImageFileNode, screenToCanvas, size.height, size.width], + [createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width], ); const handleDrop = useCallback( (event: ReactDragEvent) => { event.preventDefault(); - const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/")); + const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/")); if (!file) return; const pos = screenToCanvas(event.clientX, event.clientY); - void createImageFileNode(file, pos); + void (file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos)); }, - [createImageFileNode, screenToCanvas], + [createImageFileNode, createVideoFileNode, screenToCanvas], ); const pasteAssistantImage = useCallback( @@ -1681,6 +1721,28 @@ function InfiniteCanvasPage() { return; } + if (mode === "video") { + const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video]; + const isEmptyVideoNode = sourceNode?.type === CanvasNodeType.Video && !sourceNode.metadata?.content; + const videoId = isEmptyVideoNode ? nodeId : nanoid(); + const parent = sourceNode?.position || { x: 0, y: 0 }; + const videoNode: CanvasNodeData = { + id: videoId, + type: CanvasNodeType.Video, + title: effectivePrompt.slice(0, 32) || "Generated Video", + position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y }, + width: isEmptyVideoNode ? sourceNode.width : spec.width, + height: isEmptyVideoNode ? sourceNode.height : spec.height, + metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size }, + }; + pendingChildIds = [videoId]; + setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode])); + if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]); + const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, effectivePrompt), "video"); + setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size } } : node))); + return; + } + let streamed = ""; const isConfigNode = sourceNode?.type === CanvasNodeType.Config; const textCount = isConfigNode ? getGenerationCount(generationConfig.count) : 1; @@ -1906,12 +1968,18 @@ function InfiniteCanvasPage() { (payload: InsertAssetPayload) => { if (payload.kind === "text") { insertAssistantText(payload.content); + } else if (payload.kind === "video") { + const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Video]; + const center = screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2); + const id = `video-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + setNodes((prev) => [...prev, { id, type: CanvasNodeType.Video, title: payload.title, position: { x: center.x - spec.width / 2, y: center.y - spec.height / 2 }, width: spec.width, height: spec.height, metadata: { content: payload.url, storageKey: payload.storageKey, status: NODE_STATUS_SUCCESS } }]); + setSelectedNodeIds(new Set([id])); } else { insertAssistantImage({ id: `asset-${Date.now()}`, prompt: payload.title, dataUrl: payload.dataUrl, storageKey: payload.storageKey }); } setAssetPickerOpen(false); }, - [insertAssistantImage, insertAssistantText], + [insertAssistantImage, insertAssistantText, screenToCanvas, size.height, size.width], ); if (!projectLoaded) return ; @@ -2100,6 +2168,7 @@ function InfiniteCanvasPage() { canRedo={historyState.canRedo} backgroundMode={backgroundMode} onAddImage={() => createNode(CanvasNodeType.Image)} + onAddVideo={() => createNode(CanvasNodeType.Video)} onAddText={() => createNode(CanvasNodeType.Text)} onAddConfig={() => createNode(CanvasNodeType.Config)} onUndo={undoCanvas} @@ -2138,7 +2207,7 @@ function InfiniteCanvasPage() { /> ) : null} - + setInfoNodeId(null)} /> @@ -2406,6 +2475,10 @@ function imageMetadata(image: UploadedImage): CanvasNodeMetadata { return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType }; } +function videoMetadata(video: UploadedFile): CanvasNodeMetadata { + return { content: video.url, storageKey: video.storageKey, status: "success", bytes: video.bytes, mimeType: video.mimeType || "video/mp4" }; +} + function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata { return { generationType: type, @@ -2437,6 +2510,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) { return Promise.all( nodes.map(async (node) => { const content = node.metadata?.content; + if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } }; if (node.type !== CanvasNodeType.Image || !content) return node; if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } }; if (!content.startsWith("data:image/")) return node; @@ -2502,7 +2576,7 @@ function getInputSummary(inputs: NodeGenerationInput[]) { } function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefined, mode: CanvasNodeGenerationMode): AiConfig { - const defaultModel = mode === "image" ? config.imageModel : config.textModel; + const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : config.textModel; return { ...config, model: node?.metadata?.model || defaultModel || config.model || defaultConfig.model, diff --git a/web/src/app/(user)/canvas/components/asset-picker-modal.tsx b/web/src/app/(user)/canvas/components/asset-picker-modal.tsx index f610b2b..39818fc 100644 --- a/web/src/app/(user)/canvas/components/asset-picker-modal.tsx +++ b/web/src/app/(user)/canvas/components/asset-picker-modal.tsx @@ -12,7 +12,7 @@ import { fetchAssetLibrary, type AssetLibraryItem } from "@/services/api/assets" export type AssetPickerTab = "my-assets" | "library"; -export type InsertAssetPayload = { kind: "text"; content: string; title: string } | { kind: "image"; dataUrl: string; title: string; storageKey?: string }; +export type InsertAssetPayload = { kind: "text"; content: string; title: string } | { kind: "image"; dataUrl: string; title: string; storageKey?: string } | { kind: "video"; url: string; title: string; storageKey?: string }; type Props = { open: boolean; @@ -48,6 +48,7 @@ const kindOptions = [ { label: "全部", value: "all" }, { label: "文本", value: "text" }, { label: "图片", value: "image" }, + { label: "视频", value: "video" }, ]; function LibraryTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) => void }) { @@ -157,7 +158,7 @@ function PickerCard({ title, kind, cover, loading, onClick }: { title: string; k
{title} - {kind === "image" ? "图片" : "文本"} + {kind === "image" ? "图片" : kind === "video" ? "视频" : "文本"}
{loading && ( @@ -190,7 +191,7 @@ function MyAssetsTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) => const filtered = useMemo(() => { const query = keyword.trim().toLowerCase(); return assets - .filter((a) => a.kind === "text" || a.kind === "image") + .filter((a) => a.kind === "text" || a.kind === "image" || a.kind === "video") .filter((a) => kindFilter === "all" || a.kind === kindFilter) .filter((a) => !query || [a.title, ...(a.tags || [])].join(" ").toLowerCase().includes(query)); }, [assets, keyword, kindFilter]); @@ -206,7 +207,7 @@ function MyAssetsTab({ onInsert }: { onInsert: (payload: InsertAssetPayload) => if (asset.kind === "text") { onInsert({ kind: "text", content: asset.data.content, title: asset.title }); } else { - onInsert({ kind: "image", dataUrl: asset.data.dataUrl, storageKey: asset.data.storageKey, title: asset.title }); + onInsert(asset.kind === "video" ? { kind: "video", url: asset.data.url, storageKey: asset.data.storageKey, title: asset.title } : { kind: "image", dataUrl: asset.data.dataUrl, storageKey: asset.data.storageKey, title: asset.title }); } }; diff --git a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx index 32224a4..a88ba8f 100644 --- a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx @@ -2,7 +2,7 @@ import type { CSSProperties } from "react"; import { useState } from "react"; -import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play } from "lucide-react"; +import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play, Video } from "lucide-react"; import { App, Button, Empty, Input, InputNumber, Modal, Segmented } from "antd"; import { ModelPicker } from "@/components/model-picker"; @@ -91,6 +91,15 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o ), }, + { + value: "video", + label: ( + + + ), + }, ]} /> @@ -107,7 +116,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
event.stopPropagation()}> onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} fullWidth /> - {mode === "image" ? onConfigChange(node.id, { size: value })} /> : null} + {mode === "image" || mode === "video" ? onConfigChange(node.id, { size: value })} /> : null} onConfigChange(node.id, { count: Number(value) || 1 })} />
@@ -294,7 +303,7 @@ function InputChip({ label, value, style }: { label: string; value: string; styl } function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: CanvasGenerationMode): AiConfig { - const defaultModel = mode === "image" ? globalConfig.imageModel : globalConfig.textModel; + const defaultModel = mode === "image" ? globalConfig.imageModel : mode === "video" ? globalConfig.videoModel : globalConfig.textModel; return { ...globalConfig, model: node.metadata?.model || defaultModel || globalConfig.model || defaultConfig.model, diff --git a/web/src/app/(user)/canvas/components/canvas-node-generation.ts b/web/src/app/(user)/canvas/components/canvas-node-generation.ts index 5767aa0..83513f2 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-generation.ts +++ b/web/src/app/(user)/canvas/components/canvas-node-generation.ts @@ -11,7 +11,7 @@ export type NodeGenerationContext = { export type NodeGenerationInput = { nodeId: string; - type: "text" | "image"; + type: "text" | "image" | "video"; title: string; text?: string; image?: ReferenceImage; diff --git a/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx b/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx index b2337f3..f43c6b0 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; import { Modal, Segmented, Tooltip } from "antd"; -import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload } from "lucide-react"; +import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; import { formatBytes, getDataUrlByteSize } from "@/lib/image-utils"; @@ -57,12 +57,14 @@ export function CanvasNodeHoverToolbar({ const left = viewport.x + (node.position.x + node.width / 2) * viewport.k; const top = viewport.y + node.position.y * viewport.k - 14; const isImage = node.type === CanvasNodeType.Image; + const isVideo = node.type === CanvasNodeType.Video; const hasImage = isImage && Boolean(node.metadata?.content); + const hasVideo = isVideo && Boolean(node.metadata?.content); const isText = node.type === CanvasNodeType.Text; const isConfig = node.type === CanvasNodeType.Config; - const canOpenDialog = isText || hasImage; + const canOpenDialog = isText || hasImage || isVideo; const canRetry = node.metadata?.status === "error"; - const hasSpecificTools = canRetry || isText || isImage || isConfig; + const hasSpecificTools = canRetry || isText || isImage || isVideo || isConfig; return (
} onClick={() => onDelete(node)} danger /> {hasSpecificTools ? : null} {canRetry ? } onClick={() => onRetry(node)} /> : null} - {hasImage || isText ? } onClick={() => onSaveAsset(node)} /> : null} - {hasImage ? } onClick={() => onDownload(node)} /> : null} + {hasImage || hasVideo || isText ? } onClick={() => onSaveAsset(node)} /> : null} + {hasImage || hasVideo ? } onClick={() => onDownload(node)} /> : null} {canOpenDialog ? } onClick={() => onToggleDialog(node)} /> : null} {isText ? } onClick={() => onEditText(node)} /> : null} {isText ? } onClick={() => onGenerateImage(node)} /> : null} @@ -86,6 +88,7 @@ export function CanvasNodeHoverToolbar({ {isText ? } onClick={() => onDecreaseFont(node)} /> : null} {isText ? } onClick={() => onIncreaseFont(node)} /> : null} {isImage ? } onClick={() => onUpload(node)} /> : null} + {isVideo ? } onClick={() => onUpload(node)} /> : null} {hasImage ? ( - + diff --git a/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx b/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx index fb0f6a4..2abaac4 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx @@ -68,7 +68,7 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi }} className="thin-scrollbar h-24 w-full resize-none rounded-xl border px-3 py-2 text-sm leading-5 outline-none" style={{ background: theme.node.fill, borderColor: theme.node.stroke, color: theme.node.text }} - placeholder={mode === "image" ? (hasImageContent ? "请输入你想要把这张图修改成什么" : "描述要生成的图片内容") : hasTextContent ? "请输入你想要将本段文本修改成什么" : "请输入你想要生成的文本内容"} + placeholder={mode === "video" ? "描述要生成的视频内容" : mode === "image" ? (hasImageContent ? "请输入你想要把这张图修改成什么" : "描述要生成的图片内容") : hasTextContent ? "请输入你想要将本段文本修改成什么" : "请输入你想要生成的文本内容"} />
@@ -105,11 +105,11 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi } function defaultMode(type: CanvasNodeData["type"]): CanvasNodeGenerationMode { - return type === CanvasNodeType.Text ? "text" : "image"; + return type === CanvasNodeType.Text ? "text" : type === CanvasNodeType.Video ? "video" : "image"; } function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: CanvasNodeGenerationMode): AiConfig { - const defaultModel = mode === "image" ? globalConfig.imageModel : globalConfig.textModel; + const defaultModel = mode === "image" ? globalConfig.imageModel : mode === "video" ? globalConfig.videoModel : globalConfig.textModel; return { ...globalConfig, model: node.metadata?.model || defaultModel || globalConfig.model || defaultConfig.model, diff --git a/web/src/app/(user)/canvas/components/canvas-node.tsx b/web/src/app/(user)/canvas/components/canvas-node.tsx index f6e991a..9af34cd 100644 --- a/web/src/app/(user)/canvas/components/canvas-node.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node.tsx @@ -2,7 +2,7 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import type { ReactNode } from "react"; -import { ChevronRight, Image as ImageIcon, RefreshCw, Star } from "lucide-react"; +import { ChevronRight, Image as ImageIcon, RefreshCw, Star, Video } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; import { useThemeStore } from "@/stores/use-theme-store"; @@ -95,6 +95,7 @@ export const CanvasNode = React.memo(function CanvasNode({ const [hovered, setHovered] = useState(false); const [isEditingContent, setIsEditingContent] = useState(false); const hasImageContent = data.type === CanvasNodeType.Image && Boolean(data.metadata?.content); + const hasVideoContent = data.type === CanvasNodeType.Video && Boolean(data.metadata?.content); const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1; const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId); const isActive = isConnectionTarget || isSelected || isFocusRelated; @@ -246,7 +247,7 @@ export const CanvasNode = React.memo(function CanvasNode({
- {!hasImageContent ?
: null} + {!hasImageContent && !hasVideoContent ?
: null} @@ -325,6 +326,7 @@ const nodeContentRenderers = { [CanvasNodeType.Text]: TextContent, [CanvasNodeType.Image]: ImageNodeContent, [CanvasNodeType.Config]: EmptyImageContent, + [CanvasNodeType.Video]: VideoNodeContent, } satisfies Record ReactNode>; function LoadingContent({ theme }: Pick) { @@ -454,6 +456,17 @@ function EmptyImageContent({ theme, isBatchRoot, batchCount, batchExpanded, batc return content; } +function VideoNodeContent({ node, theme }: NodeContentRendererProps) { + if (!node.metadata?.content) + return ( +
+
+ ); + return