From 1b2d473d9cd91dcf359cb178e27a1289a9e0300b Mon Sep 17 00:00:00 2001 From: stupid-h4er Date: Tue, 26 May 2026 23:10:49 +0800 Subject: [PATCH] feat: add video node preview --- web/src/app/(user)/assets/asset-transfer.ts | 3 +- .../(user)/canvas/[id]/canvas-client-page.tsx | 28 ++++++--- .../components/canvas-config-node-panel.tsx | 40 +++++++++++- .../components/canvas-node-generation.ts | 20 ++++++ .../(user)/canvas/components/canvas-node.tsx | 2 +- web/src/app/(user)/image/page.tsx | 4 +- web/src/app/(user)/video/page.tsx | 62 ++++++++++++++++--- web/src/components/model-picker.tsx | 2 +- 8 files changed, 137 insertions(+), 24 deletions(-) diff --git a/web/src/app/(user)/assets/asset-transfer.ts b/web/src/app/(user)/assets/asset-transfer.ts index dd50cd3..bb0a2b8 100644 --- a/web/src/app/(user)/assets/asset-transfer.ts +++ b/web/src/app/(user)/assets/asset-transfer.ts @@ -26,7 +26,8 @@ export async function exportAssets(assets: Asset[]) { await Promise.all( assets.map(async (asset) => { - const storageKey = asset.kind === "image" || asset.kind === "video" ? asset.data.storageKey : undefined; + if (asset.kind !== "image" && asset.kind !== "video") return; + const storageKey = asset.data.storageKey; if (!storageKey) return; const blob = asset.kind === "image" ? await getImageBlob(storageKey) : await getMediaBlob(storageKey); if (!blob) return; diff --git a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx index 0f5671b..20020e0 100644 --- a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx +++ b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx @@ -7,7 +7,7 @@ import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settin import { saveAs } from "file-saver"; import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image"; -import { requestVideoGeneration } from "@/services/api/video"; +import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video"; import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store"; import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage"; import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage"; @@ -1749,14 +1749,14 @@ function InfiniteCanvasPage() { position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y }, width: isEmptyVideoNode ? sourceNode.width : spec.width, height: isEmptyVideoNode ? sourceNode.height : spec.height, - metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationContext.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)) }, + metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationReferenceUrls(generationContext) }, }; pendingChildIds = [videoId]; setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode])); if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]); - const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, effectivePrompt, generationContext.referenceImages), "video"); + const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, effectivePrompt, generationContext.referenceImages, generationContext.referenceVideos)); const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT); - setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 }, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationContext.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)) } } : node))); + setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 }, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationReferenceUrls(generationContext) } } : node))); return; } @@ -1857,6 +1857,7 @@ function InfiniteCanvasPage() { setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_ERROR, errorDetails: "参考图片已丢失，无法继续重试" } } : item))); return; } + const retryImages = retryReferenceImages || []; setRunningNodeId(node.id); setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_LOADING, errorDetails: undefined } } : item))); @@ -1873,19 +1874,19 @@ function InfiniteCanvasPage() { return; } if (node.type === CanvasNodeType.Video) { - const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, prompt, retryReferenceImages || []), "video"); + const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, prompt, retryImages, context?.referenceVideos || [])); const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT); setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality } } : item))); return; } - const image = useReferenceImages ? await requestEdit(generationConfig, prompt, retryReferenceImages).then((items) => items[0]) : await requestGeneration(generationConfig, prompt).then((items) => items[0]); + const image = useReferenceImages ? await requestEdit(generationConfig, prompt, retryImages).then((items) => items[0]) : await requestGeneration(generationConfig, prompt).then((items) => items[0]); const uploadedImage = await uploadImage(image.dataUrl); const imageConfig = NODE_DEFAULT_SIZE[CanvasNodeType.Image]; const imageSize = fitNodeSize(uploadedImage.width, uploadedImage.height, imageConfig.width, imageConfig.height); const generationMetadata = savedImageMetadata?.generationType ? { generationType: savedImageMetadata.generationType, model: generationConfig.model, size: generationConfig.size, quality: generationConfig.quality, count: savedImageMetadata.count || 1, references: savedImageMetadata.references } - : buildImageGenerationMetadata(useReferenceImages ? "edit" : "generation", generationConfig, 1, retryReferenceImages || []); + : buildImageGenerationMetadata(useReferenceImages ? "edit" : "generation", generationConfig, 1, retryImages); setNodes((prev) => prev.map((item) => item.id === node.id @@ -2521,6 +2522,13 @@ function referenceUrl(image: ReferenceImage) { return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined); } +function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }> }) { + return [ + ...context.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)), + ...context.referenceVideos.map((video) => video.storageKey || video.url).filter((url): url is string => Boolean(url)), + ]; +} + async function resolveMetadataReferences(metadata: CanvasNodeMetadata) { if (metadata.generationType !== "edit") return []; if (!metadata.references?.length) return null; @@ -2574,9 +2582,10 @@ function getGenerationCount(count: string) { } function applyNodeConfigPatch(node: CanvasNodeData, patch: Partial) { - const next = { ...node, metadata: { ...node.metadata, ...(patch || {}) } }; + const safePatch = patch || {}; + const next = { ...node, metadata: { ...node.metadata, ...safePatch } }; const spec = node.type === CanvasNodeType.Video ? NODE_DEFAULT_SIZE[CanvasNodeType.Video] : NODE_DEFAULT_SIZE[CanvasNodeType.Image]; - const size = typeof patch.size === "string" && !node.metadata?.content ? nodeSizeFromRatio(patch.size, spec.width, spec.height) : null; + const size = typeof safePatch.size === "string" && !node.metadata?.content ? nodeSizeFromRatio(safePatch.size, spec.width, spec.height) : null; return size && (node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video) ? { ...next, ...size, position: { x: node.position.x + node.width / 2 - size.width / 2, y: node.position.y + node.height / 2 - size.height / 2 } } : next; } @@ -2595,6 +2604,7 @@ function getInputSummary(inputs: NodeGenerationInput[]) { return { textCount: inputs.filter((input) => input.type === "text").length, imageCount: inputs.filter((input) => input.type === "image").length, + videoCount: inputs.filter((input) => input.type === "video").length, }; } diff --git a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx index 8f5c117..216b814 100644 --- a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx @@ -18,7 +18,7 @@ import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata } from ". type CanvasConfigNodePanelProps = { node: CanvasNodeData; isRunning: boolean; - inputSummary: { textCount: number; imageCount: number }; + inputSummary: { textCount: number; imageCount: number; videoCount: number }; inputs: NodeGenerationInput[]; onConfigChange: (nodeId: string, patch: Partial) => void; onTextInputChange: (nodeId: string, content: string) => void; @@ -41,6 +41,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o const chipStyle = { background: theme.node.fill, borderColor: theme.node.stroke, color: theme.node.text }; const textInputs = inputs.filter((input) => input.type === "text"); const imageInputs = inputs.filter((input) => input.type === "image"); + const videoInputs = inputs.filter((input) => input.type === "video"); const moveInput = (input: NodeGenerationInput, offset: number) => { const sameTypeInputs = inputs.filter((item) => item.type === input.type); @@ -112,6 +113,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
event.stopPropagation()}> +
+
+ +
+ {videoInputs.map((input, index) => ( + + ))} +
+ +
@@ -288,6 +299,31 @@ function ImageSortCard({ ); } +function VideoSortCard({ + input, + videoIndex, + videoTotal, + theme, + onMove, +}: { + input: NodeGenerationInput; + videoIndex: number; + videoTotal: number; + theme: (typeof canvasThemes)[keyof typeof canvasThemes]; + onMove: (input: NodeGenerationInput, offset: number) => void; +}) { + if (!input.video) return null; + return ( +
+
+
+
+ ); +} + function VerticalOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) { return ( <> diff --git a/web/src/app/(user)/canvas/components/canvas-node-generation.ts b/web/src/app/(user)/canvas/components/canvas-node-generation.ts index 83513f2..3b4c09a 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-generation.ts +++ b/web/src/app/(user)/canvas/components/canvas-node-generation.ts @@ -1,12 +1,15 @@ import type { ChatCompletionMessage } from "@/services/api/image"; import type { ReferenceImage } from "@/types/image"; +import type { ReferenceVideo } from "@/types/media"; import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../types"; export type NodeGenerationContext = { prompt: string; referenceImages: ReferenceImage[]; + referenceVideos: ReferenceVideo[]; textCount: number; imageCount: number; + videoCount: number; }; export type NodeGenerationInput = { @@ -15,6 +18,7 @@ export type NodeGenerationInput = { title: string; text?: string; image?: ReferenceImage; + video?: ReferenceVideo; }; export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string): NodeGenerationContext { @@ -24,12 +28,15 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData .filter(Boolean) .join("\n\n"); const referenceImages = inputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image)); + const referenceVideos = inputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video)); return { prompt: upstreamText ? `${prompt}\n\n${upstreamText}` : prompt, referenceImages, + referenceVideos, textCount: inputs.filter((input) => input.type === "text").length, imageCount: referenceImages.length, + videoCount: referenceVideos.length, }; } @@ -37,6 +44,8 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[ return getOrderedUpstreamNodes(nodeId, nodes, connections).flatMap((node): NodeGenerationInput[] => { const image = readReferenceImage(node); if (image) return [{ nodeId: node.id, type: "image" as const, title: node.title, image }]; + const video = readReferenceVideo(node); + if (video) return [{ nodeId: node.id, type: "video" as const, title: node.title, video }]; const text = readNodeTextInput(node); if (text) return [{ nodeId: node.id, type: "text" as const, title: node.title, text }]; return []; @@ -77,6 +86,17 @@ function readReferenceImage(node: CanvasNodeData): ReferenceImage | null { }; } +function readReferenceVideo(node: CanvasNodeData): ReferenceVideo | null { + if (node.type !== CanvasNodeType.Video || !node.metadata?.content) return null; + return { + id: node.id, + name: `${node.title || node.id}.mp4`, + type: node.metadata.mimeType || "video/mp4", + url: node.metadata.content, + storageKey: node.metadata.storageKey, + }; +} + function getOrderedUpstreamNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) { const target = nodes.find((node) => node.id === nodeId); const upstreamNodes = connections diff --git a/web/src/app/(user)/canvas/components/canvas-node.tsx b/web/src/app/(user)/canvas/components/canvas-node.tsx index 6f63b9d..7074825 100644 --- a/web/src/app/(user)/canvas/components/canvas-node.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node.tsx @@ -466,7 +466,7 @@ function VideoNodeContent({ node, theme }: NodeContentRendererProps) { return (
); return
+
+
+ 参考视频 + +
+
+ {videoReferences.map((item) => ( +
+
+ ))} + {!videoReferences.length ?
暂无参考视频，最多 3 个
: null} +
+
+
{model} · {normalizeResolution(effectiveConfig.vquality)}p · {videoSizeLabel(effectiveConfig.size)} · {normalizeVideoSeconds(effectiveConfig.videoSeconds)}s @@ -359,7 +394,7 @@ export default function VideoPage() { { @@ -542,6 +577,12 @@ async function readStoredLogs() { async function normalizeLog(log: Partial): Promise { const video = log.video?.storageKey ? { ...log.video, url: await resolveMediaUrl(log.video.storageKey, log.video.url) } : log.video; + const videoReferences = await Promise.all( + (log.videoReferences || []).map(async (item) => ({ + ...item, + url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url, + })), + ); const references = await Promise.all( (log.references || []).map(async (item) => ({ ...item, @@ -558,6 +599,7 @@ async function normalizeLog(log: Partial): Promise model: log.model || config.videoModel || "", config, references, + videoReferences, durationMs: log.durationMs || 0, size: log.size || config.size || "", resolution: normalizeResolution(log.resolution || config.vquality || ""), @@ -572,6 +614,7 @@ function serializeLog(log: GenerationLog): GenerationLog { return { ...log, references: log.references.map((item) => ({ ...item, dataUrl: item.storageKey ? "" : item.dataUrl })), + videoReferences: log.videoReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)), video: log.video?.storageKey ? { ...log.video, url: "" } : log.video, }; } @@ -586,7 +629,7 @@ function normalizeLogConfig(log: Partial): GenerationLogConfig { }; } -function buildLog({ prompt, model, config, references, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog { +function buildLog({ prompt, model, config, references, videoReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog { const logConfig = { model: config.model, videoModel: config.videoModel, @@ -603,6 +646,7 @@ function buildLog({ prompt, model, config, references, durationMs, status, video model, config: logConfig, references, + videoReferences, durationMs, size: logConfig.size, resolution: logConfig.vquality, diff --git a/web/src/components/model-picker.tsx b/web/src/components/model-picker.tsx index 9e51980..8da2b3b 100644 --- a/web/src/components/model-picker.tsx +++ b/web/src/components/model-picker.tsx @@ -20,7 +20,7 @@ type ModelPickerProps = { export function ModelPicker({ config, value, onChange, className, fullWidth = false, placeholder = "选择模型", onMissingConfig }: ModelPickerProps) { const pickerId = useId(); const [open, setOpen] = useState(false); - const options = useMemo(() => Array.from(new Set([...(config.channelMode === "local" ? [value] : []), ...config.models].filter(Boolean))), [config.channelMode, config.models, value]); + const options = useMemo(() => Array.from(new Set([...(config.channelMode === "local" ? [value] : []), ...config.models].filter((model): model is string => Boolean(model)))), [config.channelMode, config.models, value]); const current = value || ""; useEffect(() => {