Hermes/infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: improve seedance reference handling

Browse Source
This commit is contained in:
stupid-h4er
2026-05-27 21:55:05 +08:00
parent 4bffb55695
commit 74ea4b81f2
9 changed files with 165 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/pending-test.md
+1
View File
@@ -1,5 +1,6 @@
# 待测试
- Seedance 参考素材失败原因排查：后端会把火山上游错误摘要返回给前端；`/video` 和画布视频生成会按 `图片1/视频1/音频1` 自动编号参考素材，并在实际请求提示词中注入编号说明；参考视频会在请求前校验大小、时长、宽高、宽高比和像素总量。
- 画布项目导出改为下载 `.zip` 压缩包，包内包含 `projects.json` 和当前画布引用到的本地图片、视频文件，避免只导出 JSON 时丢失媒体内容。
- 画布库支持多选后一键导出多个画布项目，导出的压缩包可一次恢复多个项目。
- 画布项目导入改为读取新版 `.zip` 压缩包，会先按 `projects.json` 中的文件映射恢复图片、视频到本地存储，再插入画布项目，导入成功后仍停留在画布库。
handler/ai.go
+50 -2
View File
@@ -119,12 +119,12 @@ func copyAIResponse(w http.ResponseWriter, request *http.Request, onFailure func
defer response.Body.Close()
if response.StatusCode >= http.StatusBadRequest {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
log.Printf("AI upstream error: url=%s status=%d", request.URL.String(), response.StatusCode)
if onFailure != nil {
onFailure()
}
Fail(w, aiStatusMessage(response.StatusCode))
Fail(w, aiUpstreamStatusMessage(response.StatusCode, body))
return
}
@@ -239,6 +239,54 @@ func aiStatusMessage(statusCode int) string {
}
}
func aiUpstreamStatusMessage(statusCode int, body []byte) string {
base := aiStatusMessage(statusCode)
detail := aiUpstreamErrorDetail(body)
if detail == "" {
return base
}
return base + "：" + detail
}
func aiUpstreamErrorDetail(body []byte) string {
text := strings.TrimSpace(string(body))
if text == "" {
return ""
}
var payload struct {
Msg string `json:"msg"`
Message string `json:"message"`
Error struct {
Code string `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
if err := json.Unmarshal(body, &payload); err == nil {
if payload.Error.Message != "" {
if payload.Error.Code != "" {
return safeUpstreamText(payload.Error.Code + " " + payload.Error.Message)
}
return safeUpstreamText(payload.Error.Message)
}
if payload.Msg != "" {
return safeUpstreamText(payload.Msg)
}
if payload.Message != "" {
return safeUpstreamText(payload.Message)
}
}
return safeUpstreamText(text)
}
func safeUpstreamText(text string) string {
text = strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
runes := []rune(text)
if len(runes) > 300 {
return string(runes[:300]) + "..."
}
return text
}
type aiError struct {
message string
}
handler/ai_test.go
+20
View File
@@ -0,0 +1,20 @@
package handler
import (
"strings"
"testing"
)
func TestAIUpstreamErrorDetail(t *testing.T) {
got := aiUpstreamErrorDetail([]byte(`{"error":{"code":"InvalidParameter","message":"reference video fps is invalid"}}`))
if got != "InvalidParameter reference video fps is invalid" {
t.Fatalf("detail = %q", got)
}
}
func TestSafeUpstreamTextTruncates(t *testing.T) {
got := safeUpstreamText(strings.Repeat("错", 320))
if len([]rune(got)) != 303 {
t.Fatalf("truncated rune length = %d", len([]rune(got)))
}
}
web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx
+4 -3
View File
@@ -9,6 +9,7 @@ import { ModelPicker } from "@/components/model-picker";
import { defaultConfig, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
import { CreditSymbol, requestCreditCost } from "@/constant/credits";
import { canvasThemes } from "@/lib/canvas-theme";
import { seedanceReferenceLabel } from "@/lib/seedance-video";
import { useThemeStore } from "@/stores/use-theme-store";
import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover";
import { CanvasVideoSettingsPopover } from "./canvas-video-settings-popover";
@@ -303,7 +304,7 @@ function ImageSortCard({
<div className="w-24 shrink-0 overflow-hidden rounded-lg border" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
<div className="relative">
<img src={input.image.dataUrl} alt={input.title} className="aspect-square w-full object-cover" />
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{imageIndex + 1}</span>
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{seedanceReferenceLabel("image", imageIndex)}</span>
<HorizontalOrderButtons index={imageIndex} total={imageTotal} onMove={(offset) => onMove(input, offset)} />
</div>
</div>
@@ -328,7 +329,7 @@ function VideoSortCard({
<div className="w-32 shrink-0 overflow-hidden rounded-lg border" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
<div className="relative">
<video src={input.video.url} className="aspect-video w-full bg-black object-cover" muted preload="metadata" />
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{videoIndex + 1}</span>
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{seedanceReferenceLabel("video", videoIndex)}</span>
<HorizontalOrderButtons index={videoIndex} total={videoTotal} onMove={(offset) => onMove(input, offset)} />
</div>
</div>
@@ -358,7 +359,7 @@ function AudioSortCard({
<audio src={input.audio.url} controls className="h-8 w-full" preload="metadata" />
<div className="mt-1 flex justify-between">
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !p-0" icon={<ArrowLeft className="size-3" />} disabled={audioIndex <= 0} onClick={() => onMove(input, -1)} />
<span className="text-[10px] opacity-45">{audioIndex + 1}</span>
<span className="text-[10px] opacity-45">{seedanceReferenceLabel("audio", audioIndex)}</span>
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !p-0" icon={<ArrowRight className="size-3" />} disabled={audioIndex >= audioTotal - 1} onClick={() => onMove(input, 1)} />
</div>
</div>
web/src/app/(user)/canvas/components/canvas-node-generation.ts
+4
View File
@@ -102,6 +102,10 @@ function readReferenceVideo(node: CanvasNodeData): ReferenceVideo | null {
type: node.metadata.mimeType || "video/mp4",
url: node.metadata.content,
storageKey: node.metadata.storageKey,
bytes: node.metadata.bytes,
width: node.metadata.naturalWidth,
height: node.metadata.naturalHeight,
durationMs: node.metadata.durationMs,
};
}
web/src/app/(user)/video/page.tsx
+36 -7
View File
@@ -1,6 +1,6 @@
"use client";
import { BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, LoaderCircle, Music2, Plus, SlidersHorizontal, Sparkles, Trash2, Upload, VideoIcon } from "lucide-react";
import { ArrowLeft, ArrowRight, BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, LoaderCircle, Music2, Plus, SlidersHorizontal, Sparkles, Trash2, Upload, VideoIcon } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { App, Button, Checkbox, Drawer, Empty, Input, Modal, Tag, Typography } from "antd";
import localforage from "localforage";
@@ -13,7 +13,7 @@ import { PromptSelectDialog } from "@/components/prompts/prompt-select-dialog";
import { VideoSettingsPanel, normalizeVideoResolutionValue, normalizeVideoSizeValue, videoSizeLabel } from "@/components/video-settings-panel";
import { canvasThemes } from "@/lib/canvas-theme";
import { formatBytes, formatDuration } from "@/lib/image-utils";
import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceRatio, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceRatio, seedanceReferenceLabel, seedanceVideoReferenceError, seedanceVideoReferenceHint, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { deleteStoredMedia, resolveMediaUrl, uploadMediaFile } from "@/services/file-storage";
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video";
@@ -126,7 +126,7 @@ export default function VideoPage() {
const nextVideoReferences = await Promise.all(
videoFiles.map(async (file) => {
const video = await uploadMediaFile(file, "video-reference");
return { id: nanoid(), name: file.name, type: video.mimeType, url: video.url, storageKey: video.storageKey, durationMs: video.durationMs };
return { id: nanoid(), name: file.name, type: video.mimeType, url: video.url, storageKey: video.storageKey, bytes: video.bytes, width: video.width, height: video.height, durationMs: video.durationMs };
}),
);
const nextAudioReferences = filterAudioReferencesByDuration(
@@ -209,6 +209,11 @@ export default function VideoPage() {
openConfigDialog(true);
return null;
}
const videoReferenceError = seedanceVideoReferenceError(videoReferences);
if (videoReferenceError) {
message.error(`${videoReferenceError}。${seedanceVideoReferenceHint}`);
return null;
}
return { text, config: buildVideoConfig(effectiveConfig, model), references: [...references], videoReferences: [...videoReferences], audioReferences: [...audioReferences] };
};
@@ -240,7 +245,7 @@ export default function VideoPage() {
const stored = await uploadImage(payload.dataUrl);
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
} else if (payload.kind === "video") {
setVideoReferences((value) => [...value, { id: nanoid(), name: payload.title, type: "video/mp4", url: payload.url, storageKey: payload.storageKey }].slice(0, SEEDANCE_REFERENCE_LIMITS.videos));
setVideoReferences((value) => [...value, { id: nanoid(), name: payload.title, type: "video/mp4", url: payload.url, storageKey: payload.storageKey, width: payload.width, height: payload.height }].slice(0, SEEDANCE_REFERENCE_LIMITS.videos));
}
setAssetPickerOpen(false);
};
@@ -343,9 +348,11 @@ export default function VideoPage() {
</div>
</div>
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
{references.map((item) => (
{references.map((item, index) => (
<div key={item.id} className="group relative size-20 shrink-0 overflow-hidden rounded-md border border-stone-200 dark:border-stone-800">
<img src={item.dataUrl} alt={item.name} className="size-full object-cover" />
<span className="absolute left-1 top-1 rounded bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">{seedanceReferenceLabel("image", index)}</span>
<ReferenceOrderButtons index={index} total={references.length} onMove={(offset) => setReferences((value) => moveListItem(value, index, offset))} />
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考图">
<Trash2 className="size-3.5" />
</button>
@@ -363,9 +370,11 @@ export default function VideoPage() {
</Button>
</div>
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
{videoReferences.map((item) => (
{videoReferences.map((item, index) => (
<div key={item.id} className="group relative h-20 w-32 shrink-0 overflow-hidden rounded-md border border-stone-200 bg-black dark:border-stone-800">
<video src={item.url} className="size-full object-cover" muted preload="metadata" />
<span className="absolute left-1 top-1 rounded bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">{seedanceReferenceLabel("video", index)}</span>
<ReferenceOrderButtons index={index} total={videoReferences.length} onMove={(offset) => setVideoReferences((value) => moveListItem(value, index, offset))} />
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setVideoReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考视频">
<Trash2 className="size-3.5" />
</button>
@@ -383,13 +392,15 @@ export default function VideoPage() {
</Button>
</div>
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
{audioReferences.map((item) => (
{audioReferences.map((item, index) => (
<div key={item.id} className="group relative flex h-20 w-48 shrink-0 flex-col justify-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2 dark:border-stone-800 dark:bg-stone-900">
<div className="flex min-w-0 items-center gap-2 text-xs text-stone-500 dark:text-stone-400">
<Music2 className="size-4 shrink-0" />
<span className="shrink-0 rounded bg-stone-200 px-1 text-[10px] text-stone-700 dark:bg-stone-800 dark:text-stone-200">{seedanceReferenceLabel("audio", index)}</span>
<span className="truncate">{item.name}</span>
</div>
<audio src={item.url} controls className="h-8 w-full" preload="metadata" />
<ReferenceOrderButtons index={index} total={audioReferences.length} onMove={(offset) => setAudioReferences((value) => moveListItem(value, index, offset))} />
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setAudioReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考音频">
<Trash2 className="size-3.5" />
</button>
@@ -698,6 +709,24 @@ function filterAudioReferencesByDuration(existing: ReferenceAudio[], next: Refer
return accepted;
}
function moveListItem<T>(items: T[], index: number, offset: number) {
const targetIndex = index + offset;
if (targetIndex < 0 || targetIndex >= items.length) return items;
const next = [...items];
[next[index], next[targetIndex]] = [next[targetIndex], next[index]];
return next;
}
function ReferenceOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) {
if (total <= 1) return null;
return (
<div className="absolute inset-x-1 bottom-1 flex justify-between">
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !bg-white/85 !p-0 !shadow-sm" icon={<ArrowLeft className="size-3" />} disabled={index <= 0} onClick={() => onMove(-1)} />
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !bg-white/85 !p-0 !shadow-sm" icon={<ArrowRight className="size-3" />} disabled={index >= total - 1} onClick={() => onMove(1)} />
</div>
);
}
function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
return {
model: log.config?.model || log.model || "",
web/src/lib/seedance-video.ts
+43
View File
@@ -1,4 +1,6 @@
import type { AiConfig } from "@/stores/use-config-store";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
export const SEEDANCE_REFERENCE_LIMITS = {
images: 9,
@@ -123,3 +125,44 @@ export function boolConfig(value: string | undefined, fallback: boolean) {
if (value === "false") return false;
return fallback;
}
export function seedanceReferenceLabel(kind: "image" | "video" | "audio", index: number) {
if (kind === "image") return `图片${index + 1}`;
if (kind === "video") return `视频${index + 1}`;
return `音频${index + 1}`;
}
export function buildSeedancePromptText(prompt: string, images: ReferenceImage[], videos: ReferenceVideo[], audios: ReferenceAudio[]) {
const labels = [
...images.map((_, index) => seedanceReferenceLabel("image", index)),
...videos.map((_, index) => seedanceReferenceLabel("video", index)),
...audios.map((_, index) => seedanceReferenceLabel("audio", index)),
];
const text = prompt.trim();
if (!labels.length) return text;
return `参考素材编号：${labels.join("、")}。请按这些编号理解提示词中的图片、视频和音频引用。\n\n${text}`;
}
export function seedanceVideoReferenceError(videos: ReferenceVideo[]) {
let totalDurationMs = 0;
for (let index = 0; index < videos.length; index += 1) {
const video = videos[index];
const label = seedanceReferenceLabel("video", index);
if (video.bytes && video.bytes > SEEDANCE_REFERENCE_LIMITS.videoMaxBytes) return `${label} 超过 50MB，请压缩后再上传`;
if (video.durationMs) {
if (video.durationMs < 2000 || video.durationMs > 15000) return `${label} 时长需要在 2-15 秒之间`;
totalDurationMs += video.durationMs;
}
if (video.width && video.height) {
if (video.width < 300 || video.width > 6000 || video.height < 300 || video.height > 6000) return `${label} 宽高需要在 300-6000px 之间`;
const ratio = video.width / video.height;
if (ratio < 0.4 || ratio > 2.5) return `${label} 宽高比需要在 0.4-2.5 之间`;
const pixels = video.width * video.height;
if (pixels < 640 * 640 || pixels > 2206 * 946) return `${label} 像素总量不符合 Seedance 要求，请转成 480p/720p/1080p 后再上传`;
}
}
if (totalDurationMs > 15000) return "Seedance 参考视频总时长不能超过 15 秒";
return "";
}
export const seedanceVideoReferenceHint = "参考视频需为 mp4/mov，H.264/H.265，FPS 24-60；含真人人脸素材请使用火山授权 asset:// 素材。";
web/src/services/api/video.ts
+4 -2
View File
@@ -3,7 +3,7 @@ import axios from "axios";
import { dataUrlToFile } from "@/lib/image-utils";
import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
import { imageToDataUrl } from "@/services/image-storage";
import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { boolConfig, buildSeedancePromptText, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { buildApiUrl, type AiConfig } from "@/stores/use-config-store";
import { useUserStore } from "@/stores/use-user-store";
import type { ReferenceImage } from "@/types/image";
@@ -130,6 +130,8 @@ async function requestSeedanceGeneration(config: AiConfig, model: string, prompt
}
function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) {
const error = seedanceVideoReferenceError(videoReferences);
if (error) throw new Error(error);
let total = 0;
for (const video of videoReferences) {
if (!video.durationMs) continue;
@@ -156,7 +158,7 @@ function seedanceApiUrl(config: AiConfig, taskId?: string) {
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
const content: Array<Record<string, unknown>> = [];
const text = prompt.trim();
const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences);
if (text) content.push({ type: "text", text });
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" });
web/src/types/media.ts
+3
View File
@@ -4,6 +4,9 @@ export type ReferenceVideo = {
type: string;
url: string;
storageKey?: string;
bytes?: number;
width?: number;
height?: number;
durationMs?: number;
};