From 4bffb55695df2ff43d0487b51f7a44b4e7cc741f Mon Sep 17 00:00:00 2001 From: stupid-h4er Date: Wed, 27 May 2026 13:02:36 +0800 Subject: [PATCH] feat: add seedance media params and model visibility --- docs/features.md | 4 +- docs/pending-test.md | 6 +- docs/system-settings.md | 9 +- handler/media_reference.go | 58 +++++++- handler/media_reference_test.go | 40 ++++++ service/settings.go | 75 +++++++++- service/settings_test.go | 38 +++++ web/src/app/(admin)/admin/settings/page.tsx | 11 +- .../(user)/canvas/[id]/canvas-client-page.tsx | 97 +++++++++---- .../components/canvas-config-node-panel.tsx | 58 +++++++- .../canvas/components/canvas-mini-map.tsx | 2 +- .../components/canvas-node-generation.ts | 24 +++- .../components/canvas-node-hover-toolbar.tsx | 11 +- .../components/canvas-node-prompt-panel.tsx | 11 +- .../(user)/canvas/components/canvas-node.tsx | 25 +++- .../canvas/components/canvas-toolbar.tsx | 12 +- web/src/app/(user)/canvas/constants.ts | 5 + web/src/app/(user)/canvas/types.ts | 4 + web/src/app/(user)/video/page.tsx | 130 +++++++++++++++--- web/src/components/video-settings-panel.tsx | 103 +++++++++++++- web/src/lib/seedance-video.ts | 125 +++++++++++++++++ web/src/services/api/video.ts | 98 +++++++------ web/src/services/file-storage.ts | 18 ++- web/src/stores/use-config-store.ts | 39 +++++- web/src/types/media.ts | 10 ++ 25 files changed, 873 insertions(+), 140 deletions(-) create mode 100644 handler/media_reference_test.go create mode 100644 web/src/lib/seedance-video.ts diff --git a/docs/features.md b/docs/features.md index 5b7a1ca..a6963d0 100644 --- a/docs/features.md +++ b/docs/features.md @@ -82,11 +82,11 @@ Base URL 如果已经以 `/v1`、`/api/v3` 或 `/api/plan/v3` 结尾，系统不 - 图片比例。 - 生成数量。 - 视频模型。 -- 视频比例、清晰度和时长。 +- 视频比例、清晰度、时长、生成声音和水印。 普通图片/文本节点可以直接输入提示词生成结果。生成配置节点可以读取上游节点内容，并按节点自己的配置批量生成多个图片或文本结果。生成配置节点支持预览当前提示词和参考图输入，并调整输入顺序。 -视频生成可从文本节点读取 prompt，从图片节点读取参考图，从视频节点读取参考视频。生成成功后会把视频插入画布为视频节点并使用原生播放器预览。Seedance 参考视频需要公网可访问 URL；本地上传视频会先通过 `/api/v1/media/references` 保存到服务端，再由 `PUBLIC_BASE_URL` 生成可供火山服务器拉取的公开链接。 +视频生成可从文本节点读取 prompt，从图片节点读取参考图，从视频节点读取参考视频，从音频节点读取参考音频。Seedance 2.0 支持最多 9 张参考图、3 个参考视频、3 个参考音频；分辨率支持 `480p`、`720p`、`1080p`（fast 模型不支持 `1080p`），比例支持 `16:9`、`4:3`、`1:1`、`3:4`、`9:16`、`21:9`、`adaptive`，时长支持 4-15 秒或智能时长。生成成功后会把视频插入画布为视频节点并使用原生播放器预览。Seedance 参考视频和参考音频需要公网可访问 URL；本地上传素材会先通过 `/api/v1/media/references` 保存到服务端，再由 `PUBLIC_BASE_URL` 生成可供火山服务器拉取的公开链接。 ## 画布助手 diff --git a/docs/pending-test.md b/docs/pending-test.md index 3b1a416..e313d2b 100644 --- a/docs/pending-test.md +++ b/docs/pending-test.md @@ -22,5 +22,9 @@ - 新增 `/video` 视频创作台页面，参考生图工作台布局，支持提示词、参考图、视频参数、生成结果、保存素材、下载和本地生成记录；清晰度和秒数均支持常用值选择与手动输入，生成记录只保存媒体 `storageKey` 并可回填本次提示词、参考图和参数。 - 视频创作台生成前会把模型、尺寸、秒数、清晰度归一化为视频接口支持的参数，并展示后端返回的错误信息，避免页面侧残留的生图参数影响视频请求。 - 火山方舟 Agent Plan / Seedance 2.0 视频生成需要在真实账号下验证：`/contents/generations/tasks` 创建任务、轮询状态、`content.video_url` 回填画布，以及 401/403/429/超时错误提示。 -- `PUBLIC_BASE_URL` 已配置公网域名时，需要验证本地上传参考视频能被火山拉取；未配置或配置为内网地址时，需要验证前端能给出明确提示。 +- 管理后台保存私有渠道后，需要验证所有已启用渠道里的模型会自动出现在公开 `availableModels`，并且 `defaultVideoModel`、`defaultImageModel`、`defaultTextModel` 在为空或失效时会自动修复，前台不再显示旧的 `grok` 默认值。 +- `/video` 和画布视频设置已按 Seedance 2.0 增加分辨率、比例、4-15 秒/智能时长、生成声音和水印参数；需要在真实浏览器里验证参数回填、生成记录和画布节点配置都能保持一致。 +- `/video` 支持最多 9 张参考图、3 个参考视频、3 个参考音频；需要验证格式、大小、音频时长提示和生成请求中的 `reference_image`、`reference_video`、`reference_audio` 组装。 +- 画布新增音频节点，支持上传、拖入、播放、移动、缩放、删除，并可作为上游参考音频参与 Seedance 视频生成；需要验证刷新后本地音频 URL 能恢复。 +- `PUBLIC_BASE_URL` 已配置公网域名时，需要验证本地上传参考视频和参考音频能被火山拉取；未配置或配置为内网地址时，需要验证前端能给出明确提示。 - Seedance 返回远程视频 URL 但浏览器无法下载为 Blob 时，需要验证视频节点刷新后仍保留远程 URL，并确认上游 URL 有效期限制。 diff --git a/docs/system-settings.md b/docs/system-settings.md index 74a267b..0996665 100644 --- a/docs/system-settings.md +++ b/docs/system-settings.md @@ -41,11 +41,12 @@ | 字段 | 类型 | 说明 | | --- | --- | --- | -| `availableModels` | string[] | 系统可用模型，由管理员手动选择；页面下拉选项可来自私有渠道模型 | +| `availableModels` | string[] | 系统可用模型；保存设置时会自动合并所有已启用私有渠道的模型 | | `modelCosts` | object[] | 模型算力点配置，后端模型接口调用前按模型预扣，上游失败时返还；未配置默认不扣除 | -| `defaultModel` | string | 默认模型，从 `availableModels` 中选择 | -| `defaultImageModel` | string | 默认图片模型，从 `availableModels` 中选择 | -| `defaultTextModel` | string | 默认文本模型，从 `availableModels` 中选择 | +| `defaultModel` | string | 默认模型，从 `availableModels` 中选择；为空或失效时优先选择文本模型 | +| `defaultImageModel` | string | 默认图片模型，从 `availableModels` 中选择；为空或失效时优先选择 `seedream`、`image`、`gpt-image` 模型 | +| `defaultVideoModel` | string | 默认视频模型，从 `availableModels` 中选择；为空或失效时优先选择 `seedance`、`video` 模型 | +| `defaultTextModel` | string | 默认文本模型，从 `availableModels` 中选择；为空或失效时优先选择非图片/视频模型 | | `systemPrompt` | string | 系统提示词 | | `allowCustomChannel` | boolean | 是否允许用户在配置弹窗中切换为本地直连渠道，默认允许 | diff --git a/handler/media_reference.go b/handler/media_reference.go index d080f97..b528588 100644 --- a/handler/media_reference.go +++ b/handler/media_reference.go @@ -12,7 +12,16 @@ import ( "github.com/google/uuid" ) -const referenceMediaMaxBytes = 80 << 20 +const ( + referenceMediaMaxBytes = 80 << 20 + referenceImageMaxBytes = 30 << 20 + referenceVideoMaxBytes = 50 << 20 + referenceAudioMaxBytes = 15 << 20 + referenceImageAllowedText = "jpeg/png/webp/bmp/gif/heic/heif 图片" + referenceVideoAllowedText = "mp4/mov 视频" + referenceAudioAllowedText = "mp3/wav 音频" + referenceMediaAllowedText = referenceImageAllowedText + "、" + referenceVideoAllowedText + "或" + referenceAudioAllowedText +) type referenceMediaUploadResult struct { ID string `json:"id"` @@ -24,7 +33,7 @@ type referenceMediaUploadResult struct { func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) { publicBaseURL := strings.TrimRight(strings.TrimSpace(config.Cfg.PublicBaseURL), "/") if publicBaseURL == "" { - Fail(w, "未配置 PUBLIC_BASE_URL，无法把本地参考视频提供给火山方舟访问") + Fail(w, "未配置 PUBLIC_BASE_URL，无法把本地参考素材提供给火山方舟访问") return } r.Body = http.MaxBytesReader(w, r.Body, referenceMediaMaxBytes+1) @@ -44,7 +53,7 @@ func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) { mimeType, ext, ok := normalizeReferenceMediaType(header.Header.Get("Content-Type"), filepath.Ext(header.Filename)) if !ok { - Fail(w, "参考素材格式不支持，请使用 jpeg/png/webp/bmp/gif/heic/heif 图片或 mp4/mov 视频") + Fail(w, "参考素材格式不支持，请使用 "+referenceMediaAllowedText) return } if err := os.MkdirAll(referenceMediaDir(), 0o755); err != nil { @@ -65,9 +74,14 @@ func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) { Fail(w, "参考素材保存失败") return } - if bytes <= 0 || bytes > referenceMediaMaxBytes { + if bytes <= 0 { _ = os.Remove(targetPath) - Fail(w, "参考素材为空或超过大小限制") + Fail(w, "参考素材为空") + return + } + if limit := referenceMediaTypeMaxBytes(mimeType); limit > 0 && bytes > limit { + _ = os.Remove(targetPath) + Fail(w, referenceMediaSizeMessage(mimeType)) return } OK(w, referenceMediaUploadResult{ @@ -141,6 +155,10 @@ func referenceMediaExtByMimeType(mimeType string) string { return ".mp4" case "video/quicktime", "video/mov": return ".mov" + case "audio/mpeg", "audio/mp3": + return ".mp3" + case "audio/wav", "audio/x-wav", "audio/wave": + return ".wav" default: return "" } @@ -166,7 +184,37 @@ func mimeTypeByReferenceMediaExt(ext string) string { return "video/mp4" case ".mov": return "video/quicktime" + case ".mp3": + return "audio/mpeg" + case ".wav": + return "audio/wav" default: return "" } } + +func referenceMediaTypeMaxBytes(mimeType string) int64 { + if strings.HasPrefix(mimeType, "image/") { + return referenceImageMaxBytes + } + if strings.HasPrefix(mimeType, "video/") { + return referenceVideoMaxBytes + } + if strings.HasPrefix(mimeType, "audio/") { + return referenceAudioMaxBytes + } + return referenceMediaMaxBytes +} + +func referenceMediaSizeMessage(mimeType string) string { + if strings.HasPrefix(mimeType, "image/") { + return "参考图片超过大小限制，请使用 30MB 以内的图片" + } + if strings.HasPrefix(mimeType, "video/") { + return "参考视频超过大小限制，请使用 50MB 以内的 mp4/mov 视频" + } + if strings.HasPrefix(mimeType, "audio/") { + return "参考音频超过大小限制，请使用 15MB 以内的 mp3/wav 音频" + } + return "参考素材超过大小限制" +} diff --git a/handler/media_reference_test.go b/handler/media_reference_test.go new file mode 100644 index 0000000..95cd7fe --- /dev/null +++ b/handler/media_reference_test.go @@ -0,0 +1,40 @@ +package handler + +import "testing" + +func TestNormalizeReferenceMediaTypeSupportsAudio(t *testing.T) { + tests := []struct { + name string + contentType string + ext string + wantMime string + wantExt string + }{ + {name: "mp3 mime", contentType: "audio/mpeg", ext: ".bin", wantMime: "audio/mpeg", wantExt: ".mp3"}, + {name: "wav ext fallback", contentType: "application/octet-stream", ext: ".wav", wantMime: "audio/wav", wantExt: ".wav"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mimeType, ext, ok := normalizeReferenceMediaType(tt.contentType, tt.ext) + if !ok { + t.Fatal("expected media type to be accepted") + } + if mimeType != tt.wantMime || ext != tt.wantExt { + t.Fatalf("got (%q, %q), want (%q, %q)", mimeType, ext, tt.wantMime, tt.wantExt) + } + }) + } +} + +func TestReferenceMediaTypeMaxBytes(t *testing.T) { + if got := referenceMediaTypeMaxBytes("audio/mpeg"); got != referenceAudioMaxBytes { + t.Fatalf("audio max bytes = %d, want %d", got, referenceAudioMaxBytes) + } + if got := referenceMediaTypeMaxBytes("video/mp4"); got != referenceVideoMaxBytes { + t.Fatalf("video max bytes = %d, want %d", got, referenceVideoMaxBytes) + } + if got := referenceMediaTypeMaxBytes("image/png"); got != referenceImageMaxBytes { + t.Fatalf("image max bytes = %d, want %d", got, referenceImageMaxBytes) + } +} diff --git a/service/settings.go b/service/settings.go index 51ac403..121c802 100644 --- a/service/settings.go +++ b/service/settings.go @@ -20,7 +20,7 @@ var adminModelHTTPClient = &http.Client{Timeout: 30 * time.Second} func PublicSettings() (model.PublicSetting, error) { settings, err := repository.GetSettings() - return normalizePublicSetting(settings.Public), err + return normalizeSettings(settings).Public, err } func AdminSettings() (model.Settings, error) { @@ -63,12 +63,16 @@ func AdminTestChannelModel(index *int, channel model.ModelChannel, modelName str } func normalizeSettings(settings model.Settings) model.Settings { - settings.Public = normalizePublicSetting(settings.Public) settings.Private = normalizePrivateSetting(settings.Private) + settings.Public = normalizePublicSettingWithChannels(settings.Public, settings.Private.Channels) return settings } func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting { + return normalizePublicSettingWithChannels(setting, nil) +} + +func normalizePublicSettingWithChannels(setting model.PublicSetting, channels []model.ModelChannel) model.PublicSetting { if setting.ModelChannel.AvailableModels == nil { setting.ModelChannel.AvailableModels = []string{} } @@ -89,6 +93,16 @@ func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting { enabled := true setting.Auth.AllowRegister = &enabled } + enabledModels := enabledChannelModels(channels) + if len(enabledModels) > 0 { + setting.ModelChannel.AvailableModels = enabledModels + } else { + setting.ModelChannel.AvailableModels = uniqueModelNames(setting.ModelChannel.AvailableModels) + } + setting.ModelChannel.DefaultTextModel = repairDefaultModel(setting.ModelChannel.DefaultTextModel, setting.ModelChannel.AvailableModels, isTextModelName) + setting.ModelChannel.DefaultImageModel = repairDefaultModel(setting.ModelChannel.DefaultImageModel, setting.ModelChannel.AvailableModels, isImageModelName) + setting.ModelChannel.DefaultVideoModel = repairDefaultModel(setting.ModelChannel.DefaultVideoModel, setting.ModelChannel.AvailableModels, isVideoModelName) + setting.ModelChannel.DefaultModel = repairDefaultModel(setting.ModelChannel.DefaultModel, setting.ModelChannel.AvailableModels, isTextModelName) return setting } @@ -224,6 +238,63 @@ func isSeedanceModelName(modelName string) bool { return strings.Contains(modelName, "seedance") || strings.Contains(modelName, "doubao-seedance") } +func enabledChannelModels(channels []model.ModelChannel) []string { + models := []string{} + for _, channel := range channels { + if !channel.Enabled { + continue + } + models = append(models, channel.Models...) + } + return uniqueModelNames(models) +} + +func uniqueModelNames(models []string) []string { + result := []string{} + seen := map[string]bool{} + for _, item := range models { + name := strings.TrimSpace(item) + if name == "" || seen[name] { + continue + } + seen[name] = true + result = append(result, name) + } + return result +} + +func repairDefaultModel(current string, models []string, preferred func(string) bool) string { + current = strings.TrimSpace(current) + for _, item := range models { + if item == current { + return current + } + } + for _, item := range models { + if preferred(item) { + return item + } + } + if len(models) > 0 { + return models[0] + } + return "" +} + +func isVideoModelName(modelName string) bool { + name := strings.ToLower(strings.TrimSpace(modelName)) + return strings.Contains(name, "seedance") || strings.Contains(name, "video") +} + +func isImageModelName(modelName string) bool { + name := strings.ToLower(strings.TrimSpace(modelName)) + return strings.Contains(name, "seedream") || strings.Contains(name, "gpt-image") || strings.Contains(name, "image") +} + +func isTextModelName(modelName string) bool { + return !isImageModelName(modelName) && !isVideoModelName(modelName) +} + func normalizeModelChannel(channel model.ModelChannel) model.ModelChannel { if channel.Protocol == "" { channel.Protocol = "openai" diff --git a/service/settings_test.go b/service/settings_test.go index ea92d83..27c6b2e 100644 --- a/service/settings_test.go +++ b/service/settings_test.go @@ -60,3 +60,41 @@ func TestBuildModelChannelURLNormalizesArkPlanTaskPath(t *testing.T) { t.Fatalf("BuildModelChannelURL = %q, want %q", got, want) } } + +func TestNormalizeSettingsPublishesEnabledChannelModelsAndRepairsDefaults(t *testing.T) { + settings := normalizeSettings(model.Settings{ + Public: model.PublicSetting{ + ModelChannel: model.PublicModelChannelSetting{ + AvailableModels: []string{"grok-imagine-video", "disabled-model"}, + DefaultModel: "grok-imagine-video", + DefaultTextModel: "missing-text", + DefaultImageModel: "missing-image", + DefaultVideoModel: "missing-video", + }, + }, + Private: model.PrivateSetting{ + Channels: []model.ModelChannel{ + {Enabled: true, Models: []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast", "gpt-5.5"}}, + {Enabled: false, Models: []string{"disabled-model"}}, + }, + }, + }) + + channel := settings.Public.ModelChannel + wantModels := []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast"} + if !reflect.DeepEqual(channel.AvailableModels, wantModels) { + t.Fatalf("available models = %#v, want %#v", channel.AvailableModels, wantModels) + } + if channel.DefaultModel != "gpt-5.5" { + t.Fatalf("default model = %q, want text model", channel.DefaultModel) + } + if channel.DefaultTextModel != "gpt-5.5" { + t.Fatalf("default text model = %q, want text model", channel.DefaultTextModel) + } + if channel.DefaultImageModel != "doubao-seedream-5.0-lite" { + t.Fatalf("default image model = %q, want seedream", channel.DefaultImageModel) + } + if channel.DefaultVideoModel != "doubao-seedance-2.0-fast" { + t.Fatalf("default video model = %q, want seedance", channel.DefaultVideoModel) + } +} diff --git a/web/src/app/(admin)/admin/settings/page.tsx b/web/src/app/(admin)/admin/settings/page.tsx index c262680..7b5029f 100644 --- a/web/src/app/(admin)/admin/settings/page.tsx +++ b/web/src/app/(admin)/admin/settings/page.tsx @@ -341,7 +341,7 @@ export default function AdminSettingsPage() { const nextChannelModels = collectChannelModels(nextChannels); const nextSettings = normalizeSettings({ ...values, - public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: filterModels(values.public.modelChannel.availableModels, nextChannelModels) } }, + public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: nextChannelModels } }, private: { ...values.private, channels: nextChannels }, }); const saved = normalizeSettings(await saveAdminSettings(token, nextSettings)); @@ -414,7 +414,7 @@ export default function AdminSettingsPage() {
- + + setInfoNodeId(null)} /> @@ -2377,7 +2410,7 @@ function CanvasTopBar({ { key: "new", icon: , label: "新建画布", onClick: onCreateProject }, { key: "delete", danger: true, icon: , label: "删除当前画布", onClick: onDeleteProject }, { type: "divider" }, - { key: "import", icon: , label: "导入图片", onClick: onImportImage }, + { key: "import", icon: , label: "导入素材", onClick: onImportImage }, { type: "divider" }, { key: "undo", disabled: !canUndo, icon: , label: , onClick: onUndo }, { key: "redo", disabled: !canRedo, icon: , label: , onClick: onRedo }, @@ -2458,7 +2491,7 @@ function CanvasTopBar({ - + @@ -2499,12 +2532,20 @@ function imageExtension(dataUrl: string) { return dataUrl.match(/^data:image[/]([^;]+)/)?.[1] || dataUrl.match(/image[/]([^;]+)/)?.[1] || "png"; } +function audioExtension(mimeType?: string) { + return mimeType?.includes("wav") ? "wav" : "mp3"; +} + function imageMetadata(image: UploadedImage): CanvasNodeMetadata { return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType }; } function videoMetadata(video: UploadedFile): CanvasNodeMetadata { - return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4" }; + return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4", durationMs: video.durationMs }; +} + +function audioMetadata(audio: UploadedFile): CanvasNodeMetadata { + return { content: audio.url, storageKey: audio.storageKey, status: "success", bytes: audio.bytes, mimeType: audio.mimeType || "audio/mpeg", durationMs: audio.durationMs }; } function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata { @@ -2522,10 +2563,11 @@ function referenceUrl(image: ReferenceImage) { return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined); } -function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }> }) { +function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }>; referenceAudios?: Array<{ storageKey?: string; url?: string }> }) { return [ ...context.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)), ...context.referenceVideos.map((video) => video.storageKey || video.url).filter((url): url is string => Boolean(url)), + ...(context.referenceAudios || []).map((audio) => audio.storageKey || audio.url).filter((url): url is string => Boolean(url)), ]; } @@ -2545,7 +2587,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) { return Promise.all( nodes.map(async (node) => { const content = node.metadata?.content; - if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } }; + if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } }; if (node.type !== CanvasNodeType.Image || !content) return node; if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } }; if (!content.startsWith("data:image/")) return node; @@ -2605,6 +2647,7 @@ function getInputSummary(inputs: NodeGenerationInput[]) { textCount: inputs.filter((input) => input.type === "text").length, imageCount: inputs.filter((input) => input.type === "image").length, videoCount: inputs.filter((input) => input.type === "video").length, + audioCount: inputs.filter((input) => input.type === "audio").length, }; } @@ -2617,6 +2660,8 @@ function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefine size: node?.metadata?.size || config.size || defaultConfig.size, videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds, vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality, + videoGenerateAudio: node?.metadata?.generateAudio || config.videoGenerateAudio || defaultConfig.videoGenerateAudio, + videoWatermark: node?.metadata?.watermark || config.videoWatermark || defaultConfig.videoWatermark, count: String(node?.metadata?.count || (mode === "image" ? 3 : config.count) || defaultConfig.count), }; } @@ -2652,6 +2697,10 @@ function sourceNodeReferenceImages(node: CanvasNodeData | null) { ]; } +function isAudioFile(file: File) { + return file.type.startsWith("audio/") || /\.(mp3|wav)$/i.test(file.name); +} + function isHiddenBatchChild(node: CanvasNodeData, nodes: CanvasNodeData[], collapsingBatchIds?: Set) { const rootId = node.metadata?.batchRootId; if (!rootId) return false; diff --git a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx index 216b814..8966229 100644 --- a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx @@ -2,7 +2,7 @@ import type { CSSProperties } from "react"; import { useState } from "react"; -import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play, Video } from "lucide-react"; +import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Music2, Play, Video } from "lucide-react"; import { App, Button, Empty, Input, Modal, Segmented } from "antd"; import { ModelPicker } from "@/components/model-picker"; @@ -18,7 +18,7 @@ import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata } from ". type CanvasConfigNodePanelProps = { node: CanvasNodeData; isRunning: boolean; - inputSummary: { textCount: number; imageCount: number; videoCount: number }; + inputSummary: { textCount: number; imageCount: number; videoCount: number; audioCount: number }; inputs: NodeGenerationInput[]; onConfigChange: (nodeId: string, patch: Partial) => void; onTextInputChange: (nodeId: string, content: string) => void; @@ -42,6 +42,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o const textInputs = inputs.filter((input) => input.type === "text"); const imageInputs = inputs.filter((input) => input.type === "image"); const videoInputs = inputs.filter((input) => input.type === "video"); + const audioInputs = inputs.filter((input) => input.type === "audio"); const moveInput = (input: NodeGenerationInput, offset: number) => { const sameTypeInputs = inputs.filter((item) => item.type === input.type); @@ -114,6 +115,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o + ))} - {!references.length ?
暂无参考图，最多 7 张
: null} + {!references.length ?
暂无参考图，最多 9 张
: null} @@ -352,6 +375,30 @@ export default function VideoPage() { +
+
+ 参考音频 + +
+
+ {audioReferences.map((item) => ( +
+
+ + {item.name} +
+
+ ))} + {!audioReferences.length ?
暂无参考音频，最多 3 个，mp3/wav，单个 15MB 内
: null} +
+
+
{model} · {normalizeResolution(effectiveConfig.vquality)}p · {videoSizeLabel(effectiveConfig.size)} · {normalizeVideoSeconds(effectiveConfig.videoSeconds)}s @@ -394,7 +441,7 @@ export default function VideoPage() { { @@ -583,6 +630,12 @@ async function normalizeLog(log: Partial): Promise url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url, })), ); + const audioReferences = await Promise.all( + (log.audioReferences || []).map(async (item) => ({ + ...item, + url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url, + })), + ); const references = await Promise.all( (log.references || []).map(async (item) => ({ ...item, @@ -600,6 +653,7 @@ async function normalizeLog(log: Partial): Promise config, references, videoReferences, + audioReferences, durationMs: log.durationMs || 0, size: log.size || config.size || "", resolution: normalizeResolution(log.resolution || config.vquality || ""), @@ -615,10 +669,35 @@ function serializeLog(log: GenerationLog): GenerationLog { ...log, references: log.references.map((item) => ({ ...item, dataUrl: item.storageKey ? "" : item.dataUrl })), videoReferences: log.videoReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)), + audioReferences: log.audioReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)), video: log.video?.storageKey ? { ...log.video, url: "" } : log.video, }; } +function isSupportedAudioFile(file: File) { + return file.type === "audio/mpeg" || file.type === "audio/mp3" || file.type === "audio/wav" || file.type === "audio/x-wav" || /\.(mp3|wav)$/i.test(file.name); +} + +function filterAudioReferencesByDuration(existing: ReferenceAudio[], next: ReferenceAudio[], warn: (content: string) => void) { + let total = existing.reduce((sum, item) => sum + (item.durationMs || 0), 0); + const accepted: ReferenceAudio[] = []; + let skipped = false; + for (const item of next) { + if (item.durationMs && (item.durationMs < 2000 || item.durationMs > 15000)) { + skipped = true; + continue; + } + if (item.durationMs && total + item.durationMs > 15000) { + skipped = true; + continue; + } + total += item.durationMs || 0; + accepted.push(item); + } + if (skipped) warn("已忽略不符合时长要求的参考音频：单个 2-15 秒，总时长不超过 15 秒"); + return accepted; +} + function normalizeLogConfig(log: Partial): GenerationLogConfig { return { model: log.config?.model || log.model || "", @@ -626,16 +705,20 @@ function normalizeLogConfig(log: Partial): GenerationLogConfig { size: log.config?.size || log.size || "", vquality: normalizeResolution(log.config?.vquality || log.resolution || ""), videoSeconds: log.config?.videoSeconds || log.seconds || "", + videoGenerateAudio: log.config?.videoGenerateAudio || "true", + videoWatermark: log.config?.videoWatermark || "false", }; } -function buildLog({ prompt, model, config, references, videoReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog { +function buildLog({ prompt, model, config, references, videoReferences, audioReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; audioReferences: ReferenceAudio[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog { const logConfig = { model: config.model, videoModel: config.videoModel, size: config.size, vquality: normalizeResolution(config.vquality), videoSeconds: config.videoSeconds, + videoGenerateAudio: config.videoGenerateAudio, + videoWatermark: config.videoWatermark, }; return { id: nanoid(), @@ -647,6 +730,7 @@ function buildLog({ prompt, model, config, references, videoReferences, duration config: logConfig, references, videoReferences, + audioReferences, durationMs, size: logConfig.size, resolution: logConfig.vquality, @@ -658,17 +742,21 @@ function buildLog({ prompt, model, config, references, videoReferences, duration } function buildVideoConfig(config: AiConfig, model: string): AiConfig { + const seedance = isSeedanceVideoConfig({ ...config, model }); return { ...config, model, videoModel: model, - size: normalizeVideoSize(config.size), + size: seedance ? normalizeSeedanceRatio(config.size) : normalizeVideoSize(config.size), videoSeconds: normalizeVideoSeconds(config.videoSeconds), vquality: normalizeResolution(config.vquality), + videoGenerateAudio: String(boolConfig(config.videoGenerateAudio, true)), + videoWatermark: String(boolConfig(config.videoWatermark, false)), }; } function normalizeVideoSeconds(value: string) { + if (String(value).trim() === "-1") return "-1"; const seconds = Math.floor(Number(value) || 6); return String(Math.max(1, Math.min(20, seconds))); } diff --git a/web/src/components/video-settings-panel.tsx b/web/src/components/video-settings-panel.tsx index cae6b3f..8e608b8 100644 --- a/web/src/components/video-settings-panel.tsx +++ b/web/src/components/video-settings-panel.tsx @@ -1,8 +1,10 @@ "use client"; import { type ReactNode } from "react"; +import { Switch } from "antd"; import { ImageSettingsTheme } from "@/components/image-settings-panel"; +import { boolConfig, isSeedanceFastModel, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceDurationOptions, seedancePixelLabel, seedanceRatioOptions, seedanceResolutionOptions } from "@/lib/seedance-video"; import { type CanvasTheme } from "@/lib/canvas-theme"; import type { AiConfig } from "@/stores/use-config-store"; @@ -24,13 +26,17 @@ const secondOptions = [6, 10, 12, 16, 20]; type VideoSettingsPanelProps = { config: AiConfig; - onConfigChange: (key: "vquality" | "size" | "videoSeconds", value: string) => void; + onConfigChange: (key: "vquality" | "size" | "videoSeconds" | "videoGenerateAudio" | "videoWatermark", value: string) => void; theme: CanvasTheme; showTitle?: boolean; className?: string; }; export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = true, className = "w-[320px] space-y-4 rounded-2xl px-1 py-0.5" }: VideoSettingsPanelProps) { + if (isSeedanceVideoConfig(config)) { + return ; + } + const seconds = config.videoSeconds || "6"; const size = normalizeVideoSizeValue(config.size); const dimensions = readSizeDimensions(size); @@ -96,16 +102,84 @@ export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = ); } +function SeedanceVideoSettingsPanel({ config, onConfigChange, theme, showTitle, className }: VideoSettingsPanelProps) { + const model = config.model || config.videoModel; + const resolution = normalizeSeedanceResolution(config.vquality, model); + const ratio = normalizeSeedanceRatio(config.size); + const duration = normalizeSeedanceDuration(config.videoSeconds); + const generateAudio = boolConfig(config.videoGenerateAudio, true); + const watermark = boolConfig(config.videoWatermark, false); + + return ( + +
event.stopPropagation()}> + {showTitle ?
视频设置
: null} + +
+ {seedanceResolutionOptions.map((item) => { + const disabled = item.value === "1080p" && isSeedanceFastModel(model); + return ( + onConfigChange("vquality", item.value)}> + {item.label} + + ); + })} +
+ {isSeedanceFastModel(model) ?
fast 模型不支持 1080p，会自动使用 720p。
: null} + + +
+ {seedanceRatioOptions.map((item) => ( + + ))} +
+ + +
+ {seedanceDurationOptions.map((value) => ( + onConfigChange("videoSeconds", String(value))}> + {value === -1 ? "智能" : `${value}s`} + + ))} +
+ onConfigChange("videoSeconds", value)} /> + + +
+ onConfigChange("videoGenerateAudio", String(checked))} /> + onConfigChange("videoWatermark", String(checked))} /> +
+ +
+ + ); +} + export function videoResolutionLabel(value: string) { return `${normalizeVideoResolutionValue(value)}p`; } export function videoSizeLabel(value: string) { + const ratio = normalizeSeedanceRatio(value); + if (value === "adaptive" || value === "auto") return "自适应"; + if (ratio === value) return seedanceRatioOptions.find((item) => item.value === ratio)?.label || ratio; const size = normalizeVideoSizeValue(value); return sizeOptions.find((item) => item.value === size)?.label || size; } export function videoSecondsLabel(value: string) { + if (String(value).trim() === "-1") return "智能"; return `${value || "6"}s`; } @@ -121,9 +195,9 @@ export function normalizeVideoResolutionValue(value: string) { return value.replace(/p$/i, "") || "720"; } -function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) { +function OptionPill({ selected, disabled = false, theme, onClick, children }: { selected: boolean; disabled?: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) { return ( - ); @@ -174,6 +248,29 @@ function SizePreview({ width, height, color }: { width: number; height: number; return ; } +function ratioPreview(ratio: string) { + if (ratio === "9:16") return { width: 9, height: 16 }; + if (ratio === "1:1") return { width: 1, height: 1 }; + if (ratio === "4:3") return { width: 4, height: 3 }; + if (ratio === "3:4") return { width: 3, height: 4 }; + if (ratio === "21:9") return { width: 21, height: 9 }; + if (ratio === "adaptive") return { width: 0, height: 0 }; + return { width: 16, height: 9 }; +} + +function SwitchRow({ label, checked, theme, onChange }: { label: string; checked: boolean; theme: CanvasTheme; onChange: (checked: boolean) => void }) { + return ( +
+ + {label} + + event.stopPropagation()}> + + +
+ ); +} + function readSizeDimensions(size: string) { if (size === "auto") return { width: 0, height: 0 }; const match = size.match(/^(\d+)x(\d+)$/); diff --git a/web/src/lib/seedance-video.ts b/web/src/lib/seedance-video.ts new file mode 100644 index 0000000..35c00b9 --- /dev/null +++ b/web/src/lib/seedance-video.ts @@ -0,0 +1,125 @@ +import type { AiConfig } from "@/stores/use-config-store"; + +export const SEEDANCE_REFERENCE_LIMITS = { + images: 9, + videos: 3, + audios: 3, + imageMaxBytes: 30 * 1024 * 1024, + videoMaxBytes: 50 * 1024 * 1024, + audioMaxBytes: 15 * 1024 * 1024, +}; + +export const seedanceResolutionOptions = [ + { value: "480p", label: "480p" }, + { value: "720p", label: "720p" }, + { value: "1080p", label: "1080p" }, +] as const; + +export const seedanceRatioOptions = [ + { value: "16:9", label: "横屏" }, + { value: "9:16", label: "竖屏" }, + { value: "1:1", label: "方形" }, + { value: "4:3", label: "标准横屏" }, + { value: "3:4", label: "标准竖屏" }, + { value: "21:9", label: "宽银幕" }, + { value: "adaptive", label: "自适应" }, +] as const; + +export const seedanceDurationOptions = [-1, 4, 5, 6, 8, 10, 12, 15] as const; + +const seedancePixels = { + "480p": { + "16:9": "864x496", + "4:3": "752x560", + "1:1": "640x640", + "3:4": "560x752", + "9:16": "496x864", + "21:9": "992x432", + }, + "720p": { + "16:9": "1280x720", + "4:3": "1112x834", + "1:1": "960x960", + "3:4": "834x1112", + "9:16": "720x1280", + "21:9": "1470x630", + }, + "1080p": { + "16:9": "1920x1080", + "4:3": "1664x1248", + "1:1": "1440x1440", + "3:4": "1248x1664", + "9:16": "1080x1920", + "21:9": "2206x946", + }, +} as const; + +export function isSeedanceVideoConfig(config: Pick) { + return isSeedanceVideoModel(config.model || config.videoModel) || isArkPlanBaseUrl(config.baseUrl); +} + +export function isSeedanceVideoModel(model: string) { + const value = model.toLowerCase(); + return value.includes("seedance") || value.includes("doubao-seedance"); +} + +export function isSeedanceFastModel(model: string) { + const value = model.toLowerCase(); + return isSeedanceVideoModel(value) && value.includes("fast"); +} + +export function isArkPlanBaseUrl(baseUrl: string) { + return baseUrl.toLowerCase().includes("ark.cn-beijing.volces.com/api/plan/v3") || baseUrl.toLowerCase().includes("/api/plan/v3"); +} + +export function normalizeSeedanceResolution(value: string, model = "") { + const normalized = normalizeResolutionToken(value); + if (isSeedanceFastModel(model) && normalized === "1080p") return "720p"; + return seedanceResolutionOptions.some((item) => item.value === normalized) ? normalized : "720p"; +} + +export function normalizeResolutionToken(value: string) { + if (value === "low") return "480p"; + if (value === "auto" || value === "high" || value === "medium") return "720p"; + const resolution = String(value || "").replace(/p$/i, "") || "720"; + return `${resolution}p`; +} + +export function normalizeSeedanceDuration(value: string) { + if (String(value).trim() === "-1") return -1; + const seconds = Math.floor(Number(value) || 5); + return Math.max(4, Math.min(15, seconds)); +} + +export function normalizeSeedanceRatio(value: string) { + if (!value || value === "auto" || value === "adaptive") return "adaptive"; + if (seedanceRatioOptions.some((item) => item.value === value)) return value; + const match = value.match(/^(\d+)x(\d+)$/); + if (!match) return "adaptive"; + const width = Number(match[1]); + const height = Number(match[2]); + if (!width || !height) return "adaptive"; + const ratio = width / height; + const options = [ + ["16:9", 16 / 9], + ["4:3", 4 / 3], + ["1:1", 1], + ["3:4", 3 / 4], + ["9:16", 9 / 16], + ["21:9", 21 / 9], + ] as const; + return options.reduce((best, item) => (Math.abs(item[1] - ratio) < Math.abs(best[1] - ratio) ? item : best), options[0])[0]; +} + +export function seedancePixelLabel(resolution: string, ratio: string) { + const normalizedResolution = normalizeSeedanceResolution(resolution) as keyof typeof seedancePixels; + const normalizedRatio = normalizeSeedanceRatio(ratio) as keyof (typeof seedancePixels)[typeof normalizedResolution] | "adaptive"; + if (normalizedRatio === "adaptive") return "自动匹配"; + return seedancePixels[normalizedResolution][normalizedRatio] || ""; +} + +export function boolConfig(value: string | undefined, fallback: boolean) { + if (value === "true") return true; + if (value === "false") return false; + return fallback; +} diff --git a/web/src/services/api/video.ts b/web/src/services/api/video.ts index 56ed12a..37522f8 100644 --- a/web/src/services/api/video.ts +++ b/web/src/services/api/video.ts @@ -3,10 +3,11 @@ import axios from "axios"; import { dataUrlToFile } from "@/lib/image-utils"; import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage"; import { imageToDataUrl } from "@/services/image-storage"; +import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video"; import { buildApiUrl, type AiConfig } from "@/stores/use-config-store"; import { useUserStore } from "@/stores/use-user-store"; import type { ReferenceImage } from "@/types/image"; -import type { ReferenceVideo } from "@/types/media"; +import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; type VideoResponse = { id: string; status?: string; error?: { message?: string } }; type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string }; @@ -42,14 +43,14 @@ function refreshRemoteUser(config: AiConfig) { if (config.channelMode === "remote") void useUserStore.getState().hydrateUser(); } -export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = []): Promise { +export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = []): Promise { const model = (config.model || config.videoModel).trim(); assertVideoConfig(config, model); - if (isSeedanceConfig(config, model)) { - return requestSeedanceGeneration(config, model, prompt, references, videoReferences); + if (isSeedanceVideoConfig({ ...config, model })) { + return requestSeedanceGeneration(config, model, prompt, references, videoReferences, audioReferences); } - if (videoReferences.length) { - throw new Error("当前视频接口不支持参考视频，请切换到 Seedance 2.0 / 火山 Agent Plan 模型，或移除参考视频"); + if (videoReferences.length || audioReferences.length) { + throw new Error("当前视频接口不支持参考视频或参考音频，请切换到 Seedance 2.0 / 火山 Agent Plan 模型，或移除参考素材"); } return requestOpenAIVideoGeneration(config, model, prompt, references); } @@ -89,16 +90,22 @@ async function requestOpenAIVideoGeneration(config: AiConfig, model: string, pro } } -async function requestSeedanceGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[]) { - const content = await buildSeedanceContent(config, prompt, references, videoReferences); - if (!content.length) throw new Error("请输入视频提示词，或连接参考图片/视频"); +async function requestSeedanceGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) { + if (audioReferences.length && !references.length && !videoReferences.length) { + throw new Error("Seedance 参考音频不能单独使用，请同时添加参考图或参考视频"); + } + assertSeedanceVideoReferences(videoReferences); + assertSeedanceAudioReferences(audioReferences); + const content = await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences); + if (!content.length) throw new Error("请输入视频提示词，或连接参考图片/视频/音频"); const payload = { model, content, ratio: normalizeSeedanceRatio(config.size), - resolution: normalizeVideoResolution(config.vquality), + resolution: normalizeSeedanceResolution(config.vquality, model), duration: normalizeSeedanceDuration(config.videoSeconds), - watermark: false, + generate_audio: boolConfig(config.videoGenerateAudio, true), + watermark: boolConfig(config.videoWatermark, false), }; try { @@ -122,21 +129,44 @@ async function requestSeedanceGeneration(config: AiConfig, model: string, prompt } } +function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) { + let total = 0; + for (const video of videoReferences) { + if (!video.durationMs) continue; + if (video.durationMs < 2000 || video.durationMs > 15000) throw new Error("Seedance 参考视频单个时长需要在 2-15 秒之间"); + total += video.durationMs; + } + if (total > 15000) throw new Error("Seedance 参考视频总时长不能超过 15 秒"); +} + +function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) { + let total = 0; + for (const audio of audioReferences) { + if (!audio.durationMs) continue; + if (audio.durationMs < 2000 || audio.durationMs > 15000) throw new Error("Seedance 参考音频单个时长需要在 2-15 秒之间"); + total += audio.durationMs; + } + if (total > 15000) throw new Error("Seedance 参考音频总时长不能超过 15 秒"); +} + function seedanceApiUrl(config: AiConfig, taskId?: string) { if (config.channelMode === "remote") return taskId ? `/api/v1/videos/${encodeURIComponent(taskId)}` : "/api/v1/videos"; return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`); } -async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[]) { +async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) { const content: Array> = []; const text = prompt.trim(); if (text) content.push({ type: "text", text }); - for (const image of references.slice(0, 7)) { + for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) { content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" }); } - for (const video of videoReferences.slice(0, 3)) { + for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) { content.push({ type: "video_url", video_url: { url: await resolveSeedanceVideoUrl(video) }, role: "reference_video" }); } + for (const audio of audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios)) { + content.push({ type: "audio_url", audio_url: { url: await resolveSeedanceAudioUrl(audio) }, role: "reference_audio" }); + } return content; } @@ -161,6 +191,16 @@ async function resolveSeedanceVideoUrl(video: ReferenceVideo) { return uploadReferenceMedia(file); } +async function resolveSeedanceAudioUrl(audio: ReferenceAudio) { + if (isPublicMediaUrl(audio.url) || audio.url.startsWith("asset://")) return audio.url; + let blob: Blob | null = null; + if (audio.storageKey) blob = await getMediaBlob(audio.storageKey); + if (!blob && audio.url?.startsWith("blob:")) blob = await (await fetch(audio.url)).blob(); + if (!blob) throw new Error("参考音频必须是公网 URL、素材 ID，或本地已保存的音频"); + const file = new File([blob], audio.name || "reference-audio.mp3", { type: audio.type || blob.type || "audio/mpeg" }); + return uploadReferenceMedia(file); +} + async function uploadReferenceMedia(file: File) { const token = useUserStore.getState().token; if (!token) throw new Error("使用本地参考素材需要先登录，并在服务端配置 PUBLIC_BASE_URL"); @@ -182,11 +222,6 @@ async function videoResultFromUrl(url: string): Promise { } } -function isSeedanceConfig(config: AiConfig, model: string) { - const value = `${model} ${config.baseUrl}`.toLowerCase(); - return value.includes("seedance") || value.includes("doubao-seedance") || value.includes("ark.cn-beijing.volces.com/api/plan/v3"); -} - function assertVideoConfig(config: AiConfig, model: string) { if (!model) throw new Error("请先配置视频模型"); if (config.channelMode === "local" && !config.baseUrl.trim()) throw new Error("请先配置 Base URL"); @@ -198,11 +233,6 @@ function normalizeVideoSeconds(value: string) { return String(Math.max(1, Math.min(20, seconds))); } -function normalizeSeedanceDuration(value: string) { - const seconds = Math.floor(Number(value) || 5); - return Math.max(4, Math.min(15, seconds)); -} - function normalizeVideoSize(value: string) { if (value === "auto") return null; const size = value || "1280x720"; @@ -210,26 +240,6 @@ function normalizeVideoSize(value: string) { return ["9:16", "2:3", "3:4"].includes(size) ? "720x1280" : "1280x720"; } -function normalizeSeedanceRatio(value: string) { - if (!value || value === "auto") return "adaptive"; - if (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"].includes(value)) return value; - const match = value.match(/^(\d+)x(\d+)$/); - if (!match) return "adaptive"; - const width = Number(match[1]); - const height = Number(match[2]); - if (!width || !height) return "adaptive"; - const ratio = width / height; - const options = [ - ["16:9", 16 / 9], - ["4:3", 4 / 3], - ["1:1", 1], - ["3:4", 3 / 4], - ["9:16", 9 / 16], - ["21:9", 21 / 9], - ] as const; - return options.reduce((best, item) => (Math.abs(item[1] - ratio) < Math.abs(best[1] - ratio) ? item : best), options[0])[0]; -} - function normalizeVideoResolution(value: string) { if (value === "low") return "480p"; if (value === "auto" || value === "high" || value === "medium") return "720p"; diff --git a/web/src/services/file-storage.ts b/web/src/services/file-storage.ts index 6a1eed6..60ee07a 100644 --- a/web/src/services/file-storage.ts +++ b/web/src/services/file-storage.ts @@ -3,7 +3,7 @@ import localforage from "localforage"; import { nanoid } from "nanoid"; -export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number }; +export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number; durationMs?: number }; const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" }); const objectUrls = new Map(); @@ -14,7 +14,7 @@ export async function uploadMediaFile(input: string | Blob, prefix = "file"): Pr await store.setItem(storageKey, blob); const url = URL.createObjectURL(blob); objectUrls.set(storageKey, url); - const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : {}; + const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : blob.type.startsWith("audio/") ? await readAudioMeta(url) : {}; return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta }; } @@ -68,11 +68,21 @@ export function collectMediaStorageKeys(value: unknown, keys = new Set() } function readVideoMeta(url: string) { - return new Promise<{ width: number; height: number }>((resolve) => { + return new Promise<{ width: number; height: number; durationMs?: number }>((resolve) => { const video = document.createElement("video"); - const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720 }); + const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720, durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined }); video.onloadedmetadata = done; video.onerror = done; video.src = url; }); } + +function readAudioMeta(url: string) { + return new Promise<{ durationMs?: number }>((resolve) => { + const audio = document.createElement("audio"); + const done = () => resolve({ durationMs: Number.isFinite(audio.duration) ? Math.round(audio.duration * 1000) : undefined }); + audio.onloadedmetadata = done; + audio.onerror = done; + audio.src = url; + }); +} diff --git a/web/src/stores/use-config-store.ts b/web/src/stores/use-config-store.ts index 49e0af4..625fecb 100644 --- a/web/src/stores/use-config-store.ts +++ b/web/src/stores/use-config-store.ts @@ -17,6 +17,8 @@ export type AiConfig = { textModel: string; videoSeconds: string; vquality: string; + videoGenerateAudio: string; + videoWatermark: string; systemPrompt: string; models: string[]; quality: string; @@ -36,6 +38,8 @@ export const defaultConfig: AiConfig = { textModel: "gpt-5.5", videoSeconds: "6", vquality: "720", + videoGenerateAudio: "true", + videoWatermark: "false", systemPrompt: "", models: [], quality: "auto", @@ -61,19 +65,44 @@ function resolveEffectiveConfig(config: AiConfig, modelChannel: AdminPublicSetti const channelMode = modelChannel?.allowCustomChannel ? config.channelMode : "remote"; if (channelMode === "local" || !modelChannel) return { ...config, channelMode }; const models = modelChannel.availableModels; - const fallbackModel = modelChannel.defaultModel || models[0] || ""; + const fallbackTextModel = validDefault(modelChannel.defaultTextModel, models) || preferredModel(models, isTextModelName); + const fallbackModel = validDefault(modelChannel.defaultModel, models) || fallbackTextModel || models[0] || ""; + const fallbackImageModel = validDefault(modelChannel.defaultImageModel, models) || preferredModel(models, isImageModelName) || fallbackModel; + const fallbackVideoModel = validDefault(modelChannel.defaultVideoModel, models) || preferredModel(models, isVideoModelName) || fallbackModel; return { ...config, channelMode, models, model: models.includes(config.model) ? config.model : fallbackModel, - imageModel: models.includes(config.imageModel) ? config.imageModel : modelChannel.defaultImageModel || fallbackModel, - videoModel: models.includes(config.videoModel) ? config.videoModel : modelChannel.defaultVideoModel || fallbackModel, - textModel: models.includes(config.textModel) ? config.textModel : modelChannel.defaultTextModel || fallbackModel, + imageModel: models.includes(config.imageModel) ? config.imageModel : fallbackImageModel, + videoModel: models.includes(config.videoModel) ? config.videoModel : fallbackVideoModel, + textModel: models.includes(config.textModel) ? config.textModel : fallbackTextModel || fallbackModel, systemPrompt: modelChannel.systemPrompt, }; } +function validDefault(model: string, models: string[]) { + return models.includes(model) ? model : ""; +} + +function preferredModel(models: string[], predicate: (model: string) => boolean) { + return models.find(predicate) || ""; +} + +function isVideoModelName(model: string) { + const value = model.toLowerCase(); + return value.includes("seedance") || value.includes("video"); +} + +function isImageModelName(model: string) { + const value = model.toLowerCase(); + return value.includes("seedream") || value.includes("gpt-image") || value.includes("image"); +} + +function isTextModelName(model: string) { + return !isImageModelName(model) && !isVideoModelName(model); +} + function isAiConfigReady(config: AiConfig, model: string) { return Boolean(model.trim()) && (config.channelMode === "remote" || Boolean(config.baseUrl.trim() && config.apiKey.trim())); } @@ -112,7 +141,7 @@ export const useConfigStore = create()( partialize: (state) => ({ config: state.config }), merge: (persisted, current) => { const config = { ...defaultConfig, ...((persisted as Partial).config || {}) }; - return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720" } }; + return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720", videoGenerateAudio: config.videoGenerateAudio || "true", videoWatermark: config.videoWatermark || "false" } }; }, }, ), diff --git a/web/src/types/media.ts b/web/src/types/media.ts index 6d95955..84d4e2d 100644 --- a/web/src/types/media.ts +++ b/web/src/types/media.ts @@ -4,4 +4,14 @@ export type ReferenceVideo = { type: string; url: string; storageKey?: string; + durationMs?: number; +}; + +export type ReferenceAudio = { + id: string; + name: string; + type: string; + url: string; + storageKey?: string; + durationMs?: number; };