feat: add seedance media params and model visibility

This commit is contained in:
stupid-h4er
2026-05-27 13:02:36 +08:00
parent 871bedc072
commit 4bffb55695
25 changed files with 873 additions and 140 deletions
+2 -2
View File
@@ -82,11 +82,11 @@ Base URL 如果已经以 `/v1`、`/api/v3` 或 `/api/plan/v3` 结尾,系统不
- 图片比例。
- 生成数量。
- 视频模型。
- 视频比例、清晰度时长。
- 视频比例、清晰度时长、生成声音和水印
普通图片/文本节点可以直接输入提示词生成结果。生成配置节点可以读取上游节点内容,并按节点自己的配置批量生成多个图片或文本结果。生成配置节点支持预览当前提示词和参考图输入,并调整输入顺序。
视频生成可从文本节点读取 prompt,从图片节点读取参考图,从视频节点读取参考视频。生成成功后会把视频插入画布为视频节点并使用原生播放器预览。Seedance 参考视频需要公网可访问 URL;本地上传视频会先通过 `/api/v1/media/references` 保存到服务端,再由 `PUBLIC_BASE_URL` 生成可供火山服务器拉取的公开链接。
视频生成可从文本节点读取 prompt,从图片节点读取参考图,从视频节点读取参考视频,从音频节点读取参考音频。Seedance 2.0 支持最多 9 张参考图、3 个参考视频、3 个参考音频;分辨率支持 `480p``720p``1080p`fast 模型不支持 `1080p`),比例支持 `16:9``4:3``1:1``3:4``9:16``21:9``adaptive`,时长支持 4-15 秒或智能时长。生成成功后会把视频插入画布为视频节点并使用原生播放器预览。Seedance 参考视频和参考音频需要公网可访问 URL;本地上传素材会先通过 `/api/v1/media/references` 保存到服务端,再由 `PUBLIC_BASE_URL` 生成可供火山服务器拉取的公开链接。
## 画布助手
+5 -1
View File
@@ -22,5 +22,9 @@
- 新增 `/video` 视频创作台页面,参考生图工作台布局,支持提示词、参考图、视频参数、生成结果、保存素材、下载和本地生成记录;清晰度和秒数均支持常用值选择与手动输入,生成记录只保存媒体 `storageKey` 并可回填本次提示词、参考图和参数。
- 视频创作台生成前会把模型、尺寸、秒数、清晰度归一化为视频接口支持的参数,并展示后端返回的错误信息,避免页面侧残留的生图参数影响视频请求。
- 火山方舟 Agent Plan / Seedance 2.0 视频生成需要在真实账号下验证:`/contents/generations/tasks` 创建任务、轮询状态、`content.video_url` 回填画布,以及 401/403/429/超时错误提示。
- `PUBLIC_BASE_URL` 已配置公网域名时,需要验证本地上传参考视频能被火山拉取;未配置或配置为内网地址时,需要验证前端能给出明确提示
- 管理后台保存私有渠道后,需要验证所有已启用渠道里的模型会自动出现在公开 `availableModels`,并且 `defaultVideoModel``defaultImageModel``defaultTextModel` 在为空或失效时会自动修复,前台不再显示旧的 `grok` 默认值
- `/video` 和画布视频设置已按 Seedance 2.0 增加分辨率、比例、4-15 秒/智能时长、生成声音和水印参数;需要在真实浏览器里验证参数回填、生成记录和画布节点配置都能保持一致。
- `/video` 支持最多 9 张参考图、3 个参考视频、3 个参考音频;需要验证格式、大小、音频时长提示和生成请求中的 `reference_image``reference_video``reference_audio` 组装。
- 画布新增音频节点,支持上传、拖入、播放、移动、缩放、删除,并可作为上游参考音频参与 Seedance 视频生成;需要验证刷新后本地音频 URL 能恢复。
- `PUBLIC_BASE_URL` 已配置公网域名时,需要验证本地上传参考视频和参考音频能被火山拉取;未配置或配置为内网地址时,需要验证前端能给出明确提示。
- Seedance 返回远程视频 URL 但浏览器无法下载为 Blob 时,需要验证视频节点刷新后仍保留远程 URL,并确认上游 URL 有效期限制。
+5 -4
View File
@@ -41,11 +41,12 @@
| 字段 | 类型 | 说明 |
| --- | --- | --- |
| `availableModels` | string[] | 系统可用模型,由管理员手动选择;页面下拉选项可来自私有渠道模型 |
| `availableModels` | string[] | 系统可用模型;保存设置时会自动合并所有已启用私有渠道模型 |
| `modelCosts` | object[] | 模型算力点配置,后端模型接口调用前按模型预扣,上游失败时返还;未配置默认不扣除 |
| `defaultModel` | string | 默认模型,从 `availableModels` 中选择 |
| `defaultImageModel` | string | 默认图片模型,从 `availableModels` 中选择 |
| `defaultTextModel` | string | 默认文本模型,从 `availableModels` 中选择 |
| `defaultModel` | string | 默认模型,从 `availableModels` 中选择;为空或失效时优先选择文本模型 |
| `defaultImageModel` | string | 默认图片模型,从 `availableModels` 中选择;为空或失效时优先选择 `seedream``image``gpt-image` 模型 |
| `defaultVideoModel` | string | 默认视频模型,从 `availableModels` 中选择;为空或失效时优先选择 `seedance``video` 模型 |
| `defaultTextModel` | string | 默认文本模型,从 `availableModels` 中选择;为空或失效时优先选择非图片/视频模型 |
| `systemPrompt` | string | 系统提示词 |
| `allowCustomChannel` | boolean | 是否允许用户在配置弹窗中切换为本地直连渠道,默认允许 |
+53 -5
View File
@@ -12,7 +12,16 @@ import (
"github.com/google/uuid"
)
const referenceMediaMaxBytes = 80 << 20
const (
referenceMediaMaxBytes = 80 << 20
referenceImageMaxBytes = 30 << 20
referenceVideoMaxBytes = 50 << 20
referenceAudioMaxBytes = 15 << 20
referenceImageAllowedText = "jpeg/png/webp/bmp/gif/heic/heif 图片"
referenceVideoAllowedText = "mp4/mov 视频"
referenceAudioAllowedText = "mp3/wav 音频"
referenceMediaAllowedText = referenceImageAllowedText + "、" + referenceVideoAllowedText + "或" + referenceAudioAllowedText
)
type referenceMediaUploadResult struct {
ID string `json:"id"`
@@ -24,7 +33,7 @@ type referenceMediaUploadResult struct {
func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) {
publicBaseURL := strings.TrimRight(strings.TrimSpace(config.Cfg.PublicBaseURL), "/")
if publicBaseURL == "" {
Fail(w, "未配置 PUBLIC_BASE_URL,无法把本地参考视频提供给火山方舟访问")
Fail(w, "未配置 PUBLIC_BASE_URL,无法把本地参考素材提供给火山方舟访问")
return
}
r.Body = http.MaxBytesReader(w, r.Body, referenceMediaMaxBytes+1)
@@ -44,7 +53,7 @@ func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) {
mimeType, ext, ok := normalizeReferenceMediaType(header.Header.Get("Content-Type"), filepath.Ext(header.Filename))
if !ok {
Fail(w, "参考素材格式不支持,请使用 jpeg/png/webp/bmp/gif/heic/heif 图片或 mp4/mov 视频")
Fail(w, "参考素材格式不支持,请使用 "+referenceMediaAllowedText)
return
}
if err := os.MkdirAll(referenceMediaDir(), 0o755); err != nil {
@@ -65,9 +74,14 @@ func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) {
Fail(w, "参考素材保存失败")
return
}
if bytes <= 0 || bytes > referenceMediaMaxBytes {
if bytes <= 0 {
_ = os.Remove(targetPath)
Fail(w, "参考素材为空或超过大小限制")
Fail(w, "参考素材为空")
return
}
if limit := referenceMediaTypeMaxBytes(mimeType); limit > 0 && bytes > limit {
_ = os.Remove(targetPath)
Fail(w, referenceMediaSizeMessage(mimeType))
return
}
OK(w, referenceMediaUploadResult{
@@ -141,6 +155,10 @@ func referenceMediaExtByMimeType(mimeType string) string {
return ".mp4"
case "video/quicktime", "video/mov":
return ".mov"
case "audio/mpeg", "audio/mp3":
return ".mp3"
case "audio/wav", "audio/x-wav", "audio/wave":
return ".wav"
default:
return ""
}
@@ -166,7 +184,37 @@ func mimeTypeByReferenceMediaExt(ext string) string {
return "video/mp4"
case ".mov":
return "video/quicktime"
case ".mp3":
return "audio/mpeg"
case ".wav":
return "audio/wav"
default:
return ""
}
}
func referenceMediaTypeMaxBytes(mimeType string) int64 {
if strings.HasPrefix(mimeType, "image/") {
return referenceImageMaxBytes
}
if strings.HasPrefix(mimeType, "video/") {
return referenceVideoMaxBytes
}
if strings.HasPrefix(mimeType, "audio/") {
return referenceAudioMaxBytes
}
return referenceMediaMaxBytes
}
func referenceMediaSizeMessage(mimeType string) string {
if strings.HasPrefix(mimeType, "image/") {
return "参考图片超过大小限制,请使用 30MB 以内的图片"
}
if strings.HasPrefix(mimeType, "video/") {
return "参考视频超过大小限制,请使用 50MB 以内的 mp4/mov 视频"
}
if strings.HasPrefix(mimeType, "audio/") {
return "参考音频超过大小限制,请使用 15MB 以内的 mp3/wav 音频"
}
return "参考素材超过大小限制"
}
+40
View File
@@ -0,0 +1,40 @@
package handler
import "testing"
func TestNormalizeReferenceMediaTypeSupportsAudio(t *testing.T) {
tests := []struct {
name string
contentType string
ext string
wantMime string
wantExt string
}{
{name: "mp3 mime", contentType: "audio/mpeg", ext: ".bin", wantMime: "audio/mpeg", wantExt: ".mp3"},
{name: "wav ext fallback", contentType: "application/octet-stream", ext: ".wav", wantMime: "audio/wav", wantExt: ".wav"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mimeType, ext, ok := normalizeReferenceMediaType(tt.contentType, tt.ext)
if !ok {
t.Fatal("expected media type to be accepted")
}
if mimeType != tt.wantMime || ext != tt.wantExt {
t.Fatalf("got (%q, %q), want (%q, %q)", mimeType, ext, tt.wantMime, tt.wantExt)
}
})
}
}
func TestReferenceMediaTypeMaxBytes(t *testing.T) {
if got := referenceMediaTypeMaxBytes("audio/mpeg"); got != referenceAudioMaxBytes {
t.Fatalf("audio max bytes = %d, want %d", got, referenceAudioMaxBytes)
}
if got := referenceMediaTypeMaxBytes("video/mp4"); got != referenceVideoMaxBytes {
t.Fatalf("video max bytes = %d, want %d", got, referenceVideoMaxBytes)
}
if got := referenceMediaTypeMaxBytes("image/png"); got != referenceImageMaxBytes {
t.Fatalf("image max bytes = %d, want %d", got, referenceImageMaxBytes)
}
}
+73 -2
View File
@@ -20,7 +20,7 @@ var adminModelHTTPClient = &http.Client{Timeout: 30 * time.Second}
func PublicSettings() (model.PublicSetting, error) {
settings, err := repository.GetSettings()
return normalizePublicSetting(settings.Public), err
return normalizeSettings(settings).Public, err
}
func AdminSettings() (model.Settings, error) {
@@ -63,12 +63,16 @@ func AdminTestChannelModel(index *int, channel model.ModelChannel, modelName str
}
func normalizeSettings(settings model.Settings) model.Settings {
settings.Public = normalizePublicSetting(settings.Public)
settings.Private = normalizePrivateSetting(settings.Private)
settings.Public = normalizePublicSettingWithChannels(settings.Public, settings.Private.Channels)
return settings
}
func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting {
return normalizePublicSettingWithChannels(setting, nil)
}
func normalizePublicSettingWithChannels(setting model.PublicSetting, channels []model.ModelChannel) model.PublicSetting {
if setting.ModelChannel.AvailableModels == nil {
setting.ModelChannel.AvailableModels = []string{}
}
@@ -89,6 +93,16 @@ func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting {
enabled := true
setting.Auth.AllowRegister = &enabled
}
enabledModels := enabledChannelModels(channels)
if len(enabledModels) > 0 {
setting.ModelChannel.AvailableModels = enabledModels
} else {
setting.ModelChannel.AvailableModels = uniqueModelNames(setting.ModelChannel.AvailableModels)
}
setting.ModelChannel.DefaultTextModel = repairDefaultModel(setting.ModelChannel.DefaultTextModel, setting.ModelChannel.AvailableModels, isTextModelName)
setting.ModelChannel.DefaultImageModel = repairDefaultModel(setting.ModelChannel.DefaultImageModel, setting.ModelChannel.AvailableModels, isImageModelName)
setting.ModelChannel.DefaultVideoModel = repairDefaultModel(setting.ModelChannel.DefaultVideoModel, setting.ModelChannel.AvailableModels, isVideoModelName)
setting.ModelChannel.DefaultModel = repairDefaultModel(setting.ModelChannel.DefaultModel, setting.ModelChannel.AvailableModels, isTextModelName)
return setting
}
@@ -224,6 +238,63 @@ func isSeedanceModelName(modelName string) bool {
return strings.Contains(modelName, "seedance") || strings.Contains(modelName, "doubao-seedance")
}
func enabledChannelModels(channels []model.ModelChannel) []string {
models := []string{}
for _, channel := range channels {
if !channel.Enabled {
continue
}
models = append(models, channel.Models...)
}
return uniqueModelNames(models)
}
func uniqueModelNames(models []string) []string {
result := []string{}
seen := map[string]bool{}
for _, item := range models {
name := strings.TrimSpace(item)
if name == "" || seen[name] {
continue
}
seen[name] = true
result = append(result, name)
}
return result
}
func repairDefaultModel(current string, models []string, preferred func(string) bool) string {
current = strings.TrimSpace(current)
for _, item := range models {
if item == current {
return current
}
}
for _, item := range models {
if preferred(item) {
return item
}
}
if len(models) > 0 {
return models[0]
}
return ""
}
func isVideoModelName(modelName string) bool {
name := strings.ToLower(strings.TrimSpace(modelName))
return strings.Contains(name, "seedance") || strings.Contains(name, "video")
}
func isImageModelName(modelName string) bool {
name := strings.ToLower(strings.TrimSpace(modelName))
return strings.Contains(name, "seedream") || strings.Contains(name, "gpt-image") || strings.Contains(name, "image")
}
func isTextModelName(modelName string) bool {
return !isImageModelName(modelName) && !isVideoModelName(modelName)
}
func normalizeModelChannel(channel model.ModelChannel) model.ModelChannel {
if channel.Protocol == "" {
channel.Protocol = "openai"
+38
View File
@@ -60,3 +60,41 @@ func TestBuildModelChannelURLNormalizesArkPlanTaskPath(t *testing.T) {
t.Fatalf("BuildModelChannelURL = %q, want %q", got, want)
}
}
func TestNormalizeSettingsPublishesEnabledChannelModelsAndRepairsDefaults(t *testing.T) {
settings := normalizeSettings(model.Settings{
Public: model.PublicSetting{
ModelChannel: model.PublicModelChannelSetting{
AvailableModels: []string{"grok-imagine-video", "disabled-model"},
DefaultModel: "grok-imagine-video",
DefaultTextModel: "missing-text",
DefaultImageModel: "missing-image",
DefaultVideoModel: "missing-video",
},
},
Private: model.PrivateSetting{
Channels: []model.ModelChannel{
{Enabled: true, Models: []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast", "gpt-5.5"}},
{Enabled: false, Models: []string{"disabled-model"}},
},
},
})
channel := settings.Public.ModelChannel
wantModels := []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast"}
if !reflect.DeepEqual(channel.AvailableModels, wantModels) {
t.Fatalf("available models = %#v, want %#v", channel.AvailableModels, wantModels)
}
if channel.DefaultModel != "gpt-5.5" {
t.Fatalf("default model = %q, want text model", channel.DefaultModel)
}
if channel.DefaultTextModel != "gpt-5.5" {
t.Fatalf("default text model = %q, want text model", channel.DefaultTextModel)
}
if channel.DefaultImageModel != "doubao-seedream-5.0-lite" {
t.Fatalf("default image model = %q, want seedream", channel.DefaultImageModel)
}
if channel.DefaultVideoModel != "doubao-seedance-2.0-fast" {
t.Fatalf("default video model = %q, want seedance", channel.DefaultVideoModel)
}
}
+3 -8
View File
@@ -341,7 +341,7 @@ export default function AdminSettingsPage() {
const nextChannelModels = collectChannelModels(nextChannels);
const nextSettings = normalizeSettings({
...values,
public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: filterModels(values.public.modelChannel.availableModels, nextChannelModels) } },
public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: nextChannelModels } },
private: { ...values.private, channels: nextChannels },
});
const saved = normalizeSettings(await saveAdminSettings(token, nextSettings));
@@ -414,7 +414,7 @@ export default function AdminSettingsPage() {
<Form form={form} layout="vertical" initialValues={emptySettings} requiredMark={false}>
<Row gutter={16}>
<Col span={24}>
<Form.Item name={["public", "modelChannel", "availableModels"]} label="系统可用模型(请先在私有配置里配置渠道)" extra="可选项来自已启用渠道中选择的模型,最终开放哪些模型由这里勾选决定">
<Form.Item name={["public", "modelChannel", "availableModels"]} label="系统可用模型(请先在私有配置里配置渠道)" extra="保存设置时会自动合并所有已启用私有渠道的模型,前台模型下拉会读取这里的公开列表">
<Select mode="multiple" placeholder="请选择系统可用模型" options={channelModels.map((item) => ({ label: item, value: item }))} />
</Form.Item>
</Col>
@@ -931,11 +931,6 @@ function uniqueModels(models: string[]) {
return Array.from(new Set(models.filter(Boolean)));
}
function filterModels(models: string[], options: string[]) {
const optionSet = new Set(options);
return uniqueModels(models).filter((model) => optionSet.has(model));
}
function modelSummary(models: string[]) {
if (!models.length) return "未配置模型";
const preview = models.slice(0, 3).join(", ");
@@ -971,7 +966,7 @@ async function collectSettings(form: any, editorMode: Record<SettingsTabKey, Edi
}
values.private = privateSetting;
}
values.public.modelChannel.availableModels = filterModels(values.public.modelChannel.availableModels, collectChannelModels(values.private.channels));
values.public.modelChannel.availableModels = collectChannelModels(values.private.channels);
return normalizeSettings(values);
}
@@ -3,7 +3,7 @@
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react";
import { useParams, useRouter } from "next/navigation";
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Music2, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
import { saveAs } from "file-saver";
import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
@@ -52,6 +52,7 @@ import {
type ViewportTransform,
} from "../types";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceAudio } from "@/types/media";
type CanvasClipboard = {
nodes: CanvasNodeData[];
@@ -141,7 +142,7 @@ function CanvasRefreshShell() {
);
}
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video) => void; onClose: () => void }) {
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio) => void; onClose: () => void }) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
return (
<div
@@ -163,6 +164,7 @@ function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: Pending
<ConnectionCreateOption theme={theme} icon={<List className="size-5" />} title="文本生成" description="脚本、广告词、品牌文案" onClick={() => onCreate(CanvasNodeType.Text)} />
<ConnectionCreateOption theme={theme} icon={<ImageIcon className="size-5" />} title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} />
<ConnectionCreateOption theme={theme} icon={<Video className="size-5" />} title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} />
<ConnectionCreateOption theme={theme} icon={<Music2 className="size-5" />} title="音频参考" onClick={() => onCreate(CanvasNodeType.Audio)} />
<ConnectionCreateOption theme={theme} icon={<Settings2 className="size-5" />} title="配置节点" description="模型、尺寸、数量和输入顺序" onClick={() => onCreate(CanvasNodeType.Config)} />
</div>
</div>
@@ -483,7 +485,7 @@ function InfiniteCanvasPage() {
);
const createConnectedNode = useCallback(
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video, pending: PendingConnectionCreate) => {
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio, pending: PendingConnectionCreate) => {
const metadata = type === CanvasNodeType.Config ? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: 3 } : undefined;
const newNode = createCanvasNode(type, pending.position, metadata);
const connection = normalizeConnection(pending.connection.nodeId, newNode.id, [...nodesRef.current, newNode], pending.connection.handleType);
@@ -495,7 +497,7 @@ function InfiniteCanvasPage() {
setConnections((prev) => [...prev, { id: nanoid(), ...connection }]);
setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null);
if (type !== CanvasNodeType.Text) setDialogNodeId(newNode.id);
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
setPendingConnectionCreate(null);
setConnecting(null);
},
@@ -611,7 +613,7 @@ function InfiniteCanvasPage() {
setNodes((prev) => [...prev, newNode]);
setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null);
if (type !== CanvasNodeType.Text) setDialogNodeId(newNode.id);
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
},
[effectiveConfig.imageModel, effectiveConfig.model, effectiveConfig.size, getCanvasCenter],
);
@@ -1111,6 +1113,26 @@ function InfiniteCanvasPage() {
setDialogNodeId(id);
}, []);
const createAudioFileNode = useCallback(async (file: File, position: Position) => {
const audio = await uploadMediaFile(file, "audio");
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio];
const id = `audio-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
setNodes((prev) => [
...prev,
{
id,
type: CanvasNodeType.Audio,
title: file.name,
position: { x: position.x - spec.width / 2, y: position.y - spec.height / 2 },
width: spec.width,
height: spec.height,
metadata: audioMetadata(audio),
},
]);
setSelectedNodeIds(new Set([id]));
setSelectedConnectionId(null);
}, []);
const createTextNodeFromClipboard = useCallback(
(text: string) => {
const trimmed = text.trim();
@@ -1324,8 +1346,8 @@ function InfiniteCanvasPage() {
}, []);
const downloadNodeImage = useCallback((node: CanvasNodeData) => {
if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video) || !node.metadata?.content) return;
saveAs(node.metadata.content, `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : imageExtension(node.metadata.content)}`);
if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video && node.type !== CanvasNodeType.Audio) || !node.metadata?.content) return;
saveAs(node.metadata.content, `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : node.type === CanvasNodeType.Audio ? audioExtension(node.metadata.mimeType) : imageExtension(node.metadata.content)}`);
}, []);
const saveNodeAsset = useCallback(
@@ -1453,9 +1475,19 @@ function InfiniteCanvasPage() {
async (event: ReactChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
const target = uploadTargetRef.current;
if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/"))) return;
if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/") && !isAudioFile(file))) return;
if (target?.nodeId) {
if (isAudioFile(file)) {
const audio = await uploadMediaFile(file, "audio");
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio];
setNodes((prev) => prev.map((node) => (node.id === target.nodeId ? { ...node, type: CanvasNodeType.Audio, title: file.name, position: { x: node.position.x + node.width / 2 - spec.width / 2, y: node.position.y + node.height / 2 - spec.height / 2 }, width: spec.width, height: spec.height, metadata: { ...node.metadata, ...audioMetadata(audio), errorDetails: undefined } } : node)));
setSelectedNodeIds(new Set([target.nodeId]));
setSelectedConnectionId(null);
uploadTargetRef.current = null;
event.target.value = "";
return;
}
if (file.type.startsWith("video/")) {
const video = await uploadMediaFile(file, "video");
const nextSize = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
@@ -1505,25 +1537,25 @@ function InfiniteCanvasPage() {
setDialogNodeId(target.nodeId);
} else {
const position = target?.position || screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
void (file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position));
void (isAudioFile(file) ? createAudioFileNode(file, position) : file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position));
}
uploadTargetRef.current = null;
event.target.value = "";
},
[createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width],
[createAudioFileNode, createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width],
);
const handleDrop = useCallback(
(event: ReactDragEvent<HTMLDivElement>) => {
event.preventDefault();
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/"));
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/") || isAudioFile(item));
if (!file) return;
const pos = screenToCanvas(event.clientX, event.clientY);
void (file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos));
void (isAudioFile(file) ? createAudioFileNode(file, pos) : file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos));
},
[createImageFileNode, createVideoFileNode, screenToCanvas],
[createAudioFileNode, createImageFileNode, createVideoFileNode, screenToCanvas],
);
const pasteAssistantImage = useCallback(
@@ -1749,14 +1781,14 @@ function InfiniteCanvasPage() {
position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y },
width: isEmptyVideoNode ? sourceNode.width : spec.width,
height: isEmptyVideoNode ? sourceNode.height : spec.height,
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationReferenceUrls(generationContext) },
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, references: generationReferenceUrls(generationContext) },
};
pendingChildIds = [videoId];
setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode]));
if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]);
const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, effectivePrompt, generationContext.referenceImages, generationContext.referenceVideos));
const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, effectivePrompt, generationContext.referenceImages, generationContext.referenceVideos, generationContext.referenceAudios));
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 }, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationReferenceUrls(generationContext) } } : node)));
setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 }, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, references: generationReferenceUrls(generationContext) } } : node)));
return;
}
@@ -1874,9 +1906,9 @@ function InfiniteCanvasPage() {
return;
}
if (node.type === CanvasNodeType.Video) {
const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, prompt, retryImages, context?.referenceVideos || []));
const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, prompt, retryImages, context?.referenceVideos || [], context?.referenceAudios || []));
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality } } : item)));
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark } } : item)));
return;
}
@@ -2196,6 +2228,7 @@ function InfiniteCanvasPage() {
showImageInfo={showImageInfo}
onAddImage={() => createNode(CanvasNodeType.Image)}
onAddVideo={() => createNode(CanvasNodeType.Video)}
onAddAudio={() => createNode(CanvasNodeType.Audio)}
onAddText={() => createNode(CanvasNodeType.Text)}
onAddConfig={() => createNode(CanvasNodeType.Config)}
onUndo={undoCanvas}
@@ -2235,7 +2268,7 @@ function InfiniteCanvasPage() {
/>
) : null}
<input ref={imageInputRef} type="file" accept="image/*,video/*" className="hidden" onChange={handleImageInputChange} />
<input ref={imageInputRef} type="file" accept="image/*,video/*,audio/mpeg,audio/wav,audio/x-wav,.mp3,.wav" className="hidden" onChange={handleImageInputChange} />
<CanvasNodeInfoModal node={infoNode} open={Boolean(infoNode)} onClose={() => setInfoNodeId(null)} />
@@ -2377,7 +2410,7 @@ function CanvasTopBar({
{ key: "new", icon: <Plus className="size-4" />, label: "新建画布", onClick: onCreateProject },
{ key: "delete", danger: true, icon: <Trash2 className="size-4" />, label: "删除当前画布", onClick: onDeleteProject },
{ type: "divider" },
{ key: "import", icon: <Upload className="size-4" />, label: "导入图片", onClick: onImportImage },
{ key: "import", icon: <Upload className="size-4" />, label: "导入素材", onClick: onImportImage },
{ type: "divider" },
{ key: "undo", disabled: !canUndo, icon: <Undo2 className="size-4" />, label: <MenuLabel text="撤销" shortcut="⌘ Z" />, onClick: onUndo },
{ key: "redo", disabled: !canRedo, icon: <Redo2 className="size-4" />, label: <MenuLabel text="重做" shortcut="⌘ ⇧ Z / ⌘ Y" />, onClick: onRedo },
@@ -2458,7 +2491,7 @@ function CanvasTopBar({
<Shortcut keys={["Ctrl / Cmd", "Y"]} value="重做" />
<Shortcut keys={["Delete / Backspace"]} value="删除选中" />
<Shortcut keys={["Esc"]} value="取消选择并关闭浮层" />
<Shortcut keys={["拖入图片"]} value="上传到画布" />
<Shortcut keys={["拖入图片/视频/音频"]} value="上传到画布" />
</div>
</Modal>
</>
@@ -2499,12 +2532,20 @@ function imageExtension(dataUrl: string) {
return dataUrl.match(/^data:image[/]([^;]+)/)?.[1] || dataUrl.match(/image[/]([^;]+)/)?.[1] || "png";
}
function audioExtension(mimeType?: string) {
return mimeType?.includes("wav") ? "wav" : "mp3";
}
function imageMetadata(image: UploadedImage): CanvasNodeMetadata {
return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType };
}
function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4" };
return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4", durationMs: video.durationMs };
}
function audioMetadata(audio: UploadedFile): CanvasNodeMetadata {
return { content: audio.url, storageKey: audio.storageKey, status: "success", bytes: audio.bytes, mimeType: audio.mimeType || "audio/mpeg", durationMs: audio.durationMs };
}
function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata {
@@ -2522,10 +2563,11 @@ function referenceUrl(image: ReferenceImage) {
return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined);
}
function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }> }) {
function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }>; referenceAudios?: Array<{ storageKey?: string; url?: string }> }) {
return [
...context.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)),
...context.referenceVideos.map((video) => video.storageKey || video.url).filter((url): url is string => Boolean(url)),
...(context.referenceAudios || []).map((audio) => audio.storageKey || audio.url).filter((url): url is string => Boolean(url)),
];
}
@@ -2545,7 +2587,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
return Promise.all(
nodes.map(async (node) => {
const content = node.metadata?.content;
if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
if (node.type !== CanvasNodeType.Image || !content) return node;
if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } };
if (!content.startsWith("data:image/")) return node;
@@ -2605,6 +2647,7 @@ function getInputSummary(inputs: NodeGenerationInput[]) {
textCount: inputs.filter((input) => input.type === "text").length,
imageCount: inputs.filter((input) => input.type === "image").length,
videoCount: inputs.filter((input) => input.type === "video").length,
audioCount: inputs.filter((input) => input.type === "audio").length,
};
}
@@ -2617,6 +2660,8 @@ function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefine
size: node?.metadata?.size || config.size || defaultConfig.size,
videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds,
vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality,
videoGenerateAudio: node?.metadata?.generateAudio || config.videoGenerateAudio || defaultConfig.videoGenerateAudio,
videoWatermark: node?.metadata?.watermark || config.videoWatermark || defaultConfig.videoWatermark,
count: String(node?.metadata?.count || (mode === "image" ? 3 : config.count) || defaultConfig.count),
};
}
@@ -2652,6 +2697,10 @@ function sourceNodeReferenceImages(node: CanvasNodeData | null) {
];
}
function isAudioFile(file: File) {
return file.type.startsWith("audio/") || /\.(mp3|wav)$/i.test(file.name);
}
function isHiddenBatchChild(node: CanvasNodeData, nodes: CanvasNodeData[], collapsingBatchIds?: Set<string>) {
const rootId = node.metadata?.batchRootId;
if (!rootId) return false;
@@ -2,7 +2,7 @@
import type { CSSProperties } from "react";
import { useState } from "react";
import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play, Video } from "lucide-react";
import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Music2, Play, Video } from "lucide-react";
import { App, Button, Empty, Input, Modal, Segmented } from "antd";
import { ModelPicker } from "@/components/model-picker";
@@ -18,7 +18,7 @@ import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata } from ".
type CanvasConfigNodePanelProps = {
node: CanvasNodeData;
isRunning: boolean;
inputSummary: { textCount: number; imageCount: number; videoCount: number };
inputSummary: { textCount: number; imageCount: number; videoCount: number; audioCount: number };
inputs: NodeGenerationInput[];
onConfigChange: (nodeId: string, patch: Partial<CanvasNodeMetadata>) => void;
onTextInputChange: (nodeId: string, content: string) => void;
@@ -42,6 +42,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
const textInputs = inputs.filter((input) => input.type === "text");
const imageInputs = inputs.filter((input) => input.type === "image");
const videoInputs = inputs.filter((input) => input.type === "video");
const audioInputs = inputs.filter((input) => input.type === "audio");
const moveInput = (input: NodeGenerationInput, offset: number) => {
const sameTypeInputs = inputs.filter((item) => item.type === input.type);
@@ -114,6 +115,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
<InputChip label="提示词" value={`${inputSummary.textCount}`} style={chipStyle} />
<InputChip label="参考图" value={`${inputSummary.imageCount}`} style={chipStyle} />
<InputChip label="参考视频" value={`${inputSummary.videoCount}`} style={chipStyle} />
<InputChip label="参考音频" value={`${inputSummary.audioCount}`} style={chipStyle} />
<button type="button" className="inline-flex h-7 cursor-pointer items-center gap-1 rounded-md border px-2 text-[11px]" style={chipStyle} onClick={() => setPreviewOpen(true)}>
<Eye className="size-3.5" />
@@ -123,7 +125,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
<div className={`mb-2 grid min-w-0 cursor-default items-center gap-2 ${mode === "text" ? "grid-cols-1" : "grid-cols-[minmax(0,1fr)_148px]"}`} onMouseDown={(event) => event.stopPropagation()}>
<ModelPicker className="canvas-compact-control h-10" config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} fullWidth />
{mode === "video" ? (
<CanvasVideoSettingsPopover config={config} placement="topRight" buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, key === "videoSeconds" ? { seconds: value } : { [key]: value })} />
<CanvasVideoSettingsPopover config={config} placement="topRight" buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, videoConfigPatch(key, value))} />
) : mode === "image" ? (
<CanvasImageSettingsPopover config={config} placement="topRight" autoAdjustOverflow={false} buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, key === "count" ? { count: Number(value) || 1 } : { [key]: value })} />
) : null}
@@ -132,7 +134,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
<Button
type="primary"
className="mt-auto !h-9 !w-full !cursor-pointer !rounded-lg"
disabled={isRunning || (!inputSummary.textCount && !inputSummary.imageCount && !inputSummary.videoCount)}
disabled={isRunning || (!inputSummary.textCount && !inputSummary.imageCount && !inputSummary.videoCount && !inputSummary.audioCount)}
onMouseDown={(event) => event.stopPropagation()}
onClick={() => onGenerate(node.id)}
>
@@ -182,6 +184,15 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
</div>
</PreviewSection>
</div>
<div className="shrink-0">
<PreviewSection title="参考音频" count={audioInputs.length} empty="暂无参考音频">
<div className="thin-scrollbar flex gap-1.5 overflow-x-auto pb-1">
{audioInputs.map((input, index) => (
<AudioSortCard key={input.nodeId} input={input} audioIndex={index} audioTotal={audioInputs.length} theme={theme} onMove={moveInput} />
))}
</div>
</PreviewSection>
</div>
<div className="grid min-h-0 flex-1 grid-cols-2 gap-3 overflow-hidden">
<div className="thin-scrollbar min-h-0 overflow-y-auto pr-1.5">
<PreviewSection title="文本提示词" count={textInputs.length} empty="暂无文本提示词">
@@ -324,6 +335,36 @@ function VideoSortCard({
);
}
function AudioSortCard({
input,
audioIndex,
audioTotal,
theme,
onMove,
}: {
input: NodeGenerationInput;
audioIndex: number;
audioTotal: number;
theme: (typeof canvasThemes)[keyof typeof canvasThemes];
onMove: (input: NodeGenerationInput, offset: number) => void;
}) {
if (!input.audio) return null;
return (
<div className="w-48 shrink-0 rounded-lg border p-2" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
<div className="mb-1.5 flex min-w-0 items-center gap-1.5 text-[11px] opacity-70">
<Music2 className="size-3.5 shrink-0" />
<span className="truncate">{input.title}</span>
</div>
<audio src={input.audio.url} controls className="h-8 w-full" preload="metadata" />
<div className="mt-1 flex justify-between">
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !p-0" icon={<ArrowLeft className="size-3" />} disabled={audioIndex <= 0} onClick={() => onMove(input, -1)} />
<span className="text-[10px] opacity-45">{audioIndex + 1}</span>
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !p-0" icon={<ArrowRight className="size-3" />} disabled={audioIndex >= audioTotal - 1} onClick={() => onMove(input, 1)} />
</div>
</div>
);
}
function VerticalOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) {
return (
<>
@@ -360,6 +401,15 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can
size: node.metadata?.size || globalConfig.size || defaultConfig.size,
videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds,
vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality,
videoGenerateAudio: node.metadata?.generateAudio || globalConfig.videoGenerateAudio || defaultConfig.videoGenerateAudio,
videoWatermark: node.metadata?.watermark || globalConfig.videoWatermark || defaultConfig.videoWatermark,
count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count),
};
}
function videoConfigPatch(key: keyof AiConfig, value: string) {
if (key === "videoSeconds") return { seconds: value };
if (key === "videoGenerateAudio") return { generateAudio: value };
if (key === "videoWatermark") return { watermark: value };
return { [key]: value };
}
@@ -115,7 +115,7 @@ export function Minimap({ nodes, viewport, viewportSize, onViewportChange }: { n
>
{nodes.map((node) => {
const pos = toMinimap(node.position.x, node.position.y);
const color = node.type === CanvasNodeType.Image ? "#10b981" : node.type === CanvasNodeType.Config ? "#60a5fa" : theme.node.muted;
const color = node.type === CanvasNodeType.Image ? "#10b981" : node.type === CanvasNodeType.Video ? "#f97316" : node.type === CanvasNodeType.Audio ? "#a855f7" : node.type === CanvasNodeType.Config ? "#60a5fa" : theme.node.muted;
return (
<div
key={node.id}
@@ -1,24 +1,27 @@
import type { ChatCompletionMessage } from "@/services/api/image";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceVideo } from "@/types/media";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../types";
export type NodeGenerationContext = {
prompt: string;
referenceImages: ReferenceImage[];
referenceVideos: ReferenceVideo[];
referenceAudios: ReferenceAudio[];
textCount: number;
imageCount: number;
videoCount: number;
audioCount: number;
};
export type NodeGenerationInput = {
nodeId: string;
type: "text" | "image" | "video";
type: "text" | "image" | "video" | "audio";
title: string;
text?: string;
image?: ReferenceImage;
video?: ReferenceVideo;
audio?: ReferenceAudio;
};
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string): NodeGenerationContext {
@@ -29,14 +32,17 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
.join("\n\n");
const referenceImages = inputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
const referenceVideos = inputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video));
const referenceAudios = inputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio));
return {
prompt: upstreamText ? `${prompt}\n\n${upstreamText}` : prompt,
referenceImages,
referenceVideos,
referenceAudios,
textCount: inputs.filter((input) => input.type === "text").length,
imageCount: referenceImages.length,
videoCount: referenceVideos.length,
audioCount: referenceAudios.length,
};
}
@@ -46,6 +52,8 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[
if (image) return [{ nodeId: node.id, type: "image" as const, title: node.title, image }];
const video = readReferenceVideo(node);
if (video) return [{ nodeId: node.id, type: "video" as const, title: node.title, video }];
const audio = readReferenceAudio(node);
if (audio) return [{ nodeId: node.id, type: "audio" as const, title: node.title, audio }];
const text = readNodeTextInput(node);
if (text) return [{ nodeId: node.id, type: "text" as const, title: node.title, text }];
return [];
@@ -97,6 +105,18 @@ function readReferenceVideo(node: CanvasNodeData): ReferenceVideo | null {
};
}
function readReferenceAudio(node: CanvasNodeData): ReferenceAudio | null {
if (node.type !== CanvasNodeType.Audio || !node.metadata?.content) return null;
return {
id: node.id,
name: `${node.title || node.id}.mp3`,
type: node.metadata.mimeType || "audio/mpeg",
url: node.metadata.content,
storageKey: node.metadata.storageKey,
durationMs: node.metadata.durationMs,
};
}
function getOrderedUpstreamNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
const target = nodes.find((node) => node.id === nodeId);
const upstreamNodes = connections
@@ -2,7 +2,7 @@
import { useEffect, useMemo, useState, type ReactNode } from "react";
import { Modal, Segmented, Tooltip } from "antd";
import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react";
import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Music2, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react";
import { canvasThemes } from "@/lib/canvas-theme";
import { formatBytes, getDataUrlByteSize } from "@/lib/image-utils";
@@ -58,13 +58,15 @@ export function CanvasNodeHoverToolbar({
const top = viewport.y + node.position.y * viewport.k - 14;
const isImage = node.type === CanvasNodeType.Image;
const isVideo = node.type === CanvasNodeType.Video;
const isAudio = node.type === CanvasNodeType.Audio;
const hasImage = isImage && Boolean(node.metadata?.content);
const hasVideo = isVideo && Boolean(node.metadata?.content);
const hasAudio = isAudio && Boolean(node.metadata?.content);
const isText = node.type === CanvasNodeType.Text;
const isConfig = node.type === CanvasNodeType.Config;
const canOpenDialog = isText || hasImage || isVideo;
const canRetry = node.metadata?.status === "error";
const hasSpecificTools = canRetry || isText || isImage || isVideo || isConfig;
const hasSpecificTools = canRetry || isText || isImage || isVideo || isAudio || isConfig;
return (
<div
@@ -80,7 +82,7 @@ export function CanvasNodeHoverToolbar({
{hasSpecificTools ? <ToolbarDivider /> : null}
{canRetry ? <ToolbarAction title="重新生成" label="重试" icon={<RefreshCw className="size-4" />} onClick={() => onRetry(node)} /> : null}
{hasImage || hasVideo || isText ? <ToolbarAction title="加入我的素材" label="存素材" icon={<FolderPlus className="size-4" />} onClick={() => onSaveAsset(node)} /> : null}
{hasImage || hasVideo ? <IconAction title={hasVideo ? "下载视频" : "下载图片"} icon={<Download className="size-5" />} onClick={() => onDownload(node)} /> : null}
{hasImage || hasVideo || hasAudio ? <IconAction title={hasAudio ? "下载音频" : hasVideo ? "下载视频" : "下载图片"} icon={<Download className="size-5" />} onClick={() => onDownload(node)} /> : null}
{canOpenDialog ? <ToolbarAction title="编辑" label="编辑" icon={<MessageSquare className="size-4" />} onClick={() => onToggleDialog(node)} /> : null}
{isText ? <ToolbarAction title="编辑文本" label="编辑文字" icon={<Pencil className="size-4" />} onClick={() => onEditText(node)} /> : null}
{isText ? <ToolbarAction title="用文本生图" label="生图" icon={<ImageIcon className="size-4" />} onClick={() => onGenerateImage(node)} /> : null}
@@ -89,6 +91,7 @@ export function CanvasNodeHoverToolbar({
{isText ? <ToolbarAction title="增大字号" label="放大" icon={<Plus className="size-4" />} onClick={() => onIncreaseFont(node)} /> : null}
{isImage ? <ToolbarAction title={hasImage ? "替换图片" : "上传图片"} label={hasImage ? "替换图片" : "上传图片"} icon={<Upload className="size-4" />} onClick={() => onUpload(node)} /> : null}
{isVideo ? <ToolbarAction title={hasVideo ? "替换视频" : "上传视频"} label={hasVideo ? "替换视频" : "上传视频"} icon={<Video className="size-4" />} onClick={() => onUpload(node)} /> : null}
{isAudio ? <ToolbarAction title={hasAudio ? "替换音频" : "上传音频"} label={hasAudio ? "替换音频" : "上传音频"} icon={<Music2 className="size-4" />} onClick={() => onUpload(node)} /> : null}
{hasImage ? (
<ToolbarAction
title={node.metadata?.freeResize ? "切换为等比缩放" : "切换为自由比例"}
@@ -151,7 +154,7 @@ export function CanvasNodeInfoModal({ node, open, onClose }: { node: CanvasNodeD
{view === "info" ? (
<div className="thin-scrollbar h-full space-y-3 overflow-auto pr-1">
<InfoRow label="ID" value={node.id} />
<InfoRow label="类型" value={node.type === CanvasNodeType.Text ? "文本" : node.type === CanvasNodeType.Image ? "图片" : node.type === CanvasNodeType.Video ? "视频" : "生成配置"} />
<InfoRow label="类型" value={node.type === CanvasNodeType.Text ? "文本" : node.type === CanvasNodeType.Image ? "图片" : node.type === CanvasNodeType.Video ? "视频" : node.type === CanvasNodeType.Audio ? "音频" : "生成配置"} />
<InfoRow label="尺寸" value={`${Math.round(node.width)} x ${Math.round(node.height)}`} />
<InfoRow label="位置" value={`${Math.round(node.position.x)}, ${Math.round(node.position.y)}`} />
<InfoRow label="状态" value={node.metadata?.status || "idle"} />
@@ -93,7 +93,7 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi
) : mode === "video" ? (
<>
<ModelPicker config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} />
<CanvasVideoSettingsPopover config={config} buttonClassName="!h-10 !max-w-[170px] !justify-start !rounded-full !px-3" onConfigChange={(key, value) => onConfigChange(node.id, key === "videoSeconds" ? { seconds: value } : { [key]: value })} />
<CanvasVideoSettingsPopover config={config} buttonClassName="!h-10 !max-w-[170px] !justify-start !rounded-full !px-3" onConfigChange={(key, value) => onConfigChange(node.id, videoConfigPatch(key, value))} />
</>
) : (
<ModelPicker config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} />
@@ -132,6 +132,15 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can
size: node.metadata?.size || globalConfig.size || defaultConfig.size,
videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds,
vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality,
videoGenerateAudio: node.metadata?.generateAudio || globalConfig.videoGenerateAudio || defaultConfig.videoGenerateAudio,
videoWatermark: node.metadata?.watermark || globalConfig.videoWatermark || defaultConfig.videoWatermark,
count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count),
};
}
function videoConfigPatch(key: keyof AiConfig, value: string) {
if (key === "videoSeconds") return { seconds: value };
if (key === "videoGenerateAudio") return { generateAudio: value };
if (key === "videoWatermark") return { watermark: value };
return { [key]: value };
}
@@ -2,7 +2,7 @@
import React, { useCallback, useEffect, useRef, useState } from "react";
import type { ReactNode } from "react";
import { ChevronRight, Image as ImageIcon, RefreshCw, Star, Video } from "lucide-react";
import { ChevronRight, Image as ImageIcon, Music2, RefreshCw, Star, Video } from "lucide-react";
import { canvasThemes } from "@/lib/canvas-theme";
import { formatBytes } from "@/lib/image-utils";
@@ -99,6 +99,7 @@ export const CanvasNode = React.memo(function CanvasNode({
const [isEditingContent, setIsEditingContent] = useState(false);
const hasImageContent = data.type === CanvasNodeType.Image && Boolean(data.metadata?.content);
const hasVideoContent = data.type === CanvasNodeType.Video && Boolean(data.metadata?.content);
const hasAudioContent = data.type === CanvasNodeType.Audio && Boolean(data.metadata?.content);
const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1;
const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId);
const isActive = isConnectionTarget || isSelected || isFocusRelated;
@@ -301,7 +302,7 @@ export const CanvasNode = React.memo(function CanvasNode({
{showImageInfo && hasImageContent ? <ImageInfoBar node={data} /> : null}
{!hasImageContent && !hasVideoContent ? <div className="pointer-events-none absolute inset-x-0 bottom-0 h-12" style={{ background: `linear-gradient(to top, ${theme.canvas.background}66, transparent)` }} /> : null}
{!hasImageContent && !hasVideoContent && !hasAudioContent ? <div className="pointer-events-none absolute inset-x-0 bottom-0 h-12" style={{ background: `linear-gradient(to top, ${theme.canvas.background}66, transparent)` }} /> : null}
<ResizeHandle corner="top-left" onMouseDown={handleResizeMouseDown} />
<ResizeHandle corner="top-right" onMouseDown={handleResizeMouseDown} />
@@ -332,6 +333,7 @@ const nodeContentRenderers = {
[CanvasNodeType.Image]: ImageNodeContent,
[CanvasNodeType.Config]: EmptyImageContent,
[CanvasNodeType.Video]: VideoNodeContent,
[CanvasNodeType.Audio]: AudioNodeContent,
} satisfies Record<CanvasNodeType, (props: NodeContentRendererProps) => ReactNode>;
function LoadingContent({ theme }: Pick<NodeContentRendererProps, "theme">) {
@@ -472,6 +474,25 @@ function VideoNodeContent({ node, theme }: NodeContentRendererProps) {
return <video src={node.metadata.content} controls className="h-full w-full rounded-[18px] bg-black object-contain" data-canvas-no-zoom />;
}
function AudioNodeContent({ node, theme }: NodeContentRendererProps) {
if (!node.metadata?.content)
return (
<div className="flex h-full w-full flex-col items-center justify-center gap-2" style={{ color: theme.node.placeholder }}>
<Music2 className="size-7 opacity-35" />
<span className="text-sm"></span>
</div>
);
return (
<div className="flex h-full w-full flex-col justify-center gap-3 px-4" style={{ background: theme.node.fill, color: theme.node.text }}>
<div className="flex min-w-0 items-center gap-2 text-sm opacity-70">
<Music2 className="size-4 shrink-0" />
<span className="truncate">{node.title || "音频"}</span>
</div>
<audio src={node.metadata.content} controls className="w-full" data-canvas-no-zoom />
</div>
);
}
function ImageContent({
node,
isBatchRoot,
@@ -1,7 +1,7 @@
import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactNode, RefObject } from "react";
import { useRef, useState } from "react";
import { Button, Segmented, Switch } from "antd";
import { CircleDot, Eraser, FolderOpen, Grid2x2, Hand, Image as ImageIcon, Info, Library, Moon, Palette, Redo2, Settings2, Square, Sun, Trash2, Type, Undo2, Upload, Video } from "lucide-react";
import { CircleDot, Eraser, FolderOpen, Grid2x2, Hand, Image as ImageIcon, Info, Library, Moon, Music2, Palette, Redo2, Settings2, Square, Sun, Trash2, Type, Undo2, Upload, Video } from "lucide-react";
import { canvasThemes, type CanvasBackgroundMode, type CanvasColorTheme, type CanvasTheme } from "@/lib/canvas-theme";
import { useThemeStore } from "@/stores/use-theme-store";
@@ -15,6 +15,7 @@ export function CanvasToolbar({
showImageInfo,
onAddImage,
onAddVideo,
onAddAudio,
onAddText,
onAddConfig,
onUndo,
@@ -35,6 +36,7 @@ export function CanvasToolbar({
showImageInfo: boolean;
onAddImage: () => void;
onAddVideo: () => void;
onAddAudio: () => void;
onAddText: () => void;
onAddConfig: () => void;
onUndo: () => void;
@@ -84,10 +86,13 @@ export function CanvasToolbar({
<ToolbarButton id="tool-video" label="视频" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddVideo}>
<Video className="size-4.5" />
</ToolbarButton>
<ToolbarButton id="tool-audio" label="音频" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddAudio}>
<Music2 className="size-4.5" />
</ToolbarButton>
<ToolbarButton id="tool-config" label="生成配置" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddConfig}>
<Settings2 className="size-4.5" />
</ToolbarButton>
<ToolbarButton id="tool-upload" label="上传图片" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onUpload}>
<ToolbarButton id="tool-upload" label="上传素材" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onUpload}>
<Upload className="size-4.5" />
</ToolbarButton>
<Divider theme={theme} />
@@ -279,8 +284,9 @@ function toolLabel(id: string) {
if (id === "tool-text") return "文本";
if (id === "tool-image") return "图片";
if (id === "tool-video") return "视频";
if (id === "tool-audio") return "音频";
if (id === "tool-config") return "生成配置";
if (id === "tool-upload") return "上传图片";
if (id === "tool-upload") return "上传素材";
if (id === "tool-library") return "素材库";
if (id === "tool-assets") return "我的素材";
if (id === "tool-style") return "画布外观";
+5
View File
@@ -13,6 +13,7 @@ export const NODE_DEFAULT_SIZE = {
[CanvasNodeType.Text]: { width: 340, height: 240, title: "Note" },
[CanvasNodeType.Config]: { width: 340, height: 240, title: "生成配置" },
[CanvasNodeType.Video]: { width: 420, height: 236, title: "Video" },
[CanvasNodeType.Audio]: { width: 340, height: 120, title: "Audio" },
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
export const NODE_SPECS = {
@@ -32,6 +33,10 @@ export const NODE_SPECS = {
...NODE_DEFAULT_SIZE[CanvasNodeType.Video],
metadata: { content: "", status: "idle" },
},
[CanvasNodeType.Audio]: {
...NODE_DEFAULT_SIZE[CanvasNodeType.Audio],
metadata: { content: "", status: "idle" },
},
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
export function getNodeSpec(type: CanvasNodeType) {
+4
View File
@@ -14,6 +14,7 @@ export enum CanvasNodeType {
Text = "text",
Config = "config",
Video = "video",
Audio = "audio",
}
export type CanvasNodeStatus = "idle" | "success" | "loading" | "error";
@@ -34,6 +35,8 @@ export type CanvasNodeMetadata = {
count?: number;
seconds?: string;
vquality?: string;
generateAudio?: string;
watermark?: string;
references?: string[];
naturalWidth?: number;
naturalHeight?: number;
@@ -48,6 +51,7 @@ export type CanvasNodeMetadata = {
storageKey?: string;
mimeType?: string;
bytes?: number;
durationMs?: number;
};
export type CanvasNodeData = {
+109 -21
View File
@@ -1,6 +1,6 @@
"use client";
import { BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, LoaderCircle, Plus, SlidersHorizontal, Sparkles, Trash2, Upload, VideoIcon } from "lucide-react";
import { BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, LoaderCircle, Music2, Plus, SlidersHorizontal, Sparkles, Trash2, Upload, VideoIcon } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { App, Button, Checkbox, Drawer, Empty, Input, Modal, Tag, Typography } from "antd";
import localforage from "localforage";
@@ -13,6 +13,7 @@ import { PromptSelectDialog } from "@/components/prompts/prompt-select-dialog";
import { VideoSettingsPanel, normalizeVideoResolutionValue, normalizeVideoSizeValue, videoSizeLabel } from "@/components/video-settings-panel";
import { canvasThemes } from "@/lib/canvas-theme";
import { formatBytes, formatDuration } from "@/lib/image-utils";
import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceRatio, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { deleteStoredMedia, resolveMediaUrl, uploadMediaFile } from "@/services/file-storage";
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video";
@@ -20,7 +21,7 @@ import { useAssetStore } from "@/stores/use-asset-store";
import { useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
import { useThemeStore } from "@/stores/use-theme-store";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceVideo } from "@/types/media";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
type GeneratedVideo = {
id: string;
@@ -50,6 +51,7 @@ type GenerationLog = {
config: GenerationLogConfig;
references: ReferenceImage[];
videoReferences: ReferenceVideo[];
audioReferences: ReferenceAudio[];
durationMs: number;
size: string;
resolution: string;
@@ -59,7 +61,7 @@ type GenerationLog = {
error?: string;
};
type GenerationLogConfig = Pick<AiConfig, "model" | "videoModel" | "size" | "vquality" | "videoSeconds">;
type GenerationLogConfig = Pick<AiConfig, "model" | "videoModel" | "size" | "vquality" | "videoSeconds" | "videoGenerateAudio" | "videoWatermark">;
type UpdateAiConfig = <K extends keyof AiConfig>(key: K, value: AiConfig[K]) => void;
@@ -78,6 +80,7 @@ export default function VideoPage() {
const [prompt, setPrompt] = useState("");
const [references, setReferences] = useState<ReferenceImage[]>([]);
const [videoReferences, setVideoReferences] = useState<ReferenceVideo[]>([]);
const [audioReferences, setAudioReferences] = useState<ReferenceAudio[]>([]);
const [results, setResults] = useState<GenerationResult[]>([]);
const [logs, setLogs] = useState<GenerationLog[]>([]);
const [running, setRunning] = useState(false);
@@ -106,8 +109,14 @@ export default function VideoPage() {
const addReferences = async (files?: FileList | null) => {
const selectedFiles = Array.from(files || []);
const imageFiles = selectedFiles.filter((file) => file.type.startsWith("image/")).slice(0, 7 - references.length);
const videoFiles = selectedFiles.filter((file) => file.type.startsWith("video/")).slice(0, 3 - videoReferences.length);
const unsupported = selectedFiles.filter((file) => !file.type.startsWith("image/") && !file.type.startsWith("video/") && !isSupportedAudioFile(file));
if (unsupported.length) message.warning("已忽略不支持的参考素材,请使用图片、mp4/mov 视频或 mp3/wav 音频");
const imageFiles = selectedFiles.filter((file) => file.type.startsWith("image/") && file.size <= SEEDANCE_REFERENCE_LIMITS.imageMaxBytes).slice(0, SEEDANCE_REFERENCE_LIMITS.images - references.length);
const videoFiles = selectedFiles.filter((file) => file.type.startsWith("video/") && file.size <= SEEDANCE_REFERENCE_LIMITS.videoMaxBytes).slice(0, SEEDANCE_REFERENCE_LIMITS.videos - videoReferences.length);
const audioFiles = selectedFiles.filter((file) => isSupportedAudioFile(file) && file.size <= SEEDANCE_REFERENCE_LIMITS.audioMaxBytes).slice(0, SEEDANCE_REFERENCE_LIMITS.audios - audioReferences.length);
if (selectedFiles.some((file) => file.type.startsWith("image/") && file.size > SEEDANCE_REFERENCE_LIMITS.imageMaxBytes)) message.warning("已忽略超过 30MB 的参考图");
if (selectedFiles.some((file) => file.type.startsWith("video/") && file.size > SEEDANCE_REFERENCE_LIMITS.videoMaxBytes)) message.warning("已忽略超过 50MB 的参考视频");
if (selectedFiles.some((file) => isSupportedAudioFile(file) && file.size > SEEDANCE_REFERENCE_LIMITS.audioMaxBytes)) message.warning("已忽略超过 15MB 的参考音频");
const nextReferences = await Promise.all(
imageFiles.map(async (file) => {
const image = await uploadImage(file);
@@ -117,11 +126,22 @@ export default function VideoPage() {
const nextVideoReferences = await Promise.all(
videoFiles.map(async (file) => {
const video = await uploadMediaFile(file, "video-reference");
return { id: nanoid(), name: file.name, type: video.mimeType, url: video.url, storageKey: video.storageKey };
return { id: nanoid(), name: file.name, type: video.mimeType, url: video.url, storageKey: video.storageKey, durationMs: video.durationMs };
}),
);
setReferences((value) => [...value, ...nextReferences].slice(0, 7));
setVideoReferences((value) => [...value, ...nextVideoReferences].slice(0, 3));
const nextAudioReferences = filterAudioReferencesByDuration(
audioReferences,
await Promise.all(
audioFiles.map(async (file) => {
const audio = await uploadMediaFile(file, "audio-reference");
return { id: nanoid(), name: file.name, type: audio.mimeType, url: audio.url, storageKey: audio.storageKey, durationMs: audio.durationMs };
}),
),
message.warning,
);
setReferences((value) => [...value, ...nextReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
setVideoReferences((value) => [...value, ...nextVideoReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.videos));
setAudioReferences((value) => [...value, ...nextAudioReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.audios));
};
const addReferencesFromClipboard = async () => {
@@ -133,18 +153,17 @@ export default function VideoPage() {
return;
}
const nextReferences = await Promise.all(
blobs.slice(0, 7 - references.length).map(async (blob, index) => {
blobs.slice(0, SEEDANCE_REFERENCE_LIMITS.images - references.length).map(async (blob, index) => {
const image = await uploadImage(blob);
return { id: nanoid(), name: `clipboard-${index + 1}.png`, type: image.mimeType, dataUrl: image.url, storageKey: image.storageKey };
}),
);
setReferences((value) => [...value, ...nextReferences].slice(0, 7));
setReferences((value) => [...value, ...nextReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
message.success(`已读取 ${nextReferences.length} 张参考图`);
} catch {
message.error("剪切板里没有可读取的图片");
}
};
const generate = async () => {
const snapshot = buildRequestSnapshot();
if (!snapshot) return;
@@ -155,7 +174,7 @@ export default function VideoPage() {
const batchStartedAt = performance.now();
setStartedAt(batchStartedAt);
try {
const stored = await storeGeneratedVideo(await requestVideoGeneration(snapshot.config, snapshot.text, snapshot.references, snapshot.videoReferences));
const stored = await storeGeneratedVideo(await requestVideoGeneration(snapshot.config, snapshot.text, snapshot.references, snapshot.videoReferences, snapshot.audioReferences));
const nextVideo: GeneratedVideo = {
id: nanoid(),
url: stored.url,
@@ -167,12 +186,12 @@ export default function VideoPage() {
mimeType: stored.mimeType,
};
setResults([{ id: nextVideo.id, status: "success", video: nextVideo }]);
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, durationMs: nextVideo.durationMs, status: "成功", video: nextVideo }));
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, audioReferences: snapshot.audioReferences, durationMs: nextVideo.durationMs, status: "成功", video: nextVideo }));
message.success("视频已生成");
} catch (error) {
const errorMessage = error instanceof Error ? error.message : "生成失败";
setResults([{ id: nanoid(), status: "failed", error: errorMessage }]);
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, durationMs: performance.now() - batchStartedAt, status: "失败", error: errorMessage }));
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, audioReferences: snapshot.audioReferences, durationMs: performance.now() - batchStartedAt, status: "失败", error: errorMessage }));
message.error(errorMessage);
} finally {
setRunning(false);
@@ -190,7 +209,7 @@ export default function VideoPage() {
openConfigDialog(true);
return null;
}
return { text, config: buildVideoConfig(effectiveConfig, model), references: [...references], videoReferences: [...videoReferences] };
return { text, config: buildVideoConfig(effectiveConfig, model), references: [...references], videoReferences: [...videoReferences], audioReferences: [...audioReferences] };
};
const retryResult = () => {
@@ -219,9 +238,9 @@ export default function VideoPage() {
setPrompt(payload.content);
} else if (payload.kind === "image") {
const stored = await uploadImage(payload.dataUrl);
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }].slice(0, 7));
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
} else if (payload.kind === "video") {
setVideoReferences((value) => [...value, { id: nanoid(), name: payload.title, type: "video/mp4", url: payload.url, storageKey: payload.storageKey }].slice(0, 3));
setVideoReferences((value) => [...value, { id: nanoid(), name: payload.title, type: "video/mp4", url: payload.url, storageKey: payload.storageKey }].slice(0, SEEDANCE_REFERENCE_LIMITS.videos));
}
setAssetPickerOpen(false);
};
@@ -230,6 +249,7 @@ export default function VideoPage() {
setPrompt("");
setReferences([]);
setVideoReferences([]);
setAudioReferences([]);
setResults([]);
setElapsedMs(0);
setStartedAt(0);
@@ -263,10 +283,13 @@ export default function VideoPage() {
setPrompt(log.prompt);
setReferences(log.references || []);
setVideoReferences(log.videoReferences || []);
setAudioReferences(log.audioReferences || []);
if (log.config.videoModel || log.model) updateConfig("videoModel", log.config.videoModel || log.model);
if (log.config.size) updateConfig("size", log.config.size);
if (log.config.vquality) updateConfig("vquality", log.config.vquality);
if (log.config.videoSeconds) updateConfig("videoSeconds", log.config.videoSeconds);
if (log.config.videoGenerateAudio) updateConfig("videoGenerateAudio", log.config.videoGenerateAudio);
if (log.config.videoWatermark) updateConfig("videoWatermark", log.config.videoWatermark);
setResults(log.video ? [{ id: log.video.id, status: "success", video: log.video }] : [{ id: log.id, status: "failed", error: log.error || "生成失败" }]);
};
@@ -328,7 +351,7 @@ export default function VideoPage() {
</button>
</div>
))}
{!references.length ? <div className="flex min-w-full items-center justify-center text-sm text-stone-500"> 7 </div> : null}
{!references.length ? <div className="flex min-w-full items-center justify-center text-sm text-stone-500"> 9 </div> : null}
</div>
</div>
@@ -352,6 +375,30 @@ export default function VideoPage() {
</div>
</div>
<div className="min-w-0">
<div className="mb-2 flex items-center justify-between gap-3">
<span className="text-base font-semibold"></span>
<Button size="small" icon={<Upload className="size-3.5" />} onClick={() => fileInputRef.current?.click()}>
</Button>
</div>
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
{audioReferences.map((item) => (
<div key={item.id} className="group relative flex h-20 w-48 shrink-0 flex-col justify-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2 dark:border-stone-800 dark:bg-stone-900">
<div className="flex min-w-0 items-center gap-2 text-xs text-stone-500 dark:text-stone-400">
<Music2 className="size-4 shrink-0" />
<span className="truncate">{item.name}</span>
</div>
<audio src={item.url} controls className="h-8 w-full" preload="metadata" />
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setAudioReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考音频">
<Trash2 className="size-3.5" />
</button>
</div>
))}
{!audioReferences.length ? <div className="flex min-w-full items-center justify-center text-center text-sm text-stone-500"> 3 mp3/wav 15MB </div> : null}
</div>
</div>
<div className="flex items-center justify-between rounded-lg border border-stone-200 bg-stone-50 px-3 py-2 text-sm dark:border-stone-800 dark:bg-stone-900 sm:hidden">
<span className="truncate text-stone-500 dark:text-stone-400">
{model} · {normalizeResolution(effectiveConfig.vquality)}p · {videoSizeLabel(effectiveConfig.size)} · {normalizeVideoSeconds(effectiveConfig.videoSeconds)}s
@@ -394,7 +441,7 @@ export default function VideoPage() {
<input
ref={fileInputRef}
type="file"
accept="image/*,video/mp4,video/quicktime"
accept="image/*,video/mp4,video/quicktime,audio/mpeg,audio/wav,audio/x-wav,.mp3,.wav"
multiple
className="hidden"
onChange={(event) => {
@@ -583,6 +630,12 @@ async function normalizeLog(log: Partial<GenerationLog>): Promise<GenerationLog>
url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url,
})),
);
const audioReferences = await Promise.all(
(log.audioReferences || []).map(async (item) => ({
...item,
url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url,
})),
);
const references = await Promise.all(
(log.references || []).map(async (item) => ({
...item,
@@ -600,6 +653,7 @@ async function normalizeLog(log: Partial<GenerationLog>): Promise<GenerationLog>
config,
references,
videoReferences,
audioReferences,
durationMs: log.durationMs || 0,
size: log.size || config.size || "",
resolution: normalizeResolution(log.resolution || config.vquality || ""),
@@ -615,10 +669,35 @@ function serializeLog(log: GenerationLog): GenerationLog {
...log,
references: log.references.map((item) => ({ ...item, dataUrl: item.storageKey ? "" : item.dataUrl })),
videoReferences: log.videoReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)),
audioReferences: log.audioReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)),
video: log.video?.storageKey ? { ...log.video, url: "" } : log.video,
};
}
function isSupportedAudioFile(file: File) {
return file.type === "audio/mpeg" || file.type === "audio/mp3" || file.type === "audio/wav" || file.type === "audio/x-wav" || /\.(mp3|wav)$/i.test(file.name);
}
function filterAudioReferencesByDuration(existing: ReferenceAudio[], next: ReferenceAudio[], warn: (content: string) => void) {
let total = existing.reduce((sum, item) => sum + (item.durationMs || 0), 0);
const accepted: ReferenceAudio[] = [];
let skipped = false;
for (const item of next) {
if (item.durationMs && (item.durationMs < 2000 || item.durationMs > 15000)) {
skipped = true;
continue;
}
if (item.durationMs && total + item.durationMs > 15000) {
skipped = true;
continue;
}
total += item.durationMs || 0;
accepted.push(item);
}
if (skipped) warn("已忽略不符合时长要求的参考音频:单个 2-15 秒,总时长不超过 15 秒");
return accepted;
}
function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
return {
model: log.config?.model || log.model || "",
@@ -626,16 +705,20 @@ function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
size: log.config?.size || log.size || "",
vquality: normalizeResolution(log.config?.vquality || log.resolution || ""),
videoSeconds: log.config?.videoSeconds || log.seconds || "",
videoGenerateAudio: log.config?.videoGenerateAudio || "true",
videoWatermark: log.config?.videoWatermark || "false",
};
}
function buildLog({ prompt, model, config, references, videoReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog {
function buildLog({ prompt, model, config, references, videoReferences, audioReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; audioReferences: ReferenceAudio[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog {
const logConfig = {
model: config.model,
videoModel: config.videoModel,
size: config.size,
vquality: normalizeResolution(config.vquality),
videoSeconds: config.videoSeconds,
videoGenerateAudio: config.videoGenerateAudio,
videoWatermark: config.videoWatermark,
};
return {
id: nanoid(),
@@ -647,6 +730,7 @@ function buildLog({ prompt, model, config, references, videoReferences, duration
config: logConfig,
references,
videoReferences,
audioReferences,
durationMs,
size: logConfig.size,
resolution: logConfig.vquality,
@@ -658,17 +742,21 @@ function buildLog({ prompt, model, config, references, videoReferences, duration
}
function buildVideoConfig(config: AiConfig, model: string): AiConfig {
const seedance = isSeedanceVideoConfig({ ...config, model });
return {
...config,
model,
videoModel: model,
size: normalizeVideoSize(config.size),
size: seedance ? normalizeSeedanceRatio(config.size) : normalizeVideoSize(config.size),
videoSeconds: normalizeVideoSeconds(config.videoSeconds),
vquality: normalizeResolution(config.vquality),
videoGenerateAudio: String(boolConfig(config.videoGenerateAudio, true)),
videoWatermark: String(boolConfig(config.videoWatermark, false)),
};
}
function normalizeVideoSeconds(value: string) {
if (String(value).trim() === "-1") return "-1";
const seconds = Math.floor(Number(value) || 6);
return String(Math.max(1, Math.min(20, seconds)));
}
+100 -3
View File
@@ -1,8 +1,10 @@
"use client";
import { type ReactNode } from "react";
import { Switch } from "antd";
import { ImageSettingsTheme } from "@/components/image-settings-panel";
import { boolConfig, isSeedanceFastModel, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceDurationOptions, seedancePixelLabel, seedanceRatioOptions, seedanceResolutionOptions } from "@/lib/seedance-video";
import { type CanvasTheme } from "@/lib/canvas-theme";
import type { AiConfig } from "@/stores/use-config-store";
@@ -24,13 +26,17 @@ const secondOptions = [6, 10, 12, 16, 20];
type VideoSettingsPanelProps = {
config: AiConfig;
onConfigChange: (key: "vquality" | "size" | "videoSeconds", value: string) => void;
onConfigChange: (key: "vquality" | "size" | "videoSeconds" | "videoGenerateAudio" | "videoWatermark", value: string) => void;
theme: CanvasTheme;
showTitle?: boolean;
className?: string;
};
export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = true, className = "w-[320px] space-y-4 rounded-2xl px-1 py-0.5" }: VideoSettingsPanelProps) {
if (isSeedanceVideoConfig(config)) {
return <SeedanceVideoSettingsPanel config={config} onConfigChange={onConfigChange} theme={theme} showTitle={showTitle} className={className} />;
}
const seconds = config.videoSeconds || "6";
const size = normalizeVideoSizeValue(config.size);
const dimensions = readSizeDimensions(size);
@@ -96,16 +102,84 @@ export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle =
);
}
function SeedanceVideoSettingsPanel({ config, onConfigChange, theme, showTitle, className }: VideoSettingsPanelProps) {
const model = config.model || config.videoModel;
const resolution = normalizeSeedanceResolution(config.vquality, model);
const ratio = normalizeSeedanceRatio(config.size);
const duration = normalizeSeedanceDuration(config.videoSeconds);
const generateAudio = boolConfig(config.videoGenerateAudio, true);
const watermark = boolConfig(config.videoWatermark, false);
return (
<ImageSettingsTheme theme={theme}>
<div className={className} style={{ color: theme.node.text }} onMouseDown={(event) => event.stopPropagation()}>
{showTitle ? <div className="text-lg font-semibold"></div> : null}
<SettingGroup title="分辨率" color={theme.node.muted}>
<div className="grid grid-cols-3 gap-2.5">
{seedanceResolutionOptions.map((item) => {
const disabled = item.value === "1080p" && isSeedanceFastModel(model);
return (
<OptionPill key={item.value} selected={resolution === item.value} disabled={disabled} theme={theme} onClick={() => onConfigChange("vquality", item.value)}>
{item.label}
</OptionPill>
);
})}
</div>
{isSeedanceFastModel(model) ? <div className="text-[11px] leading-4 opacity-55">fast 1080p使 720p</div> : null}
</SettingGroup>
<SettingGroup title="比例" color={theme.node.muted}>
<div className="grid grid-cols-3 gap-2.5">
{seedanceRatioOptions.map((item) => (
<button
key={item.value}
type="button"
className="flex h-[68px] cursor-pointer flex-col items-center justify-center gap-1 rounded-xl border bg-transparent px-1 text-sm transition hover:opacity-80"
style={{ borderColor: ratio === item.value ? theme.node.text : theme.node.stroke, color: theme.node.text }}
onMouseDown={(event) => event.stopPropagation()}
onClick={() => onConfigChange("size", item.value)}
>
<SizePreview width={ratioPreview(item.value).width} height={ratioPreview(item.value).height} color={theme.node.text} />
<span>{item.label}</span>
<span className="text-[10px] leading-none opacity-55">{item.value === "adaptive" ? "adaptive" : seedancePixelLabel(resolution, item.value)}</span>
</button>
))}
</div>
</SettingGroup>
<SettingGroup title="时长" color={theme.node.muted}>
<div className="grid grid-cols-4 gap-2.5">
{seedanceDurationOptions.map((value) => (
<OptionPill key={value} selected={duration === value} theme={theme} onClick={() => onConfigChange("videoSeconds", String(value))}>
{value === -1 ? "智能" : `${value}s`}
</OptionPill>
))}
</div>
<NumberInput value={String(duration)} min={-1} max={15} theme={theme} onChange={(value) => onConfigChange("videoSeconds", value)} />
</SettingGroup>
<SettingGroup title="输出" color={theme.node.muted}>
<div className="grid gap-2 rounded-xl border p-2.5" style={{ borderColor: theme.node.stroke }}>
<SwitchRow label="生成声音" checked={generateAudio} theme={theme} onChange={(checked) => onConfigChange("videoGenerateAudio", String(checked))} />
<SwitchRow label="添加水印" checked={watermark} theme={theme} onChange={(checked) => onConfigChange("videoWatermark", String(checked))} />
</div>
</SettingGroup>
</div>
</ImageSettingsTheme>
);
}
export function videoResolutionLabel(value: string) {
return `${normalizeVideoResolutionValue(value)}p`;
}
export function videoSizeLabel(value: string) {
const ratio = normalizeSeedanceRatio(value);
if (value === "adaptive" || value === "auto") return "自适应";
if (ratio === value) return seedanceRatioOptions.find((item) => item.value === ratio)?.label || ratio;
const size = normalizeVideoSizeValue(value);
return sizeOptions.find((item) => item.value === size)?.label || size;
}
export function videoSecondsLabel(value: string) {
if (String(value).trim() === "-1") return "智能";
return `${value || "6"}s`;
}
@@ -121,9 +195,9 @@ export function normalizeVideoResolutionValue(value: string) {
return value.replace(/p$/i, "") || "720";
}
function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) {
function OptionPill({ selected, disabled = false, theme, onClick, children }: { selected: boolean; disabled?: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) {
return (
<button type="button" className="h-9 cursor-pointer rounded-full border px-2 text-sm transition hover:opacity-80" style={{ background: "transparent", borderColor: selected ? theme.node.text : theme.node.stroke, color: theme.node.text }} onMouseDown={(event) => event.stopPropagation()} onClick={onClick}>
<button type="button" disabled={disabled} className="h-9 cursor-pointer rounded-full border px-2 text-sm transition hover:opacity-80 disabled:cursor-not-allowed disabled:opacity-35" style={{ background: "transparent", borderColor: selected ? theme.node.text : theme.node.stroke, color: theme.node.text }} onMouseDown={(event) => event.stopPropagation()} onClick={onClick}>
{children}
</button>
);
@@ -174,6 +248,29 @@ function SizePreview({ width, height, color }: { width: number; height: number;
return <span className="rounded-[3px] border-2" style={{ width: previewWidth, height: previewHeight, borderColor: color }} />;
}
function ratioPreview(ratio: string) {
if (ratio === "9:16") return { width: 9, height: 16 };
if (ratio === "1:1") return { width: 1, height: 1 };
if (ratio === "4:3") return { width: 4, height: 3 };
if (ratio === "3:4") return { width: 3, height: 4 };
if (ratio === "21:9") return { width: 21, height: 9 };
if (ratio === "adaptive") return { width: 0, height: 0 };
return { width: 16, height: 9 };
}
function SwitchRow({ label, checked, theme, onChange }: { label: string; checked: boolean; theme: CanvasTheme; onChange: (checked: boolean) => void }) {
return (
<div className="flex h-8 items-center justify-between gap-3">
<span className="text-sm" style={{ color: theme.node.text }}>
{label}
</span>
<span onMouseDown={(event) => event.stopPropagation()}>
<Switch size="small" checked={checked} onChange={onChange} />
</span>
</div>
);
}
function readSizeDimensions(size: string) {
if (size === "auto") return { width: 0, height: 0 };
const match = size.match(/^(\d+)x(\d+)$/);
+125
View File
@@ -0,0 +1,125 @@
import type { AiConfig } from "@/stores/use-config-store";
export const SEEDANCE_REFERENCE_LIMITS = {
images: 9,
videos: 3,
audios: 3,
imageMaxBytes: 30 * 1024 * 1024,
videoMaxBytes: 50 * 1024 * 1024,
audioMaxBytes: 15 * 1024 * 1024,
};
export const seedanceResolutionOptions = [
{ value: "480p", label: "480p" },
{ value: "720p", label: "720p" },
{ value: "1080p", label: "1080p" },
] as const;
export const seedanceRatioOptions = [
{ value: "16:9", label: "横屏" },
{ value: "9:16", label: "竖屏" },
{ value: "1:1", label: "方形" },
{ value: "4:3", label: "标准横屏" },
{ value: "3:4", label: "标准竖屏" },
{ value: "21:9", label: "宽银幕" },
{ value: "adaptive", label: "自适应" },
] as const;
export const seedanceDurationOptions = [-1, 4, 5, 6, 8, 10, 12, 15] as const;
const seedancePixels = {
"480p": {
"16:9": "864x496",
"4:3": "752x560",
"1:1": "640x640",
"3:4": "560x752",
"9:16": "496x864",
"21:9": "992x432",
},
"720p": {
"16:9": "1280x720",
"4:3": "1112x834",
"1:1": "960x960",
"3:4": "834x1112",
"9:16": "720x1280",
"21:9": "1470x630",
},
"1080p": {
"16:9": "1920x1080",
"4:3": "1664x1248",
"1:1": "1440x1440",
"3:4": "1248x1664",
"9:16": "1080x1920",
"21:9": "2206x946",
},
} as const;
export function isSeedanceVideoConfig(config: Pick<AiConfig, "model" | "videoModel" | "baseUrl">) {
return isSeedanceVideoModel(config.model || config.videoModel) || isArkPlanBaseUrl(config.baseUrl);
}
export function isSeedanceVideoModel(model: string) {
const value = model.toLowerCase();
return value.includes("seedance") || value.includes("doubao-seedance");
}
export function isSeedanceFastModel(model: string) {
const value = model.toLowerCase();
return isSeedanceVideoModel(value) && value.includes("fast");
}
export function isArkPlanBaseUrl(baseUrl: string) {
return baseUrl.toLowerCase().includes("ark.cn-beijing.volces.com/api/plan/v3") || baseUrl.toLowerCase().includes("/api/plan/v3");
}
export function normalizeSeedanceResolution(value: string, model = "") {
const normalized = normalizeResolutionToken(value);
if (isSeedanceFastModel(model) && normalized === "1080p") return "720p";
return seedanceResolutionOptions.some((item) => item.value === normalized) ? normalized : "720p";
}
export function normalizeResolutionToken(value: string) {
if (value === "low") return "480p";
if (value === "auto" || value === "high" || value === "medium") return "720p";
const resolution = String(value || "").replace(/p$/i, "") || "720";
return `${resolution}p`;
}
export function normalizeSeedanceDuration(value: string) {
if (String(value).trim() === "-1") return -1;
const seconds = Math.floor(Number(value) || 5);
return Math.max(4, Math.min(15, seconds));
}
export function normalizeSeedanceRatio(value: string) {
if (!value || value === "auto" || value === "adaptive") return "adaptive";
if (seedanceRatioOptions.some((item) => item.value === value)) return value;
const match = value.match(/^(\d+)x(\d+)$/);
if (!match) return "adaptive";
const width = Number(match[1]);
const height = Number(match[2]);
if (!width || !height) return "adaptive";
const ratio = width / height;
const options = [
["16:9", 16 / 9],
["4:3", 4 / 3],
["1:1", 1],
["3:4", 3 / 4],
["9:16", 9 / 16],
["21:9", 21 / 9],
] as const;
return options.reduce((best, item) => (Math.abs(item[1] - ratio) < Math.abs(best[1] - ratio) ? item : best), options[0])[0];
}
export function seedancePixelLabel(resolution: string, ratio: string) {
const normalizedResolution = normalizeSeedanceResolution(resolution) as keyof typeof seedancePixels;
const normalizedRatio = normalizeSeedanceRatio(ratio) as keyof (typeof seedancePixels)[typeof normalizedResolution] | "adaptive";
if (normalizedRatio === "adaptive") return "自动匹配";
return seedancePixels[normalizedResolution][normalizedRatio] || "";
}
export function boolConfig(value: string | undefined, fallback: boolean) {
if (value === "true") return true;
if (value === "false") return false;
return fallback;
}
+54 -44
View File
@@ -3,10 +3,11 @@ import axios from "axios";
import { dataUrlToFile } from "@/lib/image-utils";
import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
import { imageToDataUrl } from "@/services/image-storage";
import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { buildApiUrl, type AiConfig } from "@/stores/use-config-store";
import { useUserStore } from "@/stores/use-user-store";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceVideo } from "@/types/media";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
type VideoResponse = { id: string; status?: string; error?: { message?: string } };
type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string };
@@ -42,14 +43,14 @@ function refreshRemoteUser(config: AiConfig) {
if (config.channelMode === "remote") void useUserStore.getState().hydrateUser();
}
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = []): Promise<VideoGenerationResult> {
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = []): Promise<VideoGenerationResult> {
const model = (config.model || config.videoModel).trim();
assertVideoConfig(config, model);
if (isSeedanceConfig(config, model)) {
return requestSeedanceGeneration(config, model, prompt, references, videoReferences);
if (isSeedanceVideoConfig({ ...config, model })) {
return requestSeedanceGeneration(config, model, prompt, references, videoReferences, audioReferences);
}
if (videoReferences.length) {
throw new Error("当前视频接口不支持参考视频,请切换到 Seedance 2.0 / 火山 Agent Plan 模型,或移除参考视频");
if (videoReferences.length || audioReferences.length) {
throw new Error("当前视频接口不支持参考视频或参考音频,请切换到 Seedance 2.0 / 火山 Agent Plan 模型,或移除参考素材");
}
return requestOpenAIVideoGeneration(config, model, prompt, references);
}
@@ -89,16 +90,22 @@ async function requestOpenAIVideoGeneration(config: AiConfig, model: string, pro
}
}
async function requestSeedanceGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[]) {
const content = await buildSeedanceContent(config, prompt, references, videoReferences);
if (!content.length) throw new Error("请输入视频提示词,或连接参考图片/视频");
async function requestSeedanceGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
if (audioReferences.length && !references.length && !videoReferences.length) {
throw new Error("Seedance 参考音频不能单独使用,请同时添加参考图或参考视频");
}
assertSeedanceVideoReferences(videoReferences);
assertSeedanceAudioReferences(audioReferences);
const content = await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences);
if (!content.length) throw new Error("请输入视频提示词,或连接参考图片/视频/音频");
const payload = {
model,
content,
ratio: normalizeSeedanceRatio(config.size),
resolution: normalizeVideoResolution(config.vquality),
resolution: normalizeSeedanceResolution(config.vquality, model),
duration: normalizeSeedanceDuration(config.videoSeconds),
watermark: false,
generate_audio: boolConfig(config.videoGenerateAudio, true),
watermark: boolConfig(config.videoWatermark, false),
};
try {
@@ -122,21 +129,44 @@ async function requestSeedanceGeneration(config: AiConfig, model: string, prompt
}
}
function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) {
let total = 0;
for (const video of videoReferences) {
if (!video.durationMs) continue;
if (video.durationMs < 2000 || video.durationMs > 15000) throw new Error("Seedance 参考视频单个时长需要在 2-15 秒之间");
total += video.durationMs;
}
if (total > 15000) throw new Error("Seedance 参考视频总时长不能超过 15 秒");
}
function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) {
let total = 0;
for (const audio of audioReferences) {
if (!audio.durationMs) continue;
if (audio.durationMs < 2000 || audio.durationMs > 15000) throw new Error("Seedance 参考音频单个时长需要在 2-15 秒之间");
total += audio.durationMs;
}
if (total > 15000) throw new Error("Seedance 参考音频总时长不能超过 15 秒");
}
function seedanceApiUrl(config: AiConfig, taskId?: string) {
if (config.channelMode === "remote") return taskId ? `/api/v1/videos/${encodeURIComponent(taskId)}` : "/api/v1/videos";
return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
}
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[]) {
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
const content: Array<Record<string, unknown>> = [];
const text = prompt.trim();
if (text) content.push({ type: "text", text });
for (const image of references.slice(0, 7)) {
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" });
}
for (const video of videoReferences.slice(0, 3)) {
for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) {
content.push({ type: "video_url", video_url: { url: await resolveSeedanceVideoUrl(video) }, role: "reference_video" });
}
for (const audio of audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios)) {
content.push({ type: "audio_url", audio_url: { url: await resolveSeedanceAudioUrl(audio) }, role: "reference_audio" });
}
return content;
}
@@ -161,6 +191,16 @@ async function resolveSeedanceVideoUrl(video: ReferenceVideo) {
return uploadReferenceMedia(file);
}
async function resolveSeedanceAudioUrl(audio: ReferenceAudio) {
if (isPublicMediaUrl(audio.url) || audio.url.startsWith("asset://")) return audio.url;
let blob: Blob | null = null;
if (audio.storageKey) blob = await getMediaBlob(audio.storageKey);
if (!blob && audio.url?.startsWith("blob:")) blob = await (await fetch(audio.url)).blob();
if (!blob) throw new Error("参考音频必须是公网 URL、素材 ID,或本地已保存的音频");
const file = new File([blob], audio.name || "reference-audio.mp3", { type: audio.type || blob.type || "audio/mpeg" });
return uploadReferenceMedia(file);
}
async function uploadReferenceMedia(file: File) {
const token = useUserStore.getState().token;
if (!token) throw new Error("使用本地参考素材需要先登录,并在服务端配置 PUBLIC_BASE_URL");
@@ -182,11 +222,6 @@ async function videoResultFromUrl(url: string): Promise<VideoGenerationResult> {
}
}
function isSeedanceConfig(config: AiConfig, model: string) {
const value = `${model} ${config.baseUrl}`.toLowerCase();
return value.includes("seedance") || value.includes("doubao-seedance") || value.includes("ark.cn-beijing.volces.com/api/plan/v3");
}
function assertVideoConfig(config: AiConfig, model: string) {
if (!model) throw new Error("请先配置视频模型");
if (config.channelMode === "local" && !config.baseUrl.trim()) throw new Error("请先配置 Base URL");
@@ -198,11 +233,6 @@ function normalizeVideoSeconds(value: string) {
return String(Math.max(1, Math.min(20, seconds)));
}
function normalizeSeedanceDuration(value: string) {
const seconds = Math.floor(Number(value) || 5);
return Math.max(4, Math.min(15, seconds));
}
function normalizeVideoSize(value: string) {
if (value === "auto") return null;
const size = value || "1280x720";
@@ -210,26 +240,6 @@ function normalizeVideoSize(value: string) {
return ["9:16", "2:3", "3:4"].includes(size) ? "720x1280" : "1280x720";
}
function normalizeSeedanceRatio(value: string) {
if (!value || value === "auto") return "adaptive";
if (["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"].includes(value)) return value;
const match = value.match(/^(\d+)x(\d+)$/);
if (!match) return "adaptive";
const width = Number(match[1]);
const height = Number(match[2]);
if (!width || !height) return "adaptive";
const ratio = width / height;
const options = [
["16:9", 16 / 9],
["4:3", 4 / 3],
["1:1", 1],
["3:4", 3 / 4],
["9:16", 9 / 16],
["21:9", 21 / 9],
] as const;
return options.reduce((best, item) => (Math.abs(item[1] - ratio) < Math.abs(best[1] - ratio) ? item : best), options[0])[0];
}
function normalizeVideoResolution(value: string) {
if (value === "low") return "480p";
if (value === "auto" || value === "high" || value === "medium") return "720p";
+14 -4
View File
@@ -3,7 +3,7 @@
import localforage from "localforage";
import { nanoid } from "nanoid";
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number };
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number; durationMs?: number };
const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" });
const objectUrls = new Map<string, string>();
@@ -14,7 +14,7 @@ export async function uploadMediaFile(input: string | Blob, prefix = "file"): Pr
await store.setItem(storageKey, blob);
const url = URL.createObjectURL(blob);
objectUrls.set(storageKey, url);
const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : {};
const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : blob.type.startsWith("audio/") ? await readAudioMeta(url) : {};
return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta };
}
@@ -68,11 +68,21 @@ export function collectMediaStorageKeys(value: unknown, keys = new Set<string>()
}
function readVideoMeta(url: string) {
return new Promise<{ width: number; height: number }>((resolve) => {
return new Promise<{ width: number; height: number; durationMs?: number }>((resolve) => {
const video = document.createElement("video");
const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720 });
const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720, durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined });
video.onloadedmetadata = done;
video.onerror = done;
video.src = url;
});
}
function readAudioMeta(url: string) {
return new Promise<{ durationMs?: number }>((resolve) => {
const audio = document.createElement("audio");
const done = () => resolve({ durationMs: Number.isFinite(audio.duration) ? Math.round(audio.duration * 1000) : undefined });
audio.onloadedmetadata = done;
audio.onerror = done;
audio.src = url;
});
}
+34 -5
View File
@@ -17,6 +17,8 @@ export type AiConfig = {
textModel: string;
videoSeconds: string;
vquality: string;
videoGenerateAudio: string;
videoWatermark: string;
systemPrompt: string;
models: string[];
quality: string;
@@ -36,6 +38,8 @@ export const defaultConfig: AiConfig = {
textModel: "gpt-5.5",
videoSeconds: "6",
vquality: "720",
videoGenerateAudio: "true",
videoWatermark: "false",
systemPrompt: "",
models: [],
quality: "auto",
@@ -61,19 +65,44 @@ function resolveEffectiveConfig(config: AiConfig, modelChannel: AdminPublicSetti
const channelMode = modelChannel?.allowCustomChannel ? config.channelMode : "remote";
if (channelMode === "local" || !modelChannel) return { ...config, channelMode };
const models = modelChannel.availableModels;
const fallbackModel = modelChannel.defaultModel || models[0] || "";
const fallbackTextModel = validDefault(modelChannel.defaultTextModel, models) || preferredModel(models, isTextModelName);
const fallbackModel = validDefault(modelChannel.defaultModel, models) || fallbackTextModel || models[0] || "";
const fallbackImageModel = validDefault(modelChannel.defaultImageModel, models) || preferredModel(models, isImageModelName) || fallbackModel;
const fallbackVideoModel = validDefault(modelChannel.defaultVideoModel, models) || preferredModel(models, isVideoModelName) || fallbackModel;
return {
...config,
channelMode,
models,
model: models.includes(config.model) ? config.model : fallbackModel,
imageModel: models.includes(config.imageModel) ? config.imageModel : modelChannel.defaultImageModel || fallbackModel,
videoModel: models.includes(config.videoModel) ? config.videoModel : modelChannel.defaultVideoModel || fallbackModel,
textModel: models.includes(config.textModel) ? config.textModel : modelChannel.defaultTextModel || fallbackModel,
imageModel: models.includes(config.imageModel) ? config.imageModel : fallbackImageModel,
videoModel: models.includes(config.videoModel) ? config.videoModel : fallbackVideoModel,
textModel: models.includes(config.textModel) ? config.textModel : fallbackTextModel || fallbackModel,
systemPrompt: modelChannel.systemPrompt,
};
}
function validDefault(model: string, models: string[]) {
return models.includes(model) ? model : "";
}
function preferredModel(models: string[], predicate: (model: string) => boolean) {
return models.find(predicate) || "";
}
function isVideoModelName(model: string) {
const value = model.toLowerCase();
return value.includes("seedance") || value.includes("video");
}
function isImageModelName(model: string) {
const value = model.toLowerCase();
return value.includes("seedream") || value.includes("gpt-image") || value.includes("image");
}
function isTextModelName(model: string) {
return !isImageModelName(model) && !isVideoModelName(model);
}
function isAiConfigReady(config: AiConfig, model: string) {
return Boolean(model.trim()) && (config.channelMode === "remote" || Boolean(config.baseUrl.trim() && config.apiKey.trim()));
}
@@ -112,7 +141,7 @@ export const useConfigStore = create<ConfigStore>()(
partialize: (state) => ({ config: state.config }),
merge: (persisted, current) => {
const config = { ...defaultConfig, ...((persisted as Partial<ConfigStore>).config || {}) };
return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720" } };
return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720", videoGenerateAudio: config.videoGenerateAudio || "true", videoWatermark: config.videoWatermark || "false" } };
},
},
),
+10
View File
@@ -4,4 +4,14 @@ export type ReferenceVideo = {
type: string;
url: string;
storageKey?: string;
durationMs?: number;
};
export type ReferenceAudio = {
id: string;
name: string;
type: string;
url: string;
storageKey?: string;
durationMs?: number;
};