Merge branch 'fork/stupid-h4er/feat/ark-agentplan-seedance-video'
# Conflicts: # docs/pending-test.md
This commit is contained in:
@@ -9,12 +9,17 @@ JWT_EXPIRE_HOURS=168
|
|||||||
# 后端监听端口
|
# 后端监听端口
|
||||||
PORT=8080
|
PORT=8080
|
||||||
|
|
||||||
|
# 公开访问地址,用于把本地上传的 Seedance 参考图/视频暴露给火山方舟拉取。
|
||||||
|
# 线上部署时填写站点根地址,例如:https://your-domain.example.com
|
||||||
|
# PUBLIC_BASE_URL=https://your-domain.example.com
|
||||||
|
|
||||||
# 前端开发代理默认使用 http://127.0.0.1:8080,如后端开发端口不同,启动前端时可单独设置 API_BASE_URL。
|
# 前端开发代理默认使用 http://127.0.0.1:8080,如后端开发端口不同,启动前端时可单独设置 API_BASE_URL。
|
||||||
# API_BASE_URL=http://127.0.0.1:8080
|
# API_BASE_URL=http://127.0.0.1:8080
|
||||||
|
|
||||||
# 数据库配置,默认使用本地 SQLite。
|
# 数据库配置,默认使用本地 SQLite。
|
||||||
STORAGE_DRIVER=sqlite
|
STORAGE_DRIVER=sqlite
|
||||||
# sqlite: DATABASE_DSN=data/infinite-canvas.db
|
# sqlite: DATABASE_DSN=data/infinite-canvas.db
|
||||||
|
# Docker 部署时建议使用绝对路径,避免工作目录变化后写入临时库:DATABASE_DSN=/app/data/infinite-canvas.db
|
||||||
# mysql: DATABASE_DSN=user:password@tcp(127.0.0.1:3306)/infinite_canvas?parseTime=true
|
# mysql: DATABASE_DSN=user:password@tcp(127.0.0.1:3306)/infinite_canvas?parseTime=true
|
||||||
# postgres: DATABASE_DSN=postgres://user:password@127.0.0.1:5432/infinite_canvas?sslmode=disable
|
# postgres: DATABASE_DSN=postgres://user:password@127.0.0.1:5432/infinite_canvas?sslmode=disable
|
||||||
DATABASE_DSN=data/infinite-canvas.db
|
DATABASE_DSN=data/infinite-canvas.db
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
## 核心功能
|
## 核心功能
|
||||||
|
|
||||||
- 无限画布:多画布项目、节点拖拽缩放、连线、小地图、撤销重做、导入导出。
|
- 无限画布:多画布项目、节点拖拽缩放、连线、小地图、撤销重做、导入导出。
|
||||||
- AI 创作:支持 OpenAI 兼容接口的文生图、图生图、参考图编辑和文本问答。
|
- AI 创作:支持 OpenAI 兼容接口的文生图、图生图、参考图编辑、文本问答和视频生成;Seedance 2.0 可通过火山方舟 Agent Plan 接入。
|
||||||
- 画布助手:围绕选中节点和上游节点对话、生图,并把结果插回画布。
|
- 画布助手:围绕选中节点和上游节点对话、生图,并把结果插回画布。
|
||||||
- 提示词库:抓取多个 GitHub 开源项目,按案例整理数百个图片提示词。
|
- 提示词库:抓取多个 GitHub 开源项目,按案例整理数百个图片提示词。
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package config
|
|||||||
import (
|
import (
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/caarlos0/env/v11"
|
"github.com/caarlos0/env/v11"
|
||||||
@@ -17,6 +19,7 @@ type Config struct {
|
|||||||
JWTExpireHours int `env:"JWT_EXPIRE_HOURS" envDefault:"168"`
|
JWTExpireHours int `env:"JWT_EXPIRE_HOURS" envDefault:"168"`
|
||||||
StorageDriver string `env:"STORAGE_DRIVER" envDefault:"sqlite"`
|
StorageDriver string `env:"STORAGE_DRIVER" envDefault:"sqlite"`
|
||||||
DatabaseDSN string `env:"DATABASE_DSN" envDefault:"data/infinite-canvas.db"`
|
DatabaseDSN string `env:"DATABASE_DSN" envDefault:"data/infinite-canvas.db"`
|
||||||
|
PublicBaseURL string `env:"PUBLIC_BASE_URL"`
|
||||||
LinuxDoAuthorizeURL string `env:"LINUX_DO_AUTHORIZE_URL" envDefault:"https://connect.linux.do/oauth2/authorize"`
|
LinuxDoAuthorizeURL string `env:"LINUX_DO_AUTHORIZE_URL" envDefault:"https://connect.linux.do/oauth2/authorize"`
|
||||||
LinuxDoTokenURL string `env:"LINUX_DO_TOKEN_URL" envDefault:"https://connect.linux.do/oauth2/token"`
|
LinuxDoTokenURL string `env:"LINUX_DO_TOKEN_URL" envDefault:"https://connect.linux.do/oauth2/token"`
|
||||||
LinuxDoUserInfoURL string `env:"LINUX_DO_USERINFO_URL" envDefault:"https://connect.linux.do/api/user"`
|
LinuxDoUserInfoURL string `env:"LINUX_DO_USERINFO_URL" envDefault:"https://connect.linux.do/api/user"`
|
||||||
@@ -29,6 +32,7 @@ func Load() error {
|
|||||||
if err := env.Parse(&Cfg); err != nil {
|
if err := env.Parse(&Cfg); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
normalizeDockerSQLiteDSN("/app/data")
|
||||||
if strings.TrimSpace(Cfg.JWTSecret) == "" || Cfg.JWTSecret == "infinite-canvas" {
|
if strings.TrimSpace(Cfg.JWTSecret) == "" || Cfg.JWTSecret == "infinite-canvas" {
|
||||||
secret, err := randomSecret()
|
secret, err := randomSecret()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -39,6 +43,33 @@ func Load() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeDockerSQLiteDSN(appDataDir string) {
|
||||||
|
driver := strings.ToLower(strings.TrimSpace(Cfg.StorageDriver))
|
||||||
|
if driver != "" && driver != "sqlite" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dsn := strings.TrimSpace(Cfg.DatabaseDSN)
|
||||||
|
if dsn == "" || dsn == ":memory:" || strings.HasPrefix(dsn, "file:") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pathPart, suffix := dsn, ""
|
||||||
|
if index := strings.Index(dsn, "?"); index >= 0 {
|
||||||
|
pathPart = dsn[:index]
|
||||||
|
suffix = dsn[index:]
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(pathPart) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
slashPath := filepath.ToSlash(pathPart)
|
||||||
|
if slashPath != "data" && !strings.HasPrefix(slashPath, "data/") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(appDataDir); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
Cfg.DatabaseDSN = filepath.Join(filepath.Dir(appDataDir), filepath.FromSlash(slashPath)) + suffix
|
||||||
|
}
|
||||||
|
|
||||||
func randomSecret() (string, error) {
|
func randomSecret() (string, error) {
|
||||||
buf := make([]byte, 32)
|
buf := make([]byte, 32)
|
||||||
if _, err := rand.Read(buf); err != nil {
|
if _, err := rand.Read(buf); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeDockerSQLiteDSNUsesMountedDataDir(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
appDataDir := filepath.Join(root, "data")
|
||||||
|
if err := os.MkdirAll(appDataDir, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
Cfg = Config{StorageDriver: "sqlite", DatabaseDSN: "data/infinite-canvas.db?_pragma=busy_timeout(5000)"}
|
||||||
|
|
||||||
|
normalizeDockerSQLiteDSN(appDataDir)
|
||||||
|
|
||||||
|
want := filepath.Join(root, "data", "infinite-canvas.db") + "?_pragma=busy_timeout(5000)"
|
||||||
|
if Cfg.DatabaseDSN != want {
|
||||||
|
t.Fatalf("DatabaseDSN = %q, want %q", Cfg.DatabaseDSN, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeDockerSQLiteDSNLeavesLocalPathWithoutMountedDataDir(t *testing.T) {
|
||||||
|
Cfg = Config{StorageDriver: "sqlite", DatabaseDSN: "data/infinite-canvas.db"}
|
||||||
|
|
||||||
|
normalizeDockerSQLiteDSN(filepath.Join(t.TempDir(), "missing-data"))
|
||||||
|
|
||||||
|
if Cfg.DatabaseDSN != "data/infinite-canvas.db" {
|
||||||
|
t.Fatalf("DatabaseDSN = %q, want relative local path", Cfg.DatabaseDSN)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
- 当文本节点已有内容时,输入框用于填写想把本段文本修改成什么;点击发送后,会在右侧生成新的文本节点,并自动连接原节点和新节点。
|
- 当文本节点已有内容时,输入框用于填写想把本段文本修改成什么;点击发送后,会在右侧生成新的文本节点,并自动连接原节点和新节点。
|
||||||
- 输入内容可以手写,也可以从提示词库选择。
|
- 输入内容可以手写,也可以从提示词库选择。
|
||||||
- 对话框里的模型下拉来自全局配置里已拉取的模型列表;选择结果只作用于当前节点,不会修改其它节点或全局默认模型。
|
- 对话框里的模型下拉来自全局配置里已拉取的模型列表;选择结果只作用于当前节点,不会修改其它节点或全局默认模型。
|
||||||
- 如果下拉中没有模型,需要先打开配置弹窗拉取模型列表,并设置默认生图模型和默认文本模型。
|
- 如果下拉中没有模型,需要先打开配置弹窗拉取模型列表,并设置默认生图模型和默认文本模型。火山方舟 Agent Plan 若提示不支持 `/models`,请手动填写模型名。
|
||||||
|
|
||||||
### 用文本节点生成图片
|
### 用文本节点生成图片
|
||||||
|
|
||||||
@@ -45,7 +45,11 @@
|
|||||||
- 视频节点使用原生播放器展示内容,可在节点内直接播放、暂停和拖动进度。
|
- 视频节点使用原生播放器展示内容,可在节点内直接播放、暂停和拖动进度。
|
||||||
- 空视频节点下方对话框可输入提示词生成视频,结果会回填到当前节点。
|
- 空视频节点下方对话框可输入提示词生成视频,结果会回填到当前节点。
|
||||||
- 从文本、图片或配置节点创建视频生成时,会在右侧生成新的视频节点并自动连接。
|
- 从文本、图片或配置节点创建视频生成时,会在右侧生成新的视频节点并自动连接。
|
||||||
- 视频生成接口使用 OpenAI 兼容的 `POST /v1/videos`、`GET /v1/videos/{id}` 和 `GET /v1/videos/{id}/content`。
|
- 生成配置节点的视频模式会读取上游文本作为 prompt,读取上游图片作为参考图,读取上游视频作为参考视频,并在输入预览里显示参考视频。
|
||||||
|
- 视频生成接口支持 OpenAI 风格的 `POST /v1/videos`、`GET /v1/videos/{id}` 和 `GET /v1/videos/{id}/content`。
|
||||||
|
- 使用火山方舟 Agent Plan / Seedance 2.0 时,Base URL 配置为 `https://ark.cn-beijing.volces.com/api/plan/v3`,模型名使用 Seedance 2.0 对应模型;系统会改用 `POST /contents/generations/tasks` 创建异步任务,并轮询 `GET /contents/generations/tasks/{id}`。
|
||||||
|
- Agent Plan 专属 `/api/plan/v3` 当前未提供 OpenAI `/models` 模型列表接口,后台不会伪造模型列表;请手动填写 `doubao-seedance-2.0` 或文档列出的其他可用模型。
|
||||||
|
- Seedance 参考视频必须是公网可访问 URL,或由本项目后端在配置 `PUBLIC_BASE_URL` 后上传并暴露的参考素材 URL。本地/内网地址无法被火山服务器拉取。
|
||||||
|
|
||||||
### 推荐流程
|
### 推荐流程
|
||||||
|
|
||||||
|
|||||||
+23
-3
@@ -52,12 +52,26 @@
|
|||||||
|
|
||||||
## AI 生成
|
## AI 生成
|
||||||
|
|
||||||
项目通过前端直接请求 OpenAI 兼容接口:
|
项目支持两种 AI 调用方式:
|
||||||
|
|
||||||
|
- 本地直连:前端使用本地配置的 Base URL、API Key 和 Model 直接请求 OpenAI 兼容接口。
|
||||||
|
- 后台渠道:前端请求本项目后端 `/api/v1/*` 代理接口,后端按模型选择管理后台配置的渠道。
|
||||||
|
|
||||||
|
OpenAI 兼容图像和文本能力继续复用现有接口:
|
||||||
|
|
||||||
- `/v1/images/generations`:文生图。
|
- `/v1/images/generations`:文生图。
|
||||||
- `/v1/images/edits`:图生图/参考图编辑。
|
- `/v1/images/edits`:图生图/参考图编辑。
|
||||||
- `/v1/chat/completions`:文本问答和带图问答。
|
- `/v1/chat/completions`:文本问答和带图问答。
|
||||||
- `/v1/models`:读取模型列表。
|
- `/v1/models`:读取模型列表;火山方舟 Agent Plan 专属 `/api/plan/v3` 当前未提供 OpenAI `/models` 模型列表接口,需要手动填写模型名。
|
||||||
|
|
||||||
|
视频能力支持两类接口:
|
||||||
|
|
||||||
|
- OpenAI 风格视频:`POST /v1/videos`、`GET /v1/videos/{id}`、`GET /v1/videos/{id}/content`。
|
||||||
|
- 火山方舟 Agent Plan / Seedance 2.0:Base URL 使用 `https://ark.cn-beijing.volces.com/api/plan/v3`,创建任务为 `POST /contents/generations/tasks`,查询任务为 `GET /contents/generations/tasks/{id}`,成功结果读取 `content.video_url`。
|
||||||
|
|
||||||
|
Base URL 如果已经以 `/v1`、`/api/v3` 或 `/api/plan/v3` 结尾,系统不会再追加 `/v1`。因此 cpa 反代或火山方舟 Agent Plan 可以继续通过现有 Base URL + API Key + Model 方式配置,不需要新增火山生图 Provider。
|
||||||
|
|
||||||
|
后台“拉取模型列表”会尝试真实请求 OpenAI `/models`,不会为 Agent Plan 伪造模型结果。如果火山方舟 Agent Plan 返回 404,请手动增加 `doubao-seedance-2.0` 或文档列出的其他模型名。
|
||||||
|
|
||||||
可配置项:
|
可配置项:
|
||||||
|
|
||||||
@@ -67,9 +81,13 @@
|
|||||||
- 图片质量。
|
- 图片质量。
|
||||||
- 图片比例。
|
- 图片比例。
|
||||||
- 生成数量。
|
- 生成数量。
|
||||||
|
- 视频模型。
|
||||||
|
- 视频比例、清晰度、时长、生成声音和水印。
|
||||||
|
|
||||||
普通图片/文本节点可以直接输入提示词生成结果。生成配置节点可以读取上游节点内容,并按节点自己的配置批量生成多个图片或文本结果。生成配置节点支持预览当前提示词和参考图输入,并调整输入顺序。
|
普通图片/文本节点可以直接输入提示词生成结果。生成配置节点可以读取上游节点内容,并按节点自己的配置批量生成多个图片或文本结果。生成配置节点支持预览当前提示词和参考图输入,并调整输入顺序。
|
||||||
|
|
||||||
|
视频生成可从文本节点读取 prompt,从图片节点读取参考图,从视频节点读取参考视频,从音频节点读取参考音频。Seedance 2.0 支持最多 9 张参考图、3 个参考视频、3 个参考音频;分辨率支持 `480p`、`720p`、`1080p`(fast 模型不支持 `1080p`),比例支持 `16:9`、`4:3`、`1:1`、`3:4`、`9:16`、`21:9`、`adaptive`,时长支持 4-15 秒或智能时长。生成成功后会把视频插入画布为视频节点并使用原生播放器预览。Seedance 参考视频和参考音频需要公网可访问 URL;本地上传素材会先通过 `/api/v1/media/references` 保存到服务端,再由 `PUBLIC_BASE_URL` 生成可供火山服务器拉取的公开链接。
|
||||||
|
|
||||||
## 画布助手
|
## 画布助手
|
||||||
|
|
||||||
画布右侧助手面板支持:
|
画布右侧助手面板支持:
|
||||||
@@ -161,6 +179,8 @@
|
|||||||
## 当前限制
|
## 当前限制
|
||||||
|
|
||||||
- 画布项目和“我的素材”目前只保存在浏览器本地,不会随账号同步。
|
- 画布项目和“我的素材”目前只保存在浏览器本地,不会随账号同步。
|
||||||
- AI API Key 目前保存在浏览器本地,并由浏览器直接请求配置的 OpenAI 兼容接口。
|
- 本地直连模式下,AI API Key 保存在浏览器本地,并由浏览器直接请求配置的 OpenAI 兼容接口;只适合本地或个人使用,公网多人使用不安全。公网部署推荐使用后台渠道,把真实密钥保存在服务端配置中。
|
||||||
- 服务器素材库目前主要保存 URL 或文本,暂未提供文件上传接口。
|
- 服务器素材库目前主要保存 URL 或文本,暂未提供文件上传接口。
|
||||||
|
- Seedance 本地参考图/视频上传依赖 `PUBLIC_BASE_URL`,如果服务部署在 localhost、内网或不可被火山访问的地址,火山无法拉取参考素材。
|
||||||
|
- Seedance 返回远程视频 URL 时,前端会尽量下载为本地 Blob 持久化;如果因 CORS 或网络限制无法下载,会保留远程 URL,后续是否可播放取决于上游 URL 的有效期。
|
||||||
- 画布更适合桌面端使用,移动端触控体验还未系统完善。
|
- 画布更适合桌面端使用,移动端触控体验还未系统完善。
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
# 待测试
|
# 待测试
|
||||||
|
|
||||||
- 外部软件可通过 URL 查询参数 `baseUrl`/`baseurl` 和 `apiKey`/`apikey` 跳转到前端;读取后会从地址栏移除这些参数,后台允许自定义渠道时会自动切到自定义渠道、填入配置并打开配置弹窗,未允许时会打开配置弹窗并提示无法导入。
|
- 外部软件可通过 URL 查询参数 `baseUrl`/`baseurl` 和 `apiKey`/`apikey` 跳转到前端;读取后会从地址栏移除这些参数,后台允许自定义渠道时会自动切到自定义渠道、填入配置并打开配置弹窗,未允许时会打开配置弹窗并提示无法导入。
|
||||||
|
- 生图工作台和画布生图会把参考图按当前顺序显示为 `图片1`、`图片2` 等编号,并在图生图请求的实际提示词中注入编号说明;需要验证 `/image` 参考图排序、画布配置节点输入顺序和画布助手参考图编号一致。
|
||||||
|
- GPT Image 生图请求会在前端把 `9:16`、`16:9` 等比例转换成合法 `WIDTHxHEIGHT` 尺寸,并在非法尺寸时直接显示中文错误,避免上游返回 `invalid_value Invalid size`。
|
||||||
|
- Docker 部署时,`DATABASE_DSN=data/infinite-canvas.db` 会在存在 `/app/data` 挂载目录时自动归一到 `/app/data/infinite-canvas.db`,需要验证后台模型配置不会再因为工作目录变为 `/app/web` 而读到空库。
|
||||||
|
- Seedance 参考视频被火山判定包含真人或隐私信息时,前端错误摘要会提示改用不含真人的视频、官方允许的模型产物或已授权的 `asset://` 素材;参考素材上传目录改为跟随 SQLite 数据目录,并补充公开素材的 HEAD 访问。
|
||||||
|
- Seedance 参考素材失败原因排查:后端会把火山上游错误摘要返回给前端;`/video` 和画布视频生成会按 `图片1/视频1/音频1` 自动编号参考素材,并在实际请求提示词中注入编号说明;参考视频会在请求前校验大小、时长、宽高、宽高比和像素总量。
|
||||||
- 画布项目导出改为下载 `.zip` 压缩包,包内包含 `projects.json` 和当前画布引用到的本地图片、视频文件,避免只导出 JSON 时丢失媒体内容。
|
- 画布项目导出改为下载 `.zip` 压缩包,包内包含 `projects.json` 和当前画布引用到的本地图片、视频文件,避免只导出 JSON 时丢失媒体内容。
|
||||||
- 画布库支持多选后一键导出多个画布项目,导出的压缩包可一次恢复多个项目。
|
- 画布库支持多选后一键导出多个画布项目,导出的压缩包可一次恢复多个项目。
|
||||||
- 画布项目导入改为读取新版 `.zip` 压缩包,会先按 `projects.json` 中的文件映射恢复图片、视频到本地存储,再插入画布项目,导入成功后仍停留在画布库。
|
- 画布项目导入改为读取新版 `.zip` 压缩包,会先按 `projects.json` 中的文件映射恢复图片、视频到本地存储,再插入画布项目,导入成功后仍停留在画布库。
|
||||||
@@ -22,3 +27,10 @@
|
|||||||
- 视频生成前端会识别后端 `{ code, msg }` 错误响应,创建失败不再继续轮询 `undefined`。
|
- 视频生成前端会识别后端 `{ code, msg }` 错误响应,创建失败不再继续轮询 `undefined`。
|
||||||
- 新增 `/video` 视频创作台页面,参考生图工作台布局,支持提示词、参考图、视频参数、生成结果、保存素材、下载和本地生成记录;清晰度和秒数均支持常用值选择与手动输入,生成记录只保存媒体 `storageKey` 并可回填本次提示词、参考图和参数。
|
- 新增 `/video` 视频创作台页面,参考生图工作台布局,支持提示词、参考图、视频参数、生成结果、保存素材、下载和本地生成记录;清晰度和秒数均支持常用值选择与手动输入,生成记录只保存媒体 `storageKey` 并可回填本次提示词、参考图和参数。
|
||||||
- 视频创作台生成前会把模型、尺寸、秒数、清晰度归一化为视频接口支持的参数,并展示后端返回的错误信息,避免页面侧残留的生图参数影响视频请求。
|
- 视频创作台生成前会把模型、尺寸、秒数、清晰度归一化为视频接口支持的参数,并展示后端返回的错误信息,避免页面侧残留的生图参数影响视频请求。
|
||||||
|
- 火山方舟 Agent Plan / Seedance 2.0 视频生成需要在真实账号下验证:`/contents/generations/tasks` 创建任务、轮询状态、`content.video_url` 回填画布,以及 401/403/429/超时错误提示。
|
||||||
|
- 管理后台保存私有渠道后,需要验证所有已启用渠道里的模型会自动出现在公开 `availableModels`,并且 `defaultVideoModel`、`defaultImageModel`、`defaultTextModel` 在为空或失效时会自动修复,前台不再显示旧的 `grok` 默认值。
|
||||||
|
- `/video` 和画布视频设置已按 Seedance 2.0 增加分辨率、比例、4-15 秒/智能时长、生成声音和水印参数;需要在真实浏览器里验证参数回填、生成记录和画布节点配置都能保持一致。
|
||||||
|
- `/video` 支持最多 9 张参考图、3 个参考视频、3 个参考音频;需要验证格式、大小、音频时长提示和生成请求中的 `reference_image`、`reference_video`、`reference_audio` 组装。
|
||||||
|
- 画布新增音频节点,支持上传、拖入、播放、移动、缩放、删除,并可作为上游参考音频参与 Seedance 视频生成;需要验证刷新后本地音频 URL 能恢复。
|
||||||
|
- `PUBLIC_BASE_URL` 已配置公网域名时,需要验证本地上传参考视频和参考音频能被火山拉取;未配置或配置为内网地址时,需要验证前端能给出明确提示。
|
||||||
|
- Seedance 返回远程视频 URL 但浏览器无法下载为 Blob 时,需要验证视频节点刷新后仍保留远程 URL,并确认上游 URL 有效期限制。
|
||||||
|
|||||||
@@ -41,11 +41,12 @@
|
|||||||
|
|
||||||
| 字段 | 类型 | 说明 |
|
| 字段 | 类型 | 说明 |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `availableModels` | string[] | 系统可用模型,由管理员手动选择;页面下拉选项可来自私有渠道模型 |
|
| `availableModels` | string[] | 系统可用模型;保存设置时会自动合并所有已启用私有渠道的模型 |
|
||||||
| `modelCosts` | object[] | 模型算力点配置,后端模型接口调用前按模型预扣,上游失败时返还;未配置默认不扣除 |
|
| `modelCosts` | object[] | 模型算力点配置,后端模型接口调用前按模型预扣,上游失败时返还;未配置默认不扣除 |
|
||||||
| `defaultModel` | string | 默认模型,从 `availableModels` 中选择 |
|
| `defaultModel` | string | 默认模型,从 `availableModels` 中选择;为空或失效时优先选择文本模型 |
|
||||||
| `defaultImageModel` | string | 默认图片模型,从 `availableModels` 中选择 |
|
| `defaultImageModel` | string | 默认图片模型,从 `availableModels` 中选择;为空或失效时优先选择 `seedream`、`image`、`gpt-image` 模型 |
|
||||||
| `defaultTextModel` | string | 默认文本模型,从 `availableModels` 中选择 |
|
| `defaultVideoModel` | string | 默认视频模型,从 `availableModels` 中选择;为空或失效时优先选择 `seedance`、`video` 模型 |
|
||||||
|
| `defaultTextModel` | string | 默认文本模型,从 `availableModels` 中选择;为空或失效时优先选择非图片/视频模型 |
|
||||||
| `systemPrompt` | string | 系统提示词 |
|
| `systemPrompt` | string | 系统提示词 |
|
||||||
| `allowCustomChannel` | boolean | 是否允许用户在配置弹窗中切换为本地直连渠道,默认允许 |
|
| `allowCustomChannel` | boolean | 是否允许用户在配置弹窗中切换为本地直连渠道,默认允许 |
|
||||||
|
|
||||||
|
|||||||
+94
-3
@@ -49,6 +49,7 @@ func proxyAIGetRequest(w http.ResponseWriter, r *http.Request, path string) {
|
|||||||
Fail(w, "AI 接口请求失败")
|
Fail(w, "AI 接口请求失败")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
path = resolveAIProxyPath(channel.BaseURL, modelName, path)
|
||||||
request, err := http.NewRequest(http.MethodGet, service.BuildModelChannelURL(channel, path), nil)
|
request, err := http.NewRequest(http.MethodGet, service.BuildModelChannelURL(channel, path), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Fail(w, "AI 接口请求失败")
|
Fail(w, "AI 接口请求失败")
|
||||||
@@ -83,6 +84,7 @@ func proxyAIRequest(w http.ResponseWriter, r *http.Request, path string) {
|
|||||||
Fail(w, "AI 接口请求失败")
|
Fail(w, "AI 接口请求失败")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
path = resolveAIProxyPath(channel.BaseURL, modelName, path)
|
||||||
request, err := http.NewRequest(http.MethodPost, service.BuildModelChannelURL(channel, path), bytes.NewReader(body))
|
request, err := http.NewRequest(http.MethodPost, service.BuildModelChannelURL(channel, path), bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("AI proxy build request failed: url=%s err=%v", service.BuildModelChannelURL(channel, path), err)
|
log.Printf("AI proxy build request failed: url=%s err=%v", service.BuildModelChannelURL(channel, path), err)
|
||||||
@@ -117,12 +119,12 @@ func copyAIResponse(w http.ResponseWriter, request *http.Request, onFailure func
|
|||||||
defer response.Body.Close()
|
defer response.Body.Close()
|
||||||
|
|
||||||
if response.StatusCode >= http.StatusBadRequest {
|
if response.StatusCode >= http.StatusBadRequest {
|
||||||
payload, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||||
log.Printf("AI upstream error: url=%s status=%d body=%s", request.URL.String(), response.StatusCode, strings.TrimSpace(string(payload)))
|
log.Printf("AI upstream error: url=%s status=%d", request.URL.String(), response.StatusCode)
|
||||||
if onFailure != nil {
|
if onFailure != nil {
|
||||||
onFailure()
|
onFailure()
|
||||||
}
|
}
|
||||||
Fail(w, "AI 接口请求失败")
|
Fail(w, aiUpstreamStatusMessage(response.StatusCode, body))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,6 +209,95 @@ func readAIRequestCount(body []byte, contentType string) int {
|
|||||||
|
|
||||||
var errMissingModel = &aiError{"缺少模型名称"}
|
var errMissingModel = &aiError{"缺少模型名称"}
|
||||||
|
|
||||||
|
func resolveAIProxyPath(baseURL string, modelName string, path string) string {
|
||||||
|
if !isArkSeedanceVideo(baseURL, modelName) {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
if path == "/videos" {
|
||||||
|
return "/contents/generations/tasks"
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(path, "/videos/") && !strings.HasSuffix(path, "/content") {
|
||||||
|
return "/contents/generations/tasks/" + strings.TrimPrefix(path, "/videos/")
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func isArkSeedanceVideo(baseURL string, modelName string) bool {
|
||||||
|
base := strings.ToLower(baseURL)
|
||||||
|
model := strings.ToLower(modelName)
|
||||||
|
return strings.Contains(model, "seedance") || strings.Contains(model, "doubao-seedance") || strings.Contains(base, "/api/plan/v3")
|
||||||
|
}
|
||||||
|
|
||||||
|
func aiStatusMessage(statusCode int) string {
|
||||||
|
switch statusCode {
|
||||||
|
case http.StatusUnauthorized, http.StatusForbidden:
|
||||||
|
return "AI 接口鉴权失败,请检查 API Key、套餐权限或模型权限"
|
||||||
|
case http.StatusTooManyRequests:
|
||||||
|
return "AI 接口限流或额度不足,请稍后重试或检查额度"
|
||||||
|
default:
|
||||||
|
return "AI 接口请求失败"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func aiUpstreamStatusMessage(statusCode int, body []byte) string {
|
||||||
|
base := aiStatusMessage(statusCode)
|
||||||
|
detail := aiUpstreamErrorDetail(body)
|
||||||
|
if detail == "" {
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
return base + ":" + detail
|
||||||
|
}
|
||||||
|
|
||||||
|
func aiUpstreamErrorDetail(body []byte) string {
|
||||||
|
text := strings.TrimSpace(string(body))
|
||||||
|
if text == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var payload struct {
|
||||||
|
Msg string `json:"msg"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Error struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
} `json:"error"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &payload); err == nil {
|
||||||
|
if payload.Error.Message != "" {
|
||||||
|
if detail := friendlyUpstreamError(payload.Error.Code, payload.Error.Message); detail != "" {
|
||||||
|
return safeUpstreamText(detail)
|
||||||
|
}
|
||||||
|
if payload.Error.Code != "" {
|
||||||
|
return safeUpstreamText(payload.Error.Code + " " + payload.Error.Message)
|
||||||
|
}
|
||||||
|
return safeUpstreamText(payload.Error.Message)
|
||||||
|
}
|
||||||
|
if payload.Msg != "" {
|
||||||
|
return safeUpstreamText(payload.Msg)
|
||||||
|
}
|
||||||
|
if payload.Message != "" {
|
||||||
|
return safeUpstreamText(payload.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return safeUpstreamText(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func friendlyUpstreamError(code string, message string) string {
|
||||||
|
lowerCode := strings.ToLower(strings.TrimSpace(code))
|
||||||
|
if strings.Contains(lowerCode, "inputvideosensitivecontentdetected") || strings.Contains(lowerCode, "privacyinformation") {
|
||||||
|
return strings.TrimSpace(code + " 参考视频疑似包含真人或隐私信息,火山方舟拒绝使用普通 URL 作为真人视频参考;请改用不含真人的视频、官方允许的模型产物,或已授权的 asset:// 素材。原始错误:" + message)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeUpstreamText(text string) string {
|
||||||
|
text = strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
|
||||||
|
runes := []rune(text)
|
||||||
|
if len(runes) > 300 {
|
||||||
|
return string(runes[:300]) + "..."
|
||||||
|
}
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
type aiError struct {
|
type aiError struct {
|
||||||
message string
|
message string
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAIUpstreamErrorDetail(t *testing.T) {
|
||||||
|
got := aiUpstreamErrorDetail([]byte(`{"error":{"code":"InvalidParameter","message":"reference video fps is invalid"}}`))
|
||||||
|
if got != "InvalidParameter reference video fps is invalid" {
|
||||||
|
t.Fatalf("detail = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAIUpstreamErrorDetailExplainsSensitiveVideo(t *testing.T) {
|
||||||
|
got := aiUpstreamErrorDetail([]byte(`{"error":{"code":"InputVideoSensitiveContentDetected.PrivacyInformation","message":"The request failed because the input video may contain real person."}}`))
|
||||||
|
if !strings.Contains(got, "参考视频疑似包含真人") || !strings.Contains(got, "asset://") {
|
||||||
|
t.Fatalf("detail = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSafeUpstreamTextTruncates(t *testing.T) {
|
||||||
|
got := safeUpstreamText(strings.Repeat("错", 320))
|
||||||
|
if len([]rune(got)) != 303 {
|
||||||
|
t.Fatalf("truncated rune length = %d", len([]rune(got)))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/basketikun/infinite-canvas/config"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
referenceMediaMaxBytes = 80 << 20
|
||||||
|
referenceImageMaxBytes = 30 << 20
|
||||||
|
referenceVideoMaxBytes = 50 << 20
|
||||||
|
referenceAudioMaxBytes = 15 << 20
|
||||||
|
referenceImageAllowedText = "jpeg/png/webp/bmp/gif/heic/heif 图片"
|
||||||
|
referenceVideoAllowedText = "mp4/mov 视频"
|
||||||
|
referenceAudioAllowedText = "mp3/wav 音频"
|
||||||
|
referenceMediaAllowedText = referenceImageAllowedText + "、" + referenceVideoAllowedText + "或" + referenceAudioAllowedText
|
||||||
|
)
|
||||||
|
|
||||||
|
type referenceMediaUploadResult struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
MimeType string `json:"mimeType"`
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) {
|
||||||
|
publicBaseURL := strings.TrimRight(strings.TrimSpace(config.Cfg.PublicBaseURL), "/")
|
||||||
|
if publicBaseURL == "" {
|
||||||
|
Fail(w, "未配置 PUBLIC_BASE_URL,无法把本地参考素材提供给火山方舟访问")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, referenceMediaMaxBytes+1)
|
||||||
|
if err := r.ParseMultipartForm(referenceMediaMaxBytes); err != nil {
|
||||||
|
Fail(w, "参考素材过大或上传格式不正确")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.MultipartForm != nil {
|
||||||
|
defer r.MultipartForm.RemoveAll()
|
||||||
|
}
|
||||||
|
file, header, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
Fail(w, "请上传参考图片或视频")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
mimeType, ext, ok := normalizeReferenceMediaType(header.Header.Get("Content-Type"), filepath.Ext(header.Filename))
|
||||||
|
if !ok {
|
||||||
|
Fail(w, "参考素材格式不支持,请使用 "+referenceMediaAllowedText)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(referenceMediaDir(), 0o755); err != nil {
|
||||||
|
Fail(w, "参考素材保存失败")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
id := uuid.NewString() + ext
|
||||||
|
targetPath := filepath.Join(referenceMediaDir(), id)
|
||||||
|
target, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
Fail(w, "参考素材保存失败")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
bytes, copyErr := io.Copy(target, file)
|
||||||
|
closeErr := target.Close()
|
||||||
|
if copyErr != nil || closeErr != nil {
|
||||||
|
_ = os.Remove(targetPath)
|
||||||
|
Fail(w, "参考素材保存失败")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if bytes <= 0 {
|
||||||
|
_ = os.Remove(targetPath)
|
||||||
|
Fail(w, "参考素材为空")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if limit := referenceMediaTypeMaxBytes(mimeType); limit > 0 && bytes > limit {
|
||||||
|
_ = os.Remove(targetPath)
|
||||||
|
Fail(w, referenceMediaSizeMessage(mimeType))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
OK(w, referenceMediaUploadResult{
|
||||||
|
ID: id,
|
||||||
|
URL: fmt.Sprintf("%s/api/media/references/%s", publicBaseURL, id),
|
||||||
|
MimeType: mimeType,
|
||||||
|
Bytes: bytes,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func ReferenceMedia(w http.ResponseWriter, r *http.Request, id string) {
|
||||||
|
if id == "" || id != filepath.Base(id) || strings.Contains(id, "..") {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path := filepath.Join(referenceMediaDir(), id)
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil || info.IsDir() {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if mimeType := mimeTypeByReferenceMediaExt(filepath.Ext(id)); mimeType != "" {
|
||||||
|
w.Header().Set("Content-Type", mimeType)
|
||||||
|
}
|
||||||
|
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||||
|
http.ServeContent(w, r, id, info.ModTime(), file)
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceMediaDir() string {
|
||||||
|
return filepath.Join(referenceDataDir(), "reference-media")
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceDataDir() string {
|
||||||
|
driver := strings.ToLower(strings.TrimSpace(config.Cfg.StorageDriver))
|
||||||
|
dsn := strings.TrimSpace(config.Cfg.DatabaseDSN)
|
||||||
|
if (driver == "" || driver == "sqlite") && dsn != "" && dsn != ":memory:" && !strings.HasPrefix(dsn, "file:") {
|
||||||
|
pathPart := dsn
|
||||||
|
if index := strings.Index(dsn, "?"); index >= 0 {
|
||||||
|
pathPart = dsn[:index]
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(pathPart) {
|
||||||
|
return filepath.Dir(pathPart)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat("/app/data"); err == nil {
|
||||||
|
return "/app/data"
|
||||||
|
}
|
||||||
|
return "data"
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeReferenceMediaType(contentType string, ext string) (string, string, bool) {
|
||||||
|
contentType = strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0]))
|
||||||
|
ext = strings.ToLower(strings.TrimSpace(ext))
|
||||||
|
if contentType == "" || contentType == "application/octet-stream" {
|
||||||
|
contentType = mimeTypeByReferenceMediaExt(ext)
|
||||||
|
}
|
||||||
|
if fixedExt := referenceMediaExtByMimeType(contentType); fixedExt != "" {
|
||||||
|
return contentType, fixedExt, true
|
||||||
|
}
|
||||||
|
if mimeType := mimeTypeByReferenceMediaExt(ext); mimeType != "" {
|
||||||
|
return mimeType, ext, true
|
||||||
|
}
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceMediaExtByMimeType(mimeType string) string {
|
||||||
|
switch strings.ToLower(mimeType) {
|
||||||
|
case "image/jpeg", "image/jpg":
|
||||||
|
return ".jpg"
|
||||||
|
case "image/png":
|
||||||
|
return ".png"
|
||||||
|
case "image/webp":
|
||||||
|
return ".webp"
|
||||||
|
case "image/bmp":
|
||||||
|
return ".bmp"
|
||||||
|
case "image/gif":
|
||||||
|
return ".gif"
|
||||||
|
case "image/heic":
|
||||||
|
return ".heic"
|
||||||
|
case "image/heif":
|
||||||
|
return ".heif"
|
||||||
|
case "video/mp4":
|
||||||
|
return ".mp4"
|
||||||
|
case "video/quicktime", "video/mov":
|
||||||
|
return ".mov"
|
||||||
|
case "audio/mpeg", "audio/mp3":
|
||||||
|
return ".mp3"
|
||||||
|
case "audio/wav", "audio/x-wav", "audio/wave":
|
||||||
|
return ".wav"
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mimeTypeByReferenceMediaExt(ext string) string {
|
||||||
|
switch strings.ToLower(ext) {
|
||||||
|
case ".jpg", ".jpeg":
|
||||||
|
return "image/jpeg"
|
||||||
|
case ".png":
|
||||||
|
return "image/png"
|
||||||
|
case ".webp":
|
||||||
|
return "image/webp"
|
||||||
|
case ".bmp":
|
||||||
|
return "image/bmp"
|
||||||
|
case ".gif":
|
||||||
|
return "image/gif"
|
||||||
|
case ".heic":
|
||||||
|
return "image/heic"
|
||||||
|
case ".heif":
|
||||||
|
return "image/heif"
|
||||||
|
case ".mp4":
|
||||||
|
return "video/mp4"
|
||||||
|
case ".mov":
|
||||||
|
return "video/quicktime"
|
||||||
|
case ".mp3":
|
||||||
|
return "audio/mpeg"
|
||||||
|
case ".wav":
|
||||||
|
return "audio/wav"
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceMediaTypeMaxBytes(mimeType string) int64 {
|
||||||
|
if strings.HasPrefix(mimeType, "image/") {
|
||||||
|
return referenceImageMaxBytes
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(mimeType, "video/") {
|
||||||
|
return referenceVideoMaxBytes
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(mimeType, "audio/") {
|
||||||
|
return referenceAudioMaxBytes
|
||||||
|
}
|
||||||
|
return referenceMediaMaxBytes
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceMediaSizeMessage(mimeType string) string {
|
||||||
|
if strings.HasPrefix(mimeType, "image/") {
|
||||||
|
return "参考图片超过大小限制,请使用 30MB 以内的图片"
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(mimeType, "video/") {
|
||||||
|
return "参考视频超过大小限制,请使用 50MB 以内的 mp4/mov 视频"
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(mimeType, "audio/") {
|
||||||
|
return "参考音频超过大小限制,请使用 15MB 以内的 mp3/wav 音频"
|
||||||
|
}
|
||||||
|
return "参考素材超过大小限制"
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/basketikun/infinite-canvas/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeReferenceMediaTypeSupportsAudio(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
contentType string
|
||||||
|
ext string
|
||||||
|
wantMime string
|
||||||
|
wantExt string
|
||||||
|
}{
|
||||||
|
{name: "mp3 mime", contentType: "audio/mpeg", ext: ".bin", wantMime: "audio/mpeg", wantExt: ".mp3"},
|
||||||
|
{name: "wav ext fallback", contentType: "application/octet-stream", ext: ".wav", wantMime: "audio/wav", wantExt: ".wav"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
mimeType, ext, ok := normalizeReferenceMediaType(tt.contentType, tt.ext)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected media type to be accepted")
|
||||||
|
}
|
||||||
|
if mimeType != tt.wantMime || ext != tt.wantExt {
|
||||||
|
t.Fatalf("got (%q, %q), want (%q, %q)", mimeType, ext, tt.wantMime, tt.wantExt)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReferenceMediaTypeMaxBytes(t *testing.T) {
|
||||||
|
if got := referenceMediaTypeMaxBytes("audio/mpeg"); got != referenceAudioMaxBytes {
|
||||||
|
t.Fatalf("audio max bytes = %d, want %d", got, referenceAudioMaxBytes)
|
||||||
|
}
|
||||||
|
if got := referenceMediaTypeMaxBytes("video/mp4"); got != referenceVideoMaxBytes {
|
||||||
|
t.Fatalf("video max bytes = %d, want %d", got, referenceVideoMaxBytes)
|
||||||
|
}
|
||||||
|
if got := referenceMediaTypeMaxBytes("image/png"); got != referenceImageMaxBytes {
|
||||||
|
t.Fatalf("image max bytes = %d, want %d", got, referenceImageMaxBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReferenceMediaDirUsesAbsoluteSQLiteDataDir(t *testing.T) {
|
||||||
|
previous := config.Cfg
|
||||||
|
t.Cleanup(func() { config.Cfg = previous })
|
||||||
|
root := t.TempDir()
|
||||||
|
config.Cfg = config.Config{StorageDriver: "sqlite", DatabaseDSN: filepath.Join(root, "infinite-canvas.db")}
|
||||||
|
|
||||||
|
if got := referenceMediaDir(); got != filepath.Join(root, "reference-media") {
|
||||||
|
t.Fatalf("referenceMediaDir = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,11 +22,18 @@ func New() *gin.Engine {
|
|||||||
api.GET("/auth/linux-do/callback", gin.WrapF(handler.LinuxDoCallback))
|
api.GET("/auth/linux-do/callback", gin.WrapF(handler.LinuxDoCallback))
|
||||||
api.GET("/auth/me", middleware.OptionalAuth, gin.WrapF(handler.CurrentUser))
|
api.GET("/auth/me", middleware.OptionalAuth, gin.WrapF(handler.CurrentUser))
|
||||||
api.GET("/settings", gin.WrapF(handler.Settings))
|
api.GET("/settings", gin.WrapF(handler.Settings))
|
||||||
|
api.GET("/media/references/:id", func(c *gin.Context) {
|
||||||
|
handler.ReferenceMedia(c.Writer, c.Request, c.Param("id"))
|
||||||
|
})
|
||||||
|
api.HEAD("/media/references/:id", func(c *gin.Context) {
|
||||||
|
handler.ReferenceMedia(c.Writer, c.Request, c.Param("id"))
|
||||||
|
})
|
||||||
v1 := api.Group("/v1", middleware.UserAuth)
|
v1 := api.Group("/v1", middleware.UserAuth)
|
||||||
v1.POST("/images/generations", gin.WrapF(handler.AIImagesGenerations))
|
v1.POST("/images/generations", gin.WrapF(handler.AIImagesGenerations))
|
||||||
v1.POST("/images/edits", gin.WrapF(handler.AIImagesEdits))
|
v1.POST("/images/edits", gin.WrapF(handler.AIImagesEdits))
|
||||||
v1.POST("/chat/completions", gin.WrapF(handler.AIChatCompletions))
|
v1.POST("/chat/completions", gin.WrapF(handler.AIChatCompletions))
|
||||||
v1.POST("/videos", gin.WrapF(handler.AIVideos))
|
v1.POST("/videos", gin.WrapF(handler.AIVideos))
|
||||||
|
v1.POST("/media/references", gin.WrapF(handler.UploadReferenceMedia))
|
||||||
v1.GET("/videos/:id", func(c *gin.Context) {
|
v1.GET("/videos/:id", func(c *gin.Context) {
|
||||||
handler.AIVideo(c.Writer, c.Request, c.Param("id"))
|
handler.AIVideo(c.Writer, c.Request, c.Param("id"))
|
||||||
})
|
})
|
||||||
|
|||||||
+141
-10
@@ -7,16 +7,20 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/basketikun/infinite-canvas/model"
|
"github.com/basketikun/infinite-canvas/model"
|
||||||
"github.com/basketikun/infinite-canvas/repository"
|
"github.com/basketikun/infinite-canvas/repository"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var adminModelHTTPClient = &http.Client{Timeout: 30 * time.Second}
|
||||||
|
|
||||||
func PublicSettings() (model.PublicSetting, error) {
|
func PublicSettings() (model.PublicSetting, error) {
|
||||||
settings, err := repository.GetSettings()
|
settings, err := repository.GetSettings()
|
||||||
return normalizePublicSetting(settings.Public), err
|
return normalizeSettings(settings).Public, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func AdminSettings() (model.Settings, error) {
|
func AdminSettings() (model.Settings, error) {
|
||||||
@@ -52,16 +56,23 @@ func AdminTestChannelModel(index *int, channel model.ModelChannel, modelName str
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
if isArkAgentPlanChannel(resolved) || isSeedanceModelName(modelName) {
|
||||||
|
return testArkSeedanceChannelModel(resolved, modelName)
|
||||||
|
}
|
||||||
return testAdminChannelModel(resolved, modelName)
|
return testAdminChannelModel(resolved, modelName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeSettings(settings model.Settings) model.Settings {
|
func normalizeSettings(settings model.Settings) model.Settings {
|
||||||
settings.Public = normalizePublicSetting(settings.Public)
|
|
||||||
settings.Private = normalizePrivateSetting(settings.Private)
|
settings.Private = normalizePrivateSetting(settings.Private)
|
||||||
|
settings.Public = normalizePublicSettingWithChannels(settings.Public, settings.Private.Channels)
|
||||||
return settings
|
return settings
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting {
|
func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting {
|
||||||
|
return normalizePublicSettingWithChannels(setting, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizePublicSettingWithChannels(setting model.PublicSetting, channels []model.ModelChannel) model.PublicSetting {
|
||||||
if setting.ModelChannel.AvailableModels == nil {
|
if setting.ModelChannel.AvailableModels == nil {
|
||||||
setting.ModelChannel.AvailableModels = []string{}
|
setting.ModelChannel.AvailableModels = []string{}
|
||||||
}
|
}
|
||||||
@@ -82,6 +93,16 @@ func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting {
|
|||||||
enabled := true
|
enabled := true
|
||||||
setting.Auth.AllowRegister = &enabled
|
setting.Auth.AllowRegister = &enabled
|
||||||
}
|
}
|
||||||
|
enabledModels := enabledChannelModels(channels)
|
||||||
|
if len(enabledModels) > 0 {
|
||||||
|
setting.ModelChannel.AvailableModels = enabledModels
|
||||||
|
} else {
|
||||||
|
setting.ModelChannel.AvailableModels = uniqueModelNames(setting.ModelChannel.AvailableModels)
|
||||||
|
}
|
||||||
|
setting.ModelChannel.DefaultTextModel = repairDefaultModel(setting.ModelChannel.DefaultTextModel, setting.ModelChannel.AvailableModels, isTextModelName)
|
||||||
|
setting.ModelChannel.DefaultImageModel = repairDefaultModel(setting.ModelChannel.DefaultImageModel, setting.ModelChannel.AvailableModels, isImageModelName)
|
||||||
|
setting.ModelChannel.DefaultVideoModel = repairDefaultModel(setting.ModelChannel.DefaultVideoModel, setting.ModelChannel.AvailableModels, isVideoModelName)
|
||||||
|
setting.ModelChannel.DefaultModel = repairDefaultModel(setting.ModelChannel.DefaultModel, setting.ModelChannel.AvailableModels, isTextModelName)
|
||||||
return setting
|
return setting
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,13 +200,101 @@ func SelectModelChannel(modelName string) (model.ModelChannel, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func BuildModelChannelURL(channel model.ModelChannel, path string) string {
|
func BuildModelChannelURL(channel model.ModelChannel, path string) string {
|
||||||
baseURL := strings.TrimRight(channel.BaseURL, "/")
|
baseURL := normalizeModelChannelBaseURL(channel.BaseURL)
|
||||||
if !strings.HasSuffix(baseURL, "/v1") {
|
lowerBaseURL := strings.ToLower(baseURL)
|
||||||
|
if !strings.HasSuffix(lowerBaseURL, "/v1") && !strings.HasSuffix(lowerBaseURL, "/api/v3") && !strings.HasSuffix(lowerBaseURL, "/api/plan/v3") {
|
||||||
baseURL += "/v1"
|
baseURL += "/v1"
|
||||||
}
|
}
|
||||||
return baseURL + path
|
return baseURL + path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeModelChannelBaseURL(baseURL string) string {
|
||||||
|
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||||
|
parsed, err := url.Parse(baseURL)
|
||||||
|
if err == nil && parsed.Scheme != "" && parsed.Host != "" {
|
||||||
|
path := strings.TrimRight(parsed.Path, "/")
|
||||||
|
lowerPath := strings.ToLower(path)
|
||||||
|
if index := strings.Index(lowerPath, "/api/plan/v3"); index >= 0 {
|
||||||
|
end := index + len("/api/plan/v3")
|
||||||
|
if len(lowerPath) == end || lowerPath[end] == '/' {
|
||||||
|
parsed.Path = path[:end]
|
||||||
|
parsed.RawPath = ""
|
||||||
|
parsed.RawQuery = ""
|
||||||
|
parsed.Fragment = ""
|
||||||
|
return strings.TrimRight(parsed.String(), "/")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return baseURL
|
||||||
|
}
|
||||||
|
|
||||||
|
func isArkAgentPlanChannel(channel model.ModelChannel) bool {
|
||||||
|
baseURL := strings.ToLower(normalizeModelChannelBaseURL(channel.BaseURL))
|
||||||
|
return strings.HasSuffix(baseURL, "/api/plan/v3")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSeedanceModelName(modelName string) bool {
|
||||||
|
modelName = strings.ToLower(strings.TrimSpace(modelName))
|
||||||
|
return strings.Contains(modelName, "seedance") || strings.Contains(modelName, "doubao-seedance")
|
||||||
|
}
|
||||||
|
|
||||||
|
func enabledChannelModels(channels []model.ModelChannel) []string {
|
||||||
|
models := []string{}
|
||||||
|
for _, channel := range channels {
|
||||||
|
if !channel.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
models = append(models, channel.Models...)
|
||||||
|
}
|
||||||
|
return uniqueModelNames(models)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueModelNames(models []string) []string {
|
||||||
|
result := []string{}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, item := range models {
|
||||||
|
name := strings.TrimSpace(item)
|
||||||
|
if name == "" || seen[name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[name] = true
|
||||||
|
result = append(result, name)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func repairDefaultModel(current string, models []string, preferred func(string) bool) string {
|
||||||
|
current = strings.TrimSpace(current)
|
||||||
|
for _, item := range models {
|
||||||
|
if item == current {
|
||||||
|
return current
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, item := range models {
|
||||||
|
if preferred(item) {
|
||||||
|
return item
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(models) > 0 {
|
||||||
|
return models[0]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isVideoModelName(modelName string) bool {
|
||||||
|
name := strings.ToLower(strings.TrimSpace(modelName))
|
||||||
|
return strings.Contains(name, "seedance") || strings.Contains(name, "video")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isImageModelName(modelName string) bool {
|
||||||
|
name := strings.ToLower(strings.TrimSpace(modelName))
|
||||||
|
return strings.Contains(name, "seedream") || strings.Contains(name, "gpt-image") || strings.Contains(name, "image")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isTextModelName(modelName string) bool {
|
||||||
|
return !isImageModelName(modelName) && !isVideoModelName(modelName)
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeModelChannel(channel model.ModelChannel) model.ModelChannel {
|
func normalizeModelChannel(channel model.ModelChannel) model.ModelChannel {
|
||||||
if channel.Protocol == "" {
|
if channel.Protocol == "" {
|
||||||
channel.Protocol = "openai"
|
channel.Protocol = "openai"
|
||||||
@@ -239,13 +348,16 @@ func fetchAdminChannelModels(channel model.ModelChannel) ([]string, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
request.Header.Set("Authorization", "Bearer "+channel.APIKey)
|
request.Header.Set("Authorization", "Bearer "+channel.APIKey)
|
||||||
response, err := http.DefaultClient.Do(request)
|
response, err := adminModelHTTPClient.Do(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, safeMessageError{message: "读取模型失败:上游接口无响应或网络不可达"}
|
||||||
}
|
}
|
||||||
defer response.Body.Close()
|
defer response.Body.Close()
|
||||||
body, _ := io.ReadAll(response.Body)
|
body, _ := io.ReadAll(response.Body)
|
||||||
if response.StatusCode >= http.StatusBadRequest {
|
if response.StatusCode >= http.StatusBadRequest {
|
||||||
|
if response.StatusCode == http.StatusNotFound && isArkAgentPlanChannel(channel) {
|
||||||
|
return nil, safeMessageError{message: "火山方舟 Agent Plan 未提供 OpenAI /models 模型列表接口,请手动填写模型名称,例如 doubao-seedance-2.0。"}
|
||||||
|
}
|
||||||
return nil, readAdminChannelError(body, response.StatusCode, "读取模型失败")
|
return nil, readAdminChannelError(body, response.StatusCode, "读取模型失败")
|
||||||
}
|
}
|
||||||
var payload struct {
|
var payload struct {
|
||||||
@@ -281,9 +393,9 @@ func testAdminChannelModel(channel model.ModelChannel, modelName string) (string
|
|||||||
}
|
}
|
||||||
request.Header.Set("Authorization", "Bearer "+channel.APIKey)
|
request.Header.Set("Authorization", "Bearer "+channel.APIKey)
|
||||||
request.Header.Set("Content-Type", "application/json")
|
request.Header.Set("Content-Type", "application/json")
|
||||||
response, err := http.DefaultClient.Do(request)
|
response, err := adminModelHTTPClient.Do(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", safeMessageError{message: "测试失败:上游接口无响应或网络不可达"}
|
||||||
}
|
}
|
||||||
defer response.Body.Close()
|
defer response.Body.Close()
|
||||||
responseBody, _ := io.ReadAll(response.Body)
|
responseBody, _ := io.ReadAll(response.Body)
|
||||||
@@ -304,6 +416,22 @@ func testAdminChannelModel(channel model.ModelChannel, modelName string) (string
|
|||||||
return "ok", nil
|
return "ok", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testArkSeedanceChannelModel(channel model.ModelChannel, modelName string) (string, error) {
|
||||||
|
if strings.TrimSpace(modelName) == "" {
|
||||||
|
return "", errors.New("缺少模型名称")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(channel.BaseURL) == "" {
|
||||||
|
return "", safeMessageError{message: "缺少接口地址"}
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(channel.APIKey) == "" {
|
||||||
|
return "", safeMessageError{message: "缺少 API Key"}
|
||||||
|
}
|
||||||
|
if !isArkAgentPlanChannel(channel) {
|
||||||
|
return "Seedance 视频模型不会发送 /chat/completions 文本测试。已检查 Base URL、API Key 和模型名非空;未调用视频生成接口,因此未验证套餐额度或模型权限。", nil
|
||||||
|
}
|
||||||
|
return "Agent Plan / Seedance 视频模型配置格式已通过。后台测试不会调用视频生成接口,因此未验证 API Key、套餐额度或模型权限;请在画布中使用视频生成验证。", nil
|
||||||
|
}
|
||||||
|
|
||||||
func readAdminChannelError(body []byte, statusCode int, fallback string) error {
|
func readAdminChannelError(body []byte, statusCode int, fallback string) error {
|
||||||
var payload struct {
|
var payload struct {
|
||||||
Error *struct {
|
Error *struct {
|
||||||
@@ -319,8 +447,11 @@ func readAdminChannelError(body []byte, statusCode int, fallback string) error {
|
|||||||
return safeMessageError{message: payload.Msg}
|
return safeMessageError{message: payload.Msg}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if statusCode == http.StatusUnauthorized {
|
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
|
||||||
return safeMessageError{message: "上游接口认证失败(401),请检查 API Key"}
|
return safeMessageError{message: fmt.Sprintf("上游接口鉴权失败(%d),请检查 API Key、套餐权限或模型权限", statusCode)}
|
||||||
|
}
|
||||||
|
if statusCode == http.StatusTooManyRequests {
|
||||||
|
return safeMessageError{message: "上游接口限流或额度不足(429),请稍后重试或检查额度"}
|
||||||
}
|
}
|
||||||
if statusCode > 0 {
|
if statusCode > 0 {
|
||||||
return safeMessageError{message: fmt.Sprintf("%s:%d", fallback, statusCode)}
|
return safeMessageError{message: fmt.Sprintf("%s:%d", fallback, statusCode)}
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/basketikun/infinite-canvas/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFetchAdminChannelModelsParsesOpenAIModels(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/models" {
|
||||||
|
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"z-model"},{"id":"a-model"},{"id":""}]}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
models, err := fetchAdminChannelModels(model.ModelChannel{
|
||||||
|
BaseURL: server.URL,
|
||||||
|
APIKey: "test-key",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fetchAdminChannelModels returned error: %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"a-model", "z-model"}; !reflect.DeepEqual(models, want) {
|
||||||
|
t.Fatalf("models = %#v, want %#v", models, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetchAdminChannelModelsReportsArkPlanModelsUnsupported(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/api/plan/v3/models" {
|
||||||
|
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
_, err := fetchAdminChannelModels(model.ModelChannel{
|
||||||
|
BaseURL: server.URL + "/api/plan/v3/contents/generations/tasks",
|
||||||
|
APIKey: "test-key",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected unsupported /models error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "Agent Plan 未提供 OpenAI /models") {
|
||||||
|
t.Fatalf("error = %q", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildModelChannelURLNormalizesArkPlanTaskPath(t *testing.T) {
|
||||||
|
got := BuildModelChannelURL(model.ModelChannel{BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3/contents/generations/tasks?debug=1"}, "/models")
|
||||||
|
want := "https://ark.cn-beijing.volces.com/api/plan/v3/models"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("BuildModelChannelURL = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSettingsPublishesEnabledChannelModelsAndRepairsDefaults(t *testing.T) {
|
||||||
|
settings := normalizeSettings(model.Settings{
|
||||||
|
Public: model.PublicSetting{
|
||||||
|
ModelChannel: model.PublicModelChannelSetting{
|
||||||
|
AvailableModels: []string{"grok-imagine-video", "disabled-model"},
|
||||||
|
DefaultModel: "grok-imagine-video",
|
||||||
|
DefaultTextModel: "missing-text",
|
||||||
|
DefaultImageModel: "missing-image",
|
||||||
|
DefaultVideoModel: "missing-video",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Private: model.PrivateSetting{
|
||||||
|
Channels: []model.ModelChannel{
|
||||||
|
{Enabled: true, Models: []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast", "gpt-5.5"}},
|
||||||
|
{Enabled: false, Models: []string{"disabled-model"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
channel := settings.Public.ModelChannel
|
||||||
|
wantModels := []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast"}
|
||||||
|
if !reflect.DeepEqual(channel.AvailableModels, wantModels) {
|
||||||
|
t.Fatalf("available models = %#v, want %#v", channel.AvailableModels, wantModels)
|
||||||
|
}
|
||||||
|
if channel.DefaultModel != "gpt-5.5" {
|
||||||
|
t.Fatalf("default model = %q, want text model", channel.DefaultModel)
|
||||||
|
}
|
||||||
|
if channel.DefaultTextModel != "gpt-5.5" {
|
||||||
|
t.Fatalf("default text model = %q, want text model", channel.DefaultTextModel)
|
||||||
|
}
|
||||||
|
if channel.DefaultImageModel != "doubao-seedream-5.0-lite" {
|
||||||
|
t.Fatalf("default image model = %q, want seedream", channel.DefaultImageModel)
|
||||||
|
}
|
||||||
|
if channel.DefaultVideoModel != "doubao-seedance-2.0-fast" {
|
||||||
|
t.Fatalf("default video model = %q, want seedance", channel.DefaultVideoModel)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -215,6 +215,10 @@ export default function AdminSettingsPage() {
|
|||||||
const channelModels = await fetchChannelModels(token, { index: editingChannelIndex ?? undefined, channel: normalizeChannel(channel) });
|
const channelModels = await fetchChannelModels(token, { index: editingChannelIndex ?? undefined, channel: normalizeChannel(channel) });
|
||||||
const current = isModelSelectorOpen ? uniqueModels(modelSelectSelected) : uniqueModels(channelForm.getFieldValue("models") || []);
|
const current = isModelSelectorOpen ? uniqueModels(modelSelectSelected) : uniqueModels(channelForm.getFieldValue("models") || []);
|
||||||
rememberModels(channelModels);
|
rememberModels(channelModels);
|
||||||
|
if (!channelModels.length) {
|
||||||
|
message.warning("上游未返回模型列表,请手动输入模型名称");
|
||||||
|
return;
|
||||||
|
}
|
||||||
setModelSelectExisting(current);
|
setModelSelectExisting(current);
|
||||||
setModelSelectSource(uniqueModels(channelModels));
|
setModelSelectSource(uniqueModels(channelModels));
|
||||||
setModelSelectSelected(uniqueModels([...current, ...channelModels]));
|
setModelSelectSelected(uniqueModels([...current, ...channelModels]));
|
||||||
@@ -337,7 +341,7 @@ export default function AdminSettingsPage() {
|
|||||||
const nextChannelModels = collectChannelModels(nextChannels);
|
const nextChannelModels = collectChannelModels(nextChannels);
|
||||||
const nextSettings = normalizeSettings({
|
const nextSettings = normalizeSettings({
|
||||||
...values,
|
...values,
|
||||||
public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: filterModels(values.public.modelChannel.availableModels, nextChannelModels) } },
|
public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: nextChannelModels } },
|
||||||
private: { ...values.private, channels: nextChannels },
|
private: { ...values.private, channels: nextChannels },
|
||||||
});
|
});
|
||||||
const saved = normalizeSettings(await saveAdminSettings(token, nextSettings));
|
const saved = normalizeSettings(await saveAdminSettings(token, nextSettings));
|
||||||
@@ -410,7 +414,7 @@ export default function AdminSettingsPage() {
|
|||||||
<Form form={form} layout="vertical" initialValues={emptySettings} requiredMark={false}>
|
<Form form={form} layout="vertical" initialValues={emptySettings} requiredMark={false}>
|
||||||
<Row gutter={16}>
|
<Row gutter={16}>
|
||||||
<Col span={24}>
|
<Col span={24}>
|
||||||
<Form.Item name={["public", "modelChannel", "availableModels"]} label="系统可用模型(请先在私有配置里配置渠道)" extra="可选项来自已启用渠道中选择的模型,最终开放哪些模型由这里勾选决定">
|
<Form.Item name={["public", "modelChannel", "availableModels"]} label="系统可用模型(请先在私有配置里配置渠道)" extra="保存设置时会自动合并所有已启用私有渠道的模型,前台模型下拉会读取这里的公开列表">
|
||||||
<Select mode="multiple" placeholder="请选择系统可用模型" options={channelModels.map((item) => ({ label: item, value: item }))} />
|
<Select mode="multiple" placeholder="请选择系统可用模型" options={channelModels.map((item) => ({ label: item, value: item }))} />
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
</Col>
|
</Col>
|
||||||
@@ -707,6 +711,7 @@ export default function AdminSettingsPage() {
|
|||||||
</Button>
|
</Button>
|
||||||
</Space.Compact>
|
</Space.Compact>
|
||||||
</Flex>
|
</Flex>
|
||||||
|
<Typography.Text type="secondary">如果上游不提供 OpenAI /models 模型列表接口,请在这里手动增加模型名称。</Typography.Text>
|
||||||
<Tabs
|
<Tabs
|
||||||
activeKey={modelSelectTab}
|
activeKey={modelSelectTab}
|
||||||
onChange={(key) => setModelSelectTab(key as ModelSelectTabKey)}
|
onChange={(key) => setModelSelectTab(key as ModelSelectTabKey)}
|
||||||
@@ -765,7 +770,7 @@ export default function AdminSettingsPage() {
|
|||||||
destroyOnHidden
|
destroyOnHidden
|
||||||
>
|
>
|
||||||
<Flex vertical gap={12}>
|
<Flex vertical gap={12}>
|
||||||
<Typography.Text type="secondary">测试会向选中模型发送一条 hi,用于确认渠道是否有响应。</Typography.Text>
|
<Typography.Text type="secondary">普通文本模型会发送一条 hi;Agent Plan / Seedance 视频模型只做配置格式检查,不会发起视频生成,也不代表模型权限已验证。</Typography.Text>
|
||||||
<Input.Search placeholder="搜索模型..." allowClear value={testKeyword} onChange={(event) => setTestKeyword(event.target.value)} />
|
<Input.Search placeholder="搜索模型..." allowClear value={testKeyword} onChange={(event) => setTestKeyword(event.target.value)} />
|
||||||
<Table
|
<Table
|
||||||
rowKey="model"
|
rowKey="model"
|
||||||
@@ -926,11 +931,6 @@ function uniqueModels(models: string[]) {
|
|||||||
return Array.from(new Set(models.filter(Boolean)));
|
return Array.from(new Set(models.filter(Boolean)));
|
||||||
}
|
}
|
||||||
|
|
||||||
function filterModels(models: string[], options: string[]) {
|
|
||||||
const optionSet = new Set(options);
|
|
||||||
return uniqueModels(models).filter((model) => optionSet.has(model));
|
|
||||||
}
|
|
||||||
|
|
||||||
function modelSummary(models: string[]) {
|
function modelSummary(models: string[]) {
|
||||||
if (!models.length) return "未配置模型";
|
if (!models.length) return "未配置模型";
|
||||||
const preview = models.slice(0, 3).join(", ");
|
const preview = models.slice(0, 3).join(", ");
|
||||||
@@ -966,7 +966,7 @@ async function collectSettings(form: any, editorMode: Record<SettingsTabKey, Edi
|
|||||||
}
|
}
|
||||||
values.private = privateSetting;
|
values.private = privateSetting;
|
||||||
}
|
}
|
||||||
values.public.modelChannel.availableModels = filterModels(values.public.modelChannel.availableModels, collectChannelModels(values.private.channels));
|
values.public.modelChannel.availableModels = collectChannelModels(values.private.channels);
|
||||||
return normalizeSettings(values);
|
return normalizeSettings(values);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,8 @@ export async function exportAssets(assets: Asset[]) {
|
|||||||
|
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
assets.map(async (asset) => {
|
assets.map(async (asset) => {
|
||||||
const storageKey = asset.kind === "image" || asset.kind === "video" ? asset.data.storageKey : undefined;
|
if (asset.kind !== "image" && asset.kind !== "video") return;
|
||||||
|
const storageKey = asset.data.storageKey;
|
||||||
if (!storageKey) return;
|
if (!storageKey) return;
|
||||||
const blob = asset.kind === "image" ? await getImageBlob(storageKey) : await getMediaBlob(storageKey);
|
const blob = asset.kind === "image" ? await getImageBlob(storageKey) : await getMediaBlob(storageKey);
|
||||||
if (!blob) return;
|
if (!blob) return;
|
||||||
|
|||||||
@@ -3,11 +3,11 @@
|
|||||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
|
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
|
||||||
import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react";
|
import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react";
|
||||||
import { useParams, useRouter } from "next/navigation";
|
import { useParams, useRouter } from "next/navigation";
|
||||||
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
|
import { Home, ImageIcon, Images, List, Menu, MessageSquare, Music2, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
|
||||||
import { saveAs } from "file-saver";
|
import { saveAs } from "file-saver";
|
||||||
|
|
||||||
import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
|
import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
|
||||||
import { requestVideoGeneration } from "@/services/api/video";
|
import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video";
|
||||||
import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
|
import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
|
||||||
import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage";
|
import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage";
|
||||||
import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
|
import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
|
||||||
@@ -52,6 +52,7 @@ import {
|
|||||||
type ViewportTransform,
|
type ViewportTransform,
|
||||||
} from "../types";
|
} from "../types";
|
||||||
import type { ReferenceImage } from "@/types/image";
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
import type { ReferenceAudio } from "@/types/media";
|
||||||
|
|
||||||
type CanvasClipboard = {
|
type CanvasClipboard = {
|
||||||
nodes: CanvasNodeData[];
|
nodes: CanvasNodeData[];
|
||||||
@@ -141,7 +142,7 @@ function CanvasRefreshShell() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video) => void; onClose: () => void }) {
|
function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: PendingConnectionCreate; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio) => void; onClose: () => void }) {
|
||||||
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
@@ -163,6 +164,7 @@ function ConnectionCreateMenu({ pending, onCreate, onClose }: { pending: Pending
|
|||||||
<ConnectionCreateOption theme={theme} icon={<List className="size-5" />} title="文本生成" description="脚本、广告词、品牌文案" onClick={() => onCreate(CanvasNodeType.Text)} />
|
<ConnectionCreateOption theme={theme} icon={<List className="size-5" />} title="文本生成" description="脚本、广告词、品牌文案" onClick={() => onCreate(CanvasNodeType.Text)} />
|
||||||
<ConnectionCreateOption theme={theme} icon={<ImageIcon className="size-5" />} title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} />
|
<ConnectionCreateOption theme={theme} icon={<ImageIcon className="size-5" />} title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} />
|
||||||
<ConnectionCreateOption theme={theme} icon={<Video className="size-5" />} title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} />
|
<ConnectionCreateOption theme={theme} icon={<Video className="size-5" />} title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} />
|
||||||
|
<ConnectionCreateOption theme={theme} icon={<Music2 className="size-5" />} title="音频参考" onClick={() => onCreate(CanvasNodeType.Audio)} />
|
||||||
<ConnectionCreateOption theme={theme} icon={<Settings2 className="size-5" />} title="配置节点" description="模型、尺寸、数量和输入顺序" onClick={() => onCreate(CanvasNodeType.Config)} />
|
<ConnectionCreateOption theme={theme} icon={<Settings2 className="size-5" />} title="配置节点" description="模型、尺寸、数量和输入顺序" onClick={() => onCreate(CanvasNodeType.Config)} />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -483,7 +485,7 @@ function InfiniteCanvasPage() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const createConnectedNode = useCallback(
|
const createConnectedNode = useCallback(
|
||||||
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video, pending: PendingConnectionCreate) => {
|
(type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio, pending: PendingConnectionCreate) => {
|
||||||
const metadata = type === CanvasNodeType.Config ? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: 3 } : undefined;
|
const metadata = type === CanvasNodeType.Config ? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: 3 } : undefined;
|
||||||
const newNode = createCanvasNode(type, pending.position, metadata);
|
const newNode = createCanvasNode(type, pending.position, metadata);
|
||||||
const connection = normalizeConnection(pending.connection.nodeId, newNode.id, [...nodesRef.current, newNode], pending.connection.handleType);
|
const connection = normalizeConnection(pending.connection.nodeId, newNode.id, [...nodesRef.current, newNode], pending.connection.handleType);
|
||||||
@@ -495,7 +497,7 @@ function InfiniteCanvasPage() {
|
|||||||
setConnections((prev) => [...prev, { id: nanoid(), ...connection }]);
|
setConnections((prev) => [...prev, { id: nanoid(), ...connection }]);
|
||||||
setSelectedNodeIds(new Set([newNode.id]));
|
setSelectedNodeIds(new Set([newNode.id]));
|
||||||
setSelectedConnectionId(null);
|
setSelectedConnectionId(null);
|
||||||
if (type !== CanvasNodeType.Text) setDialogNodeId(newNode.id);
|
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
|
||||||
setPendingConnectionCreate(null);
|
setPendingConnectionCreate(null);
|
||||||
setConnecting(null);
|
setConnecting(null);
|
||||||
},
|
},
|
||||||
@@ -611,7 +613,7 @@ function InfiniteCanvasPage() {
|
|||||||
setNodes((prev) => [...prev, newNode]);
|
setNodes((prev) => [...prev, newNode]);
|
||||||
setSelectedNodeIds(new Set([newNode.id]));
|
setSelectedNodeIds(new Set([newNode.id]));
|
||||||
setSelectedConnectionId(null);
|
setSelectedConnectionId(null);
|
||||||
if (type !== CanvasNodeType.Text) setDialogNodeId(newNode.id);
|
if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
|
||||||
},
|
},
|
||||||
[effectiveConfig.imageModel, effectiveConfig.model, effectiveConfig.size, getCanvasCenter],
|
[effectiveConfig.imageModel, effectiveConfig.model, effectiveConfig.size, getCanvasCenter],
|
||||||
);
|
);
|
||||||
@@ -1111,6 +1113,26 @@ function InfiniteCanvasPage() {
|
|||||||
setDialogNodeId(id);
|
setDialogNodeId(id);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const createAudioFileNode = useCallback(async (file: File, position: Position) => {
|
||||||
|
const audio = await uploadMediaFile(file, "audio");
|
||||||
|
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio];
|
||||||
|
const id = `audio-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
|
||||||
|
setNodes((prev) => [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
id,
|
||||||
|
type: CanvasNodeType.Audio,
|
||||||
|
title: file.name,
|
||||||
|
position: { x: position.x - spec.width / 2, y: position.y - spec.height / 2 },
|
||||||
|
width: spec.width,
|
||||||
|
height: spec.height,
|
||||||
|
metadata: audioMetadata(audio),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
setSelectedNodeIds(new Set([id]));
|
||||||
|
setSelectedConnectionId(null);
|
||||||
|
}, []);
|
||||||
|
|
||||||
const createTextNodeFromClipboard = useCallback(
|
const createTextNodeFromClipboard = useCallback(
|
||||||
(text: string) => {
|
(text: string) => {
|
||||||
const trimmed = text.trim();
|
const trimmed = text.trim();
|
||||||
@@ -1324,8 +1346,8 @@ function InfiniteCanvasPage() {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const downloadNodeImage = useCallback((node: CanvasNodeData) => {
|
const downloadNodeImage = useCallback((node: CanvasNodeData) => {
|
||||||
if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video) || !node.metadata?.content) return;
|
if ((node.type !== CanvasNodeType.Image && node.type !== CanvasNodeType.Video && node.type !== CanvasNodeType.Audio) || !node.metadata?.content) return;
|
||||||
saveAs(node.metadata.content, `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : imageExtension(node.metadata.content)}`);
|
saveAs(node.metadata.content, `canvas-${node.type}-${node.id}.${node.type === CanvasNodeType.Video ? "mp4" : node.type === CanvasNodeType.Audio ? audioExtension(node.metadata.mimeType) : imageExtension(node.metadata.content)}`);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const saveNodeAsset = useCallback(
|
const saveNodeAsset = useCallback(
|
||||||
@@ -1453,9 +1475,19 @@ function InfiniteCanvasPage() {
|
|||||||
async (event: ReactChangeEvent<HTMLInputElement>) => {
|
async (event: ReactChangeEvent<HTMLInputElement>) => {
|
||||||
const file = event.target.files?.[0];
|
const file = event.target.files?.[0];
|
||||||
const target = uploadTargetRef.current;
|
const target = uploadTargetRef.current;
|
||||||
if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/"))) return;
|
if (!file || (!file.type.startsWith("image/") && !file.type.startsWith("video/") && !isAudioFile(file))) return;
|
||||||
|
|
||||||
if (target?.nodeId) {
|
if (target?.nodeId) {
|
||||||
|
if (isAudioFile(file)) {
|
||||||
|
const audio = await uploadMediaFile(file, "audio");
|
||||||
|
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio];
|
||||||
|
setNodes((prev) => prev.map((node) => (node.id === target.nodeId ? { ...node, type: CanvasNodeType.Audio, title: file.name, position: { x: node.position.x + node.width / 2 - spec.width / 2, y: node.position.y + node.height / 2 - spec.height / 2 }, width: spec.width, height: spec.height, metadata: { ...node.metadata, ...audioMetadata(audio), errorDetails: undefined } } : node)));
|
||||||
|
setSelectedNodeIds(new Set([target.nodeId]));
|
||||||
|
setSelectedConnectionId(null);
|
||||||
|
uploadTargetRef.current = null;
|
||||||
|
event.target.value = "";
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (file.type.startsWith("video/")) {
|
if (file.type.startsWith("video/")) {
|
||||||
const video = await uploadMediaFile(file, "video");
|
const video = await uploadMediaFile(file, "video");
|
||||||
const nextSize = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
|
const nextSize = fitNodeSize(video.width || 1280, video.height || 720, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
|
||||||
@@ -1505,25 +1537,25 @@ function InfiniteCanvasPage() {
|
|||||||
setDialogNodeId(target.nodeId);
|
setDialogNodeId(target.nodeId);
|
||||||
} else {
|
} else {
|
||||||
const position = target?.position || screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
|
const position = target?.position || screenToCanvas((containerRef.current?.getBoundingClientRect().left || 0) + size.width / 2, (containerRef.current?.getBoundingClientRect().top || 0) + size.height / 2);
|
||||||
void (file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position));
|
void (isAudioFile(file) ? createAudioFileNode(file, position) : file.type.startsWith("video/") ? createVideoFileNode(file, position) : createImageFileNode(file, position));
|
||||||
}
|
}
|
||||||
|
|
||||||
uploadTargetRef.current = null;
|
uploadTargetRef.current = null;
|
||||||
event.target.value = "";
|
event.target.value = "";
|
||||||
},
|
},
|
||||||
[createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width],
|
[createAudioFileNode, createImageFileNode, createVideoFileNode, screenToCanvas, size.height, size.width],
|
||||||
);
|
);
|
||||||
|
|
||||||
const handleDrop = useCallback(
|
const handleDrop = useCallback(
|
||||||
(event: ReactDragEvent<HTMLDivElement>) => {
|
(event: ReactDragEvent<HTMLDivElement>) => {
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/"));
|
const file = Array.from(event.dataTransfer.files).find((item) => item.type.startsWith("image/") || item.type.startsWith("video/") || isAudioFile(item));
|
||||||
if (!file) return;
|
if (!file) return;
|
||||||
|
|
||||||
const pos = screenToCanvas(event.clientX, event.clientY);
|
const pos = screenToCanvas(event.clientX, event.clientY);
|
||||||
void (file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos));
|
void (isAudioFile(file) ? createAudioFileNode(file, pos) : file.type.startsWith("video/") ? createVideoFileNode(file, pos) : createImageFileNode(file, pos));
|
||||||
},
|
},
|
||||||
[createImageFileNode, createVideoFileNode, screenToCanvas],
|
[createAudioFileNode, createImageFileNode, createVideoFileNode, screenToCanvas],
|
||||||
);
|
);
|
||||||
|
|
||||||
const pasteAssistantImage = useCallback(
|
const pasteAssistantImage = useCallback(
|
||||||
@@ -1749,14 +1781,14 @@ function InfiniteCanvasPage() {
|
|||||||
position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y },
|
position: isEmptyVideoNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y },
|
||||||
width: isEmptyVideoNode ? sourceNode.width : spec.width,
|
width: isEmptyVideoNode ? sourceNode.width : spec.width,
|
||||||
height: isEmptyVideoNode ? sourceNode.height : spec.height,
|
height: isEmptyVideoNode ? sourceNode.height : spec.height,
|
||||||
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationContext.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)) },
|
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, references: generationReferenceUrls(generationContext) },
|
||||||
};
|
};
|
||||||
pendingChildIds = [videoId];
|
pendingChildIds = [videoId];
|
||||||
setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode]));
|
setNodes((prev) => (isEmptyVideoNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...videoNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), videoNode]));
|
||||||
if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]);
|
if (!isEmptyVideoNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: videoId }]);
|
||||||
const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, effectivePrompt, generationContext.referenceImages), "video");
|
const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, effectivePrompt, generationContext.referenceImages, generationContext.referenceVideos, generationContext.referenceAudios));
|
||||||
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
|
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
|
||||||
setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 }, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, references: generationContext.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)) } } : node)));
|
setNodes((prev) => prev.map((node) => (node.id === videoId ? { ...node, width: videoSize.width, height: videoSize.height, position: { x: node.position.x + node.width / 2 - videoSize.width / 2, y: node.position.y + node.height / 2 - videoSize.height / 2 }, metadata: { ...node.metadata, ...videoMetadata(video), prompt: effectivePrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, references: generationReferenceUrls(generationContext) } } : node)));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1857,6 +1889,7 @@ function InfiniteCanvasPage() {
|
|||||||
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_ERROR, errorDetails: "参考图片已丢失,无法继续重试" } } : item)));
|
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_ERROR, errorDetails: "参考图片已丢失,无法继续重试" } } : item)));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const retryImages = retryReferenceImages || [];
|
||||||
|
|
||||||
setRunningNodeId(node.id);
|
setRunningNodeId(node.id);
|
||||||
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_LOADING, errorDetails: undefined } } : item)));
|
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_LOADING, errorDetails: undefined } } : item)));
|
||||||
@@ -1873,19 +1906,19 @@ function InfiniteCanvasPage() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (node.type === CanvasNodeType.Video) {
|
if (node.type === CanvasNodeType.Video) {
|
||||||
const video = await uploadMediaFile(await requestVideoGeneration(generationConfig, prompt, retryReferenceImages || []), "video");
|
const video = await storeGeneratedVideo(await requestVideoGeneration(generationConfig, prompt, retryImages, context?.referenceVideos || [], context?.referenceAudios || []));
|
||||||
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
|
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
|
||||||
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality } } : item)));
|
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark } } : item)));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const image = useReferenceImages ? await requestEdit(generationConfig, prompt, retryReferenceImages).then((items) => items[0]) : await requestGeneration(generationConfig, prompt).then((items) => items[0]);
|
const image = useReferenceImages ? await requestEdit(generationConfig, prompt, retryImages).then((items) => items[0]) : await requestGeneration(generationConfig, prompt).then((items) => items[0]);
|
||||||
const uploadedImage = await uploadImage(image.dataUrl);
|
const uploadedImage = await uploadImage(image.dataUrl);
|
||||||
const imageConfig = NODE_DEFAULT_SIZE[CanvasNodeType.Image];
|
const imageConfig = NODE_DEFAULT_SIZE[CanvasNodeType.Image];
|
||||||
const imageSize = fitNodeSize(uploadedImage.width, uploadedImage.height, imageConfig.width, imageConfig.height);
|
const imageSize = fitNodeSize(uploadedImage.width, uploadedImage.height, imageConfig.width, imageConfig.height);
|
||||||
const generationMetadata = savedImageMetadata?.generationType
|
const generationMetadata = savedImageMetadata?.generationType
|
||||||
? { generationType: savedImageMetadata.generationType, model: generationConfig.model, size: generationConfig.size, quality: generationConfig.quality, count: savedImageMetadata.count || 1, references: savedImageMetadata.references }
|
? { generationType: savedImageMetadata.generationType, model: generationConfig.model, size: generationConfig.size, quality: generationConfig.quality, count: savedImageMetadata.count || 1, references: savedImageMetadata.references }
|
||||||
: buildImageGenerationMetadata(useReferenceImages ? "edit" : "generation", generationConfig, 1, retryReferenceImages || []);
|
: buildImageGenerationMetadata(useReferenceImages ? "edit" : "generation", generationConfig, 1, retryImages);
|
||||||
setNodes((prev) =>
|
setNodes((prev) =>
|
||||||
prev.map((item) =>
|
prev.map((item) =>
|
||||||
item.id === node.id
|
item.id === node.id
|
||||||
@@ -2195,6 +2228,7 @@ function InfiniteCanvasPage() {
|
|||||||
showImageInfo={showImageInfo}
|
showImageInfo={showImageInfo}
|
||||||
onAddImage={() => createNode(CanvasNodeType.Image)}
|
onAddImage={() => createNode(CanvasNodeType.Image)}
|
||||||
onAddVideo={() => createNode(CanvasNodeType.Video)}
|
onAddVideo={() => createNode(CanvasNodeType.Video)}
|
||||||
|
onAddAudio={() => createNode(CanvasNodeType.Audio)}
|
||||||
onAddText={() => createNode(CanvasNodeType.Text)}
|
onAddText={() => createNode(CanvasNodeType.Text)}
|
||||||
onAddConfig={() => createNode(CanvasNodeType.Config)}
|
onAddConfig={() => createNode(CanvasNodeType.Config)}
|
||||||
onUndo={undoCanvas}
|
onUndo={undoCanvas}
|
||||||
@@ -2234,7 +2268,7 @@ function InfiniteCanvasPage() {
|
|||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<input ref={imageInputRef} type="file" accept="image/*,video/*" className="hidden" onChange={handleImageInputChange} />
|
<input ref={imageInputRef} type="file" accept="image/*,video/*,audio/mpeg,audio/wav,audio/x-wav,.mp3,.wav" className="hidden" onChange={handleImageInputChange} />
|
||||||
|
|
||||||
<CanvasNodeInfoModal node={infoNode} open={Boolean(infoNode)} onClose={() => setInfoNodeId(null)} />
|
<CanvasNodeInfoModal node={infoNode} open={Boolean(infoNode)} onClose={() => setInfoNodeId(null)} />
|
||||||
|
|
||||||
@@ -2376,7 +2410,7 @@ function CanvasTopBar({
|
|||||||
{ key: "new", icon: <Plus className="size-4" />, label: "新建画布", onClick: onCreateProject },
|
{ key: "new", icon: <Plus className="size-4" />, label: "新建画布", onClick: onCreateProject },
|
||||||
{ key: "delete", danger: true, icon: <Trash2 className="size-4" />, label: "删除当前画布", onClick: onDeleteProject },
|
{ key: "delete", danger: true, icon: <Trash2 className="size-4" />, label: "删除当前画布", onClick: onDeleteProject },
|
||||||
{ type: "divider" },
|
{ type: "divider" },
|
||||||
{ key: "import", icon: <Upload className="size-4" />, label: "导入图片", onClick: onImportImage },
|
{ key: "import", icon: <Upload className="size-4" />, label: "导入素材", onClick: onImportImage },
|
||||||
{ type: "divider" },
|
{ type: "divider" },
|
||||||
{ key: "undo", disabled: !canUndo, icon: <Undo2 className="size-4" />, label: <MenuLabel text="撤销" shortcut="⌘ Z" />, onClick: onUndo },
|
{ key: "undo", disabled: !canUndo, icon: <Undo2 className="size-4" />, label: <MenuLabel text="撤销" shortcut="⌘ Z" />, onClick: onUndo },
|
||||||
{ key: "redo", disabled: !canRedo, icon: <Redo2 className="size-4" />, label: <MenuLabel text="重做" shortcut="⌘ ⇧ Z / ⌘ Y" />, onClick: onRedo },
|
{ key: "redo", disabled: !canRedo, icon: <Redo2 className="size-4" />, label: <MenuLabel text="重做" shortcut="⌘ ⇧ Z / ⌘ Y" />, onClick: onRedo },
|
||||||
@@ -2457,7 +2491,7 @@ function CanvasTopBar({
|
|||||||
<Shortcut keys={["Ctrl / Cmd", "Y"]} value="重做" />
|
<Shortcut keys={["Ctrl / Cmd", "Y"]} value="重做" />
|
||||||
<Shortcut keys={["Delete / Backspace"]} value="删除选中" />
|
<Shortcut keys={["Delete / Backspace"]} value="删除选中" />
|
||||||
<Shortcut keys={["Esc"]} value="取消选择并关闭浮层" />
|
<Shortcut keys={["Esc"]} value="取消选择并关闭浮层" />
|
||||||
<Shortcut keys={["拖入图片"]} value="上传到画布" />
|
<Shortcut keys={["拖入图片/视频/音频"]} value="上传到画布" />
|
||||||
</div>
|
</div>
|
||||||
</Modal>
|
</Modal>
|
||||||
</>
|
</>
|
||||||
@@ -2498,12 +2532,20 @@ function imageExtension(dataUrl: string) {
|
|||||||
return dataUrl.match(/^data:image[/]([^;]+)/)?.[1] || dataUrl.match(/image[/]([^;]+)/)?.[1] || "png";
|
return dataUrl.match(/^data:image[/]([^;]+)/)?.[1] || dataUrl.match(/image[/]([^;]+)/)?.[1] || "png";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function audioExtension(mimeType?: string) {
|
||||||
|
return mimeType?.includes("wav") ? "wav" : "mp3";
|
||||||
|
}
|
||||||
|
|
||||||
function imageMetadata(image: UploadedImage): CanvasNodeMetadata {
|
function imageMetadata(image: UploadedImage): CanvasNodeMetadata {
|
||||||
return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType };
|
return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType };
|
||||||
}
|
}
|
||||||
|
|
||||||
function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
|
function videoMetadata(video: UploadedFile): CanvasNodeMetadata {
|
||||||
return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4" };
|
return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4", durationMs: video.durationMs };
|
||||||
|
}
|
||||||
|
|
||||||
|
function audioMetadata(audio: UploadedFile): CanvasNodeMetadata {
|
||||||
|
return { content: audio.url, storageKey: audio.storageKey, status: "success", bytes: audio.bytes, mimeType: audio.mimeType || "audio/mpeg", durationMs: audio.durationMs };
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata {
|
function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata {
|
||||||
@@ -2521,6 +2563,14 @@ function referenceUrl(image: ReferenceImage) {
|
|||||||
return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined);
|
return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }>; referenceAudios?: Array<{ storageKey?: string; url?: string }> }) {
|
||||||
|
return [
|
||||||
|
...context.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)),
|
||||||
|
...context.referenceVideos.map((video) => video.storageKey || video.url).filter((url): url is string => Boolean(url)),
|
||||||
|
...(context.referenceAudios || []).map((audio) => audio.storageKey || audio.url).filter((url): url is string => Boolean(url)),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
async function resolveMetadataReferences(metadata: CanvasNodeMetadata) {
|
async function resolveMetadataReferences(metadata: CanvasNodeMetadata) {
|
||||||
if (metadata.generationType !== "edit") return [];
|
if (metadata.generationType !== "edit") return [];
|
||||||
if (!metadata.references?.length) return null;
|
if (!metadata.references?.length) return null;
|
||||||
@@ -2537,7 +2587,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) {
|
|||||||
return Promise.all(
|
return Promise.all(
|
||||||
nodes.map(async (node) => {
|
nodes.map(async (node) => {
|
||||||
const content = node.metadata?.content;
|
const content = node.metadata?.content;
|
||||||
if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
|
if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } };
|
||||||
if (node.type !== CanvasNodeType.Image || !content) return node;
|
if (node.type !== CanvasNodeType.Image || !content) return node;
|
||||||
if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } };
|
if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } };
|
||||||
if (!content.startsWith("data:image/")) return node;
|
if (!content.startsWith("data:image/")) return node;
|
||||||
@@ -2574,9 +2624,10 @@ function getGenerationCount(count: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyNodeConfigPatch(node: CanvasNodeData, patch: Partial<CanvasNodeData["metadata"]>) {
|
function applyNodeConfigPatch(node: CanvasNodeData, patch: Partial<CanvasNodeData["metadata"]>) {
|
||||||
const next = { ...node, metadata: { ...node.metadata, ...(patch || {}) } };
|
const safePatch = patch || {};
|
||||||
|
const next = { ...node, metadata: { ...node.metadata, ...safePatch } };
|
||||||
const spec = node.type === CanvasNodeType.Video ? NODE_DEFAULT_SIZE[CanvasNodeType.Video] : NODE_DEFAULT_SIZE[CanvasNodeType.Image];
|
const spec = node.type === CanvasNodeType.Video ? NODE_DEFAULT_SIZE[CanvasNodeType.Video] : NODE_DEFAULT_SIZE[CanvasNodeType.Image];
|
||||||
const size = typeof patch.size === "string" && !node.metadata?.content ? nodeSizeFromRatio(patch.size, spec.width, spec.height) : null;
|
const size = typeof safePatch.size === "string" && !node.metadata?.content ? nodeSizeFromRatio(safePatch.size, spec.width, spec.height) : null;
|
||||||
return size && (node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video) ? { ...next, ...size, position: { x: node.position.x + node.width / 2 - size.width / 2, y: node.position.y + node.height / 2 - size.height / 2 } } : next;
|
return size && (node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video) ? { ...next, ...size, position: { x: node.position.x + node.width / 2 - size.width / 2, y: node.position.y + node.height / 2 - size.height / 2 } } : next;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2595,6 +2646,8 @@ function getInputSummary(inputs: NodeGenerationInput[]) {
|
|||||||
return {
|
return {
|
||||||
textCount: inputs.filter((input) => input.type === "text").length,
|
textCount: inputs.filter((input) => input.type === "text").length,
|
||||||
imageCount: inputs.filter((input) => input.type === "image").length,
|
imageCount: inputs.filter((input) => input.type === "image").length,
|
||||||
|
videoCount: inputs.filter((input) => input.type === "video").length,
|
||||||
|
audioCount: inputs.filter((input) => input.type === "audio").length,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2607,6 +2660,8 @@ function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefine
|
|||||||
size: node?.metadata?.size || config.size || defaultConfig.size,
|
size: node?.metadata?.size || config.size || defaultConfig.size,
|
||||||
videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds,
|
videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds,
|
||||||
vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality,
|
vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality,
|
||||||
|
videoGenerateAudio: node?.metadata?.generateAudio || config.videoGenerateAudio || defaultConfig.videoGenerateAudio,
|
||||||
|
videoWatermark: node?.metadata?.watermark || config.videoWatermark || defaultConfig.videoWatermark,
|
||||||
count: String(node?.metadata?.count || (mode === "image" ? 3 : config.count) || defaultConfig.count),
|
count: String(node?.metadata?.count || (mode === "image" ? 3 : config.count) || defaultConfig.count),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -2642,6 +2697,10 @@ function sourceNodeReferenceImages(node: CanvasNodeData | null) {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAudioFile(file: File) {
|
||||||
|
return file.type.startsWith("audio/") || /\.(mp3|wav)$/i.test(file.name);
|
||||||
|
}
|
||||||
|
|
||||||
function isHiddenBatchChild(node: CanvasNodeData, nodes: CanvasNodeData[], collapsingBatchIds?: Set<string>) {
|
function isHiddenBatchChild(node: CanvasNodeData, nodes: CanvasNodeData[], collapsingBatchIds?: Set<string>) {
|
||||||
const rootId = node.metadata?.batchRootId;
|
const rootId = node.metadata?.batchRootId;
|
||||||
if (!rootId) return false;
|
if (!rootId) return false;
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import { requestEdit, requestGeneration, requestImageQuestion, type ChatCompleti
|
|||||||
import { imageToDataUrl, uploadImage } from "@/services/image-storage";
|
import { imageToDataUrl, uploadImage } from "@/services/image-storage";
|
||||||
import { useAssetStore } from "@/stores/use-asset-store";
|
import { useAssetStore } from "@/stores/use-asset-store";
|
||||||
import { useThemeStore } from "@/stores/use-theme-store";
|
import { useThemeStore } from "@/stores/use-theme-store";
|
||||||
|
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
|
||||||
import type { ReferenceImage } from "@/types/image";
|
import type { ReferenceImage } from "@/types/image";
|
||||||
import { DiaTextReveal } from "@/components/ui/dia-text-reveal";
|
import { DiaTextReveal } from "@/components/ui/dia-text-reveal";
|
||||||
import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover";
|
import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover";
|
||||||
@@ -398,8 +399,8 @@ function AssistantComposer({
|
|||||||
<div className="px-2 pb-2" onWheelCapture={(event) => event.stopPropagation()}>
|
<div className="px-2 pb-2" onWheelCapture={(event) => event.stopPropagation()}>
|
||||||
{references.length ? (
|
{references.length ? (
|
||||||
<div className="thin-scrollbar mb-1.5 flex max-w-full gap-1.5 overflow-x-auto px-1 pb-1">
|
<div className="thin-scrollbar mb-1.5 flex max-w-full gap-1.5 overflow-x-auto px-1 pb-1">
|
||||||
{references.map((item) => (
|
{references.map((item, index) => (
|
||||||
<AssistantReferenceChip key={item.id} item={item} onRemove={() => onRemoveReference(item.id)} />
|
<AssistantReferenceChip key={item.id} item={item} label={assistantImageReferenceLabel(references, index)} onRemove={() => onRemoveReference(item.id)} />
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -584,20 +585,23 @@ function AssistantHistory({
|
|||||||
function MessageReferences({ message }: { message: CanvasAssistantMessage }) {
|
function MessageReferences({ message }: { message: CanvasAssistantMessage }) {
|
||||||
return (
|
return (
|
||||||
<div className={cn("flex max-w-[88%] flex-wrap gap-2", message.role === "user" ? "justify-end" : "justify-start")}>
|
<div className={cn("flex max-w-[88%] flex-wrap gap-2", message.role === "user" ? "justify-end" : "justify-start")}>
|
||||||
{message.references?.map((item) => (
|
{message.references?.map((item, index, references) => (
|
||||||
<AssistantReferenceChip key={item.id} item={item} />
|
<AssistantReferenceChip key={item.id} item={item} label={assistantImageReferenceLabel(references, index)} />
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function AssistantReferenceChip({ item, onRemove }: { item: CanvasAssistantReference; onRemove?: () => void }) {
|
function AssistantReferenceChip({ item, label, onRemove }: { item: CanvasAssistantReference; label?: string; onRemove?: () => void }) {
|
||||||
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
const theme = canvasThemes[useThemeStore((state) => state.theme)];
|
||||||
const text = (item.text || item.title).replace(/\s+/g, " ").trim().slice(0, 1) || "文";
|
const text = (item.text || item.title).replace(/\s+/g, " ").trim().slice(0, 1) || "文";
|
||||||
return (
|
return (
|
||||||
<div className="group/chip relative inline-flex h-8 max-w-[150px] shrink-0 items-center gap-1.5 rounded-lg text-sm" style={{ color: theme.node.text }}>
|
<div className="group/chip relative inline-flex h-8 max-w-[150px] shrink-0 items-center gap-1.5 rounded-lg text-sm" style={{ color: theme.node.text }}>
|
||||||
{item.dataUrl ? (
|
{item.dataUrl ? (
|
||||||
<img src={item.dataUrl} alt="" className="size-8 rounded-lg object-cover" />
|
<span className="relative block size-8 shrink-0">
|
||||||
|
<img src={item.dataUrl} alt="" className="size-8 rounded-lg object-cover" />
|
||||||
|
{label ? <span className="absolute left-0.5 top-0.5 rounded bg-black/60 px-1 py-0.5 text-[8px] font-medium leading-none text-white">{label}</span> : null}
|
||||||
|
</span>
|
||||||
) : (
|
) : (
|
||||||
<span className="grid size-8 place-items-center rounded-lg border text-sm font-medium" style={{ background: theme.node.panel, borderColor: theme.node.activeStroke }}>
|
<span className="grid size-8 place-items-center rounded-lg border text-sm font-medium" style={{ background: theme.node.panel, borderColor: theme.node.activeStroke }}>
|
||||||
{text}
|
{text}
|
||||||
@@ -618,6 +622,12 @@ function AssistantReferenceChip({ item, onRemove }: { item: CanvasAssistantRefer
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function assistantImageReferenceLabel(references: CanvasAssistantReference[], index: number) {
|
||||||
|
if (!references[index]?.dataUrl) return undefined;
|
||||||
|
const imageIndex = references.slice(0, index + 1).filter((item) => item.dataUrl).length - 1;
|
||||||
|
return imageIndex >= 0 ? imageReferenceLabel(imageIndex) : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
function nodeToReference(node: CanvasNodeData): CanvasAssistantReference | null {
|
function nodeToReference(node: CanvasNodeData): CanvasAssistantReference | null {
|
||||||
if (node.type === CanvasNodeType.Image && node.metadata?.content) {
|
if (node.type === CanvasNodeType.Image && node.metadata?.content) {
|
||||||
return { id: node.id, type: node.type, title: node.title, dataUrl: node.metadata.content, storageKey: node.metadata.storageKey };
|
return { id: node.id, type: node.type, title: node.title, dataUrl: node.metadata.content, storageKey: node.metadata.storageKey };
|
||||||
|
|||||||
@@ -2,13 +2,15 @@
|
|||||||
|
|
||||||
import type { CSSProperties } from "react";
|
import type { CSSProperties } from "react";
|
||||||
import { useState } from "react";
|
import { useState } from "react";
|
||||||
import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play, Video } from "lucide-react";
|
import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Music2, Play, Video } from "lucide-react";
|
||||||
import { App, Button, Empty, Input, Modal, Segmented } from "antd";
|
import { App, Button, Empty, Input, Modal, Segmented } from "antd";
|
||||||
|
|
||||||
import { ModelPicker } from "@/components/model-picker";
|
import { ModelPicker } from "@/components/model-picker";
|
||||||
import { defaultConfig, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
import { defaultConfig, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
||||||
import { CreditSymbol, requestCreditCost } from "@/constant/credits";
|
import { CreditSymbol, requestCreditCost } from "@/constant/credits";
|
||||||
import { canvasThemes } from "@/lib/canvas-theme";
|
import { canvasThemes } from "@/lib/canvas-theme";
|
||||||
|
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
|
||||||
|
import { seedanceReferenceLabel } from "@/lib/seedance-video";
|
||||||
import { useThemeStore } from "@/stores/use-theme-store";
|
import { useThemeStore } from "@/stores/use-theme-store";
|
||||||
import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover";
|
import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover";
|
||||||
import { CanvasVideoSettingsPopover } from "./canvas-video-settings-popover";
|
import { CanvasVideoSettingsPopover } from "./canvas-video-settings-popover";
|
||||||
@@ -18,7 +20,7 @@ import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata } from ".
|
|||||||
type CanvasConfigNodePanelProps = {
|
type CanvasConfigNodePanelProps = {
|
||||||
node: CanvasNodeData;
|
node: CanvasNodeData;
|
||||||
isRunning: boolean;
|
isRunning: boolean;
|
||||||
inputSummary: { textCount: number; imageCount: number };
|
inputSummary: { textCount: number; imageCount: number; videoCount: number; audioCount: number };
|
||||||
inputs: NodeGenerationInput[];
|
inputs: NodeGenerationInput[];
|
||||||
onConfigChange: (nodeId: string, patch: Partial<CanvasNodeMetadata>) => void;
|
onConfigChange: (nodeId: string, patch: Partial<CanvasNodeMetadata>) => void;
|
||||||
onTextInputChange: (nodeId: string, content: string) => void;
|
onTextInputChange: (nodeId: string, content: string) => void;
|
||||||
@@ -41,6 +43,8 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
|
|||||||
const chipStyle = { background: theme.node.fill, borderColor: theme.node.stroke, color: theme.node.text };
|
const chipStyle = { background: theme.node.fill, borderColor: theme.node.stroke, color: theme.node.text };
|
||||||
const textInputs = inputs.filter((input) => input.type === "text");
|
const textInputs = inputs.filter((input) => input.type === "text");
|
||||||
const imageInputs = inputs.filter((input) => input.type === "image");
|
const imageInputs = inputs.filter((input) => input.type === "image");
|
||||||
|
const videoInputs = inputs.filter((input) => input.type === "video");
|
||||||
|
const audioInputs = inputs.filter((input) => input.type === "audio");
|
||||||
|
|
||||||
const moveInput = (input: NodeGenerationInput, offset: number) => {
|
const moveInput = (input: NodeGenerationInput, offset: number) => {
|
||||||
const sameTypeInputs = inputs.filter((item) => item.type === input.type);
|
const sameTypeInputs = inputs.filter((item) => item.type === input.type);
|
||||||
@@ -112,6 +116,8 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
|
|||||||
<div className="mb-2 flex flex-wrap gap-1.5" onMouseDown={(event) => event.stopPropagation()}>
|
<div className="mb-2 flex flex-wrap gap-1.5" onMouseDown={(event) => event.stopPropagation()}>
|
||||||
<InputChip label="提示词" value={`${inputSummary.textCount} 个`} style={chipStyle} />
|
<InputChip label="提示词" value={`${inputSummary.textCount} 个`} style={chipStyle} />
|
||||||
<InputChip label="参考图" value={`${inputSummary.imageCount} 张`} style={chipStyle} />
|
<InputChip label="参考图" value={`${inputSummary.imageCount} 张`} style={chipStyle} />
|
||||||
|
<InputChip label="参考视频" value={`${inputSummary.videoCount} 个`} style={chipStyle} />
|
||||||
|
<InputChip label="参考音频" value={`${inputSummary.audioCount} 个`} style={chipStyle} />
|
||||||
<button type="button" className="inline-flex h-7 cursor-pointer items-center gap-1 rounded-md border px-2 text-[11px]" style={chipStyle} onClick={() => setPreviewOpen(true)}>
|
<button type="button" className="inline-flex h-7 cursor-pointer items-center gap-1 rounded-md border px-2 text-[11px]" style={chipStyle} onClick={() => setPreviewOpen(true)}>
|
||||||
<Eye className="size-3.5" />
|
<Eye className="size-3.5" />
|
||||||
预览
|
预览
|
||||||
@@ -121,7 +127,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
|
|||||||
<div className={`mb-2 grid min-w-0 cursor-default items-center gap-2 ${mode === "text" ? "grid-cols-1" : "grid-cols-[minmax(0,1fr)_148px]"}`} onMouseDown={(event) => event.stopPropagation()}>
|
<div className={`mb-2 grid min-w-0 cursor-default items-center gap-2 ${mode === "text" ? "grid-cols-1" : "grid-cols-[minmax(0,1fr)_148px]"}`} onMouseDown={(event) => event.stopPropagation()}>
|
||||||
<ModelPicker className="canvas-compact-control h-10" config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} fullWidth />
|
<ModelPicker className="canvas-compact-control h-10" config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} fullWidth />
|
||||||
{mode === "video" ? (
|
{mode === "video" ? (
|
||||||
<CanvasVideoSettingsPopover config={config} placement="topRight" buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, key === "videoSeconds" ? { seconds: value } : { [key]: value })} />
|
<CanvasVideoSettingsPopover config={config} placement="topRight" buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, videoConfigPatch(key, value))} />
|
||||||
) : mode === "image" ? (
|
) : mode === "image" ? (
|
||||||
<CanvasImageSettingsPopover config={config} placement="topRight" autoAdjustOverflow={false} buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, key === "count" ? { count: Number(value) || 1 } : { [key]: value })} />
|
<CanvasImageSettingsPopover config={config} placement="topRight" autoAdjustOverflow={false} buttonClassName="canvas-compact-control !h-10 !w-full !justify-start !rounded-lg !px-2" onConfigChange={(key, value) => onConfigChange(node.id, key === "count" ? { count: Number(value) || 1 } : { [key]: value })} />
|
||||||
) : null}
|
) : null}
|
||||||
@@ -130,7 +136,7 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
|
|||||||
<Button
|
<Button
|
||||||
type="primary"
|
type="primary"
|
||||||
className="mt-auto !h-9 !w-full !cursor-pointer !rounded-lg"
|
className="mt-auto !h-9 !w-full !cursor-pointer !rounded-lg"
|
||||||
disabled={isRunning || (!inputSummary.textCount && !inputSummary.imageCount)}
|
disabled={isRunning || (!inputSummary.textCount && !inputSummary.imageCount && !inputSummary.videoCount && !inputSummary.audioCount)}
|
||||||
onMouseDown={(event) => event.stopPropagation()}
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
onClick={() => onGenerate(node.id)}
|
onClick={() => onGenerate(node.id)}
|
||||||
>
|
>
|
||||||
@@ -171,6 +177,24 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
|
|||||||
</div>
|
</div>
|
||||||
</PreviewSection>
|
</PreviewSection>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="shrink-0">
|
||||||
|
<PreviewSection title="参考视频" count={videoInputs.length} empty="暂无参考视频">
|
||||||
|
<div className="thin-scrollbar flex gap-1.5 overflow-x-auto pb-1">
|
||||||
|
{videoInputs.map((input, index) => (
|
||||||
|
<VideoSortCard key={input.nodeId} input={input} videoIndex={index} videoTotal={videoInputs.length} theme={theme} onMove={moveInput} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</PreviewSection>
|
||||||
|
</div>
|
||||||
|
<div className="shrink-0">
|
||||||
|
<PreviewSection title="参考音频" count={audioInputs.length} empty="暂无参考音频">
|
||||||
|
<div className="thin-scrollbar flex gap-1.5 overflow-x-auto pb-1">
|
||||||
|
{audioInputs.map((input, index) => (
|
||||||
|
<AudioSortCard key={input.nodeId} input={input} audioIndex={index} audioTotal={audioInputs.length} theme={theme} onMove={moveInput} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</PreviewSection>
|
||||||
|
</div>
|
||||||
<div className="grid min-h-0 flex-1 grid-cols-2 gap-3 overflow-hidden">
|
<div className="grid min-h-0 flex-1 grid-cols-2 gap-3 overflow-hidden">
|
||||||
<div className="thin-scrollbar min-h-0 overflow-y-auto pr-1.5">
|
<div className="thin-scrollbar min-h-0 overflow-y-auto pr-1.5">
|
||||||
<PreviewSection title="文本提示词" count={textInputs.length} empty="暂无文本提示词">
|
<PreviewSection title="文本提示词" count={textInputs.length} empty="暂无文本提示词">
|
||||||
@@ -281,13 +305,68 @@ function ImageSortCard({
|
|||||||
<div className="w-24 shrink-0 overflow-hidden rounded-lg border" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
|
<div className="w-24 shrink-0 overflow-hidden rounded-lg border" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
|
||||||
<div className="relative">
|
<div className="relative">
|
||||||
<img src={input.image.dataUrl} alt={input.title} className="aspect-square w-full object-cover" />
|
<img src={input.image.dataUrl} alt={input.title} className="aspect-square w-full object-cover" />
|
||||||
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{imageIndex + 1}</span>
|
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{imageReferenceLabel(imageIndex)}</span>
|
||||||
<HorizontalOrderButtons index={imageIndex} total={imageTotal} onMove={(offset) => onMove(input, offset)} />
|
<HorizontalOrderButtons index={imageIndex} total={imageTotal} onMove={(offset) => onMove(input, offset)} />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function VideoSortCard({
|
||||||
|
input,
|
||||||
|
videoIndex,
|
||||||
|
videoTotal,
|
||||||
|
theme,
|
||||||
|
onMove,
|
||||||
|
}: {
|
||||||
|
input: NodeGenerationInput;
|
||||||
|
videoIndex: number;
|
||||||
|
videoTotal: number;
|
||||||
|
theme: (typeof canvasThemes)[keyof typeof canvasThemes];
|
||||||
|
onMove: (input: NodeGenerationInput, offset: number) => void;
|
||||||
|
}) {
|
||||||
|
if (!input.video) return null;
|
||||||
|
return (
|
||||||
|
<div className="w-32 shrink-0 overflow-hidden rounded-lg border" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
|
||||||
|
<div className="relative">
|
||||||
|
<video src={input.video.url} className="aspect-video w-full bg-black object-cover" muted preload="metadata" />
|
||||||
|
<span className="absolute left-1 top-1 rounded bg-black/50 px-1 py-0.5 text-[9px] font-medium text-white">{seedanceReferenceLabel("video", videoIndex)}</span>
|
||||||
|
<HorizontalOrderButtons index={videoIndex} total={videoTotal} onMove={(offset) => onMove(input, offset)} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function AudioSortCard({
|
||||||
|
input,
|
||||||
|
audioIndex,
|
||||||
|
audioTotal,
|
||||||
|
theme,
|
||||||
|
onMove,
|
||||||
|
}: {
|
||||||
|
input: NodeGenerationInput;
|
||||||
|
audioIndex: number;
|
||||||
|
audioTotal: number;
|
||||||
|
theme: (typeof canvasThemes)[keyof typeof canvasThemes];
|
||||||
|
onMove: (input: NodeGenerationInput, offset: number) => void;
|
||||||
|
}) {
|
||||||
|
if (!input.audio) return null;
|
||||||
|
return (
|
||||||
|
<div className="w-48 shrink-0 rounded-lg border p-2" style={{ background: theme.node.fill, borderColor: theme.node.stroke }}>
|
||||||
|
<div className="mb-1.5 flex min-w-0 items-center gap-1.5 text-[11px] opacity-70">
|
||||||
|
<Music2 className="size-3.5 shrink-0" />
|
||||||
|
<span className="truncate">{input.title}</span>
|
||||||
|
</div>
|
||||||
|
<audio src={input.audio.url} controls className="h-8 w-full" preload="metadata" />
|
||||||
|
<div className="mt-1 flex justify-between">
|
||||||
|
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !p-0" icon={<ArrowLeft className="size-3" />} disabled={audioIndex <= 0} onClick={() => onMove(input, -1)} />
|
||||||
|
<span className="text-[10px] opacity-45">{seedanceReferenceLabel("audio", audioIndex)}</span>
|
||||||
|
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !p-0" icon={<ArrowRight className="size-3" />} disabled={audioIndex >= audioTotal - 1} onClick={() => onMove(input, 1)} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function VerticalOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) {
|
function VerticalOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) {
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
@@ -324,6 +403,15 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can
|
|||||||
size: node.metadata?.size || globalConfig.size || defaultConfig.size,
|
size: node.metadata?.size || globalConfig.size || defaultConfig.size,
|
||||||
videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds,
|
videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds,
|
||||||
vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality,
|
vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality,
|
||||||
|
videoGenerateAudio: node.metadata?.generateAudio || globalConfig.videoGenerateAudio || defaultConfig.videoGenerateAudio,
|
||||||
|
videoWatermark: node.metadata?.watermark || globalConfig.videoWatermark || defaultConfig.videoWatermark,
|
||||||
count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count),
|
count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function videoConfigPatch(key: keyof AiConfig, value: string) {
|
||||||
|
if (key === "videoSeconds") return { seconds: value };
|
||||||
|
if (key === "videoGenerateAudio") return { generateAudio: value };
|
||||||
|
if (key === "videoWatermark") return { watermark: value };
|
||||||
|
return { [key]: value };
|
||||||
|
}
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ export function Minimap({ nodes, viewport, viewportSize, onViewportChange }: { n
|
|||||||
>
|
>
|
||||||
{nodes.map((node) => {
|
{nodes.map((node) => {
|
||||||
const pos = toMinimap(node.position.x, node.position.y);
|
const pos = toMinimap(node.position.x, node.position.y);
|
||||||
const color = node.type === CanvasNodeType.Image ? "#10b981" : node.type === CanvasNodeType.Config ? "#60a5fa" : theme.node.muted;
|
const color = node.type === CanvasNodeType.Image ? "#10b981" : node.type === CanvasNodeType.Video ? "#f97316" : node.type === CanvasNodeType.Audio ? "#a855f7" : node.type === CanvasNodeType.Config ? "#60a5fa" : theme.node.muted;
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
key={node.id}
|
key={node.id}
|
||||||
|
|||||||
@@ -1,20 +1,27 @@
|
|||||||
import type { ChatCompletionMessage } from "@/services/api/image";
|
import type { ChatCompletionMessage } from "@/services/api/image";
|
||||||
import type { ReferenceImage } from "@/types/image";
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||||
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../types";
|
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../types";
|
||||||
|
|
||||||
export type NodeGenerationContext = {
|
export type NodeGenerationContext = {
|
||||||
prompt: string;
|
prompt: string;
|
||||||
referenceImages: ReferenceImage[];
|
referenceImages: ReferenceImage[];
|
||||||
|
referenceVideos: ReferenceVideo[];
|
||||||
|
referenceAudios: ReferenceAudio[];
|
||||||
textCount: number;
|
textCount: number;
|
||||||
imageCount: number;
|
imageCount: number;
|
||||||
|
videoCount: number;
|
||||||
|
audioCount: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type NodeGenerationInput = {
|
export type NodeGenerationInput = {
|
||||||
nodeId: string;
|
nodeId: string;
|
||||||
type: "text" | "image" | "video";
|
type: "text" | "image" | "video" | "audio";
|
||||||
title: string;
|
title: string;
|
||||||
text?: string;
|
text?: string;
|
||||||
image?: ReferenceImage;
|
image?: ReferenceImage;
|
||||||
|
video?: ReferenceVideo;
|
||||||
|
audio?: ReferenceAudio;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string): NodeGenerationContext {
|
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string): NodeGenerationContext {
|
||||||
@@ -24,12 +31,18 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
|
|||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join("\n\n");
|
.join("\n\n");
|
||||||
const referenceImages = inputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
|
const referenceImages = inputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
|
||||||
|
const referenceVideos = inputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video));
|
||||||
|
const referenceAudios = inputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
prompt: upstreamText ? `${prompt}\n\n${upstreamText}` : prompt,
|
prompt: upstreamText ? `${prompt}\n\n${upstreamText}` : prompt,
|
||||||
referenceImages,
|
referenceImages,
|
||||||
|
referenceVideos,
|
||||||
|
referenceAudios,
|
||||||
textCount: inputs.filter((input) => input.type === "text").length,
|
textCount: inputs.filter((input) => input.type === "text").length,
|
||||||
imageCount: referenceImages.length,
|
imageCount: referenceImages.length,
|
||||||
|
videoCount: referenceVideos.length,
|
||||||
|
audioCount: referenceAudios.length,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -37,6 +50,10 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[
|
|||||||
return getOrderedUpstreamNodes(nodeId, nodes, connections).flatMap((node): NodeGenerationInput[] => {
|
return getOrderedUpstreamNodes(nodeId, nodes, connections).flatMap((node): NodeGenerationInput[] => {
|
||||||
const image = readReferenceImage(node);
|
const image = readReferenceImage(node);
|
||||||
if (image) return [{ nodeId: node.id, type: "image" as const, title: node.title, image }];
|
if (image) return [{ nodeId: node.id, type: "image" as const, title: node.title, image }];
|
||||||
|
const video = readReferenceVideo(node);
|
||||||
|
if (video) return [{ nodeId: node.id, type: "video" as const, title: node.title, video }];
|
||||||
|
const audio = readReferenceAudio(node);
|
||||||
|
if (audio) return [{ nodeId: node.id, type: "audio" as const, title: node.title, audio }];
|
||||||
const text = readNodeTextInput(node);
|
const text = readNodeTextInput(node);
|
||||||
if (text) return [{ nodeId: node.id, type: "text" as const, title: node.title, text }];
|
if (text) return [{ nodeId: node.id, type: "text" as const, title: node.title, text }];
|
||||||
return [];
|
return [];
|
||||||
@@ -77,6 +94,33 @@ function readReferenceImage(node: CanvasNodeData): ReferenceImage | null {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readReferenceVideo(node: CanvasNodeData): ReferenceVideo | null {
|
||||||
|
if (node.type !== CanvasNodeType.Video || !node.metadata?.content) return null;
|
||||||
|
return {
|
||||||
|
id: node.id,
|
||||||
|
name: `${node.title || node.id}.mp4`,
|
||||||
|
type: node.metadata.mimeType || "video/mp4",
|
||||||
|
url: node.metadata.content,
|
||||||
|
storageKey: node.metadata.storageKey,
|
||||||
|
bytes: node.metadata.bytes,
|
||||||
|
width: node.metadata.naturalWidth,
|
||||||
|
height: node.metadata.naturalHeight,
|
||||||
|
durationMs: node.metadata.durationMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readReferenceAudio(node: CanvasNodeData): ReferenceAudio | null {
|
||||||
|
if (node.type !== CanvasNodeType.Audio || !node.metadata?.content) return null;
|
||||||
|
return {
|
||||||
|
id: node.id,
|
||||||
|
name: `${node.title || node.id}.mp3`,
|
||||||
|
type: node.metadata.mimeType || "audio/mpeg",
|
||||||
|
url: node.metadata.content,
|
||||||
|
storageKey: node.metadata.storageKey,
|
||||||
|
durationMs: node.metadata.durationMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function getOrderedUpstreamNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
|
function getOrderedUpstreamNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {
|
||||||
const target = nodes.find((node) => node.id === nodeId);
|
const target = nodes.find((node) => node.id === nodeId);
|
||||||
const upstreamNodes = connections
|
const upstreamNodes = connections
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import { useEffect, useMemo, useState, type ReactNode } from "react";
|
import { useEffect, useMemo, useState, type ReactNode } from "react";
|
||||||
import { Modal, Segmented, Tooltip } from "antd";
|
import { Modal, Segmented, Tooltip } from "antd";
|
||||||
import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react";
|
import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Music2, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react";
|
||||||
|
|
||||||
import { canvasThemes } from "@/lib/canvas-theme";
|
import { canvasThemes } from "@/lib/canvas-theme";
|
||||||
import { formatBytes, getDataUrlByteSize } from "@/lib/image-utils";
|
import { formatBytes, getDataUrlByteSize } from "@/lib/image-utils";
|
||||||
@@ -58,13 +58,15 @@ export function CanvasNodeHoverToolbar({
|
|||||||
const top = viewport.y + node.position.y * viewport.k - 14;
|
const top = viewport.y + node.position.y * viewport.k - 14;
|
||||||
const isImage = node.type === CanvasNodeType.Image;
|
const isImage = node.type === CanvasNodeType.Image;
|
||||||
const isVideo = node.type === CanvasNodeType.Video;
|
const isVideo = node.type === CanvasNodeType.Video;
|
||||||
|
const isAudio = node.type === CanvasNodeType.Audio;
|
||||||
const hasImage = isImage && Boolean(node.metadata?.content);
|
const hasImage = isImage && Boolean(node.metadata?.content);
|
||||||
const hasVideo = isVideo && Boolean(node.metadata?.content);
|
const hasVideo = isVideo && Boolean(node.metadata?.content);
|
||||||
|
const hasAudio = isAudio && Boolean(node.metadata?.content);
|
||||||
const isText = node.type === CanvasNodeType.Text;
|
const isText = node.type === CanvasNodeType.Text;
|
||||||
const isConfig = node.type === CanvasNodeType.Config;
|
const isConfig = node.type === CanvasNodeType.Config;
|
||||||
const canOpenDialog = isText || hasImage || isVideo;
|
const canOpenDialog = isText || hasImage || isVideo;
|
||||||
const canRetry = node.metadata?.status === "error";
|
const canRetry = node.metadata?.status === "error";
|
||||||
const hasSpecificTools = canRetry || isText || isImage || isVideo || isConfig;
|
const hasSpecificTools = canRetry || isText || isImage || isVideo || isAudio || isConfig;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
@@ -80,7 +82,7 @@ export function CanvasNodeHoverToolbar({
|
|||||||
{hasSpecificTools ? <ToolbarDivider /> : null}
|
{hasSpecificTools ? <ToolbarDivider /> : null}
|
||||||
{canRetry ? <ToolbarAction title="重新生成" label="重试" icon={<RefreshCw className="size-4" />} onClick={() => onRetry(node)} /> : null}
|
{canRetry ? <ToolbarAction title="重新生成" label="重试" icon={<RefreshCw className="size-4" />} onClick={() => onRetry(node)} /> : null}
|
||||||
{hasImage || hasVideo || isText ? <ToolbarAction title="加入我的素材" label="存素材" icon={<FolderPlus className="size-4" />} onClick={() => onSaveAsset(node)} /> : null}
|
{hasImage || hasVideo || isText ? <ToolbarAction title="加入我的素材" label="存素材" icon={<FolderPlus className="size-4" />} onClick={() => onSaveAsset(node)} /> : null}
|
||||||
{hasImage || hasVideo ? <IconAction title={hasVideo ? "下载视频" : "下载图片"} icon={<Download className="size-5" />} onClick={() => onDownload(node)} /> : null}
|
{hasImage || hasVideo || hasAudio ? <IconAction title={hasAudio ? "下载音频" : hasVideo ? "下载视频" : "下载图片"} icon={<Download className="size-5" />} onClick={() => onDownload(node)} /> : null}
|
||||||
{canOpenDialog ? <ToolbarAction title="编辑" label="编辑" icon={<MessageSquare className="size-4" />} onClick={() => onToggleDialog(node)} /> : null}
|
{canOpenDialog ? <ToolbarAction title="编辑" label="编辑" icon={<MessageSquare className="size-4" />} onClick={() => onToggleDialog(node)} /> : null}
|
||||||
{isText ? <ToolbarAction title="编辑文本" label="编辑文字" icon={<Pencil className="size-4" />} onClick={() => onEditText(node)} /> : null}
|
{isText ? <ToolbarAction title="编辑文本" label="编辑文字" icon={<Pencil className="size-4" />} onClick={() => onEditText(node)} /> : null}
|
||||||
{isText ? <ToolbarAction title="用文本生图" label="生图" icon={<ImageIcon className="size-4" />} onClick={() => onGenerateImage(node)} /> : null}
|
{isText ? <ToolbarAction title="用文本生图" label="生图" icon={<ImageIcon className="size-4" />} onClick={() => onGenerateImage(node)} /> : null}
|
||||||
@@ -89,6 +91,7 @@ export function CanvasNodeHoverToolbar({
|
|||||||
{isText ? <ToolbarAction title="增大字号" label="放大" icon={<Plus className="size-4" />} onClick={() => onIncreaseFont(node)} /> : null}
|
{isText ? <ToolbarAction title="增大字号" label="放大" icon={<Plus className="size-4" />} onClick={() => onIncreaseFont(node)} /> : null}
|
||||||
{isImage ? <ToolbarAction title={hasImage ? "替换图片" : "上传图片"} label={hasImage ? "替换图片" : "上传图片"} icon={<Upload className="size-4" />} onClick={() => onUpload(node)} /> : null}
|
{isImage ? <ToolbarAction title={hasImage ? "替换图片" : "上传图片"} label={hasImage ? "替换图片" : "上传图片"} icon={<Upload className="size-4" />} onClick={() => onUpload(node)} /> : null}
|
||||||
{isVideo ? <ToolbarAction title={hasVideo ? "替换视频" : "上传视频"} label={hasVideo ? "替换视频" : "上传视频"} icon={<Video className="size-4" />} onClick={() => onUpload(node)} /> : null}
|
{isVideo ? <ToolbarAction title={hasVideo ? "替换视频" : "上传视频"} label={hasVideo ? "替换视频" : "上传视频"} icon={<Video className="size-4" />} onClick={() => onUpload(node)} /> : null}
|
||||||
|
{isAudio ? <ToolbarAction title={hasAudio ? "替换音频" : "上传音频"} label={hasAudio ? "替换音频" : "上传音频"} icon={<Music2 className="size-4" />} onClick={() => onUpload(node)} /> : null}
|
||||||
{hasImage ? (
|
{hasImage ? (
|
||||||
<ToolbarAction
|
<ToolbarAction
|
||||||
title={node.metadata?.freeResize ? "切换为等比缩放" : "切换为自由比例"}
|
title={node.metadata?.freeResize ? "切换为等比缩放" : "切换为自由比例"}
|
||||||
@@ -151,7 +154,7 @@ export function CanvasNodeInfoModal({ node, open, onClose }: { node: CanvasNodeD
|
|||||||
{view === "info" ? (
|
{view === "info" ? (
|
||||||
<div className="thin-scrollbar h-full space-y-3 overflow-auto pr-1">
|
<div className="thin-scrollbar h-full space-y-3 overflow-auto pr-1">
|
||||||
<InfoRow label="ID" value={node.id} />
|
<InfoRow label="ID" value={node.id} />
|
||||||
<InfoRow label="类型" value={node.type === CanvasNodeType.Text ? "文本" : node.type === CanvasNodeType.Image ? "图片" : node.type === CanvasNodeType.Video ? "视频" : "生成配置"} />
|
<InfoRow label="类型" value={node.type === CanvasNodeType.Text ? "文本" : node.type === CanvasNodeType.Image ? "图片" : node.type === CanvasNodeType.Video ? "视频" : node.type === CanvasNodeType.Audio ? "音频" : "生成配置"} />
|
||||||
<InfoRow label="尺寸" value={`${Math.round(node.width)} x ${Math.round(node.height)}`} />
|
<InfoRow label="尺寸" value={`${Math.round(node.width)} x ${Math.round(node.height)}`} />
|
||||||
<InfoRow label="位置" value={`${Math.round(node.position.x)}, ${Math.round(node.position.y)}`} />
|
<InfoRow label="位置" value={`${Math.round(node.position.x)}, ${Math.round(node.position.y)}`} />
|
||||||
<InfoRow label="状态" value={node.metadata?.status || "idle"} />
|
<InfoRow label="状态" value={node.metadata?.status || "idle"} />
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi
|
|||||||
) : mode === "video" ? (
|
) : mode === "video" ? (
|
||||||
<>
|
<>
|
||||||
<ModelPicker config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} />
|
<ModelPicker config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} />
|
||||||
<CanvasVideoSettingsPopover config={config} buttonClassName="!h-10 !max-w-[170px] !justify-start !rounded-full !px-3" onConfigChange={(key, value) => onConfigChange(node.id, key === "videoSeconds" ? { seconds: value } : { [key]: value })} />
|
<CanvasVideoSettingsPopover config={config} buttonClassName="!h-10 !max-w-[170px] !justify-start !rounded-full !px-3" onConfigChange={(key, value) => onConfigChange(node.id, videoConfigPatch(key, value))} />
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (
|
||||||
<ModelPicker config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} />
|
<ModelPicker config={config} value={config.model} onChange={(model) => onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} />
|
||||||
@@ -132,6 +132,15 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can
|
|||||||
size: node.metadata?.size || globalConfig.size || defaultConfig.size,
|
size: node.metadata?.size || globalConfig.size || defaultConfig.size,
|
||||||
videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds,
|
videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds,
|
||||||
vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality,
|
vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality,
|
||||||
|
videoGenerateAudio: node.metadata?.generateAudio || globalConfig.videoGenerateAudio || defaultConfig.videoGenerateAudio,
|
||||||
|
videoWatermark: node.metadata?.watermark || globalConfig.videoWatermark || defaultConfig.videoWatermark,
|
||||||
count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count),
|
count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function videoConfigPatch(key: keyof AiConfig, value: string) {
|
||||||
|
if (key === "videoSeconds") return { seconds: value };
|
||||||
|
if (key === "videoGenerateAudio") return { generateAudio: value };
|
||||||
|
if (key === "videoWatermark") return { watermark: value };
|
||||||
|
return { [key]: value };
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import React, { useCallback, useEffect, useRef, useState } from "react";
|
import React, { useCallback, useEffect, useRef, useState } from "react";
|
||||||
import type { ReactNode } from "react";
|
import type { ReactNode } from "react";
|
||||||
import { ChevronRight, Image as ImageIcon, RefreshCw, Star, Video } from "lucide-react";
|
import { ChevronRight, Image as ImageIcon, Music2, RefreshCw, Star, Video } from "lucide-react";
|
||||||
|
|
||||||
import { canvasThemes } from "@/lib/canvas-theme";
|
import { canvasThemes } from "@/lib/canvas-theme";
|
||||||
import { formatBytes } from "@/lib/image-utils";
|
import { formatBytes } from "@/lib/image-utils";
|
||||||
@@ -99,6 +99,7 @@ export const CanvasNode = React.memo(function CanvasNode({
|
|||||||
const [isEditingContent, setIsEditingContent] = useState(false);
|
const [isEditingContent, setIsEditingContent] = useState(false);
|
||||||
const hasImageContent = data.type === CanvasNodeType.Image && Boolean(data.metadata?.content);
|
const hasImageContent = data.type === CanvasNodeType.Image && Boolean(data.metadata?.content);
|
||||||
const hasVideoContent = data.type === CanvasNodeType.Video && Boolean(data.metadata?.content);
|
const hasVideoContent = data.type === CanvasNodeType.Video && Boolean(data.metadata?.content);
|
||||||
|
const hasAudioContent = data.type === CanvasNodeType.Audio && Boolean(data.metadata?.content);
|
||||||
const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1;
|
const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1;
|
||||||
const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId);
|
const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId);
|
||||||
const isActive = isConnectionTarget || isSelected || isFocusRelated;
|
const isActive = isConnectionTarget || isSelected || isFocusRelated;
|
||||||
@@ -301,7 +302,7 @@ export const CanvasNode = React.memo(function CanvasNode({
|
|||||||
|
|
||||||
{showImageInfo && hasImageContent ? <ImageInfoBar node={data} /> : null}
|
{showImageInfo && hasImageContent ? <ImageInfoBar node={data} /> : null}
|
||||||
|
|
||||||
{!hasImageContent && !hasVideoContent ? <div className="pointer-events-none absolute inset-x-0 bottom-0 h-12" style={{ background: `linear-gradient(to top, ${theme.canvas.background}66, transparent)` }} /> : null}
|
{!hasImageContent && !hasVideoContent && !hasAudioContent ? <div className="pointer-events-none absolute inset-x-0 bottom-0 h-12" style={{ background: `linear-gradient(to top, ${theme.canvas.background}66, transparent)` }} /> : null}
|
||||||
|
|
||||||
<ResizeHandle corner="top-left" onMouseDown={handleResizeMouseDown} />
|
<ResizeHandle corner="top-left" onMouseDown={handleResizeMouseDown} />
|
||||||
<ResizeHandle corner="top-right" onMouseDown={handleResizeMouseDown} />
|
<ResizeHandle corner="top-right" onMouseDown={handleResizeMouseDown} />
|
||||||
@@ -332,6 +333,7 @@ const nodeContentRenderers = {
|
|||||||
[CanvasNodeType.Image]: ImageNodeContent,
|
[CanvasNodeType.Image]: ImageNodeContent,
|
||||||
[CanvasNodeType.Config]: EmptyImageContent,
|
[CanvasNodeType.Config]: EmptyImageContent,
|
||||||
[CanvasNodeType.Video]: VideoNodeContent,
|
[CanvasNodeType.Video]: VideoNodeContent,
|
||||||
|
[CanvasNodeType.Audio]: AudioNodeContent,
|
||||||
} satisfies Record<CanvasNodeType, (props: NodeContentRendererProps) => ReactNode>;
|
} satisfies Record<CanvasNodeType, (props: NodeContentRendererProps) => ReactNode>;
|
||||||
|
|
||||||
function LoadingContent({ theme }: Pick<NodeContentRendererProps, "theme">) {
|
function LoadingContent({ theme }: Pick<NodeContentRendererProps, "theme">) {
|
||||||
@@ -466,12 +468,31 @@ function VideoNodeContent({ node, theme }: NodeContentRendererProps) {
|
|||||||
return (
|
return (
|
||||||
<div className="flex h-full w-full flex-col items-center justify-center gap-3" style={{ color: theme.node.placeholder }}>
|
<div className="flex h-full w-full flex-col items-center justify-center gap-3" style={{ color: theme.node.placeholder }}>
|
||||||
<Video className="size-7 opacity-35" />
|
<Video className="size-7 opacity-35" />
|
||||||
<span className="text-sm">视频节点(未开发完,请勿使用)</span>
|
<span className="text-sm">空视频节点</span>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
return <video src={node.metadata.content} controls className="h-full w-full rounded-[18px] bg-black object-contain" data-canvas-no-zoom />;
|
return <video src={node.metadata.content} controls className="h-full w-full rounded-[18px] bg-black object-contain" data-canvas-no-zoom />;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function AudioNodeContent({ node, theme }: NodeContentRendererProps) {
|
||||||
|
if (!node.metadata?.content)
|
||||||
|
return (
|
||||||
|
<div className="flex h-full w-full flex-col items-center justify-center gap-2" style={{ color: theme.node.placeholder }}>
|
||||||
|
<Music2 className="size-7 opacity-35" />
|
||||||
|
<span className="text-sm">空音频节点</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
<div className="flex h-full w-full flex-col justify-center gap-3 px-4" style={{ background: theme.node.fill, color: theme.node.text }}>
|
||||||
|
<div className="flex min-w-0 items-center gap-2 text-sm opacity-70">
|
||||||
|
<Music2 className="size-4 shrink-0" />
|
||||||
|
<span className="truncate">{node.title || "音频"}</span>
|
||||||
|
</div>
|
||||||
|
<audio src={node.metadata.content} controls className="w-full" data-canvas-no-zoom />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function ImageContent({
|
function ImageContent({
|
||||||
node,
|
node,
|
||||||
isBatchRoot,
|
isBatchRoot,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactNode, RefObject } from "react";
|
import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactNode, RefObject } from "react";
|
||||||
import { useRef, useState } from "react";
|
import { useRef, useState } from "react";
|
||||||
import { Button, Segmented, Switch } from "antd";
|
import { Button, Segmented, Switch } from "antd";
|
||||||
import { CircleDot, Eraser, FolderOpen, Grid2x2, Hand, Image as ImageIcon, Info, Library, Moon, Palette, Redo2, Settings2, Square, Sun, Trash2, Type, Undo2, Upload, Video } from "lucide-react";
|
import { CircleDot, Eraser, FolderOpen, Grid2x2, Hand, Image as ImageIcon, Info, Library, Moon, Music2, Palette, Redo2, Settings2, Square, Sun, Trash2, Type, Undo2, Upload, Video } from "lucide-react";
|
||||||
|
|
||||||
import { canvasThemes, type CanvasBackgroundMode, type CanvasColorTheme, type CanvasTheme } from "@/lib/canvas-theme";
|
import { canvasThemes, type CanvasBackgroundMode, type CanvasColorTheme, type CanvasTheme } from "@/lib/canvas-theme";
|
||||||
import { useThemeStore } from "@/stores/use-theme-store";
|
import { useThemeStore } from "@/stores/use-theme-store";
|
||||||
@@ -15,6 +15,7 @@ export function CanvasToolbar({
|
|||||||
showImageInfo,
|
showImageInfo,
|
||||||
onAddImage,
|
onAddImage,
|
||||||
onAddVideo,
|
onAddVideo,
|
||||||
|
onAddAudio,
|
||||||
onAddText,
|
onAddText,
|
||||||
onAddConfig,
|
onAddConfig,
|
||||||
onUndo,
|
onUndo,
|
||||||
@@ -35,6 +36,7 @@ export function CanvasToolbar({
|
|||||||
showImageInfo: boolean;
|
showImageInfo: boolean;
|
||||||
onAddImage: () => void;
|
onAddImage: () => void;
|
||||||
onAddVideo: () => void;
|
onAddVideo: () => void;
|
||||||
|
onAddAudio: () => void;
|
||||||
onAddText: () => void;
|
onAddText: () => void;
|
||||||
onAddConfig: () => void;
|
onAddConfig: () => void;
|
||||||
onUndo: () => void;
|
onUndo: () => void;
|
||||||
@@ -84,10 +86,13 @@ export function CanvasToolbar({
|
|||||||
<ToolbarButton id="tool-video" label="视频" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddVideo}>
|
<ToolbarButton id="tool-video" label="视频" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddVideo}>
|
||||||
<Video className="size-4.5" />
|
<Video className="size-4.5" />
|
||||||
</ToolbarButton>
|
</ToolbarButton>
|
||||||
|
<ToolbarButton id="tool-audio" label="音频" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddAudio}>
|
||||||
|
<Music2 className="size-4.5" />
|
||||||
|
</ToolbarButton>
|
||||||
<ToolbarButton id="tool-config" label="生成配置" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddConfig}>
|
<ToolbarButton id="tool-config" label="生成配置" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onAddConfig}>
|
||||||
<Settings2 className="size-4.5" />
|
<Settings2 className="size-4.5" />
|
||||||
</ToolbarButton>
|
</ToolbarButton>
|
||||||
<ToolbarButton id="tool-upload" label="上传图片" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onUpload}>
|
<ToolbarButton id="tool-upload" label="上传素材" hovered={hovered} hoverStyle={hoverStyle} wrapRef={wrapRef} onTipX={setTipX} onHover={setHovered} onClick={onUpload}>
|
||||||
<Upload className="size-4.5" />
|
<Upload className="size-4.5" />
|
||||||
</ToolbarButton>
|
</ToolbarButton>
|
||||||
<Divider theme={theme} />
|
<Divider theme={theme} />
|
||||||
@@ -279,8 +284,9 @@ function toolLabel(id: string) {
|
|||||||
if (id === "tool-text") return "文本";
|
if (id === "tool-text") return "文本";
|
||||||
if (id === "tool-image") return "图片";
|
if (id === "tool-image") return "图片";
|
||||||
if (id === "tool-video") return "视频";
|
if (id === "tool-video") return "视频";
|
||||||
|
if (id === "tool-audio") return "音频";
|
||||||
if (id === "tool-config") return "生成配置";
|
if (id === "tool-config") return "生成配置";
|
||||||
if (id === "tool-upload") return "上传图片";
|
if (id === "tool-upload") return "上传素材";
|
||||||
if (id === "tool-library") return "素材库";
|
if (id === "tool-library") return "素材库";
|
||||||
if (id === "tool-assets") return "我的素材";
|
if (id === "tool-assets") return "我的素材";
|
||||||
if (id === "tool-style") return "画布外观";
|
if (id === "tool-style") return "画布外观";
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ export const NODE_DEFAULT_SIZE = {
|
|||||||
[CanvasNodeType.Text]: { width: 340, height: 240, title: "Note" },
|
[CanvasNodeType.Text]: { width: 340, height: 240, title: "Note" },
|
||||||
[CanvasNodeType.Config]: { width: 340, height: 240, title: "生成配置" },
|
[CanvasNodeType.Config]: { width: 340, height: 240, title: "生成配置" },
|
||||||
[CanvasNodeType.Video]: { width: 420, height: 236, title: "Video" },
|
[CanvasNodeType.Video]: { width: 420, height: 236, title: "Video" },
|
||||||
|
[CanvasNodeType.Audio]: { width: 340, height: 120, title: "Audio" },
|
||||||
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
|
} satisfies Record<CanvasNodeType, { width: number; height: number; title: string }>;
|
||||||
|
|
||||||
export const NODE_SPECS = {
|
export const NODE_SPECS = {
|
||||||
@@ -32,6 +33,10 @@ export const NODE_SPECS = {
|
|||||||
...NODE_DEFAULT_SIZE[CanvasNodeType.Video],
|
...NODE_DEFAULT_SIZE[CanvasNodeType.Video],
|
||||||
metadata: { content: "", status: "idle" },
|
metadata: { content: "", status: "idle" },
|
||||||
},
|
},
|
||||||
|
[CanvasNodeType.Audio]: {
|
||||||
|
...NODE_DEFAULT_SIZE[CanvasNodeType.Audio],
|
||||||
|
metadata: { content: "", status: "idle" },
|
||||||
|
},
|
||||||
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
|
} satisfies Record<CanvasNodeType, CanvasNodeSpec>;
|
||||||
|
|
||||||
export function getNodeSpec(type: CanvasNodeType) {
|
export function getNodeSpec(type: CanvasNodeType) {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ export enum CanvasNodeType {
|
|||||||
Text = "text",
|
Text = "text",
|
||||||
Config = "config",
|
Config = "config",
|
||||||
Video = "video",
|
Video = "video",
|
||||||
|
Audio = "audio",
|
||||||
}
|
}
|
||||||
|
|
||||||
export type CanvasNodeStatus = "idle" | "success" | "loading" | "error";
|
export type CanvasNodeStatus = "idle" | "success" | "loading" | "error";
|
||||||
@@ -34,6 +35,8 @@ export type CanvasNodeMetadata = {
|
|||||||
count?: number;
|
count?: number;
|
||||||
seconds?: string;
|
seconds?: string;
|
||||||
vquality?: string;
|
vquality?: string;
|
||||||
|
generateAudio?: string;
|
||||||
|
watermark?: string;
|
||||||
references?: string[];
|
references?: string[];
|
||||||
naturalWidth?: number;
|
naturalWidth?: number;
|
||||||
naturalHeight?: number;
|
naturalHeight?: number;
|
||||||
@@ -48,6 +51,7 @@ export type CanvasNodeMetadata = {
|
|||||||
storageKey?: string;
|
storageKey?: string;
|
||||||
mimeType?: string;
|
mimeType?: string;
|
||||||
bytes?: number;
|
bytes?: number;
|
||||||
|
durationMs?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type CanvasNodeData = {
|
export type CanvasNodeData = {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, ImagePlus, LoaderCircle, PenLine, Plus, SlidersHorizontal, Sparkles, Trash2, Upload } from "lucide-react";
|
import { ArrowLeft, ArrowRight, BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, ImagePlus, LoaderCircle, PenLine, Plus, SlidersHorizontal, Sparkles, Trash2, Upload } from "lucide-react";
|
||||||
import { useEffect, useRef, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { App, Button, Checkbox, Drawer, Empty, Image, Input, Modal, Tag, Typography } from "antd";
|
import { App, Button, Checkbox, Drawer, Empty, Image, Input, Modal, Tag, Typography } from "antd";
|
||||||
import localforage from "localforage";
|
import localforage from "localforage";
|
||||||
@@ -11,6 +11,7 @@ import { ModelPicker } from "@/components/model-picker";
|
|||||||
import { PromptSelectDialog } from "@/components/prompts/prompt-select-dialog";
|
import { PromptSelectDialog } from "@/components/prompts/prompt-select-dialog";
|
||||||
import { AssetPickerModal, type InsertAssetPayload } from "@/app/(user)/canvas/components/asset-picker-modal";
|
import { AssetPickerModal, type InsertAssetPayload } from "@/app/(user)/canvas/components/asset-picker-modal";
|
||||||
import { canvasThemes } from "@/lib/canvas-theme";
|
import { canvasThemes } from "@/lib/canvas-theme";
|
||||||
|
import { imageReferenceLabel } from "@/lib/image-reference-prompt";
|
||||||
import { useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
import { useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
||||||
import { useThemeStore } from "@/stores/use-theme-store";
|
import { useThemeStore } from "@/stores/use-theme-store";
|
||||||
import { nanoid } from "nanoid";
|
import { nanoid } from "nanoid";
|
||||||
@@ -218,9 +219,11 @@ export default function ImagePage() {
|
|||||||
const insertPickedAsset = async (payload: InsertAssetPayload) => {
|
const insertPickedAsset = async (payload: InsertAssetPayload) => {
|
||||||
if (payload.kind === "text") {
|
if (payload.kind === "text") {
|
||||||
setPrompt(payload.content);
|
setPrompt(payload.content);
|
||||||
} else {
|
} else if (payload.kind === "image") {
|
||||||
const stored = await uploadImage(payload.dataUrl);
|
const stored = await uploadImage(payload.dataUrl);
|
||||||
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }]);
|
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }]);
|
||||||
|
} else {
|
||||||
|
message.warning("生图工作台只能使用文本或图片素材");
|
||||||
}
|
}
|
||||||
setAssetPickerOpen(false);
|
setAssetPickerOpen(false);
|
||||||
};
|
};
|
||||||
@@ -371,9 +374,11 @@ export default function ImagePage() {
|
|||||||
event.currentTarget.scrollLeft += event.deltaY;
|
event.currentTarget.scrollLeft += event.deltaY;
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{references.map((item) => (
|
{references.map((item, index) => (
|
||||||
<div key={item.id} className="group relative size-20 shrink-0 overflow-hidden rounded-md border border-stone-200 dark:border-stone-800">
|
<div key={item.id} className="group relative size-20 shrink-0 overflow-hidden rounded-md border border-stone-200 dark:border-stone-800">
|
||||||
<img src={item.dataUrl} alt={item.name} className="size-full object-cover" />
|
<img src={item.dataUrl} alt={item.name} className="size-full object-cover" />
|
||||||
|
<span className="absolute left-1 top-1 rounded bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">{imageReferenceLabel(index)}</span>
|
||||||
|
<ReferenceOrderButtons index={index} total={references.length} onMove={(offset) => setReferences((value) => moveListItem(value, index, offset))} />
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex"
|
className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex"
|
||||||
@@ -740,6 +745,24 @@ function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function moveListItem<T>(items: T[], index: number, offset: number) {
|
||||||
|
const targetIndex = index + offset;
|
||||||
|
if (targetIndex < 0 || targetIndex >= items.length) return items;
|
||||||
|
const next = [...items];
|
||||||
|
[next[index], next[targetIndex]] = [next[targetIndex], next[index]];
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ReferenceOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) {
|
||||||
|
if (total <= 1) return null;
|
||||||
|
return (
|
||||||
|
<div className="absolute inset-x-1 bottom-1 flex justify-between">
|
||||||
|
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !bg-white/85 !p-0 !shadow-sm" icon={<ArrowLeft className="size-3" />} disabled={index <= 0} onClick={() => onMove(-1)} />
|
||||||
|
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !bg-white/85 !p-0 !shadow-sm" icon={<ArrowRight className="size-3" />} disabled={index >= total - 1} onClick={() => onMove(1)} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function buildLog({
|
function buildLog({
|
||||||
prompt,
|
prompt,
|
||||||
model,
|
model,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, LoaderCircle, Plus, SlidersHorizontal, Sparkles, Trash2, Upload, VideoIcon } from "lucide-react";
|
import { ArrowLeft, ArrowRight, BookOpen, CheckSquare, ClipboardPaste, Download, FolderPlus, History, LoaderCircle, Music2, Plus, SlidersHorizontal, Sparkles, Trash2, Upload, VideoIcon } from "lucide-react";
|
||||||
import { useEffect, useRef, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { App, Button, Checkbox, Drawer, Empty, Input, Modal, Tag, Typography } from "antd";
|
import { App, Button, Checkbox, Drawer, Empty, Input, Modal, Tag, Typography } from "antd";
|
||||||
import localforage from "localforage";
|
import localforage from "localforage";
|
||||||
@@ -13,13 +13,15 @@ import { PromptSelectDialog } from "@/components/prompts/prompt-select-dialog";
|
|||||||
import { VideoSettingsPanel, normalizeVideoResolutionValue, normalizeVideoSizeValue, videoSizeLabel } from "@/components/video-settings-panel";
|
import { VideoSettingsPanel, normalizeVideoResolutionValue, normalizeVideoSizeValue, videoSizeLabel } from "@/components/video-settings-panel";
|
||||||
import { canvasThemes } from "@/lib/canvas-theme";
|
import { canvasThemes } from "@/lib/canvas-theme";
|
||||||
import { formatBytes, formatDuration } from "@/lib/image-utils";
|
import { formatBytes, formatDuration } from "@/lib/image-utils";
|
||||||
|
import { boolConfig, isSeedanceVideoConfig, normalizeSeedanceRatio, seedanceReferenceLabel, seedanceVideoReferenceError, seedanceVideoReferenceHint, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
|
||||||
import { deleteStoredMedia, resolveMediaUrl, uploadMediaFile } from "@/services/file-storage";
|
import { deleteStoredMedia, resolveMediaUrl, uploadMediaFile } from "@/services/file-storage";
|
||||||
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
|
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
|
||||||
import { requestVideoGeneration } from "@/services/api/video";
|
import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video";
|
||||||
import { useAssetStore } from "@/stores/use-asset-store";
|
import { useAssetStore } from "@/stores/use-asset-store";
|
||||||
import { useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
import { useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store";
|
||||||
import { useThemeStore } from "@/stores/use-theme-store";
|
import { useThemeStore } from "@/stores/use-theme-store";
|
||||||
import type { ReferenceImage } from "@/types/image";
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||||
|
|
||||||
type GeneratedVideo = {
|
type GeneratedVideo = {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -48,6 +50,8 @@ type GenerationLog = {
|
|||||||
model: string;
|
model: string;
|
||||||
config: GenerationLogConfig;
|
config: GenerationLogConfig;
|
||||||
references: ReferenceImage[];
|
references: ReferenceImage[];
|
||||||
|
videoReferences: ReferenceVideo[];
|
||||||
|
audioReferences: ReferenceAudio[];
|
||||||
durationMs: number;
|
durationMs: number;
|
||||||
size: string;
|
size: string;
|
||||||
resolution: string;
|
resolution: string;
|
||||||
@@ -57,7 +61,7 @@ type GenerationLog = {
|
|||||||
error?: string;
|
error?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
type GenerationLogConfig = Pick<AiConfig, "model" | "videoModel" | "size" | "vquality" | "videoSeconds">;
|
type GenerationLogConfig = Pick<AiConfig, "model" | "videoModel" | "size" | "vquality" | "videoSeconds" | "videoGenerateAudio" | "videoWatermark">;
|
||||||
|
|
||||||
type UpdateAiConfig = <K extends keyof AiConfig>(key: K, value: AiConfig[K]) => void;
|
type UpdateAiConfig = <K extends keyof AiConfig>(key: K, value: AiConfig[K]) => void;
|
||||||
|
|
||||||
@@ -75,6 +79,8 @@ export default function VideoPage() {
|
|||||||
const addAsset = useAssetStore((state) => state.addAsset);
|
const addAsset = useAssetStore((state) => state.addAsset);
|
||||||
const [prompt, setPrompt] = useState("");
|
const [prompt, setPrompt] = useState("");
|
||||||
const [references, setReferences] = useState<ReferenceImage[]>([]);
|
const [references, setReferences] = useState<ReferenceImage[]>([]);
|
||||||
|
const [videoReferences, setVideoReferences] = useState<ReferenceVideo[]>([]);
|
||||||
|
const [audioReferences, setAudioReferences] = useState<ReferenceAudio[]>([]);
|
||||||
const [results, setResults] = useState<GenerationResult[]>([]);
|
const [results, setResults] = useState<GenerationResult[]>([]);
|
||||||
const [logs, setLogs] = useState<GenerationLog[]>([]);
|
const [logs, setLogs] = useState<GenerationLog[]>([]);
|
||||||
const [running, setRunning] = useState(false);
|
const [running, setRunning] = useState(false);
|
||||||
@@ -102,14 +108,40 @@ export default function VideoPage() {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const addReferences = async (files?: FileList | null) => {
|
const addReferences = async (files?: FileList | null) => {
|
||||||
const imageFiles = Array.from(files || []).filter((file) => file.type.startsWith("image/")).slice(0, 7 - references.length);
|
const selectedFiles = Array.from(files || []);
|
||||||
|
const unsupported = selectedFiles.filter((file) => !file.type.startsWith("image/") && !file.type.startsWith("video/") && !isSupportedAudioFile(file));
|
||||||
|
if (unsupported.length) message.warning("已忽略不支持的参考素材,请使用图片、mp4/mov 视频或 mp3/wav 音频");
|
||||||
|
const imageFiles = selectedFiles.filter((file) => file.type.startsWith("image/") && file.size <= SEEDANCE_REFERENCE_LIMITS.imageMaxBytes).slice(0, SEEDANCE_REFERENCE_LIMITS.images - references.length);
|
||||||
|
const videoFiles = selectedFiles.filter((file) => file.type.startsWith("video/") && file.size <= SEEDANCE_REFERENCE_LIMITS.videoMaxBytes).slice(0, SEEDANCE_REFERENCE_LIMITS.videos - videoReferences.length);
|
||||||
|
const audioFiles = selectedFiles.filter((file) => isSupportedAudioFile(file) && file.size <= SEEDANCE_REFERENCE_LIMITS.audioMaxBytes).slice(0, SEEDANCE_REFERENCE_LIMITS.audios - audioReferences.length);
|
||||||
|
if (selectedFiles.some((file) => file.type.startsWith("image/") && file.size > SEEDANCE_REFERENCE_LIMITS.imageMaxBytes)) message.warning("已忽略超过 30MB 的参考图");
|
||||||
|
if (selectedFiles.some((file) => file.type.startsWith("video/") && file.size > SEEDANCE_REFERENCE_LIMITS.videoMaxBytes)) message.warning("已忽略超过 50MB 的参考视频");
|
||||||
|
if (selectedFiles.some((file) => isSupportedAudioFile(file) && file.size > SEEDANCE_REFERENCE_LIMITS.audioMaxBytes)) message.warning("已忽略超过 15MB 的参考音频");
|
||||||
const nextReferences = await Promise.all(
|
const nextReferences = await Promise.all(
|
||||||
imageFiles.map(async (file) => {
|
imageFiles.map(async (file) => {
|
||||||
const image = await uploadImage(file);
|
const image = await uploadImage(file);
|
||||||
return { id: nanoid(), name: file.name, type: image.mimeType, dataUrl: image.url, storageKey: image.storageKey };
|
return { id: nanoid(), name: file.name, type: image.mimeType, dataUrl: image.url, storageKey: image.storageKey };
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
setReferences((value) => [...value, ...nextReferences].slice(0, 7));
|
const nextVideoReferences = await Promise.all(
|
||||||
|
videoFiles.map(async (file) => {
|
||||||
|
const video = await uploadMediaFile(file, "video-reference");
|
||||||
|
return { id: nanoid(), name: file.name, type: video.mimeType, url: video.url, storageKey: video.storageKey, bytes: video.bytes, width: video.width, height: video.height, durationMs: video.durationMs };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const nextAudioReferences = filterAudioReferencesByDuration(
|
||||||
|
audioReferences,
|
||||||
|
await Promise.all(
|
||||||
|
audioFiles.map(async (file) => {
|
||||||
|
const audio = await uploadMediaFile(file, "audio-reference");
|
||||||
|
return { id: nanoid(), name: file.name, type: audio.mimeType, url: audio.url, storageKey: audio.storageKey, durationMs: audio.durationMs };
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
message.warning,
|
||||||
|
);
|
||||||
|
setReferences((value) => [...value, ...nextReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
|
||||||
|
setVideoReferences((value) => [...value, ...nextVideoReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.videos));
|
||||||
|
setAudioReferences((value) => [...value, ...nextAudioReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.audios));
|
||||||
};
|
};
|
||||||
|
|
||||||
const addReferencesFromClipboard = async () => {
|
const addReferencesFromClipboard = async () => {
|
||||||
@@ -121,18 +153,17 @@ export default function VideoPage() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const nextReferences = await Promise.all(
|
const nextReferences = await Promise.all(
|
||||||
blobs.slice(0, 7 - references.length).map(async (blob, index) => {
|
blobs.slice(0, SEEDANCE_REFERENCE_LIMITS.images - references.length).map(async (blob, index) => {
|
||||||
const image = await uploadImage(blob);
|
const image = await uploadImage(blob);
|
||||||
return { id: nanoid(), name: `clipboard-${index + 1}.png`, type: image.mimeType, dataUrl: image.url, storageKey: image.storageKey };
|
return { id: nanoid(), name: `clipboard-${index + 1}.png`, type: image.mimeType, dataUrl: image.url, storageKey: image.storageKey };
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
setReferences((value) => [...value, ...nextReferences].slice(0, 7));
|
setReferences((value) => [...value, ...nextReferences].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
|
||||||
message.success(`已读取 ${nextReferences.length} 张参考图`);
|
message.success(`已读取 ${nextReferences.length} 张参考图`);
|
||||||
} catch {
|
} catch {
|
||||||
message.error("剪切板里没有可读取的图片");
|
message.error("剪切板里没有可读取的图片");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const generate = async () => {
|
const generate = async () => {
|
||||||
const snapshot = buildRequestSnapshot();
|
const snapshot = buildRequestSnapshot();
|
||||||
if (!snapshot) return;
|
if (!snapshot) return;
|
||||||
@@ -143,8 +174,7 @@ export default function VideoPage() {
|
|||||||
const batchStartedAt = performance.now();
|
const batchStartedAt = performance.now();
|
||||||
setStartedAt(batchStartedAt);
|
setStartedAt(batchStartedAt);
|
||||||
try {
|
try {
|
||||||
const blob = await requestVideoGeneration(snapshot.config, snapshot.text, snapshot.references);
|
const stored = await storeGeneratedVideo(await requestVideoGeneration(snapshot.config, snapshot.text, snapshot.references, snapshot.videoReferences, snapshot.audioReferences));
|
||||||
const stored = await uploadMediaFile(blob, "video");
|
|
||||||
const nextVideo: GeneratedVideo = {
|
const nextVideo: GeneratedVideo = {
|
||||||
id: nanoid(),
|
id: nanoid(),
|
||||||
url: stored.url,
|
url: stored.url,
|
||||||
@@ -156,12 +186,12 @@ export default function VideoPage() {
|
|||||||
mimeType: stored.mimeType,
|
mimeType: stored.mimeType,
|
||||||
};
|
};
|
||||||
setResults([{ id: nextVideo.id, status: "success", video: nextVideo }]);
|
setResults([{ id: nextVideo.id, status: "success", video: nextVideo }]);
|
||||||
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, durationMs: nextVideo.durationMs, status: "成功", video: nextVideo }));
|
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, audioReferences: snapshot.audioReferences, durationMs: nextVideo.durationMs, status: "成功", video: nextVideo }));
|
||||||
message.success("视频已生成");
|
message.success("视频已生成");
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const errorMessage = error instanceof Error ? error.message : "生成失败";
|
const errorMessage = error instanceof Error ? error.message : "生成失败";
|
||||||
setResults([{ id: nanoid(), status: "failed", error: errorMessage }]);
|
setResults([{ id: nanoid(), status: "failed", error: errorMessage }]);
|
||||||
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, durationMs: performance.now() - batchStartedAt, status: "失败", error: errorMessage }));
|
saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, audioReferences: snapshot.audioReferences, durationMs: performance.now() - batchStartedAt, status: "失败", error: errorMessage }));
|
||||||
message.error(errorMessage);
|
message.error(errorMessage);
|
||||||
} finally {
|
} finally {
|
||||||
setRunning(false);
|
setRunning(false);
|
||||||
@@ -179,7 +209,12 @@ export default function VideoPage() {
|
|||||||
openConfigDialog(true);
|
openConfigDialog(true);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return { text, config: buildVideoConfig(effectiveConfig, model), references: [...references] };
|
const videoReferenceError = seedanceVideoReferenceError(videoReferences);
|
||||||
|
if (videoReferenceError) {
|
||||||
|
message.error(`${videoReferenceError}。${seedanceVideoReferenceHint}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { text, config: buildVideoConfig(effectiveConfig, model), references: [...references], videoReferences: [...videoReferences], audioReferences: [...audioReferences] };
|
||||||
};
|
};
|
||||||
|
|
||||||
const retryResult = () => {
|
const retryResult = () => {
|
||||||
@@ -208,7 +243,9 @@ export default function VideoPage() {
|
|||||||
setPrompt(payload.content);
|
setPrompt(payload.content);
|
||||||
} else if (payload.kind === "image") {
|
} else if (payload.kind === "image") {
|
||||||
const stored = await uploadImage(payload.dataUrl);
|
const stored = await uploadImage(payload.dataUrl);
|
||||||
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }].slice(0, 7));
|
setReferences((value) => [...value, { id: nanoid(), name: payload.title, type: stored.mimeType, dataUrl: stored.url, storageKey: stored.storageKey }].slice(0, SEEDANCE_REFERENCE_LIMITS.images));
|
||||||
|
} else if (payload.kind === "video") {
|
||||||
|
setVideoReferences((value) => [...value, { id: nanoid(), name: payload.title, type: "video/mp4", url: payload.url, storageKey: payload.storageKey, width: payload.width, height: payload.height }].slice(0, SEEDANCE_REFERENCE_LIMITS.videos));
|
||||||
}
|
}
|
||||||
setAssetPickerOpen(false);
|
setAssetPickerOpen(false);
|
||||||
};
|
};
|
||||||
@@ -216,6 +253,8 @@ export default function VideoPage() {
|
|||||||
const createSession = () => {
|
const createSession = () => {
|
||||||
setPrompt("");
|
setPrompt("");
|
||||||
setReferences([]);
|
setReferences([]);
|
||||||
|
setVideoReferences([]);
|
||||||
|
setAudioReferences([]);
|
||||||
setResults([]);
|
setResults([]);
|
||||||
setElapsedMs(0);
|
setElapsedMs(0);
|
||||||
setStartedAt(0);
|
setStartedAt(0);
|
||||||
@@ -248,10 +287,14 @@ export default function VideoPage() {
|
|||||||
setLogsOpen(false);
|
setLogsOpen(false);
|
||||||
setPrompt(log.prompt);
|
setPrompt(log.prompt);
|
||||||
setReferences(log.references || []);
|
setReferences(log.references || []);
|
||||||
|
setVideoReferences(log.videoReferences || []);
|
||||||
|
setAudioReferences(log.audioReferences || []);
|
||||||
if (log.config.videoModel || log.model) updateConfig("videoModel", log.config.videoModel || log.model);
|
if (log.config.videoModel || log.model) updateConfig("videoModel", log.config.videoModel || log.model);
|
||||||
if (log.config.size) updateConfig("size", log.config.size);
|
if (log.config.size) updateConfig("size", log.config.size);
|
||||||
if (log.config.vquality) updateConfig("vquality", log.config.vquality);
|
if (log.config.vquality) updateConfig("vquality", log.config.vquality);
|
||||||
if (log.config.videoSeconds) updateConfig("videoSeconds", log.config.videoSeconds);
|
if (log.config.videoSeconds) updateConfig("videoSeconds", log.config.videoSeconds);
|
||||||
|
if (log.config.videoGenerateAudio) updateConfig("videoGenerateAudio", log.config.videoGenerateAudio);
|
||||||
|
if (log.config.videoWatermark) updateConfig("videoWatermark", log.config.videoWatermark);
|
||||||
setResults(log.video ? [{ id: log.video.id, status: "success", video: log.video }] : [{ id: log.id, status: "failed", error: log.error || "生成失败" }]);
|
setResults(log.video ? [{ id: log.video.id, status: "success", video: log.video }] : [{ id: log.id, status: "failed", error: log.error || "生成失败" }]);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -305,15 +348,65 @@ export default function VideoPage() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
|
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
|
||||||
{references.map((item) => (
|
{references.map((item, index) => (
|
||||||
<div key={item.id} className="group relative size-20 shrink-0 overflow-hidden rounded-md border border-stone-200 dark:border-stone-800">
|
<div key={item.id} className="group relative size-20 shrink-0 overflow-hidden rounded-md border border-stone-200 dark:border-stone-800">
|
||||||
<img src={item.dataUrl} alt={item.name} className="size-full object-cover" />
|
<img src={item.dataUrl} alt={item.name} className="size-full object-cover" />
|
||||||
|
<span className="absolute left-1 top-1 rounded bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">{seedanceReferenceLabel("image", index)}</span>
|
||||||
|
<ReferenceOrderButtons index={index} total={references.length} onMove={(offset) => setReferences((value) => moveListItem(value, index, offset))} />
|
||||||
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考图">
|
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考图">
|
||||||
<Trash2 className="size-3.5" />
|
<Trash2 className="size-3.5" />
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
{!references.length ? <div className="flex min-w-full items-center justify-center text-sm text-stone-500">暂无参考图,最多 7 张</div> : null}
|
{!references.length ? <div className="flex min-w-full items-center justify-center text-sm text-stone-500">暂无参考图,最多 9 张</div> : null}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="min-w-0">
|
||||||
|
<div className="mb-2 flex items-center justify-between gap-3">
|
||||||
|
<span className="text-base font-semibold">参考视频</span>
|
||||||
|
<Button size="small" icon={<Upload className="size-3.5" />} onClick={() => fileInputRef.current?.click()}>
|
||||||
|
上传
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
|
||||||
|
{videoReferences.map((item, index) => (
|
||||||
|
<div key={item.id} className="group relative h-20 w-32 shrink-0 overflow-hidden rounded-md border border-stone-200 bg-black dark:border-stone-800">
|
||||||
|
<video src={item.url} className="size-full object-cover" muted preload="metadata" />
|
||||||
|
<span className="absolute left-1 top-1 rounded bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">{seedanceReferenceLabel("video", index)}</span>
|
||||||
|
<ReferenceOrderButtons index={index} total={videoReferences.length} onMove={(offset) => setVideoReferences((value) => moveListItem(value, index, offset))} />
|
||||||
|
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setVideoReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考视频">
|
||||||
|
<Trash2 className="size-3.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
{!videoReferences.length ? <div className="flex min-w-full items-center justify-center text-sm text-stone-500">暂无参考视频,最多 3 个</div> : null}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="min-w-0">
|
||||||
|
<div className="mb-2 flex items-center justify-between gap-3">
|
||||||
|
<span className="text-base font-semibold">参考音频</span>
|
||||||
|
<Button size="small" icon={<Upload className="size-3.5" />} onClick={() => fileInputRef.current?.click()}>
|
||||||
|
上传
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
<div className="hover-scrollbar hover-scrollbar-hint flex min-h-24 w-full min-w-0 max-w-full gap-2 overflow-x-scroll overflow-y-hidden rounded-lg border border-dashed border-stone-300 p-2 pb-3 overscroll-x-contain dark:border-stone-700">
|
||||||
|
{audioReferences.map((item, index) => (
|
||||||
|
<div key={item.id} className="group relative flex h-20 w-48 shrink-0 flex-col justify-center gap-2 rounded-md border border-stone-200 bg-stone-50 px-2 dark:border-stone-800 dark:bg-stone-900">
|
||||||
|
<div className="flex min-w-0 items-center gap-2 text-xs text-stone-500 dark:text-stone-400">
|
||||||
|
<Music2 className="size-4 shrink-0" />
|
||||||
|
<span className="shrink-0 rounded bg-stone-200 px-1 text-[10px] text-stone-700 dark:bg-stone-800 dark:text-stone-200">{seedanceReferenceLabel("audio", index)}</span>
|
||||||
|
<span className="truncate">{item.name}</span>
|
||||||
|
</div>
|
||||||
|
<audio src={item.url} controls className="h-8 w-full" preload="metadata" />
|
||||||
|
<ReferenceOrderButtons index={index} total={audioReferences.length} onMove={(offset) => setAudioReferences((value) => moveListItem(value, index, offset))} />
|
||||||
|
<button type="button" className="absolute right-1 top-1 hidden size-6 items-center justify-center rounded bg-black/60 text-white group-hover:flex" onClick={() => setAudioReferences((value) => value.filter((ref) => ref.id !== item.id))} aria-label="移除参考音频">
|
||||||
|
<Trash2 className="size-3.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
{!audioReferences.length ? <div className="flex min-w-full items-center justify-center text-center text-sm text-stone-500">暂无参考音频,最多 3 个,mp3/wav,单个 15MB 内</div> : null}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -359,7 +452,7 @@ export default function VideoPage() {
|
|||||||
<input
|
<input
|
||||||
ref={fileInputRef}
|
ref={fileInputRef}
|
||||||
type="file"
|
type="file"
|
||||||
accept="image/*"
|
accept="image/*,video/mp4,video/quicktime,audio/mpeg,audio/wav,audio/x-wav,.mp3,.wav"
|
||||||
multiple
|
multiple
|
||||||
className="hidden"
|
className="hidden"
|
||||||
onChange={(event) => {
|
onChange={(event) => {
|
||||||
@@ -542,6 +635,18 @@ async function readStoredLogs() {
|
|||||||
|
|
||||||
async function normalizeLog(log: Partial<GenerationLog>): Promise<GenerationLog> {
|
async function normalizeLog(log: Partial<GenerationLog>): Promise<GenerationLog> {
|
||||||
const video = log.video?.storageKey ? { ...log.video, url: await resolveMediaUrl(log.video.storageKey, log.video.url) } : log.video;
|
const video = log.video?.storageKey ? { ...log.video, url: await resolveMediaUrl(log.video.storageKey, log.video.url) } : log.video;
|
||||||
|
const videoReferences = await Promise.all(
|
||||||
|
(log.videoReferences || []).map(async (item) => ({
|
||||||
|
...item,
|
||||||
|
url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
const audioReferences = await Promise.all(
|
||||||
|
(log.audioReferences || []).map(async (item) => ({
|
||||||
|
...item,
|
||||||
|
url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url,
|
||||||
|
})),
|
||||||
|
);
|
||||||
const references = await Promise.all(
|
const references = await Promise.all(
|
||||||
(log.references || []).map(async (item) => ({
|
(log.references || []).map(async (item) => ({
|
||||||
...item,
|
...item,
|
||||||
@@ -558,6 +663,8 @@ async function normalizeLog(log: Partial<GenerationLog>): Promise<GenerationLog>
|
|||||||
model: log.model || config.videoModel || "",
|
model: log.model || config.videoModel || "",
|
||||||
config,
|
config,
|
||||||
references,
|
references,
|
||||||
|
videoReferences,
|
||||||
|
audioReferences,
|
||||||
durationMs: log.durationMs || 0,
|
durationMs: log.durationMs || 0,
|
||||||
size: log.size || config.size || "",
|
size: log.size || config.size || "",
|
||||||
resolution: normalizeResolution(log.resolution || config.vquality || ""),
|
resolution: normalizeResolution(log.resolution || config.vquality || ""),
|
||||||
@@ -572,10 +679,54 @@ function serializeLog(log: GenerationLog): GenerationLog {
|
|||||||
return {
|
return {
|
||||||
...log,
|
...log,
|
||||||
references: log.references.map((item) => ({ ...item, dataUrl: item.storageKey ? "" : item.dataUrl })),
|
references: log.references.map((item) => ({ ...item, dataUrl: item.storageKey ? "" : item.dataUrl })),
|
||||||
|
videoReferences: log.videoReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)),
|
||||||
|
audioReferences: log.audioReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)),
|
||||||
video: log.video?.storageKey ? { ...log.video, url: "" } : log.video,
|
video: log.video?.storageKey ? { ...log.video, url: "" } : log.video,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isSupportedAudioFile(file: File) {
|
||||||
|
return file.type === "audio/mpeg" || file.type === "audio/mp3" || file.type === "audio/wav" || file.type === "audio/x-wav" || /\.(mp3|wav)$/i.test(file.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
function filterAudioReferencesByDuration(existing: ReferenceAudio[], next: ReferenceAudio[], warn: (content: string) => void) {
|
||||||
|
let total = existing.reduce((sum, item) => sum + (item.durationMs || 0), 0);
|
||||||
|
const accepted: ReferenceAudio[] = [];
|
||||||
|
let skipped = false;
|
||||||
|
for (const item of next) {
|
||||||
|
if (item.durationMs && (item.durationMs < 2000 || item.durationMs > 15000)) {
|
||||||
|
skipped = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (item.durationMs && total + item.durationMs > 15000) {
|
||||||
|
skipped = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
total += item.durationMs || 0;
|
||||||
|
accepted.push(item);
|
||||||
|
}
|
||||||
|
if (skipped) warn("已忽略不符合时长要求的参考音频:单个 2-15 秒,总时长不超过 15 秒");
|
||||||
|
return accepted;
|
||||||
|
}
|
||||||
|
|
||||||
|
function moveListItem<T>(items: T[], index: number, offset: number) {
|
||||||
|
const targetIndex = index + offset;
|
||||||
|
if (targetIndex < 0 || targetIndex >= items.length) return items;
|
||||||
|
const next = [...items];
|
||||||
|
[next[index], next[targetIndex]] = [next[targetIndex], next[index]];
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ReferenceOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) {
|
||||||
|
if (total <= 1) return null;
|
||||||
|
return (
|
||||||
|
<div className="absolute inset-x-1 bottom-1 flex justify-between">
|
||||||
|
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !bg-white/85 !p-0 !shadow-sm" icon={<ArrowLeft className="size-3" />} disabled={index <= 0} onClick={() => onMove(-1)} />
|
||||||
|
<Button size="small" className="!h-6 !w-6 !min-w-6 !rounded-full !bg-white/85 !p-0 !shadow-sm" icon={<ArrowRight className="size-3" />} disabled={index >= total - 1} onClick={() => onMove(1)} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
|
function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
|
||||||
return {
|
return {
|
||||||
model: log.config?.model || log.model || "",
|
model: log.config?.model || log.model || "",
|
||||||
@@ -583,16 +734,20 @@ function normalizeLogConfig(log: Partial<GenerationLog>): GenerationLogConfig {
|
|||||||
size: log.config?.size || log.size || "",
|
size: log.config?.size || log.size || "",
|
||||||
vquality: normalizeResolution(log.config?.vquality || log.resolution || ""),
|
vquality: normalizeResolution(log.config?.vquality || log.resolution || ""),
|
||||||
videoSeconds: log.config?.videoSeconds || log.seconds || "",
|
videoSeconds: log.config?.videoSeconds || log.seconds || "",
|
||||||
|
videoGenerateAudio: log.config?.videoGenerateAudio || "true",
|
||||||
|
videoWatermark: log.config?.videoWatermark || "false",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildLog({ prompt, model, config, references, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog {
|
function buildLog({ prompt, model, config, references, videoReferences, audioReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; audioReferences: ReferenceAudio[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog {
|
||||||
const logConfig = {
|
const logConfig = {
|
||||||
model: config.model,
|
model: config.model,
|
||||||
videoModel: config.videoModel,
|
videoModel: config.videoModel,
|
||||||
size: config.size,
|
size: config.size,
|
||||||
vquality: normalizeResolution(config.vquality),
|
vquality: normalizeResolution(config.vquality),
|
||||||
videoSeconds: config.videoSeconds,
|
videoSeconds: config.videoSeconds,
|
||||||
|
videoGenerateAudio: config.videoGenerateAudio,
|
||||||
|
videoWatermark: config.videoWatermark,
|
||||||
};
|
};
|
||||||
return {
|
return {
|
||||||
id: nanoid(),
|
id: nanoid(),
|
||||||
@@ -603,6 +758,8 @@ function buildLog({ prompt, model, config, references, durationMs, status, video
|
|||||||
model,
|
model,
|
||||||
config: logConfig,
|
config: logConfig,
|
||||||
references,
|
references,
|
||||||
|
videoReferences,
|
||||||
|
audioReferences,
|
||||||
durationMs,
|
durationMs,
|
||||||
size: logConfig.size,
|
size: logConfig.size,
|
||||||
resolution: logConfig.vquality,
|
resolution: logConfig.vquality,
|
||||||
@@ -614,17 +771,21 @@ function buildLog({ prompt, model, config, references, durationMs, status, video
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildVideoConfig(config: AiConfig, model: string): AiConfig {
|
function buildVideoConfig(config: AiConfig, model: string): AiConfig {
|
||||||
|
const seedance = isSeedanceVideoConfig({ ...config, model });
|
||||||
return {
|
return {
|
||||||
...config,
|
...config,
|
||||||
model,
|
model,
|
||||||
videoModel: model,
|
videoModel: model,
|
||||||
size: normalizeVideoSize(config.size),
|
size: seedance ? normalizeSeedanceRatio(config.size) : normalizeVideoSize(config.size),
|
||||||
videoSeconds: normalizeVideoSeconds(config.videoSeconds),
|
videoSeconds: normalizeVideoSeconds(config.videoSeconds),
|
||||||
vquality: normalizeResolution(config.vquality),
|
vquality: normalizeResolution(config.vquality),
|
||||||
|
videoGenerateAudio: String(boolConfig(config.videoGenerateAudio, true)),
|
||||||
|
videoWatermark: String(boolConfig(config.videoWatermark, false)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeVideoSeconds(value: string) {
|
function normalizeVideoSeconds(value: string) {
|
||||||
|
if (String(value).trim() === "-1") return "-1";
|
||||||
const seconds = Math.floor(Number(value) || 6);
|
const seconds = Math.floor(Number(value) || 6);
|
||||||
return String(Math.max(1, Math.min(20, seconds)));
|
return String(Math.max(1, Math.min(20, seconds)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,9 +17,10 @@ const aspectOptions = [
|
|||||||
{ value: "1:1", label: "1:1", width: 1024, height: 1024, icon: "square" },
|
{ value: "1:1", label: "1:1", width: 1024, height: 1024, icon: "square" },
|
||||||
{ value: "3:2", label: "3:2", width: 1536, height: 1024, icon: "landscape" },
|
{ value: "3:2", label: "3:2", width: 1536, height: 1024, icon: "landscape" },
|
||||||
{ value: "2:3", label: "2:3", width: 1024, height: 1536, icon: "portrait" },
|
{ value: "2:3", label: "2:3", width: 1024, height: 1536, icon: "portrait" },
|
||||||
{ value: "4:3", label: "4:3", width: 1344, height: 1024, icon: "landscape" },
|
{ value: "4:3", label: "4:3", width: 1360, height: 1024, icon: "landscape" },
|
||||||
{ value: "3:4", label: "3:4", width: 1024, height: 1344, icon: "portrait" },
|
{ value: "3:4", label: "3:4", width: 1024, height: 1360, icon: "portrait" },
|
||||||
{ value: "9:16", label: "9:16", width: 1024, height: 1792, icon: "portrait" },
|
{ value: "16:9", label: "16:9", width: 1824, height: 1024, icon: "landscape" },
|
||||||
|
{ value: "9:16", label: "9:16", width: 1024, height: 1824, icon: "portrait" },
|
||||||
{ value: "1:1-2k", label: "1:1(2k)", size: "2048x2048", width: 2048, height: 2048, icon: "square" },
|
{ value: "1:1-2k", label: "1:1(2k)", size: "2048x2048", width: 2048, height: 2048, icon: "square" },
|
||||||
{ value: "16:9-2k", label: "16:9(2k)", size: "2048x1152", width: 2048, height: 1152, icon: "landscape" },
|
{ value: "16:9-2k", label: "16:9(2k)", size: "2048x1152", width: 2048, height: 1152, icon: "landscape" },
|
||||||
{ value: "9:16-2k", label: "9:16(2k)", size: "1152x2048", width: 1152, height: 2048, icon: "portrait" },
|
{ value: "9:16-2k", label: "9:16(2k)", size: "1152x2048", width: 1152, height: 2048, icon: "portrait" },
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ type ModelPickerProps = {
|
|||||||
export function ModelPicker({ config, value, onChange, className, fullWidth = false, placeholder = "选择模型", onMissingConfig }: ModelPickerProps) {
|
export function ModelPicker({ config, value, onChange, className, fullWidth = false, placeholder = "选择模型", onMissingConfig }: ModelPickerProps) {
|
||||||
const pickerId = useId();
|
const pickerId = useId();
|
||||||
const [open, setOpen] = useState(false);
|
const [open, setOpen] = useState(false);
|
||||||
const options = useMemo(() => Array.from(new Set([...(config.channelMode === "local" ? [value] : []), ...config.models].filter(Boolean))), [config.channelMode, config.models, value]);
|
const options = useMemo(() => Array.from(new Set([...(config.channelMode === "local" ? [value] : []), ...config.models].filter((model): model is string => Boolean(model)))), [config.channelMode, config.models, value]);
|
||||||
const current = value || "";
|
const current = value || "";
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { type ReactNode } from "react";
|
import { type ReactNode } from "react";
|
||||||
|
import { Switch } from "antd";
|
||||||
|
|
||||||
import { ImageSettingsTheme } from "@/components/image-settings-panel";
|
import { ImageSettingsTheme } from "@/components/image-settings-panel";
|
||||||
|
import { boolConfig, isSeedanceFastModel, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceDurationOptions, seedancePixelLabel, seedanceRatioOptions, seedanceResolutionOptions } from "@/lib/seedance-video";
|
||||||
import { type CanvasTheme } from "@/lib/canvas-theme";
|
import { type CanvasTheme } from "@/lib/canvas-theme";
|
||||||
import type { AiConfig } from "@/stores/use-config-store";
|
import type { AiConfig } from "@/stores/use-config-store";
|
||||||
|
|
||||||
@@ -24,13 +26,17 @@ const secondOptions = [6, 10, 12, 16, 20];
|
|||||||
|
|
||||||
type VideoSettingsPanelProps = {
|
type VideoSettingsPanelProps = {
|
||||||
config: AiConfig;
|
config: AiConfig;
|
||||||
onConfigChange: (key: "vquality" | "size" | "videoSeconds", value: string) => void;
|
onConfigChange: (key: "vquality" | "size" | "videoSeconds" | "videoGenerateAudio" | "videoWatermark", value: string) => void;
|
||||||
theme: CanvasTheme;
|
theme: CanvasTheme;
|
||||||
showTitle?: boolean;
|
showTitle?: boolean;
|
||||||
className?: string;
|
className?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = true, className = "w-[320px] space-y-4 rounded-2xl px-1 py-0.5" }: VideoSettingsPanelProps) {
|
export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = true, className = "w-[320px] space-y-4 rounded-2xl px-1 py-0.5" }: VideoSettingsPanelProps) {
|
||||||
|
if (isSeedanceVideoConfig(config)) {
|
||||||
|
return <SeedanceVideoSettingsPanel config={config} onConfigChange={onConfigChange} theme={theme} showTitle={showTitle} className={className} />;
|
||||||
|
}
|
||||||
|
|
||||||
const seconds = config.videoSeconds || "6";
|
const seconds = config.videoSeconds || "6";
|
||||||
const size = normalizeVideoSizeValue(config.size);
|
const size = normalizeVideoSizeValue(config.size);
|
||||||
const dimensions = readSizeDimensions(size);
|
const dimensions = readSizeDimensions(size);
|
||||||
@@ -96,16 +102,84 @@ export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle =
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function SeedanceVideoSettingsPanel({ config, onConfigChange, theme, showTitle, className }: VideoSettingsPanelProps) {
|
||||||
|
const model = config.model || config.videoModel;
|
||||||
|
const resolution = normalizeSeedanceResolution(config.vquality, model);
|
||||||
|
const ratio = normalizeSeedanceRatio(config.size);
|
||||||
|
const duration = normalizeSeedanceDuration(config.videoSeconds);
|
||||||
|
const generateAudio = boolConfig(config.videoGenerateAudio, true);
|
||||||
|
const watermark = boolConfig(config.videoWatermark, false);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ImageSettingsTheme theme={theme}>
|
||||||
|
<div className={className} style={{ color: theme.node.text }} onMouseDown={(event) => event.stopPropagation()}>
|
||||||
|
{showTitle ? <div className="text-lg font-semibold">视频设置</div> : null}
|
||||||
|
<SettingGroup title="分辨率" color={theme.node.muted}>
|
||||||
|
<div className="grid grid-cols-3 gap-2.5">
|
||||||
|
{seedanceResolutionOptions.map((item) => {
|
||||||
|
const disabled = item.value === "1080p" && isSeedanceFastModel(model);
|
||||||
|
return (
|
||||||
|
<OptionPill key={item.value} selected={resolution === item.value} disabled={disabled} theme={theme} onClick={() => onConfigChange("vquality", item.value)}>
|
||||||
|
{item.label}
|
||||||
|
</OptionPill>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
{isSeedanceFastModel(model) ? <div className="text-[11px] leading-4 opacity-55">fast 模型不支持 1080p,会自动使用 720p。</div> : null}
|
||||||
|
</SettingGroup>
|
||||||
|
<SettingGroup title="比例" color={theme.node.muted}>
|
||||||
|
<div className="grid grid-cols-3 gap-2.5">
|
||||||
|
{seedanceRatioOptions.map((item) => (
|
||||||
|
<button
|
||||||
|
key={item.value}
|
||||||
|
type="button"
|
||||||
|
className="flex h-[68px] cursor-pointer flex-col items-center justify-center gap-1 rounded-xl border bg-transparent px-1 text-sm transition hover:opacity-80"
|
||||||
|
style={{ borderColor: ratio === item.value ? theme.node.text : theme.node.stroke, color: theme.node.text }}
|
||||||
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
|
onClick={() => onConfigChange("size", item.value)}
|
||||||
|
>
|
||||||
|
<SizePreview width={ratioPreview(item.value).width} height={ratioPreview(item.value).height} color={theme.node.text} />
|
||||||
|
<span>{item.label}</span>
|
||||||
|
<span className="text-[10px] leading-none opacity-55">{item.value === "adaptive" ? "adaptive" : seedancePixelLabel(resolution, item.value)}</span>
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</SettingGroup>
|
||||||
|
<SettingGroup title="时长" color={theme.node.muted}>
|
||||||
|
<div className="grid grid-cols-4 gap-2.5">
|
||||||
|
{seedanceDurationOptions.map((value) => (
|
||||||
|
<OptionPill key={value} selected={duration === value} theme={theme} onClick={() => onConfigChange("videoSeconds", String(value))}>
|
||||||
|
{value === -1 ? "智能" : `${value}s`}
|
||||||
|
</OptionPill>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<NumberInput value={String(duration)} min={-1} max={15} theme={theme} onChange={(value) => onConfigChange("videoSeconds", value)} />
|
||||||
|
</SettingGroup>
|
||||||
|
<SettingGroup title="输出" color={theme.node.muted}>
|
||||||
|
<div className="grid gap-2 rounded-xl border p-2.5" style={{ borderColor: theme.node.stroke }}>
|
||||||
|
<SwitchRow label="生成声音" checked={generateAudio} theme={theme} onChange={(checked) => onConfigChange("videoGenerateAudio", String(checked))} />
|
||||||
|
<SwitchRow label="添加水印" checked={watermark} theme={theme} onChange={(checked) => onConfigChange("videoWatermark", String(checked))} />
|
||||||
|
</div>
|
||||||
|
</SettingGroup>
|
||||||
|
</div>
|
||||||
|
</ImageSettingsTheme>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function videoResolutionLabel(value: string) {
|
export function videoResolutionLabel(value: string) {
|
||||||
return `${normalizeVideoResolutionValue(value)}p`;
|
return `${normalizeVideoResolutionValue(value)}p`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function videoSizeLabel(value: string) {
|
export function videoSizeLabel(value: string) {
|
||||||
|
const ratio = normalizeSeedanceRatio(value);
|
||||||
|
if (value === "adaptive" || value === "auto") return "自适应";
|
||||||
|
if (ratio === value) return seedanceRatioOptions.find((item) => item.value === ratio)?.label || ratio;
|
||||||
const size = normalizeVideoSizeValue(value);
|
const size = normalizeVideoSizeValue(value);
|
||||||
return sizeOptions.find((item) => item.value === size)?.label || size;
|
return sizeOptions.find((item) => item.value === size)?.label || size;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function videoSecondsLabel(value: string) {
|
export function videoSecondsLabel(value: string) {
|
||||||
|
if (String(value).trim() === "-1") return "智能";
|
||||||
return `${value || "6"}s`;
|
return `${value || "6"}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,9 +195,9 @@ export function normalizeVideoResolutionValue(value: string) {
|
|||||||
return value.replace(/p$/i, "") || "720";
|
return value.replace(/p$/i, "") || "720";
|
||||||
}
|
}
|
||||||
|
|
||||||
function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) {
|
function OptionPill({ selected, disabled = false, theme, onClick, children }: { selected: boolean; disabled?: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) {
|
||||||
return (
|
return (
|
||||||
<button type="button" className="h-9 cursor-pointer rounded-full border px-2 text-sm transition hover:opacity-80" style={{ background: "transparent", borderColor: selected ? theme.node.text : theme.node.stroke, color: theme.node.text }} onMouseDown={(event) => event.stopPropagation()} onClick={onClick}>
|
<button type="button" disabled={disabled} className="h-9 cursor-pointer rounded-full border px-2 text-sm transition hover:opacity-80 disabled:cursor-not-allowed disabled:opacity-35" style={{ background: "transparent", borderColor: selected ? theme.node.text : theme.node.stroke, color: theme.node.text }} onMouseDown={(event) => event.stopPropagation()} onClick={onClick}>
|
||||||
{children}
|
{children}
|
||||||
</button>
|
</button>
|
||||||
);
|
);
|
||||||
@@ -174,6 +248,29 @@ function SizePreview({ width, height, color }: { width: number; height: number;
|
|||||||
return <span className="rounded-[3px] border-2" style={{ width: previewWidth, height: previewHeight, borderColor: color }} />;
|
return <span className="rounded-[3px] border-2" style={{ width: previewWidth, height: previewHeight, borderColor: color }} />;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ratioPreview(ratio: string) {
|
||||||
|
if (ratio === "9:16") return { width: 9, height: 16 };
|
||||||
|
if (ratio === "1:1") return { width: 1, height: 1 };
|
||||||
|
if (ratio === "4:3") return { width: 4, height: 3 };
|
||||||
|
if (ratio === "3:4") return { width: 3, height: 4 };
|
||||||
|
if (ratio === "21:9") return { width: 21, height: 9 };
|
||||||
|
if (ratio === "adaptive") return { width: 0, height: 0 };
|
||||||
|
return { width: 16, height: 9 };
|
||||||
|
}
|
||||||
|
|
||||||
|
function SwitchRow({ label, checked, theme, onChange }: { label: string; checked: boolean; theme: CanvasTheme; onChange: (checked: boolean) => void }) {
|
||||||
|
return (
|
||||||
|
<div className="flex h-8 items-center justify-between gap-3">
|
||||||
|
<span className="text-sm" style={{ color: theme.node.text }}>
|
||||||
|
{label}
|
||||||
|
</span>
|
||||||
|
<span onMouseDown={(event) => event.stopPropagation()}>
|
||||||
|
<Switch size="small" checked={checked} onChange={onChange} />
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function readSizeDimensions(size: string) {
|
function readSizeDimensions(size: string) {
|
||||||
if (size === "auto") return { width: 0, height: 0 };
|
if (size === "auto") return { width: 0, height: 0 };
|
||||||
const match = size.match(/^(\d+)x(\d+)$/);
|
const match = size.match(/^(\d+)x(\d+)$/);
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
|
||||||
|
export function imageReferenceLabel(index: number) {
|
||||||
|
return `图片${index + 1}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildImageReferencePromptText(prompt: string, references: ReferenceImage[]) {
|
||||||
|
const text = prompt.trim();
|
||||||
|
if (!references.length) return text;
|
||||||
|
const labels = references.map((_, index) => imageReferenceLabel(index));
|
||||||
|
return `参考图片编号:${labels.join("、")}。请按这些编号理解提示词中的图片引用。\n\n${text}`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
import type { AiConfig } from "@/stores/use-config-store";
|
||||||
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||||
|
|
||||||
|
export const SEEDANCE_REFERENCE_LIMITS = {
|
||||||
|
images: 9,
|
||||||
|
videos: 3,
|
||||||
|
audios: 3,
|
||||||
|
imageMaxBytes: 30 * 1024 * 1024,
|
||||||
|
videoMaxBytes: 50 * 1024 * 1024,
|
||||||
|
audioMaxBytes: 15 * 1024 * 1024,
|
||||||
|
};
|
||||||
|
|
||||||
|
export const seedanceResolutionOptions = [
|
||||||
|
{ value: "480p", label: "480p" },
|
||||||
|
{ value: "720p", label: "720p" },
|
||||||
|
{ value: "1080p", label: "1080p" },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export const seedanceRatioOptions = [
|
||||||
|
{ value: "16:9", label: "横屏" },
|
||||||
|
{ value: "9:16", label: "竖屏" },
|
||||||
|
{ value: "1:1", label: "方形" },
|
||||||
|
{ value: "4:3", label: "标准横屏" },
|
||||||
|
{ value: "3:4", label: "标准竖屏" },
|
||||||
|
{ value: "21:9", label: "宽银幕" },
|
||||||
|
{ value: "adaptive", label: "自适应" },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export const seedanceDurationOptions = [-1, 4, 5, 6, 8, 10, 12, 15] as const;
|
||||||
|
|
||||||
|
const seedancePixels = {
|
||||||
|
"480p": {
|
||||||
|
"16:9": "864x496",
|
||||||
|
"4:3": "752x560",
|
||||||
|
"1:1": "640x640",
|
||||||
|
"3:4": "560x752",
|
||||||
|
"9:16": "496x864",
|
||||||
|
"21:9": "992x432",
|
||||||
|
},
|
||||||
|
"720p": {
|
||||||
|
"16:9": "1280x720",
|
||||||
|
"4:3": "1112x834",
|
||||||
|
"1:1": "960x960",
|
||||||
|
"3:4": "834x1112",
|
||||||
|
"9:16": "720x1280",
|
||||||
|
"21:9": "1470x630",
|
||||||
|
},
|
||||||
|
"1080p": {
|
||||||
|
"16:9": "1920x1080",
|
||||||
|
"4:3": "1664x1248",
|
||||||
|
"1:1": "1440x1440",
|
||||||
|
"3:4": "1248x1664",
|
||||||
|
"9:16": "1080x1920",
|
||||||
|
"21:9": "2206x946",
|
||||||
|
},
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export function isSeedanceVideoConfig(config: Pick<AiConfig, "model" | "videoModel" | "baseUrl">) {
|
||||||
|
return isSeedanceVideoModel(config.model || config.videoModel) || isArkPlanBaseUrl(config.baseUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isSeedanceVideoModel(model: string) {
|
||||||
|
const value = model.toLowerCase();
|
||||||
|
return value.includes("seedance") || value.includes("doubao-seedance");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isSeedanceFastModel(model: string) {
|
||||||
|
const value = model.toLowerCase();
|
||||||
|
return isSeedanceVideoModel(value) && value.includes("fast");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isArkPlanBaseUrl(baseUrl: string) {
|
||||||
|
return baseUrl.toLowerCase().includes("ark.cn-beijing.volces.com/api/plan/v3") || baseUrl.toLowerCase().includes("/api/plan/v3");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeSeedanceResolution(value: string, model = "") {
|
||||||
|
const normalized = normalizeResolutionToken(value);
|
||||||
|
if (isSeedanceFastModel(model) && normalized === "1080p") return "720p";
|
||||||
|
return seedanceResolutionOptions.some((item) => item.value === normalized) ? normalized : "720p";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeResolutionToken(value: string) {
|
||||||
|
if (value === "low") return "480p";
|
||||||
|
if (value === "auto" || value === "high" || value === "medium") return "720p";
|
||||||
|
const resolution = String(value || "").replace(/p$/i, "") || "720";
|
||||||
|
return `${resolution}p`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeSeedanceDuration(value: string) {
|
||||||
|
if (String(value).trim() === "-1") return -1;
|
||||||
|
const seconds = Math.floor(Number(value) || 5);
|
||||||
|
return Math.max(4, Math.min(15, seconds));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeSeedanceRatio(value: string) {
|
||||||
|
if (!value || value === "auto" || value === "adaptive") return "adaptive";
|
||||||
|
if (seedanceRatioOptions.some((item) => item.value === value)) return value;
|
||||||
|
const match = value.match(/^(\d+)x(\d+)$/);
|
||||||
|
if (!match) return "adaptive";
|
||||||
|
const width = Number(match[1]);
|
||||||
|
const height = Number(match[2]);
|
||||||
|
if (!width || !height) return "adaptive";
|
||||||
|
const ratio = width / height;
|
||||||
|
const options = [
|
||||||
|
["16:9", 16 / 9],
|
||||||
|
["4:3", 4 / 3],
|
||||||
|
["1:1", 1],
|
||||||
|
["3:4", 3 / 4],
|
||||||
|
["9:16", 9 / 16],
|
||||||
|
["21:9", 21 / 9],
|
||||||
|
] as const;
|
||||||
|
return options.reduce((best, item) => (Math.abs(item[1] - ratio) < Math.abs(best[1] - ratio) ? item : best), options[0])[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function seedancePixelLabel(resolution: string, ratio: string) {
|
||||||
|
const normalizedResolution = normalizeSeedanceResolution(resolution) as keyof typeof seedancePixels;
|
||||||
|
const normalizedRatio = normalizeSeedanceRatio(ratio) as keyof (typeof seedancePixels)[typeof normalizedResolution] | "adaptive";
|
||||||
|
if (normalizedRatio === "adaptive") return "自动匹配";
|
||||||
|
return seedancePixels[normalizedResolution][normalizedRatio] || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function boolConfig(value: string | undefined, fallback: boolean) {
|
||||||
|
if (value === "true") return true;
|
||||||
|
if (value === "false") return false;
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function seedanceReferenceLabel(kind: "image" | "video" | "audio", index: number) {
|
||||||
|
if (kind === "image") return `图片${index + 1}`;
|
||||||
|
if (kind === "video") return `视频${index + 1}`;
|
||||||
|
return `音频${index + 1}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSeedancePromptText(prompt: string, images: ReferenceImage[], videos: ReferenceVideo[], audios: ReferenceAudio[]) {
|
||||||
|
const labels = [
|
||||||
|
...images.map((_, index) => seedanceReferenceLabel("image", index)),
|
||||||
|
...videos.map((_, index) => seedanceReferenceLabel("video", index)),
|
||||||
|
...audios.map((_, index) => seedanceReferenceLabel("audio", index)),
|
||||||
|
];
|
||||||
|
const text = prompt.trim();
|
||||||
|
if (!labels.length) return text;
|
||||||
|
return `参考素材编号:${labels.join("、")}。请按这些编号理解提示词中的图片、视频和音频引用。\n\n${text}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function seedanceVideoReferenceError(videos: ReferenceVideo[]) {
|
||||||
|
let totalDurationMs = 0;
|
||||||
|
for (let index = 0; index < videos.length; index += 1) {
|
||||||
|
const video = videos[index];
|
||||||
|
const label = seedanceReferenceLabel("video", index);
|
||||||
|
if (video.bytes && video.bytes > SEEDANCE_REFERENCE_LIMITS.videoMaxBytes) return `${label} 超过 50MB,请压缩后再上传`;
|
||||||
|
if (video.durationMs) {
|
||||||
|
if (video.durationMs < 2000 || video.durationMs > 15000) return `${label} 时长需要在 2-15 秒之间`;
|
||||||
|
totalDurationMs += video.durationMs;
|
||||||
|
}
|
||||||
|
if (video.width && video.height) {
|
||||||
|
if (video.width < 300 || video.width > 6000 || video.height < 300 || video.height > 6000) return `${label} 宽高需要在 300-6000px 之间`;
|
||||||
|
const ratio = video.width / video.height;
|
||||||
|
if (ratio < 0.4 || ratio > 2.5) return `${label} 宽高比需要在 0.4-2.5 之间`;
|
||||||
|
const pixels = video.width * video.height;
|
||||||
|
if (pixels < 640 * 640 || pixels > 2206 * 946) return `${label} 像素总量不符合 Seedance 要求,请转成 480p/720p/1080p 后再上传`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (totalDurationMs > 15000) return "Seedance 参考视频总时长不能超过 15 秒";
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export const seedanceVideoReferenceHint = "参考视频需为 mp4/mov,H.264/H.265,FPS 24-60;含真人人脸素材请使用火山授权 asset:// 素材。";
|
||||||
@@ -4,6 +4,7 @@ import { buildApiUrl, type AiConfig } from "@/stores/use-config-store";
|
|||||||
import { useUserStore } from "@/stores/use-user-store";
|
import { useUserStore } from "@/stores/use-user-store";
|
||||||
import { nanoid } from "nanoid";
|
import { nanoid } from "nanoid";
|
||||||
import { dataUrlToFile } from "@/lib/image-utils";
|
import { dataUrlToFile } from "@/lib/image-utils";
|
||||||
|
import { buildImageReferencePromptText } from "@/lib/image-reference-prompt";
|
||||||
import { imageToDataUrl } from "@/services/image-storage";
|
import { imageToDataUrl } from "@/services/image-storage";
|
||||||
import type { ReferenceImage } from "@/types/image";
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
|
||||||
@@ -31,6 +32,12 @@ const QUALITY_ALIASES: Record<string, string> = {
|
|||||||
"2k": "medium",
|
"2k": "medium",
|
||||||
"4k": "high",
|
"4k": "high",
|
||||||
};
|
};
|
||||||
|
const DEFAULT_IMAGE_SHORT_SIDE = 1024;
|
||||||
|
const IMAGE_SIZE_STEP = 16;
|
||||||
|
const IMAGE_MIN_PIXELS = 655360;
|
||||||
|
const IMAGE_MAX_PIXELS = 8294400;
|
||||||
|
const IMAGE_MAX_EDGE = 3840;
|
||||||
|
const IMAGE_MAX_RATIO = 3;
|
||||||
|
|
||||||
function normalizeQuality(quality: string) {
|
function normalizeQuality(quality: string) {
|
||||||
const value = quality.trim().toLowerCase();
|
const value = quality.trim().toLowerCase();
|
||||||
@@ -38,36 +45,66 @@ function normalizeQuality(quality: string) {
|
|||||||
return QUALITY_BASE[normalized] ? normalized : undefined;
|
return QUALITY_BASE[normalized] ? normalized : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Map "quality + ratio" to an explicit pixel dimension like "3840x2160". Returns undefined when quality is auto. */
|
/** Map "quality + ratio" to an explicit pixel dimension like "3840x2160". */
|
||||||
function resolveSize(quality: string, ratio: string): string | undefined {
|
function resolveSize(quality: string | undefined, ratio: string): string {
|
||||||
const basePixels = QUALITY_BASE[quality];
|
const parsedRatio = parseImageRatio(ratio);
|
||||||
if (!basePixels || ratio === "auto" || !ratio) return undefined;
|
const basePixels = quality ? QUALITY_BASE[quality] : undefined;
|
||||||
|
const isLandscape = parsedRatio.width >= parsedRatio.height;
|
||||||
|
const longRatio = isLandscape ? parsedRatio.width / parsedRatio.height : parsedRatio.height / parsedRatio.width;
|
||||||
|
let longSide: number;
|
||||||
|
let shortSide: number;
|
||||||
|
|
||||||
const parts = ratio.split(":");
|
if (basePixels) {
|
||||||
if (parts.length !== 2) return undefined;
|
const targetPixels = basePixels * basePixels;
|
||||||
const w = Number(parts[0]);
|
const longSideRaw = Math.sqrt(targetPixels * longRatio);
|
||||||
const h = Number(parts[1]);
|
longSide = Math.floor(longSideRaw / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP;
|
||||||
if (!w || !h) return undefined;
|
shortSide = Math.round(longSide / longRatio / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP;
|
||||||
|
} else {
|
||||||
const targetPixels = basePixels * basePixels;
|
shortSide = DEFAULT_IMAGE_SHORT_SIDE;
|
||||||
const isLandscape = w >= h;
|
longSide = Math.round((shortSide * longRatio) / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP;
|
||||||
const longRatio = isLandscape ? w / h : h / w;
|
}
|
||||||
|
|
||||||
const longSideRaw = Math.sqrt(targetPixels * longRatio);
|
|
||||||
const longSide = Math.floor(longSideRaw / 16) * 16;
|
|
||||||
const shortSide = Math.round((longSide / longRatio) / 16) * 16;
|
|
||||||
|
|
||||||
const width = isLandscape ? longSide : shortSide;
|
const width = isLandscape ? longSide : shortSide;
|
||||||
const height = isLandscape ? shortSide : longSide;
|
const height = isLandscape ? shortSide : longSide;
|
||||||
|
validateImageSize(width, height);
|
||||||
return `${width}x${height}`;
|
return `${width}x${height}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseImageRatio(value: string) {
|
||||||
|
const parts = value.split(":");
|
||||||
|
if (parts.length !== 2) throw new Error("图像尺寸格式不支持,请使用 auto、9:16 或 1024x1024");
|
||||||
|
const w = Number(parts[0]);
|
||||||
|
const h = Number(parts[1]);
|
||||||
|
if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) throw new Error("图像比例必须是正数,例如 9:16");
|
||||||
|
if (Math.max(w, h) / Math.min(w, h) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1,请调整尺寸");
|
||||||
|
return { width: w, height: h };
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseImageDimensions(value: string) {
|
||||||
|
const match = value.match(/^(\d+)x(\d+)$/i);
|
||||||
|
if (!match) return null;
|
||||||
|
return { width: Number(match[1]), height: Number(match[2]) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateImageSize(width: number, height: number) {
|
||||||
|
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) throw new Error("图像尺寸必须是正整数,例如 1024x1024");
|
||||||
|
if (width % IMAGE_SIZE_STEP !== 0 || height % IMAGE_SIZE_STEP !== 0) throw new Error("图像尺寸的宽高必须是 16 的倍数,请调整尺寸");
|
||||||
|
if (Math.max(width, height) > IMAGE_MAX_EDGE) throw new Error("图像尺寸最长边不能超过 3840px,请调整尺寸");
|
||||||
|
if (Math.max(width, height) / Math.min(width, height) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1,请调整尺寸");
|
||||||
|
const pixels = width * height;
|
||||||
|
if (pixels < IMAGE_MIN_PIXELS || pixels > IMAGE_MAX_PIXELS) throw new Error("图像总像素需在 655360 到 8294400 之间,请调整尺寸");
|
||||||
|
}
|
||||||
|
|
||||||
function resolveRequestSize(quality: string | undefined, size: string) {
|
function resolveRequestSize(quality: string | undefined, size: string) {
|
||||||
const value = size.trim();
|
const value = size.trim();
|
||||||
if (!value || value === "auto") return undefined;
|
if (!value || value.toLowerCase() === "auto") return undefined;
|
||||||
if (/^\d+x\d+$/.test(value)) return value;
|
const dimensions = parseImageDimensions(value);
|
||||||
return (quality && resolveSize(quality, value)) || value;
|
if (dimensions) {
|
||||||
|
validateImageSize(dimensions.width, dimensions.height);
|
||||||
|
return `${dimensions.width}x${dimensions.height}`;
|
||||||
|
}
|
||||||
|
if (value.includes(":")) return resolveSize(quality, value);
|
||||||
|
throw new Error("图像尺寸格式不支持,请使用 auto、9:16 或 1024x1024");
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveImageDataUrl(item: Record<string, unknown>) {
|
function resolveImageDataUrl(item: Record<string, unknown>) {
|
||||||
@@ -100,11 +137,17 @@ function parseImagePayload(payload: ImageApiResponse) {
|
|||||||
function readAxiosError(error: unknown, fallback: string) {
|
function readAxiosError(error: unknown, fallback: string) {
|
||||||
if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) {
|
if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) {
|
||||||
const responseData = error.response?.data;
|
const responseData = error.response?.data;
|
||||||
return responseData?.msg || responseData?.error?.message || (error.response?.status ? `${fallback}:${error.response.status}` : fallback);
|
return responseData?.msg || responseData?.error?.message || readStatusError(error.response?.status, fallback);
|
||||||
}
|
}
|
||||||
return error instanceof Error ? error.message : fallback;
|
return error instanceof Error ? error.message : fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readStatusError(status: number | undefined, fallback: string) {
|
||||||
|
if (status === 401 || status === 403) return "鉴权失败,请检查 API Key、套餐权限或模型权限";
|
||||||
|
if (status === 429) return "请求被限流或额度不足,请稍后重试";
|
||||||
|
return status ? `${fallback}:${status}` : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
function parseStreamChunk(chunk: string, onDelta: (value: string) => void) {
|
function parseStreamChunk(chunk: string, onDelta: (value: string) => void) {
|
||||||
let deltaText = "";
|
let deltaText = "";
|
||||||
for (const eventBlock of chunk.split("\n\n")) {
|
for (const eventBlock of chunk.split("\n\n")) {
|
||||||
@@ -181,9 +224,10 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
|
|||||||
const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1)));
|
const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1)));
|
||||||
const quality = normalizeQuality(config.quality);
|
const quality = normalizeQuality(config.quality);
|
||||||
const requestSize = resolveRequestSize(quality, config.size);
|
const requestSize = resolveRequestSize(quality, config.size);
|
||||||
|
const requestPrompt = buildImageReferencePromptText(prompt, references);
|
||||||
const formData = new FormData();
|
const formData = new FormData();
|
||||||
formData.set("model", config.model);
|
formData.set("model", config.model);
|
||||||
formData.set("prompt", withSystemPrompt(config, prompt));
|
formData.set("prompt", withSystemPrompt(config, requestPrompt));
|
||||||
formData.set("n", String(n));
|
formData.set("n", String(n));
|
||||||
formData.set("response_format", "b64_json");
|
formData.set("response_format", "b64_json");
|
||||||
if (quality) {
|
if (quality) {
|
||||||
|
|||||||
+216
-14
@@ -1,29 +1,67 @@
|
|||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
|
|
||||||
import { dataUrlToFile } from "@/lib/image-utils";
|
import { dataUrlToFile } from "@/lib/image-utils";
|
||||||
|
import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
|
||||||
import { imageToDataUrl } from "@/services/image-storage";
|
import { imageToDataUrl } from "@/services/image-storage";
|
||||||
|
import { boolConfig, buildSeedancePromptText, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
|
||||||
import { buildApiUrl, type AiConfig } from "@/stores/use-config-store";
|
import { buildApiUrl, type AiConfig } from "@/stores/use-config-store";
|
||||||
import { useUserStore } from "@/stores/use-user-store";
|
import { useUserStore } from "@/stores/use-user-store";
|
||||||
import type { ReferenceImage } from "@/types/image";
|
import type { ReferenceImage } from "@/types/image";
|
||||||
|
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||||
|
|
||||||
type VideoResponse = { id: string; status?: string; error?: { message?: string } };
|
type VideoResponse = { id: string; status?: string; error?: { message?: string } };
|
||||||
type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string };
|
type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string };
|
||||||
|
type SeedanceTask = {
|
||||||
|
id: string;
|
||||||
|
status?: "queued" | "running" | "succeeded" | "failed" | "cancelled" | "expired";
|
||||||
|
error?: { code?: string; message?: string } | null;
|
||||||
|
content?: { video_url?: string; last_frame_url?: string } | null;
|
||||||
|
};
|
||||||
|
type ApiEnvelope<T> = T | { code?: number; data?: T | null; msg?: string };
|
||||||
|
type ReferenceMediaUploadResponse = { id: string; url: string; mimeType: string; bytes: number };
|
||||||
|
|
||||||
|
export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string };
|
||||||
|
|
||||||
function aiApiUrl(config: AiConfig, path: string) {
|
function aiApiUrl(config: AiConfig, path: string) {
|
||||||
return config.channelMode === "remote" ? `/api/v1${path}` : buildApiUrl(config.baseUrl, path);
|
return config.channelMode === "remote" ? `/api/v1${path}` : buildApiUrl(config.baseUrl, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
function aiHeaders(config: AiConfig) {
|
function aiHeaders(config: AiConfig, contentType?: string) {
|
||||||
const token = useUserStore.getState().token;
|
const token = useUserStore.getState().token;
|
||||||
return config.channelMode === "remote" ? (token ? { Authorization: `Bearer ${token}` } : undefined) : { Authorization: `Bearer ${config.apiKey}` };
|
return config.channelMode === "remote"
|
||||||
|
? {
|
||||||
|
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||||
|
...(contentType ? { "Content-Type": contentType } : {}),
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
Authorization: `Bearer ${config.apiKey}`,
|
||||||
|
...(contentType ? { "Content-Type": contentType } : {}),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function refreshRemoteUser(config: AiConfig) {
|
function refreshRemoteUser(config: AiConfig) {
|
||||||
if (config.channelMode === "remote") void useUserStore.getState().hydrateUser();
|
if (config.channelMode === "remote") void useUserStore.getState().hydrateUser();
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = []) {
|
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = []): Promise<VideoGenerationResult> {
|
||||||
const model = config.model || config.videoModel;
|
const model = (config.model || config.videoModel).trim();
|
||||||
|
assertVideoConfig(config, model);
|
||||||
|
if (isSeedanceVideoConfig({ ...config, model })) {
|
||||||
|
return requestSeedanceGeneration(config, model, prompt, references, videoReferences, audioReferences);
|
||||||
|
}
|
||||||
|
if (videoReferences.length || audioReferences.length) {
|
||||||
|
throw new Error("当前视频接口不支持参考视频或参考音频,请切换到 Seedance 2.0 / 火山 Agent Plan 模型,或移除参考素材");
|
||||||
|
}
|
||||||
|
return requestOpenAIVideoGeneration(config, model, prompt, references);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function storeGeneratedVideo(result: VideoGenerationResult): Promise<UploadedFile> {
|
||||||
|
if (result.blob) return uploadMediaFile(result.blob, "video");
|
||||||
|
if (result.url) return { url: result.url, storageKey: "", bytes: 0, mimeType: result.mimeType || "video/mp4" };
|
||||||
|
throw new Error("视频接口没有返回可播放的视频");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestOpenAIVideoGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[]) {
|
||||||
const body = new FormData();
|
const body = new FormData();
|
||||||
body.append("model", model);
|
body.append("model", model);
|
||||||
body.append("prompt", prompt);
|
body.append("prompt", prompt);
|
||||||
@@ -36,21 +74,162 @@ export async function requestVideoGeneration(config: AiConfig, prompt: string, r
|
|||||||
try {
|
try {
|
||||||
const created = unwrapVideoResponse((await axios.post<ApiVideoResponse>(aiApiUrl(config, "/videos"), body, { headers: aiHeaders(config) })).data);
|
const created = unwrapVideoResponse((await axios.post<ApiVideoResponse>(aiApiUrl(config, "/videos"), body, { headers: aiHeaders(config) })).data);
|
||||||
if (!created.id) throw new Error("视频接口没有返回任务 ID");
|
if (!created.id) throw new Error("视频接口没有返回任务 ID");
|
||||||
for (;;) {
|
for (let attempt = 0; attempt < 120; attempt += 1) {
|
||||||
const video = unwrapVideoResponse((await axios.get<ApiVideoResponse>(aiApiUrl(config, `/videos/${created.id}`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined })).data);
|
const video = unwrapVideoResponse((await axios.get<ApiVideoResponse>(aiApiUrl(config, `/videos/${created.id}`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined })).data);
|
||||||
if (video.status === "completed") break;
|
if (video.status === "completed") break;
|
||||||
if (video.status === "failed" || video.status === "cancelled") throw new Error(video.error?.message || "视频生成失败");
|
if (video.status === "failed" || video.status === "cancelled") throw new Error(video.error?.message || "视频生成失败");
|
||||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
if (attempt === 119) throw new Error("视频生成超时,请稍后重试");
|
||||||
|
await delay(2500);
|
||||||
}
|
}
|
||||||
const content = await axios.get<Blob>(aiApiUrl(config, `/videos/${created.id}/content`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined, responseType: "blob" });
|
const content = await axios.get<Blob>(aiApiUrl(config, `/videos/${created.id}/content`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined, responseType: "blob" });
|
||||||
await assertVideoBlob(content.data);
|
await assertVideoBlob(content.data);
|
||||||
refreshRemoteUser(config);
|
refreshRemoteUser(config);
|
||||||
return content.data;
|
return { blob: content.data };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(readAxiosError(error, "视频生成失败"));
|
throw new Error(readAxiosError(error, "视频生成失败"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function requestSeedanceGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
|
||||||
|
if (audioReferences.length && !references.length && !videoReferences.length) {
|
||||||
|
throw new Error("Seedance 参考音频不能单独使用,请同时添加参考图或参考视频");
|
||||||
|
}
|
||||||
|
assertSeedanceVideoReferences(videoReferences);
|
||||||
|
assertSeedanceAudioReferences(audioReferences);
|
||||||
|
const content = await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences);
|
||||||
|
if (!content.length) throw new Error("请输入视频提示词,或连接参考图片/视频/音频");
|
||||||
|
const payload = {
|
||||||
|
model,
|
||||||
|
content,
|
||||||
|
ratio: normalizeSeedanceRatio(config.size),
|
||||||
|
resolution: normalizeSeedanceResolution(config.vquality, model),
|
||||||
|
duration: normalizeSeedanceDuration(config.videoSeconds),
|
||||||
|
generate_audio: boolConfig(config.videoGenerateAudio, true),
|
||||||
|
watermark: boolConfig(config.videoWatermark, false),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const created = unwrapSeedanceTask((await axios.post<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config), payload, { headers: aiHeaders(config, "application/json") })).data);
|
||||||
|
if (!created.id) throw new Error("Seedance 接口没有返回任务 ID");
|
||||||
|
for (let attempt = 0; attempt < 120; attempt += 1) {
|
||||||
|
const task = unwrapSeedanceTask((await axios.get<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config, created.id), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined })).data);
|
||||||
|
if (task.status === "succeeded") {
|
||||||
|
const url = task.content?.video_url;
|
||||||
|
if (!url) throw new Error("Seedance 任务成功但没有返回视频 URL");
|
||||||
|
refreshRemoteUser(config);
|
||||||
|
return videoResultFromUrl(url);
|
||||||
|
}
|
||||||
|
if (task.status === "failed" || task.status === "cancelled" || task.status === "expired") throw new Error(task.error?.message || `Seedance 视频生成${task.status === "expired" ? "超时" : "失败"}`);
|
||||||
|
if (attempt === 119) throw new Error("Seedance 视频生成超时,请稍后重试");
|
||||||
|
await delay(5000);
|
||||||
|
}
|
||||||
|
throw new Error("Seedance 视频生成超时,请稍后重试");
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(readAxiosError(error, "Seedance 视频生成失败"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) {
|
||||||
|
const error = seedanceVideoReferenceError(videoReferences);
|
||||||
|
if (error) throw new Error(error);
|
||||||
|
let total = 0;
|
||||||
|
for (const video of videoReferences) {
|
||||||
|
if (!video.durationMs) continue;
|
||||||
|
if (video.durationMs < 2000 || video.durationMs > 15000) throw new Error("Seedance 参考视频单个时长需要在 2-15 秒之间");
|
||||||
|
total += video.durationMs;
|
||||||
|
}
|
||||||
|
if (total > 15000) throw new Error("Seedance 参考视频总时长不能超过 15 秒");
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) {
|
||||||
|
let total = 0;
|
||||||
|
for (const audio of audioReferences) {
|
||||||
|
if (!audio.durationMs) continue;
|
||||||
|
if (audio.durationMs < 2000 || audio.durationMs > 15000) throw new Error("Seedance 参考音频单个时长需要在 2-15 秒之间");
|
||||||
|
total += audio.durationMs;
|
||||||
|
}
|
||||||
|
if (total > 15000) throw new Error("Seedance 参考音频总时长不能超过 15 秒");
|
||||||
|
}
|
||||||
|
|
||||||
|
function seedanceApiUrl(config: AiConfig, taskId?: string) {
|
||||||
|
if (config.channelMode === "remote") return taskId ? `/api/v1/videos/${encodeURIComponent(taskId)}` : "/api/v1/videos";
|
||||||
|
return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
|
||||||
|
const content: Array<Record<string, unknown>> = [];
|
||||||
|
const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences);
|
||||||
|
if (text) content.push({ type: "text", text });
|
||||||
|
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
|
||||||
|
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" });
|
||||||
|
}
|
||||||
|
for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) {
|
||||||
|
content.push({ type: "video_url", video_url: { url: await resolveSeedanceVideoUrl(video) }, role: "reference_video" });
|
||||||
|
}
|
||||||
|
for (const audio of audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios)) {
|
||||||
|
content.push({ type: "audio_url", audio_url: { url: await resolveSeedanceAudioUrl(audio) }, role: "reference_audio" });
|
||||||
|
}
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveSeedanceImageUrl(config: AiConfig, image: ReferenceImage) {
|
||||||
|
const directUrl = image.url || image.dataUrl;
|
||||||
|
if (isPublicMediaUrl(directUrl) || directUrl.startsWith("asset://")) return directUrl;
|
||||||
|
const dataUrl = await imageToDataUrl(image);
|
||||||
|
if (!dataUrl) throw new Error("参考图读取失败,请换一张图片或重新上传");
|
||||||
|
if (config.channelMode === "remote") {
|
||||||
|
return uploadReferenceMedia(dataUrlToFile({ ...image, dataUrl }));
|
||||||
|
}
|
||||||
|
return dataUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveSeedanceVideoUrl(video: ReferenceVideo) {
|
||||||
|
if (isPublicMediaUrl(video.url) || video.url.startsWith("asset://")) return video.url;
|
||||||
|
let blob: Blob | null = null;
|
||||||
|
if (video.storageKey) blob = await getMediaBlob(video.storageKey);
|
||||||
|
if (!blob && video.url?.startsWith("blob:")) blob = await (await fetch(video.url)).blob();
|
||||||
|
if (!blob) throw new Error("参考视频必须是公网 URL、素材 ID,或本地已保存的视频");
|
||||||
|
const file = new File([blob], video.name || "reference-video.mp4", { type: video.type || blob.type || "video/mp4" });
|
||||||
|
return uploadReferenceMedia(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveSeedanceAudioUrl(audio: ReferenceAudio) {
|
||||||
|
if (isPublicMediaUrl(audio.url) || audio.url.startsWith("asset://")) return audio.url;
|
||||||
|
let blob: Blob | null = null;
|
||||||
|
if (audio.storageKey) blob = await getMediaBlob(audio.storageKey);
|
||||||
|
if (!blob && audio.url?.startsWith("blob:")) blob = await (await fetch(audio.url)).blob();
|
||||||
|
if (!blob) throw new Error("参考音频必须是公网 URL、素材 ID,或本地已保存的音频");
|
||||||
|
const file = new File([blob], audio.name || "reference-audio.mp3", { type: audio.type || blob.type || "audio/mpeg" });
|
||||||
|
return uploadReferenceMedia(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadReferenceMedia(file: File) {
|
||||||
|
const token = useUserStore.getState().token;
|
||||||
|
if (!token) throw new Error("使用本地参考素材需要先登录,并在服务端配置 PUBLIC_BASE_URL");
|
||||||
|
const body = new FormData();
|
||||||
|
body.append("file", file, file.name);
|
||||||
|
const response = await axios.post<ApiEnvelope<ReferenceMediaUploadResponse>>("/api/v1/media/references", body, { headers: { Authorization: `Bearer ${token}` } });
|
||||||
|
const payload = unwrapEnvelope(response.data, "参考素材上传失败");
|
||||||
|
if (!payload.url) throw new Error("参考素材上传后没有返回公网 URL");
|
||||||
|
return payload.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function videoResultFromUrl(url: string): Promise<VideoGenerationResult> {
|
||||||
|
try {
|
||||||
|
const response = await axios.get<Blob>(url, { responseType: "blob" });
|
||||||
|
await assertVideoBlob(response.data);
|
||||||
|
return { blob: response.data };
|
||||||
|
} catch {
|
||||||
|
return { url, mimeType: "video/mp4" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertVideoConfig(config: AiConfig, model: string) {
|
||||||
|
if (!model) throw new Error("请先配置视频模型");
|
||||||
|
if (config.channelMode === "local" && !config.baseUrl.trim()) throw new Error("请先配置 Base URL");
|
||||||
|
if (config.channelMode === "local" && !config.apiKey.trim()) throw new Error("请先配置 API Key");
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeVideoSeconds(value: string) {
|
function normalizeVideoSeconds(value: string) {
|
||||||
const seconds = Math.floor(Number(value) || 6);
|
const seconds = Math.floor(Number(value) || 6);
|
||||||
return String(Math.max(1, Math.min(20, seconds)));
|
return String(Math.max(1, Math.min(20, seconds)));
|
||||||
@@ -71,30 +250,53 @@ function normalizeVideoResolution(value: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function unwrapVideoResponse(payload: ApiVideoResponse) {
|
function unwrapVideoResponse(payload: ApiVideoResponse) {
|
||||||
if (!payload) throw new Error("接口没有返回视频任务");
|
return unwrapEnvelope(payload, "接口没有返回视频任务");
|
||||||
if ("code" in payload && typeof payload.code === "number") {
|
}
|
||||||
|
|
||||||
|
function unwrapSeedanceTask(payload: ApiEnvelope<SeedanceTask>) {
|
||||||
|
return unwrapEnvelope(payload, "Seedance 接口没有返回任务");
|
||||||
|
}
|
||||||
|
|
||||||
|
function unwrapEnvelope<T>(payload: ApiEnvelope<T>, emptyMessage: string): T {
|
||||||
|
if (!payload) throw new Error(emptyMessage);
|
||||||
|
if (typeof payload === "object" && "code" in payload && typeof payload.code === "number") {
|
||||||
if (payload.code !== 0) throw new Error(payload.msg || "请求失败");
|
if (payload.code !== 0) throw new Error(payload.msg || "请求失败");
|
||||||
if (!payload.data) throw new Error("接口没有返回视频任务");
|
if (!payload.data) throw new Error(emptyMessage);
|
||||||
return payload.data;
|
return payload.data;
|
||||||
}
|
}
|
||||||
return payload;
|
return payload as T;
|
||||||
}
|
}
|
||||||
|
|
||||||
function readAxiosError(error: unknown, fallback: string) {
|
function readAxiosError(error: unknown, fallback: string) {
|
||||||
if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) {
|
if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) {
|
||||||
const responseData = error.response?.data;
|
const responseData = error.response?.data;
|
||||||
return responseData?.msg || responseData?.error?.message || (error.response?.status ? `${fallback}:${error.response.status}` : fallback);
|
return responseData?.msg || responseData?.error?.message || statusMessage(error.response?.status, fallback);
|
||||||
}
|
}
|
||||||
return error instanceof Error ? error.message : fallback;
|
return error instanceof Error ? error.message : fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function statusMessage(status: number | undefined, fallback: string) {
|
||||||
|
if (status === 401 || status === 403) return "鉴权失败,请检查 API Key、套餐权限或模型权限";
|
||||||
|
if (status === 429) return "请求被限流或额度不足,请稍后重试";
|
||||||
|
return status ? `${fallback}(${status})` : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
async function assertVideoBlob(blob: Blob) {
|
async function assertVideoBlob(blob: Blob) {
|
||||||
if (!blob.type.includes("json")) return;
|
if (!blob.type.includes("json")) return;
|
||||||
let payload: { code?: number; msg?: string };
|
let payload: { code?: number; msg?: string; error?: { message?: string } };
|
||||||
try {
|
try {
|
||||||
payload = JSON.parse(await blob.text()) as { code?: number; msg?: string };
|
payload = JSON.parse(await blob.text()) as { code?: number; msg?: string; error?: { message?: string } };
|
||||||
} catch {
|
} catch {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (typeof payload.code === "number" && payload.code !== 0) throw new Error(payload.msg || "视频下载失败");
|
if (typeof payload.code === "number" && payload.code !== 0) throw new Error(payload.msg || "视频下载失败");
|
||||||
|
if (payload.error?.message) throw new Error(payload.error.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPublicMediaUrl(value: string) {
|
||||||
|
return /^https?:\/\//i.test(value || "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function delay(ms: number) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
import localforage from "localforage";
|
import localforage from "localforage";
|
||||||
import { nanoid } from "nanoid";
|
import { nanoid } from "nanoid";
|
||||||
|
|
||||||
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number };
|
export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number; durationMs?: number };
|
||||||
|
|
||||||
const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" });
|
const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" });
|
||||||
const objectUrls = new Map<string, string>();
|
const objectUrls = new Map<string, string>();
|
||||||
@@ -14,7 +14,7 @@ export async function uploadMediaFile(input: string | Blob, prefix = "file"): Pr
|
|||||||
await store.setItem(storageKey, blob);
|
await store.setItem(storageKey, blob);
|
||||||
const url = URL.createObjectURL(blob);
|
const url = URL.createObjectURL(blob);
|
||||||
objectUrls.set(storageKey, url);
|
objectUrls.set(storageKey, url);
|
||||||
const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : {};
|
const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : blob.type.startsWith("audio/") ? await readAudioMeta(url) : {};
|
||||||
return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta };
|
return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,11 +68,21 @@ export function collectMediaStorageKeys(value: unknown, keys = new Set<string>()
|
|||||||
}
|
}
|
||||||
|
|
||||||
function readVideoMeta(url: string) {
|
function readVideoMeta(url: string) {
|
||||||
return new Promise<{ width: number; height: number }>((resolve) => {
|
return new Promise<{ width: number; height: number; durationMs?: number }>((resolve) => {
|
||||||
const video = document.createElement("video");
|
const video = document.createElement("video");
|
||||||
const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720 });
|
const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720, durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined });
|
||||||
video.onloadedmetadata = done;
|
video.onloadedmetadata = done;
|
||||||
video.onerror = done;
|
video.onerror = done;
|
||||||
video.src = url;
|
video.src = url;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readAudioMeta(url: string) {
|
||||||
|
return new Promise<{ durationMs?: number }>((resolve) => {
|
||||||
|
const audio = document.createElement("audio");
|
||||||
|
const done = () => resolve({ durationMs: Number.isFinite(audio.duration) ? Math.round(audio.duration * 1000) : undefined });
|
||||||
|
audio.onloadedmetadata = done;
|
||||||
|
audio.onerror = done;
|
||||||
|
audio.src = url;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ export type AiConfig = {
|
|||||||
textModel: string;
|
textModel: string;
|
||||||
videoSeconds: string;
|
videoSeconds: string;
|
||||||
vquality: string;
|
vquality: string;
|
||||||
|
videoGenerateAudio: string;
|
||||||
|
videoWatermark: string;
|
||||||
systemPrompt: string;
|
systemPrompt: string;
|
||||||
models: string[];
|
models: string[];
|
||||||
quality: string;
|
quality: string;
|
||||||
@@ -36,6 +38,8 @@ export const defaultConfig: AiConfig = {
|
|||||||
textModel: "gpt-5.5",
|
textModel: "gpt-5.5",
|
||||||
videoSeconds: "6",
|
videoSeconds: "6",
|
||||||
vquality: "720",
|
vquality: "720",
|
||||||
|
videoGenerateAudio: "true",
|
||||||
|
videoWatermark: "false",
|
||||||
systemPrompt: "",
|
systemPrompt: "",
|
||||||
models: [],
|
models: [],
|
||||||
quality: "auto",
|
quality: "auto",
|
||||||
@@ -61,19 +65,44 @@ function resolveEffectiveConfig(config: AiConfig, modelChannel: AdminPublicSetti
|
|||||||
const channelMode = modelChannel?.allowCustomChannel ? config.channelMode : "remote";
|
const channelMode = modelChannel?.allowCustomChannel ? config.channelMode : "remote";
|
||||||
if (channelMode === "local" || !modelChannel) return { ...config, channelMode };
|
if (channelMode === "local" || !modelChannel) return { ...config, channelMode };
|
||||||
const models = modelChannel.availableModels;
|
const models = modelChannel.availableModels;
|
||||||
const fallbackModel = modelChannel.defaultModel || models[0] || "";
|
const fallbackTextModel = validDefault(modelChannel.defaultTextModel, models) || preferredModel(models, isTextModelName);
|
||||||
|
const fallbackModel = validDefault(modelChannel.defaultModel, models) || fallbackTextModel || models[0] || "";
|
||||||
|
const fallbackImageModel = validDefault(modelChannel.defaultImageModel, models) || preferredModel(models, isImageModelName) || fallbackModel;
|
||||||
|
const fallbackVideoModel = validDefault(modelChannel.defaultVideoModel, models) || preferredModel(models, isVideoModelName) || fallbackModel;
|
||||||
return {
|
return {
|
||||||
...config,
|
...config,
|
||||||
channelMode,
|
channelMode,
|
||||||
models,
|
models,
|
||||||
model: models.includes(config.model) ? config.model : fallbackModel,
|
model: models.includes(config.model) ? config.model : fallbackModel,
|
||||||
imageModel: models.includes(config.imageModel) ? config.imageModel : modelChannel.defaultImageModel || fallbackModel,
|
imageModel: models.includes(config.imageModel) ? config.imageModel : fallbackImageModel,
|
||||||
videoModel: models.includes(config.videoModel) ? config.videoModel : modelChannel.defaultVideoModel || fallbackModel,
|
videoModel: models.includes(config.videoModel) ? config.videoModel : fallbackVideoModel,
|
||||||
textModel: models.includes(config.textModel) ? config.textModel : modelChannel.defaultTextModel || fallbackModel,
|
textModel: models.includes(config.textModel) ? config.textModel : fallbackTextModel || fallbackModel,
|
||||||
systemPrompt: modelChannel.systemPrompt,
|
systemPrompt: modelChannel.systemPrompt,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validDefault(model: string, models: string[]) {
|
||||||
|
return models.includes(model) ? model : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function preferredModel(models: string[], predicate: (model: string) => boolean) {
|
||||||
|
return models.find(predicate) || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function isVideoModelName(model: string) {
|
||||||
|
const value = model.toLowerCase();
|
||||||
|
return value.includes("seedance") || value.includes("video");
|
||||||
|
}
|
||||||
|
|
||||||
|
function isImageModelName(model: string) {
|
||||||
|
const value = model.toLowerCase();
|
||||||
|
return value.includes("seedream") || value.includes("gpt-image") || value.includes("image");
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTextModelName(model: string) {
|
||||||
|
return !isImageModelName(model) && !isVideoModelName(model);
|
||||||
|
}
|
||||||
|
|
||||||
function isAiConfigReady(config: AiConfig, model: string) {
|
function isAiConfigReady(config: AiConfig, model: string) {
|
||||||
return Boolean(model.trim()) && (config.channelMode === "remote" || Boolean(config.baseUrl.trim() && config.apiKey.trim()));
|
return Boolean(model.trim()) && (config.channelMode === "remote" || Boolean(config.baseUrl.trim() && config.apiKey.trim()));
|
||||||
}
|
}
|
||||||
@@ -112,7 +141,7 @@ export const useConfigStore = create<ConfigStore>()(
|
|||||||
partialize: (state) => ({ config: state.config }),
|
partialize: (state) => ({ config: state.config }),
|
||||||
merge: (persisted, current) => {
|
merge: (persisted, current) => {
|
||||||
const config = { ...defaultConfig, ...((persisted as Partial<ConfigStore>).config || {}) };
|
const config = { ...defaultConfig, ...((persisted as Partial<ConfigStore>).config || {}) };
|
||||||
return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720" } };
|
return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720", videoGenerateAudio: config.videoGenerateAudio || "true", videoWatermark: config.videoWatermark || "false" } };
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -125,7 +154,27 @@ export function useEffectiveConfig() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function buildApiUrl(baseUrl: string, path: string) {
|
export function buildApiUrl(baseUrl: string, path: string) {
|
||||||
const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, "");
|
let normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, "");
|
||||||
const apiBaseUrl = normalizedBaseUrl.endsWith("/v1") ? normalizedBaseUrl : `${normalizedBaseUrl}/v1`;
|
normalizedBaseUrl = normalizeArkPlanBaseUrl(normalizedBaseUrl);
|
||||||
|
const lowerBaseUrl = normalizedBaseUrl.toLowerCase();
|
||||||
|
const apiBaseUrl = lowerBaseUrl.endsWith("/v1") || lowerBaseUrl.endsWith("/api/v3") || lowerBaseUrl.endsWith("/api/plan/v3") ? normalizedBaseUrl : `${normalizedBaseUrl}/v1`;
|
||||||
return `${apiBaseUrl}${path}`;
|
return `${apiBaseUrl}${path}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeArkPlanBaseUrl(baseUrl: string) {
|
||||||
|
try {
|
||||||
|
const url = new URL(baseUrl);
|
||||||
|
const path = url.pathname.replace(/\/+$/, "");
|
||||||
|
const lowerPath = path.toLowerCase();
|
||||||
|
const arkPlanIndex = lowerPath.indexOf("/api/plan/v3");
|
||||||
|
if (arkPlanIndex < 0) return baseUrl;
|
||||||
|
const end = arkPlanIndex + "/api/plan/v3".length;
|
||||||
|
if (lowerPath.length !== end && lowerPath[end] !== "/") return baseUrl;
|
||||||
|
url.pathname = path.slice(0, end);
|
||||||
|
url.search = "";
|
||||||
|
url.hash = "";
|
||||||
|
return url.toString().replace(/\/+$/, "");
|
||||||
|
} catch {
|
||||||
|
return baseUrl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
export type ReferenceVideo = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
url: string;
|
||||||
|
storageKey?: string;
|
||||||
|
bytes?: number;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
durationMs?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ReferenceAudio = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
url: string;
|
||||||
|
storageKey?: string;
|
||||||
|
durationMs?: number;
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user