diff --git a/.env.example b/.env.example index 4708dad..cf0319d 100644 --- a/.env.example +++ b/.env.example @@ -9,12 +9,17 @@ JWT_EXPIRE_HOURS=168 # 后端监听端口 PORT=8080 +# 公开访问地址，用于把本地上传的 Seedance 参考图/视频暴露给火山方舟拉取。 +# 线上部署时填写站点根地址，例如：https://your-domain.example.com +# PUBLIC_BASE_URL=https://your-domain.example.com + # 前端开发代理默认使用 http://127.0.0.1:8080，如后端开发端口不同，启动前端时可单独设置 API_BASE_URL。 # API_BASE_URL=http://127.0.0.1:8080 # 数据库配置，默认使用本地 SQLite。 STORAGE_DRIVER=sqlite # sqlite: DATABASE_DSN=data/infinite-canvas.db +# Docker 部署时建议使用绝对路径，避免工作目录变化后写入临时库：DATABASE_DSN=/app/data/infinite-canvas.db # mysql: DATABASE_DSN=user:password@tcp(127.0.0.1:3306)/infinite_canvas?parseTime=true # postgres: DATABASE_DSN=postgres://user:password@127.0.0.1:5432/infinite_canvas?sslmode=disable DATABASE_DSN=data/infinite-canvas.db diff --git a/README.md b/README.md index 077c2cc..15a4d6f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ## 核心功能 - 无限画布：多画布项目、节点拖拽缩放、连线、小地图、撤销重做、导入导出。 -- AI 创作：支持 OpenAI 兼容接口的文生图、图生图、参考图编辑和文本问答。 +- AI 创作：支持 OpenAI 兼容接口的文生图、图生图、参考图编辑、文本问答和视频生成；Seedance 2.0 可通过火山方舟 Agent Plan 接入。 - 画布助手：围绕选中节点和上游节点对话、生图，并把结果插回画布。 - 提示词库：抓取多个 GitHub 开源项目，按案例整理数百个图片提示词。 diff --git a/config/config.go b/config/config.go index f91f51f..a4ed42c 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,8 @@ package config import ( "crypto/rand" "encoding/base64" + "os" + "path/filepath" "strings" "github.com/caarlos0/env/v11" @@ -17,6 +19,7 @@ type Config struct { JWTExpireHours int `env:"JWT_EXPIRE_HOURS" envDefault:"168"` StorageDriver string `env:"STORAGE_DRIVER" envDefault:"sqlite"` DatabaseDSN string `env:"DATABASE_DSN" envDefault:"data/infinite-canvas.db"` + PublicBaseURL string `env:"PUBLIC_BASE_URL"` LinuxDoAuthorizeURL string `env:"LINUX_DO_AUTHORIZE_URL" envDefault:"https://connect.linux.do/oauth2/authorize"` LinuxDoTokenURL string `env:"LINUX_DO_TOKEN_URL" envDefault:"https://connect.linux.do/oauth2/token"` LinuxDoUserInfoURL string `env:"LINUX_DO_USERINFO_URL" envDefault:"https://connect.linux.do/api/user"` @@ -29,6 +32,7 @@ func Load() error { if err := env.Parse(&Cfg); err != nil { return err } + normalizeDockerSQLiteDSN("/app/data") if strings.TrimSpace(Cfg.JWTSecret) == "" || Cfg.JWTSecret == "infinite-canvas" { secret, err := randomSecret() if err != nil { @@ -39,6 +43,33 @@ func Load() error { return nil } +func normalizeDockerSQLiteDSN(appDataDir string) { + driver := strings.ToLower(strings.TrimSpace(Cfg.StorageDriver)) + if driver != "" && driver != "sqlite" { + return + } + dsn := strings.TrimSpace(Cfg.DatabaseDSN) + if dsn == "" || dsn == ":memory:" || strings.HasPrefix(dsn, "file:") { + return + } + pathPart, suffix := dsn, "" + if index := strings.Index(dsn, "?"); index >= 0 { + pathPart = dsn[:index] + suffix = dsn[index:] + } + if filepath.IsAbs(pathPart) { + return + } + slashPath := filepath.ToSlash(pathPart) + if slashPath != "data" && !strings.HasPrefix(slashPath, "data/") { + return + } + if _, err := os.Stat(appDataDir); err != nil { + return + } + Cfg.DatabaseDSN = filepath.Join(filepath.Dir(appDataDir), filepath.FromSlash(slashPath)) + suffix +} + func randomSecret() (string, error) { buf := make([]byte, 32) if _, err := rand.Read(buf); err != nil { diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..2f18680 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,33 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNormalizeDockerSQLiteDSNUsesMountedDataDir(t *testing.T) { + root := t.TempDir() + appDataDir := filepath.Join(root, "data") + if err := os.MkdirAll(appDataDir, 0755); err != nil { + t.Fatal(err) + } + Cfg = Config{StorageDriver: "sqlite", DatabaseDSN: "data/infinite-canvas.db?_pragma=busy_timeout(5000)"} + + normalizeDockerSQLiteDSN(appDataDir) + + want := filepath.Join(root, "data", "infinite-canvas.db") + "?_pragma=busy_timeout(5000)" + if Cfg.DatabaseDSN != want { + t.Fatalf("DatabaseDSN = %q, want %q", Cfg.DatabaseDSN, want) + } +} + +func TestNormalizeDockerSQLiteDSNLeavesLocalPathWithoutMountedDataDir(t *testing.T) { + Cfg = Config{StorageDriver: "sqlite", DatabaseDSN: "data/infinite-canvas.db"} + + normalizeDockerSQLiteDSN(filepath.Join(t.TempDir(), "missing-data")) + + if Cfg.DatabaseDSN != "data/infinite-canvas.db" { + t.Fatalf("DatabaseDSN = %q, want relative local path", Cfg.DatabaseDSN) + } +} diff --git a/docs/canvas-node-manual.md b/docs/canvas-node-manual.md index 845b9a6..4849bac 100644 --- a/docs/canvas-node-manual.md +++ b/docs/canvas-node-manual.md @@ -20,7 +20,7 @@ - 当文本节点已有内容时，输入框用于填写想把本段文本修改成什么；点击发送后，会在右侧生成新的文本节点，并自动连接原节点和新节点。 - 输入内容可以手写，也可以从提示词库选择。 - 对话框里的模型下拉来自全局配置里已拉取的模型列表；选择结果只作用于当前节点，不会修改其它节点或全局默认模型。 -- 如果下拉中没有模型，需要先打开配置弹窗拉取模型列表，并设置默认生图模型和默认文本模型。 +- 如果下拉中没有模型，需要先打开配置弹窗拉取模型列表，并设置默认生图模型和默认文本模型。火山方舟 Agent Plan 若提示不支持 `/models`，请手动填写模型名。 ### 用文本节点生成图片 @@ -45,7 +45,11 @@ - 视频节点使用原生播放器展示内容，可在节点内直接播放、暂停和拖动进度。 - 空视频节点下方对话框可输入提示词生成视频，结果会回填到当前节点。 - 从文本、图片或配置节点创建视频生成时，会在右侧生成新的视频节点并自动连接。 -- 视频生成接口使用 OpenAI 兼容的 `POST /v1/videos`、`GET /v1/videos/{id}` 和 `GET /v1/videos/{id}/content`。 +- 生成配置节点的视频模式会读取上游文本作为 prompt，读取上游图片作为参考图，读取上游视频作为参考视频，并在输入预览里显示参考视频。 +- 视频生成接口支持 OpenAI 风格的 `POST /v1/videos`、`GET /v1/videos/{id}` 和 `GET /v1/videos/{id}/content`。 +- 使用火山方舟 Agent Plan / Seedance 2.0 时，Base URL 配置为 `https://ark.cn-beijing.volces.com/api/plan/v3`，模型名使用 Seedance 2.0 对应模型；系统会改用 `POST /contents/generations/tasks` 创建异步任务，并轮询 `GET /contents/generations/tasks/{id}`。 +- Agent Plan 专属 `/api/plan/v3` 当前未提供 OpenAI `/models` 模型列表接口，后台不会伪造模型列表；请手动填写 `doubao-seedance-2.0` 或文档列出的其他可用模型。 +- Seedance 参考视频必须是公网可访问 URL，或由本项目后端在配置 `PUBLIC_BASE_URL` 后上传并暴露的参考素材 URL。本地/内网地址无法被火山服务器拉取。 ### 推荐流程 diff --git a/docs/features.md b/docs/features.md index bec3f05..a6963d0 100644 --- a/docs/features.md +++ b/docs/features.md @@ -52,12 +52,26 @@ ## AI 生成 -项目通过前端直接请求 OpenAI 兼容接口： +项目支持两种 AI 调用方式： + +- 本地直连：前端使用本地配置的 Base URL、API Key 和 Model 直接请求 OpenAI 兼容接口。 +- 后台渠道：前端请求本项目后端 `/api/v1/*` 代理接口，后端按模型选择管理后台配置的渠道。 + +OpenAI 兼容图像和文本能力继续复用现有接口： - `/v1/images/generations`：文生图。 - `/v1/images/edits`：图生图/参考图编辑。 - `/v1/chat/completions`：文本问答和带图问答。 -- `/v1/models`：读取模型列表。 +- `/v1/models`：读取模型列表；火山方舟 Agent Plan 专属 `/api/plan/v3` 当前未提供 OpenAI `/models` 模型列表接口，需要手动填写模型名。 + +视频能力支持两类接口： + +- OpenAI 风格视频：`POST /v1/videos`、`GET /v1/videos/{id}`、`GET /v1/videos/{id}/content`。 +- 火山方舟 Agent Plan / Seedance 2.0：Base URL 使用 `https://ark.cn-beijing.volces.com/api/plan/v3`，创建任务为 `POST /contents/generations/tasks`，查询任务为 `GET /contents/generations/tasks/{id}`，成功结果读取 `content.video_url`。 + +Base URL 如果已经以 `/v1`、`/api/v3` 或 `/api/plan/v3` 结尾，系统不会再追加 `/v1`。因此 cpa 反代或火山方舟 Agent Plan 可以继续通过现有 Base URL + API Key + Model 方式配置，不需要新增火山生图 Provider。 + +后台“拉取模型列表”会尝试真实请求 OpenAI `/models`，不会为 Agent Plan 伪造模型结果。如果火山方舟 Agent Plan 返回 404，请手动增加 `doubao-seedance-2.0` 或文档列出的其他模型名。 可配置项： @@ -67,9 +81,13 @@ - 图片质量。 - 图片比例。 - 生成数量。 +- 视频模型。 +- 视频比例、清晰度、时长、生成声音和水印。 普通图片/文本节点可以直接输入提示词生成结果。生成配置节点可以读取上游节点内容，并按节点自己的配置批量生成多个图片或文本结果。生成配置节点支持预览当前提示词和参考图输入，并调整输入顺序。 +视频生成可从文本节点读取 prompt，从图片节点读取参考图，从视频节点读取参考视频，从音频节点读取参考音频。Seedance 2.0 支持最多 9 张参考图、3 个参考视频、3 个参考音频；分辨率支持 `480p`、`720p`、`1080p`（fast 模型不支持 `1080p`），比例支持 `16:9`、`4:3`、`1:1`、`3:4`、`9:16`、`21:9`、`adaptive`，时长支持 4-15 秒或智能时长。生成成功后会把视频插入画布为视频节点并使用原生播放器预览。Seedance 参考视频和参考音频需要公网可访问 URL；本地上传素材会先通过 `/api/v1/media/references` 保存到服务端，再由 `PUBLIC_BASE_URL` 生成可供火山服务器拉取的公开链接。 + ## 画布助手 画布右侧助手面板支持： @@ -161,6 +179,8 @@ ## 当前限制 - 画布项目和“我的素材”目前只保存在浏览器本地，不会随账号同步。 -- AI API Key 目前保存在浏览器本地，并由浏览器直接请求配置的 OpenAI 兼容接口。 +- 本地直连模式下，AI API Key 保存在浏览器本地，并由浏览器直接请求配置的 OpenAI 兼容接口；只适合本地或个人使用，公网多人使用不安全。公网部署推荐使用后台渠道，把真实密钥保存在服务端配置中。 - 服务器素材库目前主要保存 URL 或文本，暂未提供文件上传接口。 +- Seedance 本地参考图/视频上传依赖 `PUBLIC_BASE_URL`，如果服务部署在 localhost、内网或不可被火山访问的地址，火山无法拉取参考素材。 +- Seedance 返回远程视频 URL 时，前端会尽量下载为本地 Blob 持久化；如果因 CORS 或网络限制无法下载，会保留远程 URL，后续是否可播放取决于上游 URL 的有效期。 - 画布更适合桌面端使用，移动端触控体验还未系统完善。 diff --git a/docs/pending-test.md b/docs/pending-test.md index 989c14d..df1e51c 100644 --- a/docs/pending-test.md +++ b/docs/pending-test.md @@ -1,6 +1,11 @@ # 待测试 - 外部软件可通过 URL 查询参数 `baseUrl`/`baseurl` 和 `apiKey`/`apikey` 跳转到前端；读取后会从地址栏移除这些参数，后台允许自定义渠道时会自动切到自定义渠道、填入配置并打开配置弹窗，未允许时会打开配置弹窗并提示无法导入。 +- 生图工作台和画布生图会把参考图按当前顺序显示为 `图片1`、`图片2` 等编号，并在图生图请求的实际提示词中注入编号说明；需要验证 `/image` 参考图排序、画布配置节点输入顺序和画布助手参考图编号一致。 +- GPT Image 生图请求会在前端把 `9:16`、`16:9` 等比例转换成合法 `WIDTHxHEIGHT` 尺寸，并在非法尺寸时直接显示中文错误，避免上游返回 `invalid_value Invalid size`。 +- Docker 部署时，`DATABASE_DSN=data/infinite-canvas.db` 会在存在 `/app/data` 挂载目录时自动归一到 `/app/data/infinite-canvas.db`，需要验证后台模型配置不会再因为工作目录变为 `/app/web` 而读到空库。 +- Seedance 参考视频被火山判定包含真人或隐私信息时，前端错误摘要会提示改用不含真人的视频、官方允许的模型产物或已授权的 `asset://` 素材；参考素材上传目录改为跟随 SQLite 数据目录，并补充公开素材的 HEAD 访问。 +- Seedance 参考素材失败原因排查：后端会把火山上游错误摘要返回给前端；`/video` 和画布视频生成会按 `图片1/视频1/音频1` 自动编号参考素材，并在实际请求提示词中注入编号说明；参考视频会在请求前校验大小、时长、宽高、宽高比和像素总量。 - 画布项目导出改为下载 `.zip` 压缩包，包内包含 `projects.json` 和当前画布引用到的本地图片、视频文件，避免只导出 JSON 时丢失媒体内容。 - 画布库支持多选后一键导出多个画布项目，导出的压缩包可一次恢复多个项目。 - 画布项目导入改为读取新版 `.zip` 压缩包，会先按 `projects.json` 中的文件映射恢复图片、视频到本地存储，再插入画布项目，导入成功后仍停留在画布库。 @@ -22,3 +27,10 @@ - 视频生成前端会识别后端 `{ code, msg }` 错误响应，创建失败不再继续轮询 `undefined`。 - 新增 `/video` 视频创作台页面，参考生图工作台布局，支持提示词、参考图、视频参数、生成结果、保存素材、下载和本地生成记录；清晰度和秒数均支持常用值选择与手动输入，生成记录只保存媒体 `storageKey` 并可回填本次提示词、参考图和参数。 - 视频创作台生成前会把模型、尺寸、秒数、清晰度归一化为视频接口支持的参数，并展示后端返回的错误信息，避免页面侧残留的生图参数影响视频请求。 +- 火山方舟 Agent Plan / Seedance 2.0 视频生成需要在真实账号下验证：`/contents/generations/tasks` 创建任务、轮询状态、`content.video_url` 回填画布，以及 401/403/429/超时错误提示。 +- 管理后台保存私有渠道后，需要验证所有已启用渠道里的模型会自动出现在公开 `availableModels`，并且 `defaultVideoModel`、`defaultImageModel`、`defaultTextModel` 在为空或失效时会自动修复，前台不再显示旧的 `grok` 默认值。 +- `/video` 和画布视频设置已按 Seedance 2.0 增加分辨率、比例、4-15 秒/智能时长、生成声音和水印参数；需要在真实浏览器里验证参数回填、生成记录和画布节点配置都能保持一致。 +- `/video` 支持最多 9 张参考图、3 个参考视频、3 个参考音频；需要验证格式、大小、音频时长提示和生成请求中的 `reference_image`、`reference_video`、`reference_audio` 组装。 +- 画布新增音频节点，支持上传、拖入、播放、移动、缩放、删除，并可作为上游参考音频参与 Seedance 视频生成；需要验证刷新后本地音频 URL 能恢复。 +- `PUBLIC_BASE_URL` 已配置公网域名时，需要验证本地上传参考视频和参考音频能被火山拉取；未配置或配置为内网地址时，需要验证前端能给出明确提示。 +- Seedance 返回远程视频 URL 但浏览器无法下载为 Blob 时，需要验证视频节点刷新后仍保留远程 URL，并确认上游 URL 有效期限制。 diff --git a/docs/system-settings.md b/docs/system-settings.md index 74a267b..0996665 100644 --- a/docs/system-settings.md +++ b/docs/system-settings.md @@ -41,11 +41,12 @@ | 字段 | 类型 | 说明 | | --- | --- | --- | -| `availableModels` | string[] | 系统可用模型，由管理员手动选择；页面下拉选项可来自私有渠道模型 | +| `availableModels` | string[] | 系统可用模型；保存设置时会自动合并所有已启用私有渠道的模型 | | `modelCosts` | object[] | 模型算力点配置，后端模型接口调用前按模型预扣，上游失败时返还；未配置默认不扣除 | -| `defaultModel` | string | 默认模型，从 `availableModels` 中选择 | -| `defaultImageModel` | string | 默认图片模型，从 `availableModels` 中选择 | -| `defaultTextModel` | string | 默认文本模型，从 `availableModels` 中选择 | +| `defaultModel` | string | 默认模型，从 `availableModels` 中选择；为空或失效时优先选择文本模型 | +| `defaultImageModel` | string | 默认图片模型，从 `availableModels` 中选择；为空或失效时优先选择 `seedream`、`image`、`gpt-image` 模型 | +| `defaultVideoModel` | string | 默认视频模型，从 `availableModels` 中选择；为空或失效时优先选择 `seedance`、`video` 模型 | +| `defaultTextModel` | string | 默认文本模型，从 `availableModels` 中选择；为空或失效时优先选择非图片/视频模型 | | `systemPrompt` | string | 系统提示词 | | `allowCustomChannel` | boolean | 是否允许用户在配置弹窗中切换为本地直连渠道，默认允许 | diff --git a/handler/ai.go b/handler/ai.go index f2c3389..26d5d7d 100644 --- a/handler/ai.go +++ b/handler/ai.go @@ -49,6 +49,7 @@ func proxyAIGetRequest(w http.ResponseWriter, r *http.Request, path string) { Fail(w, "AI 接口请求失败") return } + path = resolveAIProxyPath(channel.BaseURL, modelName, path) request, err := http.NewRequest(http.MethodGet, service.BuildModelChannelURL(channel, path), nil) if err != nil { Fail(w, "AI 接口请求失败") @@ -83,6 +84,7 @@ func proxyAIRequest(w http.ResponseWriter, r *http.Request, path string) { Fail(w, "AI 接口请求失败") return } + path = resolveAIProxyPath(channel.BaseURL, modelName, path) request, err := http.NewRequest(http.MethodPost, service.BuildModelChannelURL(channel, path), bytes.NewReader(body)) if err != nil { log.Printf("AI proxy build request failed: url=%s err=%v", service.BuildModelChannelURL(channel, path), err) @@ -117,12 +119,12 @@ func copyAIResponse(w http.ResponseWriter, request *http.Request, onFailure func defer response.Body.Close() if response.StatusCode >= http.StatusBadRequest { - payload, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - log.Printf("AI upstream error: url=%s status=%d body=%s", request.URL.String(), response.StatusCode, strings.TrimSpace(string(payload))) + body, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + log.Printf("AI upstream error: url=%s status=%d", request.URL.String(), response.StatusCode) if onFailure != nil { onFailure() } - Fail(w, "AI 接口请求失败") + Fail(w, aiUpstreamStatusMessage(response.StatusCode, body)) return } @@ -207,6 +209,95 @@ func readAIRequestCount(body []byte, contentType string) int { var errMissingModel = &aiError{"缺少模型名称"} +func resolveAIProxyPath(baseURL string, modelName string, path string) string { + if !isArkSeedanceVideo(baseURL, modelName) { + return path + } + if path == "/videos" { + return "/contents/generations/tasks" + } + if strings.HasPrefix(path, "/videos/") && !strings.HasSuffix(path, "/content") { + return "/contents/generations/tasks/" + strings.TrimPrefix(path, "/videos/") + } + return path +} + +func isArkSeedanceVideo(baseURL string, modelName string) bool { + base := strings.ToLower(baseURL) + model := strings.ToLower(modelName) + return strings.Contains(model, "seedance") || strings.Contains(model, "doubao-seedance") || strings.Contains(base, "/api/plan/v3") +} + +func aiStatusMessage(statusCode int) string { + switch statusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return "AI 接口鉴权失败，请检查 API Key、套餐权限或模型权限" + case http.StatusTooManyRequests: + return "AI 接口限流或额度不足，请稍后重试或检查额度" + default: + return "AI 接口请求失败" + } +} + +func aiUpstreamStatusMessage(statusCode int, body []byte) string { + base := aiStatusMessage(statusCode) + detail := aiUpstreamErrorDetail(body) + if detail == "" { + return base + } + return base + "：" + detail +} + +func aiUpstreamErrorDetail(body []byte) string { + text := strings.TrimSpace(string(body)) + if text == "" { + return "" + } + var payload struct { + Msg string `json:"msg"` + Message string `json:"message"` + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(body, &payload); err == nil { + if payload.Error.Message != "" { + if detail := friendlyUpstreamError(payload.Error.Code, payload.Error.Message); detail != "" { + return safeUpstreamText(detail) + } + if payload.Error.Code != "" { + return safeUpstreamText(payload.Error.Code + " " + payload.Error.Message) + } + return safeUpstreamText(payload.Error.Message) + } + if payload.Msg != "" { + return safeUpstreamText(payload.Msg) + } + if payload.Message != "" { + return safeUpstreamText(payload.Message) + } + } + return safeUpstreamText(text) +} + +func friendlyUpstreamError(code string, message string) string { + lowerCode := strings.ToLower(strings.TrimSpace(code)) + if strings.Contains(lowerCode, "inputvideosensitivecontentdetected") || strings.Contains(lowerCode, "privacyinformation") { + return strings.TrimSpace(code + " 参考视频疑似包含真人或隐私信息，火山方舟拒绝使用普通 URL 作为真人视频参考；请改用不含真人的视频、官方允许的模型产物，或已授权的 asset:// 素材。原始错误：" + message) + } + return "" +} + +func safeUpstreamText(text string) string { + text = strings.Join(strings.Fields(strings.TrimSpace(text)), " ") + runes := []rune(text) + if len(runes) > 300 { + return string(runes[:300]) + "..." + } + return text +} + type aiError struct { message string } diff --git a/handler/ai_test.go b/handler/ai_test.go new file mode 100644 index 0000000..2eaa897 --- /dev/null +++ b/handler/ai_test.go @@ -0,0 +1,27 @@ +package handler + +import ( + "strings" + "testing" +) + +func TestAIUpstreamErrorDetail(t *testing.T) { + got := aiUpstreamErrorDetail([]byte(`{"error":{"code":"InvalidParameter","message":"reference video fps is invalid"}}`)) + if got != "InvalidParameter reference video fps is invalid" { + t.Fatalf("detail = %q", got) + } +} + +func TestAIUpstreamErrorDetailExplainsSensitiveVideo(t *testing.T) { + got := aiUpstreamErrorDetail([]byte(`{"error":{"code":"InputVideoSensitiveContentDetected.PrivacyInformation","message":"The request failed because the input video may contain real person."}}`)) + if !strings.Contains(got, "参考视频疑似包含真人") || !strings.Contains(got, "asset://") { + t.Fatalf("detail = %q", got) + } +} + +func TestSafeUpstreamTextTruncates(t *testing.T) { + got := safeUpstreamText(strings.Repeat("错", 320)) + if len([]rune(got)) != 303 { + t.Fatalf("truncated rune length = %d", len([]rune(got))) + } +} diff --git a/handler/media_reference.go b/handler/media_reference.go new file mode 100644 index 0000000..94d64b7 --- /dev/null +++ b/handler/media_reference.go @@ -0,0 +1,238 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/basketikun/infinite-canvas/config" + "github.com/google/uuid" +) + +const ( + referenceMediaMaxBytes = 80 << 20 + referenceImageMaxBytes = 30 << 20 + referenceVideoMaxBytes = 50 << 20 + referenceAudioMaxBytes = 15 << 20 + referenceImageAllowedText = "jpeg/png/webp/bmp/gif/heic/heif 图片" + referenceVideoAllowedText = "mp4/mov 视频" + referenceAudioAllowedText = "mp3/wav 音频" + referenceMediaAllowedText = referenceImageAllowedText + "、" + referenceVideoAllowedText + "或" + referenceAudioAllowedText +) + +type referenceMediaUploadResult struct { + ID string `json:"id"` + URL string `json:"url"` + MimeType string `json:"mimeType"` + Bytes int64 `json:"bytes"` +} + +func UploadReferenceMedia(w http.ResponseWriter, r *http.Request) { + publicBaseURL := strings.TrimRight(strings.TrimSpace(config.Cfg.PublicBaseURL), "/") + if publicBaseURL == "" { + Fail(w, "未配置 PUBLIC_BASE_URL，无法把本地参考素材提供给火山方舟访问") + return + } + r.Body = http.MaxBytesReader(w, r.Body, referenceMediaMaxBytes+1) + if err := r.ParseMultipartForm(referenceMediaMaxBytes); err != nil { + Fail(w, "参考素材过大或上传格式不正确") + return + } + if r.MultipartForm != nil { + defer r.MultipartForm.RemoveAll() + } + file, header, err := r.FormFile("file") + if err != nil { + Fail(w, "请上传参考图片或视频") + return + } + defer file.Close() + + mimeType, ext, ok := normalizeReferenceMediaType(header.Header.Get("Content-Type"), filepath.Ext(header.Filename)) + if !ok { + Fail(w, "参考素材格式不支持，请使用 "+referenceMediaAllowedText) + return + } + if err := os.MkdirAll(referenceMediaDir(), 0o755); err != nil { + Fail(w, "参考素材保存失败") + return + } + id := uuid.NewString() + ext + targetPath := filepath.Join(referenceMediaDir(), id) + target, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + Fail(w, "参考素材保存失败") + return + } + bytes, copyErr := io.Copy(target, file) + closeErr := target.Close() + if copyErr != nil || closeErr != nil { + _ = os.Remove(targetPath) + Fail(w, "参考素材保存失败") + return + } + if bytes <= 0 { + _ = os.Remove(targetPath) + Fail(w, "参考素材为空") + return + } + if limit := referenceMediaTypeMaxBytes(mimeType); limit > 0 && bytes > limit { + _ = os.Remove(targetPath) + Fail(w, referenceMediaSizeMessage(mimeType)) + return + } + OK(w, referenceMediaUploadResult{ + ID: id, + URL: fmt.Sprintf("%s/api/media/references/%s", publicBaseURL, id), + MimeType: mimeType, + Bytes: bytes, + }) +} + +func ReferenceMedia(w http.ResponseWriter, r *http.Request, id string) { + if id == "" || id != filepath.Base(id) || strings.Contains(id, "..") { + http.NotFound(w, r) + return + } + path := filepath.Join(referenceMediaDir(), id) + file, err := os.Open(path) + if err != nil { + http.NotFound(w, r) + return + } + defer file.Close() + info, err := file.Stat() + if err != nil || info.IsDir() { + http.NotFound(w, r) + return + } + if mimeType := mimeTypeByReferenceMediaExt(filepath.Ext(id)); mimeType != "" { + w.Header().Set("Content-Type", mimeType) + } + w.Header().Set("Cache-Control", "public, max-age=86400") + http.ServeContent(w, r, id, info.ModTime(), file) +} + +func referenceMediaDir() string { + return filepath.Join(referenceDataDir(), "reference-media") +} + +func referenceDataDir() string { + driver := strings.ToLower(strings.TrimSpace(config.Cfg.StorageDriver)) + dsn := strings.TrimSpace(config.Cfg.DatabaseDSN) + if (driver == "" || driver == "sqlite") && dsn != "" && dsn != ":memory:" && !strings.HasPrefix(dsn, "file:") { + pathPart := dsn + if index := strings.Index(dsn, "?"); index >= 0 { + pathPart = dsn[:index] + } + if filepath.IsAbs(pathPart) { + return filepath.Dir(pathPart) + } + } + if _, err := os.Stat("/app/data"); err == nil { + return "/app/data" + } + return "data" +} + +func normalizeReferenceMediaType(contentType string, ext string) (string, string, bool) { + contentType = strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) + ext = strings.ToLower(strings.TrimSpace(ext)) + if contentType == "" || contentType == "application/octet-stream" { + contentType = mimeTypeByReferenceMediaExt(ext) + } + if fixedExt := referenceMediaExtByMimeType(contentType); fixedExt != "" { + return contentType, fixedExt, true + } + if mimeType := mimeTypeByReferenceMediaExt(ext); mimeType != "" { + return mimeType, ext, true + } + return "", "", false +} + +func referenceMediaExtByMimeType(mimeType string) string { + switch strings.ToLower(mimeType) { + case "image/jpeg", "image/jpg": + return ".jpg" + case "image/png": + return ".png" + case "image/webp": + return ".webp" + case "image/bmp": + return ".bmp" + case "image/gif": + return ".gif" + case "image/heic": + return ".heic" + case "image/heif": + return ".heif" + case "video/mp4": + return ".mp4" + case "video/quicktime", "video/mov": + return ".mov" + case "audio/mpeg", "audio/mp3": + return ".mp3" + case "audio/wav", "audio/x-wav", "audio/wave": + return ".wav" + default: + return "" + } +} + +func mimeTypeByReferenceMediaExt(ext string) string { + switch strings.ToLower(ext) { + case ".jpg", ".jpeg": + return "image/jpeg" + case ".png": + return "image/png" + case ".webp": + return "image/webp" + case ".bmp": + return "image/bmp" + case ".gif": + return "image/gif" + case ".heic": + return "image/heic" + case ".heif": + return "image/heif" + case ".mp4": + return "video/mp4" + case ".mov": + return "video/quicktime" + case ".mp3": + return "audio/mpeg" + case ".wav": + return "audio/wav" + default: + return "" + } +} + +func referenceMediaTypeMaxBytes(mimeType string) int64 { + if strings.HasPrefix(mimeType, "image/") { + return referenceImageMaxBytes + } + if strings.HasPrefix(mimeType, "video/") { + return referenceVideoMaxBytes + } + if strings.HasPrefix(mimeType, "audio/") { + return referenceAudioMaxBytes + } + return referenceMediaMaxBytes +} + +func referenceMediaSizeMessage(mimeType string) string { + if strings.HasPrefix(mimeType, "image/") { + return "参考图片超过大小限制，请使用 30MB 以内的图片" + } + if strings.HasPrefix(mimeType, "video/") { + return "参考视频超过大小限制，请使用 50MB 以内的 mp4/mov 视频" + } + if strings.HasPrefix(mimeType, "audio/") { + return "参考音频超过大小限制，请使用 15MB 以内的 mp3/wav 音频" + } + return "参考素材超过大小限制" +} diff --git a/handler/media_reference_test.go b/handler/media_reference_test.go new file mode 100644 index 0000000..826e045 --- /dev/null +++ b/handler/media_reference_test.go @@ -0,0 +1,56 @@ +package handler + +import ( + "path/filepath" + "testing" + + "github.com/basketikun/infinite-canvas/config" +) + +func TestNormalizeReferenceMediaTypeSupportsAudio(t *testing.T) { + tests := []struct { + name string + contentType string + ext string + wantMime string + wantExt string + }{ + {name: "mp3 mime", contentType: "audio/mpeg", ext: ".bin", wantMime: "audio/mpeg", wantExt: ".mp3"}, + {name: "wav ext fallback", contentType: "application/octet-stream", ext: ".wav", wantMime: "audio/wav", wantExt: ".wav"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mimeType, ext, ok := normalizeReferenceMediaType(tt.contentType, tt.ext) + if !ok { + t.Fatal("expected media type to be accepted") + } + if mimeType != tt.wantMime || ext != tt.wantExt { + t.Fatalf("got (%q, %q), want (%q, %q)", mimeType, ext, tt.wantMime, tt.wantExt) + } + }) + } +} + +func TestReferenceMediaTypeMaxBytes(t *testing.T) { + if got := referenceMediaTypeMaxBytes("audio/mpeg"); got != referenceAudioMaxBytes { + t.Fatalf("audio max bytes = %d, want %d", got, referenceAudioMaxBytes) + } + if got := referenceMediaTypeMaxBytes("video/mp4"); got != referenceVideoMaxBytes { + t.Fatalf("video max bytes = %d, want %d", got, referenceVideoMaxBytes) + } + if got := referenceMediaTypeMaxBytes("image/png"); got != referenceImageMaxBytes { + t.Fatalf("image max bytes = %d, want %d", got, referenceImageMaxBytes) + } +} + +func TestReferenceMediaDirUsesAbsoluteSQLiteDataDir(t *testing.T) { + previous := config.Cfg + t.Cleanup(func() { config.Cfg = previous }) + root := t.TempDir() + config.Cfg = config.Config{StorageDriver: "sqlite", DatabaseDSN: filepath.Join(root, "infinite-canvas.db")} + + if got := referenceMediaDir(); got != filepath.Join(root, "reference-media") { + t.Fatalf("referenceMediaDir = %q", got) + } +} diff --git a/router/router.go b/router/router.go index acf5bb7..6493903 100644 --- a/router/router.go +++ b/router/router.go @@ -22,11 +22,18 @@ func New() *gin.Engine { api.GET("/auth/linux-do/callback", gin.WrapF(handler.LinuxDoCallback)) api.GET("/auth/me", middleware.OptionalAuth, gin.WrapF(handler.CurrentUser)) api.GET("/settings", gin.WrapF(handler.Settings)) + api.GET("/media/references/:id", func(c *gin.Context) { + handler.ReferenceMedia(c.Writer, c.Request, c.Param("id")) + }) + api.HEAD("/media/references/:id", func(c *gin.Context) { + handler.ReferenceMedia(c.Writer, c.Request, c.Param("id")) + }) v1 := api.Group("/v1", middleware.UserAuth) v1.POST("/images/generations", gin.WrapF(handler.AIImagesGenerations)) v1.POST("/images/edits", gin.WrapF(handler.AIImagesEdits)) v1.POST("/chat/completions", gin.WrapF(handler.AIChatCompletions)) v1.POST("/videos", gin.WrapF(handler.AIVideos)) + v1.POST("/media/references", gin.WrapF(handler.UploadReferenceMedia)) v1.GET("/videos/:id", func(c *gin.Context) { handler.AIVideo(c.Writer, c.Request, c.Param("id")) }) diff --git a/service/settings.go b/service/settings.go index bbf088f..121c802 100644 --- a/service/settings.go +++ b/service/settings.go @@ -7,16 +7,20 @@ import ( "io" "math/rand" "net/http" + "net/url" "sort" "strings" + "time" "github.com/basketikun/infinite-canvas/model" "github.com/basketikun/infinite-canvas/repository" ) +var adminModelHTTPClient = &http.Client{Timeout: 30 * time.Second} + func PublicSettings() (model.PublicSetting, error) { settings, err := repository.GetSettings() - return normalizePublicSetting(settings.Public), err + return normalizeSettings(settings).Public, err } func AdminSettings() (model.Settings, error) { @@ -52,16 +56,23 @@ func AdminTestChannelModel(index *int, channel model.ModelChannel, modelName str if err != nil { return "", err } + if isArkAgentPlanChannel(resolved) || isSeedanceModelName(modelName) { + return testArkSeedanceChannelModel(resolved, modelName) + } return testAdminChannelModel(resolved, modelName) } func normalizeSettings(settings model.Settings) model.Settings { - settings.Public = normalizePublicSetting(settings.Public) settings.Private = normalizePrivateSetting(settings.Private) + settings.Public = normalizePublicSettingWithChannels(settings.Public, settings.Private.Channels) return settings } func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting { + return normalizePublicSettingWithChannels(setting, nil) +} + +func normalizePublicSettingWithChannels(setting model.PublicSetting, channels []model.ModelChannel) model.PublicSetting { if setting.ModelChannel.AvailableModels == nil { setting.ModelChannel.AvailableModels = []string{} } @@ -82,6 +93,16 @@ func normalizePublicSetting(setting model.PublicSetting) model.PublicSetting { enabled := true setting.Auth.AllowRegister = &enabled } + enabledModels := enabledChannelModels(channels) + if len(enabledModels) > 0 { + setting.ModelChannel.AvailableModels = enabledModels + } else { + setting.ModelChannel.AvailableModels = uniqueModelNames(setting.ModelChannel.AvailableModels) + } + setting.ModelChannel.DefaultTextModel = repairDefaultModel(setting.ModelChannel.DefaultTextModel, setting.ModelChannel.AvailableModels, isTextModelName) + setting.ModelChannel.DefaultImageModel = repairDefaultModel(setting.ModelChannel.DefaultImageModel, setting.ModelChannel.AvailableModels, isImageModelName) + setting.ModelChannel.DefaultVideoModel = repairDefaultModel(setting.ModelChannel.DefaultVideoModel, setting.ModelChannel.AvailableModels, isVideoModelName) + setting.ModelChannel.DefaultModel = repairDefaultModel(setting.ModelChannel.DefaultModel, setting.ModelChannel.AvailableModels, isTextModelName) return setting } @@ -179,13 +200,101 @@ func SelectModelChannel(modelName string) (model.ModelChannel, error) { } func BuildModelChannelURL(channel model.ModelChannel, path string) string { - baseURL := strings.TrimRight(channel.BaseURL, "/") - if !strings.HasSuffix(baseURL, "/v1") { + baseURL := normalizeModelChannelBaseURL(channel.BaseURL) + lowerBaseURL := strings.ToLower(baseURL) + if !strings.HasSuffix(lowerBaseURL, "/v1") && !strings.HasSuffix(lowerBaseURL, "/api/v3") && !strings.HasSuffix(lowerBaseURL, "/api/plan/v3") { baseURL += "/v1" } return baseURL + path } +func normalizeModelChannelBaseURL(baseURL string) string { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + parsed, err := url.Parse(baseURL) + if err == nil && parsed.Scheme != "" && parsed.Host != "" { + path := strings.TrimRight(parsed.Path, "/") + lowerPath := strings.ToLower(path) + if index := strings.Index(lowerPath, "/api/plan/v3"); index >= 0 { + end := index + len("/api/plan/v3") + if len(lowerPath) == end || lowerPath[end] == '/' { + parsed.Path = path[:end] + parsed.RawPath = "" + parsed.RawQuery = "" + parsed.Fragment = "" + return strings.TrimRight(parsed.String(), "/") + } + } + } + return baseURL +} + +func isArkAgentPlanChannel(channel model.ModelChannel) bool { + baseURL := strings.ToLower(normalizeModelChannelBaseURL(channel.BaseURL)) + return strings.HasSuffix(baseURL, "/api/plan/v3") +} + +func isSeedanceModelName(modelName string) bool { + modelName = strings.ToLower(strings.TrimSpace(modelName)) + return strings.Contains(modelName, "seedance") || strings.Contains(modelName, "doubao-seedance") +} + +func enabledChannelModels(channels []model.ModelChannel) []string { + models := []string{} + for _, channel := range channels { + if !channel.Enabled { + continue + } + models = append(models, channel.Models...) + } + return uniqueModelNames(models) +} + +func uniqueModelNames(models []string) []string { + result := []string{} + seen := map[string]bool{} + for _, item := range models { + name := strings.TrimSpace(item) + if name == "" || seen[name] { + continue + } + seen[name] = true + result = append(result, name) + } + return result +} + +func repairDefaultModel(current string, models []string, preferred func(string) bool) string { + current = strings.TrimSpace(current) + for _, item := range models { + if item == current { + return current + } + } + for _, item := range models { + if preferred(item) { + return item + } + } + if len(models) > 0 { + return models[0] + } + return "" +} + +func isVideoModelName(modelName string) bool { + name := strings.ToLower(strings.TrimSpace(modelName)) + return strings.Contains(name, "seedance") || strings.Contains(name, "video") +} + +func isImageModelName(modelName string) bool { + name := strings.ToLower(strings.TrimSpace(modelName)) + return strings.Contains(name, "seedream") || strings.Contains(name, "gpt-image") || strings.Contains(name, "image") +} + +func isTextModelName(modelName string) bool { + return !isImageModelName(modelName) && !isVideoModelName(modelName) +} + func normalizeModelChannel(channel model.ModelChannel) model.ModelChannel { if channel.Protocol == "" { channel.Protocol = "openai" @@ -239,13 +348,16 @@ func fetchAdminChannelModels(channel model.ModelChannel) ([]string, error) { return nil, err } request.Header.Set("Authorization", "Bearer "+channel.APIKey) - response, err := http.DefaultClient.Do(request) + response, err := adminModelHTTPClient.Do(request) if err != nil { - return nil, err + return nil, safeMessageError{message: "读取模型失败：上游接口无响应或网络不可达"} } defer response.Body.Close() body, _ := io.ReadAll(response.Body) if response.StatusCode >= http.StatusBadRequest { + if response.StatusCode == http.StatusNotFound && isArkAgentPlanChannel(channel) { + return nil, safeMessageError{message: "火山方舟 Agent Plan 未提供 OpenAI /models 模型列表接口，请手动填写模型名称，例如 doubao-seedance-2.0。"} + } return nil, readAdminChannelError(body, response.StatusCode, "读取模型失败") } var payload struct { @@ -281,9 +393,9 @@ func testAdminChannelModel(channel model.ModelChannel, modelName string) (string } request.Header.Set("Authorization", "Bearer "+channel.APIKey) request.Header.Set("Content-Type", "application/json") - response, err := http.DefaultClient.Do(request) + response, err := adminModelHTTPClient.Do(request) if err != nil { - return "", err + return "", safeMessageError{message: "测试失败：上游接口无响应或网络不可达"} } defer response.Body.Close() responseBody, _ := io.ReadAll(response.Body) @@ -304,6 +416,22 @@ func testAdminChannelModel(channel model.ModelChannel, modelName string) (string return "ok", nil } +func testArkSeedanceChannelModel(channel model.ModelChannel, modelName string) (string, error) { + if strings.TrimSpace(modelName) == "" { + return "", errors.New("缺少模型名称") + } + if strings.TrimSpace(channel.BaseURL) == "" { + return "", safeMessageError{message: "缺少接口地址"} + } + if strings.TrimSpace(channel.APIKey) == "" { + return "", safeMessageError{message: "缺少 API Key"} + } + if !isArkAgentPlanChannel(channel) { + return "Seedance 视频模型不会发送 /chat/completions 文本测试。已检查 Base URL、API Key 和模型名非空；未调用视频生成接口，因此未验证套餐额度或模型权限。", nil + } + return "Agent Plan / Seedance 视频模型配置格式已通过。后台测试不会调用视频生成接口，因此未验证 API Key、套餐额度或模型权限；请在画布中使用视频生成验证。", nil +} + func readAdminChannelError(body []byte, statusCode int, fallback string) error { var payload struct { Error *struct { @@ -319,8 +447,11 @@ func readAdminChannelError(body []byte, statusCode int, fallback string) error { return safeMessageError{message: payload.Msg} } } - if statusCode == http.StatusUnauthorized { - return safeMessageError{message: "上游接口认证失败（401），请检查 API Key"} + if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden { + return safeMessageError{message: fmt.Sprintf("上游接口鉴权失败（%d），请检查 API Key、套餐权限或模型权限", statusCode)} + } + if statusCode == http.StatusTooManyRequests { + return safeMessageError{message: "上游接口限流或额度不足（429），请稍后重试或检查额度"} } if statusCode > 0 { return safeMessageError{message: fmt.Sprintf("%s：%d", fallback, statusCode)} diff --git a/service/settings_test.go b/service/settings_test.go new file mode 100644 index 0000000..27c6b2e --- /dev/null +++ b/service/settings_test.go @@ -0,0 +1,100 @@ +package service + +import ( + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/basketikun/infinite-canvas/model" +) + +func TestFetchAdminChannelModelsParsesOpenAIModels(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/models" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"z-model"},{"id":"a-model"},{"id":""}]}`)) + })) + defer server.Close() + + models, err := fetchAdminChannelModels(model.ModelChannel{ + BaseURL: server.URL, + APIKey: "test-key", + }) + if err != nil { + t.Fatalf("fetchAdminChannelModels returned error: %v", err) + } + if want := []string{"a-model", "z-model"}; !reflect.DeepEqual(models, want) { + t.Fatalf("models = %#v, want %#v", models, want) + } +} + +func TestFetchAdminChannelModelsReportsArkPlanModelsUnsupported(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/plan/v3/models" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + http.NotFound(w, r) + })) + defer server.Close() + + _, err := fetchAdminChannelModels(model.ModelChannel{ + BaseURL: server.URL + "/api/plan/v3/contents/generations/tasks", + APIKey: "test-key", + }) + if err == nil { + t.Fatal("expected unsupported /models error") + } + if !strings.Contains(err.Error(), "Agent Plan 未提供 OpenAI /models") { + t.Fatalf("error = %q", err.Error()) + } +} + +func TestBuildModelChannelURLNormalizesArkPlanTaskPath(t *testing.T) { + got := BuildModelChannelURL(model.ModelChannel{BaseURL: "https://ark.cn-beijing.volces.com/api/plan/v3/contents/generations/tasks?debug=1"}, "/models") + want := "https://ark.cn-beijing.volces.com/api/plan/v3/models" + if got != want { + t.Fatalf("BuildModelChannelURL = %q, want %q", got, want) + } +} + +func TestNormalizeSettingsPublishesEnabledChannelModelsAndRepairsDefaults(t *testing.T) { + settings := normalizeSettings(model.Settings{ + Public: model.PublicSetting{ + ModelChannel: model.PublicModelChannelSetting{ + AvailableModels: []string{"grok-imagine-video", "disabled-model"}, + DefaultModel: "grok-imagine-video", + DefaultTextModel: "missing-text", + DefaultImageModel: "missing-image", + DefaultVideoModel: "missing-video", + }, + }, + Private: model.PrivateSetting{ + Channels: []model.ModelChannel{ + {Enabled: true, Models: []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast", "gpt-5.5"}}, + {Enabled: false, Models: []string{"disabled-model"}}, + }, + }, + }) + + channel := settings.Public.ModelChannel + wantModels := []string{"gpt-5.5", "doubao-seedream-5.0-lite", "doubao-seedance-2.0-fast"} + if !reflect.DeepEqual(channel.AvailableModels, wantModels) { + t.Fatalf("available models = %#v, want %#v", channel.AvailableModels, wantModels) + } + if channel.DefaultModel != "gpt-5.5" { + t.Fatalf("default model = %q, want text model", channel.DefaultModel) + } + if channel.DefaultTextModel != "gpt-5.5" { + t.Fatalf("default text model = %q, want text model", channel.DefaultTextModel) + } + if channel.DefaultImageModel != "doubao-seedream-5.0-lite" { + t.Fatalf("default image model = %q, want seedream", channel.DefaultImageModel) + } + if channel.DefaultVideoModel != "doubao-seedance-2.0-fast" { + t.Fatalf("default video model = %q, want seedance", channel.DefaultVideoModel) + } +} diff --git a/web/src/app/(admin)/admin/settings/page.tsx b/web/src/app/(admin)/admin/settings/page.tsx index 7ca449e..7b5029f 100644 --- a/web/src/app/(admin)/admin/settings/page.tsx +++ b/web/src/app/(admin)/admin/settings/page.tsx @@ -215,6 +215,10 @@ export default function AdminSettingsPage() { const channelModels = await fetchChannelModels(token, { index: editingChannelIndex ?? undefined, channel: normalizeChannel(channel) }); const current = isModelSelectorOpen ? uniqueModels(modelSelectSelected) : uniqueModels(channelForm.getFieldValue("models") || []); rememberModels(channelModels); + if (!channelModels.length) { + message.warning("上游未返回模型列表，请手动输入模型名称"); + return; + } setModelSelectExisting(current); setModelSelectSource(uniqueModels(channelModels)); setModelSelectSelected(uniqueModels([...current, ...channelModels])); @@ -337,7 +341,7 @@ export default function AdminSettingsPage() { const nextChannelModels = collectChannelModels(nextChannels); const nextSettings = normalizeSettings({ ...values, - public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: filterModels(values.public.modelChannel.availableModels, nextChannelModels) } }, + public: { ...values.public, modelChannel: { ...values.public.modelChannel, availableModels: nextChannelModels } }, private: { ...values.private, channels: nextChannels }, }); const saved = normalizeSettings(await saveAdminSettings(token, nextSettings)); @@ -410,7 +414,7 @@ export default function AdminSettingsPage() {
- + + setInfoNodeId(null)} /> @@ -2376,7 +2410,7 @@ function CanvasTopBar({ { key: "new", icon: , label: "新建画布", onClick: onCreateProject }, { key: "delete", danger: true, icon: , label: "删除当前画布", onClick: onDeleteProject }, { type: "divider" }, - { key: "import", icon: , label: "导入图片", onClick: onImportImage }, + { key: "import", icon: , label: "导入素材", onClick: onImportImage }, { type: "divider" }, { key: "undo", disabled: !canUndo, icon: , label: , onClick: onUndo }, { key: "redo", disabled: !canRedo, icon: , label: , onClick: onRedo }, @@ -2457,7 +2491,7 @@ function CanvasTopBar({ - + @@ -2498,12 +2532,20 @@ function imageExtension(dataUrl: string) { return dataUrl.match(/^data:image[/]([^;]+)/)?.[1] || dataUrl.match(/image[/]([^;]+)/)?.[1] || "png"; } +function audioExtension(mimeType?: string) { + return mimeType?.includes("wav") ? "wav" : "mp3"; +} + function imageMetadata(image: UploadedImage): CanvasNodeMetadata { return { content: image.url, storageKey: image.storageKey, status: "success", naturalWidth: image.width, naturalHeight: image.height, bytes: image.bytes, mimeType: image.mimeType }; } function videoMetadata(video: UploadedFile): CanvasNodeMetadata { - return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4" }; + return { content: video.url, storageKey: video.storageKey, status: "success", naturalWidth: video.width, naturalHeight: video.height, bytes: video.bytes, mimeType: video.mimeType || "video/mp4", durationMs: video.durationMs }; +} + +function audioMetadata(audio: UploadedFile): CanvasNodeMetadata { + return { content: audio.url, storageKey: audio.storageKey, status: "success", bytes: audio.bytes, mimeType: audio.mimeType || "audio/mpeg", durationMs: audio.durationMs }; } function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: AiConfig, count: number, references: ReferenceImage[]): CanvasNodeMetadata { @@ -2521,6 +2563,14 @@ function referenceUrl(image: ReferenceImage) { return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined); } +function generationReferenceUrls(context: { referenceImages: ReferenceImage[]; referenceVideos: Array<{ storageKey?: string; url?: string }>; referenceAudios?: Array<{ storageKey?: string; url?: string }> }) { + return [ + ...context.referenceImages.map(referenceUrl).filter((url): url is string => Boolean(url)), + ...context.referenceVideos.map((video) => video.storageKey || video.url).filter((url): url is string => Boolean(url)), + ...(context.referenceAudios || []).map((audio) => audio.storageKey || audio.url).filter((url): url is string => Boolean(url)), + ]; +} + async function resolveMetadataReferences(metadata: CanvasNodeMetadata) { if (metadata.generationType !== "edit") return []; if (!metadata.references?.length) return null; @@ -2537,7 +2587,7 @@ async function hydrateCanvasImages(nodes: CanvasNodeData[]) { return Promise.all( nodes.map(async (node) => { const content = node.metadata?.content; - if (node.type === CanvasNodeType.Video && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } }; + if ((node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio) && node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveMediaUrl(node.metadata.storageKey, content) } }; if (node.type !== CanvasNodeType.Image || !content) return node; if (node.metadata?.storageKey) return { ...node, metadata: { ...node.metadata, content: await resolveImageUrl(node.metadata.storageKey, content) } }; if (!content.startsWith("data:image/")) return node; @@ -2574,9 +2624,10 @@ function getGenerationCount(count: string) { } function applyNodeConfigPatch(node: CanvasNodeData, patch: Partial) { - const next = { ...node, metadata: { ...node.metadata, ...(patch || {}) } }; + const safePatch = patch || {}; + const next = { ...node, metadata: { ...node.metadata, ...safePatch } }; const spec = node.type === CanvasNodeType.Video ? NODE_DEFAULT_SIZE[CanvasNodeType.Video] : NODE_DEFAULT_SIZE[CanvasNodeType.Image]; - const size = typeof patch.size === "string" && !node.metadata?.content ? nodeSizeFromRatio(patch.size, spec.width, spec.height) : null; + const size = typeof safePatch.size === "string" && !node.metadata?.content ? nodeSizeFromRatio(safePatch.size, spec.width, spec.height) : null; return size && (node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video) ? { ...next, ...size, position: { x: node.position.x + node.width / 2 - size.width / 2, y: node.position.y + node.height / 2 - size.height / 2 } } : next; } @@ -2595,6 +2646,8 @@ function getInputSummary(inputs: NodeGenerationInput[]) { return { textCount: inputs.filter((input) => input.type === "text").length, imageCount: inputs.filter((input) => input.type === "image").length, + videoCount: inputs.filter((input) => input.type === "video").length, + audioCount: inputs.filter((input) => input.type === "audio").length, }; } @@ -2607,6 +2660,8 @@ function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefine size: node?.metadata?.size || config.size || defaultConfig.size, videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds, vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality, + videoGenerateAudio: node?.metadata?.generateAudio || config.videoGenerateAudio || defaultConfig.videoGenerateAudio, + videoWatermark: node?.metadata?.watermark || config.videoWatermark || defaultConfig.videoWatermark, count: String(node?.metadata?.count || (mode === "image" ? 3 : config.count) || defaultConfig.count), }; } @@ -2642,6 +2697,10 @@ function sourceNodeReferenceImages(node: CanvasNodeData | null) { ]; } +function isAudioFile(file: File) { + return file.type.startsWith("audio/") || /\.(mp3|wav)$/i.test(file.name); +} + function isHiddenBatchChild(node: CanvasNodeData, nodes: CanvasNodeData[], collapsingBatchIds?: Set) { const rootId = node.metadata?.batchRootId; if (!rootId) return false; diff --git a/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx b/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx index 8860bee..0b10338 100644 --- a/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx @@ -16,6 +16,7 @@ import { requestEdit, requestGeneration, requestImageQuestion, type ChatCompleti import { imageToDataUrl, uploadImage } from "@/services/image-storage"; import { useAssetStore } from "@/stores/use-asset-store"; import { useThemeStore } from "@/stores/use-theme-store"; +import { imageReferenceLabel } from "@/lib/image-reference-prompt"; import type { ReferenceImage } from "@/types/image"; import { DiaTextReveal } from "@/components/ui/dia-text-reveal"; import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover"; @@ -398,8 +399,8 @@ function AssistantComposer({
event.stopPropagation()}> {references.length ? (
- {references.map((item) => ( - onRemoveReference(item.id)} /> + {references.map((item, index) => ( + onRemoveReference(item.id)} /> ))}
) : null} @@ -584,20 +585,23 @@ function AssistantHistory({ function MessageReferences({ message }: { message: CanvasAssistantMessage }) { return (
- {message.references?.map((item) => ( - + {message.references?.map((item, index, references) => ( + ))}
); } -function AssistantReferenceChip({ item, onRemove }: { item: CanvasAssistantReference; onRemove?: () => void }) { +function AssistantReferenceChip({ item, label, onRemove }: { item: CanvasAssistantReference; label?: string; onRemove?: () => void }) { const theme = canvasThemes[useThemeStore((state) => state.theme)]; const text = (item.text || item.title).replace(/\s+/g, " ").trim().slice(0, 1) || "文"; return (
{item.dataUrl ? ( - + + + {label ? {label} : null} + ) : ( {text} @@ -618,6 +622,12 @@ function AssistantReferenceChip({ item, onRemove }: { item: CanvasAssistantRefer ); } +function assistantImageReferenceLabel(references: CanvasAssistantReference[], index: number) { + if (!references[index]?.dataUrl) return undefined; + const imageIndex = references.slice(0, index + 1).filter((item) => item.dataUrl).length - 1; + return imageIndex >= 0 ? imageReferenceLabel(imageIndex) : undefined; +} + function nodeToReference(node: CanvasNodeData): CanvasAssistantReference | null { if (node.type === CanvasNodeType.Image && node.metadata?.content) { return { id: node.id, type: node.type, title: node.title, dataUrl: node.metadata.content, storageKey: node.metadata.storageKey }; diff --git a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx index 8f5c117..4116989 100644 --- a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx @@ -2,13 +2,15 @@ import type { CSSProperties } from "react"; import { useState } from "react"; -import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Play, Video } from "lucide-react"; +import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Edit3, Eye, Image as ImageIcon, LoaderCircle, MessageSquare, Music2, Play, Video } from "lucide-react"; import { App, Button, Empty, Input, Modal, Segmented } from "antd"; import { ModelPicker } from "@/components/model-picker"; import { defaultConfig, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store"; import { CreditSymbol, requestCreditCost } from "@/constant/credits"; import { canvasThemes } from "@/lib/canvas-theme"; +import { imageReferenceLabel } from "@/lib/image-reference-prompt"; +import { seedanceReferenceLabel } from "@/lib/seedance-video"; import { useThemeStore } from "@/stores/use-theme-store"; import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover"; import { CanvasVideoSettingsPopover } from "./canvas-video-settings-popover"; @@ -18,7 +20,7 @@ import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata } from ". type CanvasConfigNodePanelProps = { node: CanvasNodeData; isRunning: boolean; - inputSummary: { textCount: number; imageCount: number }; + inputSummary: { textCount: number; imageCount: number; videoCount: number; audioCount: number }; inputs: NodeGenerationInput[]; onConfigChange: (nodeId: string, patch: Partial) => void; onTextInputChange: (nodeId: string, content: string) => void; @@ -41,6 +43,8 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o const chipStyle = { background: theme.node.fill, borderColor: theme.node.stroke, color: theme.node.text }; const textInputs = inputs.filter((input) => input.type === "text"); const imageInputs = inputs.filter((input) => input.type === "image"); + const videoInputs = inputs.filter((input) => input.type === "video"); + const audioInputs = inputs.filter((input) => input.type === "audio"); const moveInput = (input: NodeGenerationInput, offset: number) => { const sameTypeInputs = inputs.filter((item) => item.type === input.type); @@ -112,6 +116,8 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o
event.stopPropagation()}> + +
+
+ +
+ {videoInputs.map((input, index) => ( + + ))} +
+ +
+
+ +
+ {audioInputs.map((input, index) => ( + + ))} +
+ +
@@ -281,13 +305,68 @@ function ImageSortCard({
{input.title} - {imageIndex + 1} + {imageReferenceLabel(imageIndex)} onMove(input, offset)} />
); } +function VideoSortCard({ + input, + videoIndex, + videoTotal, + theme, + onMove, +}: { + input: NodeGenerationInput; + videoIndex: number; + videoTotal: number; + theme: (typeof canvasThemes)[keyof typeof canvasThemes]; + onMove: (input: NodeGenerationInput, offset: number) => void; +}) { + if (!input.video) return null; + return ( +
+
+
+
+ ); +} + +function AudioSortCard({ + input, + audioIndex, + audioTotal, + theme, + onMove, +}: { + input: NodeGenerationInput; + audioIndex: number; + audioTotal: number; + theme: (typeof canvasThemes)[keyof typeof canvasThemes]; + onMove: (input: NodeGenerationInput, offset: number) => void; +}) { + if (!input.audio) return null; + return ( +
+
+ + {input.title} +
+
+ ); +} + function VerticalOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) { return ( <> @@ -324,6 +403,15 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can size: node.metadata?.size || globalConfig.size || defaultConfig.size, videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds, vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality, + videoGenerateAudio: node.metadata?.generateAudio || globalConfig.videoGenerateAudio || defaultConfig.videoGenerateAudio, + videoWatermark: node.metadata?.watermark || globalConfig.videoWatermark || defaultConfig.videoWatermark, count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count), }; } + +function videoConfigPatch(key: keyof AiConfig, value: string) { + if (key === "videoSeconds") return { seconds: value }; + if (key === "videoGenerateAudio") return { generateAudio: value }; + if (key === "videoWatermark") return { watermark: value }; + return { [key]: value }; +} diff --git a/web/src/app/(user)/canvas/components/canvas-mini-map.tsx b/web/src/app/(user)/canvas/components/canvas-mini-map.tsx index 4557a87..1bf04f1 100644 --- a/web/src/app/(user)/canvas/components/canvas-mini-map.tsx +++ b/web/src/app/(user)/canvas/components/canvas-mini-map.tsx @@ -115,7 +115,7 @@ export function Minimap({ nodes, viewport, viewportSize, onViewportChange }: { n > {nodes.map((node) => { const pos = toMinimap(node.position.x, node.position.y); - const color = node.type === CanvasNodeType.Image ? "#10b981" : node.type === CanvasNodeType.Config ? "#60a5fa" : theme.node.muted; + const color = node.type === CanvasNodeType.Image ? "#10b981" : node.type === CanvasNodeType.Video ? "#f97316" : node.type === CanvasNodeType.Audio ? "#a855f7" : node.type === CanvasNodeType.Config ? "#60a5fa" : theme.node.muted; return (
input.image).filter((image): image is ReferenceImage => Boolean(image)); + const referenceVideos = inputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video)); + const referenceAudios = inputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio)); return { prompt: upstreamText ? `${prompt}\n\n${upstreamText}` : prompt, referenceImages, + referenceVideos, + referenceAudios, textCount: inputs.filter((input) => input.type === "text").length, imageCount: referenceImages.length, + videoCount: referenceVideos.length, + audioCount: referenceAudios.length, }; } @@ -37,6 +50,10 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[ return getOrderedUpstreamNodes(nodeId, nodes, connections).flatMap((node): NodeGenerationInput[] => { const image = readReferenceImage(node); if (image) return [{ nodeId: node.id, type: "image" as const, title: node.title, image }]; + const video = readReferenceVideo(node); + if (video) return [{ nodeId: node.id, type: "video" as const, title: node.title, video }]; + const audio = readReferenceAudio(node); + if (audio) return [{ nodeId: node.id, type: "audio" as const, title: node.title, audio }]; const text = readNodeTextInput(node); if (text) return [{ nodeId: node.id, type: "text" as const, title: node.title, text }]; return []; @@ -77,6 +94,33 @@ function readReferenceImage(node: CanvasNodeData): ReferenceImage | null { }; } +function readReferenceVideo(node: CanvasNodeData): ReferenceVideo | null { + if (node.type !== CanvasNodeType.Video || !node.metadata?.content) return null; + return { + id: node.id, + name: `${node.title || node.id}.mp4`, + type: node.metadata.mimeType || "video/mp4", + url: node.metadata.content, + storageKey: node.metadata.storageKey, + bytes: node.metadata.bytes, + width: node.metadata.naturalWidth, + height: node.metadata.naturalHeight, + durationMs: node.metadata.durationMs, + }; +} + +function readReferenceAudio(node: CanvasNodeData): ReferenceAudio | null { + if (node.type !== CanvasNodeType.Audio || !node.metadata?.content) return null; + return { + id: node.id, + name: `${node.title || node.id}.mp3`, + type: node.metadata.mimeType || "audio/mpeg", + url: node.metadata.content, + storageKey: node.metadata.storageKey, + durationMs: node.metadata.durationMs, + }; +} + function getOrderedUpstreamNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) { const target = nodes.find((node) => node.id === nodeId); const upstreamNodes = connections diff --git a/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx b/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx index f43c6b0..1c0a786 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node-hover-toolbar.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; import { Modal, Segmented, Tooltip } from "antd"; -import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react"; +import { Camera, Download, FolderPlus, Image as ImageIcon, Info, Lock, LockOpen, Maximize2, MessageSquare, Minus, Music2, Pencil, Plus, RefreshCw, Scissors, Settings2, Trash2, Upload, Video } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; import { formatBytes, getDataUrlByteSize } from "@/lib/image-utils"; @@ -58,13 +58,15 @@ export function CanvasNodeHoverToolbar({ const top = viewport.y + node.position.y * viewport.k - 14; const isImage = node.type === CanvasNodeType.Image; const isVideo = node.type === CanvasNodeType.Video; + const isAudio = node.type === CanvasNodeType.Audio; const hasImage = isImage && Boolean(node.metadata?.content); const hasVideo = isVideo && Boolean(node.metadata?.content); + const hasAudio = isAudio && Boolean(node.metadata?.content); const isText = node.type === CanvasNodeType.Text; const isConfig = node.type === CanvasNodeType.Config; const canOpenDialog = isText || hasImage || isVideo; const canRetry = node.metadata?.status === "error"; - const hasSpecificTools = canRetry || isText || isImage || isVideo || isConfig; + const hasSpecificTools = canRetry || isText || isImage || isVideo || isAudio || isConfig; return (
: null} {canRetry ? } onClick={() => onRetry(node)} /> : null} {hasImage || hasVideo || isText ? } onClick={() => onSaveAsset(node)} /> : null} - {hasImage || hasVideo ? } onClick={() => onDownload(node)} /> : null} + {hasImage || hasVideo || hasAudio ? } onClick={() => onDownload(node)} /> : null} {canOpenDialog ? } onClick={() => onToggleDialog(node)} /> : null} {isText ? } onClick={() => onEditText(node)} /> : null} {isText ? } onClick={() => onGenerateImage(node)} /> : null} @@ -89,6 +91,7 @@ export function CanvasNodeHoverToolbar({ {isText ? } onClick={() => onIncreaseFont(node)} /> : null} {isImage ? } onClick={() => onUpload(node)} /> : null} {isVideo ? } onClick={() => onUpload(node)} /> : null} + {isAudio ? } onClick={() => onUpload(node)} /> : null} {hasImage ? ( - + diff --git a/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx b/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx index 677d58d..62db038 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node-prompt-panel.tsx @@ -93,7 +93,7 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi ) : mode === "video" ? ( <> onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} /> - onConfigChange(node.id, key === "videoSeconds" ? { seconds: value } : { [key]: value })} /> + onConfigChange(node.id, videoConfigPatch(key, value))} /> ) : ( onConfigChange(node.id, { model })} onMissingConfig={() => openConfigDialog(true)} /> @@ -132,6 +132,15 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can size: node.metadata?.size || globalConfig.size || defaultConfig.size, videoSeconds: node.metadata?.seconds || globalConfig.videoSeconds || defaultConfig.videoSeconds, vquality: node.metadata?.vquality || globalConfig.vquality || defaultConfig.vquality, + videoGenerateAudio: node.metadata?.generateAudio || globalConfig.videoGenerateAudio || defaultConfig.videoGenerateAudio, + videoWatermark: node.metadata?.watermark || globalConfig.videoWatermark || defaultConfig.videoWatermark, count: String(node.metadata?.count || (mode === "image" ? 3 : globalConfig.count) || defaultConfig.count), }; } + +function videoConfigPatch(key: keyof AiConfig, value: string) { + if (key === "videoSeconds") return { seconds: value }; + if (key === "videoGenerateAudio") return { generateAudio: value }; + if (key === "videoWatermark") return { watermark: value }; + return { [key]: value }; +} diff --git a/web/src/app/(user)/canvas/components/canvas-node.tsx b/web/src/app/(user)/canvas/components/canvas-node.tsx index 6f63b9d..07bd043 100644 --- a/web/src/app/(user)/canvas/components/canvas-node.tsx +++ b/web/src/app/(user)/canvas/components/canvas-node.tsx @@ -2,7 +2,7 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import type { ReactNode } from "react"; -import { ChevronRight, Image as ImageIcon, RefreshCw, Star, Video } from "lucide-react"; +import { ChevronRight, Image as ImageIcon, Music2, RefreshCw, Star, Video } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; import { formatBytes } from "@/lib/image-utils"; @@ -99,6 +99,7 @@ export const CanvasNode = React.memo(function CanvasNode({ const [isEditingContent, setIsEditingContent] = useState(false); const hasImageContent = data.type === CanvasNodeType.Image && Boolean(data.metadata?.content); const hasVideoContent = data.type === CanvasNodeType.Video && Boolean(data.metadata?.content); + const hasAudioContent = data.type === CanvasNodeType.Audio && Boolean(data.metadata?.content); const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1; const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId); const isActive = isConnectionTarget || isSelected || isFocusRelated; @@ -301,7 +302,7 @@ export const CanvasNode = React.memo(function CanvasNode({ {showImageInfo && hasImageContent ? : null} - {!hasImageContent && !hasVideoContent ?
: null} + {!hasImageContent && !hasVideoContent && !hasAudioContent ?
: null} @@ -332,6 +333,7 @@ const nodeContentRenderers = { [CanvasNodeType.Image]: ImageNodeContent, [CanvasNodeType.Config]: EmptyImageContent, [CanvasNodeType.Video]: VideoNodeContent, + [CanvasNodeType.Audio]: AudioNodeContent, } satisfies Record ReactNode>; function LoadingContent({ theme }: Pick) { @@ -466,12 +468,31 @@ function VideoNodeContent({ node, theme }: NodeContentRendererProps) { return (
); return
- {references.map((item) => ( + {references.map((item, index) => (
{item.name} + {seedanceReferenceLabel("image", index)} + setReferences((value) => moveListItem(value, index, offset))} />
))} - {!references.length ?
暂无参考图，最多 7 张
: null} + {!references.length ?
暂无参考图，最多 9 张
: null} +
+
+ +
+
+ 参考视频 + +
+
+ {videoReferences.map((item, index) => ( +
+
+ ))} + {!videoReferences.length ?
暂无参考视频，最多 3 个
: null} +
+
+ +
+
+ 参考音频 + +
+
+ {audioReferences.map((item, index) => ( +
+
+ + {seedanceReferenceLabel("audio", index)} + {item.name} +
+
+ ))} + {!audioReferences.length ?
暂无参考音频，最多 3 个，mp3/wav，单个 15MB 内
: null}
@@ -359,7 +452,7 @@ export default function VideoPage() { { @@ -542,6 +635,18 @@ async function readStoredLogs() { async function normalizeLog(log: Partial): Promise { const video = log.video?.storageKey ? { ...log.video, url: await resolveMediaUrl(log.video.storageKey, log.video.url) } : log.video; + const videoReferences = await Promise.all( + (log.videoReferences || []).map(async (item) => ({ + ...item, + url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url, + })), + ); + const audioReferences = await Promise.all( + (log.audioReferences || []).map(async (item) => ({ + ...item, + url: item.storageKey ? await resolveMediaUrl(item.storageKey, item.url) : item.url, + })), + ); const references = await Promise.all( (log.references || []).map(async (item) => ({ ...item, @@ -558,6 +663,8 @@ async function normalizeLog(log: Partial): Promise model: log.model || config.videoModel || "", config, references, + videoReferences, + audioReferences, durationMs: log.durationMs || 0, size: log.size || config.size || "", resolution: normalizeResolution(log.resolution || config.vquality || ""), @@ -572,10 +679,54 @@ function serializeLog(log: GenerationLog): GenerationLog { return { ...log, references: log.references.map((item) => ({ ...item, dataUrl: item.storageKey ? "" : item.dataUrl })), + videoReferences: log.videoReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)), + audioReferences: log.audioReferences.map((item) => (item.storageKey ? { ...item, url: "" } : item)), video: log.video?.storageKey ? { ...log.video, url: "" } : log.video, }; } +function isSupportedAudioFile(file: File) { + return file.type === "audio/mpeg" || file.type === "audio/mp3" || file.type === "audio/wav" || file.type === "audio/x-wav" || /\.(mp3|wav)$/i.test(file.name); +} + +function filterAudioReferencesByDuration(existing: ReferenceAudio[], next: ReferenceAudio[], warn: (content: string) => void) { + let total = existing.reduce((sum, item) => sum + (item.durationMs || 0), 0); + const accepted: ReferenceAudio[] = []; + let skipped = false; + for (const item of next) { + if (item.durationMs && (item.durationMs < 2000 || item.durationMs > 15000)) { + skipped = true; + continue; + } + if (item.durationMs && total + item.durationMs > 15000) { + skipped = true; + continue; + } + total += item.durationMs || 0; + accepted.push(item); + } + if (skipped) warn("已忽略不符合时长要求的参考音频：单个 2-15 秒，总时长不超过 15 秒"); + return accepted; +} + +function moveListItem(items: T[], index: number, offset: number) { + const targetIndex = index + offset; + if (targetIndex < 0 || targetIndex >= items.length) return items; + const next = [...items]; + [next[index], next[targetIndex]] = [next[targetIndex], next[index]]; + return next; +} + +function ReferenceOrderButtons({ index, total, onMove }: { index: number; total: number; onMove: (offset: number) => void }) { + if (total <= 1) return null; + return ( +
+
+ ); +} + function normalizeLogConfig(log: Partial): GenerationLogConfig { return { model: log.config?.model || log.model || "", @@ -583,16 +734,20 @@ function normalizeLogConfig(log: Partial): GenerationLogConfig { size: log.config?.size || log.size || "", vquality: normalizeResolution(log.config?.vquality || log.resolution || ""), videoSeconds: log.config?.videoSeconds || log.seconds || "", + videoGenerateAudio: log.config?.videoGenerateAudio || "true", + videoWatermark: log.config?.videoWatermark || "false", }; } -function buildLog({ prompt, model, config, references, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog { +function buildLog({ prompt, model, config, references, videoReferences, audioReferences, durationMs, status, video, error }: { prompt: string; model: string; config: AiConfig; references: ReferenceImage[]; videoReferences: ReferenceVideo[]; audioReferences: ReferenceAudio[]; durationMs: number; status: GenerationLog["status"]; video?: GeneratedVideo; error?: string }): GenerationLog { const logConfig = { model: config.model, videoModel: config.videoModel, size: config.size, vquality: normalizeResolution(config.vquality), videoSeconds: config.videoSeconds, + videoGenerateAudio: config.videoGenerateAudio, + videoWatermark: config.videoWatermark, }; return { id: nanoid(), @@ -603,6 +758,8 @@ function buildLog({ prompt, model, config, references, durationMs, status, video model, config: logConfig, references, + videoReferences, + audioReferences, durationMs, size: logConfig.size, resolution: logConfig.vquality, @@ -614,17 +771,21 @@ function buildLog({ prompt, model, config, references, durationMs, status, video } function buildVideoConfig(config: AiConfig, model: string): AiConfig { + const seedance = isSeedanceVideoConfig({ ...config, model }); return { ...config, model, videoModel: model, - size: normalizeVideoSize(config.size), + size: seedance ? normalizeSeedanceRatio(config.size) : normalizeVideoSize(config.size), videoSeconds: normalizeVideoSeconds(config.videoSeconds), vquality: normalizeResolution(config.vquality), + videoGenerateAudio: String(boolConfig(config.videoGenerateAudio, true)), + videoWatermark: String(boolConfig(config.videoWatermark, false)), }; } function normalizeVideoSeconds(value: string) { + if (String(value).trim() === "-1") return "-1"; const seconds = Math.floor(Number(value) || 6); return String(Math.max(1, Math.min(20, seconds))); } diff --git a/web/src/components/image-settings-panel.tsx b/web/src/components/image-settings-panel.tsx index 66025ce..61095b7 100644 --- a/web/src/components/image-settings-panel.tsx +++ b/web/src/components/image-settings-panel.tsx @@ -17,9 +17,10 @@ const aspectOptions = [ { value: "1:1", label: "1:1", width: 1024, height: 1024, icon: "square" }, { value: "3:2", label: "3:2", width: 1536, height: 1024, icon: "landscape" }, { value: "2:3", label: "2:3", width: 1024, height: 1536, icon: "portrait" }, - { value: "4:3", label: "4:3", width: 1344, height: 1024, icon: "landscape" }, - { value: "3:4", label: "3:4", width: 1024, height: 1344, icon: "portrait" }, - { value: "9:16", label: "9:16", width: 1024, height: 1792, icon: "portrait" }, + { value: "4:3", label: "4:3", width: 1360, height: 1024, icon: "landscape" }, + { value: "3:4", label: "3:4", width: 1024, height: 1360, icon: "portrait" }, + { value: "16:9", label: "16:9", width: 1824, height: 1024, icon: "landscape" }, + { value: "9:16", label: "9:16", width: 1024, height: 1824, icon: "portrait" }, { value: "1:1-2k", label: "1:1(2k)", size: "2048x2048", width: 2048, height: 2048, icon: "square" }, { value: "16:9-2k", label: "16:9(2k)", size: "2048x1152", width: 2048, height: 1152, icon: "landscape" }, { value: "9:16-2k", label: "9:16(2k)", size: "1152x2048", width: 1152, height: 2048, icon: "portrait" }, diff --git a/web/src/components/model-picker.tsx b/web/src/components/model-picker.tsx index 9e51980..8da2b3b 100644 --- a/web/src/components/model-picker.tsx +++ b/web/src/components/model-picker.tsx @@ -20,7 +20,7 @@ type ModelPickerProps = { export function ModelPicker({ config, value, onChange, className, fullWidth = false, placeholder = "选择模型", onMissingConfig }: ModelPickerProps) { const pickerId = useId(); const [open, setOpen] = useState(false); - const options = useMemo(() => Array.from(new Set([...(config.channelMode === "local" ? [value] : []), ...config.models].filter(Boolean))), [config.channelMode, config.models, value]); + const options = useMemo(() => Array.from(new Set([...(config.channelMode === "local" ? [value] : []), ...config.models].filter((model): model is string => Boolean(model)))), [config.channelMode, config.models, value]); const current = value || ""; useEffect(() => { diff --git a/web/src/components/video-settings-panel.tsx b/web/src/components/video-settings-panel.tsx index cae6b3f..8e608b8 100644 --- a/web/src/components/video-settings-panel.tsx +++ b/web/src/components/video-settings-panel.tsx @@ -1,8 +1,10 @@ "use client"; import { type ReactNode } from "react"; +import { Switch } from "antd"; import { ImageSettingsTheme } from "@/components/image-settings-panel"; +import { boolConfig, isSeedanceFastModel, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceDurationOptions, seedancePixelLabel, seedanceRatioOptions, seedanceResolutionOptions } from "@/lib/seedance-video"; import { type CanvasTheme } from "@/lib/canvas-theme"; import type { AiConfig } from "@/stores/use-config-store"; @@ -24,13 +26,17 @@ const secondOptions = [6, 10, 12, 16, 20]; type VideoSettingsPanelProps = { config: AiConfig; - onConfigChange: (key: "vquality" | "size" | "videoSeconds", value: string) => void; + onConfigChange: (key: "vquality" | "size" | "videoSeconds" | "videoGenerateAudio" | "videoWatermark", value: string) => void; theme: CanvasTheme; showTitle?: boolean; className?: string; }; export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = true, className = "w-[320px] space-y-4 rounded-2xl px-1 py-0.5" }: VideoSettingsPanelProps) { + if (isSeedanceVideoConfig(config)) { + return ; + } + const seconds = config.videoSeconds || "6"; const size = normalizeVideoSizeValue(config.size); const dimensions = readSizeDimensions(size); @@ -96,16 +102,84 @@ export function VideoSettingsPanel({ config, onConfigChange, theme, showTitle = ); } +function SeedanceVideoSettingsPanel({ config, onConfigChange, theme, showTitle, className }: VideoSettingsPanelProps) { + const model = config.model || config.videoModel; + const resolution = normalizeSeedanceResolution(config.vquality, model); + const ratio = normalizeSeedanceRatio(config.size); + const duration = normalizeSeedanceDuration(config.videoSeconds); + const generateAudio = boolConfig(config.videoGenerateAudio, true); + const watermark = boolConfig(config.videoWatermark, false); + + return ( + +
event.stopPropagation()}> + {showTitle ?
视频设置
: null} + +
+ {seedanceResolutionOptions.map((item) => { + const disabled = item.value === "1080p" && isSeedanceFastModel(model); + return ( + onConfigChange("vquality", item.value)}> + {item.label} + + ); + })} +
+ {isSeedanceFastModel(model) ?
fast 模型不支持 1080p，会自动使用 720p。
: null} + + +
+ {seedanceRatioOptions.map((item) => ( + + ))} +
+ + +
+ {seedanceDurationOptions.map((value) => ( + onConfigChange("videoSeconds", String(value))}> + {value === -1 ? "智能" : `${value}s`} + + ))} +
+ onConfigChange("videoSeconds", value)} /> + + +
+ onConfigChange("videoGenerateAudio", String(checked))} /> + onConfigChange("videoWatermark", String(checked))} /> +
+ +
+ + ); +} + export function videoResolutionLabel(value: string) { return `${normalizeVideoResolutionValue(value)}p`; } export function videoSizeLabel(value: string) { + const ratio = normalizeSeedanceRatio(value); + if (value === "adaptive" || value === "auto") return "自适应"; + if (ratio === value) return seedanceRatioOptions.find((item) => item.value === ratio)?.label || ratio; const size = normalizeVideoSizeValue(value); return sizeOptions.find((item) => item.value === size)?.label || size; } export function videoSecondsLabel(value: string) { + if (String(value).trim() === "-1") return "智能"; return `${value || "6"}s`; } @@ -121,9 +195,9 @@ export function normalizeVideoResolutionValue(value: string) { return value.replace(/p$/i, "") || "720"; } -function OptionPill({ selected, theme, onClick, children }: { selected: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) { +function OptionPill({ selected, disabled = false, theme, onClick, children }: { selected: boolean; disabled?: boolean; theme: CanvasTheme; onClick: () => void; children: ReactNode }) { return ( - ); @@ -174,6 +248,29 @@ function SizePreview({ width, height, color }: { width: number; height: number; return ; } +function ratioPreview(ratio: string) { + if (ratio === "9:16") return { width: 9, height: 16 }; + if (ratio === "1:1") return { width: 1, height: 1 }; + if (ratio === "4:3") return { width: 4, height: 3 }; + if (ratio === "3:4") return { width: 3, height: 4 }; + if (ratio === "21:9") return { width: 21, height: 9 }; + if (ratio === "adaptive") return { width: 0, height: 0 }; + return { width: 16, height: 9 }; +} + +function SwitchRow({ label, checked, theme, onChange }: { label: string; checked: boolean; theme: CanvasTheme; onChange: (checked: boolean) => void }) { + return ( +
+ + {label} + + event.stopPropagation()}> + + +
+ ); +} + function readSizeDimensions(size: string) { if (size === "auto") return { width: 0, height: 0 }; const match = size.match(/^(\d+)x(\d+)$/); diff --git a/web/src/lib/image-reference-prompt.ts b/web/src/lib/image-reference-prompt.ts new file mode 100644 index 0000000..3b99ba4 --- /dev/null +++ b/web/src/lib/image-reference-prompt.ts @@ -0,0 +1,12 @@ +import type { ReferenceImage } from "@/types/image"; + +export function imageReferenceLabel(index: number) { + return `图片${index + 1}`; +} + +export function buildImageReferencePromptText(prompt: string, references: ReferenceImage[]) { + const text = prompt.trim(); + if (!references.length) return text; + const labels = references.map((_, index) => imageReferenceLabel(index)); + return `参考图片编号：${labels.join("、")}。请按这些编号理解提示词中的图片引用。\n\n${text}`; +} diff --git a/web/src/lib/seedance-video.ts b/web/src/lib/seedance-video.ts new file mode 100644 index 0000000..4194396 --- /dev/null +++ b/web/src/lib/seedance-video.ts @@ -0,0 +1,168 @@ +import type { AiConfig } from "@/stores/use-config-store"; +import type { ReferenceImage } from "@/types/image"; +import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; + +export const SEEDANCE_REFERENCE_LIMITS = { + images: 9, + videos: 3, + audios: 3, + imageMaxBytes: 30 * 1024 * 1024, + videoMaxBytes: 50 * 1024 * 1024, + audioMaxBytes: 15 * 1024 * 1024, +}; + +export const seedanceResolutionOptions = [ + { value: "480p", label: "480p" }, + { value: "720p", label: "720p" }, + { value: "1080p", label: "1080p" }, +] as const; + +export const seedanceRatioOptions = [ + { value: "16:9", label: "横屏" }, + { value: "9:16", label: "竖屏" }, + { value: "1:1", label: "方形" }, + { value: "4:3", label: "标准横屏" }, + { value: "3:4", label: "标准竖屏" }, + { value: "21:9", label: "宽银幕" }, + { value: "adaptive", label: "自适应" }, +] as const; + +export const seedanceDurationOptions = [-1, 4, 5, 6, 8, 10, 12, 15] as const; + +const seedancePixels = { + "480p": { + "16:9": "864x496", + "4:3": "752x560", + "1:1": "640x640", + "3:4": "560x752", + "9:16": "496x864", + "21:9": "992x432", + }, + "720p": { + "16:9": "1280x720", + "4:3": "1112x834", + "1:1": "960x960", + "3:4": "834x1112", + "9:16": "720x1280", + "21:9": "1470x630", + }, + "1080p": { + "16:9": "1920x1080", + "4:3": "1664x1248", + "1:1": "1440x1440", + "3:4": "1248x1664", + "9:16": "1080x1920", + "21:9": "2206x946", + }, +} as const; + +export function isSeedanceVideoConfig(config: Pick) { + return isSeedanceVideoModel(config.model || config.videoModel) || isArkPlanBaseUrl(config.baseUrl); +} + +export function isSeedanceVideoModel(model: string) { + const value = model.toLowerCase(); + return value.includes("seedance") || value.includes("doubao-seedance"); +} + +export function isSeedanceFastModel(model: string) { + const value = model.toLowerCase(); + return isSeedanceVideoModel(value) && value.includes("fast"); +} + +export function isArkPlanBaseUrl(baseUrl: string) { + return baseUrl.toLowerCase().includes("ark.cn-beijing.volces.com/api/plan/v3") || baseUrl.toLowerCase().includes("/api/plan/v3"); +} + +export function normalizeSeedanceResolution(value: string, model = "") { + const normalized = normalizeResolutionToken(value); + if (isSeedanceFastModel(model) && normalized === "1080p") return "720p"; + return seedanceResolutionOptions.some((item) => item.value === normalized) ? normalized : "720p"; +} + +export function normalizeResolutionToken(value: string) { + if (value === "low") return "480p"; + if (value === "auto" || value === "high" || value === "medium") return "720p"; + const resolution = String(value || "").replace(/p$/i, "") || "720"; + return `${resolution}p`; +} + +export function normalizeSeedanceDuration(value: string) { + if (String(value).trim() === "-1") return -1; + const seconds = Math.floor(Number(value) || 5); + return Math.max(4, Math.min(15, seconds)); +} + +export function normalizeSeedanceRatio(value: string) { + if (!value || value === "auto" || value === "adaptive") return "adaptive"; + if (seedanceRatioOptions.some((item) => item.value === value)) return value; + const match = value.match(/^(\d+)x(\d+)$/); + if (!match) return "adaptive"; + const width = Number(match[1]); + const height = Number(match[2]); + if (!width || !height) return "adaptive"; + const ratio = width / height; + const options = [ + ["16:9", 16 / 9], + ["4:3", 4 / 3], + ["1:1", 1], + ["3:4", 3 / 4], + ["9:16", 9 / 16], + ["21:9", 21 / 9], + ] as const; + return options.reduce((best, item) => (Math.abs(item[1] - ratio) < Math.abs(best[1] - ratio) ? item : best), options[0])[0]; +} + +export function seedancePixelLabel(resolution: string, ratio: string) { + const normalizedResolution = normalizeSeedanceResolution(resolution) as keyof typeof seedancePixels; + const normalizedRatio = normalizeSeedanceRatio(ratio) as keyof (typeof seedancePixels)[typeof normalizedResolution] | "adaptive"; + if (normalizedRatio === "adaptive") return "自动匹配"; + return seedancePixels[normalizedResolution][normalizedRatio] || ""; +} + +export function boolConfig(value: string | undefined, fallback: boolean) { + if (value === "true") return true; + if (value === "false") return false; + return fallback; +} + +export function seedanceReferenceLabel(kind: "image" | "video" | "audio", index: number) { + if (kind === "image") return `图片${index + 1}`; + if (kind === "video") return `视频${index + 1}`; + return `音频${index + 1}`; +} + +export function buildSeedancePromptText(prompt: string, images: ReferenceImage[], videos: ReferenceVideo[], audios: ReferenceAudio[]) { + const labels = [ + ...images.map((_, index) => seedanceReferenceLabel("image", index)), + ...videos.map((_, index) => seedanceReferenceLabel("video", index)), + ...audios.map((_, index) => seedanceReferenceLabel("audio", index)), + ]; + const text = prompt.trim(); + if (!labels.length) return text; + return `参考素材编号：${labels.join("、")}。请按这些编号理解提示词中的图片、视频和音频引用。\n\n${text}`; +} + +export function seedanceVideoReferenceError(videos: ReferenceVideo[]) { + let totalDurationMs = 0; + for (let index = 0; index < videos.length; index += 1) { + const video = videos[index]; + const label = seedanceReferenceLabel("video", index); + if (video.bytes && video.bytes > SEEDANCE_REFERENCE_LIMITS.videoMaxBytes) return `${label} 超过 50MB，请压缩后再上传`; + if (video.durationMs) { + if (video.durationMs < 2000 || video.durationMs > 15000) return `${label} 时长需要在 2-15 秒之间`; + totalDurationMs += video.durationMs; + } + if (video.width && video.height) { + if (video.width < 300 || video.width > 6000 || video.height < 300 || video.height > 6000) return `${label} 宽高需要在 300-6000px 之间`; + const ratio = video.width / video.height; + if (ratio < 0.4 || ratio > 2.5) return `${label} 宽高比需要在 0.4-2.5 之间`; + const pixels = video.width * video.height; + if (pixels < 640 * 640 || pixels > 2206 * 946) return `${label} 像素总量不符合 Seedance 要求，请转成 480p/720p/1080p 后再上传`; + } + } + if (totalDurationMs > 15000) return "Seedance 参考视频总时长不能超过 15 秒"; + return ""; +} + +export const seedanceVideoReferenceHint = "参考视频需为 mp4/mov，H.264/H.265，FPS 24-60；含真人人脸素材请使用火山授权 asset:// 素材。"; diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index 938fb85..3bc5b8f 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -4,6 +4,7 @@ import { buildApiUrl, type AiConfig } from "@/stores/use-config-store"; import { useUserStore } from "@/stores/use-user-store"; import { nanoid } from "nanoid"; import { dataUrlToFile } from "@/lib/image-utils"; +import { buildImageReferencePromptText } from "@/lib/image-reference-prompt"; import { imageToDataUrl } from "@/services/image-storage"; import type { ReferenceImage } from "@/types/image"; @@ -31,6 +32,12 @@ const QUALITY_ALIASES: Record = { "2k": "medium", "4k": "high", }; +const DEFAULT_IMAGE_SHORT_SIDE = 1024; +const IMAGE_SIZE_STEP = 16; +const IMAGE_MIN_PIXELS = 655360; +const IMAGE_MAX_PIXELS = 8294400; +const IMAGE_MAX_EDGE = 3840; +const IMAGE_MAX_RATIO = 3; function normalizeQuality(quality: string) { const value = quality.trim().toLowerCase(); @@ -38,36 +45,66 @@ function normalizeQuality(quality: string) { return QUALITY_BASE[normalized] ? normalized : undefined; } -/** Map "quality + ratio" to an explicit pixel dimension like "3840x2160". Returns undefined when quality is auto. */ -function resolveSize(quality: string, ratio: string): string | undefined { - const basePixels = QUALITY_BASE[quality]; - if (!basePixels || ratio === "auto" || !ratio) return undefined; +/** Map "quality + ratio" to an explicit pixel dimension like "3840x2160". */ +function resolveSize(quality: string | undefined, ratio: string): string { + const parsedRatio = parseImageRatio(ratio); + const basePixels = quality ? QUALITY_BASE[quality] : undefined; + const isLandscape = parsedRatio.width >= parsedRatio.height; + const longRatio = isLandscape ? parsedRatio.width / parsedRatio.height : parsedRatio.height / parsedRatio.width; + let longSide: number; + let shortSide: number; - const parts = ratio.split(":"); - if (parts.length !== 2) return undefined; - const w = Number(parts[0]); - const h = Number(parts[1]); - if (!w || !h) return undefined; - - const targetPixels = basePixels * basePixels; - const isLandscape = w >= h; - const longRatio = isLandscape ? w / h : h / w; - - const longSideRaw = Math.sqrt(targetPixels * longRatio); - const longSide = Math.floor(longSideRaw / 16) * 16; - const shortSide = Math.round((longSide / longRatio) / 16) * 16; + if (basePixels) { + const targetPixels = basePixels * basePixels; + const longSideRaw = Math.sqrt(targetPixels * longRatio); + longSide = Math.floor(longSideRaw / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP; + shortSide = Math.round(longSide / longRatio / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP; + } else { + shortSide = DEFAULT_IMAGE_SHORT_SIDE; + longSide = Math.round((shortSide * longRatio) / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP; + } const width = isLandscape ? longSide : shortSide; const height = isLandscape ? shortSide : longSide; - + validateImageSize(width, height); return `${width}x${height}`; } +function parseImageRatio(value: string) { + const parts = value.split(":"); + if (parts.length !== 2) throw new Error("图像尺寸格式不支持，请使用 auto、9:16 或 1024x1024"); + const w = Number(parts[0]); + const h = Number(parts[1]); + if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) throw new Error("图像比例必须是正数，例如 9:16"); + if (Math.max(w, h) / Math.min(w, h) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1，请调整尺寸"); + return { width: w, height: h }; +} + +function parseImageDimensions(value: string) { + const match = value.match(/^(\d+)x(\d+)$/i); + if (!match) return null; + return { width: Number(match[1]), height: Number(match[2]) }; +} + +function validateImageSize(width: number, height: number) { + if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) throw new Error("图像尺寸必须是正整数，例如 1024x1024"); + if (width % IMAGE_SIZE_STEP !== 0 || height % IMAGE_SIZE_STEP !== 0) throw new Error("图像尺寸的宽高必须是 16 的倍数，请调整尺寸"); + if (Math.max(width, height) > IMAGE_MAX_EDGE) throw new Error("图像尺寸最长边不能超过 3840px，请调整尺寸"); + if (Math.max(width, height) / Math.min(width, height) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1，请调整尺寸"); + const pixels = width * height; + if (pixels < IMAGE_MIN_PIXELS || pixels > IMAGE_MAX_PIXELS) throw new Error("图像总像素需在 655360 到 8294400 之间，请调整尺寸"); +} + function resolveRequestSize(quality: string | undefined, size: string) { const value = size.trim(); - if (!value || value === "auto") return undefined; - if (/^\d+x\d+$/.test(value)) return value; - return (quality && resolveSize(quality, value)) || value; + if (!value || value.toLowerCase() === "auto") return undefined; + const dimensions = parseImageDimensions(value); + if (dimensions) { + validateImageSize(dimensions.width, dimensions.height); + return `${dimensions.width}x${dimensions.height}`; + } + if (value.includes(":")) return resolveSize(quality, value); + throw new Error("图像尺寸格式不支持，请使用 auto、9:16 或 1024x1024"); } function resolveImageDataUrl(item: Record) { @@ -100,11 +137,17 @@ function parseImagePayload(payload: ImageApiResponse) { function readAxiosError(error: unknown, fallback: string) { if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) { const responseData = error.response?.data; - return responseData?.msg || responseData?.error?.message || (error.response?.status ? `${fallback}：${error.response.status}` : fallback); + return responseData?.msg || responseData?.error?.message || readStatusError(error.response?.status, fallback); } return error instanceof Error ? error.message : fallback; } +function readStatusError(status: number | undefined, fallback: string) { + if (status === 401 || status === 403) return "鉴权失败，请检查 API Key、套餐权限或模型权限"; + if (status === 429) return "请求被限流或额度不足，请稍后重试"; + return status ? `${fallback}：${status}` : fallback; +} + function parseStreamChunk(chunk: string, onDelta: (value: string) => void) { let deltaText = ""; for (const eventBlock of chunk.split("\n\n")) { @@ -181,9 +224,10 @@ export async function requestEdit(config: AiConfig, prompt: string, references: const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1))); const quality = normalizeQuality(config.quality); const requestSize = resolveRequestSize(quality, config.size); + const requestPrompt = buildImageReferencePromptText(prompt, references); const formData = new FormData(); formData.set("model", config.model); - formData.set("prompt", withSystemPrompt(config, prompt)); + formData.set("prompt", withSystemPrompt(config, requestPrompt)); formData.set("n", String(n)); formData.set("response_format", "b64_json"); if (quality) { diff --git a/web/src/services/api/video.ts b/web/src/services/api/video.ts index 863bcab..ef02431 100644 --- a/web/src/services/api/video.ts +++ b/web/src/services/api/video.ts @@ -1,29 +1,67 @@ import axios from "axios"; import { dataUrlToFile } from "@/lib/image-utils"; +import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage"; import { imageToDataUrl } from "@/services/image-storage"; +import { boolConfig, buildSeedancePromptText, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video"; import { buildApiUrl, type AiConfig } from "@/stores/use-config-store"; import { useUserStore } from "@/stores/use-user-store"; import type { ReferenceImage } from "@/types/image"; +import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; type VideoResponse = { id: string; status?: string; error?: { message?: string } }; type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string }; +type SeedanceTask = { + id: string; + status?: "queued" | "running" | "succeeded" | "failed" | "cancelled" | "expired"; + error?: { code?: string; message?: string } | null; + content?: { video_url?: string; last_frame_url?: string } | null; +}; +type ApiEnvelope = T | { code?: number; data?: T | null; msg?: string }; +type ReferenceMediaUploadResponse = { id: string; url: string; mimeType: string; bytes: number }; + +export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string }; function aiApiUrl(config: AiConfig, path: string) { return config.channelMode === "remote" ? `/api/v1${path}` : buildApiUrl(config.baseUrl, path); } -function aiHeaders(config: AiConfig) { +function aiHeaders(config: AiConfig, contentType?: string) { const token = useUserStore.getState().token; - return config.channelMode === "remote" ? (token ? { Authorization: `Bearer ${token}` } : undefined) : { Authorization: `Bearer ${config.apiKey}` }; + return config.channelMode === "remote" + ? { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + ...(contentType ? { "Content-Type": contentType } : {}), + } + : { + Authorization: `Bearer ${config.apiKey}`, + ...(contentType ? { "Content-Type": contentType } : {}), + }; } function refreshRemoteUser(config: AiConfig) { if (config.channelMode === "remote") void useUserStore.getState().hydrateUser(); } -export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = []) { - const model = config.model || config.videoModel; +export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = []): Promise { + const model = (config.model || config.videoModel).trim(); + assertVideoConfig(config, model); + if (isSeedanceVideoConfig({ ...config, model })) { + return requestSeedanceGeneration(config, model, prompt, references, videoReferences, audioReferences); + } + if (videoReferences.length || audioReferences.length) { + throw new Error("当前视频接口不支持参考视频或参考音频，请切换到 Seedance 2.0 / 火山 Agent Plan 模型，或移除参考素材"); + } + return requestOpenAIVideoGeneration(config, model, prompt, references); +} + +export async function storeGeneratedVideo(result: VideoGenerationResult): Promise { + if (result.blob) return uploadMediaFile(result.blob, "video"); + if (result.url) return { url: result.url, storageKey: "", bytes: 0, mimeType: result.mimeType || "video/mp4" }; + throw new Error("视频接口没有返回可播放的视频"); +} + +async function requestOpenAIVideoGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[]) { const body = new FormData(); body.append("model", model); body.append("prompt", prompt); @@ -36,21 +74,162 @@ export async function requestVideoGeneration(config: AiConfig, prompt: string, r try { const created = unwrapVideoResponse((await axios.post(aiApiUrl(config, "/videos"), body, { headers: aiHeaders(config) })).data); if (!created.id) throw new Error("视频接口没有返回任务 ID"); - for (;;) { + for (let attempt = 0; attempt < 120; attempt += 1) { const video = unwrapVideoResponse((await axios.get(aiApiUrl(config, `/videos/${created.id}`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined })).data); if (video.status === "completed") break; if (video.status === "failed" || video.status === "cancelled") throw new Error(video.error?.message || "视频生成失败"); - await new Promise((resolve) => setTimeout(resolve, 2500)); + if (attempt === 119) throw new Error("视频生成超时，请稍后重试"); + await delay(2500); } const content = await axios.get(aiApiUrl(config, `/videos/${created.id}/content`), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined, responseType: "blob" }); await assertVideoBlob(content.data); refreshRemoteUser(config); - return content.data; + return { blob: content.data }; } catch (error) { throw new Error(readAxiosError(error, "视频生成失败")); } } +async function requestSeedanceGeneration(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) { + if (audioReferences.length && !references.length && !videoReferences.length) { + throw new Error("Seedance 参考音频不能单独使用，请同时添加参考图或参考视频"); + } + assertSeedanceVideoReferences(videoReferences); + assertSeedanceAudioReferences(audioReferences); + const content = await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences); + if (!content.length) throw new Error("请输入视频提示词，或连接参考图片/视频/音频"); + const payload = { + model, + content, + ratio: normalizeSeedanceRatio(config.size), + resolution: normalizeSeedanceResolution(config.vquality, model), + duration: normalizeSeedanceDuration(config.videoSeconds), + generate_audio: boolConfig(config.videoGenerateAudio, true), + watermark: boolConfig(config.videoWatermark, false), + }; + + try { + const created = unwrapSeedanceTask((await axios.post>(seedanceApiUrl(config), payload, { headers: aiHeaders(config, "application/json") })).data); + if (!created.id) throw new Error("Seedance 接口没有返回任务 ID"); + for (let attempt = 0; attempt < 120; attempt += 1) { + const task = unwrapSeedanceTask((await axios.get>(seedanceApiUrl(config, created.id), { headers: aiHeaders(config), params: config.channelMode === "remote" ? { model } : undefined })).data); + if (task.status === "succeeded") { + const url = task.content?.video_url; + if (!url) throw new Error("Seedance 任务成功但没有返回视频 URL"); + refreshRemoteUser(config); + return videoResultFromUrl(url); + } + if (task.status === "failed" || task.status === "cancelled" || task.status === "expired") throw new Error(task.error?.message || `Seedance 视频生成${task.status === "expired" ? "超时" : "失败"}`); + if (attempt === 119) throw new Error("Seedance 视频生成超时，请稍后重试"); + await delay(5000); + } + throw new Error("Seedance 视频生成超时，请稍后重试"); + } catch (error) { + throw new Error(readAxiosError(error, "Seedance 视频生成失败")); + } +} + +function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) { + const error = seedanceVideoReferenceError(videoReferences); + if (error) throw new Error(error); + let total = 0; + for (const video of videoReferences) { + if (!video.durationMs) continue; + if (video.durationMs < 2000 || video.durationMs > 15000) throw new Error("Seedance 参考视频单个时长需要在 2-15 秒之间"); + total += video.durationMs; + } + if (total > 15000) throw new Error("Seedance 参考视频总时长不能超过 15 秒"); +} + +function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) { + let total = 0; + for (const audio of audioReferences) { + if (!audio.durationMs) continue; + if (audio.durationMs < 2000 || audio.durationMs > 15000) throw new Error("Seedance 参考音频单个时长需要在 2-15 秒之间"); + total += audio.durationMs; + } + if (total > 15000) throw new Error("Seedance 参考音频总时长不能超过 15 秒"); +} + +function seedanceApiUrl(config: AiConfig, taskId?: string) { + if (config.channelMode === "remote") return taskId ? `/api/v1/videos/${encodeURIComponent(taskId)}` : "/api/v1/videos"; + return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`); +} + +async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) { + const content: Array> = []; + const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences); + if (text) content.push({ type: "text", text }); + for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) { + content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" }); + } + for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) { + content.push({ type: "video_url", video_url: { url: await resolveSeedanceVideoUrl(video) }, role: "reference_video" }); + } + for (const audio of audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios)) { + content.push({ type: "audio_url", audio_url: { url: await resolveSeedanceAudioUrl(audio) }, role: "reference_audio" }); + } + return content; +} + +async function resolveSeedanceImageUrl(config: AiConfig, image: ReferenceImage) { + const directUrl = image.url || image.dataUrl; + if (isPublicMediaUrl(directUrl) || directUrl.startsWith("asset://")) return directUrl; + const dataUrl = await imageToDataUrl(image); + if (!dataUrl) throw new Error("参考图读取失败，请换一张图片或重新上传"); + if (config.channelMode === "remote") { + return uploadReferenceMedia(dataUrlToFile({ ...image, dataUrl })); + } + return dataUrl; +} + +async function resolveSeedanceVideoUrl(video: ReferenceVideo) { + if (isPublicMediaUrl(video.url) || video.url.startsWith("asset://")) return video.url; + let blob: Blob | null = null; + if (video.storageKey) blob = await getMediaBlob(video.storageKey); + if (!blob && video.url?.startsWith("blob:")) blob = await (await fetch(video.url)).blob(); + if (!blob) throw new Error("参考视频必须是公网 URL、素材 ID，或本地已保存的视频"); + const file = new File([blob], video.name || "reference-video.mp4", { type: video.type || blob.type || "video/mp4" }); + return uploadReferenceMedia(file); +} + +async function resolveSeedanceAudioUrl(audio: ReferenceAudio) { + if (isPublicMediaUrl(audio.url) || audio.url.startsWith("asset://")) return audio.url; + let blob: Blob | null = null; + if (audio.storageKey) blob = await getMediaBlob(audio.storageKey); + if (!blob && audio.url?.startsWith("blob:")) blob = await (await fetch(audio.url)).blob(); + if (!blob) throw new Error("参考音频必须是公网 URL、素材 ID，或本地已保存的音频"); + const file = new File([blob], audio.name || "reference-audio.mp3", { type: audio.type || blob.type || "audio/mpeg" }); + return uploadReferenceMedia(file); +} + +async function uploadReferenceMedia(file: File) { + const token = useUserStore.getState().token; + if (!token) throw new Error("使用本地参考素材需要先登录，并在服务端配置 PUBLIC_BASE_URL"); + const body = new FormData(); + body.append("file", file, file.name); + const response = await axios.post>("/api/v1/media/references", body, { headers: { Authorization: `Bearer ${token}` } }); + const payload = unwrapEnvelope(response.data, "参考素材上传失败"); + if (!payload.url) throw new Error("参考素材上传后没有返回公网 URL"); + return payload.url; +} + +async function videoResultFromUrl(url: string): Promise { + try { + const response = await axios.get(url, { responseType: "blob" }); + await assertVideoBlob(response.data); + return { blob: response.data }; + } catch { + return { url, mimeType: "video/mp4" }; + } +} + +function assertVideoConfig(config: AiConfig, model: string) { + if (!model) throw new Error("请先配置视频模型"); + if (config.channelMode === "local" && !config.baseUrl.trim()) throw new Error("请先配置 Base URL"); + if (config.channelMode === "local" && !config.apiKey.trim()) throw new Error("请先配置 API Key"); +} + function normalizeVideoSeconds(value: string) { const seconds = Math.floor(Number(value) || 6); return String(Math.max(1, Math.min(20, seconds))); @@ -71,30 +250,53 @@ function normalizeVideoResolution(value: string) { } function unwrapVideoResponse(payload: ApiVideoResponse) { - if (!payload) throw new Error("接口没有返回视频任务"); - if ("code" in payload && typeof payload.code === "number") { + return unwrapEnvelope(payload, "接口没有返回视频任务"); +} + +function unwrapSeedanceTask(payload: ApiEnvelope) { + return unwrapEnvelope(payload, "Seedance 接口没有返回任务"); +} + +function unwrapEnvelope(payload: ApiEnvelope, emptyMessage: string): T { + if (!payload) throw new Error(emptyMessage); + if (typeof payload === "object" && "code" in payload && typeof payload.code === "number") { if (payload.code !== 0) throw new Error(payload.msg || "请求失败"); - if (!payload.data) throw new Error("接口没有返回视频任务"); + if (!payload.data) throw new Error(emptyMessage); return payload.data; } - return payload; + return payload as T; } function readAxiosError(error: unknown, fallback: string) { if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) { const responseData = error.response?.data; - return responseData?.msg || responseData?.error?.message || (error.response?.status ? `${fallback}：${error.response.status}` : fallback); + return responseData?.msg || responseData?.error?.message || statusMessage(error.response?.status, fallback); } return error instanceof Error ? error.message : fallback; } +function statusMessage(status: number | undefined, fallback: string) { + if (status === 401 || status === 403) return "鉴权失败，请检查 API Key、套餐权限或模型权限"; + if (status === 429) return "请求被限流或额度不足，请稍后重试"; + return status ? `${fallback}（${status}）` : fallback; +} + async function assertVideoBlob(blob: Blob) { if (!blob.type.includes("json")) return; - let payload: { code?: number; msg?: string }; + let payload: { code?: number; msg?: string; error?: { message?: string } }; try { - payload = JSON.parse(await blob.text()) as { code?: number; msg?: string }; + payload = JSON.parse(await blob.text()) as { code?: number; msg?: string; error?: { message?: string } }; } catch { return; } if (typeof payload.code === "number" && payload.code !== 0) throw new Error(payload.msg || "视频下载失败"); + if (payload.error?.message) throw new Error(payload.error.message); +} + +function isPublicMediaUrl(value: string) { + return /^https?:\/\//i.test(value || ""); +} + +function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/web/src/services/file-storage.ts b/web/src/services/file-storage.ts index 6a1eed6..60ee07a 100644 --- a/web/src/services/file-storage.ts +++ b/web/src/services/file-storage.ts @@ -3,7 +3,7 @@ import localforage from "localforage"; import { nanoid } from "nanoid"; -export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number }; +export type UploadedFile = { url: string; storageKey: string; bytes: number; mimeType: string; width?: number; height?: number; durationMs?: number }; const store = localforage.createInstance({ name: "infinite-canvas", storeName: "media_files" }); const objectUrls = new Map(); @@ -14,7 +14,7 @@ export async function uploadMediaFile(input: string | Blob, prefix = "file"): Pr await store.setItem(storageKey, blob); const url = URL.createObjectURL(blob); objectUrls.set(storageKey, url); - const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : {}; + const meta = blob.type.startsWith("video/") ? await readVideoMeta(url) : blob.type.startsWith("audio/") ? await readAudioMeta(url) : {}; return { url, storageKey, bytes: blob.size, mimeType: blob.type || "application/octet-stream", ...meta }; } @@ -68,11 +68,21 @@ export function collectMediaStorageKeys(value: unknown, keys = new Set() } function readVideoMeta(url: string) { - return new Promise<{ width: number; height: number }>((resolve) => { + return new Promise<{ width: number; height: number; durationMs?: number }>((resolve) => { const video = document.createElement("video"); - const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720 }); + const done = () => resolve({ width: video.videoWidth || 1280, height: video.videoHeight || 720, durationMs: Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : undefined }); video.onloadedmetadata = done; video.onerror = done; video.src = url; }); } + +function readAudioMeta(url: string) { + return new Promise<{ durationMs?: number }>((resolve) => { + const audio = document.createElement("audio"); + const done = () => resolve({ durationMs: Number.isFinite(audio.duration) ? Math.round(audio.duration * 1000) : undefined }); + audio.onloadedmetadata = done; + audio.onerror = done; + audio.src = url; + }); +} diff --git a/web/src/stores/use-config-store.ts b/web/src/stores/use-config-store.ts index c4ba0f5..625fecb 100644 --- a/web/src/stores/use-config-store.ts +++ b/web/src/stores/use-config-store.ts @@ -17,6 +17,8 @@ export type AiConfig = { textModel: string; videoSeconds: string; vquality: string; + videoGenerateAudio: string; + videoWatermark: string; systemPrompt: string; models: string[]; quality: string; @@ -36,6 +38,8 @@ export const defaultConfig: AiConfig = { textModel: "gpt-5.5", videoSeconds: "6", vquality: "720", + videoGenerateAudio: "true", + videoWatermark: "false", systemPrompt: "", models: [], quality: "auto", @@ -61,19 +65,44 @@ function resolveEffectiveConfig(config: AiConfig, modelChannel: AdminPublicSetti const channelMode = modelChannel?.allowCustomChannel ? config.channelMode : "remote"; if (channelMode === "local" || !modelChannel) return { ...config, channelMode }; const models = modelChannel.availableModels; - const fallbackModel = modelChannel.defaultModel || models[0] || ""; + const fallbackTextModel = validDefault(modelChannel.defaultTextModel, models) || preferredModel(models, isTextModelName); + const fallbackModel = validDefault(modelChannel.defaultModel, models) || fallbackTextModel || models[0] || ""; + const fallbackImageModel = validDefault(modelChannel.defaultImageModel, models) || preferredModel(models, isImageModelName) || fallbackModel; + const fallbackVideoModel = validDefault(modelChannel.defaultVideoModel, models) || preferredModel(models, isVideoModelName) || fallbackModel; return { ...config, channelMode, models, model: models.includes(config.model) ? config.model : fallbackModel, - imageModel: models.includes(config.imageModel) ? config.imageModel : modelChannel.defaultImageModel || fallbackModel, - videoModel: models.includes(config.videoModel) ? config.videoModel : modelChannel.defaultVideoModel || fallbackModel, - textModel: models.includes(config.textModel) ? config.textModel : modelChannel.defaultTextModel || fallbackModel, + imageModel: models.includes(config.imageModel) ? config.imageModel : fallbackImageModel, + videoModel: models.includes(config.videoModel) ? config.videoModel : fallbackVideoModel, + textModel: models.includes(config.textModel) ? config.textModel : fallbackTextModel || fallbackModel, systemPrompt: modelChannel.systemPrompt, }; } +function validDefault(model: string, models: string[]) { + return models.includes(model) ? model : ""; +} + +function preferredModel(models: string[], predicate: (model: string) => boolean) { + return models.find(predicate) || ""; +} + +function isVideoModelName(model: string) { + const value = model.toLowerCase(); + return value.includes("seedance") || value.includes("video"); +} + +function isImageModelName(model: string) { + const value = model.toLowerCase(); + return value.includes("seedream") || value.includes("gpt-image") || value.includes("image"); +} + +function isTextModelName(model: string) { + return !isImageModelName(model) && !isVideoModelName(model); +} + function isAiConfigReady(config: AiConfig, model: string) { return Boolean(model.trim()) && (config.channelMode === "remote" || Boolean(config.baseUrl.trim() && config.apiKey.trim())); } @@ -112,7 +141,7 @@ export const useConfigStore = create()( partialize: (state) => ({ config: state.config }), merge: (persisted, current) => { const config = { ...defaultConfig, ...((persisted as Partial).config || {}) }; - return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720" } }; + return { ...current, config: { ...config, channelMode: config.channelMode || "remote", imageModel: config.imageModel || config.model, videoModel: config.videoModel || "grok-imagine-video", textModel: config.textModel || config.model, videoSeconds: config.videoSeconds || "6", vquality: config.vquality || "720", videoGenerateAudio: config.videoGenerateAudio || "true", videoWatermark: config.videoWatermark || "false" } }; }, }, ), @@ -125,7 +154,27 @@ export function useEffectiveConfig() { } export function buildApiUrl(baseUrl: string, path: string) { - const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, ""); - const apiBaseUrl = normalizedBaseUrl.endsWith("/v1") ? normalizedBaseUrl : `${normalizedBaseUrl}/v1`; + let normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, ""); + normalizedBaseUrl = normalizeArkPlanBaseUrl(normalizedBaseUrl); + const lowerBaseUrl = normalizedBaseUrl.toLowerCase(); + const apiBaseUrl = lowerBaseUrl.endsWith("/v1") || lowerBaseUrl.endsWith("/api/v3") || lowerBaseUrl.endsWith("/api/plan/v3") ? normalizedBaseUrl : `${normalizedBaseUrl}/v1`; return `${apiBaseUrl}${path}`; } + +function normalizeArkPlanBaseUrl(baseUrl: string) { + try { + const url = new URL(baseUrl); + const path = url.pathname.replace(/\/+$/, ""); + const lowerPath = path.toLowerCase(); + const arkPlanIndex = lowerPath.indexOf("/api/plan/v3"); + if (arkPlanIndex < 0) return baseUrl; + const end = arkPlanIndex + "/api/plan/v3".length; + if (lowerPath.length !== end && lowerPath[end] !== "/") return baseUrl; + url.pathname = path.slice(0, end); + url.search = ""; + url.hash = ""; + return url.toString().replace(/\/+$/, ""); + } catch { + return baseUrl; + } +} diff --git a/web/src/types/media.ts b/web/src/types/media.ts new file mode 100644 index 0000000..dd1454c --- /dev/null +++ b/web/src/types/media.ts @@ -0,0 +1,20 @@ +export type ReferenceVideo = { + id: string; + name: string; + type: string; + url: string; + storageKey?: string; + bytes?: number; + width?: number; + height?: number; + durationMs?: number; +}; + +export type ReferenceAudio = { + id: string; + name: string; + type: string; + url: string; + storageKey?: string; + durationMs?: number; +};