commit d1021a00a077a2878863f3809862ee66be2d2002 Author: oy-paddy <24783915+oy-paddy@users.noreply.github.com> Date: Thu Jun 11 15:05:25 2026 +0800 feat: DashScope 模型代理服务初始提交 支持 Anthropic 和 OpenAI 兼容协议的模型池反代，自动切换免费额度耗尽的模型。 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c6028b7 --- /dev/null +++ b/.env.example @@ -0,0 +1,32 @@ +PORT=3300 + +# 客户端调用本代理时使用的唯一密钥 +PROXY_API_KEY=sk-001 + +# 你的真实 DashScope API 密钥，请勿发送给客户端。 +# 所有密钥共享同一个 MODEL_IDS 模型池，代理会在单个密钥下尝试所有模型， +# 全部失败后才会切换到下一个密钥。 +DASHSCOPE_API_KEYS=sk-change-me-1 + +# DashScope Anthropic 兼容协议的基础 URL +UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/apps/anthropic + +# DashScope OpenAI 兼容协议（Chat Completions）的基础 URL +OPENAI_UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +# 逗号分隔的模型池，代理会用其中之一覆盖请求体中的 "model" 字段 +MODEL_IDS=deepseek-v4-flash,deepseek-v4-pro,qwen3.7-max-2026-06-08,qwen3.7-plus-2026-05-26,qwen3.7-plus,qwen3.7-max-preview,qwen3.7-max-2026-05-17,qwen3.7-max,qwen3.7-max-2026-05-20,qwen3.6-27b,qwen3.6-max-preview,qwen3.6-flash,qwen3.6-35b-a3b,qwen3.6-flash-2026-04-16,qwen3.6-plus,qwen3.6-plus-2026-04-02 + +# 免费额度耗尽后，该密钥+模型组合的禁用时长（秒）。默认 30 天。 +MODEL_COOLDOWN_SECONDS=2592000 + +# 用于持久化冷却状态的 JSON 文件路径，重启后状态不丢失 +STATE_PATH=./data/proxy-state.json + +# authorization: 使用 Authorization: Bearer +# x-api-key: 使用 x-api-key: +# both: 同时发送以上两个请求头 +UPSTREAM_AUTH_MODE=authorization + +# 设为 false 可禁用 CORS；否则填 * 或指定具体的来源地址 +CORS_ORIGIN=* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c7a5aff --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +node_modules +dist +release +release.zip +.env +data/ +*.log +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..70233e5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM node:22-alpine AS deps +WORKDIR /app +COPY package.json pnpm-lock.yaml* ./ +RUN corepack enable && pnpm install --frozen-lockfile + +FROM node:22-alpine AS build +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY . . +RUN corepack enable && pnpm build + +FROM node:22-alpine AS runner +WORKDIR /app +ENV NODE_ENV=production +COPY package.json ./ +COPY --from=deps /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +EXPOSE 3000 +CMD ["node", "dist/index.js"] diff --git a/LLM.md b/LLM.md new file mode 100644 index 0000000..616ec66 --- /dev/null +++ b/LLM.md @@ -0,0 +1,111 @@ +# LLM 接入说明 + +这个项目启动后提供本地模型代理。客户端不要使用真实 DashScope key，只使用 `.env` 里的 `PROXY_API_KEY`。 + +## 服务地址 + +默认端口来自 `.env`： + +```env +PORT=3300 +PROXY_API_KEY=sk-001 +``` + +本地代理地址： + +```text +http://127.0.0.1:3300 +``` + +客户端 API Key： + +```text +sk-001 +``` + +请求认证可以使用： + +```http +Authorization: Bearer sk-001 +``` + +或： + +```http +x-api-key: sk-001 +``` + +## Anthropic 协议客户端 + +用于 Claude Code CLI 或其他 Anthropic Messages 兼容客户端： + +```text +base URL: http://127.0.0.1:3300 +api key: sk-001 +``` + +Messages endpoint： + +```text +POST http://127.0.0.1:3300/v1/messages +``` + +## OpenAI 协议客户端 + +用于 Pi CLI 或其他 OpenAI Chat Completions 兼容客户端： + +```text +base URL: http://127.0.0.1:3300/v1 +api key: sk-001 +``` + +Chat Completions endpoint： + +```text +POST http://127.0.0.1:3300/v1/chat/completions +``` + +Models endpoint： + +```text +GET http://127.0.0.1:3300/v1/models +``` + +## Pi CLI + +在 Pi CLI 的 provider 配置里使用： + +```json +{ + "baseUrl": "http://127.0.0.1:3300/v1", + "api": "openai-completions", + "apiKey": "sk-001" +} +``` + +## Claude Code CLI + +在 Claude Code CLI 里按 Anthropic 兼容接口配置： + +```text +base URL: http://127.0.0.1:3300 +api key: sk-001 +``` + +如果使用环境变量方式，核心值仍然是： + +```text +ANTHROPIC_BASE_URL=http://127.0.0.1:3300 +ANTHROPIC_API_KEY=sk-001 +``` + +## 模型 + +客户端传入的 `model` 会被代理替换为 `.env` 里的 `MODEL_IDS` 中当前可用模型。 + +启动日志会打印实际应用的模型 ID： + +```text +[proxy] applied model ids: +[proxy] - +``` diff --git a/README.md b/README.md new file mode 100644 index 0000000..0297190 --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ +# DashScope 模型代理 + +基于 Hono.js 的反向代理服务，同时支持 DashScope Anthropic 兼容接口和 OpenAI 兼容接口。客户端只需一个代理地址和一个代理密钥，真实 DashScope API Key 和模型池均在服务端隐藏管理。当 DashScope 返回免费额度耗尽错误时，代理自动切换模型。 + +## 功能 + +- 对外只暴露一个 API Key +- 隐藏真实 DashScope API Key 和模型池 +- Anthropic 协议 `POST /v1/messages` 代理 +- OpenAI Chat Completions 协议 `POST /v1/chat/completions` 代理 +- OpenAI 协议 `GET /v1/models` 返回本地模型池列表 +- 自动替换请求体中的 `model` 字段 +- Key 优先故障转移：在当前 Key 下尝试所有可用模型，全部失败后才切换到下一个 Key +- 免费额度耗尽自动重试，触发条件（满足任意一组）： + - HTTP `403`，`code` 为 `AccessDenied`，且 `message` 同时包含 `free tier` 和 `exhausted` + - `code` 或 `type` 为 `AllocationQuota.FreeTierOnly` +- 冷却状态按 Key+模型组合持久化到 JSON 文件，重启不丢失 +- 上游返回正常状态码后，流式响应直接透传，不做缓冲 + +## 相关文档 + +- [使用说明](docs/usage.md) +- [状态文件说明](docs/state-file.md) +- [Q/A 与方案决策记录](docs/qa-and-decisions.md) + +## 快速开始 + +```bash +pnpm install +cp .env.example .env +``` + +按需修改 `.env`： + +```env +PORT=3300 + +PROXY_API_KEY=sk-001 + +DASHSCOPE_API_KEYS=sk-your-real-key-1,sk-your-real-key-2 +UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/apps/anthropic +OPENAI_UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +MODEL_IDS=deepseek-v4-flash,deepseek-v4-pro,qwen3.7-max,qwen3.7-plus +MODEL_COOLDOWN_SECONDS=2592000 +STATE_PATH=./data/proxy-state.json +UPSTREAM_AUTH_MODE=authorization +CORS_ORIGIN=* +``` + +开发模式： + +```bash +pnpm dev +``` + +生产模式： + +```bash +pnpm build +pnpm start +``` + +## 客户端调用 + +### Anthropic 协议 + +代理地址： + +```text +http://localhost:3300 +``` + +请求头二选一： + +```http +Authorization: Bearer sk-001 +``` + +或： + +```http +x-api-key: sk-001 +``` + +示例： + +```bash +curl http://localhost:3300/v1/messages \ + -H "Authorization: Bearer sk-001" \ + -H "Content-Type: application/json" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "any-model-name", + "max_tokens": 256, + "messages": [ + { + "role": "user", + "content": "你好" + } + ] + }' +``` + +### OpenAI 协议 + +代理 base URL： + +```text +http://localhost:3300/v1 +``` + +示例： + +```bash +curl http://localhost:3300/v1/chat/completions \ + -H "Authorization: Bearer sk-001" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "any-model-name", + "messages": [ + { + "role": "user", + "content": "你好" + } + ], + "stream": true + }' +``` + +客户端可以使用 OpenAI SDK 接入 `http://localhost:3300/v1`。服务端本身只是反代，使用原生 `fetch`，不需要 `openai` SDK。 + +## 服务状态 + +健康检查： + +```bash +curl http://localhost:3300/health +``` + +模型池状态： + +```bash +curl http://localhost:3300/models/status \ + -H "Authorization: Bearer sk-001" +``` + +## Docker + +```bash +docker build -t dashscope-model-proxy . +docker run --env-file .env -p 3300:3300 dashscope-model-proxy +``` + +## 说明 + +冷却状态保存在 `STATE_PATH` 指定的 JSON 文件中，存储内容包括 Key 哈希、模型 ID、冷却截止时间、失败次数和最近错误信息，不保存明文 DashScope API Key。 + +状态文件仅供单进程写入设计。若部署多实例，需各自维护独立状态文件或引入外部协调存储。 diff --git a/docs/qa-and-decisions.md b/docs/qa-and-decisions.md new file mode 100644 index 0000000..fcb5ee6 --- /dev/null +++ b/docs/qa-and-decisions.md @@ -0,0 +1,455 @@ +# Q/A 与方案决策记录 + +本文记录本地 DashScope 反代服务的需求讨论、关键 Q/A、方案选择和最终共识，方便后续继续维护。 + +## 1. 初始需求 + +用户有多个 DashScope Anthropic 兼容模型 ID，希望对外只暴露一个固定 URL 和一个代理 key。 + +上游地址： + +```text +https://dashscope.aliyuncs.com/apps/anthropic +``` + +典型免费额度耗尽错误： + +```json +{ + "request_id": "xxx", + "code": "AccessDenied", + "message": "The free tier of the model has been exhausted. If you wish to continue access the model on a paid basis, please disable the \"use free tier only\" mode in the management console." +} +``` + +判断条件： + +```text +HTTP 状态码 = 403 +code = AccessDenied +message 包含 free tier 和 exhausted +``` + +最终目标： + +1. 客户端只配置一个本地代理 URL。 +2. 客户端只使用一个代理 key。 +3. 真实 DashScope API Key 不暴露给客户端。 +4. 模型进入冷却时自动切换，不需要手动改 model。 + +## 2. 技术选型 + +### Q：用什么框架搭建？ + +A：用户指定使用 Hono.js。 + +最终方案： + +```text +Node.js + TypeScript + Hono.js +``` + +原因： + +- 项目小，Hono 足够轻量。 +- Node 20+ 原生支持 `fetch` 和 Web Stream。 +- 适合做 Anthropic 兼容 HTTP 反代。 + +## 3. 项目位置 + +### Q：放在哪里？ + +A：单独新建项目，不放到原 uni-app 项目里。 + +最终路径： + +```text +proxy-server +``` + +## 4. 对外接口 + +### Q：本地对外 URL 是什么？ + +A：默认根据 `.env` 的 `PORT` 决定。 + +当前给 Pi CLI 使用的配置： + +```env +PORT=3300 +PROXY_API_KEY=sk-001 +``` + +本地 URL： + +```text +http://localhost:3300 +``` + +Anthropic Messages 接口： + +```text +http://localhost:3300/v1/messages +``` + +客户端请求头二选一： + +```http +Authorization: Bearer sk-001 +``` + +或： + +```http +x-api-key: sk-001 +``` + +## 5. 代理是否修改 Prompt + +### Q：代理有没有加提示词、改 system 或 messages？ + +A：没有。 + +当前代理只修改请求体里的： + +```json +"model" +``` + +不会修改： + +```text +system +messages +tools +tool_choice +thinking +max_tokens +temperature +任何 prompt 内容 +``` + +Pi CLI 中模型回答“我是 Claude，由 Anthropic 开发”不代表代理加了提示词。更可能来自： + +1. Pi CLI 自己的系统提示词和代理框架上下文。 +2. DashScope Anthropic 兼容接口下模型的身份回答习惯。 + +## 6. 单个真实 key + 多模型阶段 + +### Q：最早版本怎么切换？ + +A：最早版本只有一个真实 DashScope key，多个 `MODEL_IDS`。现在即使只有一个真实 key，也统一写在 `DASHSCOPE_API_KEYS` 里。 + +配置示例： + +```env +DASHSCOPE_API_KEYS=sk-xxx +MODEL_IDS=qwen3.7-max,qwen-plus,qwen-max +``` + +切换逻辑： + +1. 使用当前模型转发。 +2. 如果命中免费额度耗尽 403，冷却这个模型。 +3. 同一个请求改用下一个模型重试。 +4. 所有模型都不可用时返回 `503`。 + +## 7. 多 Key + 多模型阶段 + +### Q：多 key 的情况下怎么切？ + +A：所有 key 共用同一组 `MODEL_IDS`。切换规则是 key 优先。 + +最终配置： + +```env +DASHSCOPE_API_KEYS=sk-key-1,sk-key-2,sk-key-3 +MODEL_IDS=model-a,model-b,model-c +``` + +最终切换规则： + +1. 优先使用当前 key。 +2. 当前 key 下某个模型触发免费额度耗尽 403，只冷却这个 `key + model` 组合。 +3. 继续尝试当前 key 下的下一个模型。 +4. 只有当前 key 下所有模型都不可用，才切换到下一个 key。 +5. 所有 key+model 组合都不可用时返回 `503`。 + +这是本项目最重要的调度共识。 + +## 8. 冷却时间 + +### Q：冷却多久？ + +A：用户确认冷却时间为 1 个月。 + +最终配置： + +```env +MODEL_COOLDOWN_SECONDS=2592000 +``` + +说明： + +```text +2592000 秒 = 30 天 +``` + +## 9. 状态持久化选择 + +### Q：为什么引入持久化状态？ + +A：服务重启后不能丢失冷却状态。 + +如果只用内存，重启后所有模型都会重新变成可用，可能继续打到已经进入免费额度冷却期的 key+model。 + +### Q：选什么存储？ + +A：最终选 JSON 状态文件。 + +原因： + +- 本地部署，不需要额外安装 MySQL/Postgres/Redis。 +- 只有冷却状态和调度游标，不需要关系型查询能力。 +- 一个文件即可持久化，适合本地工具。 +- 避免 `better-sqlite3` 原生 binding 在 Windows 上安装或打包失败。 +- 默认路径清晰，方便备份和检查。 + +最终配置： + +```env +STATE_PATH=./data/proxy-state.json +``` + +## 10. 状态文件隐私原则 + +### Q：状态文件会存真实 DashScope key 吗？ + +A：不会。 + +状态文件只存： + +```text +SHA-256(key) +``` + +也就是 `key_hash`。 + +真实 key 只存在 `.env` 里。 + +## 11. 状态文件结构 + +### Q：`modelState` 存什么？ + +A：每条记录对应一个 DashScope key 与 `MODEL_ID` 组合。 + +例如： + +```env +DASHSCOPE_API_KEYS=key1,key2 +MODEL_IDS=modelA,modelB,modelC +``` + +会生成： + +```text +2 * 3 = 6 条 modelState 记录 +``` + +字段详见： + +```text +docs/state-file.md +``` + +### Q：`runtimeState` 存什么？ + +A：只存调度游标，不存 key、不存 prompt、不存错误详情。 + +当前记录类型： + +```text +key_cursor +model_cursor: +``` + +含义： + +- `key_cursor`：当前优先使用 `.env` 里第几个 `DASHSCOPE_API_KEYS` 条目。 +- `model_cursor:`：某个 key 下当前优先使用 `MODEL_IDS` 里的第几个模型。 + +## 12. 游标策略共识 + +### Q：为什么 `model_cursor` 不直接存 modelId？ + +A：当前版本先按顺序来，使用游标绑定 `.env` 中的配置顺序。 + +共识： + +1. 当前版本保持 `key_cursor` 和 `model_cursor:`。 +2. 它们都依赖 `.env` 中 `DASHSCOPE_API_KEYS` 和 `MODEL_IDS` 的顺序。 +3. 不要随便调整这两个列表的顺序。 +4. 如果后续调整顺序，建议同时删除或重置 `data/proxy-state.json`。 +5. 后续如果需要更稳，再升级为 `current_key_hash` 和 `current_model_id:`。 + +## 13. Pi CLI 接入 + +### Q：怎么让 Pi CLI 使用本地代理？ + +A：在 `~/.pi/agent/models.json` 增加本地 provider。 + +Provider 名称： + +```text +local-dashscope-proxy +``` + +Base URL： + +```text +http://127.0.0.1:3300 +``` + +API 类型： + +```text +anthropic-messages +``` + +代理 key： + +```text +sk-001 +``` + +同时在 `~/.pi/agent/settings.json` 设置默认 provider： + +```text +defaultProvider = local-dashscope-proxy +defaultModel = qwen3.7-max +``` + +使用方式： + +```bash +pi +``` + +或： + +```bash +pi --provider local-dashscope-proxy --model qwen3.7-max +``` + +## 14. 清理旧 Pi Provider + +### Q：为什么删除 `dashscope` provider 下的两个模型？ + +A：用户希望 Pi CLI 里不再出现旧的直接 DashScope provider： + +```text +qwen3.7-max [dashscope] +qwen3.7-max[1M] [dashscope] +``` + +已删除 `~/.pi/agent/models.json` 中的 `dashscope` provider，保留本地代理 provider。 + +## 15. 当前模型列表 + +当前 `MODEL_IDS` 共 14 个： + +```text +qwen3.7-max-2026-06-08 +qwen3.7-plus-2026-05-26 +qwen3.7-plus +qwen3.7-max-preview +qwen3.7-max-2026-05-17 +qwen3.7-max +qwen3.7-max-2026-05-20 +qwen3.6-27b +qwen3.6-max-preview +qwen3.6-flash +qwen3.6-35b-a3b +qwen3.6-flash-2026-04-16 +qwen3.6-plus +qwen3.6-plus-2026-04-02 +``` + +数据库已同步初始化这些模型状态。 + +## 16. 请求日志 + +### Q：Hono 是否打印请求日志？ + +A：已增加统一请求日志中间件。 + +日志示例： + +```text +[request] GET /health status=200 duration=2ms bytes=- proxyKey=- model=- attempts=- ua="curl/8.7.1" +``` + +成功代理 `/v1/messages` 时会打印： + +```text +proxyKey= +model=<实际使用的模型 ID> +attempts=<本次请求尝试次数> +``` + +不会打印： + +```text +Authorization +x-api-key +DASHSCOPE_API_KEYS +请求 body +messages / prompt +``` + +## 17. 当前文档 + +使用说明： + +```text +docs/usage.md +``` + +状态文件说明： + +```text +docs/state-file.md +``` + +Q/A 与方案决策记录： + +```text +docs/qa-and-decisions.md +``` + +## 18. OpenAI Chat Completions 入口 + +### Q：为什么又增加 OpenAI 协议入口？ + +A：Pi CLI 的官方 DeepSeek provider 使用 `openai-completions`，说明 Pi 对 OpenAI Chat Completions 兼容链路支持成熟。为了让本地代理也能按同一类方式接入 Qwen，新增： + +```text +POST /v1/chat/completions +GET /v1/models +``` + +上游默认使用： + +```text +OPENAI_UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 +``` + +当前保留原有 Anthropic Messages 入口： + +```text +POST /v1/messages +``` + +两条链路共用同一套 DashScope key 池、模型池、冷却状态和免费额度耗尽切换逻辑。 + +服务端是反代转发，不需要安装 `openai` SDK；`openai` SDK 只适合客户端调用代理时使用。 diff --git a/docs/state-file.md b/docs/state-file.md new file mode 100644 index 0000000..2c12fd9 --- /dev/null +++ b/docs/state-file.md @@ -0,0 +1,116 @@ +# 状态文件说明 + +本服务使用 JSON 文件保存模型冷却状态。这样服务重启后，已经进入冷却期的 DashScope key 与 `MODEL_ID` 组合不会丢失。 + +默认状态文件路径： + +```env +STATE_PATH=./data/proxy-state.json +``` + +状态文件不会保存明文 DashScope API Key。代码会对每个 key 计算 SHA-256，并把结果写入 `modelState`。 + +## 文件结构 + +状态文件是一个版本化 JSON 对象： + +```json +{ + "version": 1, + "modelState": {}, + "runtimeState": {} +} +``` + +## modelState + +`modelState` 保存每个 DashScope key 与 `MODEL_ID` 组合的状态。 + +例如配置： + +```env +DASHSCOPE_API_KEYS=key1,key2 +MODEL_IDS=modelA,modelB,modelC +``` + +那么 `modelState` 会有 `2 * 3 = 6` 条记录。 + +字段说明： + +| 字段 | 说明 | +| --- | --- | +| `keyHash` | DashScope API Key 的 SHA-256 hash。用于区分不同 key，但不保存明文 key。 | +| `modelId` | 模型 ID，来自 `MODEL_IDS`。 | +| `cooldownUntil` | 冷却截止时间，Unix 毫秒时间戳。`0` 表示没有冷却。如果这个值大于当前时间，该 key+model 组合不可用。 | +| `failureCount` | 该 key+model 组合触发免费额度耗尽 403 的次数。 | +| `lastError` | 最近一次免费额度耗尽时，上游返回的错误信息。 | +| `lastUsedAt` | 最近一次成功使用该 key+model 组合的时间，Unix 毫秒时间戳。 | +| `updatedAt` | 最近一次状态更新时间，Unix 毫秒时间戳。 | + +## runtimeState + +`runtimeState` 保存调度游标，避免服务每次重启后都从第一个 key、第一条 model 开始打。 + +当前会使用的记录： + +| `name` | 说明 | +| --- | --- | +| `key_cursor` | 当前优先使用的 DashScope API Key 下标。 | +| `model_cursor:` | 某个 key 当前优先使用的模型下标。 | + +每条记录包含： + +| 字段 | 说明 | +| --- | --- | +| `value` | 状态值。当前主要保存数字游标，但统一用文本存储。 | +| `updatedAt` | 最近一次更新时间，Unix 毫秒时间戳。 | + +## 写入策略 + +状态变更会同步写入文件。写入时先写临时文件，再用 `rename` 替换正式状态文件，降低进程中断时写坏文件的风险。 + +如果启动时发现状态文件不是有效 JSON，或结构不符合当前版本，服务会把原文件重命名为： + +```text +proxy-state.json.bak. +``` + +然后创建新的空状态。 + +## 调度规则 + +当前实现是“key 优先”的切换策略： + +1. 先使用 `key_cursor` 指向的 key。 +2. 在这个 key 内部，从该 key 的 `model_cursor` 指向的模型开始尝试。 +3. 如果某个模型返回免费额度耗尽 403，只冷却这个 `keyHash + modelId` 组合。 +4. 继续尝试同一个 key 下的下一个可用模型。 +5. 只有当前 key 下所有模型都不可用时，才切换到下一个 key。 +6. 如果所有 key+model 组合都不可用，服务返回 `503`。 + +## 冷却规则 + +默认冷却时间是 1 个月： + +```env +MODEL_COOLDOWN_SECONDS=2592000 +``` + +当某个 key+model 组合进入冷却： + +```text +cooldownUntil = 当前时间 + MODEL_COOLDOWN_SECONDS +failureCount = failureCount + 1 +lastError = 上游错误信息 +updatedAt = 当前时间 +``` + +当某个 key+model 组合成功返回： + +```text +lastUsedAt = 当前时间 +lastError = null +updatedAt = 当前时间 +``` + +成功返回不会清零 `failureCount`，这个字段保留历史失败次数。 diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..c4422e4 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,338 @@ +# 使用说明 + +这个项目是一个本地 DashScope Anthropic 兼容反代服务。客户端只需要配置一个本地 URL 和一个代理 key，真实 DashScope key 和模型池都放在服务端。 + +## 1. 安装依赖 + +```bash +pnpm install +``` + +运行时依赖不包含原生编译模块，Windows 上首次安装不需要额外安装 C++ 编译环境。 + +## 2. 配置 .env + +复制配置模板： + +```bash +cp .env.example .env +``` + +核心配置： + +```env +PORT=3300 + +PROXY_API_KEY=sk-001 + +DASHSCOPE_API_KEYS=sk-key-1,sk-key-2,sk-key-3 +UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/apps/anthropic +OPENAI_UPSTREAM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +MODEL_IDS=qwen3.7-max,qwen-plus,qwen-max +MODEL_COOLDOWN_SECONDS=2592000 + +STATE_PATH=./data/proxy-state.json +UPSTREAM_AUTH_MODE=authorization +CORS_ORIGIN=* +``` + +说明： + +| 配置 | 说明 | +| --- | --- | +| `PORT` | 本地服务端口。你当前给 Pi CLI 用的是 `3300`。 | +| `PROXY_API_KEY` | 客户端调用本地代理时使用的 key。 | +| `DASHSCOPE_API_KEYS` | 多个真实 DashScope API Key，英文逗号分隔。 | +| `UPSTREAM_BASE_URL` | DashScope Anthropic 兼容接口上游地址。 | +| `OPENAI_UPSTREAM_BASE_URL` | DashScope OpenAI Chat Completions 兼容接口上游地址。 | +| `MODEL_IDS` | 所有 key 共用的模型 ID 列表，英文逗号分隔。 | +| `MODEL_COOLDOWN_SECONDS` | 触发免费额度耗尽后冷却多久。当前默认 `2592000` 秒，也就是 30 天。 | +| `STATE_PATH` | JSON 状态文件路径。用于保存冷却状态和调度游标。 | + +## 3. 初始化状态文件 + +启动服务时会自动创建状态文件。 + +也可以手动初始化： + +```bash +pnpm build +node dist/index.js +``` + +看到服务启动成功后按 `Ctrl+C` 停止即可。状态文件默认会生成在： + +```text +data/proxy-state.json +``` + +状态文件结构见： + +```text +docs/state-file.md +``` + +## 4. 启动服务 + +开发模式： + +```bash +pnpm dev +``` + +生产模式： + +```bash +pnpm build +pnpm start +``` + +启动成功后会看到类似日志： + +```text +[proxy] listening on http://localhost:3300 +[proxy] upstream base: https://dashscope.aliyuncs.com/apps/anthropic +[proxy] openai upstream base: https://dashscope.aliyuncs.com/compatible-mode/v1 +[proxy] api keys loaded: 3 +[proxy] models per key: 3 +[proxy] applied model ids: +[proxy] - qwen3.7-max +[proxy] - qwen-plus +[proxy] - qwen-max +[proxy] reminder: 请在 DashScope/百炼控制台为以上模型开启“模型用完即停”。 +[proxy] state file: ./data/proxy-state.json +``` + +## 5. 客户端调用 + +本地代理地址： + +```text +http://localhost:3300 +``` + +Anthropic Messages 接口： + +```text +http://localhost:3300/v1/messages +``` + +请求头可以用二选一： + +```http +Authorization: Bearer sk-001 +``` + +或： + +```http +x-api-key: sk-001 +``` + +测试请求： + +```bash +curl http://localhost:3300/v1/messages \ + -H "Authorization: Bearer sk-001" \ + -H "Content-Type: application/json" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "client-model-will-be-overwritten", + "max_tokens": 256, + "messages": [ + { + "role": "user", + "content": "你好" + } + ] + }' +``` + +请求体里的 `model` 会被代理覆盖成当前可用模型。 + +OpenAI Chat Completions 接口： + +```text +http://localhost:3300/v1/chat/completions +``` + +OpenAI-compatible base URL： + +```text +http://localhost:3300/v1 +``` + +测试请求： + +```bash +curl http://localhost:3300/v1/chat/completions \ + -H "Authorization: Bearer sk-001" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "client-model-will-be-overwritten", + "messages": [ + { + "role": "user", + "content": "你好" + } + ], + "stream": true + }' +``` + +查看 OpenAI 风格模型列表： + +```bash +curl http://localhost:3300/v1/models \ + -H "Authorization: Bearer sk-001" +``` + +本服务端只是反代转发，使用 Node 原生 `fetch` 即可，不需要安装 `openai` SDK。`openai` SDK 是客户端接入时才需要的。 + +## 6. Pi CLI 接入 + +Pi CLI 的 provider 已配置为： + +```text +local-dashscope-proxy +``` + +指向： + +```text +http://127.0.0.1:3300 +``` + +默认使用： + +```text +provider: local-dashscope-proxy +model: qwen3.7-max +``` + +使用前先启动代理服务： + +```bash +pnpm dev +``` + +然后运行： + +```bash +pi +``` + +也可以显式指定： + +```bash +pi --provider local-dashscope-proxy --model qwen3.7-max +``` + +查看 Pi 是否识别 provider： + +```bash +pi --list-models local --offline +``` + +如果想让 Pi CLI 按 OpenAI Chat Completions 协议接入，可以在 `~/.pi/agent/models.json` 增加类似 provider： + +```json +{ + "providers": { + "local-dashscope-openai-proxy": { + "baseUrl": "http://127.0.0.1:3300/v1", + "api": "openai-completions", + "apiKey": "sk-001", + "compat": { + "supportsDeveloperRole": false, + "supportsReasoningEffort": true, + "maxTokensField": "max_tokens", + "thinkingFormat": "qwen" + }, + "models": [ + { + "id": "qwen3.7-max", + "name": "Qwen 3.7 Max (Local OpenAI Proxy)", + "reasoning": true, + "input": ["text", "image"], + "contextWindow": 131072, + "maxTokens": 32768, + "thinkingLevelMap": { + "off": null, + "minimal": "low", + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "high" + }, + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + } + } + ] + } + } +} +``` + +## 7. 查看服务状态 + +健康检查： + +```bash +curl http://localhost:3300/health +``` + +模型池状态： + +```bash +curl http://localhost:3300/models/status \ + -H "Authorization: Bearer sk-001" +``` + +返回中重要字段： + +| 字段 | 说明 | +| --- | --- | +| `totalKeys` | 当前加载的 DashScope key 数量。 | +| `modelsPerKey` | 每个 key 下的模型数量。 | +| `totalSlots` | key+model 总组合数。 | +| `availableSlots` | 当前可用的 key+model 组合数。 | +| `models[].keyHash` | key hash 前缀，用于区分 key，不是明文 key。 | +| `models[].id` | 模型 ID。 | +| `models[].available` | 当前组合是否可用。 | +| `models[].cooldownUntil` | 冷却截止时间。 | +| `models[].failureCount` | 累计触发免费额度耗尽次数。 | + +## 8. 切换规则 + +服务按这个规则自动切换： + +1. 优先使用当前 key。 +2. 当前 key 下某个模型触发免费额度耗尽 403，只冷却这个 key+model。 +3. 继续尝试当前 key 的下一个模型。 +4. 当前 key 下所有模型都冷却后，才切换到下一个 key。 +5. 所有 key+model 都冷却时，返回 `503`。 + +## 9. 常见操作 + +重新生成状态文件： + +```bash +rm data/proxy-state.json +pnpm dev +``` + +只想清空冷却状态，也可以删除状态文件后重新启动。注意这会丢失所有历史冷却和失败计数。 + +查看状态文件： + +```text +data/proxy-state.json +``` + +状态文件和自动备份文件不需要提交。 diff --git a/package.json b/package.json new file mode 100644 index 0000000..4c2f443 --- /dev/null +++ b/package.json @@ -0,0 +1,28 @@ +{ + "name": "dashscope-model-proxy", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx watch src/index.ts", + "start": "node dist/index.js", + "build": "tsc -p tsconfig.json", + "type-check": "tsc -p tsconfig.json --noEmit", + "release:prepare": "pnpm build && node scripts/prepare-release.mjs" + }, + "dependencies": { + "@hono/node-server": "^1.14.4", + "dotenv": "^16.5.0", + "hono": "^4.7.11" + }, + "devDependencies": { + "@types/node": "^22.15.30", + "tsx": "^4.19.4", + "typescript": "^5.8.3" + }, + "engines": { + "node": ">=20", + "pnpm": ">=9" + }, + "packageManager": "pnpm@10.10.0" +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..3888f7d --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,361 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@hono/node-server': + specifier: ^1.14.4 + version: 1.19.14(hono@4.12.25) + dotenv: + specifier: ^16.5.0 + version: 16.6.1 + hono: + specifier: ^4.7.11 + version: 4.12.25 + devDependencies: + '@types/node': + specifier: ^22.15.30 + version: 22.19.20 + tsx: + specifier: ^4.19.4 + version: 4.22.4 + typescript: + specifier: ^5.8.3 + version: 5.9.3 + +packages: + + '@esbuild/aix-ppc64@0.28.0': + resolution: {integrity: sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.0': + resolution: {integrity: sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.0': + resolution: {integrity: sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.0': + resolution: {integrity: sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.0': + resolution: {integrity: sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.0': + resolution: {integrity: sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.0': + resolution: {integrity: sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.0': + resolution: {integrity: sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.0': + resolution: {integrity: sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.0': + resolution: {integrity: sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.0': + resolution: {integrity: sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.0': + resolution: {integrity: sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.0': + resolution: {integrity: sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.0': + resolution: {integrity: sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.0': + resolution: {integrity: sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.0': + resolution: {integrity: sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.0': + resolution: {integrity: sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.0': + resolution: {integrity: sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.0': + resolution: {integrity: sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.0': + resolution: {integrity: sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.0': + resolution: {integrity: sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.0': + resolution: {integrity: sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.0': + resolution: {integrity: sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.0': + resolution: {integrity: sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.0': + resolution: {integrity: sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.0': + resolution: {integrity: sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@hono/node-server@1.19.14': + resolution: {integrity: sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==} + engines: {node: '>=18.14.1'} + peerDependencies: + hono: ^4 + + '@types/node@22.19.20': + resolution: {integrity: sha512-6tELRwSDYWW9EdZhbeZmYGZ1/7Djkt+Ah3/ScEYT9cDord7UJzasR/4D3VONg9tQI5CDp+/CZC1AXj2pCFOvpw==} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + esbuild@0.28.0: + resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} + engines: {node: '>=18'} + hasBin: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + hono@4.12.25: + resolution: {integrity: sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==} + engines: {node: '>=16.9.0'} + + tsx@4.22.4: + resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + +snapshots: + + '@esbuild/aix-ppc64@0.28.0': + optional: true + + '@esbuild/android-arm64@0.28.0': + optional: true + + '@esbuild/android-arm@0.28.0': + optional: true + + '@esbuild/android-x64@0.28.0': + optional: true + + '@esbuild/darwin-arm64@0.28.0': + optional: true + + '@esbuild/darwin-x64@0.28.0': + optional: true + + '@esbuild/freebsd-arm64@0.28.0': + optional: true + + '@esbuild/freebsd-x64@0.28.0': + optional: true + + '@esbuild/linux-arm64@0.28.0': + optional: true + + '@esbuild/linux-arm@0.28.0': + optional: true + + '@esbuild/linux-ia32@0.28.0': + optional: true + + '@esbuild/linux-loong64@0.28.0': + optional: true + + '@esbuild/linux-mips64el@0.28.0': + optional: true + + '@esbuild/linux-ppc64@0.28.0': + optional: true + + '@esbuild/linux-riscv64@0.28.0': + optional: true + + '@esbuild/linux-s390x@0.28.0': + optional: true + + '@esbuild/linux-x64@0.28.0': + optional: true + + '@esbuild/netbsd-arm64@0.28.0': + optional: true + + '@esbuild/netbsd-x64@0.28.0': + optional: true + + '@esbuild/openbsd-arm64@0.28.0': + optional: true + + '@esbuild/openbsd-x64@0.28.0': + optional: true + + '@esbuild/openharmony-arm64@0.28.0': + optional: true + + '@esbuild/sunos-x64@0.28.0': + optional: true + + '@esbuild/win32-arm64@0.28.0': + optional: true + + '@esbuild/win32-ia32@0.28.0': + optional: true + + '@esbuild/win32-x64@0.28.0': + optional: true + + '@hono/node-server@1.19.14(hono@4.12.25)': + dependencies: + hono: 4.12.25 + + '@types/node@22.19.20': + dependencies: + undici-types: 6.21.0 + + dotenv@16.6.1: {} + + esbuild@0.28.0: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.0 + '@esbuild/android-arm': 0.28.0 + '@esbuild/android-arm64': 0.28.0 + '@esbuild/android-x64': 0.28.0 + '@esbuild/darwin-arm64': 0.28.0 + '@esbuild/darwin-x64': 0.28.0 + '@esbuild/freebsd-arm64': 0.28.0 + '@esbuild/freebsd-x64': 0.28.0 + '@esbuild/linux-arm': 0.28.0 + '@esbuild/linux-arm64': 0.28.0 + '@esbuild/linux-ia32': 0.28.0 + '@esbuild/linux-loong64': 0.28.0 + '@esbuild/linux-mips64el': 0.28.0 + '@esbuild/linux-ppc64': 0.28.0 + '@esbuild/linux-riscv64': 0.28.0 + '@esbuild/linux-s390x': 0.28.0 + '@esbuild/linux-x64': 0.28.0 + '@esbuild/netbsd-arm64': 0.28.0 + '@esbuild/netbsd-x64': 0.28.0 + '@esbuild/openbsd-arm64': 0.28.0 + '@esbuild/openbsd-x64': 0.28.0 + '@esbuild/openharmony-arm64': 0.28.0 + '@esbuild/sunos-x64': 0.28.0 + '@esbuild/win32-arm64': 0.28.0 + '@esbuild/win32-ia32': 0.28.0 + '@esbuild/win32-x64': 0.28.0 + + fsevents@2.3.3: + optional: true + + hono@4.12.25: {} + + tsx@4.22.4: + dependencies: + esbuild: 0.28.0 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..7d40629 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1 @@ +onlyBuiltDependencies: [] diff --git a/scripts/prepare-release.mjs b/scripts/prepare-release.mjs new file mode 100644 index 0000000..140dc92 --- /dev/null +++ b/scripts/prepare-release.mjs @@ -0,0 +1,163 @@ +import { chmodSync, cpSync, existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const scriptDir = dirname(fileURLToPath(import.meta.url)) +const projectRoot = resolve(scriptDir, '..') +const releaseDir = join(projectRoot, 'release') + +rmSync(releaseDir, { recursive: true, force: true }) +mkdirSync(releaseDir, { recursive: true }) +mkdirSync(join(releaseDir, 'data'), { recursive: true }) +mkdirSync(join(releaseDir, 'docs'), { recursive: true }) + +copyRequiredPath('dist') +copyRequiredPath('package.json') +copyRequiredPath('pnpm-lock.yaml') +copyRequiredPath('.env.example') +copyRequiredPath('LLM.md') +copyRequiredPath('docs/usage.md') +copyRequiredPath('docs/state-file.md') + +writeFileSync(join(releaseDir, 'data', '.gitkeep'), '\n') +writeFileSync(join(releaseDir, '.npmrc'), 'production=true\n') +writeFileSync(join(releaseDir, 'start.sh'), buildStartScript()) +chmodSync(join(releaseDir, 'start.sh'), 0o755) +writeFileSync(join(releaseDir, 'start.cmd'), toWindowsLineEndings(buildWindowsStartScript())) +writeFileSync(join(releaseDir, '使用方式.txt'), buildReadme()) + +console.log(`[release] prepared at ${releaseDir}`) + +function copyRequiredPath(relativePath) { + const source = join(projectRoot, relativePath) + const target = join(releaseDir, relativePath) + + if (!existsSync(source)) { + throw new Error(`Missing required path: ${relativePath}`) + } + + cpSync(source, target, { + recursive: true, + force: true, + dereference: true, + filter: (currentSource) => { + const normalized = currentSource.replace(/\\/g, '/') + if (normalized.endsWith('/data/proxy-state.json')) return false + if (normalized.includes('/data/proxy-state.json.')) return false + return true + }, + }) +} + +function buildStartScript() { + return `#!/bin/sh +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +if [ ! -f ".env" ]; then + cp .env.example .env + echo "已生成 .env，请先填写 PROXY_API_KEY、DASHSCOPE_API_KEYS、MODEL_IDS 后再重新运行 ./start.sh" + exit 1 +fi + +mkdir -p data + +if [ ! -d "node_modules" ]; then + if command -v pnpm >/dev/null 2>&1; then + pnpm install --prod --frozen-lockfile + elif command -v corepack >/dev/null 2>&1; then + corepack pnpm install --prod --frozen-lockfile + else + echo "未找到 pnpm 或 corepack，请先安装 Node.js 20+（含 corepack）或 pnpm" + exit 1 + fi +fi + +exec node dist/index.js +` +} + +function buildReadme() { + return `1. 确认机器已安装 Node.js 20 或更高版本。 +2. 首次运行： + - Windows：双击 start.cmd，或在 cmd/PowerShell 里执行 start.cmd + - macOS / Linux：执行 ./start.sh +3. 如果当前目录没有 .env，会自动从 .env.example 生成一份模板。 +4. 打开 .env，至少填写: + - PROXY_API_KEY + - DASHSCOPE_API_KEYS + - MODEL_IDS +5. 再次运行启动脚本： + - Windows：start.cmd + - macOS / Linux：./start.sh + +说明: +- 首次真正启动时会自动安装运行时依赖 +- 首次启动会自动创建新的 data/proxy-state.json +- 运行时依赖不包含原生编译模块，Windows 上不需要额外安装 C++ 编译环境 +` +} + +function buildWindowsStartScript() { + return `@echo off +setlocal +cd /d "%~dp0" + +if not exist ".env" ( + copy /Y ".env.example" ".env" >nul + echo Created .env from .env.example + echo Fill PROXY_API_KEY, DASHSCOPE_API_KEYS and MODEL_IDS + echo Then run start.cmd again + echo. + pause + exit /b 1 +) + +if not exist "data" mkdir data + +if not exist "node_modules" ( + where pnpm >nul 2>nul + if not errorlevel 1 ( + call pnpm install --prod --frozen-lockfile + if errorlevel 1 goto :install_failed + goto :run + ) + + where corepack >nul 2>nul + if not errorlevel 1 ( + call corepack pnpm install --prod --frozen-lockfile + if errorlevel 1 goto :install_failed + goto :run + ) + + echo pnpm or corepack was not found + echo Install Node.js 20+ or pnpm first + echo. + pause + exit /b 1 +) + +:run +call node dist\\index.js +set EXIT_CODE=%errorlevel% +if not "%EXIT_CODE%"=="0" ( + echo. + echo Service exited with code %EXIT_CODE% + pause +) +exit /b %EXIT_CODE% + +:install_failed +echo. +echo Dependency install failed +echo Check Node.js, pnpm and build tools +pause +exit /b 1 +` +} + +function toWindowsLineEndings(value) { + return value.replace(/\n/g, '\r\n') +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..4863be7 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,97 @@ +import 'dotenv/config' + +export type UpstreamAuthMode = 'authorization' | 'x-api-key' | 'both' + +export interface AppConfig { + port: number + proxyApiKey: string + dashscopeApiKeys: string[] + upstreamBaseUrl: string + openAIUpstreamBaseUrl: string + modelIds: string[] + cooldownMs: number + upstreamAuthMode: UpstreamAuthMode + corsOrigin: string | false + statePath: string +} + +function requireEnv(name: string): string { + const value = process.env[name]?.trim() + if (!value) { + throw new Error(`Missing required environment variable: ${name}`) + } + return value +} + +function parsePositiveNumber(name: string, fallback: number): number { + const raw = process.env[name]?.trim() + if (!raw) return fallback + + const value = Number(raw) + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${name} must be a positive number`) + } + return value +} + +function parseModelIds(): string[] { + const models = requireEnv('MODEL_IDS') + .split(',') + .map((model) => model.trim()) + .filter(Boolean) + + if (models.length === 0) { + throw new Error('MODEL_IDS must contain at least one model id') + } + + return [...new Set(models)] +} + +function parseDashscopeApiKeys(): string[] { + const raw = requireEnv('DASHSCOPE_API_KEYS') + const keys = raw + .split(',') + .map((key) => key.trim()) + .filter(Boolean) + + if (keys.length === 0) { + throw new Error('DASHSCOPE_API_KEYS must contain at least one key') + } + + return [...new Set(keys)] +} + +function parseUpstreamAuthMode(): UpstreamAuthMode { + const value = process.env.UPSTREAM_AUTH_MODE?.trim() || 'authorization' + + if (value === 'authorization' || value === 'x-api-key' || value === 'both') { + return value + } + + throw new Error('UPSTREAM_AUTH_MODE must be authorization, x-api-key, or both') +} + +function parseCorsOrigin(): string | false { + const value = process.env.CORS_ORIGIN?.trim() + if (!value) return '*' + if (value.toLowerCase() === 'false') return false + return value +} + +export function loadConfig(): AppConfig { + return { + port: parsePositiveNumber('PORT', 3000), + proxyApiKey: requireEnv('PROXY_API_KEY'), + dashscopeApiKeys: parseDashscopeApiKeys(), + upstreamBaseUrl: + process.env.UPSTREAM_BASE_URL?.trim() || 'https://dashscope.aliyuncs.com/apps/anthropic', + openAIUpstreamBaseUrl: + process.env.OPENAI_UPSTREAM_BASE_URL?.trim() || + 'https://dashscope.aliyuncs.com/compatible-mode/v1', + modelIds: parseModelIds(), + cooldownMs: parsePositiveNumber('MODEL_COOLDOWN_SECONDS', 2592000) * 1000, + upstreamAuthMode: parseUpstreamAuthMode(), + corsOrigin: parseCorsOrigin(), + statePath: process.env.STATE_PATH?.trim() || './data/proxy-state.json', + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..937e832 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,278 @@ +import { serve } from '@hono/node-server' +import { cors } from 'hono/cors' +import { Hono } from 'hono' +import type { Context, Next } from 'hono' +import { loadConfig } from './config.js' +import { ModelPool } from './model-pool.js' +import { StateStore } from './state-store.js' +import { jsonResponse, proxyAnthropicRequest, proxyOpenAIRequest } from './upstream.js' + +const config = loadConfig() +const stateStore = new StateStore(config.statePath) +const modelPool = new ModelPool( + config.dashscopeApiKeys, + config.modelIds, + config.cooldownMs, + stateStore, +) +const app = new Hono() + +if (config.corsOrigin !== false) { + app.use( + '*', + cors({ + origin: config.corsOrigin, + allowMethods: ['GET', 'POST', 'OPTIONS'], + allowHeaders: [ + 'authorization', + 'content-type', + 'x-api-key', + 'openai-organization', + 'openai-project', + 'anthropic-version', + 'anthropic-beta', + ], + exposeHeaders: ['x-proxy-key-hash', 'x-proxy-model', 'x-proxy-attempts'], + }), + ) +} + +app.use('*', requestLogger) + +app.get('/', (c) => { + return c.json({ + name: 'dashscope-model-proxy', + endpoints: ['/health', '/models/status', '/v1/models', '/v1/messages', '/v1/chat/completions'], + }) +}) + +app.get('/health', (c) => { + return c.json({ + ok: true, + totalKeys: modelPool.totalKeys(), + modelsPerKey: modelPool.totalModelsPerKey(), + totalSlots: modelPool.totalSlots(), + availableSlots: modelPool.availableCount(), + }) +}) + +app.use('/models/status', requireProxyKey) +app.get('/models/status', (c) => { + return c.json({ + totalKeys: modelPool.totalKeys(), + modelsPerKey: modelPool.totalModelsPerKey(), + totalSlots: modelPool.totalSlots(), + availableSlots: modelPool.availableCount(), + models: modelPool.snapshot(), + }) +}) + +app.use('/v1/*', requireProxyKey) +app.get('/v1/models', (c) => { + return c.json({ + object: 'list', + data: config.modelIds.map((modelId) => ({ + id: modelId, + object: 'model', + created: 0, + owned_by: 'dashscope-model-proxy', + })), + }) +}) + +app.post('/v1/chat/completions', async (c) => { + let body: unknown + + try { + body = await c.req.json() + } catch { + return c.json( + { + error: { + type: 'invalid_request_error', + message: 'Request body must be valid JSON.', + }, + }, + 400, + ) + } + + if (!isPlainObject(body)) { + return c.json( + { + error: { + type: 'invalid_request_error', + message: 'Request body must be a JSON object.', + }, + }, + 400, + ) + } + + const url = new URL(c.req.url) + const upstreamPath = url.pathname.replace(/^\/v1/, '') || url.pathname + + try { + return await proxyOpenAIRequest({ + request: c.req.raw, + body, + pathWithSearch: `${upstreamPath}${url.search}`, + config, + modelPool, + }) + } catch (error) { + console.error('[proxy] upstream request failed:', error) + return jsonResponse(502, { + error: { + type: 'upstream_error', + message: error instanceof Error ? error.message : 'Upstream request failed.', + }, + }) + } +}) + +app.post('/v1/*', async (c) => { + let body: unknown + + try { + body = await c.req.json() + } catch { + return c.json( + { + error: { + type: 'invalid_request_error', + message: 'Request body must be valid JSON.', + }, + }, + 400, + ) + } + + if (!isPlainObject(body)) { + return c.json( + { + error: { + type: 'invalid_request_error', + message: 'Request body must be a JSON object.', + }, + }, + 400, + ) + } + + const url = new URL(c.req.url) + + try { + return await proxyAnthropicRequest({ + request: c.req.raw, + body, + pathWithSearch: `${url.pathname}${url.search}`, + config, + modelPool, + }) + } catch (error) { + console.error('[proxy] upstream request failed:', error) + return jsonResponse(502, { + error: { + type: 'upstream_error', + message: error instanceof Error ? error.message : 'Upstream request failed.', + }, + }) + } +}) + +app.all('/v1/*', (c) => { + return c.json( + { + error: { + type: 'method_not_allowed', + message: 'Only POST requests are proxied under /v1/*.', + }, + }, + 405, + ) +}) + +serve( + { + fetch: app.fetch, + port: config.port, + }, + (info) => { + console.log(`[proxy] listening on http://localhost:${info.port}`) + console.log(`[proxy] upstream base: ${config.upstreamBaseUrl}`) + console.log(`[proxy] openai upstream base: ${config.openAIUpstreamBaseUrl}`) + console.log(`[proxy] api keys loaded: ${config.dashscopeApiKeys.length}`) + console.log(`[proxy] models per key: ${config.modelIds.length}`) + console.log('[proxy] applied model ids:') + for (const modelId of config.modelIds) { + console.log(`[proxy] - ${modelId}`) + } + console.log('[proxy] reminder: 请在 DashScope/百炼控制台为以上模型开启“模型用完即停”。') + console.log(`[proxy] state file: ${config.statePath}`) + }, +) + +async function requireProxyKey(c: Context, next: Next): Promise { + if (!isAuthorized(c.req.raw.headers, config.proxyApiKey)) { + return c.json( + { + error: { + type: 'authentication_error', + message: 'Invalid proxy API key.', + }, + }, + 401, + ) + } + + await next() +} + +async function requestLogger(c: Context, next: Next): Promise { + const startedAt = Date.now() + const url = new URL(c.req.url) + const contentLength = c.req.raw.headers.get('content-length') || '-' + const userAgent = c.req.raw.headers.get('user-agent') || '-' + + await next() + + const durationMs = Date.now() - startedAt + const status = c.res.status + const proxyKeyHash = c.res.headers.get('x-proxy-key-hash') || '-' + const proxyModel = c.res.headers.get('x-proxy-model') || '-' + const proxyAttempts = c.res.headers.get('x-proxy-attempts') || '-' + + console.log( + [ + '[request]', + `${c.req.method} ${url.pathname}${url.search}`, + `status=${status}`, + `duration=${durationMs}ms`, + `bytes=${contentLength}`, + `proxyKey=${proxyKeyHash}`, + `model=${proxyModel}`, + `attempts=${proxyAttempts}`, + `ua=${JSON.stringify(userAgent)}`, + ].join(' '), + ) +} + +function isAuthorized(headers: Headers, expectedKey: string): boolean { + const xApiKey = headers.get('x-api-key')?.trim() + if (xApiKey === expectedKey) return true + + const authorization = headers.get('authorization')?.trim() + if (!authorization) return false + + const bearerPrefix = 'Bearer ' + if (authorization.startsWith(bearerPrefix)) { + return authorization.slice(bearerPrefix.length).trim() === expectedKey + } + + return authorization === expectedKey +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/model-pool.ts b/src/model-pool.ts new file mode 100644 index 0000000..1eb7af3 --- /dev/null +++ b/src/model-pool.ts @@ -0,0 +1,256 @@ +import { createHash } from 'node:crypto' +import type { StateStore } from './state-store.js' + +export interface ModelCandidate { + apiKey: string + keyHash: string + keyLabel: string + keyIndex: number + modelId: string + modelIndex: number +} + +export interface ModelSnapshot { + keyHash: string + keyIndex: number + currentKey: boolean + currentModel: boolean + id: string + available: boolean + cooldownUntil: string | null + failureCount: number + lastError: string | null + lastUsedAt: string | null +} + +interface ApiKeySlot { + value: string + hash: string + label: string + index: number +} + +const KEY_CURSOR_NAME = 'key_cursor' + +export class ModelPool { + private readonly apiKeys: ApiKeySlot[] + + constructor( + apiKeys: string[], + private readonly modelIds: string[], + private readonly cooldownMs: number, + private readonly stateStore: StateStore, + ) { + this.apiKeys = apiKeys.map((value, index) => { + const hash = hashApiKey(value) + return { + value, + hash, + label: hash.slice(0, 12), + index, + } + }) + + this.stateStore.ensureModelStates( + this.apiKeys.map((key) => key.hash), + this.modelIds, + ) + } + + getAttemptModels(now = Date.now()): ModelCandidate[] { + const result: ModelCandidate[] = [] + const currentKeyIndex = this.normalizeKeyIndex(this.getKeyCursor()) + let firstAvailableKeyIndex: number | null = null + + for (let keyOffset = 0; keyOffset < this.apiKeys.length; keyOffset += 1) { + const keyIndex = (currentKeyIndex + keyOffset) % this.apiKeys.length + const key = this.apiKeys[keyIndex] + if (!key) continue + + const candidates = this.getAvailableModelsForKey(key, now) + if (candidates.length === 0) continue + + if (firstAvailableKeyIndex === null) { + firstAvailableKeyIndex = keyIndex + } + + result.push(...candidates) + } + + if (firstAvailableKeyIndex !== null && firstAvailableKeyIndex !== currentKeyIndex) { + this.setKeyCursor(firstAvailableKeyIndex, now) + } + + return result + } + + markSuccess(candidate: ModelCandidate, now = Date.now()): void { + this.stateStore.markSuccess(candidate.keyHash, candidate.modelId, now) + this.setKeyCursor(candidate.keyIndex, now) + this.setModelCursor(candidate.keyHash, candidate.modelIndex, now) + } + + markFreeTierExhausted(candidate: ModelCandidate, reason: string, now = Date.now()): void { + this.stateStore.markFreeTierExhausted( + candidate.keyHash, + candidate.modelId, + now + this.cooldownMs, + reason, + now, + ) + + const nextModelIndex = this.findNextAvailableModelIndex(candidate.keyHash, candidate.modelIndex, now) + if (nextModelIndex !== null) { + this.setModelCursor(candidate.keyHash, nextModelIndex, now) + return + } + + const nextKeyIndex = this.findNextAvailableKeyIndex(candidate.keyIndex, now) + if (nextKeyIndex !== null) { + this.setKeyCursor(nextKeyIndex, now) + } + } + + availableCount(now = Date.now()): number { + let count = 0 + + for (const key of this.apiKeys) { + for (const modelId of this.modelIds) { + const state = this.stateStore.getModelState(key.hash, modelId) + if (state.cooldownUntil <= now) count += 1 + } + } + + return count + } + + totalKeys(): number { + return this.apiKeys.length + } + + totalModelsPerKey(): number { + return this.modelIds.length + } + + totalSlots(): number { + return this.apiKeys.length * this.modelIds.length + } + + snapshot(now = Date.now()): ModelSnapshot[] { + const currentKeyIndex = this.normalizeKeyIndex(this.getKeyCursor()) + + return this.apiKeys.flatMap((key) => { + const currentModelIndex = this.normalizeModelIndex(this.getModelCursor(key.hash)) + + return this.modelIds.map((modelId, modelIndex) => { + const state = this.stateStore.getModelState(key.hash, modelId) + + return { + keyHash: key.label, + keyIndex: key.index, + currentKey: key.index === currentKeyIndex, + currentModel: key.index === currentKeyIndex && modelIndex === currentModelIndex, + id: modelId, + available: state.cooldownUntil <= now, + cooldownUntil: + state.cooldownUntil > now ? new Date(state.cooldownUntil).toISOString() : null, + failureCount: state.failureCount, + lastError: state.lastError, + lastUsedAt: state.lastUsedAt ? new Date(state.lastUsedAt).toISOString() : null, + } + }) + }) + } + + private getAvailableModelsForKey(key: ApiKeySlot, now: number): ModelCandidate[] { + const result: ModelCandidate[] = [] + const currentModelIndex = this.normalizeModelIndex(this.getModelCursor(key.hash)) + + for (let modelOffset = 0; modelOffset < this.modelIds.length; modelOffset += 1) { + const modelIndex = (currentModelIndex + modelOffset) % this.modelIds.length + const modelId = this.modelIds[modelIndex] + if (!modelId) continue + + const state = this.stateStore.getModelState(key.hash, modelId) + if (state.cooldownUntil > now) continue + + result.push({ + apiKey: key.value, + keyHash: key.hash, + keyLabel: key.label, + keyIndex: key.index, + modelId, + modelIndex, + }) + } + + return result + } + + private findNextAvailableModelIndex( + keyHash: string, + afterModelIndex: number, + now: number, + ): number | null { + for (let offset = 1; offset <= this.modelIds.length; offset += 1) { + const modelIndex = (afterModelIndex + offset) % this.modelIds.length + const modelId = this.modelIds[modelIndex] + if (!modelId) continue + + const state = this.stateStore.getModelState(keyHash, modelId) + if (state.cooldownUntil <= now) return modelIndex + } + + return null + } + + private findNextAvailableKeyIndex(afterKeyIndex: number, now: number): number | null { + for (let offset = 1; offset <= this.apiKeys.length; offset += 1) { + const keyIndex = (afterKeyIndex + offset) % this.apiKeys.length + const key = this.apiKeys[keyIndex] + if (!key) continue + + if (this.getAvailableModelsForKey(key, now).length > 0) return keyIndex + } + + return null + } + + private getKeyCursor(): number { + return this.stateStore.getRuntimeNumber(KEY_CURSOR_NAME, 0) + } + + private setKeyCursor(keyIndex: number, now: number): void { + this.stateStore.setRuntimeNumber(KEY_CURSOR_NAME, this.normalizeKeyIndex(keyIndex), now) + } + + private getModelCursor(keyHash: string): number { + return this.stateStore.getRuntimeNumber(modelCursorName(keyHash), 0) + } + + private setModelCursor(keyHash: string, modelIndex: number, now: number): void { + this.stateStore.setRuntimeNumber( + modelCursorName(keyHash), + this.normalizeModelIndex(modelIndex), + now, + ) + } + + private normalizeKeyIndex(index: number): number { + if (this.apiKeys.length === 0) return 0 + return index % this.apiKeys.length + } + + private normalizeModelIndex(index: number): number { + if (this.modelIds.length === 0) return 0 + return index % this.modelIds.length + } +} + +function hashApiKey(apiKey: string): string { + return createHash('sha256').update(apiKey).digest('hex') +} + +function modelCursorName(keyHash: string): string { + return `model_cursor:${keyHash}` +} diff --git a/src/state-store.ts b/src/state-store.ts new file mode 100644 index 0000000..700bed8 --- /dev/null +++ b/src/state-store.ts @@ -0,0 +1,326 @@ +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + fsyncSync, + writeFileSync, +} from 'node:fs' +import { dirname, resolve } from 'node:path' + +interface RuntimeStateEntry { + value: string + updatedAt: number +} + +interface StateFile { + version: 1 + modelState: Record + runtimeState: Record +} + +export interface PersistedModelState { + keyHash: string + modelId: string + cooldownUntil: number + failureCount: number + lastError: string | null + lastUsedAt: number | null + updatedAt: number +} + +export class StateStore { + private readonly statePath: string + private state: StateFile + + constructor(statePath: string) { + this.statePath = resolve(statePath) + mkdirSync(dirname(this.statePath), { recursive: true }) + this.state = this.load() + } + + ensureModelStates(keyHashes: string[], modelIds: string[], now = Date.now()): void { + let changed = false + + for (const keyHash of keyHashes) { + for (const modelId of modelIds) { + if (this.ensureModelStateEntry(keyHash, modelId, now)) { + changed = true + } + } + } + + if (changed) this.persist() + } + + getModelState(keyHash: string, modelId: string): PersistedModelState { + const entry = this.ensureModelStateEntry(keyHash, modelId, Date.now()) + + if (entry) { + this.persist() + return { ...entry } + } + + return { ...this.state.modelState[modelStateKey(keyHash, modelId)] } + } + + markSuccess(keyHash: string, modelId: string, now = Date.now()): void { + const entry = this.getOrCreateModelStateEntry(keyHash, modelId, now) + + entry.lastError = null + entry.lastUsedAt = now + entry.updatedAt = now + + this.persist() + } + + markFreeTierExhausted( + keyHash: string, + modelId: string, + cooldownUntil: number, + reason: string, + now = Date.now(), + ): void { + const entry = this.getOrCreateModelStateEntry(keyHash, modelId, now) + + entry.cooldownUntil = cooldownUntil + entry.failureCount += 1 + entry.lastError = reason + entry.updatedAt = now + + this.persist() + } + + getRuntimeNumber(name: string, fallback: number): number { + const entry = this.state.runtimeState[name] + if (!entry) return fallback + + const value = Number(entry.value) + return Number.isInteger(value) && value >= 0 ? value : fallback + } + + setRuntimeNumber(name: string, value: number, now = Date.now()): void { + this.state.runtimeState[name] = { + value: String(value), + updatedAt: now, + } + + this.persist() + } + + close(): void { + // State changes are persisted synchronously when they happen. + } + + private load(): StateFile { + if (!existsSync(this.statePath)) return emptyStateFile() + + try { + const parsed = JSON.parse(readFileSync(this.statePath, 'utf8')) as unknown + return parseStateFile(parsed) + } catch (error) { + this.backupCorruptStateFile(error) + return emptyStateFile() + } + } + + private persist(): void { + const content = `${JSON.stringify(this.state, null, 2)}\n` + const tempPath = `${this.statePath}.${process.pid}.${Date.now()}.tmp` + const fd = openSync(tempPath, 'w') + + try { + writeFileSync(fd, content, 'utf8') + fsyncSync(fd) + } finally { + closeSync(fd) + } + + renameSync(tempPath, this.statePath) + } + + private ensureModelStateEntry( + keyHash: string, + modelId: string, + now: number, + ): PersistedModelState | null { + const key = modelStateKey(keyHash, modelId) + const existing = this.state.modelState[key] + + if (existing) return null + + const entry = createModelState(keyHash, modelId, now) + this.state.modelState[key] = entry + return entry + } + + private getOrCreateModelStateEntry( + keyHash: string, + modelId: string, + now: number, + ): PersistedModelState { + const key = modelStateKey(keyHash, modelId) + const existing = this.state.modelState[key] + + if (existing) return existing + + const entry = createModelState(keyHash, modelId, now) + this.state.modelState[key] = entry + return entry + } + + private backupCorruptStateFile(error: unknown): void { + const backupPath = `${this.statePath}.bak.${new Date().toISOString().replace(/[:.]/g, '-')}` + + try { + renameSync(this.statePath, backupPath) + console.warn( + `[state] invalid state file moved to ${backupPath}: ${formatErrorMessage(error)}`, + ) + } catch (backupError) { + console.warn( + `[state] invalid state file could not be backed up: ${formatErrorMessage(backupError)}`, + ) + } + } +} + +function emptyStateFile(): StateFile { + return { + version: 1, + modelState: {}, + runtimeState: {}, + } +} + +function createModelState(keyHash: string, modelId: string, now: number): PersistedModelState { + return { + keyHash, + modelId, + cooldownUntil: 0, + failureCount: 0, + lastError: null, + lastUsedAt: null, + updatedAt: now, + } +} + +function parseStateFile(value: unknown): StateFile { + if (!isRecord(value)) { + throw new Error('state file must be a JSON object') + } + + if (value.version !== 1) { + throw new Error('unsupported state file version') + } + + if (!isRecord(value.modelState)) { + throw new Error('state file modelState must be an object') + } + + if (!isRecord(value.runtimeState)) { + throw new Error('state file runtimeState must be an object') + } + + const modelState: Record = {} + for (const entry of Object.values(value.modelState)) { + const parsedEntry = parseModelStateEntry(entry) + modelState[modelStateKey(parsedEntry.keyHash, parsedEntry.modelId)] = parsedEntry + } + + const runtimeState: Record = {} + for (const [name, entry] of Object.entries(value.runtimeState)) { + runtimeState[name] = parseRuntimeStateEntry(entry) + } + + return { + version: 1, + modelState, + runtimeState, + } +} + +function parseModelStateEntry(value: unknown): PersistedModelState { + if (!isRecord(value)) { + throw new Error('modelState entries must be objects') + } + + if (!isNonEmptyString(value.keyHash)) { + throw new Error('modelState entry keyHash must be a non-empty string') + } + + if (!isNonEmptyString(value.modelId)) { + throw new Error('modelState entry modelId must be a non-empty string') + } + + if (!isNonNegativeInteger(value.cooldownUntil)) { + throw new Error('modelState entry cooldownUntil must be a non-negative integer') + } + + if (!isNonNegativeInteger(value.failureCount)) { + throw new Error('modelState entry failureCount must be a non-negative integer') + } + + if (value.lastError !== null && typeof value.lastError !== 'string') { + throw new Error('modelState entry lastError must be a string or null') + } + + if (value.lastUsedAt !== null && !isNonNegativeInteger(value.lastUsedAt)) { + throw new Error('modelState entry lastUsedAt must be a non-negative integer or null') + } + + if (!isNonNegativeInteger(value.updatedAt)) { + throw new Error('modelState entry updatedAt must be a non-negative integer') + } + + return { + keyHash: value.keyHash, + modelId: value.modelId, + cooldownUntil: value.cooldownUntil, + failureCount: value.failureCount, + lastError: value.lastError, + lastUsedAt: value.lastUsedAt, + updatedAt: value.updatedAt, + } +} + +function parseRuntimeStateEntry(value: unknown): RuntimeStateEntry { + if (!isRecord(value)) { + throw new Error('runtimeState entries must be objects') + } + + if (typeof value.value !== 'string') { + throw new Error('runtimeState entry value must be a string') + } + + if (!isNonNegativeInteger(value.updatedAt)) { + throw new Error('runtimeState entry updatedAt must be a non-negative integer') + } + + return { + value: value.value, + updatedAt: value.updatedAt, + } +} + +function modelStateKey(keyHash: string, modelId: string): string { + return `${keyHash}:${modelId}` +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + +function isNonNegativeInteger(value: unknown): value is number { + return Number.isInteger(value) && Number(value) >= 0 +} + +function formatErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/upstream.ts b/src/upstream.ts new file mode 100644 index 0000000..4588b09 --- /dev/null +++ b/src/upstream.ts @@ -0,0 +1,376 @@ +import type { AppConfig } from './config.js' +import type { ModelPool } from './model-pool.js' + +const HOP_BY_HOP_HEADERS = new Set([ + 'connection', + 'content-length', + 'host', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', +]) + +interface FreeTierExhaustion { + request_id?: string + code?: string + message?: string +} + +interface FreeTierExhaustionCheck { + exhaustion: FreeTierExhaustion | null + parseMode: 'empty' | 'json' | 'sse' | 'text' | 'unreadable' + rawText: string +} + +export interface ProxyRequestInput { + request: Request + body: Record + pathWithSearch: string + config: AppConfig + modelPool: ModelPool +} + +export async function proxyAnthropicRequest(input: ProxyRequestInput): Promise { + return proxyModelPoolRequest({ + ...input, + upstreamBaseUrl: input.config.upstreamBaseUrl, + }) +} + +export async function proxyOpenAIRequest(input: ProxyRequestInput): Promise { + return proxyModelPoolRequest({ + ...input, + upstreamBaseUrl: input.config.openAIUpstreamBaseUrl, + blockedRequestHeaders: ['anthropic-beta', 'anthropic-version'], + }) +} + +interface ModelPoolProxyInput extends ProxyRequestInput { + upstreamBaseUrl: string + blockedRequestHeaders?: string[] +} + +async function proxyModelPoolRequest(input: ModelPoolProxyInput): Promise { + const candidates = input.modelPool.getAttemptModels() + + if (candidates.length === 0) { + return jsonResponse(503, { + error: { + type: 'all_models_in_cooldown', + message: 'All configured models are currently cooling down.', + }, + models: input.modelPool.snapshot(), + }) + } + + const attempts: Array<{ keyHash: string, model: string }> = [] + let lastExhaustion: FreeTierExhaustion | null = null + + for (const candidate of candidates) { + attempts.push({ + keyHash: candidate.keyLabel, + model: candidate.modelId, + }) + + const upstreamResponse = await fetch(buildUpstreamUrl(input.upstreamBaseUrl, input.pathWithSearch), { + method: input.request.method, + headers: buildUpstreamHeaders( + input.request.headers, + input.config, + candidate.apiKey, + input.blockedRequestHeaders, + ), + body: JSON.stringify({ + ...input.body, + model: candidate.modelId, + }), + }) + + const exhaustionCheck = await readFreeTierExhaustion(upstreamResponse.clone()) + logUpstreamRetryDecision(input.pathWithSearch, candidate, upstreamResponse, exhaustionCheck) + + if (exhaustionCheck.exhaustion) { + lastExhaustion = exhaustionCheck.exhaustion + input.modelPool.markFreeTierExhausted( + candidate, + exhaustionCheck.exhaustion.message || 'DashScope free tier exhausted', + ) + console.warn( + [ + '[proxy] free-tier exhausted; trying next model', + `path=${input.pathWithSearch}`, + `key=${candidate.keyLabel}`, + `model=${candidate.modelId}`, + `attempt=${attempts.length}/${candidates.length}`, + ].join(' '), + ) + continue + } + + input.modelPool.markSuccess(candidate) + + if (attempts.length > 1) { + console.log( + [ + '[proxy] upstream accepted after retry', + `path=${input.pathWithSearch}`, + `key=${candidate.keyLabel}`, + `model=${candidate.modelId}`, + `attempts=${attempts.length}`, + ].join(' '), + ) + } + + const headers = filterResponseHeaders(upstreamResponse.headers) + headers.set('x-proxy-key-hash', candidate.keyLabel) + headers.set('x-proxy-model', candidate.modelId) + headers.set('x-proxy-attempts', String(attempts.length)) + + return new Response(upstreamResponse.body, { + status: upstreamResponse.status, + statusText: upstreamResponse.statusText, + headers, + }) + } + + return jsonResponse(503, { + error: { + type: 'all_models_exhausted', + message: 'Every available model returned DashScope free-tier exhaustion.', + lastUpstreamError: lastExhaustion, + }, + attempts, + models: input.modelPool.snapshot(), + }) +} + +export function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { + 'content-type': 'application/json; charset=utf-8', + }, + }) +} + +function buildUpstreamUrl(baseUrl: string, pathWithSearch: string): string { + const normalizedBaseUrl = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl + return `${normalizedBaseUrl}${pathWithSearch.startsWith('/') ? '' : '/'}${pathWithSearch}` +} + +function buildUpstreamHeaders( + source: Headers, + config: AppConfig, + apiKey: string, + blockedRequestHeaders: string[] = [], +): Headers { + const headers = new Headers() + const blockedHeaderNames = new Set(blockedRequestHeaders.map((header) => header.toLowerCase())) + + for (const [name, value] of source.entries()) { + const lowerName = name.toLowerCase() + + if ( + HOP_BY_HOP_HEADERS.has(lowerName) || + blockedHeaderNames.has(lowerName) || + lowerName === 'authorization' || + lowerName === 'x-api-key' + ) { + continue + } + + headers.set(name, value) + } + + headers.set('content-type', 'application/json') + + if (config.upstreamAuthMode === 'authorization' || config.upstreamAuthMode === 'both') { + headers.set('authorization', `Bearer ${apiKey}`) + } + + if (config.upstreamAuthMode === 'x-api-key' || config.upstreamAuthMode === 'both') { + headers.set('x-api-key', apiKey) + } + + return headers +} + +function filterResponseHeaders(source: Headers): Headers { + const headers = new Headers() + + for (const [name, value] of source.entries()) { + const lowerName = name.toLowerCase() + if (HOP_BY_HOP_HEADERS.has(lowerName) || lowerName === 'content-encoding') continue + headers.set(name, value) + } + + return headers +} + +async function readFreeTierExhaustion(response: Response): Promise { + if (response.status !== 403) { + return { + exhaustion: null, + parseMode: 'empty', + rawText: '', + } + } + + const payload = await readErrorPayload(response) + if (!payload.payload) { + return { + exhaustion: null, + parseMode: payload.parseMode, + rawText: payload.rawText, + } + } + + return { + exhaustion: extractFreeTierExhaustion(payload.payload), + parseMode: payload.parseMode, + rawText: payload.rawText, + } +} + +async function readErrorPayload(response: Response): Promise<{ + payload: unknown | null + parseMode: FreeTierExhaustionCheck['parseMode'] + rawText: string +}> { + let text: string + try { + text = await response.text() + } catch { + return { + payload: null, + parseMode: 'unreadable', + rawText: '', + } + } + + const trimmed = text.trim() + if (!trimmed) { + return { + payload: null, + parseMode: 'empty', + rawText: '', + } + } + + if (trimmed.startsWith('{')) { + try { + return { + payload: JSON.parse(trimmed) as unknown, + parseMode: 'json', + rawText: trimmed, + } + } catch { + return { + payload: null, + parseMode: 'text', + rawText: trimmed, + } + } + } + + // DashScope can return streaming errors as: + // event:error + // data:{"code":"AccessDenied",...} + for (const line of trimmed.split(/\r?\n/)) { + const currentLine = line.trim() + if (!currentLine.startsWith('data:')) continue + + const rawData = currentLine.slice('data:'.length).trim() + if (!rawData || rawData === '[DONE]') continue + + try { + return { + payload: JSON.parse(rawData) as unknown, + parseMode: 'sse', + rawText: trimmed, + } + } catch { + return { + payload: null, + parseMode: 'text', + rawText: trimmed, + } + } + } + + return { + payload: null, + parseMode: 'text', + rawText: trimmed, + } +} + +function extractFreeTierExhaustion(payload: unknown): FreeTierExhaustion | null { + if (!isRecord(payload)) return null + + const error = isRecord(payload.error) ? payload.error : null + const code = getString(payload.code) || getString(error?.code) + const type = getString(payload.type) || getString(error?.type) + const message = getString(payload.message) || getString(error?.message) || '' + const requestId = + getString(payload.request_id) || + getString(payload.requestId) || + getString(error?.request_id) || + getString(error?.requestId) + const exhausted = /free\s+tier/i.test(message) && /exhausted/i.test(message) + const freeTierOnly = code === 'AllocationQuota.FreeTierOnly' || type === 'AllocationQuota.FreeTierOnly' + + if ((code === 'AccessDenied' && exhausted) || freeTierOnly) { + return { + code, + message, + request_id: requestId, + } + } + + return null +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function getString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} + +function logUpstreamRetryDecision( + pathWithSearch: string, + candidate: { keyLabel: string, modelId: string }, + response: Response, + check: FreeTierExhaustionCheck, +): void { + if (response.status < 400) return + + const contentType = response.headers.get('content-type') || '-' + const rawText = check.rawText ? previewText(check.rawText, 500) : '-' + + console.warn( + [ + '[proxy] upstream non-2xx', + `path=${pathWithSearch}`, + `status=${response.status}`, + `key=${candidate.keyLabel}`, + `model=${candidate.modelId}`, + `contentType=${JSON.stringify(contentType)}`, + `parse=${check.parseMode}`, + `freeTierExhausted=${check.exhaustion ? 'yes' : 'no'}`, + `body=${JSON.stringify(rawText)}`, + ].join(' '), + ) +} + +function previewText(value: string, maxLength: number): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, maxLength)}...` +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b056d07 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "rootDir": "src", + "outDir": "dist", + "types": ["node"], + "lib": ["ES2022", "DOM", "DOM.Iterable"] + }, + "include": ["src/**/*.ts"] +}