feat: publish xianyu hunter hermes skill package

This commit is contained in:
2026-05-06 19:26:45 +08:00
commit 6384156097
59 changed files with 9627 additions and 0 deletions
+579
View File
@@ -0,0 +1,579 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { genId, timestamp } from './utils.mjs';
import { checkLogin } from './login.mjs';
import { searchProducts } from './search.mjs';
import { coarseFilter, fineFilter } from './filter.mjs';
import { buildAnalysis } from './analyzer.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DATA_DIR = path.join(__dirname, '..', 'data');
const TASKS_FILE = path.join(DATA_DIR, 'tasks.json');
const STAGES = ['login', 'searching', 'coarse_filter', 'fine_filter'];
export class TaskManager {
constructor(broadcast) {
this.tasks = new Map();
this.broadcast = broadcast;
this._loadFromDisk();
this._startScheduler();
}
_buildTaskConfig(config = {}) {
const existingSchedule = config.schedule || {};
const scheduleEnabled = config.scheduleEnabled ?? existingSchedule.enabled ?? false;
const scheduleIntervalMinutes = config.scheduleIntervalMinutes ?? existingSchedule.intervalMinutes ?? 0;
return {
queries: Array.isArray(config.queries) ? config.queries.map(q => String(q).trim()).filter(Boolean) : [],
priceMin: config.priceMin === '' ? null : (config.priceMin ?? null),
priceMax: config.priceMax === '' ? null : (config.priceMax ?? null),
region: config.region || '',
personalSeller: !!config.personalSeller,
customRequirements: config.customRequirements || '',
coarsePrompt: config.coarsePrompt || '',
finePrompt: config.finePrompt || '',
maxPages: Math.max(1, Math.min(20, Number(config.maxPages || 3))),
maxItems: Math.max(1, Math.min(500, Number(config.maxItems || 60))),
schedule: {
enabled: !!scheduleEnabled,
intervalMinutes: Math.max(0, Number(scheduleIntervalMinutes || 0) || 0),
nextRunAt: config.scheduleNextRunAt ?? existingSchedule.nextRunAt ?? null,
lastRunAt: config.scheduleLastRunAt ?? existingSchedule.lastRunAt ?? null,
},
};
}
createTask(config) {
const id = genId();
const task = {
id,
name: config.name || `任务-${id}`,
config: this._buildTaskConfig(config),
stage: 'pending',
running: false,
stopped: false,
products: { raw: [], coarseFiltered: [], fineFiltered: [] },
hermesAnalysis: null,
chatSessions: [],
chatSessionsFull: [],
logs: [],
createdAt: Date.now(),
updatedAt: Date.now(),
};
this._normalizeSchedule(task);
this.tasks.set(id, task);
this._persistToDisk();
this._emit(id, 'task:created', this._serialize(task));
return task;
}
duplicateTask(sourceId) {
const source = this.tasks.get(sourceId);
if (!source) return null;
const id = genId();
const task = {
id,
name: `${source.name} (副本)`,
config: JSON.parse(JSON.stringify(source.config || {})),
stage: 'pending',
running: false,
stopped: false,
products: { raw: [], coarseFiltered: [], fineFiltered: [] },
hermesAnalysis: null,
chatSessions: [],
chatSessionsFull: [],
logs: [],
createdAt: Date.now(),
updatedAt: Date.now(),
};
this._normalizeSchedule(task);
this.tasks.set(id, task);
this._persistToDisk();
this._emit(id, 'task:created', this._serialize(task));
return task;
}
updateTask(id, patch = {}) {
const task = this.tasks.get(id);
if (!task) return null;
if (task.running) return null;
task.name = patch.name || task.name;
task.config = { ...task.config, ...this._buildTaskConfig({ ...task.config, ...patch }) };
task.updatedAt = Date.now();
this._normalizeSchedule(task);
this._persistToDisk();
this._emit(id, 'task:updated', this._serialize(task));
return task;
}
_resetTaskProducts(task) {
task.products = { raw: [], coarseFiltered: [], fineFiltered: [] };
task.hermesAnalysis = null;
task.chatSessions = [];
task.chatSessionsFull = [];
task.stage = 'pending';
task.stopped = false;
}
async startTask(id, configUpdates = {}) {
const task = this.tasks.get(id);
if (!task || task.running) return;
if (Object.keys(configUpdates || {}).length) {
const { reset, scheduled, ...patch } = configUpdates;
if (Object.keys(patch).length) task.config = { ...task.config, ...this._buildTaskConfig({ ...task.config, ...patch }) };
if (reset) this._resetTaskProducts(task);
if (scheduled) {
task.config.schedule.lastRunAt = Date.now();
this._normalizeSchedule(task, true);
}
}
task.running = true;
task.stopped = false;
this._persistToDisk();
this._emit(id, 'task:started', { id });
const emitLog = (msg) => {
const entry = { time: timestamp(), message: msg };
task.logs.push(entry);
this._emit(id, 'task:log', entry);
};
const shouldStop = () => task.stopped;
try {
await this._runPipeline(task, emitLog, shouldStop);
if (!task.stopped) {
task.stage = 'completed';
}
} catch (err) {
emitLog(`❌ 任务异常: ${err.message}`);
} finally {
task.running = false;
if (task.stopped) {
task.stage = 'stopped';
}
this._persistToDisk();
this._emit(id, 'task:finished', this._serialize(task));
}
}
stopTask(id) {
const task = this.tasks.get(id);
if (!task) return;
task.stopped = true;
task.stage = 'stopping';
this._persistToDisk();
this._emit(id, 'task:stopping', { id });
}
deleteTask(id) {
const task = this.tasks.get(id);
if (task) {
task.stopped = true;
this.tasks.delete(id);
this._persistToDisk();
}
}
clearTaskCache(id, { mode = 'all', minScore = 75 } = {}) {
const task = this.tasks.get(id);
if (!task || task.running) return null;
const before = {
raw: task.products?.raw?.length || 0,
coarseFiltered: task.products?.coarseFiltered?.length || 0,
fineFiltered: task.products?.fineFiltered?.length || 0,
hermesRecommended: task.hermesAnalysis?.recommendedCount || 0,
};
let kept = [];
const keepMode = mode === 'hermes-recommend' || mode === 'hermes-pass' ? mode : 'all';
if (keepMode !== 'all') {
const fine = task.products?.fineFiltered || [];
kept = fine.filter(item => {
const analysis = item.hermesAnalysis;
if (!analysis) return false;
if (keepMode === 'hermes-recommend') return analysis.verdict === 'recommend';
return (analysis.verdict === 'recommend' || analysis.verdict === 'caution') && Number(analysis.score || 0) >= Number(minScore || 0);
});
}
task.products = keepMode === 'all'
? { raw: [], coarseFiltered: [], fineFiltered: [] }
: { raw: kept, coarseFiltered: kept, fineFiltered: kept };
task.hermesAnalysis = null;
task.chatSessions = [];
task.chatSessionsFull = [];
task.stage = keepMode === 'all' ? 'pending' : (kept.length ? 'completed' : 'pending');
task.stopped = false;
task.updatedAt = Date.now();
task.logs.push({
time: timestamp(),
message: keepMode === 'all'
? `🧹 已清除本任务缓存:采集 ${before.raw} / 粗筛 ${before.coarseFiltered} / 细筛 ${before.fineFiltered}`
: `🧹 已清理缓存并保留 ${kept.length} 个优质结果(模式:${keepMode},最低分:${minScore}`,
});
this._persistToDisk();
this._emit(id, 'task:updated', this._serialize(task));
return {
ok: true,
mode: keepMode,
minScore: Number(minScore || 0),
before,
after: {
raw: task.products.raw.length,
coarseFiltered: task.products.coarseFiltered.length,
fineFiltered: task.products.fineFiltered.length,
},
kept: kept.length,
task: this._serialize(task),
};
}
getTask(id) {
const task = this.tasks.get(id);
return task ? this._serialize(task) : null;
}
getTaskFull(id) {
return this.tasks.get(id) || null;
}
analyzeTask(id, options = {}) {
const task = this.tasks.get(id);
if (!task) return null;
const analysis = buildAnalysis(task, options);
this.setTaskAnalysis(id, analysis);
return analysis;
}
setTaskAnalysis(id, analysis) {
const task = this.tasks.get(id);
if (!task || !analysis) return null;
const map = new Map((analysis.items || []).map(item => [item.id, item]));
task.products.fineFiltered = (task.products.fineFiltered || []).map(item => map.get(item.id) || item);
task.hermesAnalysis = analysis.summary;
task.updatedAt = Date.now();
this._persistToDisk();
this._emit(id, 'task:updated', this._serialize(task));
return task;
}
getAllTasks() {
return [...this.tasks.values()].map(t => this._serialize(t));
}
_resolveResumeStage(task) {
if (task.products.fineFiltered.length > 0) return 'fine_filter';
if (task.products.coarseFiltered.length > 0) return 'fine_filter';
if (task.products.raw.length > 0) return 'coarse_filter';
return null;
}
async _runPipeline(task, emitLog, shouldStop) {
const resumeStage = this._resolveResumeStage(task);
const stageOrder = ['login', 'searching', 'coarse_filter', 'fine_filter'];
const resumeIdx = resumeStage ? stageOrder.indexOf(resumeStage) : -1;
if (resumeStage) {
emitLog(`♻️ 检测到历史数据,从「${resumeStage}」阶段恢复`);
emitLog(` 已有: ${task.products.raw.length}采集 / ${task.products.coarseFiltered.length}粗筛 / ${task.products.fineFiltered.length}候选`);
}
// Stage 1: Login (always)
task.stage = 'login';
this._emitStage(task);
emitLog('========== 阶段1: 登录验证 ==========');
const loggedIn = await checkLogin(emitLog);
if (!loggedIn || shouldStop()) return;
const filterRequirements = this._buildFilterRequirements(task.config);
// Stage 2: Search
if (resumeIdx >= stageOrder.indexOf('searching')) {
emitLog(`⏭️ 跳过搜索采集(已有 ${task.products.raw.length} 个商品)`);
} else {
task.stage = 'searching';
this._emitStage(task);
emitLog('========== 阶段2: 搜索采集 ==========');
const filters = {
priceMin: task.config.priceMin,
priceMax: task.config.priceMax,
region: task.config.region,
personalSeller: task.config.personalSeller,
};
for (const query of task.config.queries) {
if (shouldStop()) break;
const products = await searchProducts(query, filters, task.config.maxPages, emitLog, shouldStop);
const deduped = products.filter(p => !task.products.raw.some(e => e.id === p.id));
const remaining = Math.max(0, (task.config.maxItems || 60) - task.products.raw.length);
const accepted = deduped.slice(0, remaining);
task.products.raw.push(...accepted);
this._emitProducts(task);
if (task.products.raw.length >= (task.config.maxItems || 60)) {
emitLog(`📦 已达到单次最大商品寻找量 ${task.config.maxItems || 60},停止继续搜索`);
break;
}
}
emitLog(`📊 总计采集 ${task.products.raw.length} 个商品(去重后)`);
this._persistToDisk();
}
if (shouldStop() || task.products.raw.length === 0) return;
// Stage 3: Coarse Filter
if (resumeIdx >= stageOrder.indexOf('coarse_filter')) {
emitLog(`⏭️ 跳过粗筛(已有 ${task.products.coarseFiltered.length} 个通过)`);
} else {
task.stage = 'coarse_filter';
this._emitStage(task);
emitLog('========== 阶段3: 粗筛(标题筛选) ==========');
const coarseRequirements = this._buildCoarseRequirements(task.config);
task.products.coarseFiltered = await coarseFilter(
task.products.raw, coarseRequirements, emitLog, shouldStop, task.config.coarsePrompt
);
this._emitProducts(task);
this._persistToDisk();
}
if (shouldStop() || task.products.coarseFiltered.length === 0) return;
// Stage 4: Fine Filter
if (resumeIdx >= stageOrder.indexOf('fine_filter')) {
emitLog(`⏭️ 跳过细筛(已有 ${task.products.fineFiltered.length} 个通过)`);
} else {
task.stage = 'fine_filter';
this._emitStage(task);
emitLog('========== 阶段4: 细筛(详情页筛选) ==========');
task.products.fineFiltered = await fineFilter(
task.products.coarseFiltered.slice(0, task.config.maxItems || 60), filterRequirements, emitLog, shouldStop, task.config.finePrompt
);
this._emitProducts(task);
this._persistToDisk();
}
if (shouldStop()) return;
task.chatSessions = [];
task.chatSessionsFull = [];
this._emit(task.id, 'task:chats', task.chatSessions);
this._persistToDisk();
emitLog(`🦐 已关闭联系卖家功能:保留 ${task.products.fineFiltered.length} 个候选商品,后续由 Hermes 读取 data/tasks.json 做深度分析报告`);
emitLog('========== 采集流程结束 ==========');
}
_buildCoarseRequirements(config) {
const parts = [];
if (config.queries.length > 0) {
parts.push(`目标商品: ${config.queries.join(', ')}`);
}
if (config.customRequirements) {
parts.push(`补充需求: ${config.customRequirements}`);
}
return parts.join('\n');
}
_buildFilterRequirements(config) {
const parts = [];
if (config.queries.length > 0) {
parts.push(`搜索关键词: ${config.queries.join(', ')}`);
}
if (config.priceMin != null || config.priceMax != null) {
parts.push(`价格范围: ${config.priceMin ?? '不限'} - ${config.priceMax ?? '不限'}`);
}
if (config.region) {
parts.push(`地区偏好: ${config.region}`);
}
if (config.personalSeller) {
parts.push('只要个人卖家');
}
if (config.customRequirements) {
parts.push(`商品需求: ${config.customRequirements}`);
}
return parts.join('\n');
}
_serialize(task) {
return {
id: task.id,
name: task.name,
config: task.config,
stage: task.stage,
running: task.running,
stopped: task.stopped,
products: {
rawCount: task.products.raw.length,
coarseCount: task.products.coarseFiltered.length,
fineCount: task.products.fineFiltered.length,
raw: task.products.raw.slice(0, 100),
coarseFiltered: task.products.coarseFiltered.slice(0, 50),
fineFiltered: task.products.fineFiltered,
},
chatSessions: task.chatSessions || [],
hermesAnalysis: task.hermesAnalysis || null,
logs: task.logs.slice(-200),
createdAt: task.createdAt,
updatedAt: task.updatedAt,
};
}
_serializeFull(task) {
return {
id: task.id,
name: task.name,
config: task.config,
stage: task.stage,
products: {
raw: task.products.raw,
coarseFiltered: task.products.coarseFiltered,
fineFiltered: task.products.fineFiltered,
},
chatSessions: task.chatSessions || [],
chatSessionsFull: task.chatSessionsFull || [],
hermesAnalysis: task.hermesAnalysis || null,
logs: task.logs.slice(-500),
createdAt: task.createdAt,
updatedAt: task.updatedAt,
};
}
_persistToDisk() {
try {
fs.mkdirSync(DATA_DIR, { recursive: true });
const data = [...this.tasks.values()].map(t => this._serializeFull(t));
fs.writeFileSync(TASKS_FILE, JSON.stringify(data, null, 2), 'utf-8');
} catch (err) {
console.error('持久化任务失败:', err.message);
}
}
_loadFromDisk() {
try {
if (!fs.existsSync(TASKS_FILE)) return;
const raw = fs.readFileSync(TASKS_FILE, 'utf-8');
const data = JSON.parse(raw);
for (const t of data) {
let chatSessionsFull = t.chatSessionsFull || [];
if (chatSessionsFull.length === 0 && (t.chatSessions || []).length > 0) {
const fineMap = new Map((t.products?.fineFiltered || []).map(p => [p.id, p]));
chatSessionsFull = t.chatSessions.map(s => {
const product = fineMap.get(s.id) || {
id: s.id,
title: s.productTitle || '',
price: s.productPrice || '',
description: s.productDescription || '',
sellerName: s.sellerName || '',
chatUrl: '',
};
const messages = (s.messages || []).map(m => ({
...m,
fingerprint: `${m.role}:${m.content?.trim()}`,
}));
const chatHistory = messages.map(m => ({
role: m.role === 'self' ? 'assistant' : 'user',
content: m.content,
}));
return {
id: s.id,
product,
chatContext: null,
status: s.status || 'waiting',
goalReason: s.goalReason || '',
messages,
chatHistory,
lastChecked: s.lastChecked || Date.now(),
backoffLevel: 0,
};
});
console.log(` 🔄 任务 ${t.id}: 从旧格式迁移了 ${chatSessionsFull.length} 个聊天会话`);
}
const task = {
id: t.id,
name: t.name,
config: this._buildTaskConfig(t.config || {}),
stage: (t.stage === 'completed' || t.stage === 'stopped') ? t.stage : 'stopped',
running: false,
stopped: true,
products: {
raw: t.products?.raw || [],
coarseFiltered: t.products?.coarseFiltered || [],
fineFiltered: t.products?.fineFiltered || [],
},
chatSessions: t.chatSessions || [],
chatSessionsFull,
hermesAnalysis: t.hermesAnalysis || null,
logs: t.logs || [],
createdAt: t.createdAt || Date.now(),
updatedAt: t.updatedAt || t.createdAt || Date.now(),
};
this._normalizeSchedule(task);
this.tasks.set(task.id, task);
}
console.log(`📂 从磁盘恢复了 ${data.length} 个任务`);
} catch (err) {
console.error('加载持久化任务失败:', err.message);
}
}
_normalizeSchedule(task, afterRun = false) {
if (!task.config.schedule) {
task.config.schedule = { enabled: false, intervalMinutes: 0, nextRunAt: null, lastRunAt: null };
}
const s = task.config.schedule;
s.intervalMinutes = Number(s.intervalMinutes || 0) || 0;
s.enabled = !!s.enabled && s.intervalMinutes > 0;
if (!s.enabled) {
s.nextRunAt = null;
return;
}
const base = afterRun ? Date.now() : (Number(s.nextRunAt) || Date.now());
if (!s.nextRunAt || Number(s.nextRunAt) <= Date.now()) {
s.nextRunAt = base + s.intervalMinutes * 60 * 1000;
}
}
_startScheduler() {
this.schedulerTimer = setInterval(() => this._tickSchedules(), 30000);
}
_tickSchedules() {
const now = Date.now();
for (const task of this.tasks.values()) {
const sched = task.config?.schedule;
if (!sched?.enabled || task.running || !sched.nextRunAt || sched.nextRunAt > now) continue;
task.logs.push({ time: timestamp(), message: `⏰ 定时任务触发,开始后台采集` });
this.startTask(task.id, { reset: true, scheduled: true });
}
}
_emit(taskId, event, data) {
this.broadcast({ event, taskId, data });
}
_emitStage(task) {
this._emit(task.id, 'task:stage', { id: task.id, stage: task.stage });
}
_emitProducts(task) {
this._emit(task.id, 'task:products', {
id: task.id,
rawCount: task.products.raw.length,
coarseCount: task.products.coarseFiltered.length,
fineCount: task.products.fineFiltered.length,
});
}
}