580 lines
19 KiB
JavaScript
580 lines
19 KiB
JavaScript
import fs from 'fs';
|
||
import path from 'path';
|
||
import { fileURLToPath } from 'url';
|
||
import { genId, timestamp } from './utils.mjs';
|
||
import { checkLogin } from './login.mjs';
|
||
import { searchProducts } from './search.mjs';
|
||
import { coarseFilter, fineFilter } from './filter.mjs';
|
||
import { buildAnalysis } from './analyzer.mjs';
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||
const DATA_DIR = path.join(__dirname, '..', 'data');
|
||
const TASKS_FILE = path.join(DATA_DIR, 'tasks.json');
|
||
|
||
const STAGES = ['login', 'searching', 'coarse_filter', 'fine_filter'];
|
||
|
||
export class TaskManager {
|
||
constructor(broadcast) {
|
||
this.tasks = new Map();
|
||
this.broadcast = broadcast;
|
||
this._loadFromDisk();
|
||
this._startScheduler();
|
||
}
|
||
|
||
_buildTaskConfig(config = {}) {
|
||
const existingSchedule = config.schedule || {};
|
||
const scheduleEnabled = config.scheduleEnabled ?? existingSchedule.enabled ?? false;
|
||
const scheduleIntervalMinutes = config.scheduleIntervalMinutes ?? existingSchedule.intervalMinutes ?? 0;
|
||
return {
|
||
queries: Array.isArray(config.queries) ? config.queries.map(q => String(q).trim()).filter(Boolean) : [],
|
||
priceMin: config.priceMin === '' ? null : (config.priceMin ?? null),
|
||
priceMax: config.priceMax === '' ? null : (config.priceMax ?? null),
|
||
region: config.region || '',
|
||
personalSeller: !!config.personalSeller,
|
||
customRequirements: config.customRequirements || '',
|
||
coarsePrompt: config.coarsePrompt || '',
|
||
finePrompt: config.finePrompt || '',
|
||
maxPages: Math.max(1, Math.min(20, Number(config.maxPages || 3))),
|
||
maxItems: Math.max(1, Math.min(500, Number(config.maxItems || 60))),
|
||
schedule: {
|
||
enabled: !!scheduleEnabled,
|
||
intervalMinutes: Math.max(0, Number(scheduleIntervalMinutes || 0) || 0),
|
||
nextRunAt: config.scheduleNextRunAt ?? existingSchedule.nextRunAt ?? null,
|
||
lastRunAt: config.scheduleLastRunAt ?? existingSchedule.lastRunAt ?? null,
|
||
},
|
||
};
|
||
}
|
||
|
||
createTask(config) {
|
||
const id = genId();
|
||
const task = {
|
||
id,
|
||
name: config.name || `任务-${id}`,
|
||
config: this._buildTaskConfig(config),
|
||
stage: 'pending',
|
||
running: false,
|
||
stopped: false,
|
||
products: { raw: [], coarseFiltered: [], fineFiltered: [] },
|
||
hermesAnalysis: null,
|
||
chatSessions: [],
|
||
chatSessionsFull: [],
|
||
logs: [],
|
||
createdAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
this._normalizeSchedule(task);
|
||
|
||
this.tasks.set(id, task);
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:created', this._serialize(task));
|
||
return task;
|
||
}
|
||
|
||
duplicateTask(sourceId) {
|
||
const source = this.tasks.get(sourceId);
|
||
if (!source) return null;
|
||
|
||
const id = genId();
|
||
const task = {
|
||
id,
|
||
name: `${source.name} (副本)`,
|
||
config: JSON.parse(JSON.stringify(source.config || {})),
|
||
stage: 'pending',
|
||
running: false,
|
||
stopped: false,
|
||
products: { raw: [], coarseFiltered: [], fineFiltered: [] },
|
||
hermesAnalysis: null,
|
||
chatSessions: [],
|
||
chatSessionsFull: [],
|
||
logs: [],
|
||
createdAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
this._normalizeSchedule(task);
|
||
|
||
this.tasks.set(id, task);
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:created', this._serialize(task));
|
||
return task;
|
||
}
|
||
|
||
updateTask(id, patch = {}) {
|
||
const task = this.tasks.get(id);
|
||
if (!task) return null;
|
||
if (task.running) return null;
|
||
|
||
task.name = patch.name || task.name;
|
||
task.config = { ...task.config, ...this._buildTaskConfig({ ...task.config, ...patch }) };
|
||
task.updatedAt = Date.now();
|
||
this._normalizeSchedule(task);
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:updated', this._serialize(task));
|
||
return task;
|
||
}
|
||
|
||
_resetTaskProducts(task) {
|
||
task.products = { raw: [], coarseFiltered: [], fineFiltered: [] };
|
||
task.hermesAnalysis = null;
|
||
task.chatSessions = [];
|
||
task.chatSessionsFull = [];
|
||
task.stage = 'pending';
|
||
task.stopped = false;
|
||
}
|
||
|
||
async startTask(id, configUpdates = {}) {
|
||
const task = this.tasks.get(id);
|
||
if (!task || task.running) return;
|
||
|
||
if (Object.keys(configUpdates || {}).length) {
|
||
const { reset, scheduled, ...patch } = configUpdates;
|
||
if (Object.keys(patch).length) task.config = { ...task.config, ...this._buildTaskConfig({ ...task.config, ...patch }) };
|
||
if (reset) this._resetTaskProducts(task);
|
||
if (scheduled) {
|
||
task.config.schedule.lastRunAt = Date.now();
|
||
this._normalizeSchedule(task, true);
|
||
}
|
||
}
|
||
|
||
task.running = true;
|
||
task.stopped = false;
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:started', { id });
|
||
|
||
const emitLog = (msg) => {
|
||
const entry = { time: timestamp(), message: msg };
|
||
task.logs.push(entry);
|
||
this._emit(id, 'task:log', entry);
|
||
};
|
||
|
||
const shouldStop = () => task.stopped;
|
||
|
||
try {
|
||
await this._runPipeline(task, emitLog, shouldStop);
|
||
if (!task.stopped) {
|
||
task.stage = 'completed';
|
||
}
|
||
} catch (err) {
|
||
emitLog(`❌ 任务异常: ${err.message}`);
|
||
} finally {
|
||
task.running = false;
|
||
if (task.stopped) {
|
||
task.stage = 'stopped';
|
||
}
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:finished', this._serialize(task));
|
||
}
|
||
}
|
||
|
||
stopTask(id) {
|
||
const task = this.tasks.get(id);
|
||
if (!task) return;
|
||
task.stopped = true;
|
||
task.stage = 'stopping';
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:stopping', { id });
|
||
}
|
||
|
||
deleteTask(id) {
|
||
const task = this.tasks.get(id);
|
||
if (task) {
|
||
task.stopped = true;
|
||
this.tasks.delete(id);
|
||
this._persistToDisk();
|
||
}
|
||
}
|
||
|
||
clearTaskCache(id, { mode = 'all', minScore = 75 } = {}) {
|
||
const task = this.tasks.get(id);
|
||
if (!task || task.running) return null;
|
||
|
||
const before = {
|
||
raw: task.products?.raw?.length || 0,
|
||
coarseFiltered: task.products?.coarseFiltered?.length || 0,
|
||
fineFiltered: task.products?.fineFiltered?.length || 0,
|
||
hermesRecommended: task.hermesAnalysis?.recommendedCount || 0,
|
||
};
|
||
|
||
let kept = [];
|
||
const keepMode = mode === 'hermes-recommend' || mode === 'hermes-pass' ? mode : 'all';
|
||
if (keepMode !== 'all') {
|
||
const fine = task.products?.fineFiltered || [];
|
||
kept = fine.filter(item => {
|
||
const analysis = item.hermesAnalysis;
|
||
if (!analysis) return false;
|
||
if (keepMode === 'hermes-recommend') return analysis.verdict === 'recommend';
|
||
return (analysis.verdict === 'recommend' || analysis.verdict === 'caution') && Number(analysis.score || 0) >= Number(minScore || 0);
|
||
});
|
||
}
|
||
|
||
task.products = keepMode === 'all'
|
||
? { raw: [], coarseFiltered: [], fineFiltered: [] }
|
||
: { raw: kept, coarseFiltered: kept, fineFiltered: kept };
|
||
task.hermesAnalysis = null;
|
||
task.chatSessions = [];
|
||
task.chatSessionsFull = [];
|
||
task.stage = keepMode === 'all' ? 'pending' : (kept.length ? 'completed' : 'pending');
|
||
task.stopped = false;
|
||
task.updatedAt = Date.now();
|
||
task.logs.push({
|
||
time: timestamp(),
|
||
message: keepMode === 'all'
|
||
? `🧹 已清除本任务缓存：采集 ${before.raw} / 粗筛 ${before.coarseFiltered} / 细筛 ${before.fineFiltered}`
|
||
: `🧹 已清理缓存并保留 ${kept.length} 个优质结果（模式：${keepMode}，最低分：${minScore}）`,
|
||
});
|
||
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:updated', this._serialize(task));
|
||
return {
|
||
ok: true,
|
||
mode: keepMode,
|
||
minScore: Number(minScore || 0),
|
||
before,
|
||
after: {
|
||
raw: task.products.raw.length,
|
||
coarseFiltered: task.products.coarseFiltered.length,
|
||
fineFiltered: task.products.fineFiltered.length,
|
||
},
|
||
kept: kept.length,
|
||
task: this._serialize(task),
|
||
};
|
||
}
|
||
|
||
getTask(id) {
|
||
const task = this.tasks.get(id);
|
||
return task ? this._serialize(task) : null;
|
||
}
|
||
|
||
getTaskFull(id) {
|
||
return this.tasks.get(id) || null;
|
||
}
|
||
|
||
analyzeTask(id, options = {}) {
|
||
const task = this.tasks.get(id);
|
||
if (!task) return null;
|
||
const analysis = buildAnalysis(task, options);
|
||
this.setTaskAnalysis(id, analysis);
|
||
return analysis;
|
||
}
|
||
|
||
setTaskAnalysis(id, analysis) {
|
||
const task = this.tasks.get(id);
|
||
if (!task || !analysis) return null;
|
||
const map = new Map((analysis.items || []).map(item => [item.id, item]));
|
||
task.products.fineFiltered = (task.products.fineFiltered || []).map(item => map.get(item.id) || item);
|
||
task.hermesAnalysis = analysis.summary;
|
||
task.updatedAt = Date.now();
|
||
this._persistToDisk();
|
||
this._emit(id, 'task:updated', this._serialize(task));
|
||
return task;
|
||
}
|
||
|
||
getAllTasks() {
|
||
return [...this.tasks.values()].map(t => this._serialize(t));
|
||
}
|
||
|
||
_resolveResumeStage(task) {
|
||
if (task.products.fineFiltered.length > 0) return 'fine_filter';
|
||
if (task.products.coarseFiltered.length > 0) return 'fine_filter';
|
||
if (task.products.raw.length > 0) return 'coarse_filter';
|
||
return null;
|
||
}
|
||
|
||
async _runPipeline(task, emitLog, shouldStop) {
|
||
const resumeStage = this._resolveResumeStage(task);
|
||
const stageOrder = ['login', 'searching', 'coarse_filter', 'fine_filter'];
|
||
const resumeIdx = resumeStage ? stageOrder.indexOf(resumeStage) : -1;
|
||
|
||
if (resumeStage) {
|
||
emitLog(`♻️ 检测到历史数据，从「${resumeStage}」阶段恢复`);
|
||
emitLog(` 已有: ${task.products.raw.length}采集 / ${task.products.coarseFiltered.length}粗筛 / ${task.products.fineFiltered.length}候选`);
|
||
}
|
||
|
||
// Stage 1: Login (always)
|
||
task.stage = 'login';
|
||
this._emitStage(task);
|
||
emitLog('========== 阶段1: 登录验证 ==========');
|
||
|
||
const loggedIn = await checkLogin(emitLog);
|
||
if (!loggedIn || shouldStop()) return;
|
||
|
||
const filterRequirements = this._buildFilterRequirements(task.config);
|
||
|
||
// Stage 2: Search
|
||
if (resumeIdx >= stageOrder.indexOf('searching')) {
|
||
emitLog(`⏭️ 跳过搜索采集（已有 ${task.products.raw.length} 个商品）`);
|
||
} else {
|
||
task.stage = 'searching';
|
||
this._emitStage(task);
|
||
emitLog('========== 阶段2: 搜索采集 ==========');
|
||
|
||
const filters = {
|
||
priceMin: task.config.priceMin,
|
||
priceMax: task.config.priceMax,
|
||
region: task.config.region,
|
||
personalSeller: task.config.personalSeller,
|
||
};
|
||
|
||
for (const query of task.config.queries) {
|
||
if (shouldStop()) break;
|
||
const products = await searchProducts(query, filters, task.config.maxPages, emitLog, shouldStop);
|
||
const deduped = products.filter(p => !task.products.raw.some(e => e.id === p.id));
|
||
const remaining = Math.max(0, (task.config.maxItems || 60) - task.products.raw.length);
|
||
const accepted = deduped.slice(0, remaining);
|
||
task.products.raw.push(...accepted);
|
||
this._emitProducts(task);
|
||
if (task.products.raw.length >= (task.config.maxItems || 60)) {
|
||
emitLog(`📦 已达到单次最大商品寻找量 ${task.config.maxItems || 60}，停止继续搜索`);
|
||
break;
|
||
}
|
||
}
|
||
|
||
emitLog(`📊 总计采集 ${task.products.raw.length} 个商品（去重后）`);
|
||
this._persistToDisk();
|
||
}
|
||
if (shouldStop() || task.products.raw.length === 0) return;
|
||
|
||
// Stage 3: Coarse Filter
|
||
if (resumeIdx >= stageOrder.indexOf('coarse_filter')) {
|
||
emitLog(`⏭️ 跳过粗筛（已有 ${task.products.coarseFiltered.length} 个通过）`);
|
||
} else {
|
||
task.stage = 'coarse_filter';
|
||
this._emitStage(task);
|
||
emitLog('========== 阶段3: 粗筛（标题筛选） ==========');
|
||
|
||
const coarseRequirements = this._buildCoarseRequirements(task.config);
|
||
task.products.coarseFiltered = await coarseFilter(
|
||
task.products.raw, coarseRequirements, emitLog, shouldStop, task.config.coarsePrompt
|
||
);
|
||
this._emitProducts(task);
|
||
this._persistToDisk();
|
||
}
|
||
if (shouldStop() || task.products.coarseFiltered.length === 0) return;
|
||
|
||
// Stage 4: Fine Filter
|
||
if (resumeIdx >= stageOrder.indexOf('fine_filter')) {
|
||
emitLog(`⏭️ 跳过细筛（已有 ${task.products.fineFiltered.length} 个通过）`);
|
||
} else {
|
||
task.stage = 'fine_filter';
|
||
this._emitStage(task);
|
||
emitLog('========== 阶段4: 细筛（详情页筛选） ==========');
|
||
|
||
task.products.fineFiltered = await fineFilter(
|
||
task.products.coarseFiltered.slice(0, task.config.maxItems || 60), filterRequirements, emitLog, shouldStop, task.config.finePrompt
|
||
);
|
||
this._emitProducts(task);
|
||
this._persistToDisk();
|
||
}
|
||
if (shouldStop()) return;
|
||
|
||
task.chatSessions = [];
|
||
task.chatSessionsFull = [];
|
||
this._emit(task.id, 'task:chats', task.chatSessions);
|
||
this._persistToDisk();
|
||
emitLog(`🦐 已关闭联系卖家功能：保留 ${task.products.fineFiltered.length} 个候选商品，后续由 Hermes 读取 data/tasks.json 做深度分析报告`);
|
||
emitLog('========== 采集流程结束 ==========');
|
||
}
|
||
|
||
_buildCoarseRequirements(config) {
|
||
const parts = [];
|
||
if (config.queries.length > 0) {
|
||
parts.push(`目标商品: ${config.queries.join(', ')}`);
|
||
}
|
||
if (config.customRequirements) {
|
||
parts.push(`补充需求: ${config.customRequirements}`);
|
||
}
|
||
return parts.join('\n');
|
||
}
|
||
|
||
_buildFilterRequirements(config) {
|
||
const parts = [];
|
||
if (config.queries.length > 0) {
|
||
parts.push(`搜索关键词: ${config.queries.join(', ')}`);
|
||
}
|
||
if (config.priceMin != null || config.priceMax != null) {
|
||
parts.push(`价格范围: ${config.priceMin ?? '不限'} - ${config.priceMax ?? '不限'}`);
|
||
}
|
||
if (config.region) {
|
||
parts.push(`地区偏好: ${config.region}`);
|
||
}
|
||
if (config.personalSeller) {
|
||
parts.push('只要个人卖家');
|
||
}
|
||
if (config.customRequirements) {
|
||
parts.push(`商品需求: ${config.customRequirements}`);
|
||
}
|
||
return parts.join('\n');
|
||
}
|
||
|
||
_serialize(task) {
|
||
return {
|
||
id: task.id,
|
||
name: task.name,
|
||
config: task.config,
|
||
stage: task.stage,
|
||
running: task.running,
|
||
stopped: task.stopped,
|
||
products: {
|
||
rawCount: task.products.raw.length,
|
||
coarseCount: task.products.coarseFiltered.length,
|
||
fineCount: task.products.fineFiltered.length,
|
||
raw: task.products.raw.slice(0, 100),
|
||
coarseFiltered: task.products.coarseFiltered.slice(0, 50),
|
||
fineFiltered: task.products.fineFiltered,
|
||
},
|
||
chatSessions: task.chatSessions || [],
|
||
hermesAnalysis: task.hermesAnalysis || null,
|
||
logs: task.logs.slice(-200),
|
||
createdAt: task.createdAt,
|
||
updatedAt: task.updatedAt,
|
||
};
|
||
}
|
||
|
||
_serializeFull(task) {
|
||
return {
|
||
id: task.id,
|
||
name: task.name,
|
||
config: task.config,
|
||
stage: task.stage,
|
||
products: {
|
||
raw: task.products.raw,
|
||
coarseFiltered: task.products.coarseFiltered,
|
||
fineFiltered: task.products.fineFiltered,
|
||
},
|
||
chatSessions: task.chatSessions || [],
|
||
chatSessionsFull: task.chatSessionsFull || [],
|
||
hermesAnalysis: task.hermesAnalysis || null,
|
||
logs: task.logs.slice(-500),
|
||
createdAt: task.createdAt,
|
||
updatedAt: task.updatedAt,
|
||
};
|
||
}
|
||
|
||
_persistToDisk() {
|
||
try {
|
||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||
const data = [...this.tasks.values()].map(t => this._serializeFull(t));
|
||
fs.writeFileSync(TASKS_FILE, JSON.stringify(data, null, 2), 'utf-8');
|
||
} catch (err) {
|
||
console.error('持久化任务失败:', err.message);
|
||
}
|
||
}
|
||
|
||
_loadFromDisk() {
|
||
try {
|
||
if (!fs.existsSync(TASKS_FILE)) return;
|
||
const raw = fs.readFileSync(TASKS_FILE, 'utf-8');
|
||
const data = JSON.parse(raw);
|
||
for (const t of data) {
|
||
let chatSessionsFull = t.chatSessionsFull || [];
|
||
|
||
if (chatSessionsFull.length === 0 && (t.chatSessions || []).length > 0) {
|
||
const fineMap = new Map((t.products?.fineFiltered || []).map(p => [p.id, p]));
|
||
chatSessionsFull = t.chatSessions.map(s => {
|
||
const product = fineMap.get(s.id) || {
|
||
id: s.id,
|
||
title: s.productTitle || '',
|
||
price: s.productPrice || '',
|
||
description: s.productDescription || '',
|
||
sellerName: s.sellerName || '',
|
||
chatUrl: '',
|
||
};
|
||
const messages = (s.messages || []).map(m => ({
|
||
...m,
|
||
fingerprint: `${m.role}:${m.content?.trim()}`,
|
||
}));
|
||
const chatHistory = messages.map(m => ({
|
||
role: m.role === 'self' ? 'assistant' : 'user',
|
||
content: m.content,
|
||
}));
|
||
return {
|
||
id: s.id,
|
||
product,
|
||
chatContext: null,
|
||
status: s.status || 'waiting',
|
||
goalReason: s.goalReason || '',
|
||
messages,
|
||
chatHistory,
|
||
lastChecked: s.lastChecked || Date.now(),
|
||
backoffLevel: 0,
|
||
};
|
||
});
|
||
console.log(` 🔄 任务 ${t.id}: 从旧格式迁移了 ${chatSessionsFull.length} 个聊天会话`);
|
||
}
|
||
|
||
const task = {
|
||
id: t.id,
|
||
name: t.name,
|
||
config: this._buildTaskConfig(t.config || {}),
|
||
stage: (t.stage === 'completed' || t.stage === 'stopped') ? t.stage : 'stopped',
|
||
running: false,
|
||
stopped: true,
|
||
products: {
|
||
raw: t.products?.raw || [],
|
||
coarseFiltered: t.products?.coarseFiltered || [],
|
||
fineFiltered: t.products?.fineFiltered || [],
|
||
},
|
||
chatSessions: t.chatSessions || [],
|
||
chatSessionsFull,
|
||
hermesAnalysis: t.hermesAnalysis || null,
|
||
logs: t.logs || [],
|
||
createdAt: t.createdAt || Date.now(),
|
||
updatedAt: t.updatedAt || t.createdAt || Date.now(),
|
||
};
|
||
this._normalizeSchedule(task);
|
||
this.tasks.set(task.id, task);
|
||
}
|
||
console.log(`📂 从磁盘恢复了 ${data.length} 个任务`);
|
||
} catch (err) {
|
||
console.error('加载持久化任务失败:', err.message);
|
||
}
|
||
}
|
||
|
||
_normalizeSchedule(task, afterRun = false) {
|
||
if (!task.config.schedule) {
|
||
task.config.schedule = { enabled: false, intervalMinutes: 0, nextRunAt: null, lastRunAt: null };
|
||
}
|
||
const s = task.config.schedule;
|
||
s.intervalMinutes = Number(s.intervalMinutes || 0) || 0;
|
||
s.enabled = !!s.enabled && s.intervalMinutes > 0;
|
||
if (!s.enabled) {
|
||
s.nextRunAt = null;
|
||
return;
|
||
}
|
||
const base = afterRun ? Date.now() : (Number(s.nextRunAt) || Date.now());
|
||
if (!s.nextRunAt || Number(s.nextRunAt) <= Date.now()) {
|
||
s.nextRunAt = base + s.intervalMinutes * 60 * 1000;
|
||
}
|
||
}
|
||
|
||
_startScheduler() {
|
||
this.schedulerTimer = setInterval(() => this._tickSchedules(), 30000);
|
||
}
|
||
|
||
_tickSchedules() {
|
||
const now = Date.now();
|
||
for (const task of this.tasks.values()) {
|
||
const sched = task.config?.schedule;
|
||
if (!sched?.enabled || task.running || !sched.nextRunAt || sched.nextRunAt > now) continue;
|
||
task.logs.push({ time: timestamp(), message: `⏰ 定时任务触发，开始后台采集` });
|
||
this.startTask(task.id, { reset: true, scheduled: true });
|
||
}
|
||
}
|
||
|
||
_emit(taskId, event, data) {
|
||
this.broadcast({ event, taskId, data });
|
||
}
|
||
|
||
_emitStage(task) {
|
||
this._emit(task.id, 'task:stage', { id: task.id, stage: task.stage });
|
||
}
|
||
|
||
_emitProducts(task) {
|
||
this._emit(task.id, 'task:products', {
|
||
id: task.id,
|
||
rawCount: task.products.raw.length,
|
||
coarseCount: task.products.coarseFiltered.length,
|
||
fineCount: task.products.fineFiltered.length,
|
||
});
|
||
}
|
||
}
|