memind/scripts/simulate-persona-intent-routing.mjs

#!/usr/bin/env node
/**
 * Simulate multi-persona chat utterances against the live chat intent router.
 *
 * Usage:
 *   node scripts/simulate-persona-intent-routing.mjs
 *   node scripts/run-scenario-test.mjs --scenario persona-intent-routing
 */
import fs from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { createDbPool } from '../db.mjs';
import { createLlmProviderService } from '../llm-providers.mjs';
import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs';
import {
  createChatIntentRouter,
  createManagedChatIntentRouter,
} from '../chat-intent-router.mjs';

const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
const JOHN_USER_ID = 'a6fb1e97-2b0f-447b-b138-4561d8e5c53e';

function sourceMatches(actual, expected) {
  if (!expected) return true;
  return String(actual ?? '') === String(expected);
}

export async function loadPersonaIntentCases(scenarioId = 'persona-intent-routing') {
  const scenarioPath = path.join(repoRoot, 'scenarios', `${scenarioId}.json`);
  const scenario = JSON.parse(await fs.readFile(scenarioPath, 'utf8'));
  return { scenario, cases: scenario.cases ?? [] };
}

export async function runPersonaIntentRouting({
  userId = JOHN_USER_ID,
  scenarioId = 'persona-intent-routing',
  reporter,
  env = process.env,
} = {}) {
  const { scenario, cases } = await loadPersonaIntentCases(scenarioId);
  if (!cases.length) {
    throw new Error(`场景 ${scenarioId} 没有 cases`);
  }

  const pool = createDbPool(env);
  const llmProviderService = createLlmProviderService(pool, {
    apiTarget: env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
    apiSecret: env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
  });
  const configService = createMemoryV2AdminConfigService(pool, { env });
  const adminRouter = createManagedChatIntentRouter({
    llmProviderService,
    configService,
    env,
    logger: { warn() {}, log() {}, info() {} },
  });
  const adminStatus = await adminRouter.getStatus().catch(() => null);
  // Local simulation follows .env (defer + LLM router). Admin-db currently
  // disables LLM routing and would swallow ambiguous tasks into fallback chat.
  const router = createChatIntentRouter({
    llmProviderService,
    env,
    logger: console,
  });

  try {
    const status = router.getStatus();
    reporter?.pass?.(
      'router 状态',
      `env llm=${status.llmRoutingEnabled ? 'on' : 'off'} shadow=${status.llmRoutingShadow ? 'on' : 'off'}；admin-db llm=${adminStatus?.llmRoutingEnabled ? 'on' : 'off'}`,
    );
    if (!status.llmRoutingEnabled) {
      reporter?.fail?.('LLM router', '本机 .env 未进入生效模式，歧义句无法精准识别');
    } else if (adminStatus && !adminStatus.llmRoutingEnabled) {
      reporter?.pass?.(
        'admin-db 对照',
        '管理后台当前关闭 LLM router，Portal 热路径仍会把歧义句打成 fallback 直聊；本模拟按 .env 验证识别精度',
      );
    }

    const rows = [];
    for (const item of cases) {
      const classification = await router.classify({
        userId,
        sessionId: 'h5direct_persona_sim',
        sessionMessageCount: 3,
        toolMode: 'chat',
        userMessage: {
          role: 'user',
          content: [{ type: 'text', text: item.text }],
          metadata: { displayText: item.text, userVisible: true },
        },
      });
      const routeOk = classification?.route === item.expectRoute;
      const sourceOk = item.expectSource === 'llm'
        ? classification?.source === 'llm' || classification?.source === 'rule'
        : sourceMatches(classification?.source, item.expectSource);
      // Rule-expected cases must keep rule source. LLM-expected cases may still
      // be caught by a stronger rule (page/news); that is acceptable if route matches.
      const sourceStrictOk = item.expectSource === 'rule'
        ? classification?.source === 'rule'
        : true;
      const ok = Boolean(routeOk && sourceStrictOk);
      const label = `${item.persona} · ${item.intent}`;
      const detail = `${classification?.route ?? 'null'}/${classification?.source ?? 'none'} conf=${classification?.confidence ?? '-'} · ${classification?.reason ?? ''}`;
      if (ok) reporter?.pass?.(label, `${item.text} → ${detail}`);
      else {
        reporter?.fail?.(
          label,
          `期望 ${item.expectRoute}/${item.expectSource ?? '*'}，实际 ${detail}；话术：${item.text}`,
        );
      }
      rows.push({
        ...item,
        ok,
        routeOk,
        sourceOk,
        actualRoute: classification?.route ?? null,
        actualSource: classification?.source ?? null,
        confidence: classification?.confidence ?? null,
        reason: classification?.reason ?? null,
      });
    }

    const failed = rows.filter((row) => !row.ok);
    console.log('\n--- 人群 × 意图 ---');
    for (const row of rows) {
      const mark = row.ok ? '✔' : '✘';
      console.log(
        `${mark} [${row.persona}] ${row.intent}: ${row.actualRoute} (${row.actualSource}) <- ${row.text}`,
      );
    }
    return {
      scenario,
      rows,
      failed,
      ok: failed.length === 0,
    };
  } finally {
    await pool.end?.();
  }
}

async function main() {
  const { createReporter } = await import('./scenario-test-lib.mjs');
  const { loadH5Environment } = await import('./load-env.mjs');
  loadH5Environment(import.meta.dirname);
  const reporter = createReporter();
  const result = await runPersonaIntentRouting({
    reporter,
    userId: process.env.MEMIND_E2E_USER_ID ?? JOHN_USER_ID,
  });
  const code = reporter.summary();
  process.exit(result.ok ? code : 1);
}

const isDirectRun = process.argv[1]
  && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
if (isDirectRun) {
  main().catch((error) => {
    console.error(error instanceof Error ? error.stack ?? error.message : error);
    process.exit(1);
  });
}