memind/memory-v2-personal-shadow.mjs

import crypto from 'node:crypto';
import { deriveUserFacingText } from './conversation-display.mjs';

const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']);
const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']);
const SECRET_PATTERNS = [
  /\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b\s*[:=]/i,
  /\bsk-[a-z0-9_-]{12,}\b/i,
  /-----BEGIN [A-Z ]+PRIVATE KEY-----/,
  /(?:密码|口令|密钥|令牌)\s*[：:=]\s*\S+/i,
];
const TRIVIAL_PATTERNS = [
  /^(?:你好|您好|谢谢|好的|可以|收到|再见|hi|hello|thanks)[！!。.\s]*$/i,
  /^(?:查一下|搜一下|看看|继续|下一步)[。.!！?？\s]*$/i,
];

function readFlag(env, key, fallback = false) {
  const raw = env?.[key];
  if (raw == null || raw === '') return fallback;
  const normalized = String(raw).trim().toLowerCase();
  if (TRUE_VALUES.has(normalized)) return true;
  if (FALSE_VALUES.has(normalized)) return false;
  return fallback;
}

function readNumber(env, key, fallback, { min = -Infinity, max = Infinity } = {}) {
  const value = Number(env?.[key]);
  if (!Number.isFinite(value)) return fallback;
  return Math.min(max, Math.max(min, value));
}

function normalizeText(value, maxLength = 2000) {
  const text = String(value ?? '').replace(/\s+/g, ' ').trim();
  if (!text) return '';
  return text.length > maxLength ? text.slice(0, maxLength) : text;
}

function messageText(message) {
  if (typeof message === 'string') return normalizeText(message);
  if (!message || typeof message !== 'object') return '';
  const displayText = message.displayText
    ?? message.display_text
    ?? message.metadata?.displayText
    ?? message.metadata?.display_text;
  if (typeof displayText === 'string' && displayText.trim()) {
    return normalizeText(displayText);
  }
  const direct = message.text ?? message.content ?? message.message ?? '';
  if (typeof direct === 'string') return normalizeText(deriveUserFacingText(direct));
  if (Array.isArray(direct)) {
    return normalizeText(deriveUserFacingText(
      direct.map((item) => item?.text ?? item?.content ?? '').join('\n'),
    ));
  }
  return '';
}

function stableHash(value) {
  return crypto.createHash('sha256').update(String(value)).digest('hex');
}

function classify(text) {
  const rules = [
    { type: 'episodic', importance: 0.95, confidence: 0.98, pattern: /(?:请记住|记住|以后要|从现在起)/i, reason: 'explicit_memory_request' },
    { type: 'episodic', importance: 0.9, confidence: 0.9, pattern: /(?:决定|确定|统一采用|必须|不允许|禁止|最终选择)/i, reason: 'decision_signal' },
    { type: 'preference', importance: 0.8, confidence: 0.88, pattern: /(?:我喜欢|我偏好|我不喜欢|我的习惯|倾向于|更希望)/i, reason: 'preference_signal' },
    { type: 'goal', importance: 0.85, confidence: 0.86, pattern: /(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i, reason: 'goal_signal' },
    { type: 'semantic', importance: 0.75, confidence: 0.82, pattern: /(?:技术栈|架构原则|项目使用|系统采用|仓库位于|运行在)/i, reason: 'stable_fact_signal' },
  ];
  return rules.find((rule) => rule.pattern.test(text)) ?? null;
}

const CANDIDATE_MODES = new Set(['off', 'shadow', 'canary', 'active']);

function normalizeCandidateMode(value, fallback = 'shadow') {
  const normalized = normalizeText(value, 20).toLowerCase();
  return CANDIDATE_MODES.has(normalized) ? normalized : fallback;
}

export function shouldAutoAcceptCandidate(candidate, config) {
  if (!candidate || !config?.enabled) return false;
  const mode = normalizeCandidateMode(config.requestedMode, 'shadow');
  if (mode === 'shadow' || mode === 'off') return false;
  if (mode === 'active') return true;
  if (candidate.policyReason === 'explicit_memory_request') return true;
  return Number(candidate.confidence) >= Math.max(Number(config.minConfidence) || 0, 0.9);
}

export function resolvePersonalShadowConfig(env = process.env) {
  const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false);
  const requestedMode = normalizeCandidateMode(
    env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'active' : 'off'),
    candidateEnabled ? 'active' : 'off',
  );
  const enabled = candidateEnabled && requestedMode !== 'off';
  return {
    enabled,
    requestedMode,
    effectiveMode: enabled ? requestedMode : 'off',
    autoReviewEnabled: enabled && requestedMode !== 'shadow',
    minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }),
    minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }),
    maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })),
    policyEnabled: readFlag(env, 'MEMORY_POLICY_ENABLED', true),
    saveExplicit: readFlag(env, 'MEMORY_POLICY_SAVE_EXPLICIT', true),
    rejectSensitive: readFlag(env, 'MEMORY_POLICY_REJECT_SENSITIVE', true),
    requireEvidence: readFlag(env, 'MEMORY_POLICY_REQUIRE_EVIDENCE', true),
    healthEnabled: readFlag(env, 'MEMORY_PLUGIN_HEALTH_ENABLED', true),
  };
}

export function createPersonalMemoryShadowPipeline({ env = process.env, now = () => Date.now(), store = null } = {}) {
  const config = resolvePersonalShadowConfig(env);
  const candidates = [];
  const candidateHashes = new Set();
  const metrics = {
    runs: 0,
    observedMessages: 0,
    accepted: 0,
    autoReviewed: 0,
    pendingReview: 0,
    rejected: 0,
    deduped: 0,
    errors: 0,
    lastRunAt: null,
    lastError: null,
  };

  function reject(reason) {
    metrics.rejected += 1;
    return { accepted: false, reason };
  }

  function evaluate({ userId, sessionId, message, index }) {
    const text = messageText(message);
    if (!text || text.length < 8) return reject('too_short');
    if (TRIVIAL_PATTERNS.some((pattern) => pattern.test(text))) return reject('trivial');
    if (config.rejectSensitive && SECRET_PATTERNS.some((pattern) => pattern.test(text))) {
      return reject('sensitive_content');
    }
    const classification = classify(text);
    if (!classification) return reject('no_durable_signal');
    if (classification.reason === 'explicit_memory_request' && !config.saveExplicit) {
      return reject('explicit_memory_disabled');
    }
    if (config.policyEnabled && classification.importance < config.minImportance) {
      return reject('below_importance_threshold');
    }
    if (config.policyEnabled && classification.confidence < config.minConfidence) {
      return reject('below_confidence_threshold');
    }
    const contentHash = stableHash(`${userId}\n${classification.type}\n${text.toLowerCase()}`);
    if (candidateHashes.has(contentHash)) {
      metrics.deduped += 1;
      return { accepted: false, reason: 'duplicate' };
    }
    const capturedAt = now();
    const evidence = {
      sourceType: 'message',
      sourceId: `${sessionId || 'unknown'}:${index}`,
      evidenceHash: stableHash(`${sessionId || ''}\n${index}\n${text}`),
      capturedAt,
    };
    if (config.requireEvidence && !evidence.sourceId) return reject('evidence_required');
    const candidate = {
      id: `pmc_${contentHash.slice(0, 24)}`,
      userId: String(userId),
      sessionId: sessionId ? String(sessionId) : null,
      memoryType: classification.type,
      content: text,
      importance: classification.importance,
      confidence: classification.confidence,
      status: 'candidate',
      policyReason: classification.reason,
      evidence,
      createdAt: capturedAt,
    };
    candidateHashes.add(contentHash);
    candidates.push(candidate);
    while (candidates.length > config.maxPending) {
      const removed = candidates.shift();
      if (removed) candidateHashes.delete(stableHash(`${removed.userId}\n${removed.memoryType}\n${removed.content.toLowerCase()}`));
    }
    metrics.accepted += 1;
    return { accepted: true, candidate };
  }

  async function observeWrite({ userId, sessionId, messages = [] } = {}) {
    if (!config.enabled) return { enabled: false, skipped: true, reason: 'disabled' };
    metrics.runs += 1;
    metrics.lastRunAt = now();
    if (!userId || !Array.isArray(messages)) {
      return { enabled: true, skipped: true, reason: 'invalid_input' };
    }
    try {
      const results = [];
      for (let index = 0; index < messages.length; index += 1) {
        const message = messages[index];
        const role = normalizeText(message?.role ?? message?.sender ?? '', 30).toLowerCase();
        if (role && !['user', 'human'].includes(role)) continue;
        metrics.observedMessages += 1;
        const result = evaluate({ userId, sessionId, message, index });
        if (result.accepted && store?.saveCandidate) {
          const autoAccept = shouldAutoAcceptCandidate(result.candidate, config);
          const persisted = await store.saveCandidate(result.candidate, { autoAccept });
          if (persisted?.inserted === false) {
            metrics.deduped += 1;
          } else if (autoAccept) {
            result.candidate.status = 'accepted';
            metrics.autoReviewed += 1;
          } else {
            metrics.pendingReview += 1;
          }
        } else if (result.accepted) {
          const autoAccept = shouldAutoAcceptCandidate(result.candidate, config);
          result.candidate.status = autoAccept ? 'accepted' : 'candidate';
          if (autoAccept) metrics.autoReviewed += 1;
          else metrics.pendingReview += 1;
        }
        results.push(result);
      }
      return {
        enabled: true,
        skipped: false,
        mode: config.effectiveMode,
        autoReviewEnabled: config.autoReviewEnabled,
        observed: results.length,
        accepted: results.filter((result) => result.accepted).length,
        autoReviewed: results.filter((result) => result.accepted && result.candidate?.status === 'accepted').length,
        pendingReview: results.filter((result) => result.accepted && result.candidate?.status === 'candidate').length,
        rejected: results.filter((result) => !result.accepted).length,
      };
    } catch (err) {
      metrics.errors += 1;
      metrics.lastError = err instanceof Error ? err.message : String(err);
      return { enabled: true, skipped: true, reason: 'pipeline_failure' };
    }
  }

  function getStatus() {
    return {
      enabled: config.enabled,
      requestedMode: config.requestedMode,
      effectiveMode: config.effectiveMode,
      phase: config.autoReviewEnabled ? 'auto-review-v1' : 'shadow-candidate-v1',
      autoReviewEnabled: config.autoReviewEnabled,
      injectionEnabled: false,
      persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory',
      pendingCandidates: candidates.length,
      health: {
        enabled: config.healthEnabled,
        state: metrics.errors > 0 ? 'degraded' : 'healthy',
        ...metrics,
      },
      policy: {
        enabled: config.policyEnabled,
        minImportance: config.minImportance,
        minConfidence: config.minConfidence,
        rejectSensitive: config.rejectSensitive,
        requireEvidence: config.requireEvidence,
      },
    };
  }

  function listCandidates({ limit = 50 } = {}) {
    return candidates.slice(-Math.max(1, Math.min(200, Number(limit) || 50))).reverse();
  }

  return { config, observeWrite, getStatus, listCandidates };
}