79 lines
2.8 KiB
JavaScript
79 lines
2.8 KiB
JavaScript
|
const CJK_QUERY_STOP_WORDS = new Set([
|
|||
|
|
'之前', '什么', '哪个', '多少', '记得', '记住', '有没有', '是否', '我的', '你是',
|
|||
|
|
'我们', '一下', '可以', '这个', '那个', '请问', '告诉', '说过', '提到', '聊过',
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
function extractQueryTokens(query) {
|
|||
|
|
const q = String(query ?? '').trim();
|
|||
|
|
if (!q) return [];
|
|||
|
|
const tokens = new Set();
|
|||
|
|
for (const part of q.replace(/[^\p{L}\p{N}]+/gu, ' ').split(/\s+/)) {
|
|||
|
|
const token = part.trim();
|
|||
|
|
if (token.length >= 2 && !CJK_QUERY_STOP_WORDS.has(token)) tokens.add(token);
|
|||
|
|
}
|
|||
|
|
for (const run of q.match(/[\p{Script=Han}]{2,}/gu) ?? []) {
|
|||
|
|
if (run.length <= 6 && !CJK_QUERY_STOP_WORDS.has(run)) tokens.add(run);
|
|||
|
|
for (let i = 0; i < run.length - 1; i += 1) {
|
|||
|
|
const bigram = run.slice(i, i + 2);
|
|||
|
|
if (!CJK_QUERY_STOP_WORDS.has(bigram)) tokens.add(bigram);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return [...tokens];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function scoreMemoryForRecall(text, tokens) {
|
|||
|
|
let score = 0;
|
|||
|
|
for (const token of tokens) {
|
|||
|
|
if (text.includes(token)) score += 1;
|
|||
|
|
}
|
|||
|
|
if (/别名.{0,8}是/u.test(text)) score += 5;
|
|||
|
|
if (/[？?]\s*$/.test(text)) score -= 3;
|
|||
|
|
if (/^用户记住/u.test(text)) score -= 2;
|
|||
|
|
return score;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export function filterMemoriesByQuery(memories, query) {
|
|||
|
|
const items = Array.isArray(memories) ? memories : [];
|
|||
|
|
const tokens = extractQueryTokens(query);
|
|||
|
|
if (!tokens.length || !items.length) return items.slice(0, Math.min(items.length, 5));
|
|||
|
|
const matched = items
|
|||
|
|
.map((item) => {
|
|||
|
|
const text = String(item?.text ?? item?.memory_text ?? '').trim();
|
|||
|
|
return { item, text, score: scoreMemoryForRecall(text, tokens) };
|
|||
|
|
})
|
|||
|
|
.filter(({ text, score }) => text && score > 0)
|
|||
|
|
.sort((a, b) => b.score - a.score)
|
|||
|
|
.map(({ item }) => item);
|
|||
|
|
return matched.length ? matched.slice(0, 5) : items.slice(0, 5);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export async function resolveMemoriesWithLegacyFallback({
|
|||
|
|
memoryV2 = null,
|
|||
|
|
conversationMemoryService = null,
|
|||
|
|
userId,
|
|||
|
|
sessionId = null,
|
|||
|
|
query = null,
|
|||
|
|
limit = 20,
|
|||
|
|
recallQuestion = false,
|
|||
|
|
} = {}) {
|
|||
|
|
if (!userId || limit <= 0) return [];
|
|||
|
|
if (recallQuestion && conversationMemoryService?.listMemories) {
|
|||
|
|
const legacy = await conversationMemoryService.listMemories(userId, { limit }).catch(() => []);
|
|||
|
|
const filtered = filterMemoriesByQuery(legacy, query);
|
|||
|
|
if (filtered.length) return filtered;
|
|||
|
|
}
|
|||
|
|
if (memoryV2?.resolve) {
|
|||
|
|
const resolved = await memoryV2.resolve({
|
|||
|
|
userId,
|
|||
|
|
sessionId,
|
|||
|
|
query,
|
|||
|
|
limit,
|
|||
|
|
}).catch(() => null);
|
|||
|
|
const memories = Array.isArray(resolved?.memories) ? resolved.memories : [];
|
|||
|
|
if (memories.length) return memories;
|
|||
|
|
}
|
|||
|
|
if (!conversationMemoryService?.listMemories) return [];
|
|||
|
|
const legacy = await conversationMemoryService.listMemories(userId, { limit }).catch(() => []);
|
|||
|
|
return filterMemoriesByQuery(legacy, query);
|
|||
|
|
}
|