Add hybrid memory retrieval and escalation history
This commit is contained in:
1 parent
1a86bded0b
commit
211de18476
38 files changed
+1602
-37
No files matched your search
@@ -0,0 +1,82 @@
|
||||
export interface BM25Document {
|
||||
id: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface BM25Index {
|
||||
rank(query: string, topK: number): Array<{ id: string; score: number }>;
|
||||
}
|
||||
|
||||
const K1 = 1.5;
|
||||
const B = 0.75;
|
||||
|
||||
function normalize(text: string): string {
|
||||
return text
|
||||
.normalize("NFKC")
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{Letter}\p{Number}\s]/gu, " ");
|
||||
}
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0);
|
||||
const compact = normalize(text).replace(/\s+/gu, "");
|
||||
const grams: string[] = [];
|
||||
for (let i = 0; i < compact.length - 1; i += 1) {
|
||||
grams.push(compact.slice(i, i + 2));
|
||||
}
|
||||
return [...new Set([...words, ...grams])];
|
||||
}
|
||||
|
||||
export function buildBM25Index(documents: BM25Document[]): BM25Index {
|
||||
const termFrequencies = new Map<string, Map<string, number>>();
|
||||
const documentLengths = new Map<string, number>();
|
||||
const documentFrequency = new Map<string, number>();
|
||||
let totalLength = 0;
|
||||
|
||||
for (const document of documents) {
|
||||
const tokens = tokenize(document.text);
|
||||
documentLengths.set(document.id, tokens.length);
|
||||
totalLength += tokens.length;
|
||||
|
||||
const frequencies = new Map<string, number>();
|
||||
for (const token of tokens) {
|
||||
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
|
||||
}
|
||||
termFrequencies.set(document.id, frequencies);
|
||||
|
||||
for (const token of frequencies.keys()) {
|
||||
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const averageLength = documents.length === 0 ? 0 : totalLength / documents.length;
|
||||
|
||||
return {
|
||||
rank(query: string, topK: number): Array<{ id: string; score: number }> {
|
||||
const queryTokens = tokenize(query);
|
||||
if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return documents
|
||||
.map((document) => {
|
||||
const frequencies = termFrequencies.get(document.id) ?? new Map<string, number>();
|
||||
const documentLength = documentLengths.get(document.id) ?? 0;
|
||||
const score = queryTokens.reduce((sum, token) => {
|
||||
const tf = frequencies.get(token) ?? 0;
|
||||
if (tf === 0) {
|
||||
return sum;
|
||||
}
|
||||
const df = documentFrequency.get(token) ?? 0;
|
||||
const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5));
|
||||
const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength));
|
||||
return sum + idf * ((tf * (K1 + 1)) / denominator);
|
||||
}, 0);
|
||||
return { id: document.id, score };
|
||||
})
|
||||
.filter((item) => item.score > 0)
|
||||
.sort((left, right) => right.score - left.score)
|
||||
.slice(0, Math.max(1, topK));
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user