Files
lolympus-agent/src/memory/bm25.ts
T

83 lines
2.7 KiB
TypeScript

export interface BM25Document {
id: string;
text: string;
}
export interface BM25Index {
rank(query: string, topK: number): Array<{ id: string; score: number }>;
}
const K1 = 1.5;
const B = 0.75;
function normalize(text: string): string {
return text
.normalize("NFKC")
.toLowerCase()
.replace(/[^\p{Letter}\p{Number}\s]/gu, " ");
}
function tokenize(text: string): string[] {
const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0);
const compact = normalize(text).replace(/\s+/gu, "");
const grams: string[] = [];
for (let i = 0; i < compact.length - 1; i += 1) {
grams.push(compact.slice(i, i + 2));
}
return [...new Set([...words, ...grams])];
}
export function buildBM25Index(documents: BM25Document[]): BM25Index {
const termFrequencies = new Map<string, Map<string, number>>();
const documentLengths = new Map<string, number>();
const documentFrequency = new Map<string, number>();
let totalLength = 0;
for (const document of documents) {
const tokens = tokenize(document.text);
documentLengths.set(document.id, tokens.length);
totalLength += tokens.length;
const frequencies = new Map<string, number>();
for (const token of tokens) {
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
}
termFrequencies.set(document.id, frequencies);
for (const token of frequencies.keys()) {
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
}
}
const averageLength = documents.length === 0 ? 0 : totalLength / documents.length;
return {
rank(query: string, topK: number): Array<{ id: string; score: number }> {
const queryTokens = tokenize(query);
if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) {
return [];
}
return documents
.map((document) => {
const frequencies = termFrequencies.get(document.id) ?? new Map<string, number>();
const documentLength = documentLengths.get(document.id) ?? 0;
const score = queryTokens.reduce((sum, token) => {
const tf = frequencies.get(token) ?? 0;
if (tf === 0) {
return sum;
}
const df = documentFrequency.get(token) ?? 0;
const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5));
const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength));
return sum + idf * ((tf * (K1 + 1)) / denominator);
}, 0);
return { id: document.id, score };
})
.filter((item) => item.score > 0)
.sort((left, right) => right.score - left.score)
.slice(0, Math.max(1, topK));
},
};
}