export interface BM25Document { id: string; text: string; } export interface BM25Index { rank(query: string, topK: number): Array<{ id: string; score: number }>; } const K1 = 1.5; const B = 0.75; function normalize(text: string): string { return text .normalize("NFKC") .toLowerCase() .replace(/[^\p{Letter}\p{Number}\s]/gu, " "); } function tokenize(text: string): string[] { const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0); const compact = normalize(text).replace(/\s+/gu, ""); const grams: string[] = []; for (let i = 0; i < compact.length - 1; i += 1) { grams.push(compact.slice(i, i + 2)); } return [...new Set([...words, ...grams])]; } export function buildBM25Index(documents: BM25Document[]): BM25Index { const termFrequencies = new Map>(); const documentLengths = new Map(); const documentFrequency = new Map(); let totalLength = 0; for (const document of documents) { const tokens = tokenize(document.text); documentLengths.set(document.id, tokens.length); totalLength += tokens.length; const frequencies = new Map(); for (const token of tokens) { frequencies.set(token, (frequencies.get(token) ?? 0) + 1); } termFrequencies.set(document.id, frequencies); for (const token of frequencies.keys()) { documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1); } } const averageLength = documents.length === 0 ? 0 : totalLength / documents.length; return { rank(query: string, topK: number): Array<{ id: string; score: number }> { const queryTokens = tokenize(query); if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) { return []; } return documents .map((document) => { const frequencies = termFrequencies.get(document.id) ?? new Map(); const documentLength = documentLengths.get(document.id) ?? 0; const score = queryTokens.reduce((sum, token) => { const tf = frequencies.get(token) ?? 0; if (tf === 0) { return sum; } const df = documentFrequency.get(token) ?? 0; const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5)); const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength)); return sum + idf * ((tf * (K1 + 1)) / denominator); }, 0); return { id: document.id, score }; }) .filter((item) => item.score > 0) .sort((left, right) => right.score - left.score) .slice(0, Math.max(1, topK)); }, }; }