83 lines
2.7 KiB
TypeScript
83 lines
2.7 KiB
TypeScript
export interface BM25Document {
|
|
id: string;
|
|
text: string;
|
|
}
|
|
|
|
export interface BM25Index {
|
|
rank(query: string, topK: number): Array<{ id: string; score: number }>;
|
|
}
|
|
|
|
const K1 = 1.5;
|
|
const B = 0.75;
|
|
|
|
function normalize(text: string): string {
|
|
return text
|
|
.normalize("NFKC")
|
|
.toLowerCase()
|
|
.replace(/[^\p{Letter}\p{Number}\s]/gu, " ");
|
|
}
|
|
|
|
function tokenize(text: string): string[] {
|
|
const words = normalize(text).split(/\s+/u).filter((token) => token.length > 0);
|
|
const compact = normalize(text).replace(/\s+/gu, "");
|
|
const grams: string[] = [];
|
|
for (let i = 0; i < compact.length - 1; i += 1) {
|
|
grams.push(compact.slice(i, i + 2));
|
|
}
|
|
return [...new Set([...words, ...grams])];
|
|
}
|
|
|
|
export function buildBM25Index(documents: BM25Document[]): BM25Index {
|
|
const termFrequencies = new Map<string, Map<string, number>>();
|
|
const documentLengths = new Map<string, number>();
|
|
const documentFrequency = new Map<string, number>();
|
|
let totalLength = 0;
|
|
|
|
for (const document of documents) {
|
|
const tokens = tokenize(document.text);
|
|
documentLengths.set(document.id, tokens.length);
|
|
totalLength += tokens.length;
|
|
|
|
const frequencies = new Map<string, number>();
|
|
for (const token of tokens) {
|
|
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
|
|
}
|
|
termFrequencies.set(document.id, frequencies);
|
|
|
|
for (const token of frequencies.keys()) {
|
|
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
|
|
}
|
|
}
|
|
|
|
const averageLength = documents.length === 0 ? 0 : totalLength / documents.length;
|
|
|
|
return {
|
|
rank(query: string, topK: number): Array<{ id: string; score: number }> {
|
|
const queryTokens = tokenize(query);
|
|
if (queryTokens.length === 0 || documents.length === 0 || averageLength === 0) {
|
|
return [];
|
|
}
|
|
|
|
return documents
|
|
.map((document) => {
|
|
const frequencies = termFrequencies.get(document.id) ?? new Map<string, number>();
|
|
const documentLength = documentLengths.get(document.id) ?? 0;
|
|
const score = queryTokens.reduce((sum, token) => {
|
|
const tf = frequencies.get(token) ?? 0;
|
|
if (tf === 0) {
|
|
return sum;
|
|
}
|
|
const df = documentFrequency.get(token) ?? 0;
|
|
const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5));
|
|
const denominator = tf + K1 * (1 - B + B * (documentLength / averageLength));
|
|
return sum + idf * ((tf * (K1 + 1)) / denominator);
|
|
}, 0);
|
|
return { id: document.id, score };
|
|
})
|
|
.filter((item) => item.score > 0)
|
|
.sort((left, right) => right.score - left.score)
|
|
.slice(0, Math.max(1, topK));
|
|
},
|
|
};
|
|
}
|