It is commonly used as a ranking function by search engines.
# bind_bm25 is given bare names -------------------
bind_bm25 <- function(tbl, term_col, document_col, n_col, k = 1.2, b = 1) {
bind_bm25_(tbl,
col_name(substitute(term_col)),
col_name(substitute(document_col)),
col_name(substitute(n_col)),
k = k,
b = b)
}
# bind_bm25_ is given strings -------------------------
bind_bm25_ <- function(tbl, term_col, document_col, n_col, k = 1.2, b = 1) {
terms <- tbl[[term_col]]
documents <- tbl[[document_col]]
n <- tbl[[n_col]]
doc_totals <- tapply(n, documents, sum)
avg_dl <- mean(doc_totals)
idf <- log(length(doc_totals) / table(terms))
tbl$tf_bm25 <- ((k+1)*n)/(n+(k*((1-b)+b*(as.numeric(doc_totals[documents])/avg_dl))))
tbl$idf <- as.numeric(idf[terms])
tbl$bm25 <- tbl$tf_bm25 * tbl$idf
tbl
}
I suggest to add a function to bind BM25 score (which is based on a probabilistic term weighting model). It is useful in some cases as it gives control over:
It is commonly used as a ranking function by search engines.
I implemented a function
bind_bm25in the forked repo HERE