NLP - prune ngrams by finding the minimum number of ngrams that cover X percent of the word instances

ngram_prune.R

prune_ngram_df_by_cover_percentage <- function(df, percentage) {
  # assumes df contains columns (word, freq)
  # assumes df is sorted by freq in descending order 
  # prune ngrams by finding the minimum number of ngrams that cover X percent of the word instances
  sums <- cumsum(df$freq)
  cover <- which(sums >= sum(df$freq) * percentage)[1]
  print(sprintf("%s of %s (%s%%) cover %s%% of word instances", 
                cover, 
                nrow(df), 
                cover/nrow(df)*100,
                percentage*100))
  
  df[1:cover,]
}