Just a little reminder: be careful not to use leave-one-out with a perfectly balanced problem

loo_cv_comparison.R

library(C50)

# Test data
group <- c(rep('groupA', 10), rep('groupB', 10))
data <- data.frame(group = group, var = c(rep(0, 10), rep(0, 10)))

# Leave-one-out
probs <-  lapply(1:nrow(data), function(i) {
    train <- data[-i, ]
    test <- data[i, ]
    model <- C5.0(group ~ ., train)
    return(predict(model, test, type = "prob"))
})

probs <- do.call(rbind, probs)
# OOOOOOOOPS
cat("Wrong!\n")
print(probs)

# 5-fold CV
kfolds <- createFolds(data$group, 5)

probs <- lapply(kfolds, function(i) {
    train <- data[-i, ]
    test <- data[i, ]
    model <- C5.0(group ~ ., train)
    return(predict(model, test, type = "prob"))
})

probs <- do.call(rbind, probs)
probs <- probs[order(as.numeric(rownames(probs))), ]

# Good
cat("Yes!\n")
print(probs)