Prob. classifiers
library(ggplot2)
library(learningmachine)
library(caret)
library(palmerpenguins)
library(mlbench)
library(skimr)
library(reshape2)
library(pROC)1 - Using Classifier object
set.seed(43)
X <- as.matrix(iris[, 1:4])
# y <- factor(as.numeric(iris$Species))
y <- iris$Species
index_train <- base::sample.int(n = nrow(X),
size = floor(0.8*nrow(X)),
replace = FALSE)
X_train <- X[index_train, ]
y_train <- y[index_train]
X_test <- X[-index_train, ]
y_test <- y[-index_train]
dim(X_train)## [1] 120 4## [1] 30 4obj <- learningmachine::Classifier$new(method = "ranger",
pi_method="kdesplitconformal",
type_prediction_set="score")
obj$get_type()## [1] "classification"## [1] "Classifier"obj$set_B(10)
obj$set_level(95)
t0 <- proc.time()[3]
obj$fit(X_train, y_train)
cat("Elapsed: ", proc.time()[3] - t0, "s \n")## Elapsed: 0.073 sdf <- reshape2::melt(probs$sims$setosa[1:3, ])
df$Var2 <- NULL
colnames(df) <- c("individual", "prob_setosa")
df$individual <- as.factor(df$individual)
ggplot2::ggplot(df, aes(x=individual, y=prob_setosa)) + geom_boxplot() + coord_flip()
## Sepal.Length Sepal.Width Petal.Length Petal.Width
## [1,] 5.1 3.5 1.4 0.2
## [2,] 4.6 3.1 1.5 0.2
## [3,] 4.9 3.1 1.5 0.1
## [4,] 5.4 3.7 1.5 0.2
## [5,] 5.2 3.5 1.5 0.2
## [6,] 4.8 3.1 1.6 0.2## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## Sepal.Length -0.0393826289 -0.06047290 -0.018292358 0.0006522623 ***
## Sepal.Width 0.0608279754 0.02516647 0.096489483 0.0015709739 **
## Petal.Length -0.0247974949 -0.04713525 -0.002459736 0.0307883308 *
## Petal.Width -0.0005907565 -0.04448090 0.043299382 0.9782267230
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 30
## Number of columns 4
## _______________________
## Column type frequency:
## numeric 4
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75 p100 hist
## 1 Sepal.Length -0.0394 0.0565 -0.214 -0.0726 -0.0103 0.00220 0.0146 ▁▁▃▃▇
## 2 Sepal.Width 0.0608 0.0955 -0.0585 0.000681 0.0424 0.0668 0.386 ▆▇▁▁▁
## 3 Petal.Length -0.0248 0.0598 -0.183 0 0 0 0.0200 ▁▁▁▁▇
## 4 Petal.Width -0.000591 0.118 -0.251 -0.0194 0 0 0.416 ▁▇▂▁▁## Sepal.Length Sepal.Width Petal.Length Petal.Width
## [1,] 5.1 3.5 1.4 0.2
## [2,] 4.6 3.1 1.5 0.2
## [3,] 4.9 3.1 1.5 0.1
## [4,] 5.4 3.7 1.5 0.2
## [5,] 5.2 3.5 1.5 0.2
## [6,] 4.8 3.1 1.6 0.2## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## Sepal.Length -0.04905915 -0.08288493 -0.022702016 1.268525e-83 ***
## Sepal.Width 0.04387477 0.01043957 0.083874087 1.331000e-83 ***
## Petal.Length -0.01558657 -0.04333975 0.003951706 1.653044e-76 ***
## Petal.Width 0.02050216 -0.02171052 0.093805784 3.677758e-48 ***
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 30
## Number of columns 4
## _______________________
## Column type frequency:
## numeric 4
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75 p100 hist
## 1 Sepal.Length -0.0394 0.0565 -0.214 -0.0726 -0.0103 0.00220 0.0146 ▁▁▃▃▇
## 2 Sepal.Width 0.0608 0.0955 -0.0585 0.000681 0.0424 0.0668 0.386 ▆▇▁▁▁
## 3 Petal.Length -0.0248 0.0598 -0.183 0 0 0 0.0200 ▁▁▁▁▇
## 4 Petal.Width -0.000591 0.118 -0.251 -0.0194 0 0 0.416 ▁▇▂▁▁2 - ranger classification
obj <- learningmachine::Classifier$new(method = "ranger",
type_prediction_set="score")
obj$set_level(95)
obj$set_pi_method("bootsplitconformal")
t0 <- proc.time()[3]
obj$fit(X_train, y_train)
cat("Elapsed: ", proc.time()[3] - t0, "s \n")## Elapsed: 0.074 sdf <- reshape2::melt(probs$sims$setosa[1:3, ])
df$Var2 <- NULL
colnames(df) <- c("individual", "prob_setosa")
df$individual <- as.factor(df$individual)
ggplot2::ggplot(df, aes(x=individual, y=prob_setosa)) + geom_boxplot() + coord_flip()
## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## Sepal.Length -0.0393317322 -0.06040555 -0.018257914 0.0006557049 ***
## Sepal.Width 0.0608229848 0.02515105 0.096494916 0.0015763685 **
## Petal.Length -0.0237068539 -0.04523145 -0.002182253 0.0320241179 *
## Petal.Width 0.0008587577 -0.04423383 0.045951343 0.9691971509
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 30
## Number of columns 4
## _______________________
## Column type frequency:
## numeric 4
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75 p100 hist
## 1 Sepal.Length -0.0393 0.0564 -0.214 -0.0727 -0.00980 0.00220 0.0139 ▁▁▃▃▇
## 2 Sepal.Width 0.0608 0.0955 -0.0584 0.000668 0.0424 0.0667 0.387 ▆▇▁▁▁
## 3 Petal.Length -0.0237 0.0576 -0.164 0 0 0 0.0205 ▁▁▁▁▇
## 4 Petal.Width 0.000859 0.121 -0.249 -0.0198 0 0 0.423 ▁▇▁▁▁## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## Sepal.Length 0.01539821 -0.03700392 0.053982986 1.971876e-29 ***
## Sepal.Width -0.05163956 -0.11618377 0.002941184 6.757869e-82 ***
## Petal.Length -0.36646725 -0.86034485 -0.001290106 2.886398e-83 ***
## Petal.Width -0.31840103 -0.82767983 -0.062043001 1.268425e-83 ***
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 30
## Number of columns 4
## _______________________
## Column type frequency:
## numeric 4
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75 p100 hist
## 1 Sepal.Length -0.00212 0.0951 -0.271 -0.0357 -0.00147 0.0626 0.167 ▁▂▆▇▃
## 2 Sepal.Width -0.0927 0.173 -0.797 -0.0919 -0.0709 -0.00538 0.128 ▁▁▂▇▇
## 3 Petal.Length -0.426 0.903 -3.21 -0.192 0 0 0.240 ▁▁▁▁▇
## 4 Petal.Width -0.499 1.20 -5.94 -0.432 -0.0667 0 0 ▁▁▁▁▇3 - extratrees classification
obj <- learningmachine::Classifier$new(method = "extratrees",
pi_method = "bootsplitconformal", type_prediction_set="score")
obj$set_level(95)
t0 <- proc.time()[3]
obj$fit(X_train, y_train)
cat("Elapsed: ", proc.time()[3] - t0, "s \n")## Elapsed: 0.087 sdf <- reshape2::melt(probs$sims$virginica[1:3, ])
df$Var2 <- NULL
colnames(df) <- c("individual", "prob_virginica")
df$individual <- as.factor(df$individual)
ggplot2::ggplot(df, aes(x=individual, y=prob_virginica)) + geom_boxplot() + coord_flip()
## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## Sepal.Length 0.036541255 -0.01605960 0.08914210 0.1660389146
## Sepal.Width -0.008665406 -0.08293934 0.06560852 0.8130836281
## Petal.Length 0.367528646 0.17051921 0.56453808 0.0006587430 ***
## Petal.Width 0.657891051 0.31550619 1.00027591 0.0004837876 ***
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 30
## Number of columns 4
## _______________________
## Column type frequency:
## numeric 4
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75 p100 hist
## 1 Sepal.Length 0.0365 0.141 -0.247 -0.0246 0.00605 0.0720 0.571 ▁▇▂▁▁
## 2 Sepal.Width -0.00867 0.199 -0.378 -0.0921 -0.0117 0.0154 0.629 ▂▇▁▁▁
## 3 Petal.Length 0.368 0.528 -0.00377 0.00117 0.0920 0.525 1.63 ▇▁▁▁▁
## 4 Petal.Width 0.658 0.917 0 0.0426 0.388 0.678 4.17 ▇▁▁▁▁## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## Sepal.Length -0.03655713 -0.06897040 -0.008440761 1.355211e-83 ***
## Sepal.Width 0.11311135 0.04720126 0.171467174 1.268525e-83 ***
## Petal.Length -0.06650335 -0.18681406 0.005096492 1.523688e-82 ***
## Petal.Width -0.08250986 -0.28946732 0.033922868 9.179632e-63 ***
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 30
## Number of columns 4
## _______________________
## Column type frequency:
## numeric 4
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75 p100 hist
## 1 Sepal.Length -0.0310 0.0555 -0.171 -0.0616 -0.00914 0.00495 0.0525 ▂▂▃▇▅
## 2 Sepal.Width 0.118 0.161 -0.0922 0.00719 0.0673 0.240 0.565 ▇▇▆▁▂
## 3 Petal.Length -0.0461 0.148 -0.756 -0.0701 0 0.0142 0.0681 ▁▁▁▁▇
## 4 Petal.Width -0.0757 0.271 -1.38 -0.0676 0 0.0273 0.140 ▁▁▁▁▇4 - Penguins dataset
penguins_ <- as.data.frame(palmerpenguins::penguins)
replacement <- median(penguins$bill_length_mm, na.rm = TRUE)
penguins_$bill_length_mm[is.na(penguins$bill_length_mm)] <- replacement
replacement <- median(penguins$bill_depth_mm, na.rm = TRUE)
penguins_$bill_depth_mm[is.na(penguins$bill_depth_mm)] <- replacement
replacement <- median(penguins$flipper_length_mm, na.rm = TRUE)
penguins_$flipper_length_mm[is.na(penguins$flipper_length_mm)] <- replacement
replacement <- median(penguins$body_mass_g, na.rm = TRUE)
penguins_$body_mass_g[is.na(penguins$body_mass_g)] <- replacement
# replacing NA's by the most frequent occurence
penguins_$sex[is.na(penguins$sex)] <- "male" # most frequent
# one-hot encoding for covariates
penguins_mat <- model.matrix(species ~., data=penguins_)[,-1]
penguins_mat <- cbind.data.frame(penguins_$species, penguins_mat)
penguins_mat <- as.data.frame(penguins_mat)
colnames(penguins_mat)[1] <- "species"
y <- penguins_mat$species
X <- as.matrix(penguins_mat[,2:ncol(penguins_mat)])
n <- nrow(X)
p <- ncol(X)
set.seed(1234)
index_train <- sample(1:n, size=floor(0.8*n))
X_train <- X[index_train, ]
y_train <- factor(y[index_train])
X_test <- X[-index_train, ][1:5, ]
y_test <- factor(y[-index_train][1:5])obj <- learningmachine::Classifier$new(method = "extratrees",
type_prediction_set="score")
obj$set_pi_method("bootsplitconformal")
obj$set_level(95)
obj$set_B(10L)
t0 <- proc.time()[3]
obj$fit(X_train, y_train)
cat("Elapsed: ", proc.time()[3] - t0, "s \n")## Elapsed: 0.149 sdf <- reshape2::melt(probs$sims[[1]][1:3, ])
df$Var2 <- NULL
colnames(df) <- c("individual", "prob_Adelie")
df$individual <- as.factor(df$individual)
ggplot2::ggplot(df, aes(x=individual, y=prob_Adelie)) + geom_boxplot() + coord_flip()
## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## islandDream 0.000000e+00 NaN NaN NaN
## islandTorgersen -1.778468e-02 -3.459725e-02 -9.721090e-04 0.04251686 *
## bill_length_mm 5.776485e-03 -5.537377e-03 1.709035e-02 0.22929028
## bill_depth_mm 3.407051e-03 -1.958522e-03 8.772623e-03 0.15268182
## flipper_length_mm -4.827102e-04 -1.785223e-03 8.198026e-04 0.36165192
## body_mass_g -2.325034e-05 -5.796378e-05 1.146310e-05 0.13646280
## sexmale -7.978043e-03 -2.213090e-02 6.174818e-03 0.19261298
## year -3.352928e-05 -8.057931e-05 1.352074e-05 0.11899484
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 5
## Number of columns 8
## _______________________
## Column type frequency:
## numeric 8
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50
## 1 islandDream 0 0 0 0 0
## 2 islandTorgersen -0.0178 0.0135 -0.0344 -0.0270 -0.0190
## 3 bill_length_mm 0.00578 0.00911 0.000434 0.00100 0.00263
## 4 bill_depth_mm 0.00341 0.00432 -0.00118 0.00175 0.00275
## 5 flipper_length_mm -0.000483 0.00105 -0.00233 -0.000239 -0.0000659
## 6 body_mass_g -0.0000233 0.0000280 -0.0000606 -0.0000442 -0.00000943
## 7 sexmale -0.00798 0.0114 -0.0223 -0.0184 0
## 8 year -0.0000335 0.0000379 -0.0000969 -0.0000349 -0.0000213
## p75 p100 hist
## 1 0 0 ▁▁▇▁▁
## 2 -0.00591 -0.00263 ▃▃▃▁▇
## 3 0.00284 0.0220 ▇▁▁▁▂
## 4 0.00320 0.0105 ▂▇▁▁▂
## 5 -0.0000350 0.000258 ▂▁▁▁▇
## 6 -0.00000863 0.00000657 ▃▃▁▇▃
## 7 0 0.000842 ▅▁▁▁▇
## 8 -0.0000173 0.00000271 ▂▁▁▇▂rvfl
obj <- learningmachine::Classifier$new(method = "rvfl",
type_prediction_set="score")
obj$set_pi_method("bootsplitconformal")
obj$set_level(95)
obj$set_B(10L)
t0 <- proc.time()[3]
obj$fit(X_train, y_train)
cat("Elapsed: ", proc.time()[3] - t0, "s \n")## Elapsed: 0.009 sdf <- reshape2::melt(probs$sims[[1]][1:3, ])
df$Var2 <- NULL
colnames(df) <- c("individual", "prob_Adelie")
df$individual <- as.factor(df$individual)
ggplot2::ggplot(df, aes(x=individual, y=prob_Adelie)) + geom_boxplot() + coord_flip()
## $Coverage_rate
## [1] 100
##
## $citests
## estimate lower upper p-value signif
## islandDream 0.4802248563 0.4780044748 0.4824452379 4.614437e-11 ***
## islandTorgersen -0.2959594072 -0.3063930678 -0.2855257465 1.557925e-07 ***
## bill_length_mm 0.2003956489 0.1963455026 0.2044457951 1.684113e-08 ***
## bill_depth_mm 0.0508209573 0.0479212470 0.0537206676 1.067137e-06 ***
## flipper_length_mm -0.0224653400 -0.0227350465 -0.0221956335 2.097271e-09 ***
## body_mass_g -0.0001692677 -0.0001723101 -0.0001662252 1.053542e-08 ***
## sexmale -0.4310257052 -0.4393042529 -0.4227471575 1.373600e-08 ***
## year -0.0102992883 -0.0110718998 -0.0095266769 3.181998e-06 ***
##
## $signif_codes
## [1] "Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
##
## $effects
## ── Data Summary ────────────────────────
## Values
## Name effects
## Number of rows 5
## Number of columns 8
## _______________________
## Column type frequency:
## numeric 8
## ________________________
## Group variables None
##
## ── Variable type: numeric ──────────────────────────────────────────────────────
## skim_variable mean sd p0 p25 p50 p75
## 1 islandDream 0.480 0.00179 0.478 0.479 0.481 0.482
## 2 islandTorgersen -0.296 0.00840 -0.311 -0.294 -0.293 -0.292
## 3 bill_length_mm 0.200 0.00326 0.198 0.199 0.200 0.200
## 4 bill_depth_mm 0.0508 0.00234 0.0467 0.0512 0.0516 0.0521
## 5 flipper_length_mm -0.0225 0.000217 -0.0226 -0.0226 -0.0226 -0.0224
## 6 body_mass_g -0.000169 0.00000245 -0.000173 -0.000169 -0.000169 -0.000168
## 7 sexmale -0.431 0.00667 -0.442 -0.430 -0.429 -0.427
## 8 year -0.0103 0.000622 -0.0112 -0.0105 -0.0102 -0.00980
## p100 hist
## 1 0.482 ▂▂▁▁▇
## 2 -0.289 ▂▁▁▂▇
## 3 0.206 ▇▇▁▁▃
## 4 0.0524 ▂▁▁▂▇
## 5 -0.0221 ▇▁▂▁▂
## 6 -0.000167 ▃▁▁▇▇
## 7 -0.426 ▃▁▁▇▇
## 8 -0.00972 ▃▁▃▃▇