Examples for regression, time series, and classification.
Linear model
# 'X' contains the explanatory variables
# 'y' is the response
# 'k' is the number of folds in k-fold cross-validation
# 'repeats' is the number of repeats of the k-fold cross-validation procedure
# linear model example -----
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3, show_progress = FALSE)## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.0657917 0.7188426 1.0338031
## fold_2 1.0253419 0.6820600 0.7632532
## fold_3 0.8726629 1.0294080 0.7830872
## fold_4 0.8899336 1.2441376 0.9776853
## fold_5 0.9724975 1.0578021 1.0921075
##
## $mean
## [1] 0.9472276
##
## $sd
## [1] 0.15799
##
## $median
## [1] 0.9776853
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.8727 Min. :0.6821 Min. :0.7633
## 1st Qu.:0.8899 1st Qu.:0.7188 1st Qu.:0.7831
## Median :0.9725 Median :1.0294 Median :0.9777
## Mean :0.9652 Mean :0.9465 Mean :0.9300
## 3rd Qu.:1.0253 3rd Qu.:1.0578 3rd Qu.:1.0338
## Max. :1.0658 Max. :1.2441 Max. :1.0921# linear model example, with validation set
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3, p = 0.8, show_progress = FALSE)## $folds
## repeat_1 repeat_2 repeat_3
## fold_training_1 0.8589887 0.6618209 0.8028635
## fold_validation_1 1.0689051 1.1206828 1.1719136
## fold_training_2 0.7093379 1.0424256 0.9199860
## fold_validation_2 1.0827333 1.1363834 1.0731990
## fold_training_3 1.0489754 1.1053316 1.1602932
## fold_validation_3 1.2271907 1.1456095 1.0881138
## fold_training_4 0.9703262 0.9576335 0.8823505
## fold_validation_4 1.1098708 1.0933379 1.0563444
## fold_training_5 1.0689683 0.9692094 1.1509957
## fold_validation_5 1.0925497 1.0641650 1.1304265
##
## $mean_training
## [1] 0.9539671
##
## $mean_validation
## [1] 1.110762
##
## $sd_training
## [1] 0.1507651
##
## $sd_validation
## [1] 0.04619929
##
## $median_training
## [1] 0.9692094
##
## $median_validation
## [1] 1.093338
##
## $summary_training
## repeat_1 repeat_2 repeat_3
## Min. :0.7093 Min. :0.6618 Min. :0.8029
## 1st Qu.:0.8590 1st Qu.:0.9576 1st Qu.:0.8824
## Median :0.9703 Median :0.9692 Median :0.9200
## Mean :0.9313 Mean :0.9473 Mean :0.9833
## 3rd Qu.:1.0490 3rd Qu.:1.0424 3rd Qu.:1.1510
## Max. :1.0690 Max. :1.1053 Max. :1.1603
##
## $summary_validation
## repeat_1 repeat_2 repeat_3
## Min. :1.069 Min. :1.064 Min. :1.056
## 1st Qu.:1.083 1st Qu.:1.093 1st Qu.:1.073
## Median :1.093 Median :1.121 Median :1.088
## Mean :1.116 Mean :1.112 Mean :1.104
## 3rd Qu.:1.110 3rd Qu.:1.136 3rd Qu.:1.130
## Max. :1.227 Max. :1.146 Max. :1.172glmnet
# glmnet example -----
# fit glmnet, with alpha = 1, lambda = 0.1
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3, show_progress = FALSE,
fit_func = glmnet::glmnet, predict_func = predict.glmnet,
packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0.5, lambda = 0.1))## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.0514770 0.7505046 1.0142623
## fold_2 1.0195407 0.6996108 0.7214830
## fold_3 0.8858994 1.0033483 0.7838630
## fold_4 0.8101031 1.2110573 0.9747158
## fold_5 0.9562564 1.0257926 1.0952526
##
## $mean
## [1] 0.9335445
##
## $sd
## [1] 0.1511116
##
## $median
## [1] 0.9747158
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.8101 Min. :0.6996 Min. :0.7215
## 1st Qu.:0.8859 1st Qu.:0.7505 1st Qu.:0.7839
## Median :0.9563 Median :1.0033 Median :0.9747
## Mean :0.9447 Mean :0.9381 Mean :0.9179
## 3rd Qu.:1.0195 3rd Qu.:1.0258 3rd Qu.:1.0143
## Max. :1.0515 Max. :1.2111 Max. :1.0953# fit glmnet, with alpha = 0, lambda = 0.01
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3, show_progress = FALSE,
fit_func = glmnet::glmnet, predict_func = predict.glmnet,
packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0, lambda = 0.01))## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.0605785 0.7192835 1.0253503
## fold_2 1.0266807 0.6757645 0.7594241
## fold_3 0.8895544 1.0344944 0.7733901
## fold_4 0.8938092 1.2470820 1.0042933
## fold_5 0.9641163 1.0703400 1.0952784
##
## $mean
## [1] 0.949296
##
## $sd
## [1] 0.160505
##
## $median
## [1] 1.004293
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.8896 Min. :0.6758 Min. :0.7594
## 1st Qu.:0.8938 1st Qu.:0.7193 1st Qu.:0.7734
## Median :0.9641 Median :1.0345 Median :1.0043
## Mean :0.9669 Mean :0.9494 Mean :0.9315
## 3rd Qu.:1.0267 3rd Qu.:1.0703 3rd Qu.:1.0254
## Max. :1.0606 Max. :1.2471 Max. :1.0953 # fit glmnet, with alpha = 0, lambda = 0.01, with validation set
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 2, p = 0.8,
show_progress = FALSE,
fit_func = glmnet::glmnet, predict_func = predict.glmnet,
packages = c("glmnet", "Matrix"), fit_params = list(alpha = 0, lambda = 0.01))## $folds
## repeat_1 repeat_2
## fold_training_1 0.8581977 0.6617929
## fold_validation_1 1.0648167 1.1204975
## fold_training_2 0.7000929 1.0383779
## fold_validation_2 1.0683314 1.1451865
## fold_training_3 1.0515575 1.1083551
## fold_validation_3 1.1922896 1.1370166
## fold_training_4 0.9847907 0.9510162
## fold_validation_4 1.0888078 1.0477593
## fold_training_5 1.0600201 0.9665769
## fold_validation_5 1.0568172 1.0571621
##
## $mean_training
## [1] 0.9380778
##
## $mean_validation
## [1] 1.097868
##
## $sd_training
## [1] 0.1525031
##
## $sd_validation
## [1] 0.04841842
##
## $median_training
## [1] 0.9756838
##
## $median_validation
## [1] 1.07857
##
## $summary_training
## repeat_1 repeat_2
## Min. :0.7001 Min. :0.6618
## 1st Qu.:0.8582 1st Qu.:0.9510
## Median :0.9848 Median :0.9666
## Mean :0.9309 Mean :0.9452
## 3rd Qu.:1.0516 3rd Qu.:1.0384
## Max. :1.0600 Max. :1.1084
##
## $summary_validation
## repeat_1 repeat_2
## Min. :1.057 Min. :1.048
## 1st Qu.:1.065 1st Qu.:1.057
## Median :1.068 Median :1.120
## Mean :1.094 Mean :1.102
## 3rd Qu.:1.089 3rd Qu.:1.137
## Max. :1.192 Max. :1.145Random Forest
# randomForest example -----
# fit randomForest with mtry = 2
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3,
show_progress = FALSE,
fit_func = randomForest::randomForest, predict_func = predict,
packages = "randomForest", fit_params = list(mtry = 2))## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.0932266 0.8306226 1.0178042
## fold_2 1.0016687 0.8546975 0.8257244
## fold_3 0.8960080 1.1016916 0.7720573
## fold_4 0.7837822 1.2375085 1.0627761
## fold_5 0.9827901 1.0636167 1.1402827
##
## $mean
## [1] 0.9776172
##
## $sd
## [1] 0.1429053
##
## $median
## [1] 1.001669
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.7838 Min. :0.8306 Min. :0.7721
## 1st Qu.:0.8960 1st Qu.:0.8547 1st Qu.:0.8257
## Median :0.9828 Median :1.0636 Median :1.0178
## Mean :0.9515 Mean :1.0176 Mean :0.9637
## 3rd Qu.:1.0017 3rd Qu.:1.1017 3rd Qu.:1.0628
## Max. :1.0932 Max. :1.2375 Max. :1.1403# fit randomForest with mtry = 4
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3,
show_progress = FALSE,
fit_func = randomForest::randomForest, predict_func = predict,
packages = "randomForest", fit_params = list(mtry = 4))## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.0938938 0.8294432 1.0274141
## fold_2 0.9948985 0.8837450 0.8508324
## fold_3 0.8830305 1.1455328 0.7690735
## fold_4 0.8326955 1.2709843 1.0781753
## fold_5 0.9802129 1.0722765 1.1277285
##
## $mean
## [1] 0.9893291
##
## $sd
## [1] 0.1440225
##
## $median
## [1] 0.9948985
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.8327 Min. :0.8294 Min. :0.7691
## 1st Qu.:0.8830 1st Qu.:0.8837 1st Qu.:0.8508
## Median :0.9802 Median :1.0723 Median :1.0274
## Mean :0.9569 Mean :1.0404 Mean :0.9706
## 3rd Qu.:0.9949 3rd Qu.:1.1455 3rd Qu.:1.0782
## Max. :1.0939 Max. :1.2710 Max. :1.1277# fit randomForest with mtry = 4, with validation set
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 2, p = 0.8,
show_progress = FALSE,
fit_func = randomForest::randomForest, predict_func = predict,
packages = "randomForest", fit_params = list(mtry = 4))## $folds
## repeat_1 repeat_2
## fold_training_1 0.8159825 0.7779928
## fold_validation_1 0.6071914 0.7095150
## fold_training_2 0.8055798 0.9620625
## fold_validation_2 0.6124228 0.6857506
## fold_training_3 1.0334201 1.1550091
## fold_validation_3 0.9720091 0.9593153
## fold_training_4 0.9259389 1.0479121
## fold_validation_4 0.6447842 0.6833361
## fold_training_5 0.9795790 0.9959987
## fold_validation_5 0.8065167 0.6781677
##
## $mean_training
## [1] 0.9499476
##
## $mean_validation
## [1] 0.7359009
##
## $sd_training
## [1] 0.1205054
##
## $sd_validation
## [1] 0.1333626
##
## $median_training
## [1] 0.9708207
##
## $median_validation
## [1] 0.6845434
##
## $summary_training
## repeat_1 repeat_2
## Min. :0.8056 Min. :0.7780
## 1st Qu.:0.8160 1st Qu.:0.9621
## Median :0.9259 Median :0.9960
## Mean :0.9121 Mean :0.9878
## 3rd Qu.:0.9796 3rd Qu.:1.0479
## Max. :1.0334 Max. :1.1550
##
## $summary_validation
## repeat_1 repeat_2
## Min. :0.6072 Min. :0.6782
## 1st Qu.:0.6124 1st Qu.:0.6833
## Median :0.6448 Median :0.6858
## Mean :0.7286 Mean :0.7432
## 3rd Qu.:0.8065 3rd Qu.:0.7095
## Max. :0.9720 Max. :0.9593xgboost
# xgboost example -----
# The response and covariates are named 'label' and 'data'
# So, we do this:
f_xgboost <- function(x, y, ...) xgboost::xgboost(data = x, label = y, ...)
# fit xgboost with nrounds = 5
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3,
show_progress = FALSE,
fit_func = f_xgboost, predict_func = predict,
packages = "xgboost", fit_params = list(nrounds = 5,
verbose = FALSE))## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.0923221 0.7236469 1.1619487
## fold_2 0.9351413 0.9599840 0.8286488
## fold_3 0.9963314 1.1720342 0.6582968
## fold_4 0.8584912 1.3177231 1.1886943
## fold_5 1.1337121 0.9694089 1.2350702
##
## $mean
## [1] 1.01543
##
## $sd
## [1] 0.1926768
##
## $median
## [1] 0.9963314
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.8585 Min. :0.7236 Min. :0.6583
## 1st Qu.:0.9351 1st Qu.:0.9600 1st Qu.:0.8286
## Median :0.9963 Median :0.9694 Median :1.1619
## Mean :1.0032 Mean :1.0286 Mean :1.0145
## 3rd Qu.:1.0923 3rd Qu.:1.1720 3rd Qu.:1.1887
## Max. :1.1337 Max. :1.3177 Max. :1.2351# fit xgboost with nrounds = 10
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3,
show_progress = FALSE,
fit_func = f_xgboost, predict_func = predict,
packages = "xgboost", fit_params = list(nrounds = 10,
verbose = FALSE))## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 1.1910244 0.7427528 1.2616161
## fold_2 0.9065975 1.0807476 0.8485347
## fold_3 1.0471434 1.2457897 0.6648973
## fold_4 1.0050769 1.3496909 1.2477930
## fold_5 1.1346344 1.0062300 1.2326640
##
## $mean
## [1] 1.064346
##
## $sd
## [1] 0.2032111
##
## $median
## [1] 1.080748
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.9066 Min. :0.7428 Min. :0.6649
## 1st Qu.:1.0051 1st Qu.:1.0062 1st Qu.:0.8485
## Median :1.0471 Median :1.0807 Median :1.2327
## Mean :1.0569 Mean :1.0850 Mean :1.0511
## 3rd Qu.:1.1346 3rd Qu.:1.2458 3rd Qu.:1.2478
## Max. :1.1910 Max. :1.3497 Max. :1.2616# fit xgboost with nrounds = 10, with validation set
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 2, p = 0.8,
show_progress = FALSE,
fit_func = f_xgboost, predict_func = predict,
packages = "xgboost", fit_params = list(nrounds = 10,
verbose = FALSE))## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Passed unrecognized parameters: ",
## paste(head(names_unrecognized), : Passed unrecognized parameters: verbose. This
## warning will become an error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'data' has been renamed to 'x'. This warning will become an
## error in a future version.## Warning in throw_err_or_depr_msg("Parameter '", match_old, "' has been renamed
## to '", : Parameter 'label' has been renamed to 'y'. This warning will become an
## error in a future version.## $folds
## repeat_1 repeat_2
## fold_training_1 0.9190310 0.7364255
## fold_validation_1 0.4569952 0.6786064
## fold_training_2 0.9481533 1.1548803
## fold_validation_2 0.5632529 0.6893498
## fold_training_3 1.1415977 1.0943176
## fold_validation_3 0.9537433 0.7608019
## fold_training_4 0.9348288 1.2765564
## fold_validation_4 0.7181834 0.6285656
## fold_training_5 0.9635096 0.9326149
## fold_validation_5 0.6841376 0.6020802
##
## $mean_training
## [1] 1.010192
##
## $mean_validation
## [1] 0.6735716
##
## $sd_training
## [1] 0.1553539
##
## $sd_validation
## [1] 0.1311037
##
## $median_training
## [1] 0.9558315
##
## $median_validation
## [1] 0.681372
##
## $summary_training
## repeat_1 repeat_2
## Min. :0.9190 Min. :0.7364
## 1st Qu.:0.9348 1st Qu.:0.9326
## Median :0.9482 Median :1.0943
## Mean :0.9814 Mean :1.0390
## 3rd Qu.:0.9635 3rd Qu.:1.1549
## Max. :1.1416 Max. :1.2766
##
## $summary_validation
## repeat_1 repeat_2
## Min. :0.4570 Min. :0.6021
## 1st Qu.:0.5633 1st Qu.:0.6286
## Median :0.6841 Median :0.6786
## Mean :0.6753 Mean :0.6719
## 3rd Qu.:0.7182 3rd Qu.:0.6893
## Max. :0.9537 Max. :0.7608Theta method (time series)
res <- crossvalidation::crossval_ts(y=AirPassengers, initial_window = 10, fcast_func = thetaf, show_progress = FALSE)
print(colMeans(res))## ME RMSE MAE MPE MAPE
## 2.657082195 51.427170382 46.511874693 0.003423843 0.155428590# Input data
# Transforming model response into a factor
y <- as.factor(as.numeric(iris$Species))
# Explanatory variables
X <- as.matrix(iris[, c("Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width")])# 5-fold cross-validation repeated 3 times
# default error metric, when y is a factor: accuracy
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3,
fit_func = randomForest::randomForest,
predict_func = predict,
fit_params = list(mtry = 2),
packages = "randomForest",
show_progress = FALSE)## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 0.9666667 0.9666667 1.0000000
## fold_2 0.9666667 0.9000000 0.9333333
## fold_3 1.0000000 0.9666667 0.9333333
## fold_4 0.9333333 1.0000000 0.9333333
## fold_5 0.9333333 0.9333333 0.9666667
##
## $mean
## [1] 0.9555556
##
## $sd
## [1] 0.02999118
##
## $median
## [1] 0.9666667
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.9333 Min. :0.9000 Min. :0.9333
## 1st Qu.:0.9333 1st Qu.:0.9333 1st Qu.:0.9333
## Median :0.9667 Median :0.9667 Median :0.9333
## Mean :0.9600 Mean :0.9533 Mean :0.9533
## 3rd Qu.:0.9667 3rd Qu.:0.9667 3rd Qu.:0.9667
## Max. :1.0000 Max. :1.0000 Max. :1.0000# We can specify custom error metrics for crossvalidation::crossval_ml
# here, the error rate
eval_metric <- function (preds, actual)
{
stopifnot(length(preds) == length(actual))
res <- 1-mean(preds == actual)
names(res) <- "error rate"
return(res)
}
# specify `eval_metric` argument for measuring the error rate
# instead of the (default) accuracy
crossvalidation::crossval_ml(x = X, y = y, k = 5, repeats = 3,
fit_func = randomForest::randomForest,
predict_func = predict,
fit_params = list(mtry = 2),
packages = "randomForest",
eval_metric=eval_metric,
show_progress = FALSE)## $folds
## repeat_1 repeat_2 repeat_3
## fold_1 0.03333333 0.03333333 0.00000000
## fold_2 0.03333333 0.10000000 0.06666667
## fold_3 0.00000000 0.03333333 0.06666667
## fold_4 0.06666667 0.00000000 0.06666667
## fold_5 0.06666667 0.06666667 0.03333333
##
## $mean
## [1] 0.04444444
##
## $sd
## [1] 0.02999118
##
## $median
## [1] 0.03333333
##
## $summary
## repeat_1 repeat_2 repeat_3
## Min. :0.00000 Min. :0.00000 Min. :0.00000
## 1st Qu.:0.03333 1st Qu.:0.03333 1st Qu.:0.03333
## Median :0.03333 Median :0.03333 Median :0.06667
## Mean :0.04000 Mean :0.04667 Mean :0.04667
## 3rd Qu.:0.06667 3rd Qu.:0.06667 3rd Qu.:0.06667
## Max. :0.06667 Max. :0.10000 Max. :0.06667