conformalize

Simulated Data

We will generate a simple dataset for demonstration purposes.

set.seed(123)
n <- 200
x <- matrix(runif(n * 2), ncol = 2)
y <- 3 * x[, 1] + 2 * x[, 2] + rnorm(n, sd = 0.5)
data <- data.frame(x1 = x[, 1], x2 = x[, 2], y = y)

Fit Conformal Model

We will use a linear model (lm) as the fit_func and its corresponding predict function as the predict_func.

library(stats)

# Define fit and predict functions
fit_func <- function(formula, data, ...) lm(formula, data = data, ...)
predict_func <- function(fit, newdata, ...) predict(fit, newdata = newdata, ...)

# Apply conformalize
conformal_model <- misc::conformalize(
  formula = y ~ x1 + x2,
  data = data,
  fit_func = fit_func,
  predict_func = predict_func,
  split_ratio = 0.8,
  seed = 123
)

Generate Predictions and Prediction Intervals

We will use the predict.conformalize method to generate predictions and calculate prediction intervals.

# New data for prediction
new_data <- data.frame(x1 = runif(50), x2 = runif(50))

# Predict with split conformal method
predictions <- predict(
  conformal_model,
  newdata = new_data,
  level = 0.95,
  method = "split"
)

## 
## [1] "object's value:"
## $fit
## 
## Call:
## lm(formula = formula, data = data)
## 
## Coefficients:
## (Intercept)           x1           x2  
##   -0.000171     3.115230     1.954085  
## 
## 
## $residuals
##          10          15          18          19          28          33 
## -0.07155337 -0.50575404 -0.34743168 -0.61805759 -0.40117678  0.05274032 
##          45          47          49          58          59          61 
## -0.09260136 -0.05885147 -0.31844333  0.24731261 -0.13813234 -0.29452465 
##          65          66          73          75          82          95 
##  1.07655235  0.74875010  0.26380004  0.40559198  0.29773432  0.99912730 
##         102         104         106         107         113         114 
## -0.41409168 -0.60257935  0.08524682 -1.10511279 -0.83744059 -0.40724958 
##         115         119         120         133         136         142 
## -0.24259646  0.79117708  0.44859913 -0.40430463 -1.07935716  0.43797248 
##         146         148         150         151         152         160 
## -0.88608587 -0.27676132 -0.14552706  0.41625998 -1.00911160  1.29913204 
##         161         174         175         199 
## -0.15375511  0.34773043  0.20273890 -0.64231123 
## 
## $sd_residuals
## [1] 0.5868316
## 
## $scaled_residuals
## [1] -0.1249228
## 
## attr(,"class")
## [1] "conformalize"

head(predictions)

##         fit        lwr      upr
## 1 1.6023773  0.5217324 2.683022
## 2 2.4634938  1.3828489 3.544139
## 3 0.6216433 -0.4590017 1.702288
## 4 0.9257140 -0.1549310 2.006359
## 5 2.0106565  0.9300115 3.091301
## 6 0.7427247 -0.3379203 1.823370

Calculate Out-of-Sample Coverage Rate

The coverage rate is the proportion of true values that fall within the prediction intervals.

# Simulate true values for the new data
true_y <- 3 * new_data$x1 + 2 * new_data$x2 + rnorm(50, sd = 0.5)

# Check if true values fall within the prediction intervals
coverage <- mean(true_y >= predictions[, "lwr"] & true_y <= predictions[, "upr"])

cat("Out-of-sample coverage rate:", coverage)

## Out-of-sample coverage rate: 0.98

Results

• The prediction intervals are calculated using the split conformal method.
• The out-of-sample coverage rate is displayed, which should be close to the specified confidence level (e.g., 0.95).

Load the Data

We will use the MASS package to access the Boston dataset.

library(MASS)

# Load the Boston dataset
data(Boston)

# Inspect the dataset
head(Boston)

##      crim zn indus chas   nox    rm  age    dis rad tax ptratio  black lstat
## 1 0.00632 18  2.31    0 0.538 6.575 65.2 4.0900   1 296    15.3 396.90  4.98
## 2 0.02731  0  7.07    0 0.469 6.421 78.9 4.9671   2 242    17.8 396.90  9.14
## 3 0.02729  0  7.07    0 0.469 7.185 61.1 4.9671   2 242    17.8 392.83  4.03
## 4 0.03237  0  2.18    0 0.458 6.998 45.8 6.0622   3 222    18.7 394.63  2.94
## 5 0.06905  0  2.18    0 0.458 7.147 54.2 6.0622   3 222    18.7 396.90  5.33
## 6 0.02985  0  2.18    0 0.458 6.430 58.7 6.0622   3 222    18.7 394.12  5.21
##   medv
## 1 24.0
## 2 21.6
## 3 34.7
## 4 33.4
## 5 36.2
## 6 28.7

Split the Data

We will split the data into training and test sets to ensure they are disjoint.

set.seed(123)
n <- nrow(Boston)
train_indices <- sample(seq_len(n), size = floor(0.8 * n))
train_data <- Boston[train_indices, ]
test_data <- Boston[-train_indices, ]

Fit Conformal Model 1

# Define fit and predict functions
fit_func <- function(formula, data, ...) MASS::rlm(formula, data = data, ...)
predict_func <- function(fit, newdata, ...) predict(fit, newdata, ...)

# Apply conformalize using the training data
conformal_model_boston <- misc::conformalize(
  formula = medv ~ .,
  data = train_data,
  fit_func = fit_func,
  predict_func = predict_func,
  seed = 123
)

Generate Predictions and Prediction Intervals 1

We will use the predict.conformalize method to generate predictions and calculate prediction intervals for the test set.

# Predict with split conformal method on the test data
predictions_boston <- predict(
  conformal_model_boston,
  newdata = test_data,
  level = 0.95,
  method = "split"
)

## 
## [1] "object's value:"
## $fit
## Call:
## rlm(formula = formula, data = data)
## Converged in 12 iterations
## 
## Coefficients:
##  (Intercept)         crim           zn        indus         chas          nox 
## 10.053413833 -0.163015431  0.009957524 -0.013355275  1.557351564 -5.289064676 
##           rm          age          dis          rad          tax      ptratio 
##  6.050082402 -0.014130776 -1.046959248  0.204156664 -0.010752932 -0.815953461 
##        black        lstat 
##  0.014196664 -0.436463429 
## 
## Degrees of freedom: 202 total; 188 residual
## Scale estimate: 2.78 
## 
## $residuals
##          415          463          179          426          118          503 
##  17.75101224  -0.42301639  -1.61859953   1.62139385  -3.34677969  -1.60897767 
##           90          256          197          491          348          355 
##  -3.19085491   1.54228923  -1.78273122   5.81481859  -0.79919511   5.50240802 
##            7          501          485          254          211           43 
##   0.53762811  -3.44980333   2.53375234   8.03603880   1.44113352   0.08310029 
##          373          332          425          330           23          411 
##  28.09729049  -1.37881939   2.08155331  -2.10520186  -1.24294958   7.86360572 
##          309          135          290           72           76           63 
##  -5.85410610   3.32023507  -0.94636403   0.80984792  -1.85752392  -1.35877320 
##          210          493          294           41          492          316 
##   6.10840592   4.26900808  -1.23403353   2.06751492  -0.11235268  -3.23281956 
##           16           94          342           39          240            4 
##   0.81505531  -1.98229092   1.20380307   2.23545905  -4.28334192   3.17274197 
##           13          409          308           89           25          291 
##   1.08193007   6.47568699  -3.13637115  -7.61851115  -0.14593182  -2.55986770 
##          286          396          110          158          398           67 
##  -3.95756249  -6.92122367  -0.08380848   8.39453871  -6.20029505  -2.16451929 
##          335           85          136          178          236           98 
##  -2.13636104  -0.97950834  -0.01953636  -3.55277580  -0.09793921   0.42318320 
##          214          127          212          273          310          232 
##   2.77058176   2.35196874   5.06669808  -3.18238722  -2.29399575  -2.62510928 
##          366          416          350          407          280          154 
##  20.43126404  -0.62828007   2.65823308  10.23724986   0.74559441   1.59669803 
##          102          255          326          272          470          288 
##   0.19552077   0.18581742  -1.05991028  -1.16045291   4.23230707  -1.66642094 
##          440           55          331          478          184          459 
##   0.71422248   5.50936970  -2.00530538   4.02428952   2.53299654  -1.93859807 
##          196          432          352           20          502          177 
##   9.48644109  -2.62067731   3.24264261   0.51984714  -2.24793764  -1.58613996 
##           42          405          380          395          194          249 
##  -2.75990583   4.56727197  -5.27523660  -3.43589886   0.10886146   3.11941696 
##          200          377          250          292          434           33 
##   6.02359042  -3.63739240   1.09081050   4.60846568  -1.70144111   4.07698311 
##          152           54          430          289          185          205 
##   1.29737169   0.24062338  -1.99405324  -2.80371431   6.00421913   7.74226900 
##          334          215          318           57          105          279 
##  -1.14753812  12.85549644   1.83703448   1.44486879  -0.60194612   0.46171405 
##          129          106          480          369          406          287 
##  -1.75885183   2.26221951   1.79691136  31.89913520   1.75716435   1.61365538 
##          356          225          117          452          400          341 
##   5.24072632   3.87927876  -1.67191387  -4.92493185  -3.94064347  -2.50204595 
##          176          220          191           53          104          447 
##  -1.26482459  -4.90616974   6.42986225  -2.64155010  -0.47874865  -2.81083340 
##          320          354          275          261            2          336 
##  -0.53303073   5.21706683  -1.12106092  -2.06641580  -3.33379070  -0.19269560 
##          497          394          448          327          419          112 
##   6.50546425  -5.49102250  -5.42896409  -1.29979669   9.33706323  -4.11746097 
##           36           87          399          111          461          351 
##  -3.60153400   0.90077494   1.54781531   0.97506146  -3.11921704   1.75920907 
##           31           73          424          333          122           92 
##   1.35292531  -0.96867871   3.25415805  -2.67640505  -1.09730455  -4.87346776 
##          234          338          472          300          323          314 
##   8.19783868  -0.82543739  -1.15016922  -2.15925065  -2.00661608  -3.39475797 
##          443          387          376          208          499          206 
##  -0.32010786   8.62390298 -10.57350717   5.50712569   0.08881949   0.54973502 
##          297          219           82          169          413          228 
##  -0.11392277  -1.13639371  -2.23070320  -1.15670018  22.68901628  -1.11958292 
##          423          258          385          482          498          192 
##   6.06871179   2.90343078  10.38887008  -2.11698974  -0.34891269   0.99064382 
##          386          414           50          260           97          183 
##   1.34099245  10.71923362   3.13590006  -4.57405529  -2.25277923   3.81266117 
##          100          274          484          296          357          227 
##  -0.46119518  -0.96786208   2.97493526  -0.20959741  -0.95380842  -1.96415748 
##          190          329          156          131            8          468 
##   1.00965741  -1.42530635  -4.17827374  -1.58936371   7.51853676   4.12047455 
##           70          213          382          182 
##   1.16853787   1.78909033  -7.07189205   9.67559663 
## 
## $sd_residuals
## [1] 5.384367
## 
## $scaled_residuals
## [1] 0.1888384
## 
## attr(,"class")
## [1] "conformalize"

head(predictions_boston)

##         fit       lwr      upr
## 1  29.92942 20.263283 39.59556
## 15 19.30837  9.642229 28.97451
## 17 20.71124 11.045100 30.37738
## 19 14.86650  5.200365 24.53264
## 28 14.79883  5.132688 24.46497
## 37 20.98752 11.321382 30.65366

Calculate Out-of-Sample Coverage Rate 1

The coverage rate is the proportion of true values in the test set that fall within the prediction intervals.

# True values for the test set
true_y_boston <- test_data$medv

# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston[, "lwr"] & true_y_boston <= predictions_boston[, "upr"])

cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)

## Out-of-sample coverage rate for Boston dataset: 0.9509804

Fit Conformal Model 2

# Define fit and predict functions
fit_func <- function(formula, data, ...) stats::glm(formula, data = data, ...)
predict_func <- function(fit, newdata, ...) predict(fit, newdata, ...)

# Apply conformalize using the training data
conformal_model_boston <- misc::conformalize(
  formula = medv ~ .,
  data = train_data,
  fit_func = fit_func,
  predict_func = predict_func,
  seed = 123
)

Generate Predictions and Prediction Intervals 2

We will use the predict.conformalize method to generate predictions and calculate prediction intervals for the test set.

# Predict with split conformal method on the test data
predictions_boston <- predict(
  conformal_model_boston,
  newdata = test_data,
  level = 0.95,
  method = "split"
)

## 
## [1] "object's value:"
## $fit
## 
## Call:  stats::glm(formula = formula, data = data)
## 
## Coefficients:
## (Intercept)         crim           zn        indus         chas          nox  
##    29.78825     -0.14238      0.03251      0.03061      2.17196    -16.17824  
##          rm          age          dis          rad          tax      ptratio  
##     4.70344      0.01813     -1.50717      0.35519     -0.01344     -1.04058  
##       black        lstat  
##     0.01303     -0.58559  
## 
## Degrees of Freedom: 201 Total (i.e. Null);  188 Residual
## Null Deviance:       18580 
## Residual Deviance: 3991  AIC: 1206
## 
## $residuals
##           415           463           179           426           118 
##  15.764477831  -0.988770013  -2.543179984   0.246263589  -4.612173369 
##           503            90           256           197           491 
##  -1.693827822  -2.950069022   1.869965859  -2.430610387   6.641430762 
##           348           355             7           501           485 
##  -0.387896963   7.048338684   0.194748907  -3.493814316   1.693920234 
##           254           211            43           373           332 
##  12.292956642   0.138159179   1.154004128  24.027816881  -0.803632969 
##           425           330            23           411           309 
##  -0.839920969  -0.722612132  -0.167767308   2.167940990  -6.825169566 
##           135           290            72            76            63 
##   3.275100495  -0.759414026   1.430389678  -2.020087653  -1.674552399 
##           210           493           294            41           492 
##   4.586854531   4.552309079  -1.126062979   1.696969057   0.262178313 
##           316            16            94           342            39 
##  -3.970341530   1.333549618  -3.241784129   2.197383166   2.775323621 
##           240             4            13           409           308 
##  -4.694843972   4.306893821   1.896411213   4.373885971  -4.894902111 
##            89            25           291           286           396 
##  -8.248709022   0.398861978  -3.821840049  -3.993434219  -8.052811215 
##           110           158           398            67           335 
##  -0.174707897   6.170637038  -7.912137723  -3.779100281  -0.420307060 
##            85           136           178           236            98 
##  -0.505685158   0.722000550  -5.082576649  -1.112554638   1.000305006 
##           214           127           212           273           310 
##   3.417047314   2.068581362   4.029083985  -3.973489628  -3.257778331 
##           232           366           416           350           407 
##  -3.217537805  14.504200597  -0.789956450   5.421465622   5.957527419 
##           280           154           102           255           326 
##  -0.388687016   1.987991575   0.466096118   0.007181811   0.558481278 
##           272           470           288           440            55 
##  -0.918384377   2.276754357  -2.553856315   0.008825628   6.349577246 
##           331           478           184           459           196 
##  -0.804163641   1.663888937   0.439328371  -2.359964178   8.911889804 
##           432           352            20           502           177 
##  -4.146198759   5.045979801   0.563337346  -1.280802168  -1.992182379 
##            42           405           380           395           194 
##  -0.902292374   3.327617862  -6.667225863  -5.400633140   0.231787928 
##           249           200           377           250           292 
##   3.977422436   6.478740182  -4.023381192   2.932225794   3.772826861 
##           434            33           152            54           430 
##  -2.448673653   6.358540758   0.946984605   0.628531680  -2.546717499 
##           289           185           205           334           215 
##  -3.941265332   4.102284871   6.607098719   0.430018524  15.728088344 
##           318            57           105           279           129 
##   2.099510664   1.644595746  -1.270661910  -0.426073914  -1.271359683 
##           106           480           369           406           287 
##   1.426383844  -0.792647956  25.844187265  -0.968727677   2.656329159 
##           356           225           117           452           400 
##   6.852693030   3.881995799  -2.152249451  -5.166074745  -3.503250830 
##           341           176           220           191            53 
##  -2.262305509  -1.417549048  -7.035616819   6.863606354  -2.022958762 
##           104           447           320           354           275 
##  -0.802709122  -3.305050943   0.093391012   5.989158592  -2.851763168 
##           261             2           336           497           394 
##  -2.597612328  -3.713695585   1.263629192   6.975450133  -7.050402445 
##           448           327           419           112            36 
##  -6.187067749  -0.138726762   6.104096769  -4.466775330  -4.616862136 
##            87           399           111           461           351 
##   1.284500325   0.300778763   1.758188903  -3.221342364   3.619381597 
##            31            73           424           333           122 
##   2.428740626  -0.527694477   1.845675138  -2.221711574  -2.189798218 
##            92           234           338           472           300 
##  -5.666921491   8.775318305  -0.162437018  -4.000684029  -1.784296138 
##           323           314           443           387           376 
##  -1.977525789  -4.395997397  -1.201281672   6.307724361 -11.738018736 
##           208           499           206           297           219 
##   5.537880093   0.246994887   1.102222072  -0.460269305  -2.987158004 
##            82           169           413           228           423 
##  -3.021002286  -3.205200388  19.853308439  -2.192024195   2.797529204 
##           258           385           482           498           192 
##   3.414178564   8.271338091  -4.451362820  -0.266787057   0.792207439 
##           386           414            50           260            97 
##   0.442623562   6.487624676   3.388454435  -6.629745005  -2.978426708 
##           183           100           274           484           296 
##   2.427214778  -0.031584384  -0.570256827   1.238172323   0.095873701 
##           357           227           190           329           156 
##  -1.933201226  -2.531726289   0.382566988  -0.545690834  -3.581922258 
##           131             8           468            70           213 
##  -1.346444916   7.621427858   2.329583294   1.528233254   1.223808670 
##           382           182 
##  -7.999559935   8.543358664 
## 
## $sd_residuals
## [1] 5.091575
## 
## $scaled_residuals
## [1] 0.1125522
## 
## attr(,"class")
## [1] "conformalize"

head(predictions_boston)

##         fit       lwr      upr
## 1  30.39434 21.864586 38.92410
## 15 19.10867 10.578908 27.63842
## 17 19.48015 10.950390 28.00990
## 19 14.18587  5.656113 22.71563
## 28 13.85324  5.323482 22.38300
## 37 21.66451 13.134753 30.19427

# Predict with split conformal method on the test data
predictions_boston2 <- predict(
  conformal_model_boston,
  newdata = test_data,
  level = 0.95,
  method = "kde"
)

## 
## [1] "object's value:"
## $fit
## 
## Call:  stats::glm(formula = formula, data = data)
## 
## Coefficients:
## (Intercept)         crim           zn        indus         chas          nox  
##    29.78825     -0.14238      0.03251      0.03061      2.17196    -16.17824  
##          rm          age          dis          rad          tax      ptratio  
##     4.70344      0.01813     -1.50717      0.35519     -0.01344     -1.04058  
##       black        lstat  
##     0.01303     -0.58559  
## 
## Degrees of Freedom: 201 Total (i.e. Null);  188 Residual
## Null Deviance:       18580 
## Residual Deviance: 3991  AIC: 1206
## 
## $residuals
##           415           463           179           426           118 
##  15.764477831  -0.988770013  -2.543179984   0.246263589  -4.612173369 
##           503            90           256           197           491 
##  -1.693827822  -2.950069022   1.869965859  -2.430610387   6.641430762 
##           348           355             7           501           485 
##  -0.387896963   7.048338684   0.194748907  -3.493814316   1.693920234 
##           254           211            43           373           332 
##  12.292956642   0.138159179   1.154004128  24.027816881  -0.803632969 
##           425           330            23           411           309 
##  -0.839920969  -0.722612132  -0.167767308   2.167940990  -6.825169566 
##           135           290            72            76            63 
##   3.275100495  -0.759414026   1.430389678  -2.020087653  -1.674552399 
##           210           493           294            41           492 
##   4.586854531   4.552309079  -1.126062979   1.696969057   0.262178313 
##           316            16            94           342            39 
##  -3.970341530   1.333549618  -3.241784129   2.197383166   2.775323621 
##           240             4            13           409           308 
##  -4.694843972   4.306893821   1.896411213   4.373885971  -4.894902111 
##            89            25           291           286           396 
##  -8.248709022   0.398861978  -3.821840049  -3.993434219  -8.052811215 
##           110           158           398            67           335 
##  -0.174707897   6.170637038  -7.912137723  -3.779100281  -0.420307060 
##            85           136           178           236            98 
##  -0.505685158   0.722000550  -5.082576649  -1.112554638   1.000305006 
##           214           127           212           273           310 
##   3.417047314   2.068581362   4.029083985  -3.973489628  -3.257778331 
##           232           366           416           350           407 
##  -3.217537805  14.504200597  -0.789956450   5.421465622   5.957527419 
##           280           154           102           255           326 
##  -0.388687016   1.987991575   0.466096118   0.007181811   0.558481278 
##           272           470           288           440            55 
##  -0.918384377   2.276754357  -2.553856315   0.008825628   6.349577246 
##           331           478           184           459           196 
##  -0.804163641   1.663888937   0.439328371  -2.359964178   8.911889804 
##           432           352            20           502           177 
##  -4.146198759   5.045979801   0.563337346  -1.280802168  -1.992182379 
##            42           405           380           395           194 
##  -0.902292374   3.327617862  -6.667225863  -5.400633140   0.231787928 
##           249           200           377           250           292 
##   3.977422436   6.478740182  -4.023381192   2.932225794   3.772826861 
##           434            33           152            54           430 
##  -2.448673653   6.358540758   0.946984605   0.628531680  -2.546717499 
##           289           185           205           334           215 
##  -3.941265332   4.102284871   6.607098719   0.430018524  15.728088344 
##           318            57           105           279           129 
##   2.099510664   1.644595746  -1.270661910  -0.426073914  -1.271359683 
##           106           480           369           406           287 
##   1.426383844  -0.792647956  25.844187265  -0.968727677   2.656329159 
##           356           225           117           452           400 
##   6.852693030   3.881995799  -2.152249451  -5.166074745  -3.503250830 
##           341           176           220           191            53 
##  -2.262305509  -1.417549048  -7.035616819   6.863606354  -2.022958762 
##           104           447           320           354           275 
##  -0.802709122  -3.305050943   0.093391012   5.989158592  -2.851763168 
##           261             2           336           497           394 
##  -2.597612328  -3.713695585   1.263629192   6.975450133  -7.050402445 
##           448           327           419           112            36 
##  -6.187067749  -0.138726762   6.104096769  -4.466775330  -4.616862136 
##            87           399           111           461           351 
##   1.284500325   0.300778763   1.758188903  -3.221342364   3.619381597 
##            31            73           424           333           122 
##   2.428740626  -0.527694477   1.845675138  -2.221711574  -2.189798218 
##            92           234           338           472           300 
##  -5.666921491   8.775318305  -0.162437018  -4.000684029  -1.784296138 
##           323           314           443           387           376 
##  -1.977525789  -4.395997397  -1.201281672   6.307724361 -11.738018736 
##           208           499           206           297           219 
##   5.537880093   0.246994887   1.102222072  -0.460269305  -2.987158004 
##            82           169           413           228           423 
##  -3.021002286  -3.205200388  19.853308439  -2.192024195   2.797529204 
##           258           385           482           498           192 
##   3.414178564   8.271338091  -4.451362820  -0.266787057   0.792207439 
##           386           414            50           260            97 
##   0.442623562   6.487624676   3.388454435  -6.629745005  -2.978426708 
##           183           100           274           484           296 
##   2.427214778  -0.031584384  -0.570256827   1.238172323   0.095873701 
##           357           227           190           329           156 
##  -1.933201226  -2.531726289   0.382566988  -0.545690834  -3.581922258 
##           131             8           468            70           213 
##  -1.346444916   7.621427858   2.329583294   1.528233254   1.223808670 
##           382           182 
##  -7.999559935   8.543358664 
## 
## $sd_residuals
## [1] 5.091575
## 
## $scaled_residuals
## [1] 0.1125522
## 
## attr(,"class")
## [1] "conformalize"

head(predictions_boston2)

##         fit      lwr      upr
## 1  30.39434 22.37589 49.43768
## 15 19.10867 11.96002 28.22319
## 17 19.48015 11.91617 34.06139
## 19 14.18587  6.77628 23.56133
## 28 13.85324  6.61761 27.71907
## 37 21.66451 13.49167 37.83532

# Predict with split conformal method on the test data
predictions_boston3 <- predict(
  conformal_model_boston,
  newdata = test_data,
  level = 0.95,
  method = "surrogate"
)

## 
## [1] "object's value:"
## $fit
## 
## Call:  stats::glm(formula = formula, data = data)
## 
## Coefficients:
## (Intercept)         crim           zn        indus         chas          nox  
##    29.78825     -0.14238      0.03251      0.03061      2.17196    -16.17824  
##          rm          age          dis          rad          tax      ptratio  
##     4.70344      0.01813     -1.50717      0.35519     -0.01344     -1.04058  
##       black        lstat  
##     0.01303     -0.58559  
## 
## Degrees of Freedom: 201 Total (i.e. Null);  188 Residual
## Null Deviance:       18580 
## Residual Deviance: 3991  AIC: 1206
## 
## $residuals
##           415           463           179           426           118 
##  15.764477831  -0.988770013  -2.543179984   0.246263589  -4.612173369 
##           503            90           256           197           491 
##  -1.693827822  -2.950069022   1.869965859  -2.430610387   6.641430762 
##           348           355             7           501           485 
##  -0.387896963   7.048338684   0.194748907  -3.493814316   1.693920234 
##           254           211            43           373           332 
##  12.292956642   0.138159179   1.154004128  24.027816881  -0.803632969 
##           425           330            23           411           309 
##  -0.839920969  -0.722612132  -0.167767308   2.167940990  -6.825169566 
##           135           290            72            76            63 
##   3.275100495  -0.759414026   1.430389678  -2.020087653  -1.674552399 
##           210           493           294            41           492 
##   4.586854531   4.552309079  -1.126062979   1.696969057   0.262178313 
##           316            16            94           342            39 
##  -3.970341530   1.333549618  -3.241784129   2.197383166   2.775323621 
##           240             4            13           409           308 
##  -4.694843972   4.306893821   1.896411213   4.373885971  -4.894902111 
##            89            25           291           286           396 
##  -8.248709022   0.398861978  -3.821840049  -3.993434219  -8.052811215 
##           110           158           398            67           335 
##  -0.174707897   6.170637038  -7.912137723  -3.779100281  -0.420307060 
##            85           136           178           236            98 
##  -0.505685158   0.722000550  -5.082576649  -1.112554638   1.000305006 
##           214           127           212           273           310 
##   3.417047314   2.068581362   4.029083985  -3.973489628  -3.257778331 
##           232           366           416           350           407 
##  -3.217537805  14.504200597  -0.789956450   5.421465622   5.957527419 
##           280           154           102           255           326 
##  -0.388687016   1.987991575   0.466096118   0.007181811   0.558481278 
##           272           470           288           440            55 
##  -0.918384377   2.276754357  -2.553856315   0.008825628   6.349577246 
##           331           478           184           459           196 
##  -0.804163641   1.663888937   0.439328371  -2.359964178   8.911889804 
##           432           352            20           502           177 
##  -4.146198759   5.045979801   0.563337346  -1.280802168  -1.992182379 
##            42           405           380           395           194 
##  -0.902292374   3.327617862  -6.667225863  -5.400633140   0.231787928 
##           249           200           377           250           292 
##   3.977422436   6.478740182  -4.023381192   2.932225794   3.772826861 
##           434            33           152            54           430 
##  -2.448673653   6.358540758   0.946984605   0.628531680  -2.546717499 
##           289           185           205           334           215 
##  -3.941265332   4.102284871   6.607098719   0.430018524  15.728088344 
##           318            57           105           279           129 
##   2.099510664   1.644595746  -1.270661910  -0.426073914  -1.271359683 
##           106           480           369           406           287 
##   1.426383844  -0.792647956  25.844187265  -0.968727677   2.656329159 
##           356           225           117           452           400 
##   6.852693030   3.881995799  -2.152249451  -5.166074745  -3.503250830 
##           341           176           220           191            53 
##  -2.262305509  -1.417549048  -7.035616819   6.863606354  -2.022958762 
##           104           447           320           354           275 
##  -0.802709122  -3.305050943   0.093391012   5.989158592  -2.851763168 
##           261             2           336           497           394 
##  -2.597612328  -3.713695585   1.263629192   6.975450133  -7.050402445 
##           448           327           419           112            36 
##  -6.187067749  -0.138726762   6.104096769  -4.466775330  -4.616862136 
##            87           399           111           461           351 
##   1.284500325   0.300778763   1.758188903  -3.221342364   3.619381597 
##            31            73           424           333           122 
##   2.428740626  -0.527694477   1.845675138  -2.221711574  -2.189798218 
##            92           234           338           472           300 
##  -5.666921491   8.775318305  -0.162437018  -4.000684029  -1.784296138 
##           323           314           443           387           376 
##  -1.977525789  -4.395997397  -1.201281672   6.307724361 -11.738018736 
##           208           499           206           297           219 
##   5.537880093   0.246994887   1.102222072  -0.460269305  -2.987158004 
##            82           169           413           228           423 
##  -3.021002286  -3.205200388  19.853308439  -2.192024195   2.797529204 
##           258           385           482           498           192 
##   3.414178564   8.271338091  -4.451362820  -0.266787057   0.792207439 
##           386           414            50           260            97 
##   0.442623562   6.487624676   3.388454435  -6.629745005  -2.978426708 
##           183           100           274           484           296 
##   2.427214778  -0.031584384  -0.570256827   1.238172323   0.095873701 
##           357           227           190           329           156 
##  -1.933201226  -2.531726289   0.382566988  -0.545690834  -3.581922258 
##           131             8           468            70           213 
##  -1.346444916   7.621427858   2.329583294   1.528233254   1.223808670 
##           382           182 
##  -7.999559935   8.543358664 
## 
## $sd_residuals
## [1] 5.091575
## 
## $scaled_residuals
## [1] 0.1125522
## 
## attr(,"class")
## [1] "conformalize"

## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo

head(predictions_boston3)

##         fit       lwr      upr
## 1  30.39434 22.482206 39.30623
## 15 19.10867 12.073048 31.40162
## 17 19.48015 12.698948 34.93286
## 19 14.18587  7.396239 29.63858
## 28 13.85324  6.134993 22.68221
## 37 21.66451 14.614108 33.95747

# Predict with split conformal method on the test data
predictions_boston4 <- predict(
  conformal_model_boston,
  newdata = test_data,
  level = 0.95,
  method = "bootstrap"
)

## 
## [1] "object's value:"
## $fit
## 
## Call:  stats::glm(formula = formula, data = data)
## 
## Coefficients:
## (Intercept)         crim           zn        indus         chas          nox  
##    29.78825     -0.14238      0.03251      0.03061      2.17196    -16.17824  
##          rm          age          dis          rad          tax      ptratio  
##     4.70344      0.01813     -1.50717      0.35519     -0.01344     -1.04058  
##       black        lstat  
##     0.01303     -0.58559  
## 
## Degrees of Freedom: 201 Total (i.e. Null);  188 Residual
## Null Deviance:       18580 
## Residual Deviance: 3991  AIC: 1206
## 
## $residuals
##           415           463           179           426           118 
##  15.764477831  -0.988770013  -2.543179984   0.246263589  -4.612173369 
##           503            90           256           197           491 
##  -1.693827822  -2.950069022   1.869965859  -2.430610387   6.641430762 
##           348           355             7           501           485 
##  -0.387896963   7.048338684   0.194748907  -3.493814316   1.693920234 
##           254           211            43           373           332 
##  12.292956642   0.138159179   1.154004128  24.027816881  -0.803632969 
##           425           330            23           411           309 
##  -0.839920969  -0.722612132  -0.167767308   2.167940990  -6.825169566 
##           135           290            72            76            63 
##   3.275100495  -0.759414026   1.430389678  -2.020087653  -1.674552399 
##           210           493           294            41           492 
##   4.586854531   4.552309079  -1.126062979   1.696969057   0.262178313 
##           316            16            94           342            39 
##  -3.970341530   1.333549618  -3.241784129   2.197383166   2.775323621 
##           240             4            13           409           308 
##  -4.694843972   4.306893821   1.896411213   4.373885971  -4.894902111 
##            89            25           291           286           396 
##  -8.248709022   0.398861978  -3.821840049  -3.993434219  -8.052811215 
##           110           158           398            67           335 
##  -0.174707897   6.170637038  -7.912137723  -3.779100281  -0.420307060 
##            85           136           178           236            98 
##  -0.505685158   0.722000550  -5.082576649  -1.112554638   1.000305006 
##           214           127           212           273           310 
##   3.417047314   2.068581362   4.029083985  -3.973489628  -3.257778331 
##           232           366           416           350           407 
##  -3.217537805  14.504200597  -0.789956450   5.421465622   5.957527419 
##           280           154           102           255           326 
##  -0.388687016   1.987991575   0.466096118   0.007181811   0.558481278 
##           272           470           288           440            55 
##  -0.918384377   2.276754357  -2.553856315   0.008825628   6.349577246 
##           331           478           184           459           196 
##  -0.804163641   1.663888937   0.439328371  -2.359964178   8.911889804 
##           432           352            20           502           177 
##  -4.146198759   5.045979801   0.563337346  -1.280802168  -1.992182379 
##            42           405           380           395           194 
##  -0.902292374   3.327617862  -6.667225863  -5.400633140   0.231787928 
##           249           200           377           250           292 
##   3.977422436   6.478740182  -4.023381192   2.932225794   3.772826861 
##           434            33           152            54           430 
##  -2.448673653   6.358540758   0.946984605   0.628531680  -2.546717499 
##           289           185           205           334           215 
##  -3.941265332   4.102284871   6.607098719   0.430018524  15.728088344 
##           318            57           105           279           129 
##   2.099510664   1.644595746  -1.270661910  -0.426073914  -1.271359683 
##           106           480           369           406           287 
##   1.426383844  -0.792647956  25.844187265  -0.968727677   2.656329159 
##           356           225           117           452           400 
##   6.852693030   3.881995799  -2.152249451  -5.166074745  -3.503250830 
##           341           176           220           191            53 
##  -2.262305509  -1.417549048  -7.035616819   6.863606354  -2.022958762 
##           104           447           320           354           275 
##  -0.802709122  -3.305050943   0.093391012   5.989158592  -2.851763168 
##           261             2           336           497           394 
##  -2.597612328  -3.713695585   1.263629192   6.975450133  -7.050402445 
##           448           327           419           112            36 
##  -6.187067749  -0.138726762   6.104096769  -4.466775330  -4.616862136 
##            87           399           111           461           351 
##   1.284500325   0.300778763   1.758188903  -3.221342364   3.619381597 
##            31            73           424           333           122 
##   2.428740626  -0.527694477   1.845675138  -2.221711574  -2.189798218 
##            92           234           338           472           300 
##  -5.666921491   8.775318305  -0.162437018  -4.000684029  -1.784296138 
##           323           314           443           387           376 
##  -1.977525789  -4.395997397  -1.201281672   6.307724361 -11.738018736 
##           208           499           206           297           219 
##   5.537880093   0.246994887   1.102222072  -0.460269305  -2.987158004 
##            82           169           413           228           423 
##  -3.021002286  -3.205200388  19.853308439  -2.192024195   2.797529204 
##           258           385           482           498           192 
##   3.414178564   8.271338091  -4.451362820  -0.266787057   0.792207439 
##           386           414            50           260            97 
##   0.442623562   6.487624676   3.388454435  -6.629745005  -2.978426708 
##           183           100           274           484           296 
##   2.427214778  -0.031584384  -0.570256827   1.238172323   0.095873701 
##           357           227           190           329           156 
##  -1.933201226  -2.531726289   0.382566988  -0.545690834  -3.581922258 
##           131             8           468            70           213 
##  -1.346444916   7.621427858   2.329583294   1.528233254   1.223808670 
##           382           182 
##  -7.999559935   8.543358664 
## 
## $sd_residuals
## [1] 5.091575
## 
## $scaled_residuals
## [1] 0.1125522
## 
## attr(,"class")
## [1] "conformalize"

head(predictions_boston4)

##         fit       lwr      upr
## 1  30.39434 23.343942 45.84706
## 15 19.10867 11.055854 34.56138
## 17 19.48015 11.427336 33.98435
## 19 14.18587  6.133060 25.25477
## 28 13.85324  5.800429 21.80538
## 37 21.66451 13.611700 36.16871

Fit Conformal Model 2

# Define fit and predict functions
fit_func <- function(formula, data, ...) ranger::ranger(formula, data = data)
predict_func <- function(fit, newdata, ...) predict(fit, newdata)$predictions

# Apply conformalize using the training data
conformal_model_boston_rf <- misc::conformalize(
  formula = medv ~ .,
  data = train_data,
  fit_func = fit_func,
  predict_func = predict_func,
  seed = 123
)

# Predict with split conformal method on the test data
predictions_boston_rf <- predict(
  conformal_model_boston_rf,
  newdata = test_data,
  predict_func = predict_func,
  level = 0.95,
  method = "kde"
)

## 
## [1] "object's value:"
## $fit
## Ranger result
## 
## Call:
##  ranger::ranger(formula, data = data) 
## 
## Type:                             Regression 
## Number of trees:                  500 
## Sample size:                      202 
## Number of independent variables:  13 
## Mtry:                             3 
## Target node size:                 5 
## Variable importance mode:         none 
## Splitrule:                        variance 
## OOB prediction error (MSE):       13.88378 
## R squared (OOB):                  0.8497858 
## 
## $residuals
##   [1]  -3.28995224  -0.76321776   2.43128905  -2.66240151  -0.99287952
##   [6]   2.70121471  -2.42204429  -1.55376000  -2.56573810  -3.26182053
##  [11]  -2.14750667  -3.32581873   2.63805000  -3.16368111  -0.18665491
##  [16]   4.75706000   1.91775692   0.17052333  23.22278310  -3.92153000
##  [21]  -2.90377188  -2.01840810  -1.43485155  -5.79080450  -6.06995808
##  [26]   0.29719359   0.28491333   0.27125333  -0.99347000  -1.85960667
##  [31]   2.10630692   1.28491786   1.08958000  -0.03567000  -0.61526291
##  [36]  -4.23196833  -0.18508529   0.65849333  -0.65427429   3.12121000
##  [41]  -1.74895000  -1.48603000   2.20137667   2.93456194  -1.10091333
##  [46]  -7.04727762  -1.25147789  -2.56449333  -2.72583333  -0.31294785
##  [51]   0.79210756   8.54976667  -3.74300005  -2.86267333  -2.72342333
##  [56]   0.24300667   1.35786058  -0.21181667   0.88757333  -0.42359000
##  [61]   4.69163000  -0.96359889   0.66065359  -0.75930000  -1.30661167
##  [66]  -4.03423333   5.21864760  -3.41794288  -1.17415476  -5.75970684
##  [71]   3.99781333   2.82440860   3.23760137  -2.09188333  -0.43265667
##  [76]   0.28025667   1.90285132  -0.66357667   0.37622751  -1.72012333
##  [81]  -3.60811810  -1.11760463   3.47579692  -0.93810272   5.71293333
##  [86]  -1.19071621  -1.40754143  -1.02234815   1.73892137   0.66767333
##  [91]  -1.84796667  -1.89577701  -1.67581062   0.91369779   2.46615571
##  [96]   1.82967667   3.23964571   1.33185565   0.59930333   2.67968333
## [101]  -1.69234339  -2.04407466   1.67464874   1.35952333  -3.74173287
## [106]  -1.14513333   2.09679930   4.16788000  -1.80788667   4.88675667
## [111]   0.61670667  -0.97427667   0.30204090   2.81902667   1.33740701
## [116]   1.35159447   2.71234556  18.28314843  -5.38726210  -4.98795000
## [121]  -1.93378921   1.11416333   0.45888857  -0.64968798  -5.63412045
## [126]  -1.79377333   2.19376667   0.29618905   8.15903905  -0.81654000
## [131]  -0.43899577   0.13727578  -0.73637667  -1.56695667   0.61336333
## [136]  -1.61457714  -1.85142641  -0.04208333   2.18082333  -0.05410923
## [141]  -1.69643645  -1.05970000  -1.41252805   0.24018524  -2.90524333
## [146]   0.87718000  -4.74593554   1.14160137  -0.36370946  -2.36319333
## [151]  -1.52786832  -0.82554667  -0.66698403  -3.51356000  -0.22256070
## [156]  -2.46117333   5.40267000  -1.87395333  -1.54967593  -1.27451143
## [161]  -1.77721333  -1.18105474   3.10490370   0.25962197 -10.30566648
## [166]   3.36484000   0.55351667   0.76607333   2.61153667   1.55455333
## [171]  -1.59643667   1.81080762   4.28905158  -2.09246429   2.04975681
## [176]   7.35960095  -2.43278149  -1.33287801  -2.12767333   2.59084667
## [181]  -3.68412954   2.79636778   0.26392333  -2.15853429  -2.89614000
## [186]   2.09732000  -0.29039667  -0.33629000   0.82993794   3.32955000
## [191]   1.82944150  -6.48816667   1.79487190  -3.49737476  -2.38569872
## [196]   0.62226415   9.56466667   3.69821468   0.15197667   1.87070333
## [201]  -0.41697375   9.91625667
## 
## $sd_residuals
## [1] 3.558924
## 
## $scaled_residuals
## [1] 0.01476135
## 
## attr(,"class")
## [1] "conformalize"

head(predictions_boston_rf)

##           fit       lwr      upr
## [1,] 27.06333 20.953499 32.71876
## [2,] 19.08097 13.197909 25.55087
## [3,] 21.23130 15.544766 27.99221
## [4,] 18.69498 12.585151 25.22157
## [5,] 15.74951  9.883451 21.74887
## [6,] 21.32154 14.682593 30.42000

Calculate Out-of-Sample Coverage Rate 2

The coverage rate is the proportion of true values in the test set that fall within the prediction intervals.

# True values for the test set
true_y_boston <- test_data$medv

# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston[, "lwr"] & true_y_boston <= predictions_boston[, "upr"])

cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)

## Out-of-sample coverage rate for Boston dataset: 0.9411765

# True values for the test set
true_y_boston <- test_data$medv

# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston2[, "lwr"] & true_y_boston <= predictions_boston2[, "upr"])

cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)

## Out-of-sample coverage rate for Boston dataset: 0.9607843

# True values for the test set
true_y_boston <- test_data$medv

# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston3[, "lwr"] & true_y_boston <= predictions_boston3[, "upr"])

cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)

## Out-of-sample coverage rate for Boston dataset: 0.9705882

# True values for the test set
true_y_boston <- test_data$medv

# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston4[, "lwr"] & true_y_boston <= predictions_boston4[, "upr"])

cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)

## Out-of-sample coverage rate for Boston dataset: 0.9607843

# True values for the test set
true_y_boston <- test_data$medv

# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston_rf[, "lwr"] & true_y_boston <= predictions_boston_rf[, "upr"])

cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)

## Out-of-sample coverage rate for Boston dataset: 0.9215686

Results

• The prediction intervals are calculated using the split conformal method.
• The out-of-sample coverage rate is displayed, which should be close to the specified confidence level (e.g., 0.95).