---
title: "xgboost: Binary Classification"
vignette: >
  %\VignetteEncoding{UTF-8}
  %\VignetteIndexEntry{xgboost: Binary Classification}
  %\VignetteEngine{quarto::html}
editor_options:
  chunk_output_type: console
execute:
  eval: false
  collapse: true
  comment: "#>"
---

```{r setup}
# nolint start
library(mlexperiments)
library(mllrnrs)
```

See [https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R](https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R) for implementation details.

# Preprocessing

## Import and Prepare Data

```{r}
library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
  data.table::as.data.table() |>
  na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"
```

## General Configurations

```{r}
seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
  # on cran
  ncores <- 2L
} else {
  ncores <- ifelse(
    test = parallel::detectCores() > 4,
    yes = 4L,
    no = ifelse(
      test = parallel::detectCores() < 2L,
      yes = 1L,
      no = parallel::detectCores()
    )
  )
}
options("mlexperiments.bayesian.max_init" = 4L)
options("mlexperiments.optim.xgb.nrounds" = 20L)
options("mlexperiments.optim.xgb.early_stopping_rounds" = 5L)
```

## Generate Training- and Test Data

```{r}
data_split <- splitTools::partition(
  y = dataset[, get(target_col)],
  p = c(train = 0.7, test = 0.3),
  type = "stratified",
  seed = seed
)

train_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L


test_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1L
```

## Generate Training Data Folds

```{r}
fold_list <- splitTools::create_folds(
  y = train_y,
  k = 3,
  type = "stratified",
  seed = seed
)
```

# Experiments

## Prepare Experiments

```{r}
# required learner arguments, not optimized
learner_args <- list(
  objective = "binary:logistic",
  eval_metric = "logloss"
)

# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- NULL
performance_metric <- metric("auc")
performance_metric_args <- list(positive = "1", negative = "0")
return_models <- FALSE

# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
  subsample = seq(0.6, 1, .2),
  colsample_bytree = seq(0.6, 1, .2),
  min_child_weight = seq(1, 5, 4),
  learning_rate = seq(0.1, 0.2, 0.1),
  max_depth = seq(1, 5, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
  set.seed(123)
  sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
  parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}

# required for bayesian optimization
parameter_bounds <- list(
  subsample = c(0.2, 1),
  colsample_bytree = c(0.2, 1),
  min_child_weight = c(1L, 10L),
  learning_rate = c(0.1, 0.2),
  max_depth =  c(1L, 10L)
)
optim_args <- list(
  n_iter = ncores,
  kappa = 3.5,
  acq = "ucb"
)
```


## Hyperparameter Tuning

### Grid Search

```{r}
tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerXgboost$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "grid",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)

head(tuner_results_grid)
#>    setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth       objective eval_metric
#>         <int>             <num>   <int>     <num>            <num>            <num>         <num>     <num>          <char>      <char>
#> 1:          1         0.1473724      20       0.6              0.8                5           0.2         1 binary:logistic     logloss
#> 2:          2         0.1836365      20       1.0              0.8                5           0.1         5 binary:logistic     logloss
#> 3:          3         0.2004256      20       0.8              0.8                5           0.1         1 binary:logistic     logloss
#> 4:          4         0.1431941      20       0.6              0.8                5           0.2         5 binary:logistic     logloss
#> 5:          5         0.1766953      20       1.0              0.8                1           0.1         5 binary:logistic     logloss
#> 6:          6         0.1866896      20       0.8              0.8                5           0.1         5 binary:logistic     logloss
```


### Bayesian Optimization

```{r}
tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerXgboost$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "bayesian",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds

tuner$learner_args <- learner_args
tuner$optim_args <- optim_args

tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.045  Round = 1   subsample = 0.8000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 1.0000  Value = -0.2004256
#> elapsed = 0.037  Round = 2   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 1.0000  Value = -0.146329
#> elapsed = 0.033  Round = 3   subsample = 1.0000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 5.0000  Value = -0.1836365
#> elapsed = 0.033  Round = 4   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 5.0000   learning_rate = 0.2000  max_depth = 5.0000  Value = -0.143957
#> elapsed = 0.026  Round = 5   subsample = 0.2000  colsample_bytree = 0.2253119    min_child_weight = 6.0000   learning_rate = 0.1478213   max_depth = 9.0000  Value = -0.2851956
#> elapsed = 0.048  Round = 6   subsample = 1.0000  colsample_bytree = 0.6428195    min_child_weight = 7.0000   learning_rate = 0.2000  max_depth = 10.0000 Value = -0.1407496
#> elapsed = 0.029  Round = 7   subsample = 0.6370911   colsample_bytree = 0.8772461    min_child_weight = 3.0000   learning_rate = 0.1746002   max_depth = 4.0000  Value = -0.1423412
#> elapsed = 0.034  Round = 8   subsample = 1.0000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.1753007   max_depth = 10.0000 Value = -0.1424366
#>
#>  Best Parameters Found:
#> Round = 6    subsample = 1.0000  colsample_bytree = 0.6428195    min_child_weight = 7.0000   learning_rate = 0.2000  max_depth = 10.0000 Value = -0.1407496

head(tuner_results_bayesian)
#>    setting_id subsample colsample_bytree min_child_weight learning_rate max_depth      Value       objective eval_metric metric_optim_mean
#>         <int>     <num>            <num>            <num>         <num>     <num>      <num>          <char>      <char>             <num>
#> 1:          1       0.8        0.8000000                5     0.1000000         1 -0.2004256 binary:logistic     logloss         0.2004256
#> 2:          2       0.6        1.0000000                1     0.2000000         1 -0.1463290 binary:logistic     logloss         0.1463290
#> 3:          3       1.0        0.8000000                5     0.1000000         5 -0.1836365 binary:logistic     logloss         0.1836365
#> 4:          4       0.6        1.0000000                5     0.2000000         5 -0.1439570 binary:logistic     logloss         0.1439570
#> 5:          5       0.2        0.2253119                6     0.1478213         9 -0.2851956 binary:logistic     logloss         0.2851956
#> 6:          6       1.0        0.6428195                7     0.2000000        10 -0.1407496 binary:logistic     logloss         0.1407496
```

## k-Fold Cross Validation

```{r}
validator <- mlexperiments::MLCrossValidation$new(
  learner = mllrnrs::LearnerXgboost$new(
    metric_optimization_higher_better = FALSE
  ),
  fold_list = fold_list,
  ncores = ncores,
  seed = seed
)

validator$learner_args <- tuner$results$best.setting[-1]

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3

head(validator_results)
#>      fold performance colsample_bytree min_child_weight learning_rate max_depth       objective eval_metric nrounds
#>    <char>       <num>            <num>            <num>         <num>     <num>          <char>      <char>   <int>
#> 1:  Fold1   0.9920565        0.6428195                7           0.2        10 binary:logistic     logloss      20
#> 2:  Fold2   0.9928918        0.6428195                7           0.2        10 binary:logistic     logloss      20
#> 3:  Fold3   0.9818853        0.6428195                7           0.2        10 binary:logistic     logloss      20
```


## Nested Cross Validation

### Inner Grid Search

```{r}
validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerXgboost$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "grid",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)

head(validator_results)
#>      fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth       objective eval_metric
#>    <char>       <num>   <int>     <num>            <num>            <num>         <num>     <num>          <char>      <char>
#> 1:  Fold1   0.9876434      20       0.6                1                1           0.2         1 binary:logistic     logloss
#> 2:  Fold2   0.9905513      20       0.6                1                1           0.2         1 binary:logistic     logloss
#> 3:  Fold3   0.9843750      20       0.6                1                1           0.2         1 binary:logistic     logloss
```


### Inner Bayesian Optimization

```{r}
validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerXgboost$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "bayesian",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"


validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.045  Round = 1   subsample = 0.8000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 1.0000  Value = -0.2144507
#> elapsed = 0.025  Round = 2   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 1.0000  Value = -0.1632438
#> elapsed = 0.028  Round = 3   subsample = 1.0000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 5.0000  Value = -0.1995263
#> elapsed = 0.037  Round = 4   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 5.0000   learning_rate = 0.2000  max_depth = 5.0000  Value = -0.1765465
#> elapsed = 0.052  Round = 5   subsample = 0.6587262   colsample_bytree = 0.2441275    min_child_weight = 4.0000   learning_rate = 0.1583349   max_depth = 8.0000  Value = -0.1751771
#> elapsed = 0.032  Round = 6   subsample = 0.2000  colsample_bytree = 0.7380485    min_child_weight = 1.0000   learning_rate = 0.1221712   max_depth = 6.0000  Value = -0.2001825
#> elapsed = 0.026  Round = 7   subsample = 0.4372448   colsample_bytree = 0.304417 min_child_weight = 1.0000   learning_rate = 0.1949787   max_depth = 8.0000  Value = -0.1653602
#> elapsed = 0.029  Round = 8   subsample = 1.0000  colsample_bytree = 0.3752773    min_child_weight = 1.0000   learning_rate = 0.176386    max_depth = 9.0000  Value = -0.151424
#>
#>  Best Parameters Found:
#> Round = 8    subsample = 1.0000  colsample_bytree = 0.3752773    min_child_weight = 1.0000   learning_rate = 0.176386    max_depth = 9.0000  Value = -0.151424
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.024  Round = 1   subsample = 0.8000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 1.0000  Value = -0.2016618
#> elapsed = 0.024  Round = 2   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 1.0000  Value = -0.1358879
#> elapsed = 0.036  Round = 3   subsample = 1.0000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 5.0000  Value = -0.1937136
#> elapsed = 0.025  Round = 4   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 5.0000   learning_rate = 0.2000  max_depth = 5.0000  Value = -0.1629105
#> elapsed = 0.026  Round = 5   subsample = 0.9597928   colsample_bytree = 0.2087203    min_child_weight = 1.0000   learning_rate = 0.1807996   max_depth = 3.0000  Value = -0.1409365
#> elapsed = 0.026  Round = 6   subsample = 0.9648661   colsample_bytree = 0.2000   min_child_weight = 10.0000  learning_rate = 0.2000  max_depth = 10.0000 Value = -0.1876724
#> elapsed = 0.027  Round = 7   subsample = 0.2232804   colsample_bytree = 0.89331  min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 10.0000 Value = -0.147581
#> elapsed = 0.051  Round = 8   subsample = 0.2000  colsample_bytree = 0.8707313    min_child_weight = 1.0000   learning_rate = 0.1000  max_depth = 10.0000 Value = -0.2066387
#>
#>  Best Parameters Found:
#> Round = 2    subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 1.0000  Value = -0.1358879
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.042  Round = 1   subsample = 0.8000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 1.0000  Value = -0.1996737
#> elapsed = 0.023  Round = 2   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 1.0000  Value = -0.1550507
#> elapsed = 0.03   Round = 3   subsample = 1.0000  colsample_bytree = 0.8000   min_child_weight = 5.0000   learning_rate = 0.1000  max_depth = 5.0000  Value = -0.1921469
#> elapsed = 0.024  Round = 4   subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 5.0000   learning_rate = 0.2000  max_depth = 5.0000  Value = -0.1630041
#> elapsed = 0.028  Round = 5   subsample = 0.8696134   colsample_bytree = 0.5493993    min_child_weight = 2.0000   learning_rate = 0.1541663   max_depth = 4.0000  Value = -0.156581
#> elapsed = 0.012  Round = 6   subsample = 0.2000  colsample_bytree = 0.4853539    min_child_weight = 10.0000  learning_rate = 0.1531602   max_depth = 10.0000 Value = -0.6478907
#> elapsed = 0.05   Round = 7   subsample = 0.2232806   colsample_bytree = 1.0000   min_child_weight = 3.0000   learning_rate = 0.1040753   max_depth = 10.0000 Value = -0.2244361
#> elapsed = 0.024  Round = 8   subsample = 0.5157127   colsample_bytree = 0.9291875    min_child_weight = 7.0000   learning_rate = 0.1998166   max_depth = 3.0000  Value = -0.2151049
#>
#>  Best Parameters Found:
#> Round = 2    subsample = 0.6000  colsample_bytree = 1.0000   min_child_weight = 1.0000   learning_rate = 0.2000  max_depth = 1.0000  Value = -0.1550507

head(validator_results)
#>      fold performance subsample colsample_bytree min_child_weight learning_rate max_depth       objective eval_metric nrounds
#>    <char>       <num>     <num>            <num>            <num>         <num>     <num>          <char>      <char>   <int>
#> 1:  Fold1   0.9955869       1.0        0.3752773                1      0.176386         9 binary:logistic     logloss      20
#> 2:  Fold2   0.9905513       0.6        1.0000000                1      0.200000         1 binary:logistic     logloss      20
#> 3:  Fold3   0.9843750       0.6        1.0000000                1      0.200000         1 binary:logistic     logloss      20
```


## Holdout Test Dataset Performance

### Predict Outcome in Holdout Test Dataset

```{r}
preds_xgboost <- mlexperiments::predictions(
  object = validator,
  newdata = test_x
)
```

### Evaluate Performance on Holdout Test Dataset

```{r}
perf_xgboost <- mlexperiments::performance(
  object = validator,
  prediction_results = preds_xgboost,
  y_ground_truth = test_y,
  type = "binary"
)
perf_xgboost
#>     model performance       AUC      Brier BrierScaled       BAC    TP    TN    FP    FN       TPR       TNR         FPR       FNR       PPV
#>    <char>       <num>     <num>      <num>       <num>     <num> <int> <int> <int> <int>     <num>     <num>       <num>     <num>     <num>
#> 1:  Fold1   0.9839863 0.9839863 0.04247381   0.8131821 0.9369818    64   132     2     8 0.8888889 0.9850746 0.014925373 0.1111111 0.9696970
#> 2:  Fold2   0.9812915 0.9812915 0.04675659   0.7943447 0.9059909    59   133     1    13 0.8194444 0.9925373 0.007462687 0.1805556 0.9833333
#> 3:  Fold3   0.9830535 0.9830535 0.04884222   0.7851712 0.9161484    61   132     2    11 0.8472222 0.9850746 0.014925373 0.1527778 0.9682540
#>          NPV        FDR       MCC        F1     GMEAN       GPR       ACC       MMCE        BER
#>        <num>      <num>     <num>     <num>     <num>     <num>     <num>      <num>      <num>
#> 1: 0.9428571 0.03030303 0.8930504 0.9275362 0.9357467 0.9284142 0.9514563 0.04854369 0.06301824
#> 2: 0.9109589 0.01666667 0.8521438 0.8939394 0.9018477 0.8976564 0.9320388 0.06796117 0.09400912
#> 3: 0.9230769 0.03174603 0.8613082 0.9037037 0.9135519 0.9057187 0.9368932 0.06310680 0.08385158
```

```{r include=FALSE}
# nolint end
```
