---
title: "lightgbm: Binary Classification"
vignette: >
  %\VignetteEncoding{UTF-8}
  %\VignetteIndexEntry{lightgbm: Binary Classification}
  %\VignetteEngine{quarto::html}
editor_options: 
  chunk_output_type: console
execute:
  eval: false
  collapse: true
  comment: "#>"
---

```{r setup}
# nolint start
library(mlexperiments)
library(mllrnrs)
```

See [https://github.com/kapsner/mllrnrs/blob/main/R/learner_lightgbm.R](https://github.com/kapsner/mllrnrs/blob/main/R/learner_lightgbm.R) for implementation details.

# Preprocessing

## Import and Prepare Data

```{r}
library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
  data.table::as.data.table() |>
  na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"
```

## General Configurations

```{r}
seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
  # on cran
  ncores <- 2L
} else {
  ncores <- ifelse(
    test = parallel::detectCores() > 4,
    yes = 4L,
    no = ifelse(
      test = parallel::detectCores() < 2L,
      yes = 1L,
      no = parallel::detectCores()
    )
  )
}
options("mlexperiments.bayesian.max_init" = 4L)
options("mlexperiments.optim.lgb.nrounds" = 20L)
options("mlexperiments.optim.lgb.early_stopping_rounds" = 5L)
```

## Generate Training- and Test Data

```{r}
data_split <- splitTools::partition(
  y = dataset[, get(target_col)],
  p = c(train = 0.7, test = 0.3),
  type = "stratified",
  seed = seed
)

train_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L


test_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1L
```

## Generate Training Data Folds

```{r}
fold_list <- splitTools::create_folds(
  y = train_y,
  k = 3,
  type = "stratified",
  seed = seed
)
```

# Experiments

## Prepare Experiments

```{r}
# required learner arguments, not optimized
learner_args <- list(
  max_depth = -1L,
  verbose = -1L,
  objective = "binary",
  metric = "binary_logloss"
)

# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- NULL
performance_metric <- metric("auc")
performance_metric_args <- list(positive = "1", negative = "0")
return_models <- FALSE

# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
  bagging_fraction = seq(0.6, 1, .2),
  feature_fraction = seq(0.6, 1, .2),
  min_data_in_leaf = seq(2, 10, 2),
  learning_rate = seq(0.1, 0.2, 0.1),
  num_leaves = seq(2, 20, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
  set.seed(123)
  sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
  parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}

# required for bayesian optimization
parameter_bounds <- list(
  bagging_fraction = c(0.2, 1),
  feature_fraction = c(0.2, 1),
  min_data_in_leaf = c(2L, 12L),
  learning_rate = c(0.1, 0.2),
  num_leaves =  c(2L, 20L)
)
optim_args <- list(
  n_iter = ncores,
  kappa = 3.5,
  acq = "ucb"
)
```


## Hyperparameter Tuning

### Grid Search

```{r}
tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerLightgbm$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "grid",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)

head(tuner_results_grid)
#>    setting_id metric_optim_mean nrounds bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective
#>         <int>             <num>   <int>            <num>            <num>            <num>         <num>      <num>     <int>   <int>    <char>
#> 1:          1         0.1413020      18              0.6              0.6                4           0.2         18        -1      -1    binary
#> 2:          2         0.1336577      18              0.8              1.0               10           0.2          6        -1      -1    binary
#> 3:          3         0.1959308      20              0.8              0.8                4           0.1          2        -1      -1    binary
#> 4:          4         0.1665362      20              1.0              0.8                4           0.1         10        -1      -1    binary
#> 5:          5         0.1325429      20              1.0              0.6                6           0.2         18        -1      -1    binary
#> 6:          6         0.1651197      20              1.0              1.0                8           0.1         14        -1      -1    binary
#>            metric
#>            <char>
#> 1: binary_logloss
#> 2: binary_logloss
#> 3: binary_logloss
#> 4: binary_logloss
#> 5: binary_logloss
#> 6: binary_logloss
```


### Bayesian Optimization

```{r}
tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerLightgbm$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "bayesian",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds

tuner$learner_args <- learner_args
tuner$optim_args <- optim_args

tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.032  Round = 1   bagging_fraction = 0.8000   feature_fraction = 0.8000   min_data_in_leaf = 4.0000   learning_rate = 0.1000  num_leaves = 2.0000 Value = -0.1959308
#> elapsed = 0.04   Round = 2   bagging_fraction = 0.6000   feature_fraction = 0.6000   min_data_in_leaf = 6.0000   learning_rate = 0.2000  num_leaves = 10.0000    Value = -0.1288185
#> elapsed = 0.034  Round = 3   bagging_fraction = 0.8000   feature_fraction = 1.0000   min_data_in_leaf = 10.0000  learning_rate = 0.2000  num_leaves = 6.0000 Value = -0.1336577
#> elapsed = 0.038  Round = 4   bagging_fraction = 0.8000   feature_fraction = 0.6000   min_data_in_leaf = 8.0000   learning_rate = 0.1000  num_leaves = 14.0000    Value = -0.1622234
#> elapsed = 0.048  Round = 5   bagging_fraction = 0.8514907    feature_fraction = 0.5359993    min_data_in_leaf = 9.0000   learning_rate = 0.1801683   num_leaves = 20.0000    Value = -0.1392806
#> elapsed = 0.043  Round = 6   bagging_fraction = 0.2079895    feature_fraction = 0.385567 min_data_in_leaf = 3.0000   learning_rate = 0.1618143   num_leaves = 11.0000    Value = -0.137801
#> elapsed = 0.036  Round = 7   bagging_fraction = 0.8973112    feature_fraction = 0.2399434    min_data_in_leaf = 12.0000  learning_rate = 0.2000  num_leaves = 15.0000    Value = -0.1243682
#> elapsed = 0.044  Round = 8   bagging_fraction = 0.4285091    feature_fraction = 0.2176898    min_data_in_leaf = 12.0000  learning_rate = 0.1330769   num_leaves = 20.0000    Value = -0.1403338
#>
#>  Best Parameters Found:
#> Round = 7    bagging_fraction = 0.8973112    feature_fraction = 0.2399434    min_data_in_leaf = 12.0000  learning_rate = 0.2000  num_leaves = 15.0000    Value = -0.1243682

head(tuner_results_bayesian)
#>    setting_id bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves      Value max_depth verbose objective         metric
#>         <int>            <num>            <num>            <num>         <num>      <num>      <num>     <int>   <int>    <char>         <char>
#> 1:          1        0.8000000        0.8000000                4     0.1000000          2 -0.1959308        -1      -1    binary binary_logloss
#> 2:          2        0.6000000        0.6000000                6     0.2000000         10 -0.1288185        -1      -1    binary binary_logloss
#> 3:          3        0.8000000        1.0000000               10     0.2000000          6 -0.1336577        -1      -1    binary binary_logloss
#> 4:          4        0.8000000        0.6000000                8     0.1000000         14 -0.1622234        -1      -1    binary binary_logloss
#> 5:          5        0.8514907        0.5359993                9     0.1801683         20 -0.1392806        -1      -1    binary binary_logloss
#> 6:          6        0.2079895        0.3855670                3     0.1618143         11 -0.1378010        -1      -1    binary binary_logloss
#>    metric_optim_mean
#>                <num>
#> 1:         0.1959308
#> 2:         0.1288185
#> 3:         0.1336577
#> 4:         0.1622234
#> 5:         0.1392806
#> 6:         0.1378010
```

## k-Fold Cross Validation

```{r}
validator <- mlexperiments::MLCrossValidation$new(
  learner = mllrnrs::LearnerLightgbm$new(
    metric_optimization_higher_better = FALSE
  ),
  fold_list = fold_list,
  ncores = ncores,
  seed = seed
)

validator$learner_args <- tuner$results$best.setting[-1]

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3

head(validator_results)
#>      fold performance feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective         metric nrounds
#>    <char>       <num>            <num>            <num>         <num>      <num>     <int>   <int>    <char>         <char>   <int>
#> 1:  Fold1   0.9890556        0.2399434               12           0.2         15        -1      -1    binary binary_logloss      20
#> 2:  Fold2   0.9932386        0.2399434               12           0.2         15        -1      -1    binary binary_logloss      20
#> 3:  Fold3   0.9876374        0.2399434               12           0.2         15        -1      -1    binary binary_logloss      20
```


## Nested Cross Validation

### Inner Grid Search

```{r}
validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerLightgbm$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "grid",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)

head(validator_results)
#>      fold performance nrounds bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective
#>    <char>       <num>   <int>            <num>            <num>            <num>         <num>      <num>     <int>   <int>    <char>
#> 1:  Fold1   0.9902913      20              0.8              1.0               10           0.2          6        -1      -1    binary
#> 2:  Fold2   0.9883842      19              0.8              1.0               10           0.2          6        -1      -1    binary
#> 3:  Fold3   0.9862637      18              0.6              0.6                6           0.2         10        -1      -1    binary
#>            metric
#>            <char>
#> 1: binary_logloss
#> 2: binary_logloss
#> 3: binary_logloss
```


### Inner Bayesian Optimization

```{r}
validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerLightgbm$new(
    metric_optimization_higher_better = FALSE
  ),
  strategy = "bayesian",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"


validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.027  Round = 1   bagging_fraction = 0.8000   feature_fraction = 0.8000   min_data_in_leaf = 4.0000   learning_rate = 0.1000  num_leaves = 2.0000 Value = -0.2081399
#> elapsed = 0.035  Round = 2   bagging_fraction = 0.6000   feature_fraction = 0.6000   min_data_in_leaf = 6.0000   learning_rate = 0.2000  num_leaves = 10.0000    Value = -0.1575606
#> elapsed = 0.03   Round = 3   bagging_fraction = 0.8000   feature_fraction = 1.0000   min_data_in_leaf = 10.0000  learning_rate = 0.2000  num_leaves = 6.0000 Value = -0.1462635
#> elapsed = 0.036  Round = 4   bagging_fraction = 0.8000   feature_fraction = 0.6000   min_data_in_leaf = 8.0000   learning_rate = 0.1000  num_leaves = 14.0000    Value = -0.1733679
#> elapsed = 0.029  Round = 5   bagging_fraction = 0.5596134    feature_fraction = 0.9300915    min_data_in_leaf = 7.0000   learning_rate = 0.1800818   num_leaves = 2.0000 Value = -0.1645675
#> elapsed = 0.032  Round = 6   bagging_fraction = 0.8950514    feature_fraction = 0.2196616    min_data_in_leaf = 12.0000  learning_rate = 0.1620435   num_leaves = 20.0000    Value = -0.149935
#> elapsed = 0.025  Round = 7   bagging_fraction = 0.2000   feature_fraction = 0.991415 min_data_in_leaf = 12.0000  learning_rate = 0.1000  num_leaves = 2.0000 Value = -0.210863
#> elapsed = 0.043  Round = 8   bagging_fraction = 0.2000   feature_fraction = 0.8495294    min_data_in_leaf = 2.0000   learning_rate = 0.1000  num_leaves = 20.0000    Value = -0.2048307
#>
#>  Best Parameters Found:
#> Round = 3    bagging_fraction = 0.8000   feature_fraction = 1.0000   min_data_in_leaf = 10.0000  learning_rate = 0.2000  num_leaves = 6.0000 Value = -0.1462635
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.029  Round = 1   bagging_fraction = 0.8000   feature_fraction = 0.8000   min_data_in_leaf = 4.0000   learning_rate = 0.1000  num_leaves = 2.0000 Value = -0.1968844
#> elapsed = 0.04   Round = 2   bagging_fraction = 0.6000   feature_fraction = 0.6000   min_data_in_leaf = 6.0000   learning_rate = 0.2000  num_leaves = 10.0000    Value = -0.1448359
#> elapsed = 0.032  Round = 3   bagging_fraction = 0.8000   feature_fraction = 1.0000   min_data_in_leaf = 10.0000  learning_rate = 0.2000  num_leaves = 6.0000 Value = -0.1431115
#> elapsed = 0.035  Round = 4   bagging_fraction = 0.8000   feature_fraction = 0.6000   min_data_in_leaf = 8.0000   learning_rate = 0.1000  num_leaves = 14.0000    Value = -0.1744139
#> elapsed = 0.037  Round = 5   bagging_fraction = 0.5157127    feature_fraction = 0.9291875    min_data_in_leaf = 9.0000   learning_rate = 0.1626326   num_leaves = 6.0000 Value = -0.1437751
#> elapsed = 0.042  Round = 6   bagging_fraction = 0.803404 feature_fraction = 0.8105688    min_data_in_leaf = 9.0000   learning_rate = 0.1840982   num_leaves = 6.0000 Value = -0.1449229
#> elapsed = 0.034  Round = 7   bagging_fraction = 0.3970573    feature_fraction = 0.271388 min_data_in_leaf = 9.0000   learning_rate = 0.16237 num_leaves = 10.0000    Value = -0.1432943
#> elapsed = 0.039  Round = 8   bagging_fraction = 0.3970573    feature_fraction = 0.2713869    min_data_in_leaf = 9.0000   learning_rate = 0.16237 num_leaves = 10.0000    Value = -0.1432943
#>
#>  Best Parameters Found:
#> Round = 3    bagging_fraction = 0.8000   feature_fraction = 1.0000   min_data_in_leaf = 10.0000  learning_rate = 0.2000  num_leaves = 6.0000 Value = -0.1431115
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.026  Round = 1   bagging_fraction = 0.8000   feature_fraction = 0.8000   min_data_in_leaf = 4.0000   learning_rate = 0.1000  num_leaves = 2.0000 Value = -0.1935595
#> elapsed = 0.034  Round = 2   bagging_fraction = 0.6000   feature_fraction = 0.6000   min_data_in_leaf = 6.0000   learning_rate = 0.2000  num_leaves = 10.0000    Value = -0.1456779
#> elapsed = 0.03   Round = 3   bagging_fraction = 0.8000   feature_fraction = 1.0000   min_data_in_leaf = 10.0000  learning_rate = 0.2000  num_leaves = 6.0000 Value = -0.1592296
#> elapsed = 0.039  Round = 4   bagging_fraction = 0.8000   feature_fraction = 0.6000   min_data_in_leaf = 8.0000   learning_rate = 0.1000  num_leaves = 14.0000    Value = -0.1740933
#> elapsed = 0.029  Round = 5   bagging_fraction = 0.5157127    feature_fraction = 0.9291875    min_data_in_leaf = 9.0000   learning_rate = 0.1998166   num_leaves = 6.0000 Value = -0.1436106
#> elapsed = 0.042  Round = 6   bagging_fraction = 0.2000   feature_fraction = 0.4928526    min_data_in_leaf = 6.0000   learning_rate = 0.1736983   num_leaves = 20.0000    Value = -0.1451436
#> elapsed = 0.039  Round = 7   bagging_fraction = 0.8514907    feature_fraction = 0.5359993    min_data_in_leaf = 9.0000   learning_rate = 0.1068914   num_leaves = 20.0000    Value = -0.1693466
#> elapsed = 0.034  Round = 8   bagging_fraction = 0.2000   feature_fraction = 0.2000   min_data_in_leaf = 12.0000  learning_rate = 0.1000  num_leaves = 20.0000    Value = -0.1729361
#>
#>  Best Parameters Found:
#> Round = 5    bagging_fraction = 0.5157127    feature_fraction = 0.9291875    min_data_in_leaf = 9.0000   learning_rate = 0.1998166   num_leaves = 6.0000 Value = -0.1436106

head(validator_results)
#>      fold performance bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective         metric
#>    <char>       <num>            <num>            <num>            <num>         <num>      <num>     <int>   <int>    <char>         <char>
#> 1:  Fold1   0.9902913        0.8000000        1.0000000               10     0.2000000          6        -1      -1    binary binary_logloss
#> 2:  Fold2   0.9883842        0.8000000        1.0000000               10     0.2000000          6        -1      -1    binary binary_logloss
#> 3:  Fold3   0.9869505        0.5157127        0.9291875                9     0.1998166          6        -1      -1    binary binary_logloss
#>    nrounds
#>      <int>
#> 1:      20
#> 2:      19
#> 3:      19
```


## Holdout Test Dataset Performance

### Predict Outcome in Holdout Test Dataset

```{r}
preds_lightgbm <- mlexperiments::predictions(
  object = validator,
  newdata = test_x
)
```

### Evaluate Performance on Holdout Test Dataset

```{r}
perf_lightgbm <- mlexperiments::performance(
  object = validator,
  prediction_results = preds_lightgbm,
  y_ground_truth = test_y,
  type = "binary"
)
perf_lightgbm
#>     model performance       AUC      Brier BrierScaled       BAC    TP    TN    FP    FN       TPR       TNR         FPR        FNR       PPV
#>    <char>       <num>     <num>      <num>       <num>     <num> <int> <int> <int> <int>     <num>     <num>       <num>      <num>     <num>
#> 1:  Fold1   0.9764718 0.9764718 0.03740030   0.8354976 0.9407131    64   133     1     8 0.8888889 0.9925373 0.007462687 0.11111111 0.9846154
#> 2:  Fold2   0.9906198 0.9906198 0.04184296   0.8159569 0.9268242    62   133     1    10 0.8611111 0.9925373 0.007462687 0.13888889 0.9841270
#> 3:  Fold3   0.9811360 0.9811360 0.04052871   0.8217375 0.9401949    65   131     3     7 0.9027778 0.9776119 0.022388060 0.09722222 0.9558824
#>          NPV        FDR       MCC        F1     GMEAN       GPR       ACC       MMCE        BER
#>        <num>      <num>     <num>     <num>     <num>     <num>     <num>      <num>      <num>
#> 1: 0.9432624 0.01538462 0.9043538 0.9343066 0.9392845 0.9355286 0.9563107 0.04368932 0.05928690
#> 2: 0.9300699 0.01587302 0.8834041 0.9185185 0.9244917 0.9205665 0.9466019 0.05339806 0.07317579
#> 3: 0.9492754 0.04411765 0.8926878 0.9285714 0.9394500 0.9289507 0.9514563 0.04854369 0.05980514
```

```{r include=FALSE}
# nolint end
```
