Fit logistic regression model

Fit logistic regression model#

  • based on different imputation methods

  • baseline: reference

  • model: any other selected imputation method

Hide code cell source

 
import logging
from pathlib import Path
from typing import List

import matplotlib.pyplot as plt
import njab.sklearn
import pandas as pd
import sklearn
from njab.plotting.metrics import plot_split_auc, plot_split_prc
from njab.sklearn.types import Splits

import pimmslearn
import pimmslearn.analyzers
import pimmslearn.io.datasplits

plt.rcParams['figure.figsize'] = (2.5, 2.5)
plt.rcParams['lines.linewidth'] = 1
plt.rcParams['lines.markersize'] = 2
fontsize = 5
figsize = (2.5, 2.5)
pimmslearn.plotting.make_large_descriptors(fontsize)


logger = pimmslearn.logging.setup_nb_logger()
logging.getLogger('fontTools').setLevel(logging.ERROR)


def parse_roc(*res: List[njab.sklearn.types.Results]) -> pd.DataFrame:
    ret = list()
    for _r in res:
        _roc = (pd.DataFrame(_r.test.roc,
                             index='fpr tpr cutoffs'.split()
                             )).loc[['fpr', 'tpr']]
        _roc = _roc.T
        _roc.columns = pd.MultiIndex.from_product([[_r.name], _roc.columns])
        ret.append(_roc)
    ret = pd.concat(ret, axis=1)
    return ret


def parse_prc(*res: List[njab.sklearn.types.Results]) -> pd.DataFrame:
    ret = list()
    for _r in res:
        _prc = pd.DataFrame(_r.test.prc,
                            index='precision recall cutoffs'.split()
                            ).loc[['precision', 'recall']]
        _prc = _prc.T.rename(columns={'recall': 'tpr'})
        _prc.columns = pd.MultiIndex.from_product([[_r.name], _prc.columns])
        ret.append(_prc)
    ret = pd.concat(ret, axis=1)
    return ret


# catch passed parameters
args = None
args = dict(globals()).keys()

Parameters#

Default and set parameters for the notebook.

folder_data: str = ''  # specify data directory if needed
fn_clinical_data = "data/ALD_study/processed/ald_metadata_cli.csv"
folder_experiment = "runs/appl_ald_data/plasma/proteinGroups"
model_key = 'VAE'
target = 'kleiner'
sample_id_col = 'Sample ID'
cutoff_target: int = 2  # => for binarization target >= cutoff_target
file_format = "csv"
out_folder = 'diff_analysis'
fn_qc_samples = ''  # 'data/ALD_study/processed/qc_plasma_proteinGroups.pkl'

baseline = 'RSN'  # default is RSN, as this was used in the original ALD Niu. et. al 2022
template_pred = 'pred_real_na_{}.csv'  # fixed, do not change

# Parameters
cutoff_target = 0.5
folder_experiment = "runs/alzheimer_study"
target = "AD"
baseline = "PI"
model_key = "QRILC"
out_folder = "diff_analysis"
fn_clinical_data = "runs/alzheimer_study/data/clinical_data.csv"

Hide code cell source

 
params = pimmslearn.nb.get_params(args, globals=globals())
args = pimmslearn.nb.Config()
args.folder_experiment = Path(params["folder_experiment"])
args = pimmslearn.nb.add_default_paths(args,
                                 out_root=(args.folder_experiment
                                           / params["out_folder"]
                                           / params["target"]
                                           / f"{params['baseline']}_vs_{params['model_key']}"))
args.update_from_dict(params)
files_out = dict()
args

root - INFO     Removed from global namespace: folder_data
root - INFO     Removed from global namespace: fn_clinical_data
root - INFO     Removed from global namespace: folder_experiment
root - INFO     Removed from global namespace: model_key
root - INFO     Removed from global namespace: target
root - INFO     Removed from global namespace: sample_id_col
root - INFO     Removed from global namespace: cutoff_target
root - INFO     Removed from global namespace: file_format
root - INFO     Removed from global namespace: out_folder
root - INFO     Removed from global namespace: fn_qc_samples
root - INFO     Removed from global namespace: baseline
root - INFO     Removed from global namespace: template_pred
root - INFO     Already set attribute: folder_experiment has value runs/alzheimer_study
root - INFO     Already set attribute: out_folder has value diff_analysis

{'baseline': 'PI',
 'cutoff_target': 0.5,
 'data': PosixPath('runs/alzheimer_study/data'),
 'file_format': 'csv',
 'fn_clinical_data': 'runs/alzheimer_study/data/clinical_data.csv',
 'fn_qc_samples': '',
 'folder_data': '',
 'folder_experiment': PosixPath('runs/alzheimer_study'),
 'model_key': 'QRILC',
 'out_figures': PosixPath('runs/alzheimer_study/figures'),
 'out_folder': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC'),
 'out_metrics': PosixPath('runs/alzheimer_study'),
 'out_models': PosixPath('runs/alzheimer_study'),
 'out_preds': PosixPath('runs/alzheimer_study/preds'),
 'sample_id_col': 'Sample ID',
 'target': 'AD',
 'template_pred': 'pred_real_na_{}.csv'}

Load data#

Load target#

target = pd.read_csv(args.fn_clinical_data,
                     index_col=0,
                     usecols=[args.sample_id_col, args.target])
target = target.dropna()
target
AD
Sample ID
Sample_000 0
Sample_001 1
Sample_002 1
Sample_003 1
Sample_004 1
... ...
Sample_205 1
Sample_206 0
Sample_207 0
Sample_208 0
Sample_209 0

210 rows × 1 columns

MS proteomics or specified omics data#

Aggregated from data splits of the imputation workflow run before.

Hide code cell source

 
data = pimmslearn.io.datasplits.DataSplits.from_folder(
    args.data, file_format=args.file_format)
data = pd.concat([data.train_X, data.val_y, data.test_y])
data.sample(5)

pimmslearn.io.datasplits - INFO     Loaded 'train_X' from file: runs/alzheimer_study/data/train_X.csv
pimmslearn.io.datasplits - INFO     Loaded 'val_y' from file: runs/alzheimer_study/data/val_y.csv
pimmslearn.io.datasplits - INFO     Loaded 'test_y' from file: runs/alzheimer_study/data/test_y.csv

Sample ID   protein groups                   
Sample_175  Q504Y0;Q504Y0-3;Q504Y0-4;Q504Y0-5   12.502
Sample_032  P01780                              20.278
Sample_072  A0A182DWH7;P49908                   19.087
Sample_067  A0A0C4DGV8;Q13214;Q13214-2          15.283
Sample_136  Q96MK3                              15.706
Name: intensity, dtype: float64

Get overlap between independent features and target

Select by ALD criteria#

Use parameters as specified in ALD study.

Hide code cell source

 
DATA_COMPLETENESS = 0.6
MIN_N_PROTEIN_GROUPS: int = 200
FRAC_PROTEIN_GROUPS: int = 0.622
CV_QC_SAMPLE: float = 0.4

ald_study, cutoffs = pimmslearn.analyzers.diff_analysis.select_raw_data(data.unstack(
), data_completeness=DATA_COMPLETENESS, frac_protein_groups=FRAC_PROTEIN_GROUPS)

if args.fn_qc_samples:
    qc_samples = pd.read_pickle(args.fn_qc_samples)
    qc_samples = qc_samples[ald_study.columns]
    qc_cv_feat = qc_samples.std() / qc_samples.mean()
    qc_cv_feat = qc_cv_feat.rename(qc_samples.columns.name)
    fig, ax = plt.subplots(figsize=(4, 7))
    ax = qc_cv_feat.plot.box(ax=ax)
    ax.set_ylabel('Coefficient of Variation')
    print((qc_cv_feat < CV_QC_SAMPLE).value_counts())
    ald_study = ald_study[pimmslearn.analyzers.diff_analysis.select_feat(qc_samples)]

column_name_first_prot_to_pg = {
    pg.split(';')[0]: pg for pg in data.unstack().columns}

ald_study = ald_study.rename(columns=column_name_first_prot_to_pg)
ald_study

root - INFO     Initally: N samples: 210, M feat: 1421
root - INFO     Dropped features quantified in less than 126 samples.
root - INFO     After feat selection: N samples: 210, M feat: 1213
root - INFO     Min No. of Protein-Groups in single sample: 754
root - INFO     Finally: N samples: 210, M feat: 1213
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 A0A075B6J9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 15.912 16.852 15.570 16.481 20.246 16.764 17.584 16.988 20.054 NaN ... 16.012 15.178 NaN 15.050 16.842 19.863 NaN 19.563 12.837 12.805
Sample_001 15.936 16.874 15.519 16.387 19.941 18.786 17.144 NaN 19.067 16.188 ... 15.528 15.576 NaN 14.833 16.597 20.299 15.556 19.386 13.970 12.442
Sample_002 16.111 14.523 15.935 16.416 19.251 16.832 15.671 17.012 18.569 NaN ... 15.229 14.728 13.757 15.118 17.440 19.598 15.735 20.447 12.636 12.505
Sample_003 16.107 17.032 15.802 16.979 19.628 17.852 18.877 14.182 18.985 13.438 ... 15.495 14.590 14.682 15.140 17.356 19.429 NaN 20.216 12.627 12.445
Sample_004 15.603 15.331 15.375 16.679 20.450 18.682 17.081 14.140 19.686 14.495 ... 14.757 15.094 14.048 15.256 17.075 19.582 15.328 19.867 13.145 12.235
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 17.705 17.039 NaN 16.413 19.102 16.064 ... 15.235 15.684 14.236 15.415 17.551 17.922 16.340 19.928 12.929 11.802
Sample_206 15.798 17.554 15.600 15.938 18.154 18.152 16.503 16.860 18.538 15.288 ... 15.422 16.106 NaN 15.345 17.084 18.708 14.249 19.433 NaN NaN
Sample_207 15.739 16.877 15.469 16.898 18.636 17.950 16.321 16.401 18.849 17.580 ... 15.808 16.098 14.403 15.715 16.586 18.725 16.138 19.599 13.637 11.174
Sample_208 15.477 16.779 14.995 16.132 14.908 17.530 NaN 16.119 18.368 15.202 ... 15.157 16.712 NaN 14.640 16.533 19.411 15.807 19.545 13.216 NaN
Sample_209 15.727 17.261 15.175 16.235 17.893 17.744 16.371 15.780 18.806 16.532 ... 15.237 15.652 15.211 14.205 16.749 19.275 15.732 19.577 11.042 11.791

210 rows × 1213 columns

Number of complete cases which can be used:

Hide code cell source

 
mask_has_target = data.index.levels[0].intersection(target.index)
assert not mask_has_target.empty, f"No data for target: {data.index.levels[0]} and {target.index}"
print(
    f"Samples available both in proteomics data and for target: {len(mask_has_target)}")
target, data, ald_study = target.loc[mask_has_target], data.loc[mask_has_target], ald_study.loc[mask_has_target]

Samples available both in proteomics data and for target: 210

Load imputations from specified model#

Hide code cell source

 
fname = args.out_preds / args.template_pred.format(args.model_key)
print(f"missing values pred. by {args.model_key}: {fname}")
load_single_csv_pred_file = pimmslearn.analyzers.compare_predictions.load_single_csv_pred_file
pred_real_na = load_single_csv_pred_file(fname).loc[mask_has_target]
pred_real_na.sample(3)

missing values pred. by QRILC: runs/alzheimer_study/preds/pred_real_na_QRILC.csv

Sample ID   protein groups                    
Sample_181  E9PKE3;P11142                        13.805
Sample_049  J3QQR8;J3QQX6;J3QRQ1;J3QRT5;P13598   15.078
Sample_168  Q9H3T2;Q9H3T2-3                      12.336
Name: intensity, dtype: float64

Load imputations from baseline model#

Hide code cell source

 
fname = args.out_preds / args.template_pred.format(args.baseline)
pred_real_na_baseline = load_single_csv_pred_file(fname)  # .loc[mask_has_target]
pred_real_na_baseline

Sample ID   protein groups          
Sample_000  A0A075B6J9                 14.940
            A0A075B6Q5                 13.757
            A0A075B6R2                 12.682
            A0A075B6S5                 12.439
            A0A087WSY4                 13.868
                                        ...  
Sample_209  Q9P1W8;Q9P1W8-2;Q9P1W8-4   12.389
            Q9UI40;Q9UI40-2            12.263
            Q9UIW2                     13.217
            Q9UMX0;Q9UMX0-2;Q9UMX0-4   12.512
            Q9UP79                     12.945
Name: intensity, Length: 46401, dtype: float64

Modeling setup#

General approach:

  • use one train, test split of the data

  • select best 10 features from training data X_train, y_train before binarization of target

  • dichotomize (binarize) data into to groups (zero and 1)

  • evaluate model on the test data X_test, y_test

Repeat general approach for

  1. all original ald data: all features justed in original ALD study

  2. all model data: all features available my using the self supervised deep learning model

  3. newly available feat only: the subset of features available from the self supervised deep learning model which were newly retained using the new approach

All data:

Hide code cell source

 
X = pd.concat([data, pred_real_na]).unstack()
X
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H7 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 15.912 16.852 15.570 16.481 17.301 20.246 16.764 17.584 16.988 20.054 ... 16.012 15.178 10.370 15.050 16.842 19.863 13.442 19.563 12.837 12.805
Sample_001 15.936 16.874 15.519 16.387 13.796 19.941 18.786 17.144 14.561 19.067 ... 15.528 15.576 12.473 14.833 16.597 20.299 15.556 19.386 13.970 12.442
Sample_002 16.111 14.523 15.935 16.416 18.175 19.251 16.832 15.671 17.012 18.569 ... 15.229 14.728 13.757 15.118 17.440 19.598 15.735 20.447 12.636 12.505
Sample_003 16.107 17.032 15.802 16.979 15.963 19.628 17.852 18.877 14.182 18.985 ... 15.495 14.590 14.682 15.140 17.356 19.429 13.728 20.216 12.627 12.445
Sample_004 15.603 15.331 15.375 16.679 15.473 20.450 18.682 17.081 14.140 19.686 ... 14.757 15.094 14.048 15.256 17.075 19.582 15.328 19.867 13.145 12.235
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 13.441 17.705 17.039 14.193 16.413 19.102 ... 15.235 15.684 14.236 15.415 17.551 17.922 16.340 19.928 12.929 11.802
Sample_206 15.798 17.554 15.600 15.938 13.013 18.154 18.152 16.503 16.860 18.538 ... 15.422 16.106 11.659 15.345 17.084 18.708 14.249 19.433 10.770 8.330
Sample_207 15.739 16.877 15.469 16.898 10.220 18.636 17.950 16.321 16.401 18.849 ... 15.808 16.098 14.403 15.715 16.586 18.725 16.138 19.599 13.637 11.174
Sample_208 15.477 16.779 14.995 16.132 14.313 14.908 17.530 14.092 16.119 18.368 ... 15.157 16.712 11.500 14.640 16.533 19.411 15.807 19.545 13.216 8.985
Sample_209 15.727 17.261 15.175 16.235 14.822 17.893 17.744 16.371 15.780 18.806 ... 15.237 15.652 15.211 14.205 16.749 19.275 15.732 19.577 11.042 11.791

210 rows × 1421 columns

Subset of data by ALD criteria#

Hide code cell source

 
# could be just observed, drop columns with missing values
ald_study = pd.concat(
    [ald_study.stack(),
     pred_real_na_baseline.loc[
        # only select columns in selected in ald_study
        pd.IndexSlice[:, pred_real_na.index.levels[-1].intersection(ald_study.columns)]
    ]
    ]
).unstack()
ald_study
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 A0A075B6K4 ... O14793 O95479;R4GMU1 P01282;P01282-2 P10619;P10619-2;X6R5C5;X6R8A1 P21810 Q14956;Q14956-2 Q6ZMP0;Q6ZMP0-2 Q9HBW1 Q9NY15 P17050
Sample ID
Sample_000 15.912 16.852 15.570 16.481 20.246 16.764 17.584 16.988 20.054 16.148 ... 14.051 12.078 12.537 13.475 14.440 13.662 12.857 14.353 12.748 13.655
Sample_001 15.936 16.874 15.519 16.387 19.941 18.786 17.144 13.042 19.067 16.127 ... 12.330 12.439 12.044 12.454 13.382 12.739 12.921 13.182 13.122 13.202
Sample_002 16.111 14.523 15.935 16.416 19.251 16.832 15.671 17.012 18.569 15.387 ... 12.583 12.572 12.103 13.894 11.879 12.361 12.745 12.834 13.456 12.397
Sample_003 16.107 17.032 15.802 16.979 19.628 17.852 18.877 14.182 18.985 16.565 ... 12.065 12.135 12.925 13.803 12.223 12.104 11.823 14.041 14.192 13.144
Sample_004 15.603 15.331 15.375 16.679 20.450 18.682 17.081 14.140 19.686 16.418 ... 12.839 12.940 13.356 12.339 12.095 13.547 12.575 13.088 13.740 11.047
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 17.705 17.039 13.027 16.413 19.102 15.350 ... 14.269 14.064 16.826 18.182 15.225 15.044 14.192 16.605 14.995 14.257
Sample_206 15.798 17.554 15.600 15.938 18.154 18.152 16.503 16.860 18.538 16.582 ... 14.273 17.700 16.802 20.202 15.280 15.086 13.978 18.086 15.557 14.171
Sample_207 15.739 16.877 15.469 16.898 18.636 17.950 16.321 16.401 18.849 15.768 ... 14.473 16.882 16.917 20.105 15.690 15.135 13.138 17.066 15.706 15.690
Sample_208 15.477 16.779 14.995 16.132 14.908 17.530 12.648 16.119 18.368 17.560 ... 15.234 17.175 16.521 18.859 15.305 15.161 13.006 17.917 15.396 14.371
Sample_209 15.727 17.261 15.175 16.235 17.893 17.744 16.371 15.780 18.806 16.338 ... 14.556 16.656 16.954 18.493 15.823 14.626 13.385 17.767 15.687 13.573

210 rows × 1213 columns

Features which would not have been included using ALD criteria:

Hide code cell source

 
new_features = X.columns.difference(ald_study.columns)
new_features

Index(['A0A075B6H7', 'A0A075B6Q5', 'A0A075B7B8', 'A0A087WSY4',
       'A0A087WTT8;A0A0A0MQX5;O94779;O94779-2', 'A0A087WXB8;Q9Y274',
       'A0A087WXE9;E9PQ70;Q6UXH9;Q6UXH9-2;Q6UXH9-3',
       'A0A087X1Z2;C9JTV4;H0Y4Y4;Q8WYH2;Q96C19;Q9BUP0;Q9BUP0-2',
       'A0A0A0MQS9;A0A0A0MTC7;Q16363;Q16363-2', 'A0A0A0MSN4;P12821;P12821-2',
       ...
       'Q9NZ94;Q9NZ94-2;Q9NZ94-3', 'Q9NZU1', 'Q9P1W8;Q9P1W8-2;Q9P1W8-4',
       'Q9UHI8', 'Q9UI40;Q9UI40-2',
       'Q9UIB8;Q9UIB8-2;Q9UIB8-3;Q9UIB8-4;Q9UIB8-5;Q9UIB8-6',
       'Q9UKZ4;Q9UKZ4-2', 'Q9UMX0;Q9UMX0-2;Q9UMX0-4', 'Q9Y281;Q9Y281-3',
       'Q9Y490'],
      dtype='object', name='protein groups', length=208)

Binarize targets, but also keep groups for stratification

Hide code cell source

 
target_to_group = target.copy()
target = target >= args.cutoff_target
pd.crosstab(target.squeeze(), target_to_group.squeeze())
AD 0 1
AD
False 122 0
True 0 88

Determine best number of parameters by cross validation procedure#

using subset of data by ALD criteria:

Hide code cell source

 
cv_feat_ald = njab.sklearn.find_n_best_features(X=ald_study, y=target, name=args.target,
                                                groups=target_to_group)
cv_feat_ald = (cv_feat_ald
               .drop('test_case', axis=1)
               .groupby('n_features')
               .agg(['mean', 'std']))
cv_feat_ald

  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 164.12it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
100%|██████████| 2/2 [00:00<00:00,  5.84it/s]
100%|██████████| 2/2 [00:00<00:00,  5.83it/s]
  0%|          | 0/3 [00:00<?, ?it/s]
 67%|██████▋   | 2/3 [00:00<00:00,  4.73it/s]
100%|██████████| 3/3 [00:00<00:00,  3.63it/s]
100%|██████████| 3/3 [00:00<00:00,  3.80it/s]
  0%|          | 0/4 [00:00<?, ?it/s]
 50%|█████     | 2/4 [00:00<00:00,  8.19it/s]
 75%|███████▌  | 3/4 [00:00<00:00,  5.49it/s]
100%|██████████| 4/4 [00:00<00:00,  4.42it/s]
100%|██████████| 4/4 [00:00<00:00,  4.93it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 40%|████      | 2/5 [00:00<00:00,  7.46it/s]
 60%|██████    | 3/5 [00:00<00:00,  5.55it/s]
 80%|████████  | 4/5 [00:00<00:00,  3.79it/s]
100%|██████████| 5/5 [00:01<00:00,  3.76it/s]
100%|██████████| 5/5 [00:01<00:00,  4.21it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
 33%|███▎      | 2/6 [00:00<00:00,  9.16it/s]
 50%|█████     | 3/6 [00:00<00:00,  4.69it/s]
 67%|██████▋   | 4/6 [00:01<00:00,  3.24it/s]
 83%|████████▎ | 5/6 [00:01<00:00,  2.56it/s]
100%|██████████| 6/6 [00:02<00:00,  2.32it/s]
100%|██████████| 6/6 [00:02<00:00,  2.85it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 29%|██▊       | 2/7 [00:00<00:01,  4.08it/s]
 43%|████▎     | 3/7 [00:00<00:01,  2.91it/s]
 57%|█████▋    | 4/7 [00:01<00:01,  2.36it/s]
 71%|███████▏  | 5/7 [00:01<00:00,  2.35it/s]
 86%|████████▌ | 6/7 [00:02<00:00,  2.42it/s]
100%|██████████| 7/7 [00:02<00:00,  2.67it/s]
100%|██████████| 7/7 [00:02<00:00,  2.65it/s]
  0%|          | 0/8 [00:00<?, ?it/s]
 25%|██▌       | 2/8 [00:00<00:01,  3.74it/s]
 38%|███▊      | 3/8 [00:01<00:01,  2.71it/s]
 50%|█████     | 4/8 [00:01<00:01,  2.40it/s]
 62%|██████▎   | 5/8 [00:02<00:01,  2.16it/s]
 75%|███████▌  | 6/8 [00:02<00:00,  2.19it/s]
 88%|████████▊ | 7/8 [00:02<00:00,  2.49it/s]
100%|██████████| 8/8 [00:03<00:00,  2.82it/s]
100%|██████████| 8/8 [00:03<00:00,  2.60it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
 22%|██▏       | 2/9 [00:00<00:01,  3.81it/s]
 33%|███▎      | 3/9 [00:00<00:01,  3.13it/s]
 44%|████▍     | 4/9 [00:01<00:01,  3.06it/s]
 56%|█████▌    | 5/9 [00:01<00:01,  2.99it/s]
 67%|██████▋   | 6/9 [00:01<00:01,  2.99it/s]
 78%|███████▊  | 7/9 [00:02<00:00,  3.07it/s]
 89%|████████▉ | 8/9 [00:02<00:00,  3.13it/s]
100%|██████████| 9/9 [00:02<00:00,  3.35it/s]
100%|██████████| 9/9 [00:02<00:00,  3.20it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 20%|██        | 2/10 [00:00<00:01,  5.87it/s]
 30%|███       | 3/10 [00:00<00:01,  4.31it/s]
 40%|████      | 4/10 [00:00<00:01,  3.89it/s]
 50%|█████     | 5/10 [00:01<00:01,  3.39it/s]
 60%|██████    | 6/10 [00:01<00:01,  3.24it/s]
 70%|███████   | 7/10 [00:01<00:00,  3.16it/s]
 80%|████████  | 8/10 [00:02<00:00,  3.24it/s]
 90%|█████████ | 9/10 [00:02<00:00,  3.52it/s]
100%|██████████| 10/10 [00:02<00:00,  3.66it/s]
100%|██████████| 10/10 [00:02<00:00,  3.62it/s]
  0%|          | 0/11 [00:00<?, ?it/s]
 18%|█▊        | 2/11 [00:00<00:01,  5.47it/s]
 27%|██▋       | 3/11 [00:00<00:01,  4.22it/s]
 36%|███▋      | 4/11 [00:01<00:01,  3.64it/s]
 45%|████▌     | 5/11 [00:01<00:01,  3.44it/s]
 55%|█████▍    | 6/11 [00:01<00:01,  3.49it/s]
 64%|██████▎   | 7/11 [00:01<00:01,  3.61it/s]
 73%|███████▎  | 8/11 [00:02<00:00,  3.62it/s]
 82%|████████▏ | 9/11 [00:02<00:00,  3.75it/s]
 91%|█████████ | 10/11 [00:02<00:00,  3.83it/s]
100%|██████████| 11/11 [00:02<00:00,  3.95it/s]
100%|██████████| 11/11 [00:02<00:00,  3.82it/s]
  0%|          | 0/12 [00:00<?, ?it/s]
 17%|█▋        | 2/12 [00:00<00:01,  6.01it/s]
 25%|██▌       | 3/12 [00:00<00:01,  4.86it/s]
 33%|███▎      | 4/12 [00:00<00:01,  4.09it/s]
 42%|████▏     | 5/12 [00:01<00:01,  4.04it/s]
 50%|█████     | 6/12 [00:01<00:01,  3.53it/s]
 58%|█████▊    | 7/12 [00:01<00:01,  3.49it/s]
 67%|██████▋   | 8/12 [00:02<00:01,  3.49it/s]
 75%|███████▌  | 9/12 [00:02<00:00,  3.61it/s]
 83%|████████▎ | 10/12 [00:02<00:00,  3.58it/s]
 92%|█████████▏| 11/12 [00:02<00:00,  3.76it/s]
100%|██████████| 12/12 [00:03<00:00,  3.90it/s]
100%|██████████| 12/12 [00:03<00:00,  3.87it/s]
  0%|          | 0/13 [00:00<?, ?it/s]
 15%|█▌        | 2/13 [00:00<00:01,  5.91it/s]
 23%|██▎       | 3/13 [00:00<00:02,  4.06it/s]
 31%|███       | 4/13 [00:01<00:02,  3.65it/s]
 38%|███▊      | 5/13 [00:01<00:02,  3.37it/s]
 46%|████▌     | 6/13 [00:01<00:02,  3.46it/s]
 54%|█████▍    | 7/13 [00:01<00:01,  3.36it/s]
 62%|██████▏   | 8/13 [00:02<00:01,  3.21it/s]
 69%|██████▉   | 9/13 [00:02<00:01,  3.30it/s]
 77%|███████▋  | 10/13 [00:02<00:00,  3.51it/s]
 85%|████████▍ | 11/13 [00:03<00:00,  3.49it/s]
 92%|█████████▏| 12/13 [00:03<00:00,  3.49it/s]
100%|██████████| 13/13 [00:03<00:00,  3.68it/s]
100%|██████████| 13/13 [00:03<00:00,  3.58it/s]
  0%|          | 0/14 [00:00<?, ?it/s]
 14%|█▍        | 2/14 [00:00<00:02,  5.98it/s]
 21%|██▏       | 3/14 [00:00<00:02,  4.59it/s]
 29%|██▊       | 4/14 [00:00<00:02,  3.76it/s]
 36%|███▌      | 5/14 [00:01<00:02,  3.41it/s]
 43%|████▎     | 6/14 [00:01<00:02,  3.42it/s]
 50%|█████     | 7/14 [00:01<00:02,  3.45it/s]
 57%|█████▋    | 8/14 [00:02<00:01,  3.53it/s]
 64%|██████▍   | 9/14 [00:02<00:01,  3.81it/s]
 71%|███████▏  | 10/14 [00:02<00:01,  3.92it/s]
 79%|███████▊  | 11/14 [00:02<00:00,  3.96it/s]
 86%|████████▌ | 12/14 [00:03<00:00,  4.02it/s]
 93%|█████████▎| 13/14 [00:03<00:00,  3.86it/s]
100%|██████████| 14/14 [00:03<00:00,  3.90it/s]
100%|██████████| 14/14 [00:03<00:00,  3.85it/s]
  0%|          | 0/15 [00:00<?, ?it/s]
 13%|█▎        | 2/15 [00:00<00:01,  6.98it/s]
 20%|██        | 3/15 [00:00<00:02,  4.73it/s]
 27%|██▋       | 4/15 [00:00<00:02,  4.06it/s]
 33%|███▎      | 5/15 [00:01<00:02,  3.53it/s]
 40%|████      | 6/15 [00:01<00:02,  3.47it/s]
 47%|████▋     | 7/15 [00:01<00:02,  3.20it/s]
 53%|█████▎    | 8/15 [00:02<00:02,  3.22it/s]
 60%|██████    | 9/15 [00:02<00:01,  3.12it/s]
 67%|██████▋   | 10/15 [00:02<00:01,  3.39it/s]
 73%|███████▎  | 11/15 [00:03<00:01,  3.51it/s]
 80%|████████  | 12/15 [00:03<00:00,  3.60it/s]
 87%|████████▋ | 13/15 [00:03<00:00,  3.76it/s]
 93%|█████████▎| 14/15 [00:03<00:00,  3.69it/s]
100%|██████████| 15/15 [00:04<00:00,  3.81it/s]
100%|██████████| 15/15 [00:04<00:00,  3.67it/s]
fit_time score_time test_precision test_recall test_f1 test_balanced_accuracy test_roc_auc test_average_precision n_observations
mean std mean std mean std mean std mean std mean std mean std mean std mean std
n_features
1 0.006 0.003 0.066 0.026 0.469 0.448 0.055 0.069 0.095 0.108 0.522 0.031 0.844 0.063 0.818 0.086 210.000 0.000
2 0.004 0.002 0.041 0.014 0.649 0.138 0.506 0.142 0.557 0.117 0.648 0.079 0.711 0.081 0.660 0.092 210.000 0.000
3 0.005 0.003 0.059 0.025 0.718 0.097 0.604 0.112 0.649 0.086 0.713 0.060 0.768 0.060 0.722 0.083 210.000 0.000
4 0.005 0.002 0.052 0.020 0.773 0.101 0.640 0.110 0.694 0.084 0.749 0.060 0.793 0.060 0.755 0.090 210.000 0.000
5 0.007 0.004 0.071 0.037 0.698 0.091 0.659 0.104 0.673 0.079 0.724 0.062 0.850 0.052 0.827 0.056 210.000 0.000
6 0.005 0.002 0.050 0.024 0.719 0.081 0.687 0.099 0.699 0.076 0.745 0.059 0.853 0.054 0.827 0.059 210.000 0.000
7 0.004 0.001 0.042 0.012 0.726 0.088 0.722 0.116 0.719 0.084 0.760 0.068 0.857 0.057 0.823 0.067 210.000 0.000
8 0.004 0.001 0.041 0.013 0.777 0.073 0.763 0.095 0.765 0.062 0.799 0.051 0.895 0.048 0.886 0.048 210.000 0.000
9 0.004 0.001 0.041 0.013 0.774 0.074 0.761 0.091 0.764 0.062 0.798 0.051 0.893 0.049 0.884 0.049 210.000 0.000
10 0.004 0.001 0.038 0.005 0.812 0.081 0.821 0.093 0.813 0.071 0.840 0.061 0.909 0.049 0.902 0.046 210.000 0.000
11 0.005 0.002 0.050 0.024 0.811 0.081 0.818 0.099 0.812 0.075 0.839 0.064 0.908 0.050 0.900 0.048 210.000 0.000
12 0.005 0.003 0.050 0.022 0.810 0.083 0.815 0.100 0.809 0.076 0.837 0.065 0.907 0.050 0.898 0.048 210.000 0.000
13 0.006 0.004 0.065 0.032 0.804 0.080 0.792 0.110 0.793 0.075 0.824 0.063 0.905 0.050 0.895 0.048 210.000 0.000
14 0.006 0.002 0.060 0.021 0.802 0.075 0.797 0.108 0.795 0.072 0.826 0.060 0.903 0.051 0.892 0.050 210.000 0.000
15 0.006 0.002 0.059 0.019 0.796 0.073 0.782 0.104 0.784 0.067 0.817 0.056 0.901 0.051 0.890 0.049 210.000 0.000

Using all data:

Hide code cell source

 
cv_feat_all = njab.sklearn.find_n_best_features(X=X, y=target, name=args.target,
                                                groups=target_to_group)
cv_feat_all = cv_feat_all.drop('test_case', axis=1).groupby('n_features').agg(['mean', 'std'])
cv_feat_all

  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 486.18it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
100%|██████████| 2/2 [00:00<00:00,  5.34it/s]
100%|██████████| 2/2 [00:00<00:00,  5.31it/s]
  0%|          | 0/3 [00:00<?, ?it/s]
 67%|██████▋   | 2/3 [00:00<00:00,  5.07it/s]
100%|██████████| 3/3 [00:00<00:00,  3.49it/s]
100%|██████████| 3/3 [00:00<00:00,  3.71it/s]
  0%|          | 0/4 [00:00<?, ?it/s]
 50%|█████     | 2/4 [00:00<00:00,  4.07it/s]
 75%|███████▌  | 3/4 [00:01<00:00,  2.74it/s]
100%|██████████| 4/4 [00:01<00:00,  2.48it/s]
100%|██████████| 4/4 [00:01<00:00,  2.68it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 40%|████      | 2/5 [00:00<00:00,  5.80it/s]
 60%|██████    | 3/5 [00:00<00:00,  4.72it/s]
 80%|████████  | 4/5 [00:00<00:00,  3.90it/s]
100%|██████████| 5/5 [00:01<00:00,  3.76it/s]
100%|██████████| 5/5 [00:01<00:00,  4.07it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
 33%|███▎      | 2/6 [00:00<00:00,  7.38it/s]
 50%|█████     | 3/6 [00:00<00:00,  4.20it/s]
 67%|██████▋   | 4/6 [00:01<00:00,  3.51it/s]
 83%|████████▎ | 5/6 [00:01<00:00,  3.54it/s]
100%|██████████| 6/6 [00:01<00:00,  3.70it/s]
100%|██████████| 6/6 [00:01<00:00,  3.90it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 29%|██▊       | 2/7 [00:00<00:00,  6.89it/s]
 43%|████▎     | 3/7 [00:00<00:00,  4.11it/s]
 57%|█████▋    | 4/7 [00:01<00:00,  3.37it/s]
 71%|███████▏  | 5/7 [00:01<00:00,  3.44it/s]
 86%|████████▌ | 6/7 [00:01<00:00,  3.56it/s]
100%|██████████| 7/7 [00:01<00:00,  3.49it/s]
100%|██████████| 7/7 [00:01<00:00,  3.69it/s]
  0%|          | 0/8 [00:00<?, ?it/s]
 25%|██▌       | 2/8 [00:00<00:00,  6.24it/s]
 38%|███▊      | 3/8 [00:00<00:01,  4.43it/s]
 50%|█████     | 4/8 [00:00<00:00,  4.14it/s]
 62%|██████▎   | 5/8 [00:01<00:00,  3.90it/s]
 75%|███████▌  | 6/8 [00:01<00:00,  3.87it/s]
 88%|████████▊ | 7/8 [00:01<00:00,  3.54it/s]
100%|██████████| 8/8 [00:02<00:00,  3.34it/s]
100%|██████████| 8/8 [00:02<00:00,  3.77it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
 22%|██▏       | 2/9 [00:00<00:01,  6.19it/s]
 33%|███▎      | 3/9 [00:00<00:01,  4.75it/s]
 44%|████▍     | 4/9 [00:00<00:01,  4.33it/s]
 56%|█████▌    | 5/9 [00:01<00:01,  3.55it/s]
 67%|██████▋   | 6/9 [00:01<00:00,  3.18it/s]
 78%|███████▊  | 7/9 [00:02<00:00,  2.87it/s]
 89%|████████▉ | 8/9 [00:02<00:00,  2.97it/s]
100%|██████████| 9/9 [00:02<00:00,  3.13it/s]
100%|██████████| 9/9 [00:02<00:00,  3.41it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 20%|██        | 2/10 [00:00<00:01,  5.25it/s]
 30%|███       | 3/10 [00:00<00:01,  4.21it/s]
 40%|████      | 4/10 [00:00<00:01,  3.78it/s]
 50%|█████     | 5/10 [00:01<00:01,  3.79it/s]
 60%|██████    | 6/10 [00:01<00:01,  3.21it/s]
 70%|███████   | 7/10 [00:02<00:01,  2.96it/s]
 80%|████████  | 8/10 [00:02<00:00,  3.15it/s]
 90%|█████████ | 9/10 [00:02<00:00,  3.21it/s]
100%|██████████| 10/10 [00:02<00:00,  3.32it/s]
100%|██████████| 10/10 [00:02<00:00,  3.44it/s]
  0%|          | 0/11 [00:00<?, ?it/s]
 18%|█▊        | 2/11 [00:00<00:01,  7.20it/s]
 27%|██▋       | 3/11 [00:00<00:01,  5.28it/s]
 36%|███▋      | 4/11 [00:00<00:01,  4.54it/s]
 45%|████▌     | 5/11 [00:01<00:01,  4.38it/s]
 55%|█████▍    | 6/11 [00:01<00:01,  4.16it/s]
 64%|██████▎   | 7/11 [00:01<00:01,  2.87it/s]
 73%|███████▎  | 8/11 [00:02<00:01,  2.61it/s]
 82%|████████▏ | 9/11 [00:02<00:00,  2.46it/s]
 91%|█████████ | 10/11 [00:03<00:00,  2.24it/s]
100%|██████████| 11/11 [00:03<00:00,  2.41it/s]
100%|██████████| 11/11 [00:03<00:00,  2.98it/s]
  0%|          | 0/12 [00:00<?, ?it/s]
 17%|█▋        | 2/12 [00:00<00:01,  5.21it/s]
 25%|██▌       | 3/12 [00:00<00:02,  4.17it/s]
 33%|███▎      | 4/12 [00:01<00:02,  3.72it/s]
 42%|████▏     | 5/12 [00:01<00:02,  3.49it/s]
 50%|█████     | 6/12 [00:01<00:01,  3.43it/s]
 58%|█████▊    | 7/12 [00:02<00:01,  3.02it/s]
 67%|██████▋   | 8/12 [00:02<00:01,  3.11it/s]
 75%|███████▌  | 9/12 [00:02<00:00,  3.28it/s]
 83%|████████▎ | 10/12 [00:02<00:00,  3.37it/s]
 92%|█████████▏| 11/12 [00:03<00:00,  2.96it/s]
100%|██████████| 12/12 [00:03<00:00,  2.97it/s]
100%|██████████| 12/12 [00:03<00:00,  3.28it/s]
  0%|          | 0/13 [00:00<?, ?it/s]
 15%|█▌        | 2/13 [00:00<00:01,  6.42it/s]
 23%|██▎       | 3/13 [00:00<00:02,  4.21it/s]
 31%|███       | 4/13 [00:00<00:02,  4.03it/s]
 38%|███▊      | 5/13 [00:01<00:02,  3.88it/s]
 46%|████▌     | 6/13 [00:01<00:01,  3.84it/s]
 54%|█████▍    | 7/13 [00:01<00:01,  3.76it/s]
 62%|██████▏   | 8/13 [00:02<00:01,  3.30it/s]
 69%|██████▉   | 9/13 [00:02<00:01,  2.80it/s]
 77%|███████▋  | 10/13 [00:03<00:01,  2.62it/s]
 85%|████████▍ | 11/13 [00:03<00:00,  2.65it/s]
 92%|█████████▏| 12/13 [00:03<00:00,  2.29it/s]
100%|██████████| 13/13 [00:04<00:00,  2.33it/s]
100%|██████████| 13/13 [00:04<00:00,  2.95it/s]
  0%|          | 0/14 [00:00<?, ?it/s]
 14%|█▍        | 2/14 [00:00<00:02,  5.24it/s]
 21%|██▏       | 3/14 [00:00<00:03,  3.15it/s]
 29%|██▊       | 4/14 [00:01<00:03,  2.76it/s]
 36%|███▌      | 5/14 [00:01<00:03,  2.75it/s]
 43%|████▎     | 6/14 [00:02<00:03,  2.57it/s]
 50%|█████     | 7/14 [00:02<00:02,  2.54it/s]
 57%|█████▋    | 8/14 [00:03<00:02,  2.31it/s]
 64%|██████▍   | 9/14 [00:03<00:02,  2.21it/s]
 71%|███████▏  | 10/14 [00:03<00:01,  2.24it/s]
 79%|███████▊  | 11/14 [00:04<00:01,  2.11it/s]
 86%|████████▌ | 12/14 [00:05<00:00,  2.03it/s]
 93%|█████████▎| 13/14 [00:05<00:00,  2.10it/s]
100%|██████████| 14/14 [00:05<00:00,  2.32it/s]
100%|██████████| 14/14 [00:05<00:00,  2.41it/s]
  0%|          | 0/15 [00:00<?, ?it/s]
 13%|█▎        | 2/15 [00:00<00:01,  7.25it/s]
 20%|██        | 3/15 [00:00<00:02,  5.15it/s]
 27%|██▋       | 4/15 [00:00<00:02,  4.68it/s]
 33%|███▎      | 5/15 [00:01<00:02,  4.15it/s]
 40%|████      | 6/15 [00:01<00:02,  4.08it/s]
 47%|████▋     | 7/15 [00:01<00:01,  4.06it/s]
 53%|█████▎    | 8/15 [00:01<00:01,  3.97it/s]
 60%|██████    | 9/15 [00:02<00:02,  2.99it/s]
 67%|██████▋   | 10/15 [00:02<00:01,  2.64it/s]
 73%|███████▎  | 11/15 [00:03<00:01,  2.59it/s]
 80%|████████  | 12/15 [00:03<00:01,  2.85it/s]
 87%|████████▋ | 13/15 [00:03<00:00,  3.15it/s]
 93%|█████████▎| 14/15 [00:03<00:00,  3.42it/s]
100%|██████████| 15/15 [00:04<00:00,  3.67it/s]
100%|██████████| 15/15 [00:04<00:00,  3.55it/s]
fit_time score_time test_precision test_recall test_f1 test_balanced_accuracy test_roc_auc test_average_precision n_observations
mean std mean std mean std mean std mean std mean std mean std mean std mean std
n_features
1 0.005 0.002 0.049 0.016 0.823 0.277 0.136 0.090 0.224 0.132 0.560 0.045 0.849 0.065 0.824 0.088 210.000 0.000
2 0.006 0.002 0.066 0.020 0.716 0.112 0.578 0.125 0.630 0.101 0.701 0.068 0.760 0.073 0.725 0.096 210.000 0.000
3 0.004 0.001 0.042 0.010 0.679 0.100 0.666 0.133 0.665 0.096 0.716 0.076 0.791 0.074 0.745 0.098 210.000 0.000
4 0.005 0.002 0.049 0.019 0.776 0.090 0.740 0.124 0.751 0.083 0.790 0.068 0.892 0.055 0.871 0.067 210.000 0.000
5 0.004 0.002 0.041 0.016 0.824 0.091 0.769 0.115 0.790 0.081 0.823 0.067 0.904 0.051 0.868 0.075 210.000 0.000
6 0.004 0.001 0.041 0.010 0.826 0.092 0.769 0.113 0.791 0.080 0.824 0.066 0.902 0.054 0.867 0.076 210.000 0.000
7 0.004 0.001 0.041 0.010 0.821 0.076 0.805 0.112 0.808 0.074 0.837 0.063 0.910 0.050 0.875 0.073 210.000 0.000
8 0.004 0.001 0.037 0.005 0.828 0.073 0.804 0.114 0.810 0.069 0.839 0.058 0.909 0.050 0.874 0.073 210.000 0.000
9 0.006 0.002 0.062 0.027 0.826 0.076 0.804 0.115 0.809 0.073 0.839 0.060 0.908 0.050 0.873 0.072 210.000 0.000
10 0.005 0.002 0.050 0.022 0.816 0.076 0.797 0.115 0.802 0.075 0.832 0.063 0.905 0.050 0.871 0.070 210.000 0.000
11 0.004 0.002 0.040 0.010 0.826 0.076 0.821 0.104 0.819 0.070 0.846 0.058 0.904 0.048 0.868 0.068 210.000 0.000
12 0.005 0.002 0.050 0.018 0.817 0.077 0.806 0.098 0.807 0.065 0.835 0.055 0.905 0.049 0.871 0.067 210.000 0.000
13 0.008 0.004 0.073 0.032 0.838 0.077 0.831 0.097 0.831 0.070 0.856 0.060 0.911 0.052 0.887 0.065 210.000 0.000
14 0.006 0.004 0.060 0.033 0.834 0.084 0.842 0.111 0.834 0.082 0.859 0.070 0.913 0.051 0.890 0.065 210.000 0.000
15 0.003 0.001 0.027 0.012 0.841 0.072 0.858 0.096 0.846 0.067 0.869 0.059 0.918 0.051 0.896 0.067 210.000 0.000

Using only new features:

Hide code cell source

 
cv_feat_new = njab.sklearn.find_n_best_features(X=X.loc[:, new_features],
                                                y=target, name=args.target,
                                                groups=target_to_group)
cv_feat_new = cv_feat_new.drop('test_case', axis=1).groupby('n_features').agg(['mean', 'std'])
cv_feat_new

  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 1282.27it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
100%|██████████| 2/2 [00:00<00:00, 29.17it/s]
  0%|          | 0/3 [00:00<?, ?it/s]
100%|██████████| 3/3 [00:00<00:00, 25.57it/s]
100%|██████████| 3/3 [00:00<00:00, 25.41it/s]
  0%|          | 0/4 [00:00<?, ?it/s]
 75%|███████▌  | 3/4 [00:00<00:00, 15.62it/s]
100%|██████████| 4/4 [00:00<00:00, 14.21it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 60%|██████    | 3/5 [00:00<00:00, 20.08it/s]
100%|██████████| 5/5 [00:00<00:00, 15.95it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
 50%|█████     | 3/6 [00:00<00:00, 16.13it/s]
 83%|████████▎ | 5/6 [00:00<00:00, 11.90it/s]
100%|██████████| 6/6 [00:00<00:00, 11.92it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 43%|████▎     | 3/7 [00:00<00:00, 20.80it/s]
 86%|████████▌ | 6/7 [00:00<00:00, 17.13it/s]
100%|██████████| 7/7 [00:00<00:00, 16.93it/s]
  0%|          | 0/8 [00:00<?, ?it/s]
 38%|███▊      | 3/8 [00:00<00:00, 25.56it/s]
 75%|███████▌  | 6/8 [00:00<00:00, 14.66it/s]
100%|██████████| 8/8 [00:00<00:00, 12.51it/s]
100%|██████████| 8/8 [00:00<00:00, 13.54it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
 33%|███▎      | 3/9 [00:00<00:00, 18.17it/s]
 56%|█████▌    | 5/9 [00:00<00:00, 14.83it/s]
 78%|███████▊  | 7/9 [00:00<00:00, 15.23it/s]
100%|██████████| 9/9 [00:00<00:00, 14.83it/s]
100%|██████████| 9/9 [00:00<00:00, 15.15it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 30%|███       | 3/10 [00:00<00:00, 22.38it/s]
 60%|██████    | 6/10 [00:00<00:00, 16.19it/s]
 80%|████████  | 8/10 [00:00<00:00, 14.10it/s]
100%|██████████| 10/10 [00:00<00:00, 11.77it/s]
100%|██████████| 10/10 [00:00<00:00, 13.21it/s]
  0%|          | 0/11 [00:00<?, ?it/s]
 18%|█▊        | 2/11 [00:00<00:00, 14.68it/s]
 36%|███▋      | 4/11 [00:00<00:00, 13.91it/s]
 55%|█████▍    | 6/11 [00:00<00:00, 11.89it/s]
 73%|███████▎  | 8/11 [00:00<00:00, 12.71it/s]
 91%|█████████ | 10/11 [00:00<00:00, 12.63it/s]
100%|██████████| 11/11 [00:00<00:00, 12.98it/s]
  0%|          | 0/12 [00:00<?, ?it/s]
 25%|██▌       | 3/12 [00:00<00:00, 22.03it/s]
 50%|█████     | 6/12 [00:00<00:00, 13.71it/s]
 67%|██████▋   | 8/12 [00:00<00:00, 13.04it/s]
 83%|████████▎ | 10/12 [00:00<00:00, 11.82it/s]
100%|██████████| 12/12 [00:00<00:00, 10.92it/s]
100%|██████████| 12/12 [00:00<00:00, 12.06it/s]
  0%|          | 0/13 [00:00<?, ?it/s]
 23%|██▎       | 3/13 [00:00<00:00, 21.94it/s]
 46%|████▌     | 6/13 [00:00<00:00, 15.61it/s]
 62%|██████▏   | 8/13 [00:00<00:00, 14.07it/s]
 77%|███████▋  | 10/13 [00:00<00:00, 13.92it/s]
 92%|█████████▏| 12/13 [00:00<00:00, 13.90it/s]
100%|██████████| 13/13 [00:00<00:00, 14.51it/s]
  0%|          | 0/14 [00:00<?, ?it/s]
 21%|██▏       | 3/14 [00:00<00:00, 24.60it/s]
 43%|████▎     | 6/14 [00:00<00:00, 18.77it/s]
 57%|█████▋    | 8/14 [00:00<00:00, 17.04it/s]
 71%|███████▏  | 10/14 [00:00<00:00, 17.73it/s]
 86%|████████▌ | 12/14 [00:00<00:00, 13.54it/s]
100%|██████████| 14/14 [00:00<00:00, 11.88it/s]
100%|██████████| 14/14 [00:00<00:00, 14.22it/s]
  0%|          | 0/15 [00:00<?, ?it/s]
 20%|██        | 3/15 [00:00<00:00, 20.60it/s]
 40%|████      | 6/15 [00:00<00:00, 17.14it/s]
 53%|█████▎    | 8/15 [00:00<00:00, 15.59it/s]
 67%|██████▋   | 10/15 [00:00<00:00, 15.51it/s]
 80%|████████  | 12/15 [00:00<00:00, 15.67it/s]
 93%|█████████▎| 14/15 [00:00<00:00, 15.33it/s]
100%|██████████| 15/15 [00:00<00:00, 15.56it/s]
fit_time score_time test_precision test_recall test_f1 test_balanced_accuracy test_roc_auc test_average_precision n_observations
mean std mean std mean std mean std mean std mean std mean std mean std mean std
n_features
1 0.003 0.002 0.032 0.021 0.070 0.248 0.006 0.021 0.011 0.039 0.503 0.010 0.746 0.059 0.658 0.078 210.000 0.000
2 0.004 0.002 0.046 0.017 0.567 0.227 0.204 0.092 0.286 0.108 0.537 0.050 0.638 0.086 0.581 0.087 210.000 0.000
3 0.004 0.001 0.041 0.010 0.610 0.210 0.236 0.095 0.331 0.114 0.559 0.061 0.612 0.077 0.580 0.087 210.000 0.000
4 0.004 0.001 0.036 0.010 0.620 0.125 0.342 0.099 0.434 0.098 0.593 0.055 0.625 0.079 0.597 0.079 210.000 0.000
5 0.003 0.000 0.034 0.002 0.606 0.125 0.340 0.096 0.429 0.094 0.587 0.058 0.610 0.085 0.585 0.086 210.000 0.000
6 0.004 0.002 0.041 0.012 0.595 0.123 0.337 0.088 0.425 0.090 0.583 0.053 0.603 0.080 0.585 0.084 210.000 0.000
7 0.004 0.001 0.037 0.003 0.572 0.123 0.327 0.093 0.411 0.096 0.573 0.057 0.592 0.078 0.571 0.077 210.000 0.000
8 0.004 0.001 0.043 0.014 0.556 0.128 0.329 0.102 0.406 0.102 0.568 0.060 0.577 0.080 0.559 0.078 210.000 0.000
9 0.004 0.002 0.039 0.006 0.525 0.120 0.319 0.102 0.391 0.102 0.554 0.060 0.566 0.079 0.549 0.078 210.000 0.000
10 0.005 0.002 0.052 0.019 0.519 0.121 0.319 0.106 0.389 0.104 0.551 0.063 0.555 0.078 0.540 0.074 210.000 0.000
11 0.004 0.001 0.040 0.008 0.486 0.117 0.318 0.100 0.380 0.099 0.537 0.062 0.559 0.079 0.538 0.083 210.000 0.000
12 0.005 0.002 0.045 0.019 0.548 0.088 0.428 0.095 0.474 0.076 0.583 0.053 0.632 0.069 0.576 0.067 210.000 0.000
13 0.004 0.002 0.041 0.011 0.555 0.099 0.436 0.107 0.482 0.093 0.590 0.062 0.631 0.075 0.586 0.077 210.000 0.000
14 0.004 0.002 0.042 0.012 0.546 0.095 0.429 0.108 0.473 0.088 0.583 0.059 0.621 0.074 0.576 0.077 210.000 0.000
15 0.006 0.003 0.054 0.020 0.567 0.113 0.449 0.124 0.493 0.107 0.598 0.070 0.626 0.070 0.586 0.079 210.000 0.000

Best number of features by subset of the data:#

Hide code cell source

 
n_feat_best = pd.DataFrame(
    {'ald': cv_feat_ald.loc[:, pd.IndexSlice[:, 'mean']].idxmax(),
     'all': cv_feat_all.loc[:, pd.IndexSlice[:, 'mean']].idxmax(),
     'new': cv_feat_new.loc[:, pd.IndexSlice[:, 'mean']].idxmax()
     }
).droplevel(-1)
n_feat_best
ald all new
fit_time 5 13 15
score_time 5 13 15
test_precision 10 15 4
test_recall 10 15 15
test_f1 10 15 15
test_balanced_accuracy 10 15 15
test_roc_auc 10 15 1
test_average_precision 10 15 1
n_observations 1 1 1

Train, test split#

Show number of cases in train and test data

Hide code cell source

 
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
    X,
    target,
    test_size=.2,
    stratify=target_to_group,
    random_state=42)
idx_train = X_train.index
idx_test = X_test.index

njab.pandas.combine_value_counts(
    pd.concat([y_train, y_test],
              axis=1,
              ignore_index=True,
              ).rename(columns={0: 'train', 1: 'test'})
)
train test
False 98 24
True 70 18

Results#

  • run_model returns dataclasses with the further needed results

  • add mrmr selection of data (select best number of features to use instead of fixing it)

Save results for final model on entire data, new features and ALD study criteria selected data.

Hide code cell source

 
splits = Splits(X_train=X.loc[idx_train],
                X_test=X.loc[idx_test],
                y_train=y_train,
                y_test=y_test)
results_model_full = njab.sklearn.run_model(
    splits,
    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'all'])
results_model_full.name = f'{args.model_key} all'
fname = args.out_folder / f'results_{results_model_full.name}.pkl'
files_out[fname.name] = fname
pimmslearn.io.to_pickle(results_model_full, fname)

splits = Splits(X_train=X.loc[idx_train, new_features],
                X_test=X.loc[idx_test, new_features],
                y_train=y_train,
                y_test=y_test)
results_model_new = njab.sklearn.run_model(
    splits,
    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'new'])
results_model_new.name = f'{args.model_key} new'
fname = args.out_folder / f'results_{results_model_new.name}.pkl'
files_out[fname.name] = fname
pimmslearn.io.to_pickle(results_model_new, fname)

splits_ald = Splits(
    X_train=ald_study.loc[idx_train],
    X_test=ald_study.loc[idx_test],
    y_train=y_train,
    y_test=y_test)
results_ald_full = njab.sklearn.run_model(
    splits_ald,
    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'ald'])
results_ald_full.name = 'ALD study all'
fname = args.out_folder / f'results_{results_ald_full.name}.pkl'
files_out[fname.name] = fname
pimmslearn.io.to_pickle(results_ald_full, fname)

  0%|          | 0/15 [00:00<?, ?it/s]
 13%|█▎        | 2/15 [00:00<00:02,  4.48it/s]
 20%|██        | 3/15 [00:00<00:03,  3.21it/s]
 27%|██▋       | 4/15 [00:01<00:03,  3.23it/s]
 33%|███▎      | 5/15 [00:01<00:03,  3.32it/s]
 40%|████      | 6/15 [00:01<00:02,  3.34it/s]
 47%|████▋     | 7/15 [00:02<00:02,  3.42it/s]
 53%|█████▎    | 8/15 [00:02<00:01,  3.52it/s]
 60%|██████    | 9/15 [00:02<00:01,  3.60it/s]
 67%|██████▋   | 10/15 [00:02<00:01,  3.64it/s]
 73%|███████▎  | 11/15 [00:03<00:01,  3.60it/s]
 80%|████████  | 12/15 [00:03<00:00,  3.42it/s]
 87%|████████▋ | 13/15 [00:03<00:00,  3.41it/s]
 93%|█████████▎| 14/15 [00:04<00:00,  3.41it/s]
100%|██████████| 15/15 [00:04<00:00,  3.01it/s]
100%|██████████| 15/15 [00:04<00:00,  3.35it/s]
  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 1074.09it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 20%|██        | 2/10 [00:00<00:01,  6.69it/s]
 30%|███       | 3/10 [00:00<00:01,  5.41it/s]
 40%|████      | 4/10 [00:00<00:01,  4.38it/s]
 50%|█████     | 5/10 [00:01<00:01,  4.41it/s]
 60%|██████    | 6/10 [00:01<00:00,  4.29it/s]
 70%|███████   | 7/10 [00:01<00:00,  4.08it/s]
 80%|████████  | 8/10 [00:01<00:00,  3.88it/s]
 90%|█████████ | 9/10 [00:02<00:00,  3.90it/s]
100%|██████████| 10/10 [00:02<00:00,  3.83it/s]
100%|██████████| 10/10 [00:02<00:00,  4.19it/s]

ROC-AUC on test split#

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)
plot_split_auc(results_ald_full.test, results_ald_full.name, ax)
plot_split_auc(results_model_full.test, results_model_full.name, ax)
plot_split_auc(results_model_new.test, results_model_new.name, ax)
fname = args.out_folder / 'auc_roc_curve.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/auc_roc_curve.pdf
../../../_images/fb95aff1447da63fe60824dd7bf66e27b0e292a3fd73ed75f644af9e82dd9d33.png

Data used to plot ROC:

Hide code cell source

 
res = [results_ald_full, results_model_full, results_model_new]

auc_roc_curve = parse_roc(*res)
auc_roc_curve.to_excel(fname.with_suffix('.xlsx'))
auc_roc_curve
ALD study all QRILC all QRILC new
fpr tpr fpr tpr fpr tpr
0 0.000 0.000 0.000 0.000 0.000 0.000
1 0.000 0.056 0.000 0.056 0.042 0.000
2 0.000 0.611 0.000 0.500 0.042 0.111
3 0.042 0.611 0.042 0.500 0.125 0.111
4 0.042 0.722 0.042 0.722 0.125 0.222
5 0.125 0.722 0.292 0.722 0.167 0.222
6 0.125 0.833 0.292 0.833 0.167 0.667
7 0.292 0.833 0.333 0.833 0.292 0.667
8 0.292 0.889 0.333 0.889 0.292 0.722
9 0.417 0.889 0.375 0.889 0.500 0.722
10 0.417 0.944 0.375 0.944 0.500 0.778
11 0.917 0.944 0.500 0.944 0.542 0.778
12 0.917 1.000 0.500 1.000 0.542 0.833
13 1.000 1.000 1.000 1.000 0.625 0.833
14 NaN NaN NaN NaN 0.625 0.944
15 NaN NaN NaN NaN 0.958 0.944
16 NaN NaN NaN NaN 0.958 1.000
17 NaN NaN NaN NaN 1.000 1.000

Features selected for final models#

Hide code cell source

 
selected_features = pd.DataFrame(
    [results_ald_full.selected_features,
     results_model_full.selected_features,
     results_model_new.selected_features],
    index=[
        results_ald_full.name,
        results_model_full.name,
        results_model_new.name]
).T
selected_features.index.name = 'rank'
fname = args.out_folder / 'mrmr_feat_by_model.xlsx'
files_out[fname.name] = fname
selected_features.to_excel(fname)
selected_features
ALD study all QRILC all QRILC new
rank
0 P10636-2;P10636-6 Q9Y2T3;Q9Y2T3-3 A6PVN5;F6WIT2;Q15257;Q15257-2;Q15257-3
1 Q15848 P60709;P63261 None
2 P51888 A0A0C4DH07;Q8N2S1;Q8N2S1-2;Q8N2S1-3 None
3 P61981 P10636-2;P10636-6 None
4 P04075 P61981 None
5 P14174 A0A0A0MRJ7;P12259 None
6 P08294 P14174 None
7 Q9Y2T3;Q9Y2T3-3 P04075 None
8 P63104 A6PVN5;F6WIT2;Q15257;Q15257-2;Q15257-3 None
9 C9JF17;P05090 C9JF17;P05090 None
10 None P63104 None
11 None P00338;P00338-3 None
12 None P05413;S4R371 None
13 None Q6EMK4 None
14 None P00492 None

Precision-Recall plot on test data#

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)

ax = plot_split_prc(results_ald_full.test, results_ald_full.name, ax)
ax = plot_split_prc(results_model_full.test, results_model_full.name, ax)
ax = plot_split_prc(results_model_new.test, results_model_new.name, ax)
fname = folder = args.out_folder / 'prec_recall_curve.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/prec_recall_curve.pdf
../../../_images/b5415c6449b503a8dc53c31b6d1dc3542ce60108cf80a9dabd26c31bc15ee41d.png

Data used to plot PRC:

Hide code cell source

 
prec_recall_curve = parse_prc(*res)
prec_recall_curve.to_excel(fname.with_suffix('.xlsx'))
prec_recall_curve
ALD study all QRILC all QRILC new
precision tpr precision tpr precision tpr
0 0.429 1.000 0.429 1.000 0.429 1.000
1 0.439 1.000 0.439 1.000 0.439 1.000
2 0.450 1.000 0.450 1.000 0.425 0.944
3 0.436 0.944 0.462 1.000 0.436 0.944
4 0.447 0.944 0.474 1.000 0.447 0.944
5 0.459 0.944 0.486 1.000 0.459 0.944
6 0.472 0.944 0.500 1.000 0.472 0.944
7 0.486 0.944 0.514 1.000 0.486 0.944
8 0.500 0.944 0.529 1.000 0.500 0.944
9 0.515 0.944 0.545 1.000 0.515 0.944
10 0.531 0.944 0.562 1.000 0.531 0.944
11 0.548 0.944 0.581 1.000 0.516 0.889
12 0.567 0.944 0.600 1.000 0.500 0.833
13 0.586 0.944 0.586 0.944 0.517 0.833
14 0.607 0.944 0.607 0.944 0.536 0.833
15 0.630 0.944 0.630 0.944 0.519 0.778
16 0.615 0.889 0.654 0.944 0.538 0.778
17 0.640 0.889 0.640 0.889 0.520 0.722
18 0.667 0.889 0.667 0.889 0.542 0.722
19 0.696 0.889 0.652 0.833 0.565 0.722
20 0.682 0.833 0.682 0.833 0.591 0.722
21 0.714 0.833 0.667 0.778 0.619 0.722
22 0.750 0.833 0.650 0.722 0.650 0.722
23 0.789 0.833 0.684 0.722 0.632 0.667
24 0.833 0.833 0.722 0.722 0.667 0.667
25 0.824 0.778 0.765 0.722 0.706 0.667
26 0.812 0.722 0.812 0.722 0.750 0.667
27 0.867 0.722 0.867 0.722 0.733 0.611
28 0.929 0.722 0.929 0.722 0.714 0.556
29 0.923 0.667 0.923 0.667 0.692 0.500
30 0.917 0.611 0.917 0.611 0.667 0.444
31 1.000 0.611 0.909 0.556 0.636 0.389
32 1.000 0.556 0.900 0.500 0.600 0.333
33 1.000 0.500 1.000 0.500 0.556 0.278
34 1.000 0.444 1.000 0.444 0.500 0.222
35 1.000 0.389 1.000 0.389 0.571 0.222
36 1.000 0.333 1.000 0.333 0.500 0.167
37 1.000 0.278 1.000 0.278 0.400 0.111
38 1.000 0.222 1.000 0.222 0.500 0.111
39 1.000 0.167 1.000 0.167 0.667 0.111
40 1.000 0.111 1.000 0.111 0.500 0.056
41 1.000 0.056 1.000 0.056 0.000 0.000
42 1.000 0.000 1.000 0.000 1.000 0.000

Train data plots#

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)

ax = plot_split_prc(results_ald_full.train, results_ald_full.name, ax)
ax = plot_split_prc(results_model_full.train, results_model_full.name, ax)
ax = plot_split_prc(results_model_new.train, results_model_new.name, ax)
fname = folder = args.out_folder / 'prec_recall_curve_train.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/prec_recall_curve_train.pdf
../../../_images/32ef6a9e07e34c4e1b3dc4a411beea28be9ed3c85d291d5e822bcad4b5de8f11.png

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)
plot_split_auc(results_ald_full.train, results_ald_full.name, ax)
plot_split_auc(results_model_full.train, results_model_full.name, ax)
plot_split_auc(results_model_new.train, results_model_new.name, ax)
fname = folder = args.out_folder / 'auc_roc_curve_train.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/auc_roc_curve_train.pdf
../../../_images/9d8b1d163b159870749d42a4750f818c583bac8515b9be48ce90d0c8d7bc932a.png

Output files:

Hide code cell source

 
files_out

{'results_QRILC all.pkl': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/results_QRILC all.pkl'),
 'results_QRILC new.pkl': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/results_QRILC new.pkl'),
 'results_ALD study all.pkl': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/results_ALD study all.pkl'),
 'auc_roc_curve.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/auc_roc_curve.pdf'),
 'mrmr_feat_by_model.xlsx': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/mrmr_feat_by_model.xlsx'),
 'prec_recall_curve.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/prec_recall_curve.pdf'),
 'prec_recall_curve_train.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/prec_recall_curve_train.pdf'),
 'auc_roc_curve_train.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_QRILC/auc_roc_curve_train.pdf')}