Fit logistic regression model

Fit logistic regression model#

  • based on different imputation methods

  • baseline: reference

  • model: any other selected imputation method

Hide code cell source

 
import logging
from pathlib import Path
from typing import List

import matplotlib.pyplot as plt
import njab.sklearn
import pandas as pd
import sklearn
from njab.plotting.metrics import plot_split_auc, plot_split_prc
from njab.sklearn.types import Splits

import pimmslearn
import pimmslearn.analyzers
import pimmslearn.io.datasplits

plt.rcParams['figure.figsize'] = (2.5, 2.5)
plt.rcParams['lines.linewidth'] = 1
plt.rcParams['lines.markersize'] = 2
fontsize = 5
figsize = (2.5, 2.5)
pimmslearn.plotting.make_large_descriptors(fontsize)


logger = pimmslearn.logging.setup_nb_logger()
logging.getLogger('fontTools').setLevel(logging.ERROR)


def parse_roc(*res: List[njab.sklearn.types.Results]) -> pd.DataFrame:
    ret = list()
    for _r in res:
        _roc = (pd.DataFrame(_r.test.roc,
                             index='fpr tpr cutoffs'.split()
                             )).loc[['fpr', 'tpr']]
        _roc = _roc.T
        _roc.columns = pd.MultiIndex.from_product([[_r.name], _roc.columns])
        ret.append(_roc)
    ret = pd.concat(ret, axis=1)
    return ret


def parse_prc(*res: List[njab.sklearn.types.Results]) -> pd.DataFrame:
    ret = list()
    for _r in res:
        _prc = pd.DataFrame(_r.test.prc,
                            index='precision recall cutoffs'.split()
                            ).loc[['precision', 'recall']]
        _prc = _prc.T.rename(columns={'recall': 'tpr'})
        _prc.columns = pd.MultiIndex.from_product([[_r.name], _prc.columns])
        ret.append(_prc)
    ret = pd.concat(ret, axis=1)
    return ret


# catch passed parameters
args = None
args = dict(globals()).keys()

Parameters#

Default and set parameters for the notebook.

folder_data: str = ''  # specify data directory if needed
fn_clinical_data = "data/ALD_study/processed/ald_metadata_cli.csv"
folder_experiment = "runs/appl_ald_data/plasma/proteinGroups"
model_key = 'VAE'
target = 'kleiner'
sample_id_col = 'Sample ID'
cutoff_target: int = 2  # => for binarization target >= cutoff_target
file_format = "csv"
out_folder = 'diff_analysis'
fn_qc_samples = ''  # 'data/ALD_study/processed/qc_plasma_proteinGroups.pkl'

baseline = 'RSN'  # default is RSN, as this was used in the original ALD Niu. et. al 2022
template_pred = 'pred_real_na_{}.csv'  # fixed, do not change

# Parameters
cutoff_target = 0.5
folder_experiment = "runs/alzheimer_study"
target = "AD"
baseline = "PI"
model_key = "VAE"
out_folder = "diff_analysis"
fn_clinical_data = "runs/alzheimer_study/data/clinical_data.csv"

Hide code cell source

 
params = pimmslearn.nb.get_params(args, globals=globals())
args = pimmslearn.nb.Config()
args.folder_experiment = Path(params["folder_experiment"])
args = pimmslearn.nb.add_default_paths(args,
                                 out_root=(args.folder_experiment
                                           / params["out_folder"]
                                           / params["target"]
                                           / f"{params['baseline']}_vs_{params['model_key']}"))
args.update_from_dict(params)
files_out = dict()
args

root - INFO     Removed from global namespace: folder_data
root - INFO     Removed from global namespace: fn_clinical_data
root - INFO     Removed from global namespace: folder_experiment
root - INFO     Removed from global namespace: model_key
root - INFO     Removed from global namespace: target
root - INFO     Removed from global namespace: sample_id_col
root - INFO     Removed from global namespace: cutoff_target
root - INFO     Removed from global namespace: file_format
root - INFO     Removed from global namespace: out_folder
root - INFO     Removed from global namespace: fn_qc_samples
root - INFO     Removed from global namespace: baseline
root - INFO     Removed from global namespace: template_pred
root - INFO     Already set attribute: folder_experiment has value runs/alzheimer_study
root - INFO     Already set attribute: out_folder has value diff_analysis

{'baseline': 'PI',
 'cutoff_target': 0.5,
 'data': PosixPath('runs/alzheimer_study/data'),
 'file_format': 'csv',
 'fn_clinical_data': 'runs/alzheimer_study/data/clinical_data.csv',
 'fn_qc_samples': '',
 'folder_data': '',
 'folder_experiment': PosixPath('runs/alzheimer_study'),
 'model_key': 'VAE',
 'out_figures': PosixPath('runs/alzheimer_study/figures'),
 'out_folder': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE'),
 'out_metrics': PosixPath('runs/alzheimer_study'),
 'out_models': PosixPath('runs/alzheimer_study'),
 'out_preds': PosixPath('runs/alzheimer_study/preds'),
 'sample_id_col': 'Sample ID',
 'target': 'AD',
 'template_pred': 'pred_real_na_{}.csv'}

Load data#

Load target#

target = pd.read_csv(args.fn_clinical_data,
                     index_col=0,
                     usecols=[args.sample_id_col, args.target])
target = target.dropna()
target
AD
Sample ID
Sample_000 0
Sample_001 1
Sample_002 1
Sample_003 1
Sample_004 1
... ...
Sample_205 1
Sample_206 0
Sample_207 0
Sample_208 0
Sample_209 0

210 rows × 1 columns

MS proteomics or specified omics data#

Aggregated from data splits of the imputation workflow run before.

Hide code cell source

 
data = pimmslearn.io.datasplits.DataSplits.from_folder(
    args.data, file_format=args.file_format)
data = pd.concat([data.train_X, data.val_y, data.test_y])
data.sample(5)

pimmslearn.io.datasplits - INFO     Loaded 'train_X' from file: runs/alzheimer_study/data/train_X.csv
pimmslearn.io.datasplits - INFO     Loaded 'val_y' from file: runs/alzheimer_study/data/val_y.csv
pimmslearn.io.datasplits - INFO     Loaded 'test_y' from file: runs/alzheimer_study/data/test_y.csv

Sample ID   protein groups                                                                                
Sample_188  P07686                                                                                           17.227
Sample_093  P22692;P22692-2                                                                                  16.207
Sample_055  Q14118                                                                                           21.489
Sample_172  A0A0G2JHN4;A0A0G2JII5;A0A0G2JIT9;A0A0G2JJ47;A0A0G2JJK2;A0A0G2JJZ1;A0A0G2JP87;A0A0G2JPU2;B0S7V6   18.582
Sample_177  P37837                                                                                           18.031
Name: intensity, dtype: float64

Get overlap between independent features and target

Select by ALD criteria#

Use parameters as specified in ALD study.

Hide code cell source

 
DATA_COMPLETENESS = 0.6
MIN_N_PROTEIN_GROUPS: int = 200
FRAC_PROTEIN_GROUPS: int = 0.622
CV_QC_SAMPLE: float = 0.4

ald_study, cutoffs = pimmslearn.analyzers.diff_analysis.select_raw_data(data.unstack(
), data_completeness=DATA_COMPLETENESS, frac_protein_groups=FRAC_PROTEIN_GROUPS)

if args.fn_qc_samples:
    qc_samples = pd.read_pickle(args.fn_qc_samples)
    qc_samples = qc_samples[ald_study.columns]
    qc_cv_feat = qc_samples.std() / qc_samples.mean()
    qc_cv_feat = qc_cv_feat.rename(qc_samples.columns.name)
    fig, ax = plt.subplots(figsize=(4, 7))
    ax = qc_cv_feat.plot.box(ax=ax)
    ax.set_ylabel('Coefficient of Variation')
    print((qc_cv_feat < CV_QC_SAMPLE).value_counts())
    ald_study = ald_study[pimmslearn.analyzers.diff_analysis.select_feat(qc_samples)]

column_name_first_prot_to_pg = {
    pg.split(';')[0]: pg for pg in data.unstack().columns}

ald_study = ald_study.rename(columns=column_name_first_prot_to_pg)
ald_study

root - INFO     Initally: N samples: 210, M feat: 1421
root - INFO     Dropped features quantified in less than 126 samples.
root - INFO     After feat selection: N samples: 210, M feat: 1213
root - INFO     Min No. of Protein-Groups in single sample: 754
root - INFO     Finally: N samples: 210, M feat: 1213
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 A0A075B6J9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 15.912 16.852 15.570 16.481 20.246 16.764 17.584 16.988 20.054 NaN ... 16.012 15.178 NaN 15.050 16.842 19.863 NaN 19.563 12.837 12.805
Sample_001 15.936 16.874 15.519 16.387 19.941 18.786 17.144 NaN 19.067 16.188 ... 15.528 15.576 NaN 14.833 16.597 20.299 15.556 19.386 13.970 12.442
Sample_002 16.111 14.523 15.935 16.416 19.251 16.832 15.671 17.012 18.569 NaN ... 15.229 14.728 13.757 15.118 17.440 19.598 15.735 20.447 12.636 12.505
Sample_003 16.107 17.032 15.802 16.979 19.628 17.852 18.877 14.182 18.985 13.438 ... 15.495 14.590 14.682 15.140 17.356 19.429 NaN 20.216 12.627 12.445
Sample_004 15.603 15.331 15.375 16.679 20.450 18.682 17.081 14.140 19.686 14.495 ... 14.757 15.094 14.048 15.256 17.075 19.582 15.328 19.867 13.145 12.235
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 17.705 17.039 NaN 16.413 19.102 16.064 ... 15.235 15.684 14.236 15.415 17.551 17.922 16.340 19.928 12.929 11.802
Sample_206 15.798 17.554 15.600 15.938 18.154 18.152 16.503 16.860 18.538 15.288 ... 15.422 16.106 NaN 15.345 17.084 18.708 14.249 19.433 NaN NaN
Sample_207 15.739 16.877 15.469 16.898 18.636 17.950 16.321 16.401 18.849 17.580 ... 15.808 16.098 14.403 15.715 16.586 18.725 16.138 19.599 13.637 11.174
Sample_208 15.477 16.779 14.995 16.132 14.908 17.530 NaN 16.119 18.368 15.202 ... 15.157 16.712 NaN 14.640 16.533 19.411 15.807 19.545 13.216 NaN
Sample_209 15.727 17.261 15.175 16.235 17.893 17.744 16.371 15.780 18.806 16.532 ... 15.237 15.652 15.211 14.205 16.749 19.275 15.732 19.577 11.042 11.791

210 rows × 1213 columns

Number of complete cases which can be used:

Hide code cell source

 
mask_has_target = data.index.levels[0].intersection(target.index)
assert not mask_has_target.empty, f"No data for target: {data.index.levels[0]} and {target.index}"
print(
    f"Samples available both in proteomics data and for target: {len(mask_has_target)}")
target, data, ald_study = target.loc[mask_has_target], data.loc[mask_has_target], ald_study.loc[mask_has_target]

Samples available both in proteomics data and for target: 210

Load imputations from specified model#

Hide code cell source

 
fname = args.out_preds / args.template_pred.format(args.model_key)
print(f"missing values pred. by {args.model_key}: {fname}")
load_single_csv_pred_file = pimmslearn.analyzers.compare_predictions.load_single_csv_pred_file
pred_real_na = load_single_csv_pred_file(fname).loc[mask_has_target]
pred_real_na.sample(3)

missing values pred. by VAE: runs/alzheimer_study/preds/pred_real_na_VAE.csv

Sample ID   protein groups                       
Sample_108  Q9NS00;Q9NS00-2                         12.206
Sample_047  A0A087WTT8;A0A0A0MQX5;O94779;O94779-2   12.973
Sample_145  Q5R3F8                                  14.399
Name: intensity, dtype: float64

Load imputations from baseline model#

Hide code cell source

 
fname = args.out_preds / args.template_pred.format(args.baseline)
pred_real_na_baseline = load_single_csv_pred_file(fname)  # .loc[mask_has_target]
pred_real_na_baseline

Sample ID   protein groups          
Sample_000  A0A075B6J9                 14.940
            A0A075B6Q5                 13.757
            A0A075B6R2                 12.682
            A0A075B6S5                 12.439
            A0A087WSY4                 13.868
                                        ...  
Sample_209  Q9P1W8;Q9P1W8-2;Q9P1W8-4   12.389
            Q9UI40;Q9UI40-2            12.263
            Q9UIW2                     13.217
            Q9UMX0;Q9UMX0-2;Q9UMX0-4   12.512
            Q9UP79                     12.945
Name: intensity, Length: 46401, dtype: float64

Modeling setup#

General approach:

  • use one train, test split of the data

  • select best 10 features from training data X_train, y_train before binarization of target

  • dichotomize (binarize) data into to groups (zero and 1)

  • evaluate model on the test data X_test, y_test

Repeat general approach for

  1. all original ald data: all features justed in original ALD study

  2. all model data: all features available my using the self supervised deep learning model

  3. newly available feat only: the subset of features available from the self supervised deep learning model which were newly retained using the new approach

All data:

Hide code cell source

 
X = pd.concat([data, pred_real_na]).unstack()
X
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H7 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 15.912 16.852 15.570 16.481 17.301 20.246 16.764 17.584 16.988 20.054 ... 16.012 15.178 14.199 15.050 16.842 19.863 15.865 19.563 12.837 12.805
Sample_001 15.936 16.874 15.519 16.387 13.796 19.941 18.786 17.144 16.865 19.067 ... 15.528 15.576 14.066 14.833 16.597 20.299 15.556 19.386 13.970 12.442
Sample_002 16.111 14.523 15.935 16.416 18.175 19.251 16.832 15.671 17.012 18.569 ... 15.229 14.728 13.757 15.118 17.440 19.598 15.735 20.447 12.636 12.505
Sample_003 16.107 17.032 15.802 16.979 15.963 19.628 17.852 18.877 14.182 18.985 ... 15.495 14.590 14.682 15.140 17.356 19.429 16.071 20.216 12.627 12.445
Sample_004 15.603 15.331 15.375 16.679 15.473 20.450 18.682 17.081 14.140 19.686 ... 14.757 15.094 14.048 15.256 17.075 19.582 15.328 19.867 13.145 12.235
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 15.429 17.705 17.039 15.975 16.413 19.102 ... 15.235 15.684 14.236 15.415 17.551 17.922 16.340 19.928 12.929 11.802
Sample_206 15.798 17.554 15.600 15.938 15.741 18.154 18.152 16.503 16.860 18.538 ... 15.422 16.106 14.638 15.345 17.084 18.708 14.249 19.433 11.626 11.056
Sample_207 15.739 16.877 15.469 16.898 14.777 18.636 17.950 16.321 16.401 18.849 ... 15.808 16.098 14.403 15.715 16.586 18.725 16.138 19.599 13.637 11.174
Sample_208 15.477 16.779 14.995 16.132 14.366 14.908 17.530 16.983 16.119 18.368 ... 15.157 16.712 14.610 14.640 16.533 19.411 15.807 19.545 13.216 11.262
Sample_209 15.727 17.261 15.175 16.235 15.231 17.893 17.744 16.371 15.780 18.806 ... 15.237 15.652 15.211 14.205 16.749 19.275 15.732 19.577 11.042 11.791

210 rows × 1421 columns

Subset of data by ALD criteria#

Hide code cell source

 
# could be just observed, drop columns with missing values
ald_study = pd.concat(
    [ald_study.stack(),
     pred_real_na_baseline.loc[
        # only select columns in selected in ald_study
        pd.IndexSlice[:, pred_real_na.index.levels[-1].intersection(ald_study.columns)]
    ]
    ]
).unstack()
ald_study
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 A0A075B6K4 ... O14793 O95479;R4GMU1 P01282;P01282-2 P10619;P10619-2;X6R5C5;X6R8A1 P21810 Q14956;Q14956-2 Q6ZMP0;Q6ZMP0-2 Q9HBW1 Q9NY15 P17050
Sample ID
Sample_000 15.912 16.852 15.570 16.481 20.246 16.764 17.584 16.988 20.054 16.148 ... 14.051 12.078 12.537 13.475 14.440 13.662 12.857 14.353 12.748 13.655
Sample_001 15.936 16.874 15.519 16.387 19.941 18.786 17.144 13.042 19.067 16.127 ... 12.330 12.439 12.044 12.454 13.382 12.739 12.921 13.182 13.122 13.202
Sample_002 16.111 14.523 15.935 16.416 19.251 16.832 15.671 17.012 18.569 15.387 ... 12.583 12.572 12.103 13.894 11.879 12.361 12.745 12.834 13.456 12.397
Sample_003 16.107 17.032 15.802 16.979 19.628 17.852 18.877 14.182 18.985 16.565 ... 12.065 12.135 12.925 13.803 12.223 12.104 11.823 14.041 14.192 13.144
Sample_004 15.603 15.331 15.375 16.679 20.450 18.682 17.081 14.140 19.686 16.418 ... 12.839 12.940 13.356 12.339 12.095 13.547 12.575 13.088 13.740 11.047
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 17.705 17.039 13.027 16.413 19.102 15.350 ... 14.269 14.064 16.826 18.182 15.225 15.044 14.192 16.605 14.995 14.257
Sample_206 15.798 17.554 15.600 15.938 18.154 18.152 16.503 16.860 18.538 16.582 ... 14.273 17.700 16.802 20.202 15.280 15.086 13.978 18.086 15.557 14.171
Sample_207 15.739 16.877 15.469 16.898 18.636 17.950 16.321 16.401 18.849 15.768 ... 14.473 16.882 16.917 20.105 15.690 15.135 13.138 17.066 15.706 15.690
Sample_208 15.477 16.779 14.995 16.132 14.908 17.530 12.648 16.119 18.368 17.560 ... 15.234 17.175 16.521 18.859 15.305 15.161 13.006 17.917 15.396 14.371
Sample_209 15.727 17.261 15.175 16.235 17.893 17.744 16.371 15.780 18.806 16.338 ... 14.556 16.656 16.954 18.493 15.823 14.626 13.385 17.767 15.687 13.573

210 rows × 1213 columns

Features which would not have been included using ALD criteria:

Hide code cell source

 
new_features = X.columns.difference(ald_study.columns)
new_features

Index(['A0A075B6H7', 'A0A075B6Q5', 'A0A075B7B8', 'A0A087WSY4',
       'A0A087WTT8;A0A0A0MQX5;O94779;O94779-2', 'A0A087WXB8;Q9Y274',
       'A0A087WXE9;E9PQ70;Q6UXH9;Q6UXH9-2;Q6UXH9-3',
       'A0A087X1Z2;C9JTV4;H0Y4Y4;Q8WYH2;Q96C19;Q9BUP0;Q9BUP0-2',
       'A0A0A0MQS9;A0A0A0MTC7;Q16363;Q16363-2', 'A0A0A0MSN4;P12821;P12821-2',
       ...
       'Q9NZ94;Q9NZ94-2;Q9NZ94-3', 'Q9NZU1', 'Q9P1W8;Q9P1W8-2;Q9P1W8-4',
       'Q9UHI8', 'Q9UI40;Q9UI40-2',
       'Q9UIB8;Q9UIB8-2;Q9UIB8-3;Q9UIB8-4;Q9UIB8-5;Q9UIB8-6',
       'Q9UKZ4;Q9UKZ4-2', 'Q9UMX0;Q9UMX0-2;Q9UMX0-4', 'Q9Y281;Q9Y281-3',
       'Q9Y490'],
      dtype='object', name='protein groups', length=208)

Binarize targets, but also keep groups for stratification

Hide code cell source

 
target_to_group = target.copy()
target = target >= args.cutoff_target
pd.crosstab(target.squeeze(), target_to_group.squeeze())
AD 0 1
AD
False 122 0
True 0 88

Determine best number of parameters by cross validation procedure#

using subset of data by ALD criteria:

Hide code cell source

 
cv_feat_ald = njab.sklearn.find_n_best_features(X=ald_study, y=target, name=args.target,
                                                groups=target_to_group)
cv_feat_ald = (cv_feat_ald
               .drop('test_case', axis=1)
               .groupby('n_features')
               .agg(['mean', 'std']))
cv_feat_ald

  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 113.77it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
100%|██████████| 2/2 [00:00<00:00,  6.18it/s]
100%|██████████| 2/2 [00:00<00:00,  6.14it/s]
  0%|          | 0/3 [00:00<?, ?it/s]
 67%|██████▋   | 2/3 [00:00<00:00,  4.87it/s]
100%|██████████| 3/3 [00:00<00:00,  3.45it/s]
100%|██████████| 3/3 [00:00<00:00,  3.65it/s]
  0%|          | 0/4 [00:00<?, ?it/s]
 50%|█████     | 2/4 [00:00<00:00,  7.92it/s]
 75%|███████▌  | 3/4 [00:00<00:00,  5.71it/s]
100%|██████████| 4/4 [00:00<00:00,  5.16it/s]
100%|██████████| 4/4 [00:00<00:00,  5.53it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 40%|████      | 2/5 [00:00<00:00,  5.19it/s]
 60%|██████    | 3/5 [00:00<00:00,  4.67it/s]
 80%|████████  | 4/5 [00:01<00:00,  3.42it/s]
100%|██████████| 5/5 [00:01<00:00,  3.05it/s]
100%|██████████| 5/5 [00:01<00:00,  3.45it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
 33%|███▎      | 2/6 [00:00<00:00,  8.28it/s]
 50%|█████     | 3/6 [00:00<00:00,  6.04it/s]
 67%|██████▋   | 4/6 [00:00<00:00,  4.98it/s]
 83%|████████▎ | 5/6 [00:01<00:00,  4.45it/s]
100%|██████████| 6/6 [00:01<00:00,  3.99it/s]
100%|██████████| 6/6 [00:01<00:00,  4.59it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 29%|██▊       | 2/7 [00:00<00:00,  5.95it/s]
 43%|████▎     | 3/7 [00:00<00:00,  5.18it/s]
 57%|█████▋    | 4/7 [00:00<00:00,  4.55it/s]
 71%|███████▏  | 5/7 [00:01<00:00,  4.12it/s]
 86%|████████▌ | 6/7 [00:01<00:00,  3.76it/s]
100%|██████████| 7/7 [00:01<00:00,  3.69it/s]
100%|██████████| 7/7 [00:01<00:00,  4.09it/s]
  0%|          | 0/8 [00:00<?, ?it/s]
 25%|██▌       | 2/8 [00:00<00:01,  4.23it/s]
 38%|███▊      | 3/8 [00:00<00:01,  3.01it/s]
 50%|█████     | 4/8 [00:01<00:01,  2.79it/s]
 62%|██████▎   | 5/8 [00:01<00:01,  2.90it/s]
 75%|███████▌  | 6/8 [00:01<00:00,  3.03it/s]
 88%|████████▊ | 7/8 [00:02<00:00,  3.17it/s]
100%|██████████| 8/8 [00:02<00:00,  3.32it/s]
100%|██████████| 8/8 [00:02<00:00,  3.18it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
 22%|██▏       | 2/9 [00:00<00:00,  7.05it/s]
 33%|███▎      | 3/9 [00:00<00:01,  4.96it/s]
 44%|████▍     | 4/9 [00:00<00:01,  4.44it/s]
 56%|█████▌    | 5/9 [00:01<00:00,  4.13it/s]
 67%|██████▋   | 6/9 [00:01<00:00,  3.92it/s]
 78%|███████▊  | 7/9 [00:01<00:00,  4.07it/s]
 89%|████████▉ | 8/9 [00:01<00:00,  4.10it/s]
100%|██████████| 9/9 [00:02<00:00,  4.20it/s]
100%|██████████| 9/9 [00:02<00:00,  4.32it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 20%|██        | 2/10 [00:00<00:01,  7.63it/s]
 30%|███       | 3/10 [00:00<00:01,  5.37it/s]
 40%|████      | 4/10 [00:00<00:01,  4.48it/s]
 50%|█████     | 5/10 [00:01<00:01,  4.31it/s]
 60%|██████    | 6/10 [00:01<00:01,  3.91it/s]
 70%|███████   | 7/10 [00:01<00:00,  3.90it/s]
 80%|████████  | 8/10 [00:01<00:00,  3.75it/s]
 90%|█████████ | 9/10 [00:02<00:00,  3.72it/s]
100%|██████████| 10/10 [00:02<00:00,  3.93it/s]
100%|██████████| 10/10 [00:02<00:00,  4.15it/s]
  0%|          | 0/11 [00:00<?, ?it/s]
 18%|█▊        | 2/11 [00:00<00:01,  7.41it/s]
 27%|██▋       | 3/11 [00:00<00:01,  5.05it/s]
 36%|███▋      | 4/11 [00:00<00:01,  4.56it/s]
 45%|████▌     | 5/11 [00:01<00:01,  4.18it/s]
 55%|█████▍    | 6/11 [00:01<00:01,  3.97it/s]
 64%|██████▎   | 7/11 [00:01<00:01,  3.87it/s]
 73%|███████▎  | 8/11 [00:01<00:00,  3.95it/s]
 82%|████████▏ | 9/11 [00:02<00:00,  4.04it/s]
 91%|█████████ | 10/11 [00:02<00:00,  4.23it/s]
100%|██████████| 11/11 [00:02<00:00,  3.74it/s]
100%|██████████| 11/11 [00:02<00:00,  4.11it/s]
  0%|          | 0/12 [00:00<?, ?it/s]
 17%|█▋        | 2/12 [00:00<00:01,  7.45it/s]
 25%|██▌       | 3/12 [00:00<00:01,  5.12it/s]
 33%|███▎      | 4/12 [00:00<00:01,  4.47it/s]
 42%|████▏     | 5/12 [00:01<00:01,  4.02it/s]
 50%|█████     | 6/12 [00:01<00:01,  3.99it/s]
 58%|█████▊    | 7/12 [00:01<00:01,  3.91it/s]
 67%|██████▋   | 8/12 [00:01<00:01,  3.80it/s]
 75%|███████▌  | 9/12 [00:02<00:00,  3.57it/s]
 83%|████████▎ | 10/12 [00:02<00:00,  3.35it/s]
 92%|█████████▏| 11/12 [00:02<00:00,  3.34it/s]
100%|██████████| 12/12 [00:03<00:00,  3.54it/s]
100%|██████████| 12/12 [00:03<00:00,  3.84it/s]
  0%|          | 0/13 [00:00<?, ?it/s]
 15%|█▌        | 2/13 [00:00<00:01,  8.26it/s]
 23%|██▎       | 3/13 [00:00<00:01,  5.44it/s]
 31%|███       | 4/13 [00:00<00:02,  4.46it/s]
 38%|███▊      | 5/13 [00:01<00:01,  4.24it/s]
 46%|████▌     | 6/13 [00:01<00:01,  4.18it/s]
 54%|█████▍    | 7/13 [00:01<00:01,  3.79it/s]
 62%|██████▏   | 8/13 [00:01<00:01,  3.76it/s]
 69%|██████▉   | 9/13 [00:02<00:01,  3.29it/s]
 77%|███████▋  | 10/13 [00:02<00:00,  3.42it/s]
 85%|████████▍ | 11/13 [00:02<00:00,  3.48it/s]
 92%|█████████▏| 12/13 [00:03<00:00,  3.75it/s]
100%|██████████| 13/13 [00:03<00:00,  3.87it/s]
100%|██████████| 13/13 [00:03<00:00,  3.96it/s]
  0%|          | 0/14 [00:00<?, ?it/s]
 14%|█▍        | 2/14 [00:00<00:01,  8.64it/s]
 21%|██▏       | 3/14 [00:00<00:01,  5.56it/s]
 29%|██▊       | 4/14 [00:00<00:02,  4.43it/s]
 36%|███▌      | 5/14 [00:01<00:02,  4.02it/s]
 43%|████▎     | 6/14 [00:01<00:02,  3.95it/s]
 50%|█████     | 7/14 [00:01<00:01,  3.76it/s]
 57%|█████▋    | 8/14 [00:02<00:01,  3.15it/s]
 64%|██████▍   | 9/14 [00:02<00:01,  3.12it/s]
 71%|███████▏  | 10/14 [00:02<00:01,  3.21it/s]
 79%|███████▊  | 11/14 [00:02<00:00,  3.39it/s]
 86%|████████▌ | 12/14 [00:03<00:00,  3.54it/s]
 93%|█████████▎| 13/14 [00:03<00:00,  3.49it/s]
100%|██████████| 14/14 [00:03<00:00,  3.53it/s]
100%|██████████| 14/14 [00:03<00:00,  3.70it/s]
  0%|          | 0/15 [00:00<?, ?it/s]
 13%|█▎        | 2/15 [00:00<00:02,  5.15it/s]
 20%|██        | 3/15 [00:00<00:03,  3.73it/s]
 27%|██▋       | 4/15 [00:01<00:03,  3.18it/s]
 33%|███▎      | 5/15 [00:01<00:02,  3.55it/s]
 40%|████      | 6/15 [00:01<00:02,  3.72it/s]
 47%|████▋     | 7/15 [00:01<00:02,  3.87it/s]
 53%|█████▎    | 8/15 [00:02<00:01,  4.02it/s]
 60%|██████    | 9/15 [00:02<00:01,  4.05it/s]
 67%|██████▋   | 10/15 [00:02<00:01,  3.52it/s]
 73%|███████▎  | 11/15 [00:02<00:01,  3.57it/s]
 80%|████████  | 12/15 [00:03<00:00,  3.63it/s]
 87%|████████▋ | 13/15 [00:03<00:00,  3.61it/s]
 93%|█████████▎| 14/15 [00:03<00:00,  3.66it/s]
100%|██████████| 15/15 [00:04<00:00,  3.48it/s]
100%|██████████| 15/15 [00:04<00:00,  3.66it/s]
fit_time score_time test_precision test_recall test_f1 test_balanced_accuracy test_roc_auc test_average_precision n_observations
mean std mean std mean std mean std mean std mean std mean std mean std mean std
n_features
1 0.006 0.003 0.062 0.026 0.469 0.448 0.055 0.069 0.095 0.108 0.522 0.031 0.844 0.063 0.818 0.086 210.000 0.000
2 0.004 0.002 0.050 0.020 0.649 0.138 0.506 0.142 0.557 0.117 0.648 0.079 0.711 0.081 0.660 0.092 210.000 0.000
3 0.006 0.002 0.065 0.024 0.718 0.097 0.604 0.112 0.649 0.086 0.713 0.060 0.768 0.060 0.722 0.083 210.000 0.000
4 0.004 0.001 0.042 0.012 0.773 0.101 0.640 0.110 0.694 0.084 0.749 0.060 0.793 0.060 0.755 0.090 210.000 0.000
5 0.005 0.002 0.054 0.022 0.698 0.091 0.659 0.104 0.673 0.079 0.724 0.062 0.850 0.052 0.827 0.056 210.000 0.000
6 0.006 0.004 0.063 0.037 0.719 0.081 0.687 0.099 0.699 0.076 0.745 0.059 0.853 0.054 0.827 0.059 210.000 0.000
7 0.005 0.002 0.048 0.018 0.726 0.088 0.722 0.116 0.719 0.084 0.760 0.068 0.857 0.057 0.823 0.067 210.000 0.000
8 0.005 0.002 0.058 0.027 0.777 0.073 0.763 0.095 0.765 0.062 0.799 0.051 0.895 0.048 0.886 0.048 210.000 0.000
9 0.005 0.002 0.049 0.021 0.774 0.074 0.761 0.091 0.764 0.062 0.798 0.051 0.893 0.049 0.884 0.049 210.000 0.000
10 0.004 0.002 0.046 0.020 0.812 0.081 0.821 0.093 0.813 0.071 0.840 0.061 0.909 0.049 0.902 0.046 210.000 0.000
11 0.004 0.002 0.044 0.016 0.811 0.081 0.818 0.099 0.812 0.075 0.839 0.064 0.908 0.050 0.900 0.048 210.000 0.000
12 0.004 0.002 0.046 0.020 0.810 0.083 0.815 0.100 0.809 0.076 0.837 0.065 0.907 0.050 0.898 0.048 210.000 0.000
13 0.003 0.000 0.035 0.003 0.804 0.080 0.792 0.110 0.793 0.075 0.824 0.063 0.905 0.050 0.895 0.048 210.000 0.000
14 0.004 0.002 0.043 0.015 0.802 0.075 0.797 0.108 0.795 0.072 0.826 0.060 0.903 0.051 0.892 0.050 210.000 0.000
15 0.006 0.003 0.055 0.023 0.796 0.073 0.782 0.104 0.784 0.067 0.817 0.056 0.901 0.051 0.890 0.049 210.000 0.000

Using all data:

Hide code cell source

 
cv_feat_all = njab.sklearn.find_n_best_features(X=X, y=target, name=args.target,
                                                groups=target_to_group)
cv_feat_all = cv_feat_all.drop('test_case', axis=1).groupby('n_features').agg(['mean', 'std'])
cv_feat_all

  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 211.72it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
100%|██████████| 2/2 [00:00<00:00,  4.51it/s]
100%|██████████| 2/2 [00:00<00:00,  4.47it/s]
  0%|          | 0/3 [00:00<?, ?it/s]
 67%|██████▋   | 2/3 [00:00<00:00,  3.56it/s]
100%|██████████| 3/3 [00:01<00:00,  2.57it/s]
100%|██████████| 3/3 [00:01<00:00,  2.71it/s]
  0%|          | 0/4 [00:00<?, ?it/s]
 50%|█████     | 2/4 [00:00<00:00,  3.28it/s]
 75%|███████▌  | 3/4 [00:01<00:00,  2.84it/s]
100%|██████████| 4/4 [00:01<00:00,  2.66it/s]
100%|██████████| 4/4 [00:01<00:00,  2.77it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 40%|████      | 2/5 [00:00<00:00,  5.51it/s]
 60%|██████    | 3/5 [00:00<00:00,  3.46it/s]
 80%|████████  | 4/5 [00:01<00:00,  2.87it/s]
100%|██████████| 5/5 [00:01<00:00,  2.75it/s]
100%|██████████| 5/5 [00:01<00:00,  3.04it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
 33%|███▎      | 2/6 [00:00<00:00, 10.43it/s]
 67%|██████▋   | 4/6 [00:00<00:00,  5.97it/s]
 83%|████████▎ | 5/6 [00:00<00:00,  4.71it/s]
100%|██████████| 6/6 [00:01<00:00,  3.75it/s]
100%|██████████| 6/6 [00:01<00:00,  4.45it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 29%|██▊       | 2/7 [00:00<00:00,  7.71it/s]
 43%|████▎     | 3/7 [00:00<00:00,  5.15it/s]
 57%|█████▋    | 4/7 [00:00<00:00,  4.46it/s]
 71%|███████▏  | 5/7 [00:01<00:00,  4.05it/s]
 86%|████████▌ | 6/7 [00:01<00:00,  3.72it/s]
100%|██████████| 7/7 [00:01<00:00,  3.78it/s]
100%|██████████| 7/7 [00:01<00:00,  4.18it/s]
  0%|          | 0/8 [00:00<?, ?it/s]
 25%|██▌       | 2/8 [00:00<00:00,  7.44it/s]
 38%|███▊      | 3/8 [00:00<00:00,  5.73it/s]
 50%|█████     | 4/8 [00:00<00:00,  4.90it/s]
 62%|██████▎   | 5/8 [00:01<00:00,  4.54it/s]
 75%|███████▌  | 6/8 [00:01<00:00,  4.08it/s]
 88%|████████▊ | 7/8 [00:01<00:00,  3.91it/s]
100%|██████████| 8/8 [00:01<00:00,  4.03it/s]
100%|██████████| 8/8 [00:01<00:00,  4.42it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
 22%|██▏       | 2/9 [00:00<00:01,  6.56it/s]
 33%|███▎      | 3/9 [00:00<00:01,  4.57it/s]
 44%|████▍     | 4/9 [00:00<00:01,  4.04it/s]
 56%|█████▌    | 5/9 [00:01<00:01,  3.64it/s]
 67%|██████▋   | 6/9 [00:01<00:00,  3.78it/s]
 78%|███████▊  | 7/9 [00:01<00:00,  3.67it/s]
 89%|████████▉ | 8/9 [00:02<00:00,  3.45it/s]
100%|██████████| 9/9 [00:02<00:00,  3.62it/s]
100%|██████████| 9/9 [00:02<00:00,  3.83it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 20%|██        | 2/10 [00:00<00:02,  3.52it/s]
 30%|███       | 3/10 [00:01<00:02,  2.62it/s]
 40%|████      | 4/10 [00:01<00:02,  2.30it/s]
 50%|█████     | 5/10 [00:02<00:02,  2.25it/s]
 60%|██████    | 6/10 [00:02<00:01,  2.28it/s]
 70%|███████   | 7/10 [00:02<00:01,  2.20it/s]
 80%|████████  | 8/10 [00:03<00:00,  2.16it/s]
 90%|█████████ | 9/10 [00:03<00:00,  2.12it/s]
100%|██████████| 10/10 [00:04<00:00,  2.21it/s]
100%|██████████| 10/10 [00:04<00:00,  2.29it/s]
  0%|          | 0/11 [00:00<?, ?it/s]
 18%|█▊        | 2/11 [00:00<00:02,  4.41it/s]
 27%|██▋       | 3/11 [00:00<00:02,  2.85it/s]
 36%|███▋      | 4/11 [00:01<00:02,  2.54it/s]
 45%|████▌     | 5/11 [00:01<00:02,  2.79it/s]
 55%|█████▍    | 6/11 [00:02<00:01,  2.72it/s]
 64%|██████▎   | 7/11 [00:02<00:01,  2.75it/s]
 73%|███████▎  | 8/11 [00:02<00:01,  2.85it/s]
 82%|████████▏ | 9/11 [00:03<00:00,  2.96it/s]
 91%|█████████ | 10/11 [00:03<00:00,  2.96it/s]
100%|██████████| 11/11 [00:03<00:00,  3.09it/s]
100%|██████████| 11/11 [00:03<00:00,  2.94it/s]
  0%|          | 0/12 [00:00<?, ?it/s]
 17%|█▋        | 2/12 [00:00<00:01,  6.34it/s]
 25%|██▌       | 3/12 [00:00<00:01,  4.70it/s]
 33%|███▎      | 4/12 [00:00<00:01,  4.02it/s]
 42%|████▏     | 5/12 [00:01<00:01,  3.51it/s]
 50%|█████     | 6/12 [00:01<00:02,  2.92it/s]
 58%|█████▊    | 7/12 [00:02<00:01,  2.57it/s]
 67%|██████▋   | 8/12 [00:02<00:01,  2.49it/s]
 75%|███████▌  | 9/12 [00:03<00:01,  2.38it/s]
 83%|████████▎ | 10/12 [00:03<00:00,  2.33it/s]
 92%|█████████▏| 11/12 [00:03<00:00,  2.36it/s]
100%|██████████| 12/12 [00:04<00:00,  2.42it/s]
100%|██████████| 12/12 [00:04<00:00,  2.75it/s]
  0%|          | 0/13 [00:00<?, ?it/s]
 15%|█▌        | 2/13 [00:00<00:02,  4.77it/s]
 23%|██▎       | 3/13 [00:00<00:02,  3.46it/s]
 31%|███       | 4/13 [00:01<00:02,  3.06it/s]
 38%|███▊      | 5/13 [00:01<00:02,  2.99it/s]
 46%|████▌     | 6/13 [00:01<00:02,  3.08it/s]
 54%|█████▍    | 7/13 [00:02<00:01,  3.05it/s]
 62%|██████▏   | 8/13 [00:02<00:01,  3.08it/s]
 69%|██████▉   | 9/13 [00:02<00:01,  3.10it/s]
 77%|███████▋  | 10/13 [00:03<00:00,  3.11it/s]
 85%|████████▍ | 11/13 [00:03<00:00,  2.99it/s]
 92%|█████████▏| 12/13 [00:03<00:00,  3.00it/s]
100%|██████████| 13/13 [00:04<00:00,  3.02it/s]
100%|██████████| 13/13 [00:04<00:00,  3.11it/s]
  0%|          | 0/14 [00:00<?, ?it/s]
 14%|█▍        | 2/14 [00:00<00:01,  7.76it/s]
 21%|██▏       | 3/14 [00:00<00:02,  4.74it/s]
 29%|██▊       | 4/14 [00:00<00:02,  3.98it/s]
 36%|███▌      | 5/14 [00:01<00:02,  3.59it/s]
 43%|████▎     | 6/14 [00:01<00:02,  3.31it/s]
 50%|█████     | 7/14 [00:02<00:02,  2.55it/s]
 57%|█████▋    | 8/14 [00:02<00:02,  2.41it/s]
 64%|██████▍   | 9/14 [00:03<00:02,  2.19it/s]
 71%|███████▏  | 10/14 [00:03<00:01,  2.32it/s]
 79%|███████▊  | 11/14 [00:03<00:01,  2.42it/s]
 86%|████████▌ | 12/14 [00:04<00:00,  2.60it/s]
 93%|█████████▎| 13/14 [00:04<00:00,  2.57it/s]
100%|██████████| 14/14 [00:05<00:00,  2.63it/s]
100%|██████████| 14/14 [00:05<00:00,  2.80it/s]
  0%|          | 0/15 [00:00<?, ?it/s]
 13%|█▎        | 2/15 [00:00<00:02,  6.17it/s]
 20%|██        | 3/15 [00:00<00:02,  4.31it/s]
 27%|██▋       | 4/15 [00:00<00:02,  3.87it/s]
 33%|███▎      | 5/15 [00:01<00:02,  3.54it/s]
 40%|████      | 6/15 [00:01<00:02,  3.44it/s]
 47%|████▋     | 7/15 [00:01<00:02,  3.49it/s]
 53%|█████▎    | 8/15 [00:02<00:02,  3.33it/s]
 60%|██████    | 9/15 [00:02<00:01,  3.38it/s]
 67%|██████▋   | 10/15 [00:02<00:01,  3.12it/s]
 73%|███████▎  | 11/15 [00:03<00:01,  3.32it/s]
 80%|████████  | 12/15 [00:03<00:00,  3.37it/s]
 87%|████████▋ | 13/15 [00:03<00:00,  3.48it/s]
 93%|█████████▎| 14/15 [00:03<00:00,  3.64it/s]
100%|██████████| 15/15 [00:04<00:00,  3.45it/s]
100%|██████████| 15/15 [00:04<00:00,  3.53it/s]
fit_time score_time test_precision test_recall test_f1 test_balanced_accuracy test_roc_auc test_average_precision n_observations
mean std mean std mean std mean std mean std mean std mean std mean std mean std
n_features
1 0.005 0.002 0.054 0.020 0.010 0.071 0.001 0.008 0.002 0.015 0.497 0.008 0.858 0.063 0.826 0.087 210.000 0.000
2 0.004 0.002 0.051 0.021 0.768 0.122 0.632 0.126 0.684 0.097 0.742 0.074 0.787 0.076 0.733 0.095 210.000 0.000
3 0.005 0.003 0.054 0.022 0.745 0.090 0.675 0.118 0.703 0.089 0.752 0.067 0.852 0.060 0.818 0.073 210.000 0.000
4 0.004 0.002 0.047 0.019 0.741 0.092 0.669 0.119 0.698 0.090 0.748 0.069 0.849 0.061 0.815 0.072 210.000 0.000
5 0.004 0.001 0.039 0.009 0.734 0.102 0.656 0.129 0.686 0.094 0.739 0.073 0.866 0.056 0.837 0.066 210.000 0.000
6 0.004 0.002 0.042 0.022 0.734 0.095 0.667 0.131 0.693 0.096 0.744 0.074 0.866 0.056 0.839 0.065 210.000 0.000
7 0.004 0.001 0.042 0.014 0.751 0.106 0.691 0.132 0.714 0.101 0.761 0.081 0.873 0.060 0.846 0.070 210.000 0.000
8 0.004 0.002 0.047 0.022 0.750 0.103 0.692 0.128 0.714 0.097 0.761 0.078 0.871 0.061 0.843 0.069 210.000 0.000
9 0.004 0.001 0.042 0.017 0.759 0.099 0.738 0.122 0.744 0.094 0.783 0.078 0.881 0.059 0.861 0.065 210.000 0.000
10 0.004 0.002 0.044 0.014 0.767 0.097 0.750 0.125 0.755 0.099 0.792 0.082 0.886 0.057 0.862 0.066 210.000 0.000
11 0.005 0.002 0.049 0.020 0.792 0.084 0.795 0.111 0.789 0.079 0.820 0.067 0.912 0.048 0.889 0.056 210.000 0.000
12 0.004 0.001 0.038 0.006 0.793 0.093 0.794 0.116 0.788 0.085 0.820 0.073 0.914 0.048 0.894 0.055 210.000 0.000
13 0.005 0.003 0.053 0.022 0.794 0.091 0.795 0.114 0.790 0.083 0.820 0.071 0.912 0.048 0.891 0.055 210.000 0.000
14 0.004 0.001 0.040 0.009 0.793 0.090 0.796 0.117 0.790 0.084 0.821 0.071 0.912 0.049 0.892 0.056 210.000 0.000
15 0.006 0.003 0.054 0.021 0.804 0.085 0.790 0.109 0.793 0.080 0.824 0.066 0.914 0.046 0.896 0.054 210.000 0.000

Using only new features:

Hide code cell source

 
cv_feat_new = njab.sklearn.find_n_best_features(X=X.loc[:, new_features],
                                                y=target, name=args.target,
                                                groups=target_to_group)
cv_feat_new = cv_feat_new.drop('test_case', axis=1).groupby('n_features').agg(['mean', 'std'])
cv_feat_new

  0%|          | 0/1 [00:00<?, ?it/s]
100%|██████████| 1/1 [00:00<00:00, 1019.77it/s]
  0%|          | 0/2 [00:00<?, ?it/s]
100%|██████████| 2/2 [00:00<00:00, 20.86it/s]
  0%|          | 0/3 [00:00<?, ?it/s]
100%|██████████| 3/3 [00:00<00:00, 19.70it/s]
100%|██████████| 3/3 [00:00<00:00, 19.60it/s]
  0%|          | 0/4 [00:00<?, ?it/s]
 75%|███████▌  | 3/4 [00:00<00:00, 17.93it/s]
100%|██████████| 4/4 [00:00<00:00, 13.91it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 60%|██████    | 3/5 [00:00<00:00, 22.31it/s]
100%|██████████| 5/5 [00:00<00:00, 17.92it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
 50%|█████     | 3/6 [00:00<00:00, 21.56it/s]
100%|██████████| 6/6 [00:00<00:00, 16.93it/s]
100%|██████████| 6/6 [00:00<00:00, 17.33it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 43%|████▎     | 3/7 [00:00<00:00, 24.87it/s]
 86%|████████▌ | 6/7 [00:00<00:00, 15.54it/s]
100%|██████████| 7/7 [00:00<00:00, 14.56it/s]
  0%|          | 0/8 [00:00<?, ?it/s]
 38%|███▊      | 3/8 [00:00<00:00, 24.11it/s]
 75%|███████▌  | 6/8 [00:00<00:00, 17.72it/s]
100%|██████████| 8/8 [00:00<00:00, 15.69it/s]
100%|██████████| 8/8 [00:00<00:00, 16.58it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
 22%|██▏       | 2/9 [00:00<00:00, 19.20it/s]
 44%|████▍     | 4/9 [00:00<00:00, 13.66it/s]
 67%|██████▋   | 6/9 [00:00<00:00, 12.71it/s]
 89%|████████▉ | 8/9 [00:00<00:00, 14.83it/s]
100%|██████████| 9/9 [00:00<00:00, 13.56it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 30%|███       | 3/10 [00:00<00:00, 19.08it/s]
 50%|█████     | 5/10 [00:00<00:00, 17.23it/s]
 70%|███████   | 7/10 [00:00<00:00, 17.10it/s]
 90%|█████████ | 9/10 [00:00<00:00, 16.61it/s]
100%|██████████| 10/10 [00:00<00:00, 17.23it/s]
  0%|          | 0/11 [00:00<?, ?it/s]
 27%|██▋       | 3/11 [00:00<00:00, 24.35it/s]
 55%|█████▍    | 6/11 [00:00<00:00, 13.73it/s]
 73%|███████▎  | 8/11 [00:00<00:00, 12.61it/s]
 91%|█████████ | 10/11 [00:00<00:00, 13.57it/s]
100%|██████████| 11/11 [00:00<00:00, 13.57it/s]
  0%|          | 0/12 [00:00<?, ?it/s]
 25%|██▌       | 3/12 [00:00<00:00, 26.28it/s]
 50%|█████     | 6/12 [00:00<00:00, 20.13it/s]
 75%|███████▌  | 9/12 [00:00<00:00, 16.74it/s]
 92%|█████████▏| 11/12 [00:00<00:00, 13.64it/s]
100%|██████████| 12/12 [00:00<00:00, 15.39it/s]
  0%|          | 0/13 [00:00<?, ?it/s]
 23%|██▎       | 3/13 [00:00<00:00, 25.03it/s]
 46%|████▌     | 6/13 [00:00<00:00, 15.43it/s]
 62%|██████▏   | 8/13 [00:00<00:00, 13.51it/s]
 77%|███████▋  | 10/13 [00:00<00:00, 14.32it/s]
 92%|█████████▏| 12/13 [00:00<00:00, 13.06it/s]
100%|██████████| 13/13 [00:00<00:00, 14.38it/s]
  0%|          | 0/14 [00:00<?, ?it/s]
 21%|██▏       | 3/14 [00:00<00:00, 22.07it/s]
 43%|████▎     | 6/14 [00:00<00:00, 18.95it/s]
 57%|█████▋    | 8/14 [00:00<00:00, 17.72it/s]
 71%|███████▏  | 10/14 [00:00<00:00, 15.13it/s]
 86%|████████▌ | 12/14 [00:00<00:00, 13.74it/s]
100%|██████████| 14/14 [00:00<00:00, 13.00it/s]
100%|██████████| 14/14 [00:00<00:00, 14.69it/s]
  0%|          | 0/15 [00:00<?, ?it/s]
 20%|██        | 3/15 [00:00<00:00, 16.06it/s]
 33%|███▎      | 5/15 [00:00<00:00, 15.11it/s]
 47%|████▋     | 7/15 [00:00<00:00, 13.00it/s]
 60%|██████    | 9/15 [00:00<00:00, 12.75it/s]
 73%|███████▎  | 11/15 [00:00<00:00, 13.17it/s]
 87%|████████▋ | 13/15 [00:00<00:00, 14.61it/s]
100%|██████████| 15/15 [00:01<00:00, 13.42it/s]
100%|██████████| 15/15 [00:01<00:00, 13.65it/s]
fit_time score_time test_precision test_recall test_f1 test_balanced_accuracy test_roc_auc test_average_precision n_observations
mean std mean std mean std mean std mean std mean std mean std mean std mean std
n_features
1 0.004 0.002 0.042 0.015 0.000 0.000 0.000 0.000 0.000 0.000 0.500 0.000 0.738 0.066 0.696 0.076 210.000 0.000
2 0.005 0.003 0.051 0.019 0.650 0.092 0.560 0.101 0.594 0.070 0.666 0.052 0.731 0.065 0.686 0.072 210.000 0.000
3 0.003 0.001 0.038 0.007 0.699 0.090 0.615 0.099 0.648 0.070 0.708 0.054 0.802 0.054 0.769 0.061 210.000 0.000
4 0.004 0.001 0.042 0.010 0.687 0.081 0.634 0.118 0.653 0.082 0.710 0.062 0.812 0.055 0.768 0.071 210.000 0.000
5 0.004 0.001 0.038 0.010 0.683 0.083 0.643 0.113 0.655 0.076 0.710 0.056 0.821 0.055 0.779 0.069 210.000 0.000
6 0.004 0.002 0.045 0.018 0.662 0.089 0.638 0.116 0.643 0.082 0.698 0.063 0.813 0.059 0.770 0.075 210.000 0.000
7 0.004 0.003 0.041 0.014 0.664 0.090 0.635 0.117 0.641 0.082 0.697 0.063 0.810 0.059 0.766 0.076 210.000 0.000
8 0.005 0.002 0.048 0.017 0.673 0.092 0.645 0.117 0.651 0.082 0.705 0.063 0.806 0.061 0.763 0.076 210.000 0.000
9 0.004 0.001 0.039 0.010 0.670 0.092 0.639 0.108 0.646 0.076 0.701 0.060 0.803 0.061 0.759 0.076 210.000 0.000
10 0.004 0.001 0.043 0.012 0.673 0.089 0.638 0.107 0.648 0.074 0.702 0.059 0.800 0.061 0.755 0.076 210.000 0.000
11 0.004 0.002 0.040 0.014 0.660 0.103 0.638 0.122 0.642 0.093 0.696 0.074 0.797 0.065 0.749 0.082 210.000 0.000
12 0.006 0.003 0.057 0.020 0.674 0.094 0.634 0.121 0.648 0.090 0.704 0.069 0.807 0.064 0.756 0.080 210.000 0.000
13 0.004 0.002 0.040 0.011 0.672 0.101 0.630 0.125 0.644 0.097 0.701 0.073 0.808 0.063 0.763 0.077 210.000 0.000
14 0.005 0.003 0.054 0.023 0.671 0.105 0.634 0.126 0.646 0.100 0.703 0.078 0.803 0.064 0.755 0.079 210.000 0.000
15 0.006 0.004 0.058 0.031 0.670 0.104 0.634 0.125 0.646 0.099 0.702 0.076 0.799 0.064 0.753 0.078 210.000 0.000

Best number of features by subset of the data:#

Hide code cell source

 
n_feat_best = pd.DataFrame(
    {'ald': cv_feat_ald.loc[:, pd.IndexSlice[:, 'mean']].idxmax(),
     'all': cv_feat_all.loc[:, pd.IndexSlice[:, 'mean']].idxmax(),
     'new': cv_feat_new.loc[:, pd.IndexSlice[:, 'mean']].idxmax()
     }
).droplevel(-1)
n_feat_best
ald all new
fit_time 1 15 15
score_time 3 3 15
test_precision 10 15 3
test_recall 10 14 8
test_f1 10 15 5
test_balanced_accuracy 10 15 5
test_roc_auc 10 15 5
test_average_precision 10 15 5
n_observations 1 1 1

Train, test split#

Show number of cases in train and test data

Hide code cell source

 
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
    X,
    target,
    test_size=.2,
    stratify=target_to_group,
    random_state=42)
idx_train = X_train.index
idx_test = X_test.index

njab.pandas.combine_value_counts(
    pd.concat([y_train, y_test],
              axis=1,
              ignore_index=True,
              ).rename(columns={0: 'train', 1: 'test'})
)
train test
False 98 24
True 70 18

Results#

  • run_model returns dataclasses with the further needed results

  • add mrmr selection of data (select best number of features to use instead of fixing it)

Save results for final model on entire data, new features and ALD study criteria selected data.

Hide code cell source

 
splits = Splits(X_train=X.loc[idx_train],
                X_test=X.loc[idx_test],
                y_train=y_train,
                y_test=y_test)
results_model_full = njab.sklearn.run_model(
    splits,
    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'all'])
results_model_full.name = f'{args.model_key} all'
fname = args.out_folder / f'results_{results_model_full.name}.pkl'
files_out[fname.name] = fname
pimmslearn.io.to_pickle(results_model_full, fname)

splits = Splits(X_train=X.loc[idx_train, new_features],
                X_test=X.loc[idx_test, new_features],
                y_train=y_train,
                y_test=y_test)
results_model_new = njab.sklearn.run_model(
    splits,
    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'new'])
results_model_new.name = f'{args.model_key} new'
fname = args.out_folder / f'results_{results_model_new.name}.pkl'
files_out[fname.name] = fname
pimmslearn.io.to_pickle(results_model_new, fname)

splits_ald = Splits(
    X_train=ald_study.loc[idx_train],
    X_test=ald_study.loc[idx_test],
    y_train=y_train,
    y_test=y_test)
results_ald_full = njab.sklearn.run_model(
    splits_ald,
    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'ald'])
results_ald_full.name = 'ALD study all'
fname = args.out_folder / f'results_{results_ald_full.name}.pkl'
files_out[fname.name] = fname
pimmslearn.io.to_pickle(results_ald_full, fname)

  0%|          | 0/15 [00:00<?, ?it/s]
 13%|█▎        | 2/15 [00:00<00:02,  5.61it/s]
 20%|██        | 3/15 [00:00<00:02,  4.14it/s]
 27%|██▋       | 4/15 [00:01<00:03,  3.33it/s]
 33%|███▎      | 5/15 [00:01<00:03,  2.74it/s]
 40%|████      | 6/15 [00:02<00:03,  2.38it/s]
 47%|████▋     | 7/15 [00:02<00:03,  2.38it/s]
 53%|█████▎    | 8/15 [00:02<00:02,  2.53it/s]
 60%|██████    | 9/15 [00:03<00:02,  2.78it/s]
 67%|██████▋   | 10/15 [00:03<00:01,  2.81it/s]
 73%|███████▎  | 11/15 [00:03<00:01,  2.96it/s]
 80%|████████  | 12/15 [00:04<00:00,  3.23it/s]
 87%|████████▋ | 13/15 [00:04<00:00,  3.46it/s]
 93%|█████████▎| 14/15 [00:04<00:00,  3.63it/s]
100%|██████████| 15/15 [00:04<00:00,  3.87it/s]
100%|██████████| 15/15 [00:04<00:00,  3.17it/s]
  0%|          | 0/5 [00:00<?, ?it/s]
 80%|████████  | 4/5 [00:00<00:00, 32.24it/s]
100%|██████████| 5/5 [00:00<00:00, 30.00it/s]
  0%|          | 0/10 [00:00<?, ?it/s]
 20%|██        | 2/10 [00:00<00:01,  5.11it/s]
 30%|███       | 3/10 [00:00<00:01,  4.76it/s]
 40%|████      | 4/10 [00:00<00:01,  4.98it/s]
 50%|█████     | 5/10 [00:01<00:01,  4.77it/s]
 60%|██████    | 6/10 [00:01<00:00,  4.33it/s]
 70%|███████   | 7/10 [00:01<00:00,  4.23it/s]
 80%|████████  | 8/10 [00:01<00:00,  4.23it/s]
 90%|█████████ | 9/10 [00:02<00:00,  4.38it/s]
100%|██████████| 10/10 [00:02<00:00,  4.07it/s]
100%|██████████| 10/10 [00:02<00:00,  4.36it/s]

ROC-AUC on test split#

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)
plot_split_auc(results_ald_full.test, results_ald_full.name, ax)
plot_split_auc(results_model_full.test, results_model_full.name, ax)
plot_split_auc(results_model_new.test, results_model_new.name, ax)
fname = args.out_folder / 'auc_roc_curve.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/auc_roc_curve.pdf
../../../_images/32b0b28d5f8bd41c5702c7801a3066644d1ea996b3b063f9aff22f62974e3760.png

Data used to plot ROC:

Hide code cell source

 
res = [results_ald_full, results_model_full, results_model_new]

auc_roc_curve = parse_roc(*res)
auc_roc_curve.to_excel(fname.with_suffix('.xlsx'))
auc_roc_curve
ALD study all VAE all VAE new
fpr tpr fpr tpr fpr tpr
0 0.000 0.000 0.000 0.000 0.000 0.000
1 0.000 0.056 0.000 0.056 0.042 0.000
2 0.000 0.611 0.000 0.556 0.083 0.000
3 0.042 0.611 0.083 0.556 0.083 0.278
4 0.042 0.722 0.083 0.611 0.292 0.278
5 0.125 0.722 0.250 0.611 0.292 0.667
6 0.125 0.833 0.250 0.722 0.500 0.667
7 0.292 0.833 0.292 0.722 0.500 0.722
8 0.292 0.889 0.292 0.778 0.542 0.722
9 0.417 0.889 0.333 0.778 0.542 0.833
10 0.417 0.944 0.333 0.833 0.667 0.833
11 0.917 0.944 0.375 0.833 0.667 1.000
12 0.917 1.000 0.375 0.889 1.000 1.000
13 1.000 1.000 0.500 0.889 NaN NaN
14 NaN NaN 0.500 0.944 NaN NaN
15 NaN NaN 0.583 0.944 NaN NaN
16 NaN NaN 0.583 1.000 NaN NaN
17 NaN NaN 1.000 1.000 NaN NaN

Features selected for final models#

Hide code cell source

 
selected_features = pd.DataFrame(
    [results_ald_full.selected_features,
     results_model_full.selected_features,
     results_model_new.selected_features],
    index=[
        results_ald_full.name,
        results_model_full.name,
        results_model_new.name]
).T
selected_features.index.name = 'rank'
fname = args.out_folder / 'mrmr_feat_by_model.xlsx'
files_out[fname.name] = fname
selected_features.to_excel(fname)
selected_features
ALD study all VAE all VAE new
rank
0 P10636-2;P10636-6 P10636-2;P10636-6 Q14894
1 Q15848 P26572 P01704
2 P51888 P15086 P51688
3 P61981 P04075 A0A0C4DGV4;E9PLX3;O43504;R4GMU8
4 P04075 Q16674;W4VSR3 Q96GD0
5 P14174 Q9Y2T3;Q9Y2T3-3 None
6 P08294 P15151-2 None
7 Q9Y2T3;Q9Y2T3-3 P61981 None
8 P63104 P14174 None
9 C9JF17;P05090 Q14894 None
10 None P63104 None
11 None P00492 None
12 None P00338;P00338-3 None
13 None Q6EMK4 None
14 None A0A0C4DGY8;D6RA00;Q9UHY7 None

Precision-Recall plot on test data#

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)

ax = plot_split_prc(results_ald_full.test, results_ald_full.name, ax)
ax = plot_split_prc(results_model_full.test, results_model_full.name, ax)
ax = plot_split_prc(results_model_new.test, results_model_new.name, ax)
fname = folder = args.out_folder / 'prec_recall_curve.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/prec_recall_curve.pdf
../../../_images/31faaf75b22323a012e14c41d49fd4ed97afb7a7f75a53e801d98c93e5efaa6f.png

Data used to plot PRC:

Hide code cell source

 
prec_recall_curve = parse_prc(*res)
prec_recall_curve.to_excel(fname.with_suffix('.xlsx'))
prec_recall_curve
ALD study all VAE all VAE new
precision tpr precision tpr precision tpr
0 0.429 1.000 0.429 1.000 0.429 1.000
1 0.439 1.000 0.439 1.000 0.439 1.000
2 0.450 1.000 0.450 1.000 0.450 1.000
3 0.436 0.944 0.462 1.000 0.462 1.000
4 0.447 0.944 0.474 1.000 0.474 1.000
5 0.459 0.944 0.486 1.000 0.486 1.000
6 0.472 0.944 0.500 1.000 0.500 1.000
7 0.486 0.944 0.514 1.000 0.514 1.000
8 0.500 0.944 0.529 1.000 0.529 1.000
9 0.515 0.944 0.545 1.000 0.515 0.944
10 0.531 0.944 0.562 1.000 0.500 0.889
11 0.548 0.944 0.548 0.944 0.484 0.833
12 0.567 0.944 0.567 0.944 0.500 0.833
13 0.586 0.944 0.586 0.944 0.517 0.833
14 0.607 0.944 0.571 0.889 0.536 0.833
15 0.630 0.944 0.593 0.889 0.519 0.778
16 0.615 0.889 0.615 0.889 0.500 0.722
17 0.640 0.889 0.640 0.889 0.520 0.722
18 0.667 0.889 0.625 0.833 0.500 0.667
19 0.696 0.889 0.652 0.833 0.522 0.667
20 0.682 0.833 0.636 0.778 0.545 0.667
21 0.714 0.833 0.667 0.778 0.571 0.667
22 0.750 0.833 0.650 0.722 0.600 0.667
23 0.789 0.833 0.684 0.722 0.632 0.667
24 0.833 0.833 0.667 0.667 0.611 0.611
25 0.824 0.778 0.647 0.611 0.588 0.556
26 0.812 0.722 0.688 0.611 0.562 0.500
27 0.867 0.722 0.733 0.611 0.533 0.444
28 0.929 0.722 0.786 0.611 0.500 0.389
29 0.923 0.667 0.846 0.611 0.462 0.333
30 0.917 0.611 0.833 0.556 0.417 0.278
31 1.000 0.611 0.909 0.556 0.455 0.278
32 1.000 0.556 1.000 0.556 0.500 0.278
33 1.000 0.500 1.000 0.500 0.556 0.278
34 1.000 0.444 1.000 0.444 0.625 0.278
35 1.000 0.389 1.000 0.389 0.714 0.278
36 1.000 0.333 1.000 0.333 0.667 0.222
37 1.000 0.278 1.000 0.278 0.600 0.167
38 1.000 0.222 1.000 0.222 0.500 0.111
39 1.000 0.167 1.000 0.167 0.333 0.056
40 1.000 0.111 1.000 0.111 0.000 0.000
41 1.000 0.056 1.000 0.056 0.000 0.000
42 1.000 0.000 1.000 0.000 1.000 0.000

Train data plots#

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)

ax = plot_split_prc(results_ald_full.train, results_ald_full.name, ax)
ax = plot_split_prc(results_model_full.train, results_model_full.name, ax)
ax = plot_split_prc(results_model_new.train, results_model_new.name, ax)
fname = folder = args.out_folder / 'prec_recall_curve_train.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/prec_recall_curve_train.pdf
../../../_images/6d0e8e25a59a49d9498219a2badf99c125fc3171a20b399e35e6bd7c6fe3a442.png

Hide code cell source

 
fig, ax = plt.subplots(1, 1, figsize=figsize)
plot_split_auc(results_ald_full.train, results_ald_full.name, ax)
plot_split_auc(results_model_full.train, results_model_full.name, ax)
plot_split_auc(results_model_new.train, results_model_new.name, ax)
fname = folder = args.out_folder / 'auc_roc_curve_train.pdf'
files_out[fname.name] = fname
pimmslearn.savefig(fig, name=fname)

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/auc_roc_curve_train.pdf
../../../_images/f92b66a9311fe06db921588a2deff0a7ff3ea605df4c4c3c4cbab39ea5ffc51a.png

Output files:

Hide code cell source

 
files_out

{'results_VAE all.pkl': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/results_VAE all.pkl'),
 'results_VAE new.pkl': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/results_VAE new.pkl'),
 'results_ALD study all.pkl': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/results_ALD study all.pkl'),
 'auc_roc_curve.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/auc_roc_curve.pdf'),
 'mrmr_feat_by_model.xlsx': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/mrmr_feat_by_model.xlsx'),
 'prec_recall_curve.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/prec_recall_curve.pdf'),
 'prec_recall_curve_train.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/prec_recall_curve_train.pdf'),
 'auc_roc_curve_train.pdf': PosixPath('runs/alzheimer_study/diff_analysis/AD/PI_vs_VAE/auc_roc_curve_train.pdf')}