Variational Autoencoder

Variational Autoencoder#

Hide code cell source

 
import logging
from functools import partial

import pandas as pd
import sklearn
import torch
from fastai import learner
from fastai.basics import *
from fastai.callback.all import *
from fastai.callback.all import EarlyStoppingCallback
from fastai.learner import Learner
from fastai.torch_basics import *
from IPython.display import display
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from torch.nn import Sigmoid

import pimmslearn
import pimmslearn.model
import pimmslearn.models as models
import pimmslearn.nb
from pimmslearn.analyzers import analyzers
from pimmslearn.io import datasplits
# overwriting Recorder callback with custom plot_loss
from pimmslearn.models import ae, plot_loss

learner.Recorder.plot_loss = plot_loss


logger = pimmslearn.logging.setup_logger(logging.getLogger('pimmslearn'))
logger.info(
    "Experiment 03 - Analysis of latent spaces and performance comparisions")

figures = {}  # collection of ax or figures

pimmslearn - INFO     Experiment 03 - Analysis of latent spaces and performance comparisions

Hide code cell source

 
# catch passed parameters
args = None
args = dict(globals()).keys()

Papermill script parameters:

# files and folders
# Datasplit folder with data for experiment
folder_experiment: str = 'runs/example'
folder_data: str = ''  # specify data directory if needed
file_format: str = 'csv'  # file format of create splits, default pickle (pkl)
# Machine parsed metadata from rawfile workflow
fn_rawfile_metadata: str = 'data/dev_datasets/HeLa_6070/files_selected_metadata_N50.csv'
# training
epochs_max: int = 50  # Maximum number of epochs
batch_size: int = 64  # Batch size for training (and evaluation)
cuda: bool = True  # Whether to use a GPU for training
# model
# Dimensionality of encoding dimension (latent space of model)
latent_dim: int = 25
# A underscore separated string of layers, '256_128' for the encoder, reverse will be use for decoder
hidden_layers: str = '256_128'
# force_train:bool = True # Force training when saved model could be used. Per default re-train model
patience: int = 50  # Patience for early stopping
sample_idx_position: int = 0  # position of index which is sample ID
model: str = 'VAE'  # model name
model_key: str = 'VAE'  # potentially alternative key for model (grid search)
save_pred_real_na: bool = True  # Save all predictions for missing values
# metadata -> defaults for metadata extracted from machine data
meta_date_col: str = None  # date column in meta data
meta_cat_col: str = None  # category column in meta data

# Parameters
model = "VAE"
latent_dim = 10
batch_size = 64
epochs_max = 300
hidden_layers = "64"
sample_idx_position = 0
cuda = False
save_pred_real_na = True
fn_rawfile_metadata = "https://raw.githubusercontent.com/RasmussenLab/njab/HEAD/docs/tutorial/data/alzheimer/meta.csv"
folder_experiment = "runs/alzheimer_study"
model_key = "VAE"

Some argument transformations

Hide code cell source

 
args = pimmslearn.nb.get_params(args, globals=globals())
args

{'folder_experiment': 'runs/alzheimer_study',
 'folder_data': '',
 'file_format': 'csv',
 'fn_rawfile_metadata': 'https://raw.githubusercontent.com/RasmussenLab/njab/HEAD/docs/tutorial/data/alzheimer/meta.csv',
 'epochs_max': 300,
 'batch_size': 64,
 'cuda': False,
 'latent_dim': 10,
 'hidden_layers': '64',
 'patience': 50,
 'sample_idx_position': 0,
 'model': 'VAE',
 'model_key': 'VAE',
 'save_pred_real_na': True,
 'meta_date_col': None,
 'meta_cat_col': None}

Hide code cell source

 
args = pimmslearn.nb.args_from_dict(args)

if isinstance(args.hidden_layers, str):
    args.overwrite_entry("hidden_layers", [int(x)
                         for x in args.hidden_layers.split('_')])
else:
    raise ValueError(
        f"hidden_layers is of unknown type {type(args.hidden_layers)}")
args

{'batch_size': 64,
 'cuda': False,
 'data': Path('runs/alzheimer_study/data'),
 'epochs_max': 300,
 'file_format': 'csv',
 'fn_rawfile_metadata': 'https://raw.githubusercontent.com/RasmussenLab/njab/HEAD/docs/tutorial/data/alzheimer/meta.csv',
 'folder_data': '',
 'folder_experiment': Path('runs/alzheimer_study'),
 'hidden_layers': [64],
 'latent_dim': 10,
 'meta_cat_col': None,
 'meta_date_col': None,
 'model': 'VAE',
 'model_key': 'VAE',
 'out_figures': Path('runs/alzheimer_study/figures'),
 'out_folder': Path('runs/alzheimer_study'),
 'out_metrics': Path('runs/alzheimer_study'),
 'out_models': Path('runs/alzheimer_study'),
 'out_preds': Path('runs/alzheimer_study/preds'),
 'patience': 50,
 'sample_idx_position': 0,
 'save_pred_real_na': True}

Some naming conventions

Hide code cell source

 
TEMPLATE_MODEL_PARAMS = 'model_params_{}.json'

Load data in long format#

Hide code cell source

 
data = datasplits.DataSplits.from_folder(
    args.data, file_format=args.file_format)

pimmslearn.io.datasplits - INFO     Loaded 'train_X' from file: runs/alzheimer_study/data/train_X.csv
pimmslearn.io.datasplits - INFO     Loaded 'val_y' from file: runs/alzheimer_study/data/val_y.csv
pimmslearn.io.datasplits - INFO     Loaded 'test_y' from file: runs/alzheimer_study/data/test_y.csv

data is loaded in long format

Hide code cell source

 
data.train_X.sample(5)

Sample ID   protein groups          
Sample_111  O43291                     18.804
Sample_063  H7C2F2                     20.415
Sample_060  P35241;P35241-4;P35241-5   15.784
Sample_022  Q96S96                     20.158
Sample_047  J3KNP4;Q9NPR2;Q9NPR2-2     16.921
Name: intensity, dtype: float64

Infer index names from long format

Hide code cell source

 
index_columns = list(data.train_X.index.names)
sample_id = index_columns.pop(args.sample_idx_position)
if len(index_columns) == 1:
    index_column = index_columns.pop()
    index_columns = None
    logger.info(f"{sample_id = }, single feature: {index_column = }")
else:
    logger.info(f"{sample_id = }, multiple features: {index_columns = }")

if not index_columns:
    index_columns = [sample_id, index_column]
else:
    raise NotImplementedError(
        "More than one feature: Needs to be implemented. see above logging output.")

pimmslearn - INFO     sample_id = 'Sample ID', single feature: index_column = 'protein groups'

load meta data for splits

Hide code cell source

 
if args.fn_rawfile_metadata:
    df_meta = pd.read_csv(args.fn_rawfile_metadata, index_col=0)
    display(df_meta.loc[data.train_X.index.levels[0]])
else:
    df_meta = None
_collection site _age at CSF collection _gender _t-tau [ng/L] _p-tau [ng/L] _Abeta-42 [ng/L] _Abeta-40 [ng/L] _Abeta-42/Abeta-40 ratio _primary biochemical AD classification _clinical AD diagnosis _MMSE score
Sample ID
Sample_000 Sweden 71.000 f 703.000 85.000 562.000 NaN NaN biochemical control NaN NaN
Sample_001 Sweden 77.000 m 518.000 91.000 334.000 NaN NaN biochemical AD NaN NaN
Sample_002 Sweden 75.000 m 974.000 87.000 515.000 NaN NaN biochemical AD NaN NaN
Sample_003 Sweden 72.000 f 950.000 109.000 394.000 NaN NaN biochemical AD NaN NaN
Sample_004 Sweden 63.000 f 873.000 88.000 234.000 NaN NaN biochemical AD NaN NaN
... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 Berlin 69.000 f 1,945.000 NaN 699.000 12,140.000 0.058 biochemical AD AD 17.000
Sample_206 Berlin 73.000 m 299.000 NaN 1,420.000 16,571.000 0.086 biochemical control non-AD 28.000
Sample_207 Berlin 71.000 f 262.000 NaN 639.000 9,663.000 0.066 biochemical control non-AD 28.000
Sample_208 Berlin 83.000 m 289.000 NaN 1,436.000 11,285.000 0.127 biochemical control non-AD 24.000
Sample_209 Berlin 63.000 f 591.000 NaN 1,299.000 11,232.000 0.116 biochemical control non-AD 29.000

210 rows × 11 columns

Initialize Comparison#

  • replicates idea for truely missing values: Define truth as by using n=3 replicates to impute each sample

  • real test data:

    • Not used for predictions or early stopping.

    • [x] add some additional NAs based on distribution of data

Hide code cell source

 
freq_feat = pimmslearn.io.datasplits.load_freq(args.data)
freq_feat.head()  # training data

protein groups
A0A024QZX5;A0A087X1N8;P35237                                                     197
A0A024R0T9;K7ER74;P02655                                                         208
A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8   185
A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503                                          208
A0A075B6H7                                                                        97
Name: freq, dtype: int64

Produce some addional simulated samples#

The validation simulated NA is used to by all models to evaluate training performance.

Hide code cell source

 
val_pred_simulated_na = data.val_y.to_frame(name='observed')
val_pred_simulated_na
observed
Sample ID protein groups
Sample_158 Q9UN70;Q9UN70-2 14.630
Sample_050 Q9Y287 15.755
Sample_107 Q8N475;Q8N475-2 15.029
Sample_199 P06307 19.376
Sample_067 Q5VUB5 15.309
... ... ...
Sample_111 F6SYF8;Q9UBP4 22.822
Sample_002 A0A0A0MT36 18.165
Sample_049 Q8WY21;Q8WY21-2;Q8WY21-3;Q8WY21-4 15.525
Sample_182 Q8NFT8 14.379
Sample_123 Q16853;Q16853-2 14.504

12600 rows × 1 columns

Hide code cell source

 
test_pred_simulated_na = data.test_y.to_frame(name='observed')
test_pred_simulated_na.describe()
observed
count 12,600.000
mean 16.339
std 2.741
min 7.209
25% 14.412
50% 15.935
75% 17.910
max 30.140

Data in wide format#

  • Autoencoder need data in wide format

Hide code cell source

 
data.to_wide_format()
args.M = data.train_X.shape[-1]
data.train_X.head()
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H7 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 15.912 16.852 15.570 16.481 17.301 20.246 16.764 17.584 16.988 20.054 ... 16.012 15.178 NaN 15.050 16.842 NaN NaN 19.563 NaN 12.805
Sample_001 NaN 16.874 15.519 16.387 NaN 19.941 18.786 17.144 NaN 19.067 ... 15.528 15.576 NaN 14.833 16.597 20.299 15.556 19.386 13.970 12.442
Sample_002 16.111 NaN 15.935 16.416 18.175 19.251 16.832 15.671 17.012 18.569 ... 15.229 14.728 13.757 15.118 17.440 19.598 15.735 20.447 12.636 12.505
Sample_003 16.107 17.032 15.802 16.979 15.963 19.628 17.852 18.877 14.182 18.985 ... 15.495 14.590 14.682 15.140 17.356 19.429 NaN 20.216 NaN 12.445
Sample_004 15.603 15.331 15.375 16.679 NaN 20.450 18.682 17.081 14.140 19.686 ... 14.757 NaN NaN 15.256 17.075 19.582 15.328 NaN 13.145 NaN

5 rows × 1421 columns

Add interpolation performance#

Fill Validation data with potentially missing features#

Hide code cell source

 
data.train_X
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H7 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 15.912 16.852 15.570 16.481 17.301 20.246 16.764 17.584 16.988 20.054 ... 16.012 15.178 NaN 15.050 16.842 NaN NaN 19.563 NaN 12.805
Sample_001 NaN 16.874 15.519 16.387 NaN 19.941 18.786 17.144 NaN 19.067 ... 15.528 15.576 NaN 14.833 16.597 20.299 15.556 19.386 13.970 12.442
Sample_002 16.111 NaN 15.935 16.416 18.175 19.251 16.832 15.671 17.012 18.569 ... 15.229 14.728 13.757 15.118 17.440 19.598 15.735 20.447 12.636 12.505
Sample_003 16.107 17.032 15.802 16.979 15.963 19.628 17.852 18.877 14.182 18.985 ... 15.495 14.590 14.682 15.140 17.356 19.429 NaN 20.216 NaN 12.445
Sample_004 15.603 15.331 15.375 16.679 NaN 20.450 18.682 17.081 14.140 19.686 ... 14.757 NaN NaN 15.256 17.075 19.582 15.328 NaN 13.145 NaN
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 15.682 16.886 14.910 16.482 NaN 17.705 17.039 NaN 16.413 19.102 ... NaN 15.684 14.236 15.415 17.551 17.922 16.340 19.928 12.929 NaN
Sample_206 15.798 17.554 15.600 15.938 NaN 18.154 18.152 16.503 16.860 18.538 ... 15.422 16.106 NaN 15.345 17.084 18.708 NaN 19.433 NaN NaN
Sample_207 15.739 NaN 15.469 16.898 NaN 18.636 17.950 16.321 16.401 18.849 ... 15.808 16.098 14.403 15.715 NaN 18.725 16.138 19.599 13.637 11.174
Sample_208 15.477 16.779 14.995 16.132 NaN 14.908 NaN NaN 16.119 18.368 ... 15.157 16.712 NaN 14.640 16.533 19.411 15.807 19.545 NaN NaN
Sample_209 NaN 17.261 15.175 16.235 NaN 17.893 17.744 16.371 15.780 18.806 ... 15.237 15.652 15.211 14.205 16.749 19.275 15.732 19.577 11.042 11.791

210 rows × 1421 columns

Hide code cell source

 
data.val_y  # potentially has less features
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H7 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN 19.863 NaN NaN NaN NaN
Sample_001 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_002 NaN 14.523 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_003 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_004 NaN NaN NaN NaN 15.473 NaN NaN NaN NaN NaN ... NaN NaN 14.048 NaN NaN NaN NaN 19.867 NaN 12.235
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN 11.802
Sample_206 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_207 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_208 NaN NaN NaN NaN NaN NaN 17.530 NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_209 15.727 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN

210 rows × 1419 columns

Hide code cell source

 
data.val_y = pd.DataFrame(pd.NA, index=data.train_X.index,
                          columns=data.train_X.columns).fillna(data.val_y)
data.val_y
protein groups A0A024QZX5;A0A087X1N8;P35237 A0A024R0T9;K7ER74;P02655 A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 A0A075B6H7 A0A075B6H9 A0A075B6I0 A0A075B6I1 A0A075B6I6 A0A075B6I9 ... Q9Y653;Q9Y653-2;Q9Y653-3 Q9Y696 Q9Y6C2 Q9Y6N6 Q9Y6N7;Q9Y6N7-2;Q9Y6N7-4 Q9Y6R7 Q9Y6X5 Q9Y6Y8;Q9Y6Y8-2 Q9Y6Y9 S4R3U6
Sample ID
Sample_000 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN 19.863 NaN NaN NaN NaN
Sample_001 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_002 NaN 14.523 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_003 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_004 NaN NaN NaN NaN 15.473 NaN NaN NaN NaN NaN ... NaN NaN 14.048 NaN NaN NaN NaN 19.867 NaN 12.235
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
Sample_205 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN 11.802
Sample_206 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_207 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_208 NaN NaN NaN NaN NaN NaN 17.530 NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
Sample_209 15.727 NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN

210 rows × 1421 columns

Variational Autoencoder#

Analysis: DataLoaders, Model, transform#

Hide code cell source

 
default_pipeline = sklearn.pipeline.Pipeline(
    [
        ('normalize', StandardScaler()),
        ('impute', SimpleImputer(add_indicator=False))
    ])

Analysis: DataLoaders, Model#

Hide code cell source

 
analysis = ae.AutoEncoderAnalysis(  # datasplits=data,
    train_df=data.train_X,
    val_df=data.val_y,
    model=models.vae.VAE,
    model_kwargs=dict(n_features=data.train_X.shape[-1],
                      n_neurons=args.hidden_layers,
                      # last_encoder_activation=None,
                      last_decoder_activation=None,
                      dim_latent=args.latent_dim),
    transform=default_pipeline,
    decode=['normalize'],
    bs=args.batch_size)
args.n_params = analysis.n_params_ae
if args.cuda:
    analysis.model = analysis.model.cuda()
analysis.model

VAE(
  (encoder): Sequential(
    (0): Linear(in_features=1421, out_features=64, bias=True)
    (1): Dropout(p=0.2, inplace=False)
    (2): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (3): LeakyReLU(negative_slope=0.1)
    (4): Linear(in_features=64, out_features=20, bias=True)
  )
  (decoder): Sequential(
    (0): Linear(in_features=10, out_features=64, bias=True)
    (1): Dropout(p=0.2, inplace=False)
    (2): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (3): LeakyReLU(negative_slope=0.1)
    (4): Linear(in_features=64, out_features=2842, bias=True)
  )
)

Training#

Hide code cell source

 
results = []
loss_fct = partial(models.vae.loss_fct, results=results)

Hide code cell source

 
analysis.learn = Learner(dls=analysis.dls,
                         model=analysis.model,
                         loss_func=loss_fct,
                         cbs=[ae.ModelAdapterVAE(),
                              EarlyStoppingCallback(patience=args.patience)
                              ])

analysis.learn.show_training_loop()

Start Fit
   - before_fit     : [TrainEvalCallback, Recorder, ProgressCallback, EarlyStoppingCallback]
  Start Epoch Loop
     - before_epoch   : [Recorder, ProgressCallback]
    Start Train
       - before_train   : [TrainEvalCallback, Recorder, ProgressCallback]
      Start Batch Loop
         - before_batch   : [ModelAdapterVAE, CastToTensor]
         - after_pred     : [ModelAdapterVAE]
         - after_loss     : [ModelAdapterVAE]
         - before_backward: []
         - before_step    : []
         - after_step     : []
         - after_cancel_batch: []
         - after_batch    : [TrainEvalCallback, Recorder, ProgressCallback]
      End Batch Loop
    End Train
     - after_cancel_train: [Recorder]
     - after_train    : [Recorder, ProgressCallback]
    Start Valid
       - before_validate: [TrainEvalCallback, Recorder, ProgressCallback]
      Start Batch Loop
         - **CBs same as train batch**: []
      End Batch Loop
    End Valid
     - after_cancel_validate: [Recorder]
     - after_validate : [Recorder, ProgressCallback]
  End Epoch Loop
   - after_cancel_epoch: []
   - after_epoch    : [Recorder, EarlyStoppingCallback]
End Fit
 - after_cancel_fit: []
 - after_fit      : [ProgressCallback, EarlyStoppingCallback]

Adding a EarlyStoppingCallback results in an error. Potential fix in PR3509 is not yet in current version. Try again later

Hide code cell source

 
# learn.summary()

Hide code cell source

 
suggested_lr = analysis.learn.lr_find()
analysis.params['suggested_inital_lr'] = suggested_lr.valley
suggested_lr

SuggestedLRs(valley=0.0030199517495930195)
_images/c62ee516a0b15f9a822cffb46cc70b2e9683c1253ddfe2b73e3bf677f12f0b18.png

Hide code cell source

 
results.clear()  # reset results

dump model config

Hide code cell source

 
# needs class as argument, not instance, but serialization needs instance
analysis.params['last_decoder_activation'] = Sigmoid()

pimmslearn.io.dump_json(
    pimmslearn.io.parse_dict(
        analysis.params, types=[
            (torch.nn.modules.module.Module, lambda m: str(m))
        ]),
    args.out_models / TEMPLATE_MODEL_PARAMS.format(args.model_key))

# restore original value
analysis.params['last_decoder_activation'] = Sigmoid

Hide code cell source

 
# papermill_description=train
analysis.learn.fit_one_cycle(args.epochs_max, lr_max=suggested_lr.valley)
epoch train_loss valid_loss time
0 1701.175293 93.328133 00:00
1 1690.376343 93.924973 00:00
2 1687.467407 94.548965 00:00
3 1683.436401 94.405281 00:00
4 1679.548096 94.293472 00:00
5 1678.122437 95.086693 00:00
6 1673.897827 95.175476 00:00
7 1672.194824 94.896873 00:00
8 1668.538574 94.623993 00:00
9 1666.478149 95.353432 00:00
10 1663.490356 94.830406 00:00
11 1660.132446 94.572517 00:00
12 1656.048096 94.789543 00:00
13 1652.768311 95.245514 00:00
14 1650.696899 94.491600 00:00
15 1646.301880 94.475868 00:00
16 1641.806641 94.609879 00:00
17 1637.537354 94.569901 00:00
18 1632.284058 94.738457 00:00
19 1626.488403 94.096031 00:00
20 1619.968750 93.563400 00:00
21 1613.468018 93.610352 00:00
22 1606.483521 93.037575 00:00
23 1598.677856 93.466507 00:00
24 1590.847534 92.849319 00:00
25 1582.922729 93.007713 00:00
26 1573.967773 92.723648 00:00
27 1565.328369 93.070068 00:00
28 1555.866943 92.895668 00:00
29 1545.809692 93.230072 00:00
30 1535.542358 93.129463 00:00
31 1524.850708 93.292076 00:00
32 1514.004272 93.547432 00:00
33 1503.508423 93.532066 00:00
34 1492.338867 93.213402 00:00
35 1480.102417 92.769699 00:00
36 1469.326782 92.987892 00:00
37 1459.264038 93.491920 00:00
38 1447.944336 94.016022 00:00
39 1437.075928 94.541069 00:00
40 1426.525146 94.324005 00:00
41 1414.924805 94.235168 00:00
42 1404.228149 94.401466 00:00
43 1396.221558 94.494942 00:00
44 1385.866577 94.149971 00:00
45 1376.761475 93.747124 00:00
46 1367.018066 93.549683 00:00
47 1358.119019 93.837875 00:00
48 1350.043457 93.959671 00:00
49 1340.775757 93.198608 00:00
50 1331.328857 92.392624 00:00
51 1324.231567 92.478699 00:00
52 1316.158447 92.427269 00:00
53 1307.685791 92.033554 00:00
54 1299.277954 92.150909 00:00
55 1290.935303 92.289749 00:00
56 1283.127075 91.753387 00:00
57 1276.848022 91.855843 00:00
58 1270.368042 91.506409 00:00
59 1263.855469 91.712555 00:00
60 1257.921021 91.767616 00:00
61 1251.389404 91.834969 00:00
62 1244.571533 91.916328 00:00
63 1238.156494 91.858940 00:00
64 1232.901001 91.325676 00:00
65 1228.177734 92.128067 00:00
66 1223.536133 92.071602 00:00
67 1216.561890 91.795822 00:00
68 1211.360962 92.003090 00:00
69 1204.472290 91.842010 00:00
70 1198.730957 91.755898 00:00
71 1194.042480 92.395157 00:00
72 1190.488525 92.347099 00:00
73 1185.557739 91.414536 00:00
74 1180.156250 91.902924 00:00
75 1176.907471 92.528923 00:00
76 1173.411621 91.907028 00:00
77 1169.287598 92.070450 00:00
78 1164.298218 91.820183 00:00
79 1160.719482 91.722221 00:00
80 1156.317261 91.674156 00:00
81 1152.143921 91.492653 00:00
82 1149.084229 91.469864 00:00
83 1147.286743 91.308762 00:00
84 1144.587280 91.408875 00:00
85 1140.294434 91.784225 00:00
86 1139.907959 91.708038 00:00
87 1136.847778 91.758270 00:00
88 1134.164185 91.563469 00:00
89 1130.312500 91.044243 00:00
90 1129.452148 90.432632 00:00
91 1125.999756 90.150612 00:00
92 1122.544556 90.797729 00:00
93 1119.757446 91.018127 00:00
94 1116.208618 91.975914 00:00
95 1113.968628 92.612335 00:00
96 1110.997559 92.026329 00:00
97 1110.015259 92.163612 00:00
98 1108.436523 92.210213 00:00
99 1105.441528 92.199738 00:00
100 1104.292969 91.926987 00:00
101 1101.078735 92.005592 00:00
102 1099.702026 91.923904 00:00
103 1099.697876 92.695145 00:00
104 1100.849854 92.279625 00:00
105 1099.303345 91.310921 00:00
106 1096.897705 90.907890 00:00
107 1096.534668 91.057747 00:00
108 1093.661865 90.959747 00:00
109 1092.260498 90.994858 00:00
110 1090.592529 91.552483 00:00
111 1088.287476 91.709740 00:00
112 1085.657471 92.195526 00:00
113 1084.306641 92.396149 00:00
114 1081.707275 92.016853 00:00
115 1080.371582 91.974854 00:00
116 1078.323120 91.693314 00:00
117 1077.419678 92.124664 00:00
118 1074.969849 91.791069 00:00
119 1072.675049 91.589417 00:00
120 1072.918701 91.491241 00:00
121 1072.260132 91.481476 00:00
122 1070.536499 91.428894 00:00
123 1070.626221 91.515533 00:00
124 1068.718750 92.155899 00:00
125 1065.842407 92.073799 00:00
126 1063.175659 92.558609 00:00
127 1061.248047 92.314072 00:00
128 1059.534058 92.128555 00:00
129 1058.202148 92.143623 00:00
130 1058.327148 92.285332 00:00
131 1057.238159 91.942284 00:00
132 1057.599976 91.844208 00:00
133 1057.063843 92.063644 00:00
134 1056.683716 92.062706 00:00
135 1056.043335 92.395218 00:00
136 1055.895264 92.039963 00:00
137 1055.122681 92.357513 00:00
138 1053.537109 92.525764 00:00
139 1052.039062 92.744690 00:00
140 1051.083862 92.581940 00:00
141 1051.428833 92.452278 00:00 
No improvement since epoch 91: early stopping

Save number of actually trained epochs

Hide code cell source

 
args.epoch_trained = analysis.learn.epoch + 1
args.epoch_trained

142

Loss normalized by total number of measurements#

Hide code cell source

 
N_train_notna = data.train_X.notna().sum().sum()
N_val_notna = data.val_y.notna().sum().sum()
fig = models.plot_training_losses(analysis.learn, args.model_key,
                                  folder=args.out_figures,
                                  norm_factors=[N_train_notna, N_val_notna])

pimmslearn.plotting - INFO     Saved Figures to runs/alzheimer_study/figures/vae_training
_images/425037c689a0da82526fdf827a6b0d38a5d91c1c0ab50bc51633a2baf3a5cd21.png

Predictions#

create predictions and select validation data predictions

Hide code cell source

 
analysis.model.eval()
pred, target = res = ae.get_preds_from_df(df=data.train_X, learn=analysis.learn,
                                          position_pred_tuple=0,
                                          transformer=analysis.transform)
pred = pred.stack()
pred

Sample ID   protein groups                                                                
Sample_000  A0A024QZX5;A0A087X1N8;P35237                                                     15.975
            A0A024R0T9;K7ER74;P02655                                                         16.667
            A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8   15.853
            A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503                                          16.658
            A0A075B6H7                                                                       17.277
                                                                                              ...  
Sample_209  Q9Y6R7                                                                           19.189
            Q9Y6X5                                                                           15.555
            Q9Y6Y8;Q9Y6Y8-2                                                                  19.277
            Q9Y6Y9                                                                           11.740
            S4R3U6                                                                           11.585
Length: 298410, dtype: float32

Hide code cell source

 
val_pred_simulated_na['VAE'] = pred  # 'model_key' ?
val_pred_simulated_na
observed VAE
Sample ID protein groups
Sample_158 Q9UN70;Q9UN70-2 14.630 15.728
Sample_050 Q9Y287 15.755 16.880
Sample_107 Q8N475;Q8N475-2 15.029 14.722
Sample_199 P06307 19.376 19.024
Sample_067 Q5VUB5 15.309 15.009
... ... ... ...
Sample_111 F6SYF8;Q9UBP4 22.822 22.927
Sample_002 A0A0A0MT36 18.165 16.110
Sample_049 Q8WY21;Q8WY21-2;Q8WY21-3;Q8WY21-4 15.525 15.718
Sample_182 Q8NFT8 14.379 13.400
Sample_123 Q16853;Q16853-2 14.504 14.481

12600 rows × 2 columns

Hide code cell source

 
test_pred_simulated_na['VAE'] = pred  # model_key?
test_pred_simulated_na
observed VAE
Sample ID protein groups
Sample_000 A0A075B6P5;P01615 17.016 17.206
A0A087X089;Q16627;Q16627-2 18.280 17.919
A0A0B4J2B5;S4R460 21.735 22.218
A0A140T971;O95865;Q5SRR8;Q5SSV3 14.603 15.279
A0A140TA33;A0A140TA41;A0A140TA52;P22105;P22105-3;P22105-4 16.143 16.684
... ... ... ...
Sample_209 Q96ID5 16.074 16.093
Q9H492;Q9H492-2 13.173 13.051
Q9HC57 14.207 14.030
Q9NPH3;Q9NPH3-2;Q9NPH3-5 14.962 15.202
Q9UGM5;Q9UGM5-2 16.871 16.396

12600 rows × 2 columns

save missing values predictions

Hide code cell source

 
if args.save_pred_real_na:
    pred_real_na = ae.get_missing_values(df_train_wide=data.train_X,
                                         val_idx=val_pred_simulated_na.index,
                                         test_idx=test_pred_simulated_na.index,
                                         pred=pred)
    display(pred_real_na)
    pred_real_na.to_csv(args.out_preds / f"pred_real_na_{args.model_key}.csv")

Sample ID   protein groups          
Sample_000  A0A075B6J9                 15.513
            A0A075B6Q5                 16.136
            A0A075B6R2                 16.669
            A0A075B6S5                 16.126
            A0A087WSY4                 16.227
                                        ...  
Sample_209  Q9P1W8;Q9P1W8-2;Q9P1W8-4   16.046
            Q9UI40;Q9UI40-2            16.062
            Q9UIW2                     16.785
            Q9UMX0;Q9UMX0-2;Q9UMX0-4   13.881
            Q9UP79                     16.019
Name: intensity, Length: 46401, dtype: float32

Plots#

  • validation data

Hide code cell source

 
analysis.model = analysis.model.cpu()
# underlying data is train_X for both
# assert analysis.dls.valid.data.equals(analysis.dls.train.data)
# Reconstruct DataLoader for case that during training singleton batches were dropped
_dl = torch.utils.data.DataLoader(
    pimmslearn.io.datasets.DatasetWithTarget(
        analysis.dls.valid.data),
    batch_size=args.batch_size,
    shuffle=False)
df_latent = pimmslearn.model.get_latent_space(analysis.model.get_mu_and_logvar,
                                              dl=_dl,
                                              dl_index=analysis.dls.valid.data.index)
df_latent
latent dimension 1 latent dimension 2 latent dimension 3 latent dimension 4 latent dimension 5 latent dimension 6 latent dimension 7 latent dimension 8 latent dimension 9 latent dimension 10
Sample ID
Sample_000 1.348 0.102 -0.398 -0.825 0.492 1.611 1.778 0.008 2.030 -1.281
Sample_001 0.080 -0.205 0.804 -1.505 0.183 1.130 0.888 0.140 2.717 -0.640
Sample_002 2.700 -0.830 -0.329 -0.738 -0.442 -0.057 1.140 0.398 1.662 1.159
Sample_003 2.074 0.244 -0.481 -1.411 -0.188 0.478 1.367 0.455 1.639 -0.587
Sample_004 1.151 -0.149 0.305 -0.938 -0.220 0.074 1.505 0.015 2.270 -1.392
... ... ... ... ... ... ... ... ... ... ...
Sample_205 1.715 -0.781 -0.721 -1.075 1.098 -0.163 -1.194 -1.537 0.740 -0.242
Sample_206 -0.405 -1.891 -1.175 1.607 1.171 2.035 -1.266 -0.285 1.390 0.203
Sample_207 0.366 -1.235 -0.338 -0.018 -0.520 1.123 -0.348 -1.819 0.802 -2.922
Sample_208 -0.968 -2.778 -0.215 -1.215 0.081 0.763 -0.413 -0.886 0.896 0.248
Sample_209 -0.114 -2.281 -0.848 -0.278 -1.964 -0.075 -0.596 -0.197 0.732 0.654

210 rows × 10 columns

Hide code cell source

 
ana_latent = analyzers.LatentAnalysis(df_latent,
                                      df_meta,
                                      args.model_key,
                                      folder=args.out_figures)
if args.meta_date_col and df_meta is not None:
    figures[f'latent_{args.model_key}_by_date'], ax = ana_latent.plot_by_date(
        args.meta_date_col)

Hide code cell source

 
if args.meta_cat_col and df_meta is not None:
    figures[f'latent_{args.model_key}_by_{"_".join(args.meta_cat_col.split())}'], ax = ana_latent.plot_by_category(
        args.meta_cat_col)

Hide code cell source

 
feat_freq_val = val_pred_simulated_na['observed'].groupby(level=-1).count()
feat_freq_val.name = 'freq_val'
ax = feat_freq_val.plot.box()
_images/68bed08d665702d1c44d3abbb18ec078524e649f113fb48c023babd4a133cb1c.png

Hide code cell source

 
feat_freq_val.value_counts().sort_index().head()  # require more than one feat?

freq_val
1    12
2    18
3    50
4    82
5   108
Name: count, dtype: int64

Hide code cell source

 
errors_val = val_pred_simulated_na.drop('observed', axis=1).sub(
    val_pred_simulated_na['observed'], axis=0)
errors_val = errors_val.abs().groupby(level=-1).mean()
errors_val = errors_val.join(freq_feat).sort_values(by='freq', ascending=True)


errors_val_smoothed = errors_val.copy()  # .loc[feat_freq_val > 1]
errors_val_smoothed[errors_val.columns[:-1]] = errors_val[errors_val.columns[:-1]
                                                          ].rolling(window=200, min_periods=1).mean()
ax = errors_val_smoothed.plot(x='freq', figsize=(15, 10))
# errors_val_smoothed
_images/21ed35ec9cd4e5819d74972b4a89f6d65245f88ed307bf556c7474811ae182b3.png

Hide code cell source

 
errors_val = val_pred_simulated_na.drop('observed', axis=1).sub(
    val_pred_simulated_na['observed'], axis=0)
errors_val.abs().groupby(level=-1).agg(['mean', 'count'])
VAE
mean count
protein groups
A0A024QZX5;A0A087X1N8;P35237 0.143 7
A0A024R0T9;K7ER74;P02655 1.258 4
A0A024R3W6;A0A024R412;O60462;O60462-2;O60462-3;O60462-4;O60462-5;Q7LBX6;X5D2Q8 0.242 9
A0A024R644;A0A0A0MRU5;A0A1B0GWI2;O75503 0.241 6
A0A075B6H7 0.563 6
... ... ...
Q9Y6R7 0.373 10
Q9Y6X5 0.274 7
Q9Y6Y8;Q9Y6Y8-2 0.344 9
Q9Y6Y9 0.415 15
S4R3U6 0.474 24

1419 rows × 2 columns

Hide code cell source

 
errors_val
VAE
Sample ID protein groups
Sample_158 Q9UN70;Q9UN70-2 1.097
Sample_050 Q9Y287 1.125
Sample_107 Q8N475;Q8N475-2 -0.307
Sample_199 P06307 -0.352
Sample_067 Q5VUB5 -0.300
... ... ...
Sample_111 F6SYF8;Q9UBP4 0.105
Sample_002 A0A0A0MT36 -2.055
Sample_049 Q8WY21;Q8WY21-2;Q8WY21-3;Q8WY21-4 0.193
Sample_182 Q8NFT8 -0.978
Sample_123 Q16853;Q16853-2 -0.023

12600 rows × 1 columns

Comparisons#

Simulated NAs : Artificially created NAs. Some data was sampled and set explicitly to misssing before it was fed to the model for reconstruction.

Validation data#

  • all measured (identified, observed) peptides in validation data

Hide code cell source

 
# papermill_description=metrics
# d_metrics = models.Metrics(no_na_key='NA interpolated', with_na_key='NA not interpolated')
d_metrics = models.Metrics()

The simulated NA for the validation step are real test data (not used for training nor early stopping)

Hide code cell source

 
added_metrics = d_metrics.add_metrics(val_pred_simulated_na, 'valid_simulated_na')
added_metrics

Selected as truth to compare to: observed

{'VAE': {'MSE': 0.45432558057872835,
  'MAE': 0.42999764576512867,
  'N': 12600,
  'prop': 1.0}}

Test Datasplit#

Hide code cell source

 
added_metrics = d_metrics.add_metrics(test_pred_simulated_na, 'test_simulated_na')
added_metrics

Selected as truth to compare to: observed

{'VAE': {'MSE': 0.48234073988947856,
  'MAE': 0.437808936420482,
  'N': 12600,
  'prop': 1.0}}

Save all metrics as json

Hide code cell source

 
pimmslearn.io.dump_json(d_metrics.metrics, args.out_metrics /
                        f'metrics_{args.model_key}.json')
d_metrics

{ 'test_simulated_na': { 'VAE': { 'MAE': 0.437808936420482,
                                  'MSE': 0.48234073988947856,
                                  'N': 12600,
                                  'prop': 1.0}},
  'valid_simulated_na': { 'VAE': { 'MAE': 0.42999764576512867,
                                   'MSE': 0.45432558057872835,
                                   'N': 12600,
                                   'prop': 1.0}}}

Hide code cell source

 
metrics_df = models.get_df_from_nested_dict(
    d_metrics.metrics, column_levels=['model', 'metric_name']).T
metrics_df
subset valid_simulated_na test_simulated_na
model metric_name
VAE MSE 0.454 0.482
MAE 0.430 0.438
N 12,600.000 12,600.000
prop 1.000 1.000

Save predictions#

Hide code cell source

 
# save simulated missing values for both splits
val_pred_simulated_na.to_csv(args.out_preds / f"pred_val_{args.model_key}.csv")
test_pred_simulated_na.to_csv(args.out_preds / f"pred_test_{args.model_key}.csv")

Config#

Hide code cell source

 
figures  # switch to fnames?

{}

Hide code cell source

 
args.dump(fname=args.out_models / f"model_config_{args.model_key}.yaml")
args

{'M': 1421,
 'batch_size': 64,
 'cuda': False,
 'data': Path('runs/alzheimer_study/data'),
 'epoch_trained': 142,
 'epochs_max': 300,
 'file_format': 'csv',
 'fn_rawfile_metadata': 'https://raw.githubusercontent.com/RasmussenLab/njab/HEAD/docs/tutorial/data/alzheimer/meta.csv',
 'folder_data': '',
 'folder_experiment': Path('runs/alzheimer_study'),
 'hidden_layers': [64],
 'latent_dim': 10,
 'meta_cat_col': None,
 'meta_date_col': None,
 'model': 'VAE',
 'model_key': 'VAE',
 'n_params': 277998,
 'out_figures': Path('runs/alzheimer_study/figures'),
 'out_folder': Path('runs/alzheimer_study'),
 'out_metrics': Path('runs/alzheimer_study'),
 'out_models': Path('runs/alzheimer_study'),
 'out_preds': Path('runs/alzheimer_study/preds'),
 'patience': 50,
 'sample_idx_position': 0,
 'save_pred_real_na': True}
Contents