From df2f59f7908a1b43b2371280f338c8c519bfbe01 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Sat, 22 Jan 2022 07:40:17 +0000 Subject: [PATCH 01/21] initial commit --- InnerEye/ML/Histopathology/models/deepmil.py | 22 +++++++++++++------ InnerEye/ML/Histopathology/models/encoders.py | 7 +++++- .../aggregate_metrics_crossvalidation.py | 20 +++++++++++++---- .../classification/DeepSMILEPanda.py | 2 +- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 827d23a03..e27921735 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -14,7 +14,7 @@ from pytorch_lightning import LightningModule from torch import Tensor, argmax, mode, nn, no_grad, optim, round -from torchmetrics import AUROC, F1, Accuracy, Precision, Recall +from torchmetrics import AUROC, F1, Accuracy, Precision, Recall, ConfusionMatrix from InnerEye.Common import fixed_paths from InnerEye.ML.Histopathology.datasets.base_dataset import TilesDataset, SlidesDataset @@ -94,6 +94,11 @@ def __init__(self, self.tile_size = tile_size self.level = level + # Metrics Objects + self.train_metrics = self.get_metrics() + self.val_metrics = self.get_metrics() + self.test_metrics = self.get_metrics() + self.save_hyperparameters() self.verbose = verbose @@ -103,10 +108,6 @@ def __init__(self, self.loss_fn = self.get_loss() self.activation_fn = self.get_activation() - # Metrics Objects - self.train_metrics = self.get_metrics() - self.val_metrics = self.get_metrics() - self.test_metrics = self.get_metrics() def get_pooling(self) -> Tuple[Callable, int]: pooling_layer = self.pooling_layer(self.num_encoding, @@ -148,7 +149,8 @@ def get_metrics(self) -> nn.ModuleDict: if self.n_classes > 1: return nn.ModuleDict({'accuracy': Accuracy(num_classes=self.n_classes, average='micro'), 'macro_accuracy': Accuracy(num_classes=self.n_classes, average='macro'), - 'weighted_accuracy': Accuracy(num_classes=self.n_classes, average='weighted')}) + 'weighted_accuracy': Accuracy(num_classes=self.n_classes, average='weighted'), + 'confusion_matrix': ConfusionMatrix(num_classes=self.n_classes)}) else: return nn.ModuleDict({'accuracy': Accuracy(), 'auroc': AUROC(num_classes=self.n_classes), @@ -162,7 +164,13 @@ def log_metrics(self, if stage not in valid_stages: raise Exception(f"Invalid stage. Chose one of {valid_stages}") for metric_name, metric_object in self.get_metrics_dict(stage).items(): - self.log(f'{stage}/{metric_name}', metric_object, on_epoch=True, on_step=False, logger=True, sync_dist=True) + if metric_name == "confusion_matrix": # and stage in ['val']: + # We can't log tensors in the normal way - just print it to console + metric_value = metric_object.compute() + print(f'{stage}/{metric_name}:') + print(np.array(metric_value.cpu())) + else: + self.log(f'{stage}/{metric_name}', metric_object, on_epoch=True, on_step=False, logger=True, sync_dist=True) def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore with no_grad(): diff --git a/InnerEye/ML/Histopathology/models/encoders.py b/InnerEye/ML/Histopathology/models/encoders.py index 04f454bba..ca502e2bd 100644 --- a/InnerEye/ML/Histopathology/models/encoders.py +++ b/InnerEye/ML/Histopathology/models/encoders.py @@ -137,7 +137,12 @@ class HistoSSLEncoder(TileEncoder): WEIGHTS_URL = ("https://github.com/ozanciga/self-supervised-histopathology/releases/" "download/tenpercent/tenpercent_resnet18.ckpt") + def _get_preprocessing(self) -> Callable: + return get_imagenet_preprocessing() + def _get_encoder(self) -> Tuple[Callable, int]: resnet18_model = resnet18(pretrained=False) histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) - return setup_feature_extractor(histossl_encoder, self.input_dim) # type: ignore + histossl_encoder.fc = torch.nn.Sequential() + _, num_features = setup_feature_extractor(histossl_encoder, self.input_dim) + return histossl_encoder, num_features # type: ignore diff --git a/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py b/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py index 7c96b0349..d0d81df8d 100644 --- a/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py +++ b/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py @@ -6,11 +6,23 @@ """ Script to find mean and standard deviation of desired metrics from cross validation child runs. """ -import os +import sys, os import pandas as pd +from pathlib import Path -from health_azure import aggregate_hyperdrive_metrics, get_workspace +current_dir = Path(os.getcwd()) +radiomics_root = current_dir +if (radiomics_root / "InnerEyePrivate").is_dir(): + radiomics_root_str = str(radiomics_root) + if radiomics_root_str not in sys.path: + print(f"Adding to sys.path: {radiomics_root_str}") + sys.path.insert(0, radiomics_root_str) + sys.path.insert(0, str(radiomics_root / "innereye-deeplearning")) + sys.path.insert(0, str(radiomics_root / "innereye-deeplearning/hi-ml/hi-ml-azure/src")) + sys.path.insert(0, str(radiomics_root / "innereye-deeplearning/hi-ml/hi-ml/src")) + print(f"Sys path {sys.path}") +from health_azure import aggregate_hyperdrive_metrics, get_workspace from InnerEye.Common import fixed_paths @@ -28,8 +40,8 @@ def get_cross_validation_metrics_df(run_id: str) -> pd.DataFrame: if __name__ == "__main__": - metrics_list = ['test/accuracy', 'test/auroc', 'test/f1score', 'test/precision', 'test/recall'] - run_id = "hsharma_features_viz:HD_eff4c009-2f9f-4c2c-94c6-c0c84944a412" + metrics_list = ['test/accuracy', 'test/auroc', 'test/f1score', 'test/precision', 'test/recall', 'test/macro_accuracy', 'test/weighted_accuracy'] + run_id = "hsharma_panda_tiles_ssl:HD_b5be4968-4896-4fc4-8d62-291ebe5c57c2" metrics_df = get_cross_validation_metrics_df(run_id=run_id) for metric in metrics_list: if metric in metrics_df.index.values: diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 59a617b74..26e0476d5 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -52,7 +52,7 @@ def __init__(self, **kwargs: Any) -> None: number_of_cross_validation_splits=5, cross_validation_split_index=0, # declared in OptimizerParams: - l_rate=5e-4, + l_rate=1e-3, weight_decay=1e-4, adam_betas=(0.9, 0.99)) default_kwargs.update(kwargs) From 00d8e8df788c7a81e2f8d954de9cb2f55fe8ac08 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Sat, 22 Jan 2022 16:33:18 +0000 Subject: [PATCH 02/21] panda histosslmil investigate --- InnerEye/ML/Histopathology/models/deepmil.py | 19 ++++++++++--------- InnerEye/ML/Histopathology/models/encoders.py | 11 ++++++++--- .../ML/Histopathology/utils/download_utils.py | 2 +- .../histo_configs/classification/BaseMIL.py | 2 +- .../classification/DeepSMILEPanda.py | 2 +- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index e27921735..87edf5743 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -94,11 +94,6 @@ def __init__(self, self.tile_size = tile_size self.level = level - # Metrics Objects - self.train_metrics = self.get_metrics() - self.val_metrics = self.get_metrics() - self.test_metrics = self.get_metrics() - self.save_hyperparameters() self.verbose = verbose @@ -108,6 +103,10 @@ def __init__(self, self.loss_fn = self.get_loss() self.activation_fn = self.get_activation() + # Metrics Objects + self.train_metrics = self.get_metrics() + self.val_metrics = self.get_metrics() + self.test_metrics = self.get_metrics() def get_pooling(self) -> Tuple[Callable, int]: pooling_layer = self.pooling_layer(self.num_encoding, @@ -167,14 +166,16 @@ def log_metrics(self, if metric_name == "confusion_matrix": # and stage in ['val']: # We can't log tensors in the normal way - just print it to console metric_value = metric_object.compute() - print(f'{stage}/{metric_name}:') - print(np.array(metric_value.cpu())) + if stage == 'test': + print(f'{stage}/{metric_name}:') + print(np.array(metric_value.cpu())) else: self.log(f'{stage}/{metric_name}', metric_object, on_epoch=True, on_step=False, logger=True, sync_dist=True) def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore - with no_grad(): - H = self.encoder(images) # N X L x 1 x 1 + # with no_grad(): + print(self.encoder) + H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/Histopathology/models/encoders.py b/InnerEye/ML/Histopathology/models/encoders.py index ca502e2bd..d567407bc 100644 --- a/InnerEye/ML/Histopathology/models/encoders.py +++ b/InnerEye/ML/Histopathology/models/encoders.py @@ -137,12 +137,17 @@ class HistoSSLEncoder(TileEncoder): WEIGHTS_URL = ("https://github.com/ozanciga/self-supervised-histopathology/releases/" "download/tenpercent/tenpercent_resnet18.ckpt") - def _get_preprocessing(self) -> Callable: - return get_imagenet_preprocessing() + # def _get_preprocessing(self) -> Callable: + # return get_imagenet_preprocessing() + + # def _get_encoder(self) -> Tuple[Callable, int]: + # resnet18_model = resnet18(pretrained=False) + # histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) + # return setup_feature_extractor(histossl_encoder, self.input_dim) # type: ignore def _get_encoder(self) -> Tuple[Callable, int]: resnet18_model = resnet18(pretrained=False) histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) histossl_encoder.fc = torch.nn.Sequential() - _, num_features = setup_feature_extractor(histossl_encoder, self.input_dim) + num_features = 512 return histossl_encoder, num_features # type: ignore diff --git a/InnerEye/ML/Histopathology/utils/download_utils.py b/InnerEye/ML/Histopathology/utils/download_utils.py index 1addcbc55..862a8439d 100644 --- a/InnerEye/ML/Histopathology/utils/download_utils.py +++ b/InnerEye/ML/Histopathology/utils/download_utils.py @@ -30,7 +30,7 @@ def download_file_if_necessary(run_id: str, remote_dir: Path, download_dir: Path download_files_from_run_id(run_id=run_id, output_folder=local_dir, prefix=str(remote_path), - aml_workspace=aml_workspace, + workspace=aml_workspace, validate_checksum=True) assert local_path.exists() print("File is downloaded at", local_path) diff --git a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py index 521711b17..66ffa196a 100644 --- a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py +++ b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py @@ -35,7 +35,7 @@ class BaseMIL(LightningContainer): n_channels: int = param.Integer(3, bounds=(1, None), doc="Number of channels in the tile.") # Data module parameters: - batch_size: int = param.Integer(16, bounds=(1, None), doc="Number of slides to load per batch.") + batch_size: int = param.Integer(32, bounds=(1, None), doc="Number of slides to load per batch.") max_bag_size: int = param.Integer(1000, bounds=(0, None), doc="Upper bound on number of tiles in each loaded bag. " "If 0 (default), will return all samples in each bag. " diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 26e0476d5..bc96f4b62 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -122,7 +122,7 @@ def create_model(self) -> DeepMILModule: # no-op IdentityEncoder to be used inside the model self.slide_dataset = self.get_slide_dataset() self.level = 1 - return DeepMILModule(encoder=IdentityEncoder(input_dim=(self.encoder.num_encoding,)), + return DeepMILModule(encoder=self.encoder, # IdentityEncoder(input_dim=(self.encoder.num_encoding,)), label_column=self.data_module.train_dataset.LABEL_COLUMN, n_classes=self.data_module.train_dataset.N_CLASSES, pooling_layer=self.get_pooling_layer(), From c5ab194f37bda3f8ddf8899e26d627a78434dce4 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Mon, 24 Jan 2022 10:18:25 +0000 Subject: [PATCH 03/21] batch size and lr increase --- InnerEye/ML/Histopathology/models/deepmil.py | 5 ++--- InnerEye/ML/configs/histo_configs/classification/BaseMIL.py | 2 +- .../configs/histo_configs/classification/DeepSMILEPanda.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 87edf5743..ab6cea7ed 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -173,9 +173,8 @@ def log_metrics(self, self.log(f'{stage}/{metric_name}', metric_object, on_epoch=True, on_step=False, logger=True, sync_dist=True) def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore - # with no_grad(): - print(self.encoder) - H = self.encoder(images) # N X L x 1 x 1 + with no_grad(): + H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py index 66ffa196a..95727c40f 100644 --- a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py +++ b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py @@ -35,7 +35,7 @@ class BaseMIL(LightningContainer): n_channels: int = param.Integer(3, bounds=(1, None), doc="Number of channels in the tile.") # Data module parameters: - batch_size: int = param.Integer(32, bounds=(1, None), doc="Number of slides to load per batch.") + batch_size: int = param.Integer(64, bounds=(1, None), doc="Number of slides to load per batch.") max_bag_size: int = param.Integer(1000, bounds=(0, None), doc="Upper bound on number of tiles in each loaded bag. " "If 0 (default), will return all samples in each bag. " diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index bc96f4b62..37fb32f46 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -52,7 +52,7 @@ def __init__(self, **kwargs: Any) -> None: number_of_cross_validation_splits=5, cross_validation_split_index=0, # declared in OptimizerParams: - l_rate=1e-3, + l_rate=5e-3, weight_decay=1e-4, adam_betas=(0.9, 0.99)) default_kwargs.update(kwargs) @@ -122,7 +122,7 @@ def create_model(self) -> DeepMILModule: # no-op IdentityEncoder to be used inside the model self.slide_dataset = self.get_slide_dataset() self.level = 1 - return DeepMILModule(encoder=self.encoder, # IdentityEncoder(input_dim=(self.encoder.num_encoding,)), + return DeepMILModule(encoder=IdentityEncoder(input_dim=(self.encoder.num_encoding,)), label_column=self.data_module.train_dataset.LABEL_COLUMN, n_classes=self.data_module.train_dataset.N_CLASSES, pooling_layer=self.get_pooling_layer(), From 808a02e84b1f3fad19c4fffee4cf6ec5e419ed0c Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Wed, 26 Jan 2022 13:19:46 +0000 Subject: [PATCH 04/21] initial finetuning setup --- InnerEye/ML/Histopathology/models/deepmil.py | 4 ++-- .../ML/configs/histo_configs/classification/BaseMIL.py | 6 +++--- .../histo_configs/classification/DeepSMILEPanda.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index ab6cea7ed..4e987dcb6 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -173,8 +173,8 @@ def log_metrics(self, self.log(f'{stage}/{metric_name}', metric_object, on_epoch=True, on_step=False, logger=True, sync_dist=True) def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore - with no_grad(): - H = self.encoder(images) # N X L x 1 x 1 + # with no_grad(): + H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py index 95727c40f..f2aa73d5e 100644 --- a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py +++ b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py @@ -35,16 +35,16 @@ class BaseMIL(LightningContainer): n_channels: int = param.Integer(3, bounds=(1, None), doc="Number of channels in the tile.") # Data module parameters: - batch_size: int = param.Integer(64, bounds=(1, None), doc="Number of slides to load per batch.") + batch_size: int = param.Integer(8, bounds=(1, None), doc="Number of slides to load per batch.") max_bag_size: int = param.Integer(1000, bounds=(0, None), doc="Upper bound on number of tiles in each loaded bag. " "If 0 (default), will return all samples in each bag. " "If > 0, bags larger than `max_bag_size` will yield " "random subsets of instances.") - cache_mode: CacheMode = param.ClassSelector(default=CacheMode.MEMORY, class_=CacheMode, + cache_mode: CacheMode = param.ClassSelector(default=CacheMode.NONE, class_=CacheMode, doc="The type of caching to perform: " "'memory' (default), 'disk', or 'none'.") - save_precache: bool = param.Boolean(True, doc="Whether to pre-cache the entire transformed " + save_precache: bool = param.Boolean(False, doc="Whether to pre-cache the entire transformed " "dataset upfront and save it to disk.") # local_dataset (used as data module root_path) is declared in DatasetParams superclass diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 37fb32f46..208f85139 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -46,13 +46,13 @@ def __init__(self, **kwargs: Any) -> None: extra_local_dataset_paths=[Path("/tmp/datasets/PANDA")], # To mount the dataset instead of downloading in AML, pass --use_dataset_mount in the CLI # declared in TrainerParams: - num_epochs=200, + num_epochs=20, # use_mixed_precision = True, # declared in WorkflowParams: number_of_cross_validation_splits=5, cross_validation_split_index=0, # declared in OptimizerParams: - l_rate=5e-3, + l_rate=5e-4, weight_decay=1e-4, adam_betas=(0.9, 0.99)) default_kwargs.update(kwargs) @@ -101,7 +101,7 @@ def get_data_module(self) -> PandaTilesDataModule: transform = Compose( [ LoadTilesBatchd(image_key, progress=True), - EncodeTilesBatchd(image_key, self.encoder), + #EncodeTilesBatchd(image_key, self.encoder), ] ) return PandaTilesDataModule( @@ -122,7 +122,7 @@ def create_model(self) -> DeepMILModule: # no-op IdentityEncoder to be used inside the model self.slide_dataset = self.get_slide_dataset() self.level = 1 - return DeepMILModule(encoder=IdentityEncoder(input_dim=(self.encoder.num_encoding,)), + return DeepMILModule(encoder=self.encoder, #IdentityEncoder(input_dim=(self.encoder.num_encoding,)), label_column=self.data_module.train_dataset.LABEL_COLUMN, n_classes=self.data_module.train_dataset.N_CLASSES, pooling_layer=self.get_pooling_layer(), From 0b0eaa6b0ff3070044839547c4f55d682f57f052 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Mon, 31 Jan 2022 16:10:31 +0000 Subject: [PATCH 05/21] finetuning with caching --- InnerEye/ML/configs/histo_configs/classification/BaseMIL.py | 2 +- .../ML/configs/histo_configs/classification/DeepSMILEPanda.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py index 14b762d8a..2b8e1e5b4 100644 --- a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py +++ b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py @@ -35,7 +35,7 @@ class BaseMIL(LightningContainer): n_channels: int = param.Integer(3, bounds=(1, None), doc="Number of channels in the tile.") # Data module parameters: - batch_size: int = param.Integer(8, bounds=(1, None), doc="Number of slides to load per batch.") + batch_size: int = param.Integer(32, bounds=(1, None), doc="Number of slides to load per batch.") max_bag_size: int = param.Integer(1000, bounds=(0, None), doc="Upper bound on number of tiles in each loaded bag. " "If 0 (default), will return all samples in each bag. " diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index f6271b2bd..6d82af3a0 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -14,6 +14,7 @@ from health_azure.utils import get_workspace, is_running_in_azure_ml from health_ml.networks.layers.attention_layers import GatedAttentionLayer from InnerEye.Common import fixed_paths +from InnerEye.ML.Histopathology.datamodules.base_module import CacheMode, CacheLocation from InnerEye.ML.Histopathology.datamodules.panda_module import PandaTilesDataModule from InnerEye.ML.Histopathology.datasets.panda_tiles_dataset import PandaTilesDataset from InnerEye.ML.common import get_best_checkpoint_path @@ -41,6 +42,8 @@ def __init__(self, **kwargs: Any) -> None: pooling_type=GatedAttentionLayer.__name__, # average number of tiles is 56 for PANDA encoding_chunk_size=60, + cache_mode=CacheMode.MEMORY, + precache_location=CacheLocation.SAME, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), From 5cfad63bdfe2aec1df178acf10154e99764639db Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Feb 2022 13:58:22 +0000 Subject: [PATCH 06/21] finetuning and caching --- .../ML/configs/histo_configs/classification/DeepSMILEPanda.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 6d82af3a0..808b1ac57 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -44,6 +44,7 @@ def __init__(self, **kwargs: Any) -> None: encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.SAME, + batch_size=16, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), From e63005c2d46d4991db44797b0be79884da320196 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Feb 2022 20:26:56 +0000 Subject: [PATCH 07/21] class names --- .../configs/histo_configs/classification/DeepSMILEPanda.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index c1eda2818..85d3c8a4d 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -41,10 +41,10 @@ def __init__(self, **kwargs: Any) -> None: # declared in BaseMIL: pooling_type=GatedAttentionLayer.__name__, # average number of tiles is 56 for PANDA - encoding_chunk_size=60, + encoding_chunk_size=30, cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.SAME, - batch_size=16, + batch_size=8, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), @@ -131,6 +131,7 @@ def create_model(self) -> DeepMILModule: # no-op IdentityEncoder to be used inside the model self.slide_dataset = self.get_slide_dataset() self.level = 1 + self.class_names = ["ISUP 0", "ISUP 1", "ISUP 2", "ISUP 3", "ISUP 4", "ISUP 5"] return DeepMILModule(encoder=self.encoder, # IdentityEncoder(input_dim=(self.encoder.num_encoding,)), label_column=self.data_module.train_dataset.LABEL_COLUMN, n_classes=self.data_module.train_dataset.N_CLASSES, From 4fd511207671e8a51a9ee0ad38b401bf9a37a0f4 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Wed, 2 Feb 2022 09:16:45 +0000 Subject: [PATCH 08/21] enable finetuning --- InnerEye/ML/Histopathology/models/deepmil.py | 13 +++++++--- .../classification/DeepSMILEPanda.py | 26 ++++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 02d9e75a5..7f7e58155 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -54,7 +54,8 @@ def __init__(self, slide_dataset: SlidesDataset = None, tile_size: int = 224, level: int = 1, - class_names: Optional[List[str]] = None) -> None: + class_names: Optional[List[str]] = None, + is_finetune: Optional[bool] = False) -> None: """ :param label_column: Label key for input batch dictionary. :param n_classes: Number of output classes for MIL prediction. For binary classification, n_classes should be set to 1. @@ -73,6 +74,7 @@ def __init__(self, :param tile_size: The size of each tile (default=224). :param level: The downsampling level (e.g. 0, 1, 2) of the tiles if available (default=1). :param class_names: The names of the classes if available (default=None). + :param is_finetune: Boolean value to enable/disable finetuning (default=False) """ super().__init__() @@ -112,6 +114,8 @@ def __init__(self, self.verbose = verbose + self.is_finetune = is_finetune + self.aggregation_fn, self.num_pooling = self.get_pooling() self.classifier_fn = self.get_classifier() self.loss_fn = self.get_loss() @@ -187,8 +191,11 @@ def log_metrics(self, log_on_epoch(self, f'{stage}/{metric_name}', metric_object) def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore - # with no_grad(): - H = self.encoder(images) # N X L x 1 x 1 + if self.is_finetune: + H = self.encoder(images) # N X L x 1 x 1 + else: + with no_grad(): + H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 85d3c8a4d..fe24460b1 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -41,7 +41,7 @@ def __init__(self, **kwargs: Any) -> None: # declared in BaseMIL: pooling_type=GatedAttentionLayer.__name__, # average number of tiles is 56 for PANDA - encoding_chunk_size=30, + encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.SAME, batch_size=8, @@ -82,6 +82,7 @@ def __init__(self, **kwargs: Any) -> None: mode="max", ) self.callbacks = best_checkpoint_callback + self.is_finetune = True @property def cache_dir(self) -> Path: @@ -107,12 +108,14 @@ def setup(self) -> None: def get_data_module(self) -> PandaTilesDataModule: image_key = PandaTilesDataset.IMAGE_COLUMN - transform = Compose( - [ - LoadTilesBatchd(image_key, progress=True), - # EncodeTilesBatchd(image_key, self.encoder), - ] - ) + if self.is_finetune: + transform = Compose([LoadTilesBatchd(image_key, progress=True)]) + else: + transform = Compose([ + LoadTilesBatchd(image_key, progress=True), + EncodeTilesBatchd(image_key, self.encoder) + ]) + return PandaTilesDataModule( root_path=self.local_dataset, max_bag_size=self.max_bag_size, @@ -132,7 +135,11 @@ def create_model(self) -> DeepMILModule: self.slide_dataset = self.get_slide_dataset() self.level = 1 self.class_names = ["ISUP 0", "ISUP 1", "ISUP 2", "ISUP 3", "ISUP 4", "ISUP 5"] - return DeepMILModule(encoder=self.encoder, # IdentityEncoder(input_dim=(self.encoder.num_encoding,)), + if self.is_finetune: + self.model_encoder = self.encoder + else: + self.model_encoder = IdentityEncoder(input_dim=(self.encoder.num_encoding,)) + return DeepMILModule(encoder=self.model_encoder, label_column=self.data_module.train_dataset.LABEL_COLUMN, n_classes=self.data_module.train_dataset.N_CLASSES, pooling_layer=self.get_pooling_layer(), @@ -143,7 +150,8 @@ def create_model(self) -> DeepMILModule: slide_dataset=self.get_slide_dataset(), tile_size=self.tile_size, level=self.level, - class_names=self.class_names) + class_names=self.class_names, + is_finetune=self.is_finetune) def get_slide_dataset(self) -> PandaDataset: return PandaDataset(root=self.extra_local_dataset_paths[0]) # type: ignore From c3e0675c9eebdbc1df80c1df5ca85ea70737123b Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Wed, 2 Feb 2022 09:26:37 +0000 Subject: [PATCH 09/21] allign with main --- InnerEye/ML/Histopathology/models/deepmil.py | 2 +- InnerEye/ML/Histopathology/models/encoders.py | 8 -------- .../histo_configs/classification/DeepSMILEPanda.py | 6 +++--- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 7f7e58155..7529bb217 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -74,7 +74,7 @@ def __init__(self, :param tile_size: The size of each tile (default=224). :param level: The downsampling level (e.g. 0, 1, 2) of the tiles if available (default=1). :param class_names: The names of the classes if available (default=None). - :param is_finetune: Boolean value to enable/disable finetuning (default=False) + :param is_finetune: Boolean value to enable/disable finetuning (default=False). """ super().__init__() diff --git a/InnerEye/ML/Histopathology/models/encoders.py b/InnerEye/ML/Histopathology/models/encoders.py index d567407bc..e6c609e51 100644 --- a/InnerEye/ML/Histopathology/models/encoders.py +++ b/InnerEye/ML/Histopathology/models/encoders.py @@ -137,14 +137,6 @@ class HistoSSLEncoder(TileEncoder): WEIGHTS_URL = ("https://github.com/ozanciga/self-supervised-histopathology/releases/" "download/tenpercent/tenpercent_resnet18.ckpt") - # def _get_preprocessing(self) -> Callable: - # return get_imagenet_preprocessing() - - # def _get_encoder(self) -> Tuple[Callable, int]: - # resnet18_model = resnet18(pretrained=False) - # histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) - # return setup_feature_extractor(histossl_encoder, self.input_dim) # type: ignore - def _get_encoder(self) -> Tuple[Callable, int]: resnet18_model = resnet18(pretrained=False) histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index fe24460b1..e5b4cde31 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -53,7 +53,7 @@ def __init__(self, **kwargs: Any) -> None: extra_local_dataset_paths=[Path("/tmp/datasets/PANDA")], # To mount the dataset instead of downloading in AML, pass --use_dataset_mount in the CLI # declared in TrainerParams: - num_epochs=20, + num_epochs=200, # use_mixed_precision = True, # declared in WorkflowParams: @@ -82,7 +82,7 @@ def __init__(self, **kwargs: Any) -> None: mode="max", ) self.callbacks = best_checkpoint_callback - self.is_finetune = True + self.is_finetune = False @property def cache_dir(self) -> Path: @@ -113,7 +113,7 @@ def get_data_module(self) -> PandaTilesDataModule: else: transform = Compose([ LoadTilesBatchd(image_key, progress=True), - EncodeTilesBatchd(image_key, self.encoder) + EncodeTilesBatchd(image_key, self.encoder, chunk_size=self.encoding_chunk_size) ]) return PandaTilesDataModule( From 0855c6ffddadff91f0fb3df8c62e93ce9167da7e Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Wed, 2 Feb 2022 09:34:30 +0000 Subject: [PATCH 10/21] restore files from main --- .../aggregate_metrics_crossvalidation.py | 20 ++++--------------- .../ML/Histopathology/utils/download_utils.py | 2 +- .../histo_configs/classification/BaseMIL.py | 4 ++-- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py b/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py index d0d81df8d..7c96b0349 100644 --- a/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py +++ b/InnerEye/ML/Histopathology/scripts/aggregate_metrics_crossvalidation.py @@ -6,23 +6,11 @@ """ Script to find mean and standard deviation of desired metrics from cross validation child runs. """ -import sys, os +import os import pandas as pd -from pathlib import Path - -current_dir = Path(os.getcwd()) -radiomics_root = current_dir -if (radiomics_root / "InnerEyePrivate").is_dir(): - radiomics_root_str = str(radiomics_root) - if radiomics_root_str not in sys.path: - print(f"Adding to sys.path: {radiomics_root_str}") - sys.path.insert(0, radiomics_root_str) - sys.path.insert(0, str(radiomics_root / "innereye-deeplearning")) - sys.path.insert(0, str(radiomics_root / "innereye-deeplearning/hi-ml/hi-ml-azure/src")) - sys.path.insert(0, str(radiomics_root / "innereye-deeplearning/hi-ml/hi-ml/src")) - print(f"Sys path {sys.path}") from health_azure import aggregate_hyperdrive_metrics, get_workspace + from InnerEye.Common import fixed_paths @@ -40,8 +28,8 @@ def get_cross_validation_metrics_df(run_id: str) -> pd.DataFrame: if __name__ == "__main__": - metrics_list = ['test/accuracy', 'test/auroc', 'test/f1score', 'test/precision', 'test/recall', 'test/macro_accuracy', 'test/weighted_accuracy'] - run_id = "hsharma_panda_tiles_ssl:HD_b5be4968-4896-4fc4-8d62-291ebe5c57c2" + metrics_list = ['test/accuracy', 'test/auroc', 'test/f1score', 'test/precision', 'test/recall'] + run_id = "hsharma_features_viz:HD_eff4c009-2f9f-4c2c-94c6-c0c84944a412" metrics_df = get_cross_validation_metrics_df(run_id=run_id) for metric in metrics_list: if metric in metrics_df.index.values: diff --git a/InnerEye/ML/Histopathology/utils/download_utils.py b/InnerEye/ML/Histopathology/utils/download_utils.py index 862a8439d..1addcbc55 100644 --- a/InnerEye/ML/Histopathology/utils/download_utils.py +++ b/InnerEye/ML/Histopathology/utils/download_utils.py @@ -30,7 +30,7 @@ def download_file_if_necessary(run_id: str, remote_dir: Path, download_dir: Path download_files_from_run_id(run_id=run_id, output_folder=local_dir, prefix=str(remote_path), - workspace=aml_workspace, + aml_workspace=aml_workspace, validate_checksum=True) assert local_path.exists() print("File is downloaded at", local_path) diff --git a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py index 2b8e1e5b4..4328de36b 100644 --- a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py +++ b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py @@ -35,13 +35,13 @@ class BaseMIL(LightningContainer): n_channels: int = param.Integer(3, bounds=(1, None), doc="Number of channels in the tile.") # Data module parameters: - batch_size: int = param.Integer(32, bounds=(1, None), doc="Number of slides to load per batch.") + batch_size: int = param.Integer(16, bounds=(1, None), doc="Number of slides to load per batch.") max_bag_size: int = param.Integer(1000, bounds=(0, None), doc="Upper bound on number of tiles in each loaded bag. " "If 0 (default), will return all samples in each bag. " "If > 0, bags larger than `max_bag_size` will yield " "random subsets of instances.") - cache_mode: CacheMode = param.ClassSelector(default=CacheMode.NONE, class_=CacheMode, + cache_mode: CacheMode = param.ClassSelector(default=CacheMode.MEMORY, class_=CacheMode, doc="The type of caching to perform: " "'memory' (default), 'disk', or 'none'.") precache_location: str = param.ClassSelector(default=CacheLocation.NONE, class_=CacheLocation, From 6eea4c70954625915e3d782e630bb62aa860e6df Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Wed, 2 Feb 2022 10:20:48 +0000 Subject: [PATCH 11/21] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a7b7a71..53e0c9e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ jobs that run in AzureML. - ([#634](https://github.com/microsoft/InnerEye-DeepLearning/pull/634)) Add WSI heatmaps and thumbnails to standard test outputs - ([#635](https://github.com/microsoft/InnerEye-DeepLearning/pull/635)) Add tile selection and binary label for online evaluation of PANDA SSL - ([#647](https://github.com/microsoft/InnerEye-DeepLearning/pull/647)) Add class-wise accuracy logging and confusion matrix to DeepMIL +- ([#650](https://github.com/microsoft/InnerEye-DeepLearning/pull/650)) Enable fine-tuning in DeepMIL using PANDA as the classification task ### Changed - ([#588](https://github.com/microsoft/InnerEye-DeepLearning/pull/588)) Replace SciPy with PIL.PngImagePlugin.PngImageFile to load png files. From 4aeb3e304fcc693acfd57a85b57e6999c3162498 Mon Sep 17 00:00:00 2001 From: Max Ilse Date: Wed, 2 Feb 2022 15:36:05 +0000 Subject: [PATCH 12/21] fixed test --- Tests/SSL/test_ssl_containers.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Tests/SSL/test_ssl_containers.py b/Tests/SSL/test_ssl_containers.py index 1a55a53e6..49569378b 100644 --- a/Tests/SSL/test_ssl_containers.py +++ b/Tests/SSL/test_ssl_containers.py @@ -634,7 +634,24 @@ def test_simclr_dataloader_type() -> None: """ This test checks if the transform pipeline of a SSL job can handle different data types coming from the dataloader. """ - def check_types_in_dataloader(dataloader: CombinedLoader) -> None: + # TODO: Once the pytorch lightning bug is fixed the following test can be removed. + # The training and val loader will be both CombinedLoaders + def check_types_in_train_dataloader(dataloader: dict) -> None: + for i, batch in enumerate(dataloader[SSLDataModuleType.ENCODER]): + assert isinstance(batch[0][0], torch.Tensor) + assert isinstance(batch[0][1], torch.Tensor) + assert isinstance(batch[1], torch.Tensor) + if i == 1: + break + + for i, batch in enumerate(dataloader[SSLDataModuleType.LINEAR_HEAD]): + assert isinstance(batch[0], torch.Tensor) + assert isinstance(batch[1], torch.Tensor) + assert isinstance(batch[2], torch.Tensor) + if i == 1: + break + + def check_types_in_val_dataloader(dataloader: CombinedLoader) -> None: for i, batch in enumerate(dataloader): assert isinstance(batch[SSLDataModuleType.ENCODER][0][0], torch.Tensor) assert isinstance(batch[SSLDataModuleType.ENCODER][0][1], torch.Tensor) @@ -646,8 +663,8 @@ def check_types_in_dataloader(dataloader: CombinedLoader) -> None: break def check_types_in_train_and_val(data: CombinedDataModule) -> None: - check_types_in_dataloader(data.train_dataloader()) - check_types_in_dataloader(data.val_dataloader()) + check_types_in_train_dataloader(data.train_dataloader()) + check_types_in_val_dataloader(data.val_dataloader()) container = DummySimCLR() container.setup() From 9185ab1af5b352d855745c45f5850496e3f33bea Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Thu, 3 Feb 2022 15:17:05 +0000 Subject: [PATCH 13/21] PR comments addressed --- InnerEye/ML/Histopathology/models/deepmil.py | 7 ++----- InnerEye/ML/Histopathology/models/encoders.py | 2 +- .../histo_configs/classification/DeepSMILEPanda.py | 9 ++++++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 7529bb217..18dd1c740 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -13,7 +13,7 @@ import more_itertools as mi from pytorch_lightning import LightningModule -from torch import Tensor, argmax, mode, nn, no_grad, optim, round +from torch import Tensor, argmax, mode, nn, set_grad_enabled, optim, round from torchmetrics import AUROC, F1, Accuracy, Precision, Recall, ConfusionMatrix from InnerEye.Common import fixed_paths @@ -191,11 +191,8 @@ def log_metrics(self, log_on_epoch(self, f'{stage}/{metric_name}', metric_object) def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore - if self.is_finetune: + with set_grad_enabled(self.is_finetune): H = self.encoder(images) # N X L x 1 x 1 - else: - with no_grad(): - H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/Histopathology/models/encoders.py b/InnerEye/ML/Histopathology/models/encoders.py index e6c609e51..132c35a3c 100644 --- a/InnerEye/ML/Histopathology/models/encoders.py +++ b/InnerEye/ML/Histopathology/models/encoders.py @@ -141,5 +141,5 @@ def _get_encoder(self) -> Tuple[Callable, int]: resnet18_model = resnet18(pretrained=False) histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) histossl_encoder.fc = torch.nn.Sequential() - num_features = 512 + num_features = resnet18_model.fc.in_features return histossl_encoder, num_features # type: ignore diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index e5b4cde31..f4feae3da 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -36,6 +36,9 @@ class DeepSMILEPanda(BaseMIL): + """`is_finetune` sets the fine-tuning mode. If this is set, setting cache_mode=CacheMode.NONE takes ~30 min/epoch and + cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.SAME takes ~ 5 min/epoch. Fine-tuning is tested with batch size 8 on PANDA. + """ def __init__(self, **kwargs: Any) -> None: default_kwargs = dict( # declared in BaseMIL: @@ -43,8 +46,7 @@ def __init__(self, **kwargs: Any) -> None: # average number of tiles is 56 for PANDA encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, - precache_location=CacheLocation.SAME, - batch_size=8, + precache_location=CacheLocation.CPU, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), @@ -104,7 +106,8 @@ def setup(self) -> None: self.downloader.download_checkpoint_if_necessary() self.encoder = self.get_encoder() self.encoder.cuda() - self.encoder.eval() + if not self.is_finetune: + self.encoder.eval() def get_data_module(self) -> PandaTilesDataModule: image_key = PandaTilesDataset.IMAGE_COLUMN From ad5872e82289cd12dc0fc980b5ce7bb8720e4bf4 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Thu, 3 Feb 2022 16:54:20 +0000 Subject: [PATCH 14/21] ddp debug --- InnerEye/ML/Histopathology/models/deepmil.py | 2 +- .../configs/histo_configs/classification/DeepSMILEPanda.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 18dd1c740..70f103065 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -55,7 +55,7 @@ def __init__(self, tile_size: int = 224, level: int = 1, class_names: Optional[List[str]] = None, - is_finetune: Optional[bool] = False) -> None: + is_finetune: bool = False) -> None: """ :param label_column: Label key for input batch dictionary. :param n_classes: Number of output classes for MIL prediction. For binary classification, n_classes should be set to 1. diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index f4feae3da..9e2cdf778 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -46,7 +46,8 @@ def __init__(self, **kwargs: Any) -> None: # average number of tiles is 56 for PANDA encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, - precache_location=CacheLocation.CPU, + precache_location=CacheLocation.NONE, + max_bag_size=10, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), @@ -84,7 +85,7 @@ def __init__(self, **kwargs: Any) -> None: mode="max", ) self.callbacks = best_checkpoint_callback - self.is_finetune = False + self.is_finetune = True @property def cache_dir(self) -> Path: @@ -105,7 +106,7 @@ def setup(self) -> None: os.chdir(fixed_paths.repository_parent_directory()) self.downloader.download_checkpoint_if_necessary() self.encoder = self.get_encoder() - self.encoder.cuda() + # self.encoder.cuda() if not self.is_finetune: self.encoder.eval() From 0bbd85f99b27c7c03663b5e545cd052ef63323eb Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Fri, 4 Feb 2022 14:02:01 +0000 Subject: [PATCH 15/21] finetuning ddp --- InnerEye/ML/Histopathology/models/encoders.py | 2 +- .../ML/configs/histo_configs/classification/DeepSMILEPanda.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/encoders.py b/InnerEye/ML/Histopathology/models/encoders.py index 132c35a3c..155b73ef7 100644 --- a/InnerEye/ML/Histopathology/models/encoders.py +++ b/InnerEye/ML/Histopathology/models/encoders.py @@ -139,7 +139,7 @@ class HistoSSLEncoder(TileEncoder): def _get_encoder(self) -> Tuple[Callable, int]: resnet18_model = resnet18(pretrained=False) + num_features = resnet18_model.fc.in_features histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) histossl_encoder.fc = torch.nn.Sequential() - num_features = resnet18_model.fc.in_features return histossl_encoder, num_features # type: ignore diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 9e2cdf778..d9adaab2a 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -46,8 +46,8 @@ def __init__(self, **kwargs: Any) -> None: # average number of tiles is 56 for PANDA encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, - precache_location=CacheLocation.NONE, - max_bag_size=10, + precache_location=CacheLocation.CPU, + max_bag_size=100, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), From 571f66dcaa93ffaa49cecb3ddd0485812b43e3ed Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Mon, 7 Feb 2022 08:10:59 +0000 Subject: [PATCH 16/21] finetuning config --- .../configs/histo_configs/classification/DeepSMILEPanda.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index d9adaab2a..f46f2a9c0 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -37,7 +37,8 @@ class DeepSMILEPanda(BaseMIL): """`is_finetune` sets the fine-tuning mode. If this is set, setting cache_mode=CacheMode.NONE takes ~30 min/epoch and - cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.SAME takes ~ 5 min/epoch. Fine-tuning is tested with batch size 8 on PANDA. + cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU takes ~[5-10] min/epoch. + Fine-tuning with caching completes using batch_size=8, max_bag_size=100, num_peochs=20, max_num_gpus=1 on PANDA. """ def __init__(self, **kwargs: Any) -> None: default_kwargs = dict( @@ -47,6 +48,7 @@ def __init__(self, **kwargs: Any) -> None: encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU, + batch_size=8, max_bag_size=100, # declared in DatasetParams: @@ -56,7 +58,7 @@ def __init__(self, **kwargs: Any) -> None: extra_local_dataset_paths=[Path("/tmp/datasets/PANDA")], # To mount the dataset instead of downloading in AML, pass --use_dataset_mount in the CLI # declared in TrainerParams: - num_epochs=200, + num_epochs=20, # use_mixed_precision = True, # declared in WorkflowParams: @@ -106,7 +108,6 @@ def setup(self) -> None: os.chdir(fixed_paths.repository_parent_directory()) self.downloader.download_checkpoint_if_necessary() self.encoder = self.get_encoder() - # self.encoder.cuda() if not self.is_finetune: self.encoder.eval() From bd4802feb97c1e2948da9789b194216c2b604876 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Mon, 7 Feb 2022 13:54:30 +0000 Subject: [PATCH 17/21] unfreeze encoders finetuning and chunking in encoder --- InnerEye/ML/Histopathology/models/deepmil.py | 14 ++++++++++++-- .../histo_configs/classification/BaseMIL.py | 14 +++++++++++--- .../histo_configs/classification/DeepSMILEPanda.py | 12 +++++++----- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 70f103065..8189a44c0 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -24,6 +24,7 @@ plot_slide, plot_normalized_confusion_matrix) from InnerEye.ML.Histopathology.utils.naming import SlideKey, ResultsKey, MetricsKey from InnerEye.ML.Histopathology.utils.viz_utils import load_image_dict +from InnerEye.ML.Histopathology.models.transforms import EncodeTilesBatchd from health_ml.utils import log_on_epoch RESULTS_COLS = [ResultsKey.SLIDE_ID, ResultsKey.TILE_ID, ResultsKey.IMAGE_PATH, ResultsKey.PROB, @@ -55,7 +56,8 @@ def __init__(self, tile_size: int = 224, level: int = 1, class_names: Optional[List[str]] = None, - is_finetune: bool = False) -> None: + is_finetune: bool = False, + encoding_chunk_size: int = 0) -> None: """ :param label_column: Label key for input batch dictionary. :param n_classes: Number of output classes for MIL prediction. For binary classification, n_classes should be set to 1. @@ -75,6 +77,7 @@ def __init__(self, :param level: The downsampling level (e.g. 0, 1, 2) of the tiles if available (default=1). :param class_names: The names of the classes if available (default=None). :param is_finetune: Boolean value to enable/disable finetuning (default=False). + :param encoding_chunk_size: Only used in fine-tuning mode. If > 0 performs encoding in chunks, by loading enconding_chunk_size tiles per chunk (default=0). """ super().__init__() @@ -114,7 +117,9 @@ def __init__(self, self.verbose = verbose + #finetuning attributes self.is_finetune = is_finetune + self.encoding_chunk_size = encoding_chunk_size self.aggregation_fn, self.num_pooling = self.get_pooling() self.classifier_fn = self.get_classifier() @@ -192,7 +197,12 @@ def log_metrics(self, def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore with set_grad_enabled(self.is_finetune): - H = self.encoder(images) # N X L x 1 x 1 + if self.is_finetune: + image_key = TilesDataset.IMAGE_COLUMN + transform = EncodeTilesBatchd(image_key, self.encoder, chunk_size=self.encoding_chunk_size) + H = transform(images) + else: + H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py index 4328de36b..1d4a98dd8 100644 --- a/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py +++ b/InnerEye/ML/configs/histo_configs/classification/BaseMIL.py @@ -27,6 +27,8 @@ class BaseMIL(LightningContainer): # Model parameters: pooling_type: str = param.String(doc="Name of the pooling layer class to use.") + is_finetune: bool = param.Boolean(doc="Whether to fine-tune the encoder. Options:" + "`False` (default), or `True`.") # l_rate, weight_decay, adam_betas are already declared in OptimizerParams superclass # Encoder parameters: @@ -61,8 +63,8 @@ def setup(self) -> None: raise NotImplementedError("InnerEyeSSLEncoder requires a pre-trained checkpoint.") self.encoder = self.get_encoder() - self.encoder.cuda() - self.encoder.eval() + if not self.is_finetune: + self.encoder.eval() def get_encoder(self) -> TileEncoder: if self.encoder_type == ImageNetEncoder.__name__: @@ -94,7 +96,13 @@ def create_model(self) -> DeepMILModule: self.data_module = self.get_data_module() # Encoding is done in the datamodule, so here we provide instead a dummy # no-op IdentityEncoder to be used inside the model - return DeepMILModule(encoder=IdentityEncoder(input_dim=(self.encoder.num_encoding,)), + if self.is_finetune: + self.model_encoder = self.encoder + for params in self.model_encoder.parameters(): + params.requires_grad = True + else: + self.model_encoder = IdentityEncoder(input_dim=(self.encoder.num_encoding,)) + return DeepMILModule(encoder=self.model_encoder, label_column=self.data_module.train_dataset.LABEL_COLUMN, n_classes=self.data_module.train_dataset.N_CLASSES, pooling_layer=self.get_pooling_layer(), diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index f46f2a9c0..82736b1c3 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -38,7 +38,7 @@ class DeepSMILEPanda(BaseMIL): """`is_finetune` sets the fine-tuning mode. If this is set, setting cache_mode=CacheMode.NONE takes ~30 min/epoch and cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU takes ~[5-10] min/epoch. - Fine-tuning with caching completes using batch_size=8, max_bag_size=100, num_peochs=20, max_num_gpus=1 on PANDA. + Fine-tuning with caching completes using batch_size=8, max_bag_size=100, num_epochs=20, max_num_gpus=1 on PANDA. """ def __init__(self, **kwargs: Any) -> None: default_kwargs = dict( @@ -48,8 +48,8 @@ def __init__(self, **kwargs: Any) -> None: encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU, - batch_size=8, - max_bag_size=100, + is_finetune=True, + # batch_size=4, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), @@ -87,7 +87,6 @@ def __init__(self, **kwargs: Any) -> None: mode="max", ) self.callbacks = best_checkpoint_callback - self.is_finetune = True @property def cache_dir(self) -> Path: @@ -142,6 +141,8 @@ def create_model(self) -> DeepMILModule: self.class_names = ["ISUP 0", "ISUP 1", "ISUP 2", "ISUP 3", "ISUP 4", "ISUP 5"] if self.is_finetune: self.model_encoder = self.encoder + for params in self.model_encoder.parameters(): + params.requires_grad = True else: self.model_encoder = IdentityEncoder(input_dim=(self.encoder.num_encoding,)) return DeepMILModule(encoder=self.model_encoder, @@ -156,7 +157,8 @@ def create_model(self) -> DeepMILModule: tile_size=self.tile_size, level=self.level, class_names=self.class_names, - is_finetune=self.is_finetune) + is_finetune=self.is_finetune, + encoding_chunk_size=self.encoding_chunk_size) def get_slide_dataset(self) -> PandaDataset: return PandaDataset(root=self.extra_local_dataset_paths[0]) # type: ignore From adba7d6bb46f23a06626f20686d5e5cced0021a9 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 8 Feb 2022 06:07:52 +0000 Subject: [PATCH 18/21] revert encoding chunks --- InnerEye/ML/Histopathology/models/deepmil.py | 15 +++------------ .../classification/DeepSMILEPanda.py | 7 +++---- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 8189a44c0..2f50b3a04 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -24,7 +24,6 @@ plot_slide, plot_normalized_confusion_matrix) from InnerEye.ML.Histopathology.utils.naming import SlideKey, ResultsKey, MetricsKey from InnerEye.ML.Histopathology.utils.viz_utils import load_image_dict -from InnerEye.ML.Histopathology.models.transforms import EncodeTilesBatchd from health_ml.utils import log_on_epoch RESULTS_COLS = [ResultsKey.SLIDE_ID, ResultsKey.TILE_ID, ResultsKey.IMAGE_PATH, ResultsKey.PROB, @@ -56,8 +55,7 @@ def __init__(self, tile_size: int = 224, level: int = 1, class_names: Optional[List[str]] = None, - is_finetune: bool = False, - encoding_chunk_size: int = 0) -> None: + is_finetune: bool = False) -> None: """ :param label_column: Label key for input batch dictionary. :param n_classes: Number of output classes for MIL prediction. For binary classification, n_classes should be set to 1. @@ -77,7 +75,6 @@ def __init__(self, :param level: The downsampling level (e.g. 0, 1, 2) of the tiles if available (default=1). :param class_names: The names of the classes if available (default=None). :param is_finetune: Boolean value to enable/disable finetuning (default=False). - :param encoding_chunk_size: Only used in fine-tuning mode. If > 0 performs encoding in chunks, by loading enconding_chunk_size tiles per chunk (default=0). """ super().__init__() @@ -117,9 +114,8 @@ def __init__(self, self.verbose = verbose - #finetuning attributes + # Finetuning attributes self.is_finetune = is_finetune - self.encoding_chunk_size = encoding_chunk_size self.aggregation_fn, self.num_pooling = self.get_pooling() self.classifier_fn = self.get_classifier() @@ -197,12 +193,7 @@ def log_metrics(self, def forward(self, images: Tensor) -> Tuple[Tensor, Tensor]: # type: ignore with set_grad_enabled(self.is_finetune): - if self.is_finetune: - image_key = TilesDataset.IMAGE_COLUMN - transform = EncodeTilesBatchd(image_key, self.encoder, chunk_size=self.encoding_chunk_size) - H = transform(images) - else: - H = self.encoder(images) # N X L x 1 x 1 + H = self.encoder(images) # N X L x 1 x 1 A, M = self.aggregation_fn(H) # A: K x N | M: K x L M = M.view(-1, self.num_encoding * self.pool_out_dim) Y_prob = self.classifier_fn(M) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 82736b1c3..204c8439b 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -38,7 +38,7 @@ class DeepSMILEPanda(BaseMIL): """`is_finetune` sets the fine-tuning mode. If this is set, setting cache_mode=CacheMode.NONE takes ~30 min/epoch and cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU takes ~[5-10] min/epoch. - Fine-tuning with caching completes using batch_size=8, max_bag_size=100, num_epochs=20, max_num_gpus=1 on PANDA. + Fine-tuning with caching completes using batch_size=4, max_bag_size=1000, num_epochs=20, max_num_gpus=1 on PANDA. """ def __init__(self, **kwargs: Any) -> None: default_kwargs = dict( @@ -49,7 +49,7 @@ def __init__(self, **kwargs: Any) -> None: cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU, is_finetune=True, - # batch_size=4, + batch_size=4, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), @@ -157,8 +157,7 @@ def create_model(self) -> DeepMILModule: tile_size=self.tile_size, level=self.level, class_names=self.class_names, - is_finetune=self.is_finetune, - encoding_chunk_size=self.encoding_chunk_size) + is_finetune=self.is_finetune) def get_slide_dataset(self) -> PandaDataset: return PandaDataset(root=self.extra_local_dataset_paths[0]) # type: ignore From be3e78d1554d02f2ef03f8019b68cb85f931d2b4 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 8 Feb 2022 06:20:07 +0000 Subject: [PATCH 19/21] change batch size and num epochs --- .../ML/configs/histo_configs/classification/DeepSMILEPanda.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 204c8439b..7520b62d4 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -49,7 +49,6 @@ def __init__(self, **kwargs: Any) -> None: cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU, is_finetune=True, - batch_size=4, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"), @@ -58,7 +57,7 @@ def __init__(self, **kwargs: Any) -> None: extra_local_dataset_paths=[Path("/tmp/datasets/PANDA")], # To mount the dataset instead of downloading in AML, pass --use_dataset_mount in the CLI # declared in TrainerParams: - num_epochs=20, + num_epochs=200, # use_mixed_precision = True, # declared in WorkflowParams: From 3e83b5ba2a38ae35b0e03f49d46762262016d77f Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 8 Feb 2022 07:48:53 +0000 Subject: [PATCH 20/21] histossl encoder freeze --- InnerEye/ML/Histopathology/models/encoders.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/InnerEye/ML/Histopathology/models/encoders.py b/InnerEye/ML/Histopathology/models/encoders.py index 155b73ef7..570254298 100644 --- a/InnerEye/ML/Histopathology/models/encoders.py +++ b/InnerEye/ML/Histopathology/models/encoders.py @@ -142,4 +142,6 @@ def _get_encoder(self) -> Tuple[Callable, int]: num_features = resnet18_model.fc.in_features histossl_encoder = load_weights_to_model(self.WEIGHTS_URL, resnet18_model) histossl_encoder.fc = torch.nn.Sequential() + for param in histossl_encoder.parameters(): + param.requires_grad = False return histossl_encoder, num_features # type: ignore From 62715d68177b2015f6ea79d267bc761bc2117c73 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Wed, 9 Feb 2022 09:11:28 +0000 Subject: [PATCH 21/21] finetuning glag false --- .../ML/configs/histo_configs/classification/DeepSMILEPanda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 7520b62d4..9296d6492 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -48,7 +48,7 @@ def __init__(self, **kwargs: Any) -> None: encoding_chunk_size=60, cache_mode=CacheMode.MEMORY, precache_location=CacheLocation.CPU, - is_finetune=True, + is_finetune=False, # declared in DatasetParams: local_dataset=Path("/tmp/datasets/PANDA_tiles"),