From 4286bf0fe7919f47ef4d26ab6dec4c11d7558821 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Mon, 28 Feb 2022 15:58:07 +0000 Subject: [PATCH 01/11] initial commit add FP TN and multiclass outputs --- InnerEye/ML/Histopathology/models/deepmil.py | 48 ++++++++++++++----- .../ML/Histopathology/utils/metrics_utils.py | 21 ++++---- .../classification/DeepSMILECrck.py | 4 +- .../classification/DeepSMILEPanda.py | 4 +- InnerEye/ML/configs/histo_configs/run_ids.py | 2 +- hi-ml | 2 +- 6 files changed, 54 insertions(+), 27 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index fc30c2ec3..2254766d5 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -241,8 +241,10 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK probs = self.activation_fn(bag_logits) if self.n_classes > 1: preds = argmax(probs, dim=1) + probs_perclass = probs else: preds = round(probs) + probs_perclass = Tensor([[1.0-probs[i][0].item(), probs[i][0].item()] for i in range(len(probs))]).cuda() loss = loss.view(-1, 1) preds = preds.view(-1, 1) @@ -255,7 +257,7 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, - ResultsKey.PROB: probs, ResultsKey.PRED_LABEL: preds, + ResultsKey.PROB: probs_perclass, ResultsKey.PRED_LABEL: preds, ResultsKey.TRUE_LABEL: bag_labels, ResultsKey.BAG_ATTN: bag_attn_list, ResultsKey.IMAGE: batch[TilesDataset.IMAGE_COLUMN]}) @@ -338,11 +340,27 @@ def test_epoch_end(self, outputs: List[Dict[str, Any]]) -> None: # type: ignore torch.save(features_list, encoded_features_filename) print("Selecting tiles ...") - fn_top_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('lowest_pred', 'highest_att')) - fn_bottom_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('lowest_pred', 'lowest_att')) - tp_top_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('highest_pred', 'highest_att')) - tp_bottom_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('highest_pred', 'lowest_att')) - report_cases = {'TP': [tp_top_tiles, tp_bottom_tiles], 'FN': [fn_top_tiles, fn_bottom_tiles]} + # Class 0 + tn_top_tiles = select_k_tiles(results, n_slides=10, label=0, n_tiles=10, select=('highest_pred', 'highest_att')) + tn_bottom_tiles = select_k_tiles(results, n_slides=10, label=0, n_tiles=10, select=('highest_pred', 'lowest_att')) + fp_top_tiles = select_k_tiles(results, n_slides=10, label=0, n_tiles=10, select=('lowest_pred', 'highest_att')) + fp_bottom_tiles = select_k_tiles(results, n_slides=10, label=0, n_tiles=10, select=('lowest_pred', 'lowest_att')) + report_cases = {'TN': [tn_top_tiles, tn_bottom_tiles], 'FP': [fp_top_tiles, fp_bottom_tiles]} + + # Class 1 to n_classes-1 + if self.n_classes > 1: + for i in range(1, self.n_classes): + fn_top_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('lowest_pred', 'highest_att')) + fn_bottom_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('lowest_pred', 'lowest_att')) + tp_top_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('highest_pred', 'highest_att')) + tp_bottom_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('highest_pred', 'lowest_att')) + report_cases.update({'TP_'+str(i): [tp_top_tiles, tp_bottom_tiles], 'FN_'+str(i): [fn_top_tiles, fn_bottom_tiles]}) + else: + fn_top_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('lowest_pred', 'highest_att')) + fn_bottom_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('lowest_pred', 'lowest_att')) + tp_top_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('highest_pred', 'highest_att')) + tp_bottom_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('highest_pred', 'lowest_att')) + report_cases.update({'TP_1': [tp_top_tiles, tp_bottom_tiles], 'FN_1': [fn_top_tiles, fn_bottom_tiles]}) for key in report_cases.keys(): print(f"Plotting {key} (tiles, thumbnails, attention heatmaps)...") @@ -396,13 +414,19 @@ def normalize_dict_for_df(dict_old: Dict[str, Any], use_gpu: bool) -> Dict: # these steps are required to convert the dictionary to pandas dataframe. device = 'cuda' if use_gpu else 'cpu' dict_new = dict() + bag_size = len(dict_old[ResultsKey.SLIDE_ID]) for key, value in dict_old.items(): - if isinstance(value, Tensor): - value = value.squeeze(0).to(device).numpy() - if value.ndim == 0: - bag_size = len(dict_old[ResultsKey.SLIDE_ID]) - value = np.full(bag_size, fill_value=value) - dict_new[key] = value + if not key == ResultsKey.PROB: + if isinstance(value, Tensor): + value = value.squeeze(0).to(device).numpy() + if value.ndim == 0: + value = np.full(bag_size, fill_value=value) + dict_new[key] = value + else: + if isinstance(value, Tensor): + value = value.squeeze(0).to(device).numpy() + for i in range(len(value)): + dict_new[key+str(i)] = np.repeat(value[i], bag_size) return dict_new @staticmethod diff --git a/InnerEye/ML/Histopathology/utils/metrics_utils.py b/InnerEye/ML/Histopathology/utils/metrics_utils.py index c0d47a664..4e4a96832 100644 --- a/InnerEye/ML/Histopathology/utils/metrics_utils.py +++ b/InnerEye/ML/Histopathology/utils/metrics_utils.py @@ -35,7 +35,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: in :param return_col: column name of the values we want to return for each tile :return: tuple containing the slides id, the slide score, the tile ids, the tiles scores """ - tmp_s = [(results[prob_col][i], i) for i, gt in enumerate(results[gt_col]) if gt == label] # type ignore + tmp_s = [(results[prob_col][i][label], i) for i, gt in enumerate(results[gt_col]) if gt == label] # type ignore if select[0] == 'lowest_pred': tmp_s.sort(reverse=False) elif select[0] == 'highest_pred': @@ -58,7 +58,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: in scores.append(results[attn_col][slide_idx][0][t_idx]) # slide_ids are duplicated k_idx.append((results[slide_col][slide_idx][0], - results[prob_col][slide_idx].item(), + results[prob_col][slide_idx], k_tiles, scores)) return k_idx @@ -71,20 +71,23 @@ def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.PROB, :param gt_col: column name that contains the true label :return: matplotlib figure of the scores histogram by class """ - pos_scores = [results[prob_col][i][0].cpu().item() for i, gt in enumerate(results[gt_col]) if gt == 1] - neg_scores = [results[prob_col][i][0].cpu().item() for i, gt in enumerate(results[gt_col]) if gt == 0] - fig, ax = plt.subplots() - ax.hist([pos_scores, neg_scores], label=['1', '0'], alpha=0.5) + n_classes = len(results[prob_col][0]) + scores_class = [] + for j in range(n_classes): + scores = [results[prob_col][i][j].cpu().item() for i, gt in enumerate(results[gt_col]) if gt == j] + scores_class.append(scores) + fig, ax = plt.subplots() + ax.hist(scores_class, label=[str(i) for i in range(n_classes)], alpha=0.5) ax.set_xlabel("Predicted Score") ax.legend() return fig -def plot_attention_tiles(slide: str, score: float, paths: List, attn: List, case: str, ncols: int = 5, +def plot_attention_tiles(slide: str, score: List[float], paths: List, attn: List, case: str, ncols: int = 5, size: Tuple = (10, 10)) -> plt.figure: """ :param slide: slide identifier - :param score: predicted score for the slide + :param score: predicted scores of each class for the slide :param paths: list of paths to tiles belonging to the slide :param attn: list of scores belonging to the tiles in paths. paths and attn are expected to have the same shape :param case: string used to define the title of the plot e.g. TP @@ -94,7 +97,7 @@ def plot_attention_tiles(slide: str, score: float, paths: List, attn: List, case """ nrows = int(ceil(len(paths) / ncols)) fig, axs = plt.subplots(nrows=nrows, ncols=ncols, figsize=size) - fig.suptitle(f"{case}: {slide} P=%.2f" % score) + fig.suptitle(f"{case}: {slide} P=%.2f" % max(score)) for i in range(len(paths)): img = load_pil_image(paths[i]) axs.ravel()[i].imshow(img, clim=(0, 255), cmap='gray') diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py index 78d0bd1f1..17a1eca24 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py @@ -59,8 +59,8 @@ def __init__(self, **kwargs: Any) -> None: # declared in TrainerParams: num_epochs=50, # declared in WorkflowParams: - number_of_cross_validation_splits=5, - cross_validation_split_index=0, + number_of_cross_validation_splits=0, + cross_validation_split_index=-1, # declared in OptimizerParams: l_rate=5e-4, weight_decay=1e-4, diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 9296d6492..7dadcdb1c 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -61,8 +61,8 @@ def __init__(self, **kwargs: Any) -> None: # use_mixed_precision = True, # declared in WorkflowParams: - number_of_cross_validation_splits=5, - cross_validation_split_index=0, + number_of_cross_validation_splits=0, + cross_validation_split_index=-1, # declared in OptimizerParams: l_rate=5e-4, diff --git a/InnerEye/ML/configs/histo_configs/run_ids.py b/InnerEye/ML/configs/histo_configs/run_ids.py index 7692fc3e7..99550d563 100644 --- a/InnerEye/ML/configs/histo_configs/run_ids.py +++ b/InnerEye/ML/configs/histo_configs/run_ids.py @@ -1,6 +1,6 @@ innereye_ssl_checkpoint = "hsharma_panda_explore:hsharma_panda_explore_1638437076_357167ae" innereye_ssl_checkpoint_binary = "hsharma_panda_tiles_ssl:hsharma_panda_tiles_ssl_1639766433_161e03b9" -innereye_ssl_checkpoint_crck_4ws = "ModifyOldSSLCheckpoint:a9259fdb-3964-4c5b-8962-4660e0b79d44" +innereye_ssl_checkpoint_crck_4ws = "dev_histo:dev_histo_1645447795_643569ed" innereye_ssl_checkpoint_crck_radiomics = "ModifyOldSSLCheckpoint:704b1af8-7c75-46ed-8460-d80a0e603194" # outdated checkpoints diff --git a/hi-ml b/hi-ml index 2bc397b47..8cbb81387 160000 --- a/hi-ml +++ b/hi-ml @@ -1 +1 @@ -Subproject commit 2bc397b4707b56fecca624ce81e6883e0170b24b +Subproject commit 8cbb81387dbe8580fba533ad9344b4430d9a392b From 22b6ac30d99b38501cf28c3cfe366633f7c76a03 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Mar 2022 10:40:56 +0000 Subject: [PATCH 02/11] modify tests metrics_utils --- .../classification/DeepSMILECrck.py | 2 +- .../utils/test_metrics_utils.py | 69 ++++++++++++------- .../test_data/histo_heatmaps/score_hist.png | 4 +- 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py index 17a1eca24..181fe9585 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py @@ -96,7 +96,7 @@ def setup(self) -> None: self.downloader = CheckpointDownloader( azure_config_json_path=get_workspace(), run_id=innereye_ssl_checkpoint_crck_4ws, - checkpoint_filename="best_checkpoint.ckpt", + checkpoint_filename="last.ckpt", download_dir="outputs/", remote_checkpoint_dir=Path("outputs/checkpoints") ) diff --git a/Tests/ML/histopathology/utils/test_metrics_utils.py b/Tests/ML/histopathology/utils/test_metrics_utils.py index b1cc4c1ef..a79523800 100644 --- a/Tests/ML/histopathology/utils/test_metrics_utils.py +++ b/Tests/ML/histopathology/utils/test_metrics_utils.py @@ -31,6 +31,9 @@ def assert_equal_lists(pred: List, expected: List) -> None: for j, value in enumerate(slide): if type(value) in [int, float]: assert math.isclose(value, expected[i][j], rel_tol=1e-06) + elif (type(value) == Tensor) and (value.ndim >= 1): + for k, idx in enumerate(value): + assert math.isclose(idx, expected[i][j][k], rel_tol=1e-06) elif isinstance(value, List): for k, idx in enumerate(value): if type(idx) in [int, float]: @@ -41,15 +44,20 @@ def assert_equal_lists(pred: List, expected: List) -> None: raise TypeError("Unexpected list composition") -test_dict = {ResultsKey.SLIDE_ID: [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]], - ResultsKey.IMAGE_PATH: [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], - ResultsKey.PROB: [Tensor([0.5]), Tensor([0.7]), Tensor([0.4]), Tensor([1.0])], - ResultsKey.TRUE_LABEL: [0, 1, 1, 1], +test_dict = {ResultsKey.SLIDE_ID: [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]], + ResultsKey.IMAGE_PATH: [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], + ResultsKey.PROB: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), + Tensor([0.7, 0.3]), Tensor([0.8, 0.2]), Tensor([0.1, 0.9]), Tensor([0.01, 0.99])], + ResultsKey.TRUE_LABEL: [0, 1, 1, 1, 1, 0, 0, 0], ResultsKey.BAG_ATTN: - [Tensor([[0.1, 0.0, 0.2, 0.15]]), + [Tensor([[0.10, 0.00, 0.20, 0.15]]), Tensor([[0.10, 0.18, 0.15, 0.13]]), Tensor([[0.25, 0.23, 0.20, 0.21]]), - Tensor([[0.33, 0.31, 0.37, 0.35]])], + Tensor([[0.33, 0.31, 0.37, 0.35]]), + Tensor([[0.43, 0.01, 0.07, 0.25]]), + Tensor([[0.53, 0.11, 0.17, 0.55]]), + Tensor([[0.63, 0.21, 0.27, 0.05]]), + Tensor([[0.73, 0.31, 0.37, 0.15]])], ResultsKey.TILE_X: [Tensor([200, 200, 424, 424]), Tensor([200, 200, 424, 424]), @@ -64,27 +72,40 @@ def assert_equal_lists(pred: List, expected: List) -> None: def test_select_k_tiles() -> None: - top_tn = select_k_tiles(test_dict, n_slides=1, label=0, n_tiles=2, select=('lowest_pred', 'highest_att')) - assert_equal_lists(top_tn, [(1, 0.5, [3, 4], [Tensor([0.2]), Tensor([0.15])])]) - nslides = 2 ntiles = 2 + # TP + top_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('highest_pred', 'highest_att')) + bottom_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('highest_pred', 'lowest_att')) + print(top_tp) + assert_equal_lists(top_tp, [(4, Tensor([0.0, 1.0]), [3, 4], [Tensor([0.37]), Tensor([0.35])]), + (2, Tensor([0.3, 0.7]), [2, 3], [Tensor([0.18]), Tensor([0.15])])]) + assert_equal_lists(bottom_tp, [(4, Tensor([0.0, 1.0]), [2, 1], [Tensor([0.31]), Tensor([0.33])]), + (2, Tensor([0.3, 0.7]), [1, 4], [Tensor([0.10]), Tensor([0.13])])]) + + # FN top_fn = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('lowest_pred', 'highest_att')) - bottom_fn = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, - select=('lowest_pred', 'lowest_att')) - assert_equal_lists(top_fn, [(3, 0.4, [1, 2], [Tensor([0.25]), Tensor([0.23])]), - (2, 0.7, [2, 3], [Tensor([0.18]), Tensor([0.15])])]) - assert_equal_lists(bottom_fn, [(3, 0.4, [3, 4], [Tensor([0.20]), Tensor([0.21])]), - (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) - - top_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, - select=('highest_pred', 'highest_att')) - bottom_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, - select=('highest_pred', 'lowest_att')) - assert_equal_lists(top_tp, [(4, 1.0, [3, 4], [Tensor([0.37]), Tensor([0.35])]), - (2, 0.7, [2, 3], [Tensor([0.18]), Tensor([0.15])])]) - assert_equal_lists(bottom_tp, [(4, 1.0, [2, 1], [Tensor([0.31]), Tensor([0.33])]), - (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) + bottom_fn = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('lowest_pred', 'lowest_att')) + assert_equal_lists(top_fn, [(5, Tensor([0.7, 0.3]), [1, 4], [Tensor([0.43]), Tensor([0.25])]), + (3, Tensor([0.6, 0.4]), [1, 2], [Tensor([0.25]), Tensor([0.23])])]) + assert_equal_lists(bottom_fn, [(5, Tensor([0.7, 0.3]), [2, 3], [Tensor([0.01]), Tensor([0.07])]), + (3, Tensor([0.6, 0.4]), [3, 4], [Tensor([0.20]), Tensor([0.21])])]) + + # TN + top_tn = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('highest_pred', 'highest_att')) + bottom_tn = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('highest_pred', 'lowest_att')) + assert_equal_lists(top_tn, [(6, Tensor([0.8, 0.2]), [4, 1], [Tensor([0.55]), Tensor([0.53])]), + (1, Tensor([0.6, 0.4]), [3, 4], [Tensor([0.2]), Tensor([0.15])])]) + assert_equal_lists(bottom_tn, [(6, Tensor([0.8, 0.2]), [2, 3], [Tensor([0.11]), Tensor([0.17])]), + (1, Tensor([0.6, 0.4]), [2, 1], [Tensor([0.00]), Tensor([0.10])])]) + + # FP + top_tn = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('lowest_pred', 'highest_att')) + bottom_tn = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('lowest_pred', 'lowest_att')) + assert_equal_lists(top_tn, [(8, Tensor([0.01, 0.99]), [1, 3], [Tensor([0.73]), Tensor([0.37])]), + (7, Tensor([0.1, 0.9]), [1, 3], [Tensor([0.63]), Tensor([0.27])])]) + assert_equal_lists(bottom_tn, [(8, Tensor([0.01, 0.99]), [4, 2], [Tensor([0.15]), Tensor([0.31])]), + (7, Tensor([0.1, 0.9]), [4, 2], [Tensor([0.05]), Tensor([0.21])])]) @pytest.mark.skipif(is_windows(), reason="Rendering is different on Windows") diff --git a/Tests/ML/test_data/histo_heatmaps/score_hist.png b/Tests/ML/test_data/histo_heatmaps/score_hist.png index bced47d8b..bdc6d5b59 100644 --- a/Tests/ML/test_data/histo_heatmaps/score_hist.png +++ b/Tests/ML/test_data/histo_heatmaps/score_hist.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ca95c0017d0a51d75d118e54f21c1e907b3d90dcca822b23622e369267907198 -size 17057 +oid sha256:6ddc430ffcade51a072e9452833143840b1e5726148fd850ad3f370f1315bb32 +size 20452 From 2e30e943d6a4540b423fa6b328892c174723773e Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Mar 2022 10:58:16 +0000 Subject: [PATCH 03/11] restore extra files from main --- .../configs/histo_configs/classification/DeepSMILECrck.py | 6 +++--- .../configs/histo_configs/classification/DeepSMILEPanda.py | 4 ++-- InnerEye/ML/configs/histo_configs/run_ids.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py index 181fe9585..78d0bd1f1 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILECrck.py @@ -59,8 +59,8 @@ def __init__(self, **kwargs: Any) -> None: # declared in TrainerParams: num_epochs=50, # declared in WorkflowParams: - number_of_cross_validation_splits=0, - cross_validation_split_index=-1, + number_of_cross_validation_splits=5, + cross_validation_split_index=0, # declared in OptimizerParams: l_rate=5e-4, weight_decay=1e-4, @@ -96,7 +96,7 @@ def setup(self) -> None: self.downloader = CheckpointDownloader( azure_config_json_path=get_workspace(), run_id=innereye_ssl_checkpoint_crck_4ws, - checkpoint_filename="last.ckpt", + checkpoint_filename="best_checkpoint.ckpt", download_dir="outputs/", remote_checkpoint_dir=Path("outputs/checkpoints") ) diff --git a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py index 7dadcdb1c..9296d6492 100644 --- a/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py +++ b/InnerEye/ML/configs/histo_configs/classification/DeepSMILEPanda.py @@ -61,8 +61,8 @@ def __init__(self, **kwargs: Any) -> None: # use_mixed_precision = True, # declared in WorkflowParams: - number_of_cross_validation_splits=0, - cross_validation_split_index=-1, + number_of_cross_validation_splits=5, + cross_validation_split_index=0, # declared in OptimizerParams: l_rate=5e-4, diff --git a/InnerEye/ML/configs/histo_configs/run_ids.py b/InnerEye/ML/configs/histo_configs/run_ids.py index 99550d563..7692fc3e7 100644 --- a/InnerEye/ML/configs/histo_configs/run_ids.py +++ b/InnerEye/ML/configs/histo_configs/run_ids.py @@ -1,6 +1,6 @@ innereye_ssl_checkpoint = "hsharma_panda_explore:hsharma_panda_explore_1638437076_357167ae" innereye_ssl_checkpoint_binary = "hsharma_panda_tiles_ssl:hsharma_panda_tiles_ssl_1639766433_161e03b9" -innereye_ssl_checkpoint_crck_4ws = "dev_histo:dev_histo_1645447795_643569ed" +innereye_ssl_checkpoint_crck_4ws = "ModifyOldSSLCheckpoint:a9259fdb-3964-4c5b-8962-4660e0b79d44" innereye_ssl_checkpoint_crck_radiomics = "ModifyOldSSLCheckpoint:704b1af8-7c75-46ed-8460-d80a0e603194" # outdated checkpoints From 04671c604bca3a5b4491ae0252d23ae2561cb7d9 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Mar 2022 11:04:40 +0000 Subject: [PATCH 04/11] changelog --- CHANGELOG.md | 1 + hi-ml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c0d3520f..08d862099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ jobs that run in AzureML. - ([#653](https://github.com/microsoft/InnerEye-DeepLearning/pull/653)) Add dropout to DeepMIL and fix feature extractor setup. - ([#650](https://github.com/microsoft/InnerEye-DeepLearning/pull/650)) Enable fine-tuning in DeepMIL using PANDA as the classification task. - ([#656](https://github.com/microsoft/InnerEye-DeepLearning/pull/656)) Add subsampling transform and support for MIL mean pooling. +- ([#679](https://github.com/microsoft/InnerEye-DeepLearning/pull/679)) Add FP and TN slides/tiles to DeepMIL outputs and extend outputs to multi-class problems. ### Changed - ([#659](https://github.com/microsoft/InnerEye-DeepLearning/pull/659)) Update cudatoolkit version from 11.1 to 11.3. diff --git a/hi-ml b/hi-ml index 8cbb81387..30854eae4 160000 --- a/hi-ml +++ b/hi-ml @@ -1 +1 @@ -Subproject commit 8cbb81387dbe8580fba533ad9344b4430d9a392b +Subproject commit 30854eae4fd27776be9f0105099ddba663ef3eb5 From 8f6ff38b6349383cb9331c1c0952a9e020a60e82 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Mar 2022 12:15:19 +0000 Subject: [PATCH 05/11] separate PROB and PROB_CLASS keys --- InnerEye/ML/Histopathology/models/deepmil.py | 8 ++++---- InnerEye/ML/Histopathology/utils/naming.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index a118e0123..15c85888c 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -27,7 +27,7 @@ from health_ml.utils import log_on_epoch RESULTS_COLS = [ResultsKey.SLIDE_ID, ResultsKey.TILE_ID, ResultsKey.IMAGE_PATH, ResultsKey.PROB, - ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN] + ResultsKey.PROB_CLASS, ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN] def _format_cuda_memory_stats() -> str: @@ -245,7 +245,7 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK probs_perclass = predicted_probs else: predicted_labels = round(predicted_probs) - probs_perclass = Tensor([[1.0-predicted_probs[i][0].item(), predicted_probs[i][0].item()] for i in range(len(predicted_probs))]).cuda() + probs_perclass = Tensor([[1.0-predicted_probs[i][0].item(), predicted_probs[i][0].item()] for i in range(len(predicted_probs))]) loss = loss.view(-1, 1) predicted_labels = predicted_labels.view(-1, 1) @@ -258,7 +258,7 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, - ResultsKey.PROB: probs_perclass, ResultsKey.PRED_LABEL: predicted_labels, + ResultsKey.PROB: predicted_probs, ResultsKey.PROB_CLASS: probs_perclass, ResultsKey.PRED_LABEL: predicted_labels, ResultsKey.TRUE_LABEL: bag_labels, ResultsKey.BAG_ATTN: bag_attn_list, ResultsKey.IMAGE: batch[TilesDataset.IMAGE_COLUMN]}) @@ -417,7 +417,7 @@ def normalize_dict_for_df(dict_old: Dict[str, Any], use_gpu: bool) -> Dict: dict_new = dict() bag_size = len(dict_old[ResultsKey.SLIDE_ID]) for key, value in dict_old.items(): - if not key == ResultsKey.PROB: + if not key == ResultsKey.PROB_CLASS: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() if value.ndim == 0: diff --git a/InnerEye/ML/Histopathology/utils/naming.py b/InnerEye/ML/Histopathology/utils/naming.py index f499b7b51..fbc2b07fd 100644 --- a/InnerEye/ML/Histopathology/utils/naming.py +++ b/InnerEye/ML/Histopathology/utils/naming.py @@ -48,6 +48,7 @@ class ResultsKey(str, Enum): IMAGE_PATH = 'image_path' LOSS = 'loss' PROB = 'prob' + PROB_CLASS = 'prob_class' PRED_LABEL = 'pred_label' TRUE_LABEL = 'true_label' BAG_ATTN = 'bag_attn' From dd413d7dcb2cb1a6a489cccdd160e9e1d120f4fd Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Mar 2022 12:22:49 +0000 Subject: [PATCH 06/11] separate PROB and PROB_CLASS keys test and utils --- InnerEye/ML/Histopathology/models/deepmil.py | 3 ++- InnerEye/ML/Histopathology/utils/metrics_utils.py | 4 ++-- Tests/ML/histopathology/utils/test_metrics_utils.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 15c85888c..46d63abbc 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -258,7 +258,8 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, - ResultsKey.PROB: predicted_probs, ResultsKey.PROB_CLASS: probs_perclass, ResultsKey.PRED_LABEL: predicted_labels, + ResultsKey.PROB: predicted_probs, ResultsKey.PROB_CLASS: probs_perclass, + ResultsKey.PRED_LABEL: predicted_labels, ResultsKey.TRUE_LABEL: bag_labels, ResultsKey.BAG_ATTN: bag_attn_list, ResultsKey.IMAGE: batch[TilesDataset.IMAGE_COLUMN]}) diff --git a/InnerEye/ML/Histopathology/utils/metrics_utils.py b/InnerEye/ML/Histopathology/utils/metrics_utils.py index 4e4a96832..9c3df42a5 100644 --- a/InnerEye/ML/Histopathology/utils/metrics_utils.py +++ b/InnerEye/ML/Histopathology/utils/metrics_utils.py @@ -20,7 +20,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: int = 1, select: Tuple = ('lowest_pred', 'highest_att'), slide_col: str = ResultsKey.SLIDE_ID, gt_col: str = ResultsKey.TRUE_LABEL, - attn_col: str = ResultsKey.BAG_ATTN, prob_col: str = ResultsKey.PROB, + attn_col: str = ResultsKey.BAG_ATTN, prob_col: str = ResultsKey.PROB_CLASS, return_col: str = ResultsKey.IMAGE_PATH) -> List[Tuple[Any, Any, List[Any], List[Any]]]: """ :param results: List that contains slide_level dicts @@ -63,7 +63,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: in return k_idx -def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.PROB, +def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.PROB_CLASS, gt_col: str = ResultsKey.TRUE_LABEL) -> plt.figure: """ :param results: List that contains slide_level dicts diff --git a/Tests/ML/histopathology/utils/test_metrics_utils.py b/Tests/ML/histopathology/utils/test_metrics_utils.py index a79523800..73e82a226 100644 --- a/Tests/ML/histopathology/utils/test_metrics_utils.py +++ b/Tests/ML/histopathology/utils/test_metrics_utils.py @@ -46,7 +46,7 @@ def assert_equal_lists(pred: List, expected: List) -> None: test_dict = {ResultsKey.SLIDE_ID: [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]], ResultsKey.IMAGE_PATH: [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], - ResultsKey.PROB: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), + ResultsKey.PROB_CLASS: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), Tensor([0.7, 0.3]), Tensor([0.8, 0.2]), Tensor([0.1, 0.9]), Tensor([0.01, 0.99])], ResultsKey.TRUE_LABEL: [0, 1, 1, 1, 1, 0, 0, 0], ResultsKey.BAG_ATTN: From 74af2922c46f900de82659d6f557c08b3d5f33af Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Tue, 1 Mar 2022 13:46:06 +0000 Subject: [PATCH 07/11] correct metrics multiclass --- InnerEye/ML/Histopathology/models/deepmil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 46d63abbc..1eb8578d5 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -254,7 +254,10 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results = dict() for metric_object in self.get_metrics_dict(stage).values(): - metric_object.update(predicted_probs, bag_labels) + if self.n_classes > 1: + metric_object.update(predicted_labels, bag_labels) + else: + metric_object.update(predicted_probs, bag_labels) results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, From 540a1cda6baf7af8442d476e72136c8fb1f60568 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Thu, 3 Mar 2022 15:17:03 +0000 Subject: [PATCH 08/11] PR comments addressed --- InnerEye/ML/Histopathology/models/deepmil.py | 30 ++++++++----------- .../ML/Histopathology/utils/metrics_utils.py | 10 +++---- InnerEye/ML/Histopathology/utils/naming.py | 2 +- .../utils/test_metrics_utils.py | 10 +++---- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 1eb8578d5..7b5903163 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -27,7 +27,7 @@ from health_ml.utils import log_on_epoch RESULTS_COLS = [ResultsKey.SLIDE_ID, ResultsKey.TILE_ID, ResultsKey.IMAGE_PATH, ResultsKey.PROB, - ResultsKey.PROB_CLASS, ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN] + ResultsKey.CLASS_PROB, ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN] def _format_cuda_memory_stats() -> str: @@ -245,11 +245,11 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK probs_perclass = predicted_probs else: predicted_labels = round(predicted_probs) - probs_perclass = Tensor([[1.0-predicted_probs[i][0].item(), predicted_probs[i][0].item()] for i in range(len(predicted_probs))]) + probs_perclass = Tensor([[1.0 - predicted_probs[i][0].item(), predicted_probs[i][0].item()] for i in range(len(predicted_probs))]) loss = loss.view(-1, 1) predicted_labels = predicted_labels.view(-1, 1) - predicted_probs = predicted_probs.view(-1, 1) + # predicted_probs = predicted_probs.view(-1, 1) bag_labels = bag_labels.view(-1, 1) results = dict() @@ -261,7 +261,7 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, - ResultsKey.PROB: predicted_probs, ResultsKey.PROB_CLASS: probs_perclass, + ResultsKey.PROB: predicted_probs, ResultsKey.CLASS_PROB: probs_perclass, ResultsKey.PRED_LABEL: predicted_labels, ResultsKey.TRUE_LABEL: bag_labels, ResultsKey.BAG_ATTN: bag_attn_list, ResultsKey.IMAGE: batch[TilesDataset.IMAGE_COLUMN]}) @@ -353,19 +353,13 @@ def test_epoch_end(self, outputs: List[Dict[str, Any]]) -> None: # type: ignore report_cases = {'TN': [tn_top_tiles, tn_bottom_tiles], 'FP': [fp_top_tiles, fp_bottom_tiles]} # Class 1 to n_classes-1 - if self.n_classes > 1: - for i in range(1, self.n_classes): - fn_top_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('lowest_pred', 'highest_att')) - fn_bottom_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('lowest_pred', 'lowest_att')) - tp_top_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('highest_pred', 'highest_att')) - tp_bottom_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('highest_pred', 'lowest_att')) - report_cases.update({'TP_'+str(i): [tp_top_tiles, tp_bottom_tiles], 'FN_'+str(i): [fn_top_tiles, fn_bottom_tiles]}) - else: - fn_top_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('lowest_pred', 'highest_att')) - fn_bottom_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('lowest_pred', 'lowest_att')) - tp_top_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('highest_pred', 'highest_att')) - tp_bottom_tiles = select_k_tiles(results, n_slides=10, label=1, n_tiles=10, select=('highest_pred', 'lowest_att')) - report_cases.update({'TP_1': [tp_top_tiles, tp_bottom_tiles], 'FN_1': [fn_top_tiles, fn_bottom_tiles]}) + n_classes_to_select = self.n_classes if self.n_classes > 1 else 2 + for i in range(1, n_classes_to_select): + fn_top_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('lowest_pred', 'highest_att')) + fn_bottom_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('lowest_pred', 'lowest_att')) + tp_top_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('highest_pred', 'highest_att')) + tp_bottom_tiles = select_k_tiles(results, n_slides=10, label=i, n_tiles=10, select=('highest_pred', 'lowest_att')) + report_cases.update({'TP_'+str(i): [tp_top_tiles, tp_bottom_tiles], 'FN_'+str(i): [fn_top_tiles, fn_bottom_tiles]}) for key in report_cases.keys(): print(f"Plotting {key} (tiles, thumbnails, attention heatmaps)...") @@ -421,7 +415,7 @@ def normalize_dict_for_df(dict_old: Dict[str, Any], use_gpu: bool) -> Dict: dict_new = dict() bag_size = len(dict_old[ResultsKey.SLIDE_ID]) for key, value in dict_old.items(): - if not key == ResultsKey.PROB_CLASS: + if not key == ResultsKey.CLASS_PROB: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() if value.ndim == 0: diff --git a/InnerEye/ML/Histopathology/utils/metrics_utils.py b/InnerEye/ML/Histopathology/utils/metrics_utils.py index 9c3df42a5..5c854fb4e 100644 --- a/InnerEye/ML/Histopathology/utils/metrics_utils.py +++ b/InnerEye/ML/Histopathology/utils/metrics_utils.py @@ -20,7 +20,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: int = 1, select: Tuple = ('lowest_pred', 'highest_att'), slide_col: str = ResultsKey.SLIDE_ID, gt_col: str = ResultsKey.TRUE_LABEL, - attn_col: str = ResultsKey.BAG_ATTN, prob_col: str = ResultsKey.PROB_CLASS, + attn_col: str = ResultsKey.BAG_ATTN, prob_col: str = ResultsKey.CLASS_PROB, return_col: str = ResultsKey.IMAGE_PATH) -> List[Tuple[Any, Any, List[Any], List[Any]]]: """ :param results: List that contains slide_level dicts @@ -63,7 +63,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: in return k_idx -def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.PROB_CLASS, +def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.CLASS_PROB, gt_col: str = ResultsKey.TRUE_LABEL) -> plt.figure: """ :param results: List that contains slide_level dicts @@ -83,11 +83,11 @@ def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.PROB_CLASS, return fig -def plot_attention_tiles(slide: str, score: List[float], paths: List, attn: List, case: str, ncols: int = 5, +def plot_attention_tiles(slide: str, scores: List[float], paths: List, attn: List, case: str, ncols: int = 5, size: Tuple = (10, 10)) -> plt.figure: """ :param slide: slide identifier - :param score: predicted scores of each class for the slide + :param scores: predicted scores of each class for the slide :param paths: list of paths to tiles belonging to the slide :param attn: list of scores belonging to the tiles in paths. paths and attn are expected to have the same shape :param case: string used to define the title of the plot e.g. TP @@ -97,7 +97,7 @@ def plot_attention_tiles(slide: str, score: List[float], paths: List, attn: List """ nrows = int(ceil(len(paths) / ncols)) fig, axs = plt.subplots(nrows=nrows, ncols=ncols, figsize=size) - fig.suptitle(f"{case}: {slide} P=%.2f" % max(score)) + fig.suptitle(f"{case}: {slide} P=%.2f" % max(scores)) for i in range(len(paths)): img = load_pil_image(paths[i]) axs.ravel()[i].imshow(img, clim=(0, 255), cmap='gray') diff --git a/InnerEye/ML/Histopathology/utils/naming.py b/InnerEye/ML/Histopathology/utils/naming.py index fbc2b07fd..5c065bc18 100644 --- a/InnerEye/ML/Histopathology/utils/naming.py +++ b/InnerEye/ML/Histopathology/utils/naming.py @@ -48,7 +48,7 @@ class ResultsKey(str, Enum): IMAGE_PATH = 'image_path' LOSS = 'loss' PROB = 'prob' - PROB_CLASS = 'prob_class' + CLASS_PROB = 'prob_class' PRED_LABEL = 'pred_label' TRUE_LABEL = 'true_label' BAG_ATTN = 'bag_attn' diff --git a/Tests/ML/histopathology/utils/test_metrics_utils.py b/Tests/ML/histopathology/utils/test_metrics_utils.py index 73e82a226..0ebcbf31e 100644 --- a/Tests/ML/histopathology/utils/test_metrics_utils.py +++ b/Tests/ML/histopathology/utils/test_metrics_utils.py @@ -46,7 +46,7 @@ def assert_equal_lists(pred: List, expected: List) -> None: test_dict = {ResultsKey.SLIDE_ID: [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]], ResultsKey.IMAGE_PATH: [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], - ResultsKey.PROB_CLASS: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), + ResultsKey.CLASS_PROB: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), Tensor([0.7, 0.3]), Tensor([0.8, 0.2]), Tensor([0.1, 0.9]), Tensor([0.01, 0.99])], ResultsKey.TRUE_LABEL: [0, 1, 1, 1, 1, 0, 0, 0], ResultsKey.BAG_ATTN: @@ -100,11 +100,11 @@ def test_select_k_tiles() -> None: (1, Tensor([0.6, 0.4]), [2, 1], [Tensor([0.00]), Tensor([0.10])])]) # FP - top_tn = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('lowest_pred', 'highest_att')) - bottom_tn = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('lowest_pred', 'lowest_att')) - assert_equal_lists(top_tn, [(8, Tensor([0.01, 0.99]), [1, 3], [Tensor([0.73]), Tensor([0.37])]), + top_fp = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('lowest_pred', 'highest_att')) + bottom_fp = select_k_tiles(test_dict, n_slides=nslides, label=0, n_tiles=ntiles, select=('lowest_pred', 'lowest_att')) + assert_equal_lists(top_fp, [(8, Tensor([0.01, 0.99]), [1, 3], [Tensor([0.73]), Tensor([0.37])]), (7, Tensor([0.1, 0.9]), [1, 3], [Tensor([0.63]), Tensor([0.27])])]) - assert_equal_lists(bottom_tn, [(8, Tensor([0.01, 0.99]), [4, 2], [Tensor([0.15]), Tensor([0.31])]), + assert_equal_lists(bottom_fp, [(8, Tensor([0.01, 0.99]), [4, 2], [Tensor([0.15]), Tensor([0.31])]), (7, Tensor([0.1, 0.9]), [4, 2], [Tensor([0.05]), Tensor([0.21])])]) From eb51db152919c2b99e920a5eed193392190da557 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Fri, 4 Mar 2022 13:14:32 +0000 Subject: [PATCH 09/11] attempt multiclass bug fix --- InnerEye/ML/Histopathology/models/deepmil.py | 12 +++++------- hi-ml | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 7b5903163..56d23f8b7 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -249,15 +249,13 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK loss = loss.view(-1, 1) predicted_labels = predicted_labels.view(-1, 1) - # predicted_probs = predicted_probs.view(-1, 1) + if self.n_classes == 1: + predicted_probs = predicted_probs.view(-1, 1) bag_labels = bag_labels.view(-1, 1) results = dict() for metric_object in self.get_metrics_dict(stage).values(): - if self.n_classes > 1: - metric_object.update(predicted_labels, bag_labels) - else: - metric_object.update(predicted_probs, bag_labels) + metric_object.update(predicted_probs, bag_labels.squeeze()) results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, @@ -415,13 +413,13 @@ def normalize_dict_for_df(dict_old: Dict[str, Any], use_gpu: bool) -> Dict: dict_new = dict() bag_size = len(dict_old[ResultsKey.SLIDE_ID]) for key, value in dict_old.items(): - if not key == ResultsKey.CLASS_PROB: + if not key in [ResultsKey.CLASS_PROB, ResultsKey.PROB]: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() if value.ndim == 0: value = np.full(bag_size, fill_value=value) dict_new[key] = value - else: + elif key == ResultsKey.CLASS_PROB: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() for i in range(len(value)): diff --git a/hi-ml b/hi-ml index 30854eae4..0250715c5 160000 --- a/hi-ml +++ b/hi-ml @@ -1 +1 @@ -Subproject commit 30854eae4fd27776be9f0105099ddba663ef3eb5 +Subproject commit 0250715c5ac1ef09227b51388df44b568a496f65 From 72e27b549ab2981e81cd3f1633bf359e32c7f04a Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Fri, 4 Mar 2022 13:28:23 +0000 Subject: [PATCH 10/11] attempt2 multiclass bug fix --- InnerEye/ML/Histopathology/models/deepmil.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 56d23f8b7..8657cf4ca 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -255,7 +255,10 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results = dict() for metric_object in self.get_metrics_dict(stage).values(): - metric_object.update(predicted_probs, bag_labels.squeeze()) + if self.n_classes > 1: + metric_object.update(predicted_probs, bag_labels.squeeze()) + else: + metric_object.update(predicted_probs, bag_labels) results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, @@ -413,7 +416,7 @@ def normalize_dict_for_df(dict_old: Dict[str, Any], use_gpu: bool) -> Dict: dict_new = dict() bag_size = len(dict_old[ResultsKey.SLIDE_ID]) for key, value in dict_old.items(): - if not key in [ResultsKey.CLASS_PROB, ResultsKey.PROB]: + if key not in [ResultsKey.CLASS_PROB, ResultsKey.PROB]: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() if value.ndim == 0: From 8dc9ed3bafd27d131801da4d00010725fd32f420 Mon Sep 17 00:00:00 2001 From: Harshita Sharma Date: Fri, 4 Mar 2022 15:47:47 +0000 Subject: [PATCH 11/11] CLASS_PROB to CLASS_PROBS --- InnerEye/ML/Histopathology/models/deepmil.py | 8 ++++---- InnerEye/ML/Histopathology/utils/metrics_utils.py | 4 ++-- InnerEye/ML/Histopathology/utils/naming.py | 2 +- Tests/ML/histopathology/utils/test_metrics_utils.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/InnerEye/ML/Histopathology/models/deepmil.py b/InnerEye/ML/Histopathology/models/deepmil.py index 8657cf4ca..692033bc9 100644 --- a/InnerEye/ML/Histopathology/models/deepmil.py +++ b/InnerEye/ML/Histopathology/models/deepmil.py @@ -27,7 +27,7 @@ from health_ml.utils import log_on_epoch RESULTS_COLS = [ResultsKey.SLIDE_ID, ResultsKey.TILE_ID, ResultsKey.IMAGE_PATH, ResultsKey.PROB, - ResultsKey.CLASS_PROB, ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN] + ResultsKey.CLASS_PROBS, ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN] def _format_cuda_memory_stats() -> str: @@ -262,7 +262,7 @@ def _shared_step(self, batch: Dict, batch_idx: int, stage: str) -> Dict[ResultsK results.update({ResultsKey.SLIDE_ID: batch[TilesDataset.SLIDE_ID_COLUMN], ResultsKey.TILE_ID: batch[TilesDataset.TILE_ID_COLUMN], ResultsKey.IMAGE_PATH: batch[TilesDataset.PATH_COLUMN], ResultsKey.LOSS: loss, - ResultsKey.PROB: predicted_probs, ResultsKey.CLASS_PROB: probs_perclass, + ResultsKey.PROB: predicted_probs, ResultsKey.CLASS_PROBS: probs_perclass, ResultsKey.PRED_LABEL: predicted_labels, ResultsKey.TRUE_LABEL: bag_labels, ResultsKey.BAG_ATTN: bag_attn_list, ResultsKey.IMAGE: batch[TilesDataset.IMAGE_COLUMN]}) @@ -416,13 +416,13 @@ def normalize_dict_for_df(dict_old: Dict[str, Any], use_gpu: bool) -> Dict: dict_new = dict() bag_size = len(dict_old[ResultsKey.SLIDE_ID]) for key, value in dict_old.items(): - if key not in [ResultsKey.CLASS_PROB, ResultsKey.PROB]: + if key not in [ResultsKey.CLASS_PROBS, ResultsKey.PROB]: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() if value.ndim == 0: value = np.full(bag_size, fill_value=value) dict_new[key] = value - elif key == ResultsKey.CLASS_PROB: + elif key == ResultsKey.CLASS_PROBS: if isinstance(value, Tensor): value = value.squeeze(0).to(device).numpy() for i in range(len(value)): diff --git a/InnerEye/ML/Histopathology/utils/metrics_utils.py b/InnerEye/ML/Histopathology/utils/metrics_utils.py index 5c854fb4e..a966e4cf5 100644 --- a/InnerEye/ML/Histopathology/utils/metrics_utils.py +++ b/InnerEye/ML/Histopathology/utils/metrics_utils.py @@ -20,7 +20,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: int = 1, select: Tuple = ('lowest_pred', 'highest_att'), slide_col: str = ResultsKey.SLIDE_ID, gt_col: str = ResultsKey.TRUE_LABEL, - attn_col: str = ResultsKey.BAG_ATTN, prob_col: str = ResultsKey.CLASS_PROB, + attn_col: str = ResultsKey.BAG_ATTN, prob_col: str = ResultsKey.CLASS_PROBS, return_col: str = ResultsKey.IMAGE_PATH) -> List[Tuple[Any, Any, List[Any], List[Any]]]: """ :param results: List that contains slide_level dicts @@ -63,7 +63,7 @@ def select_k_tiles(results: Dict, n_tiles: int = 5, n_slides: int = 5, label: in return k_idx -def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.CLASS_PROB, +def plot_scores_hist(results: Dict, prob_col: str = ResultsKey.CLASS_PROBS, gt_col: str = ResultsKey.TRUE_LABEL) -> plt.figure: """ :param results: List that contains slide_level dicts diff --git a/InnerEye/ML/Histopathology/utils/naming.py b/InnerEye/ML/Histopathology/utils/naming.py index 5c065bc18..04dc01dc9 100644 --- a/InnerEye/ML/Histopathology/utils/naming.py +++ b/InnerEye/ML/Histopathology/utils/naming.py @@ -48,7 +48,7 @@ class ResultsKey(str, Enum): IMAGE_PATH = 'image_path' LOSS = 'loss' PROB = 'prob' - CLASS_PROB = 'prob_class' + CLASS_PROBS = 'prob_class' PRED_LABEL = 'pred_label' TRUE_LABEL = 'true_label' BAG_ATTN = 'bag_attn' diff --git a/Tests/ML/histopathology/utils/test_metrics_utils.py b/Tests/ML/histopathology/utils/test_metrics_utils.py index 0ebcbf31e..9db87a914 100644 --- a/Tests/ML/histopathology/utils/test_metrics_utils.py +++ b/Tests/ML/histopathology/utils/test_metrics_utils.py @@ -46,7 +46,7 @@ def assert_equal_lists(pred: List, expected: List) -> None: test_dict = {ResultsKey.SLIDE_ID: [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]], ResultsKey.IMAGE_PATH: [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]], - ResultsKey.CLASS_PROB: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), + ResultsKey.CLASS_PROBS: [Tensor([0.6, 0.4]), Tensor([0.3, 0.7]), Tensor([0.6, 0.4]), Tensor([0.0, 1.0]), Tensor([0.7, 0.3]), Tensor([0.8, 0.2]), Tensor([0.1, 0.9]), Tensor([0.01, 0.99])], ResultsKey.TRUE_LABEL: [0, 1, 1, 1, 1, 0, 0, 0], ResultsKey.BAG_ATTN: