From c848bf6b34105fac32e723cc7bca5dff90eb6883 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Tue, 1 Feb 2022 15:32:41 +0000 Subject: [PATCH 01/30] fix --- azure-pipelines/build-pr.yml | 8 ++++---- azure-pipelines/build.yaml | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index da6454f42..1fbd2abcd 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -54,19 +54,19 @@ jobs: pytest_mark: after_training_single_run test_run_title: tests_after_training_single_run - - job: RunGpuTestsInAzureML + - job: RunAllTestsInAzureML dependsOn: CancelPreviousJobs variables: - name: tag - value: 'RunGpuTests' + value: 'RunAllTests' pool: vmImage: 'ubuntu-18.04' steps: - template: train_template.yml parameters: wait_for_completion: 'True' - pytest_mark: 'gpu or cpu_and_gpu or azureml' - max_run_duration: '30m' + pytest_mark: 'not (after_training_single_run or after_training_ensemble_run or inference or after_training_2node or after_training_glaucoma_cv_run or after_training_hello_container)' + max_run_duration: '40m' - task: PublishTestResults@2 inputs: testResultsFiles: '**/test-*.xml' diff --git a/azure-pipelines/build.yaml b/azure-pipelines/build.yaml index e98a98920..5bd63e341 100644 --- a/azure-pipelines/build.yaml +++ b/azure-pipelines/build.yaml @@ -30,18 +30,18 @@ steps: condition: succeededOrFailed() displayName: Install InnerEye (Dev) Package - # First run all tests that require the InnerEye package. All code should be consumed via the InnerEye package, - # hence don't set PYTHONPATH to InnerEye - but do set it to hi-ml if that has been included as a submodule for dev - # work on the package - - bash: | - source activate InnerEye - pytest ./Tests/ -m "not (gpu or azureml or after_training_single_run or after_training_ensemble_run or inference or after_training_2node or after_training_glaucoma_cv_run or after_training_hello_container)" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-config=.coveragerc --cov-report=xml --verbose - env: - APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey) - DATASETS_ACCOUNT_KEY: $(InnerEyePublicDatasetsStorageKey) - failOnStderr: false - condition: succeededOrFailed() - displayName: Run pytests on InnerEye package + # # First run all tests that require the InnerEye package. All code should be consumed via the InnerEye package, + # # hence don't set PYTHONPATH to InnerEye - but do set it to hi-ml if that has been included as a submodule for dev + # # work on the package + # - bash: | + # source activate InnerEye + # pytest ./Tests/ -m "not (gpu or azureml or after_training_single_run or after_training_ensemble_run or inference or after_training_2node or after_training_glaucoma_cv_run or after_training_hello_container)" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-config=.coveragerc --cov-report=xml --verbose + # env: + # APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey) + # DATASETS_ACCOUNT_KEY: $(InnerEyePublicDatasetsStorageKey) + # failOnStderr: false + # condition: succeededOrFailed() + # displayName: Run pytests on InnerEye package # Run all tests for code that does not live in the InnerEye package. For that, set PYTHONPATH - bash: | From 6317a1a3139cfcc59184e0966426a5c6cd165b67 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Tue, 1 Feb 2022 15:50:50 +0000 Subject: [PATCH 02/30] rename workflows --- .github/workflows/check_changelog.yml | 3 ++- .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/issues_to_ado.yml | 3 ++- .github/workflows/linting_and_hello_world.yml | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml index 7cced82dc..0ca7fb52e 100644 --- a/.github/workflows/check_changelog.yml +++ b/.github/workflows/check_changelog.yml @@ -6,7 +6,8 @@ name: Check Changelog on: pull_request: jobs: - check: + check_changelog: + name: Check Changelog runs-on: ubuntu-latest if: ${{ contains(github.event.pull_request.labels.*.name, 'no changelog needed') == 0 }} steps: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5a81e8493..592674cb2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -10,8 +10,8 @@ on: - cron: '45 4 * * 1' jobs: - analyze: - name: Analyze + codeql_analyze: + name: CodeQL Analyze runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/issues_to_ado.yml b/.github/workflows/issues_to_ado.yml index ec24f968f..e7a2728f0 100644 --- a/.github/workflows/issues_to_ado.yml +++ b/.github/workflows/issues_to_ado.yml @@ -6,7 +6,8 @@ on: [opened, edited, deleted, closed, reopened, labeled, unlabeled, assigned] jobs: - alert: + issues_to_ado: + name: Sync issues with Azure DevOps runs-on: ubuntu-latest steps: - uses: danhellem/github-actions-issue-to-work-item@master diff --git a/.github/workflows/linting_and_hello_world.yml b/.github/workflows/linting_and_hello_world.yml index 9ed6405bd..b1a5c3410 100644 --- a/.github/workflows/linting_and_hello_world.yml +++ b/.github/workflows/linting_and_hello_world.yml @@ -7,7 +7,8 @@ on: pull_request: jobs: - linux: + flake_mypy_helloworld_linux: + name: Flake8, MyPy, HelloWorld on Linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -53,7 +54,8 @@ jobs: PYTHONPATH: ${{ github.workspace }} if: always() - windows: + hello_world_windows: + name: HelloWorld on Windows runs-on: windows-latest steps: - uses: actions/checkout@v2 From fb0901b6b07baed12825033897f1702bea75f911 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 09:27:25 +0000 Subject: [PATCH 03/30] Revert "fix" This reverts commit c848bf6b34105fac32e723cc7bca5dff90eb6883. --- azure-pipelines/build-pr.yml | 8 ++++---- azure-pipelines/build.yaml | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index 1fbd2abcd..da6454f42 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -54,19 +54,19 @@ jobs: pytest_mark: after_training_single_run test_run_title: tests_after_training_single_run - - job: RunAllTestsInAzureML + - job: RunGpuTestsInAzureML dependsOn: CancelPreviousJobs variables: - name: tag - value: 'RunAllTests' + value: 'RunGpuTests' pool: vmImage: 'ubuntu-18.04' steps: - template: train_template.yml parameters: wait_for_completion: 'True' - pytest_mark: 'not (after_training_single_run or after_training_ensemble_run or inference or after_training_2node or after_training_glaucoma_cv_run or after_training_hello_container)' - max_run_duration: '40m' + pytest_mark: 'gpu or cpu_and_gpu or azureml' + max_run_duration: '30m' - task: PublishTestResults@2 inputs: testResultsFiles: '**/test-*.xml' diff --git a/azure-pipelines/build.yaml b/azure-pipelines/build.yaml index 5bd63e341..e98a98920 100644 --- a/azure-pipelines/build.yaml +++ b/azure-pipelines/build.yaml @@ -30,18 +30,18 @@ steps: condition: succeededOrFailed() displayName: Install InnerEye (Dev) Package - # # First run all tests that require the InnerEye package. All code should be consumed via the InnerEye package, - # # hence don't set PYTHONPATH to InnerEye - but do set it to hi-ml if that has been included as a submodule for dev - # # work on the package - # - bash: | - # source activate InnerEye - # pytest ./Tests/ -m "not (gpu or azureml or after_training_single_run or after_training_ensemble_run or inference or after_training_2node or after_training_glaucoma_cv_run or after_training_hello_container)" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-config=.coveragerc --cov-report=xml --verbose - # env: - # APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey) - # DATASETS_ACCOUNT_KEY: $(InnerEyePublicDatasetsStorageKey) - # failOnStderr: false - # condition: succeededOrFailed() - # displayName: Run pytests on InnerEye package + # First run all tests that require the InnerEye package. All code should be consumed via the InnerEye package, + # hence don't set PYTHONPATH to InnerEye - but do set it to hi-ml if that has been included as a submodule for dev + # work on the package + - bash: | + source activate InnerEye + pytest ./Tests/ -m "not (gpu or azureml or after_training_single_run or after_training_ensemble_run or inference or after_training_2node or after_training_glaucoma_cv_run or after_training_hello_container)" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-config=.coveragerc --cov-report=xml --verbose + env: + APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey) + DATASETS_ACCOUNT_KEY: $(InnerEyePublicDatasetsStorageKey) + failOnStderr: false + condition: succeededOrFailed() + displayName: Run pytests on InnerEye package # Run all tests for code that does not live in the InnerEye package. For that, set PYTHONPATH - bash: | From 88b0f96f993c5549b79e2f90bb585581cc67bbd5 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 09:55:42 +0000 Subject: [PATCH 04/30] mark linux-only tests --- .../Histopathology/datasets/panda_dataset.py | 7 +- .../Histopathology/preprocessing/loading.py | 7 +- Tests/Azure/test_azure_config.py | 7 +- .../preprocessing/test_slide_loading.py | 18 ++-- .../utils/test_metrics_utils.py | 84 +++++++++++-------- 5 files changed, 72 insertions(+), 51 deletions(-) diff --git a/InnerEye/ML/Histopathology/datasets/panda_dataset.py b/InnerEye/ML/Histopathology/datasets/panda_dataset.py index b84571257..b9795ad68 100644 --- a/InnerEye/ML/Histopathology/datasets/panda_dataset.py +++ b/InnerEye/ML/Histopathology/datasets/panda_dataset.py @@ -7,7 +7,6 @@ from typing import Any, Dict, Union, Optional import pandas as pd -from cucim import CuImage from health_ml.utils import box_utils from monai.config import KeysCollection from monai.data.image_reader import ImageReader, WSIReader @@ -88,7 +87,7 @@ def __init__(self, reader: WSIReader, image_key: str = 'image', mask_key: str = self.margin = margin self.kwargs = kwargs - def _get_bounding_box(self, mask_obj: CuImage) -> box_utils.Box: + def _get_bounding_box(self, mask_obj: 'CuImage') -> box_utils.Box: # Estimate bounding box at the lowest resolution (i.e. highest level) highest_level = mask_obj.resolutions['level_count'] - 1 scale = mask_obj.resolutions['level_downsamples'][highest_level] @@ -99,8 +98,8 @@ def _get_bounding_box(self, mask_obj: CuImage) -> box_utils.Box: return bbox def __call__(self, data: Dict) -> Dict: - mask_obj: CuImage = self.reader.read(data[self.mask_key]) - image_obj: CuImage = self.reader.read(data[self.image_key]) + mask_obj: 'CuImage' = self.reader.read(data[self.mask_key]) + image_obj: 'CuImage' = self.reader.read(data[self.image_key]) level0_bbox = self._get_bounding_box(mask_obj) diff --git a/InnerEye/ML/Histopathology/preprocessing/loading.py b/InnerEye/ML/Histopathology/preprocessing/loading.py index 77942e555..48eadb8a6 100644 --- a/InnerEye/ML/Histopathology/preprocessing/loading.py +++ b/InnerEye/ML/Histopathology/preprocessing/loading.py @@ -2,7 +2,6 @@ import numpy as np import skimage.filters -from cucim import CuImage from health_ml.utils import box_utils from monai.data.image_reader import WSIReader from monai.transforms import MapTransform @@ -35,7 +34,7 @@ def segment_foreground(slide: np.ndarray, threshold: Optional[float] = None) \ return luminance < threshold, threshold -def load_slide_at_level(reader: WSIReader, slide_obj: CuImage, level: int) -> np.ndarray: +def load_slide_at_level(reader: WSIReader, slide_obj: 'CuImage', level: int) -> np.ndarray: """Load full slide array at the given magnification level. This is a manual workaround for a MONAI bug (https://github.com/Project-MONAI/MONAI/issues/3415) @@ -77,7 +76,7 @@ def __init__(self, reader: WSIReader, image_key: str = SlideKey.IMAGE, level: in self.margin = margin self.foreground_threshold = foreground_threshold - def _get_bounding_box(self, slide_obj: CuImage) -> Tuple[box_utils.Box, float]: + def _get_bounding_box(self, slide_obj: 'CuImage') -> Tuple[box_utils.Box, float]: # Estimate bounding box at the lowest resolution (i.e. highest level) highest_level = slide_obj.resolutions['level_count'] - 1 scale = slide_obj.resolutions['level_downsamples'][highest_level] @@ -88,7 +87,7 @@ def _get_bounding_box(self, slide_obj: CuImage) -> Tuple[box_utils.Box, float]: return bbox, threshold def __call__(self, data: Dict) -> Dict: - image_obj: CuImage = self.reader.read(data[self.image_key]) + image_obj: 'CuImage' = self.reader.read(data[self.image_key]) level0_bbox, threshold = self._get_bounding_box(image_obj) diff --git a/Tests/Azure/test_azure_config.py b/Tests/Azure/test_azure_config.py index b889779f7..6a375eb30 100644 --- a/Tests/Azure/test_azure_config.py +++ b/Tests/Azure/test_azure_config.py @@ -9,6 +9,7 @@ from InnerEye.Azure.azure_config import AzureConfig from InnerEye.Azure.azure_runner import create_dataset_configs +from InnerEye.Common.common_util import is_linux from InnerEye.ML.deep_learning_config import DatasetParams from Tests.ML.util import get_default_azure_config @@ -65,8 +66,10 @@ def test_dataset_consumption2() -> None: assert datasets[1].name == "2" assert datasets[0].local_folder == Path("l1") assert datasets[1].local_folder == Path("l2") - assert datasets[0].target_folder == PosixPath("mp1") - assert datasets[1].target_folder == PosixPath("mp2") + if is_linux(): + # PosixPath cannot be instantiated on Windows + assert datasets[0].target_folder == PosixPath("mp1") + assert datasets[1].target_folder == PosixPath("mp2") def test_dataset_consumption3() -> None: diff --git a/Tests/ML/histopathology/preprocessing/test_slide_loading.py b/Tests/ML/histopathology/preprocessing/test_slide_loading.py index 60d9717ed..95bc2e743 100644 --- a/Tests/ML/histopathology/preprocessing/test_slide_loading.py +++ b/Tests/ML/histopathology/preprocessing/test_slide_loading.py @@ -2,22 +2,24 @@ import numpy as np import pytest -from cucim import CuImage from monai.data.image_reader import WSIReader +from InnerEye.Common.common_util import is_windows from InnerEye.Common.fixed_paths_for_tests import tests_root_directory from InnerEye.ML.Histopathology.preprocessing.tiling import tile_array_2d -from InnerEye.ML.Histopathology.preprocessing.loading import LoadROId, get_luminance, load_slide_at_level, segment_foreground +from InnerEye.ML.Histopathology.preprocessing.loading import (LoadROId, get_luminance, load_slide_at_level, + segment_foreground) from InnerEye.ML.Histopathology.utils.naming import SlideKey from Tests.ML.histopathology.datasets.test_slides_dataset import MockSlidesDataset TEST_IMAGE_PATH = str(tests_root_directory("ML/histopathology/test_data/panda_wsi_example.tiff")) +@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") def test_load_slide() -> None: level = 2 reader = WSIReader('cuCIM') - slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) + slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) dims = slide_obj.resolutions['level_dimensions'][level][::-1] slide = load_slide_at_level(reader, slide_obj, level) @@ -39,10 +41,11 @@ def test_load_slide() -> None: assert np.array_equiv(larger_slide[:, :, dims[1]:], empty_fill_value) +@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") def test_get_luminance() -> None: level = 2 # here we only need to test at a single resolution reader = WSIReader('cuCIM') - slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) + slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) slide = load_slide_at_level(reader, slide_obj, level) slide_luminance = get_luminance(slide) @@ -61,10 +64,11 @@ def test_get_luminance() -> None: assert np.array_equal(slide_luminance_tiles.squeeze(1), tiles_luminance) +@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") def test_segment_foreground() -> None: level = 2 # here we only need to test at a single resolution reader = WSIReader('cuCIM') - slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) + slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) slide = load_slide_at_level(reader, slide_obj, level) auto_mask, auto_threshold = segment_foreground(slide, threshold=None) @@ -95,12 +99,13 @@ def test_segment_foreground() -> None: @pytest.mark.parametrize('level', [1, 2]) @pytest.mark.parametrize('foreground_threshold', [None, 215]) +@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") def test_get_bounding_box(level: int, foreground_threshold: Optional[float]) -> None: margin = 0 reader = WSIReader('cuCIM') loader = LoadROId(reader, image_key=SlideKey.IMAGE, level=level, margin=margin, foreground_threshold=foreground_threshold) - slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) + slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) level0_bbox, _ = loader._get_bounding_box(slide_obj) highest_level = slide_obj.resolutions['level_count'] - 1 @@ -130,6 +135,7 @@ def test_get_bounding_box(level: int, foreground_threshold: Optional[float]) -> @pytest.mark.parametrize('level', [1, 2]) @pytest.mark.parametrize('margin', [0, 42]) @pytest.mark.parametrize('foreground_threshold', [None, 215]) +@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") def test_load_roi(level: int, margin: int, foreground_threshold: Optional[float]) -> None: dataset = MockSlidesDataset() sample = dataset[0] diff --git a/Tests/ML/histopathology/utils/test_metrics_utils.py b/Tests/ML/histopathology/utils/test_metrics_utils.py index c226519d8..0d8a8a29f 100644 --- a/Tests/ML/histopathology/utils/test_metrics_utils.py +++ b/Tests/ML/histopathology/utils/test_metrics_utils.py @@ -13,7 +13,9 @@ from torch.functional import Tensor import pytest -from InnerEye.ML.Histopathology.utils.metrics_utils import plot_scores_hist, select_k_tiles, plot_slide, plot_heatmap_overlay, plot_normalized_confusion_matrix +from InnerEye.Common.common_util import is_windows +from InnerEye.ML.Histopathology.utils.metrics_utils import plot_scores_hist, select_k_tiles, plot_slide, \ + plot_heatmap_overlay, plot_normalized_confusion_matrix from InnerEye.ML.Histopathology.utils.naming import ResultsKey from InnerEye.ML.Histopathology.utils.heatmap_utils import location_selected_tiles from InnerEye.Common.fixed_paths_for_tests import full_ml_test_data_path @@ -44,22 +46,23 @@ def assert_equal_lists(pred: List, expected: List) -> None: ResultsKey.PROB: [Tensor([0.5]), Tensor([0.7]), Tensor([0.4]), Tensor([1.0])], ResultsKey.TRUE_LABEL: [0, 1, 1, 1], ResultsKey.BAG_ATTN: - [Tensor([[0.1, 0.0, 0.2, 0.15]]), + [Tensor([[0.1, 0.0, 0.2, 0.15]]), Tensor([[0.10, 0.18, 0.15, 0.13]]), Tensor([[0.25, 0.23, 0.20, 0.21]]), Tensor([[0.33, 0.31, 0.37, 0.35]])], ResultsKey.TILE_X: - [Tensor([200, 200, 424, 424]), + [Tensor([200, 200, 424, 424]), + Tensor([200, 200, 424, 424]), Tensor([200, 200, 424, 424]), - Tensor([200, 200, 424, 424]), Tensor([200, 200, 424, 424])], - ResultsKey.TILE_Y: - [Tensor([200, 424, 200, 424]), + ResultsKey.TILE_Y: + [Tensor([200, 424, 200, 424]), + Tensor([200, 200, 424, 424]), Tensor([200, 200, 424, 424]), - Tensor([200, 200, 424, 424]), Tensor([200, 200, 424, 424])] } + def test_select_k_tiles() -> None: top_tn = select_k_tiles(test_dict, n_slides=1, label=0, n_tiles=2, select=('lowest_pred', 'highest_att')) assert_equal_lists(top_tn, [(1, 0.5, [3, 4], [Tensor([0.2]), Tensor([0.15])])]) @@ -67,16 +70,24 @@ def test_select_k_tiles() -> None: nslides = 2 ntiles = 2 top_fn = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('lowest_pred', 'highest_att')) - bottom_fn = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('lowest_pred', 'lowest_att')) - assert_equal_lists(top_fn, [(3, 0.4, [1, 2], [Tensor([0.25]), Tensor([0.23])]), (2, 0.7, [2, 3], [Tensor([0.18]), Tensor([0.15])])]) - assert_equal_lists(bottom_fn, [(3, 0.4, [3, 4], [Tensor([0.20]), Tensor([0.21])]), (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) - - top_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('highest_pred', 'highest_att')) - bottom_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, select=('highest_pred', 'lowest_att')) - assert_equal_lists(top_tp, [(4, 1.0, [3, 4], [Tensor([0.37]), Tensor([0.35])]), (2, 0.7, [2, 3], [Tensor([0.18]), Tensor([0.15])])]) - assert_equal_lists(bottom_tp, [(4, 1.0, [2, 1], [Tensor([0.31]), Tensor([0.33])]), (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) - - + bottom_fn = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, + select=('lowest_pred', 'lowest_att')) + assert_equal_lists(top_fn, [(3, 0.4, [1, 2], [Tensor([0.25]), Tensor([0.23])]), + (2, 0.7, [2, 3], [Tensor([0.18]), Tensor([0.15])])]) + assert_equal_lists(bottom_fn, [(3, 0.4, [3, 4], [Tensor([0.20]), Tensor([0.21])]), + (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) + + top_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, + select=('highest_pred', 'highest_att')) + bottom_tp = select_k_tiles(test_dict, n_slides=nslides, label=1, n_tiles=ntiles, + select=('highest_pred', 'lowest_att')) + assert_equal_lists(top_tp, [(4, 1.0, [3, 4], [Tensor([0.37]), Tensor([0.35])]), + (2, 0.7, [2, 3], [Tensor([0.18]), Tensor([0.15])])]) + assert_equal_lists(bottom_tp, [(4, 1.0, [2, 1], [Tensor([0.31]), Tensor([0.33])]), + (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) + + +@pytest.mark.skipif(is_windows(), "Rendering is different on Windows") def test_plot_scores_hist(test_output_dirs: OutputFolderForTests) -> None: fig = plot_scores_hist(test_dict) assert isinstance(fig, matplotlib.figure.Figure) @@ -104,16 +115,17 @@ def test_plot_slide(test_output_dirs: OutputFolderForTests, scale: int) -> None: assert_binary_files_match(file, expected) +@pytest.mark.skipif(is_windows(), "Rendering is different on Windows") def test_plot_heatmap_overlay(test_output_dirs: OutputFolderForTests) -> None: set_random_seed(0) slide_image = np.random.rand(3, 1000, 2000) location_bbox = [100, 100] - slide = 1 + slide = 1 tile_size = 224 level = 0 - fig = plot_heatmap_overlay(slide=slide, # type: ignore + fig = plot_heatmap_overlay(slide=slide, # type: ignore slide_image=slide_image, - results=test_dict, # type: ignore + results=test_dict, # type: ignore location_bbox=location_bbox, tile_size=tile_size, level=level) @@ -128,15 +140,16 @@ def test_plot_heatmap_overlay(test_output_dirs: OutputFolderForTests) -> None: @pytest.mark.parametrize("n_classes", [1, 3]) +@pytest.mark.skipif(is_windows(), "Rendering is different on Windows") def test_plot_normalized_confusion_matrix(test_output_dirs: OutputFolderForTests, n_classes: int) -> None: set_random_seed(0) if n_classes > 1: cm = np.random.randint(1, 1000, size=(n_classes, n_classes)) class_names = [str(i) for i in range(n_classes)] else: - cm = np.random.randint(1, 1000, size=(n_classes+1, n_classes+1)) - class_names = [str(i) for i in range(n_classes+1)] - cm_n = cm/cm.sum(axis=1, keepdims=True) + cm = np.random.randint(1, 1000, size=(n_classes + 1, n_classes + 1)) + class_names = [str(i) for i in range(n_classes + 1)] + cm_n = cm / cm.sum(axis=1, keepdims=True) assert (cm_n <= 1).all() fig = plot_normalized_confusion_matrix(cm=cm_n, class_names=class_names) @@ -153,26 +166,27 @@ def test_plot_normalized_confusion_matrix(test_output_dirs: OutputFolderForTests @pytest.mark.parametrize("level", [0, 1, 2]) def test_location_selected_tiles(level: int) -> None: set_random_seed(0) - slide = 1 + slide = 1 location_bbox = [100, 100] slide_image = np.random.rand(3, 1000, 2000) coords = [] - slide_ids = [item[0] for item in test_dict[ResultsKey.SLIDE_ID]] # type: ignore + slide_ids = [item[0] for item in test_dict[ResultsKey.SLIDE_ID]] # type: ignore slide_idx = slide_ids.index(slide) - for tile_idx in range(len(test_dict[ResultsKey.IMAGE_PATH][slide_idx])): # type: ignore - tile_coords = np.transpose(np.array([test_dict[ResultsKey.TILE_X][slide_idx][tile_idx].cpu().numpy(), # type: ignore - test_dict[ResultsKey.TILE_Y][slide_idx][tile_idx].cpu().numpy()])) # type: ignore + for tile_idx in range(len(test_dict[ResultsKey.IMAGE_PATH][slide_idx])): # type: ignore + tile_coords = np.transpose( + np.array([test_dict[ResultsKey.TILE_X][slide_idx][tile_idx].cpu().numpy(), # type: ignore + test_dict[ResultsKey.TILE_Y][slide_idx][tile_idx].cpu().numpy()])) # type: ignore coords.append(tile_coords) coords = np.array(coords) - tile_coords_transformed = location_selected_tiles(tile_coords=coords, - location_bbox=location_bbox, - level=level) + tile_coords_transformed = location_selected_tiles(tile_coords=coords, + location_bbox=location_bbox, + level=level) tile_xs, tile_ys = tile_coords_transformed.T level_dict = {0: 1, 1: 4, 2: 16} factor = level_dict[level] - assert min(tile_xs) >= 0 - assert max(tile_xs) <= slide_image.shape[2]//factor - assert min(tile_ys) >= 0 - assert max(tile_ys) <= slide_image.shape[1]//factor + assert min(tile_xs) >= 0 + assert max(tile_xs) <= slide_image.shape[2] // factor + assert min(tile_ys) >= 0 + assert max(tile_ys) <= slide_image.shape[1] // factor From 977dd70a6cf1fff73ad6db0cab8f1ef6c0179dcf Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 10:25:09 +0000 Subject: [PATCH 05/30] fix code --- Tests/ML/histopathology/models/test_deepmil.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Tests/ML/histopathology/models/test_deepmil.py b/Tests/ML/histopathology/models/test_deepmil.py index 797b3080c..39e6b7173 100644 --- a/Tests/ML/histopathology/models/test_deepmil.py +++ b/Tests/ML/histopathology/models/test_deepmil.py @@ -7,6 +7,7 @@ from typing import Callable, Dict, List, Type # noqa import pytest +import torch from torch import Tensor, argmax, nn, rand, randint, randn, round, stack, allclose from torchvision.models import resnet18 @@ -29,7 +30,7 @@ ) from InnerEye.ML.Histopathology.models.deepmil import DeepMILModule from InnerEye.ML.Histopathology.models.encoders import ImageNetEncoder, TileEncoder -from InnerEye.ML.Histopathology.utils.naming import ResultsKey +from InnerEye.ML.Histopathology.utils.naming import MetricsKey, ResultsKey def get_supervised_imagenet_encoder() -> TileEncoder: @@ -108,9 +109,12 @@ def test_lightningmodule( assert preds.shape[0] == batch_size for metric_name, metric_object in module.train_metrics.items(): - if (batch_size > 1) or (not metric_name == "auroc"): + if metric_name == MetricsKey.CONF_MATRIX or metric_name == MetricsKey.AUROC: + continue + if batch_size > 1: score = metric_object(preds.view(-1, 1), bag_labels.view(-1, 1)) - assert score >= 0 and score <= 1 + assert torch.all(score >= 0) + assert torch.all(score <= 1) def move_batch_to_expected_device(batch: Dict[str, List], use_gpu: bool) -> Dict: From 067d249f9104c5f106286e1f40b6664549fbb998 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 10:32:12 +0000 Subject: [PATCH 06/30] speed up test --- Tests/ML/histopathology/models/test_deepmil.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/ML/histopathology/models/test_deepmil.py b/Tests/ML/histopathology/models/test_deepmil.py index 39e6b7173..96ccd3fc8 100644 --- a/Tests/ML/histopathology/models/test_deepmil.py +++ b/Tests/ML/histopathology/models/test_deepmil.py @@ -39,10 +39,10 @@ def get_supervised_imagenet_encoder() -> TileEncoder: @pytest.mark.parametrize("n_classes", [1, 3]) @pytest.mark.parametrize("pooling_layer", [AttentionLayer, GatedAttentionLayer]) -@pytest.mark.parametrize("batch_size", [1, 15]) -@pytest.mark.parametrize("max_bag_size", [1, 7]) -@pytest.mark.parametrize("pool_hidden_dim", [1, 5]) -@pytest.mark.parametrize("pool_out_dim", [1, 6]) +@pytest.mark.parametrize("batch_size", [1, 2]) +@pytest.mark.parametrize("max_bag_size", [1, 3]) +@pytest.mark.parametrize("pool_hidden_dim", [1, 4]) +@pytest.mark.parametrize("pool_out_dim", [1, 5]) def test_lightningmodule( n_classes: int, pooling_layer: Callable[[int, int, int], nn.Module], From 930731e2cff5f2c0d26d317f6d219804fe351427 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 11:20:20 +0000 Subject: [PATCH 07/30] conda for windows --- azure-pipelines/prepare_conda.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/prepare_conda.yml b/azure-pipelines/prepare_conda.yml index 55b8e27c5..634c2aa10 100644 --- a/azure-pipelines/prepare_conda.yml +++ b/azure-pipelines/prepare_conda.yml @@ -1,6 +1,9 @@ steps: - bash: | - subdir=bin + if [ $(Agent.OS) = 'Windows_NT' ] + then subdir=Scripts + else subdir=bin + fi echo "Adding this directory to PATH: $CONDA/$subdir" echo "##vso[task.prependpath]$CONDA/$subdir" displayName: Add conda to PATH @@ -9,4 +12,4 @@ steps: - bash: | sudo chown -R $USER /usr/share/miniconda condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' )) - displayName: Take ownership of conda installation + displayName: Take ownership of conda installation (Linux only) From 2e94f1bc84748d5b38bd4e34192a3993c584feb1 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 11:21:34 +0000 Subject: [PATCH 08/30] job rename --- azure-pipelines/build-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index da6454f42..f63401a0a 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -23,13 +23,13 @@ jobs: steps: - template: cancel_aml_jobs.yml - - job: Windows + - job: CredScan_ComponentGov pool: vmImage: 'windows-2019' steps: - template: build_windows.yaml - - job: Linux + - job: PyTest pool: vmImage: 'ubuntu-20.04' steps: From 626b36bf9108214ff5eb62c98d9fc90874cf3b38 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 11:22:48 +0000 Subject: [PATCH 09/30] disable caching for now --- azure-pipelines/build-pr.yml | 2 +- azure-pipelines/inner_eye_env.yml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index f63401a0a..a09c3746b 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -31,7 +31,7 @@ jobs: - job: PyTest pool: - vmImage: 'ubuntu-20.04' + vmImage: 'windows-2019' steps: - template: build.yaml diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 49020f994..9fbc05f7d 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -5,14 +5,14 @@ steps: - template: prepare_conda.yml - # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda - - task: Cache@2 - displayName: Use cached Conda environment - inputs: - # Beware of changing the cache key or path independently, safest to change in sync - key: 'usr_share_miniconda_envs | "$(Agent.OS)" | environment.yml' - path: /usr/share/miniconda/envs - cacheHitVar: CONDA_CACHE_RESTORED + # # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda + # - task: Cache@2 + # displayName: Use cached Conda environment + # inputs: + # # Beware of changing the cache key or path independently, safest to change in sync + # key: 'usr_share_miniconda_envs | "$(Agent.OS)" | environment.yml' + # path: /usr/share/miniconda/envs + # cacheHitVar: CONDA_CACHE_RESTORED - script: conda env create --file environment.yml displayName: Create Anaconda environment From 4b0ee80ef764e45a8b5d7b0fa300d86686f5c055 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 11:25:48 +0000 Subject: [PATCH 10/30] disable caching part 2 --- azure-pipelines/inner_eye_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 9fbc05f7d..b86f1110d 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -17,7 +17,7 @@ steps: - script: conda env create --file environment.yml displayName: Create Anaconda environment failOnStderr: false # Conda env create does not have an option to suppress warnings generated in wheel.py - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + # condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: source activate InnerEye displayName: Check if InnerEye is present From 49a072ef95b1b00011deaa4cc600c9c8c434b2ed Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 11:26:41 +0000 Subject: [PATCH 11/30] disable most jobs --- azure-pipelines/build-pr.yml | 355 ++++++++++++++++++----------------- 1 file changed, 178 insertions(+), 177 deletions(-) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index a09c3746b..29aea8a8b 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -35,180 +35,181 @@ jobs: steps: - template: build.yaml - - job: TrainInAzureML - dependsOn: CancelPreviousJobs - variables: - - name: tag - value: 'TrainBasicModel' - - name: more_switches - value: '--log_level=DEBUG --pl_deterministic --use_dataset_mount=True --regression_test_folder=RegressionTestResults/PR_BasicModel2Epochs' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - max_run_duration: '30m' - - template: tests_after_training.yml - parameters: - pytest_mark: after_training_single_run - test_run_title: tests_after_training_single_run - - - job: RunGpuTestsInAzureML - dependsOn: CancelPreviousJobs - variables: - - name: tag - value: 'RunGpuTests' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - pytest_mark: 'gpu or cpu_and_gpu or azureml' - max_run_duration: '30m' - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'tests_on_AzureML' - condition: succeededOrFailed() - displayName: Publish test results - - # Now train a module, using the Github code as a submodule. Here, a simpler 1 channel model - # is trained, because we use this build to also check the "submit_for_inference" code, that - # presently only handles single channel models. - - job: TrainInAzureMLViaSubmodule - dependsOn: CancelPreviousJobs - variables: - - name: model - value: 'BasicModel2Epochs1Channel' - - name: tag - value: 'Train1ChannelSubmodule' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_via_submodule.yml - parameters: - wait_for_completion: 'True' - max_run_duration: '30m' - - template: tests_after_training.yml - parameters: - pytest_mark: "inference or after_training" - test_run_title: tests_after_train_submodule - - - # Train a 2-element ensemble model - - job: TrainEnsemble - dependsOn: CancelPreviousJobs - variables: - - name: model - value: 'BasicModelForEnsembleTest' - - name: number_of_cross_validation_splits - value: 2 - - name: tag - value: 'TrainEnsemble' - - name: more_switches - value: '--pl_deterministic --log_level=DEBUG --regression_test_folder=RegressionTestResults/PR_TrainEnsemble' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - pytest_mark: '' - max_run_duration: '1h' - - template: tests_after_training.yml - parameters: - pytest_mark: after_training_ensemble_run - test_run_title: tests_after_training_ensemble_run - - # Train a model on 2 nodes - - job: Train2Nodes - dependsOn: CancelPreviousJobs - variables: - - name: model - value: 'BasicModel2EpochsMoreData' - - name: tag - value: 'Train2Nodes' - - name: more_switches - value: '--log_level=DEBUG --pl_deterministic --num_nodes=2 --regression_test_folder=RegressionTestResults/PR_Train2Nodes' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - pytest_mark: '' - max_run_duration: '1h' - - template: tests_after_training.yml - parameters: - pytest_mark: after_training_2node - test_run_title: tests_after_training_2node_run - - - job: TrainHelloWorld - dependsOn: CancelPreviousJobs - variables: - - name: model - value: 'HelloWorld' - - name: tag - value: 'HelloWorldPR' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - pytest_mark: '' - max_run_duration: '30m' - - # Run HelloContainer on 2 nodes. HelloContainer uses native Lighting test set inference, which can get - # confused after doing multi-node training in the same script. - - job: TrainHelloContainer - dependsOn: CancelPreviousJobs - variables: - - name: model - value: 'HelloContainer' - - name: tag - value: 'HelloContainerPR' - - name: more_switches - value: '--pl_deterministic --num_nodes=2 --max_num_gpus=2 --regression_test_folder=RegressionTestResults/PR_HelloContainer' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - pytest_mark: '' - max_run_duration: '30m' - - template: tests_after_training.yml - parameters: - pytest_mark: after_training_hello_container - test_run_title: tests_after_training_hello_container - - # Run the Lung model. This is a large model requiring a docker image with large memory. This tests against - # regressions in AML when requesting more than the default amount of memory. This needs to run with all subjects to - # trigger the bug, total runtime 10min - - job: TrainLung - dependsOn: CancelPreviousJobs - variables: - - name: model - value: 'Lung' - - name: tag - value: 'LungPR' - - name: more_switches - value: '--pl_deterministic --num_epochs=1 --feature_channels=16 --show_patch_sampling=0 --train_batch_size=4 --inference_on_val_set=False --inference_on_test_set=False ' - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: train_template.yml - parameters: - wait_for_completion: 'True' - pytest_mark: '' - max_run_duration: '30m' - - # Now run the build for the Data Selection sub-folder - - job: BuildDataSelection - pool: - vmImage: 'ubuntu-18.04' - steps: - - template: build_data_quality.yaml + # - job: TrainInAzureML + # dependsOn: CancelPreviousJobs + # variables: + # - name: tag + # value: 'TrainBasicModel' + # - name: more_switches + # value: '--log_level=DEBUG --pl_deterministic --use_dataset_mount=True --regression_test_folder=RegressionTestResults/PR_BasicModel2Epochs' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # max_run_duration: '30m' + # - template: tests_after_training.yml + # parameters: + # pytest_mark: after_training_single_run + # test_run_title: tests_after_training_single_run +# + # - job: RunGpuTestsInAzureML + # dependsOn: CancelPreviousJobs + # variables: + # - name: tag + # value: 'RunGpuTests' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # pytest_mark: 'gpu or cpu_and_gpu or azureml' + # max_run_duration: '30m' + # - task: PublishTestResults@2 + # inputs: + # testResultsFiles: '**/test-*.xml' + # testRunTitle: 'tests_on_AzureML' + # condition: succeededOrFailed() + # displayName: Publish test results +# + # # Now train a module, using the Github code as a submodule. Here, a simpler 1 channel model + # # is trained, because we use this build to also check the "submit_for_inference" code, that + # # presently only handles single channel models. + # - job: TrainInAzureMLViaSubmodule + # dependsOn: CancelPreviousJobs + # variables: + # - name: model + # value: 'BasicModel2Epochs1Channel' + # - name: tag + # value: 'Train1ChannelSubmodule' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_via_submodule.yml + # parameters: + # wait_for_completion: 'True' + # max_run_duration: '30m' + # - template: tests_after_training.yml + # parameters: + # pytest_mark: "inference or after_training" + # test_run_title: tests_after_train_submodule +# +# + # # Train a 2-element ensemble model + # - job: TrainEnsemble + # dependsOn: CancelPreviousJobs + # variables: + # - name: model + # value: 'BasicModelForEnsembleTest' + # - name: number_of_cross_validation_splits + # value: 2 + # - name: tag + # value: 'TrainEnsemble' + # - name: more_switches + # value: '--pl_deterministic --log_level=DEBUG --regression_test_folder=RegressionTestResults/PR_TrainEnsemble' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # pytest_mark: '' + # max_run_duration: '1h' + # - template: tests_after_training.yml + # parameters: + # pytest_mark: after_training_ensemble_run + # test_run_title: tests_after_training_ensemble_run +# + # # Train a model on 2 nodes + # - job: Train2Nodes + # dependsOn: CancelPreviousJobs + # variables: + # - name: model + # value: 'BasicModel2EpochsMoreData' + # - name: tag + # value: 'Train2Nodes' + # - name: more_switches + # value: '--log_level=DEBUG --pl_deterministic --num_nodes=2 --regression_test_folder=RegressionTestResults/PR_Train2Nodes' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # pytest_mark: '' + # max_run_duration: '1h' + # - template: tests_after_training.yml + # parameters: + # pytest_mark: after_training_2node + # test_run_title: tests_after_training_2node_run +# + # - job: TrainHelloWorld + # dependsOn: CancelPreviousJobs + # variables: + # - name: model + # value: 'HelloWorld' + # - name: tag + # value: 'HelloWorldPR' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # pytest_mark: '' + # max_run_duration: '30m' +# + # # Run HelloContainer on 2 nodes. HelloContainer uses native Lighting test set inference, which can get + # # confused after doing multi-node training in the same script. + # - job: TrainHelloContainer + # dependsOn: CancelPreviousJobs + # variables: + # - name: model + # value: 'HelloContainer' + # - name: tag + # value: 'HelloContainerPR' + # - name: more_switches + # value: '--pl_deterministic --num_nodes=2 --max_num_gpus=2 --regression_test_folder=RegressionTestResults/PR_HelloContainer' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # pytest_mark: '' + # max_run_duration: '30m' + # - template: tests_after_training.yml + # parameters: + # pytest_mark: after_training_hello_container + # test_run_title: tests_after_training_hello_container +# + # # Run the Lung model. This is a large model requiring a docker image with large memory. This tests against + # # regressions in AML when requesting more than the default amount of memory. This needs to run with all subjects to + # # trigger the bug, total runtime 10min + # - job: TrainLung + # dependsOn: CancelPreviousJobs + # variables: + # - name: model + # value: 'Lung' + # - name: tag + # value: 'LungPR' + # - name: more_switches + # value: '--pl_deterministic --num_epochs=1 --feature_channels=16 --show_patch_sampling=0 --train_batch_size=4 --inference_on_val_set=False --inference_on_test_set=False ' + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: train_template.yml + # parameters: + # wait_for_completion: 'True' + # pytest_mark: '' + # max_run_duration: '30m' +# + # # Now run the build for the Data Selection sub-folder + # - job: BuildDataSelection + # pool: + # vmImage: 'ubuntu-18.04' + # steps: + # - template: build_data_quality.yaml +# \ No newline at end of file From a50a9cc69dc582d2261444f0ce0832c7bc73d4b6 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 11:48:18 +0000 Subject: [PATCH 12/30] fix script/bash problem --- azure-pipelines/inner_eye_env.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index b86f1110d..7f1aa8712 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -5,6 +5,10 @@ steps: - template: prepare_conda.yml + - bash: | + conda info + displayName: Print Conda info + # # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda # - task: Cache@2 # displayName: Use cached Conda environment @@ -14,14 +18,11 @@ steps: # path: /usr/share/miniconda/envs # cacheHitVar: CONDA_CACHE_RESTORED - - script: conda env create --file environment.yml + - bash: conda env create --file environment.yml displayName: Create Anaconda environment failOnStderr: false # Conda env create does not have an option to suppress warnings generated in wheel.py # condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - - script: source activate InnerEye - displayName: Check if InnerEye is present - - bash: | source activate InnerEye which python @@ -29,3 +30,6 @@ steps: pip freeze failOnStderr: false displayName: Print package list and Conda info + + - bash: source activate InnerEye + displayName: Check if InnerEye is present From e3965ae97a1c64b839545f0e639129259d02a6ac Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 12:07:52 +0000 Subject: [PATCH 13/30] fix skipif errors --- .../histopathology/preprocessing/test_slide_loading.py | 10 +++++----- Tests/ML/histopathology/utils/test_metrics_utils.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Tests/ML/histopathology/preprocessing/test_slide_loading.py b/Tests/ML/histopathology/preprocessing/test_slide_loading.py index 95bc2e743..5d9e2f819 100644 --- a/Tests/ML/histopathology/preprocessing/test_slide_loading.py +++ b/Tests/ML/histopathology/preprocessing/test_slide_loading.py @@ -15,7 +15,7 @@ TEST_IMAGE_PATH = str(tests_root_directory("ML/histopathology/test_data/panda_wsi_example.tiff")) -@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") +@pytest.mark.skipif(is_windows(), reason="cucim package is not available on Windows") def test_load_slide() -> None: level = 2 reader = WSIReader('cuCIM') @@ -41,7 +41,7 @@ def test_load_slide() -> None: assert np.array_equiv(larger_slide[:, :, dims[1]:], empty_fill_value) -@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") +@pytest.mark.skipif(is_windows(), reason="cucim package is not available on Windows") def test_get_luminance() -> None: level = 2 # here we only need to test at a single resolution reader = WSIReader('cuCIM') @@ -64,7 +64,7 @@ def test_get_luminance() -> None: assert np.array_equal(slide_luminance_tiles.squeeze(1), tiles_luminance) -@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") +@pytest.mark.skipif(is_windows(), reason="cucim package is not available on Windows") def test_segment_foreground() -> None: level = 2 # here we only need to test at a single resolution reader = WSIReader('cuCIM') @@ -99,7 +99,7 @@ def test_segment_foreground() -> None: @pytest.mark.parametrize('level', [1, 2]) @pytest.mark.parametrize('foreground_threshold', [None, 215]) -@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") +@pytest.mark.skipif(is_windows(), reason="cucim package is not available on Windows") def test_get_bounding_box(level: int, foreground_threshold: Optional[float]) -> None: margin = 0 reader = WSIReader('cuCIM') @@ -135,7 +135,7 @@ def test_get_bounding_box(level: int, foreground_threshold: Optional[float]) -> @pytest.mark.parametrize('level', [1, 2]) @pytest.mark.parametrize('margin', [0, 42]) @pytest.mark.parametrize('foreground_threshold', [None, 215]) -@pytest.mark.skipif(is_windows(), "cucim package is not available on Windows") +@pytest.mark.skipif(is_windows(), reason="cucim package is not available on Windows") def test_load_roi(level: int, margin: int, foreground_threshold: Optional[float]) -> None: dataset = MockSlidesDataset() sample = dataset[0] diff --git a/Tests/ML/histopathology/utils/test_metrics_utils.py b/Tests/ML/histopathology/utils/test_metrics_utils.py index 0d8a8a29f..b1cc4c1ef 100644 --- a/Tests/ML/histopathology/utils/test_metrics_utils.py +++ b/Tests/ML/histopathology/utils/test_metrics_utils.py @@ -87,7 +87,7 @@ def test_select_k_tiles() -> None: (2, 0.7, [1, 4], [Tensor([0.10]), Tensor([0.13])])]) -@pytest.mark.skipif(is_windows(), "Rendering is different on Windows") +@pytest.mark.skipif(is_windows(), reason="Rendering is different on Windows") def test_plot_scores_hist(test_output_dirs: OutputFolderForTests) -> None: fig = plot_scores_hist(test_dict) assert isinstance(fig, matplotlib.figure.Figure) @@ -115,7 +115,7 @@ def test_plot_slide(test_output_dirs: OutputFolderForTests, scale: int) -> None: assert_binary_files_match(file, expected) -@pytest.mark.skipif(is_windows(), "Rendering is different on Windows") +@pytest.mark.skipif(is_windows(), reason="Rendering is different on Windows") def test_plot_heatmap_overlay(test_output_dirs: OutputFolderForTests) -> None: set_random_seed(0) slide_image = np.random.rand(3, 1000, 2000) @@ -140,7 +140,7 @@ def test_plot_heatmap_overlay(test_output_dirs: OutputFolderForTests) -> None: @pytest.mark.parametrize("n_classes", [1, 3]) -@pytest.mark.skipif(is_windows(), "Rendering is different on Windows") +@pytest.mark.skipif(is_windows(), reason="Rendering is different on Windows") def test_plot_normalized_confusion_matrix(test_output_dirs: OutputFolderForTests, n_classes: int) -> None: set_random_seed(0) if n_classes > 1: From fb949f36fda406e7e1b008242b6bed7c1d28e6d0 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 12:08:38 +0000 Subject: [PATCH 14/30] multiplatform cache --- azure-pipelines/inner_eye_env.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 7f1aa8712..32c06670d 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -1,3 +1,9 @@ +variables: + ${{ if eq( variables['Agent.OS'], 'Linux' ) }}: + CONDA_ENV_DIR: '/usr/share/miniconda/envs' + ${{ if eq( variables['Agent.OS'], 'Windows_NT' ) }}: + CONDA_ENV_DIR: 'C:\Miniconda\envs' + steps: - template: checkout.yml @@ -9,19 +15,19 @@ steps: conda info displayName: Print Conda info - # # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda - # - task: Cache@2 - # displayName: Use cached Conda environment - # inputs: - # # Beware of changing the cache key or path independently, safest to change in sync - # key: 'usr_share_miniconda_envs | "$(Agent.OS)" | environment.yml' - # path: /usr/share/miniconda/envs - # cacheHitVar: CONDA_CACHE_RESTORED + # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda + - task: Cache@2 + displayName: Use cached Conda environment + inputs: + # Beware of changing the cache key or path independently, safest to change in sync + key: 'conda_env_dir | "$(Agent.OS)" | environment.yml' + path: $(CONDA_ENV_DIR) + cacheHitVar: CONDA_CACHE_RESTORED - bash: conda env create --file environment.yml displayName: Create Anaconda environment failOnStderr: false # Conda env create does not have an option to suppress warnings generated in wheel.py - # condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - bash: | source activate InnerEye From 32e1da36bf6835de4a1b206096a87fbbb373a090 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 12:11:04 +0000 Subject: [PATCH 15/30] cleanup --- azure-pipelines/inner_eye_env.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 32c06670d..b3a1f3538 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -11,10 +11,6 @@ steps: - template: prepare_conda.yml - - bash: | - conda info - displayName: Print Conda info - # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda - task: Cache@2 displayName: Use cached Conda environment @@ -36,6 +32,7 @@ steps: pip freeze failOnStderr: false displayName: Print package list and Conda info + condition: succeededOrFailed() - bash: source activate InnerEye - displayName: Check if InnerEye is present + displayName: Check if InnerEye environment is present From 5f2416621b9c994c7c89986c7bd996d227c6a417 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 13:00:10 +0000 Subject: [PATCH 16/30] yaml fix --- azure-pipelines/inner_eye_env.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index b3a1f3538..07daee9a6 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -1,9 +1,3 @@ -variables: - ${{ if eq( variables['Agent.OS'], 'Linux' ) }}: - CONDA_ENV_DIR: '/usr/share/miniconda/envs' - ${{ if eq( variables['Agent.OS'], 'Windows_NT' ) }}: - CONDA_ENV_DIR: 'C:\Miniconda\envs' - steps: - template: checkout.yml @@ -17,8 +11,11 @@ steps: inputs: # Beware of changing the cache key or path independently, safest to change in sync key: 'conda_env_dir | "$(Agent.OS)" | environment.yml' - path: $(CONDA_ENV_DIR) cacheHitVar: CONDA_CACHE_RESTORED + ${{ if eq( variables['Agent.OS'], 'Linux' ) }}: + path: '/usr/share/miniconda/envs' + ${{ if eq( variables['Agent.OS'], 'Windows_NT' ) }}: + path: 'C:\Miniconda\envs' - bash: conda env create --file environment.yml displayName: Create Anaconda environment From 4d292389ee8a3b00753ee71053310f52c0b25d45 Mon Sep 17 00:00:00 2001 From: Max Ilse Date: Wed, 2 Feb 2022 15:36:05 +0000 Subject: [PATCH 17/30] fixed test --- Tests/SSL/test_ssl_containers.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Tests/SSL/test_ssl_containers.py b/Tests/SSL/test_ssl_containers.py index 1a55a53e6..49569378b 100644 --- a/Tests/SSL/test_ssl_containers.py +++ b/Tests/SSL/test_ssl_containers.py @@ -634,7 +634,24 @@ def test_simclr_dataloader_type() -> None: """ This test checks if the transform pipeline of a SSL job can handle different data types coming from the dataloader. """ - def check_types_in_dataloader(dataloader: CombinedLoader) -> None: + # TODO: Once the pytorch lightning bug is fixed the following test can be removed. + # The training and val loader will be both CombinedLoaders + def check_types_in_train_dataloader(dataloader: dict) -> None: + for i, batch in enumerate(dataloader[SSLDataModuleType.ENCODER]): + assert isinstance(batch[0][0], torch.Tensor) + assert isinstance(batch[0][1], torch.Tensor) + assert isinstance(batch[1], torch.Tensor) + if i == 1: + break + + for i, batch in enumerate(dataloader[SSLDataModuleType.LINEAR_HEAD]): + assert isinstance(batch[0], torch.Tensor) + assert isinstance(batch[1], torch.Tensor) + assert isinstance(batch[2], torch.Tensor) + if i == 1: + break + + def check_types_in_val_dataloader(dataloader: CombinedLoader) -> None: for i, batch in enumerate(dataloader): assert isinstance(batch[SSLDataModuleType.ENCODER][0][0], torch.Tensor) assert isinstance(batch[SSLDataModuleType.ENCODER][0][1], torch.Tensor) @@ -646,8 +663,8 @@ def check_types_in_dataloader(dataloader: CombinedLoader) -> None: break def check_types_in_train_and_val(data: CombinedDataModule) -> None: - check_types_in_dataloader(data.train_dataloader()) - check_types_in_dataloader(data.val_dataloader()) + check_types_in_train_dataloader(data.train_dataloader()) + check_types_in_val_dataloader(data.val_dataloader()) container = DummySimCLR() container.setup() From c5512238cf222d65bc43fa46a99c94a59fffd10f Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 13:42:06 +0000 Subject: [PATCH 18/30] put all jobs back in --- azure-pipelines/build-pr.yml | 355 +++++++++++++++++------------------ 1 file changed, 177 insertions(+), 178 deletions(-) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index 29aea8a8b..a09c3746b 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -35,181 +35,180 @@ jobs: steps: - template: build.yaml - # - job: TrainInAzureML - # dependsOn: CancelPreviousJobs - # variables: - # - name: tag - # value: 'TrainBasicModel' - # - name: more_switches - # value: '--log_level=DEBUG --pl_deterministic --use_dataset_mount=True --regression_test_folder=RegressionTestResults/PR_BasicModel2Epochs' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # max_run_duration: '30m' - # - template: tests_after_training.yml - # parameters: - # pytest_mark: after_training_single_run - # test_run_title: tests_after_training_single_run -# - # - job: RunGpuTestsInAzureML - # dependsOn: CancelPreviousJobs - # variables: - # - name: tag - # value: 'RunGpuTests' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # pytest_mark: 'gpu or cpu_and_gpu or azureml' - # max_run_duration: '30m' - # - task: PublishTestResults@2 - # inputs: - # testResultsFiles: '**/test-*.xml' - # testRunTitle: 'tests_on_AzureML' - # condition: succeededOrFailed() - # displayName: Publish test results -# - # # Now train a module, using the Github code as a submodule. Here, a simpler 1 channel model - # # is trained, because we use this build to also check the "submit_for_inference" code, that - # # presently only handles single channel models. - # - job: TrainInAzureMLViaSubmodule - # dependsOn: CancelPreviousJobs - # variables: - # - name: model - # value: 'BasicModel2Epochs1Channel' - # - name: tag - # value: 'Train1ChannelSubmodule' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_via_submodule.yml - # parameters: - # wait_for_completion: 'True' - # max_run_duration: '30m' - # - template: tests_after_training.yml - # parameters: - # pytest_mark: "inference or after_training" - # test_run_title: tests_after_train_submodule -# -# - # # Train a 2-element ensemble model - # - job: TrainEnsemble - # dependsOn: CancelPreviousJobs - # variables: - # - name: model - # value: 'BasicModelForEnsembleTest' - # - name: number_of_cross_validation_splits - # value: 2 - # - name: tag - # value: 'TrainEnsemble' - # - name: more_switches - # value: '--pl_deterministic --log_level=DEBUG --regression_test_folder=RegressionTestResults/PR_TrainEnsemble' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # pytest_mark: '' - # max_run_duration: '1h' - # - template: tests_after_training.yml - # parameters: - # pytest_mark: after_training_ensemble_run - # test_run_title: tests_after_training_ensemble_run -# - # # Train a model on 2 nodes - # - job: Train2Nodes - # dependsOn: CancelPreviousJobs - # variables: - # - name: model - # value: 'BasicModel2EpochsMoreData' - # - name: tag - # value: 'Train2Nodes' - # - name: more_switches - # value: '--log_level=DEBUG --pl_deterministic --num_nodes=2 --regression_test_folder=RegressionTestResults/PR_Train2Nodes' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # pytest_mark: '' - # max_run_duration: '1h' - # - template: tests_after_training.yml - # parameters: - # pytest_mark: after_training_2node - # test_run_title: tests_after_training_2node_run -# - # - job: TrainHelloWorld - # dependsOn: CancelPreviousJobs - # variables: - # - name: model - # value: 'HelloWorld' - # - name: tag - # value: 'HelloWorldPR' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # pytest_mark: '' - # max_run_duration: '30m' -# - # # Run HelloContainer on 2 nodes. HelloContainer uses native Lighting test set inference, which can get - # # confused after doing multi-node training in the same script. - # - job: TrainHelloContainer - # dependsOn: CancelPreviousJobs - # variables: - # - name: model - # value: 'HelloContainer' - # - name: tag - # value: 'HelloContainerPR' - # - name: more_switches - # value: '--pl_deterministic --num_nodes=2 --max_num_gpus=2 --regression_test_folder=RegressionTestResults/PR_HelloContainer' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # pytest_mark: '' - # max_run_duration: '30m' - # - template: tests_after_training.yml - # parameters: - # pytest_mark: after_training_hello_container - # test_run_title: tests_after_training_hello_container -# - # # Run the Lung model. This is a large model requiring a docker image with large memory. This tests against - # # regressions in AML when requesting more than the default amount of memory. This needs to run with all subjects to - # # trigger the bug, total runtime 10min - # - job: TrainLung - # dependsOn: CancelPreviousJobs - # variables: - # - name: model - # value: 'Lung' - # - name: tag - # value: 'LungPR' - # - name: more_switches - # value: '--pl_deterministic --num_epochs=1 --feature_channels=16 --show_patch_sampling=0 --train_batch_size=4 --inference_on_val_set=False --inference_on_test_set=False ' - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: train_template.yml - # parameters: - # wait_for_completion: 'True' - # pytest_mark: '' - # max_run_duration: '30m' -# - # # Now run the build for the Data Selection sub-folder - # - job: BuildDataSelection - # pool: - # vmImage: 'ubuntu-18.04' - # steps: - # - template: build_data_quality.yaml -# \ No newline at end of file + - job: TrainInAzureML + dependsOn: CancelPreviousJobs + variables: + - name: tag + value: 'TrainBasicModel' + - name: more_switches + value: '--log_level=DEBUG --pl_deterministic --use_dataset_mount=True --regression_test_folder=RegressionTestResults/PR_BasicModel2Epochs' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + max_run_duration: '30m' + - template: tests_after_training.yml + parameters: + pytest_mark: after_training_single_run + test_run_title: tests_after_training_single_run + + - job: RunGpuTestsInAzureML + dependsOn: CancelPreviousJobs + variables: + - name: tag + value: 'RunGpuTests' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + pytest_mark: 'gpu or cpu_and_gpu or azureml' + max_run_duration: '30m' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'tests_on_AzureML' + condition: succeededOrFailed() + displayName: Publish test results + + # Now train a module, using the Github code as a submodule. Here, a simpler 1 channel model + # is trained, because we use this build to also check the "submit_for_inference" code, that + # presently only handles single channel models. + - job: TrainInAzureMLViaSubmodule + dependsOn: CancelPreviousJobs + variables: + - name: model + value: 'BasicModel2Epochs1Channel' + - name: tag + value: 'Train1ChannelSubmodule' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_via_submodule.yml + parameters: + wait_for_completion: 'True' + max_run_duration: '30m' + - template: tests_after_training.yml + parameters: + pytest_mark: "inference or after_training" + test_run_title: tests_after_train_submodule + + + # Train a 2-element ensemble model + - job: TrainEnsemble + dependsOn: CancelPreviousJobs + variables: + - name: model + value: 'BasicModelForEnsembleTest' + - name: number_of_cross_validation_splits + value: 2 + - name: tag + value: 'TrainEnsemble' + - name: more_switches + value: '--pl_deterministic --log_level=DEBUG --regression_test_folder=RegressionTestResults/PR_TrainEnsemble' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + pytest_mark: '' + max_run_duration: '1h' + - template: tests_after_training.yml + parameters: + pytest_mark: after_training_ensemble_run + test_run_title: tests_after_training_ensemble_run + + # Train a model on 2 nodes + - job: Train2Nodes + dependsOn: CancelPreviousJobs + variables: + - name: model + value: 'BasicModel2EpochsMoreData' + - name: tag + value: 'Train2Nodes' + - name: more_switches + value: '--log_level=DEBUG --pl_deterministic --num_nodes=2 --regression_test_folder=RegressionTestResults/PR_Train2Nodes' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + pytest_mark: '' + max_run_duration: '1h' + - template: tests_after_training.yml + parameters: + pytest_mark: after_training_2node + test_run_title: tests_after_training_2node_run + + - job: TrainHelloWorld + dependsOn: CancelPreviousJobs + variables: + - name: model + value: 'HelloWorld' + - name: tag + value: 'HelloWorldPR' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + pytest_mark: '' + max_run_duration: '30m' + + # Run HelloContainer on 2 nodes. HelloContainer uses native Lighting test set inference, which can get + # confused after doing multi-node training in the same script. + - job: TrainHelloContainer + dependsOn: CancelPreviousJobs + variables: + - name: model + value: 'HelloContainer' + - name: tag + value: 'HelloContainerPR' + - name: more_switches + value: '--pl_deterministic --num_nodes=2 --max_num_gpus=2 --regression_test_folder=RegressionTestResults/PR_HelloContainer' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + pytest_mark: '' + max_run_duration: '30m' + - template: tests_after_training.yml + parameters: + pytest_mark: after_training_hello_container + test_run_title: tests_after_training_hello_container + + # Run the Lung model. This is a large model requiring a docker image with large memory. This tests against + # regressions in AML when requesting more than the default amount of memory. This needs to run with all subjects to + # trigger the bug, total runtime 10min + - job: TrainLung + dependsOn: CancelPreviousJobs + variables: + - name: model + value: 'Lung' + - name: tag + value: 'LungPR' + - name: more_switches + value: '--pl_deterministic --num_epochs=1 --feature_channels=16 --show_patch_sampling=0 --train_batch_size=4 --inference_on_val_set=False --inference_on_test_set=False ' + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: train_template.yml + parameters: + wait_for_completion: 'True' + pytest_mark: '' + max_run_duration: '30m' + + # Now run the build for the Data Selection sub-folder + - job: BuildDataSelection + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: build_data_quality.yaml From 34d0807fb579da984520dae5c0189ec37bb3c9e3 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 13:52:32 +0000 Subject: [PATCH 19/30] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a7b7a71..45f3dfb24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ jobs that run in AzureML. - ([#647](https://github.com/microsoft/InnerEye-DeepLearning/pull/647)) Add class-wise accuracy logging and confusion matrix to DeepMIL ### Changed +- ([#652](https://github.com/microsoft/InnerEye-DeepLearning/pull/652)) Run pytest build on Windows after Linux agent version upgrade - ([#588](https://github.com/microsoft/InnerEye-DeepLearning/pull/588)) Replace SciPy with PIL.PngImagePlugin.PngImageFile to load png files. - ([#585](https://github.com/microsoft/InnerEye-DeepLearning/pull/585)) Switching to PyTorch 1.10.0 and torchvision 0.11.1 - ([#576](https://github.com/microsoft/InnerEye-DeepLearning/pull/576)) The console output is no longer written to stdout.txt because AzureML handles that better now From bc16040d7759702b662b13879ff41310549eb72f Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 13:53:25 +0000 Subject: [PATCH 20/30] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f3dfb24..c18a62698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,6 @@ jobs that run in AzureML. - ([#647](https://github.com/microsoft/InnerEye-DeepLearning/pull/647)) Add class-wise accuracy logging and confusion matrix to DeepMIL ### Changed -- ([#652](https://github.com/microsoft/InnerEye-DeepLearning/pull/652)) Run pytest build on Windows after Linux agent version upgrade - ([#588](https://github.com/microsoft/InnerEye-DeepLearning/pull/588)) Replace SciPy with PIL.PngImagePlugin.PngImageFile to load png files. - ([#585](https://github.com/microsoft/InnerEye-DeepLearning/pull/585)) Switching to PyTorch 1.10.0 and torchvision 0.11.1 - ([#576](https://github.com/microsoft/InnerEye-DeepLearning/pull/576)) The console output is no longer written to stdout.txt because AzureML handles that better now @@ -120,6 +119,7 @@ in inference-only runs when using lightning containers. - ([#628](https://github.com/microsoft/InnerEye-DeepLearning/pull/628)) SSL SimCLR using the wrong LR schedule when running on multiple nodes - ([#638](https://github.com/microsoft/InnerEye-DeepLearning/pull/638)) SimClr cosine LR scheduler was using wrong length information when using with long linear head datasets - ([#612](https://github.com/microsoft/InnerEye-DeepLearning/pull/612)) SSL online evaluator was not doing distributed training +- ([#652](https://github.com/microsoft/InnerEye-DeepLearning/pull/652)) Run pytest build on Windows after Linux agent version upgrade ### Removed From c0335dde6a3382b372c84feff414e81681d7c0bc Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 14:05:30 +0000 Subject: [PATCH 21/30] cache-only job --- azure-pipelines/build-pr.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index a09c3746b..4d76c08ee 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -29,6 +29,18 @@ jobs: steps: - template: build_windows.yaml + - job: CreateCondaEnvCache_Windows + pool: + vmImage: 'windows-2019' + steps: + - template: inner_eye_env.yml + + - job: CreateCondaEnvAndCache_Linux + pool: + vmImage: 'ubuntu-18.04' + steps: + - template: inner_eye_env.yml + - job: PyTest pool: vmImage: 'windows-2019' From 62e09e62b17855d92fdedbffdfede65840a352e0 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 14:06:38 +0000 Subject: [PATCH 22/30] comment --- azure-pipelines/build-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index 4d76c08ee..1294d6a10 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -29,6 +29,8 @@ jobs: steps: - template: build_windows.yaml + # Run jobs that only build the environment. These jobs have a high chance of succeeding and filling the build + # cache. Pytest, etc legs will only fill the cache if they succeed. - job: CreateCondaEnvCache_Windows pool: vmImage: 'windows-2019' From 98b3c3ce4ca603dfb6004d70da0a2fa802c8b022 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 17:13:55 +0000 Subject: [PATCH 23/30] fix flake&mypy --- .../ML/Histopathology/datasets/panda_dataset.py | 13 ++++++++++--- InnerEye/ML/Histopathology/preprocessing/loading.py | 10 +++++++++- .../preprocessing/test_slide_loading.py | 12 ++++++++---- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/InnerEye/ML/Histopathology/datasets/panda_dataset.py b/InnerEye/ML/Histopathology/datasets/panda_dataset.py index b9795ad68..baf2379b9 100644 --- a/InnerEye/ML/Histopathology/datasets/panda_dataset.py +++ b/InnerEye/ML/Histopathology/datasets/panda_dataset.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. # ------------------------------------------------------------------------------------------ - +import logging from pathlib import Path from typing import Any, Dict, Union, Optional @@ -14,6 +14,11 @@ from InnerEye.ML.Histopathology.datasets.base_dataset import SlidesDataset +try: + from cucim import CuImage +except: + logging.warning("cucim library not available, code may fail.") + class PandaDataset(SlidesDataset): """Dataset class for loading files from the PANDA challenge dataset. @@ -47,6 +52,7 @@ def __init__(self, # MONAI's convention is that dictionary transforms have a 'd' suffix in the class name class ReadImaged(MapTransform): """Basic transform to read image files.""" + def __init__(self, reader: ImageReader, keys: KeysCollection, allow_missing_keys: bool = False, **kwargs: Any) -> None: super().__init__(keys, allow_missing_keys=allow_missing_keys) @@ -70,6 +76,7 @@ class LoadPandaROId(MapTransform): - `'level'` (int): chosen magnification level - `'scale'` (float): corresponding scale, loaded from the file """ + def __init__(self, reader: WSIReader, image_key: str = 'image', mask_key: str = 'mask', level: int = 0, margin: int = 0, **kwargs: Any) -> None: """ @@ -98,8 +105,8 @@ def _get_bounding_box(self, mask_obj: 'CuImage') -> box_utils.Box: return bbox def __call__(self, data: Dict) -> Dict: - mask_obj: 'CuImage' = self.reader.read(data[self.mask_key]) - image_obj: 'CuImage' = self.reader.read(data[self.image_key]) + mask_obj: CuImage = self.reader.read(data[self.mask_key]) + image_obj: CuImage = self.reader.read(data[self.image_key]) level0_bbox = self._get_bounding_box(mask_obj) diff --git a/InnerEye/ML/Histopathology/preprocessing/loading.py b/InnerEye/ML/Histopathology/preprocessing/loading.py index 48eadb8a6..03de14165 100644 --- a/InnerEye/ML/Histopathology/preprocessing/loading.py +++ b/InnerEye/ML/Histopathology/preprocessing/loading.py @@ -1,3 +1,4 @@ +import logging from typing import Dict, Optional, Tuple import numpy as np @@ -8,6 +9,11 @@ from InnerEye.ML.Histopathology.utils.naming import SlideKey +try: + from cucim import CuImage +except: + logging.warning("cucim library not available, code may fail.") + def get_luminance(slide: np.ndarray) -> np.ndarray: """Compute a grayscale version of the input slide. @@ -59,6 +65,7 @@ class LoadROId(MapTransform): - `SlideKey.SCALE` (float): corresponding scale, loaded from the file - `SlideKey.FOREGROUND_THRESHOLD` (float): threshold used to segment the foreground """ + def __init__(self, reader: WSIReader, image_key: str = SlideKey.IMAGE, level: int = 0, margin: int = 0, foreground_threshold: Optional[float] = None) -> None: """ @@ -87,7 +94,8 @@ def _get_bounding_box(self, slide_obj: 'CuImage') -> Tuple[box_utils.Box, float] return bbox, threshold def __call__(self, data: Dict) -> Dict: - image_obj: 'CuImage' = self.reader.read(data[self.image_key]) + from cucim import CuImage + image_obj: CuImage = self.reader.read(data[self.image_key]) level0_bbox, threshold = self._get_bounding_box(image_obj) diff --git a/Tests/ML/histopathology/preprocessing/test_slide_loading.py b/Tests/ML/histopathology/preprocessing/test_slide_loading.py index 5d9e2f819..a090eecd0 100644 --- a/Tests/ML/histopathology/preprocessing/test_slide_loading.py +++ b/Tests/ML/histopathology/preprocessing/test_slide_loading.py @@ -19,7 +19,8 @@ def test_load_slide() -> None: level = 2 reader = WSIReader('cuCIM') - slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) + from cucim import CuImage + slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) dims = slide_obj.resolutions['level_dimensions'][level][::-1] slide = load_slide_at_level(reader, slide_obj, level) @@ -45,7 +46,8 @@ def test_load_slide() -> None: def test_get_luminance() -> None: level = 2 # here we only need to test at a single resolution reader = WSIReader('cuCIM') - slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) + from cucim import CuImage + slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) slide = load_slide_at_level(reader, slide_obj, level) slide_luminance = get_luminance(slide) @@ -68,7 +70,8 @@ def test_get_luminance() -> None: def test_segment_foreground() -> None: level = 2 # here we only need to test at a single resolution reader = WSIReader('cuCIM') - slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) + from cucim import CuImage + slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) slide = load_slide_at_level(reader, slide_obj, level) auto_mask, auto_threshold = segment_foreground(slide, threshold=None) @@ -105,7 +108,8 @@ def test_get_bounding_box(level: int, foreground_threshold: Optional[float]) -> reader = WSIReader('cuCIM') loader = LoadROId(reader, image_key=SlideKey.IMAGE, level=level, margin=margin, foreground_threshold=foreground_threshold) - slide_obj: 'CuImage' = reader.read(TEST_IMAGE_PATH) + from cucim import CuImage + slide_obj: CuImage = reader.read(TEST_IMAGE_PATH) level0_bbox, _ = loader._get_bounding_box(slide_obj) highest_level = slide_obj.resolutions['level_count'] - 1 From ee8370ca41c5012b4974c7aea49721d1d4ac424f Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 19:21:19 +0000 Subject: [PATCH 24/30] fix build --- azure-pipelines/inner_eye_env.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 07daee9a6..a0cb3d67d 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -5,17 +5,25 @@ steps: - template: prepare_conda.yml + - script: echo '##vso[task.setvariable variable=conda_env_dir]/usr/share/miniconda/envs' + displayName: "Set the Conda environment folder (Linux)" + condition: eq(variables['Agent.OS'], 'Linux') + + - script: echo '##vso[task.setvariable variable=conda_env_dir]C:\Miniconda\envs' + displayName: "Set the Conda environment folder(Windows)" + condition: eq(variables['Agent.OS'], 'Windows_NT') + + - script: echo $(buildVersion) + displayName: 'Printing Conda environment folder' + # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda - task: Cache@2 displayName: Use cached Conda environment inputs: # Beware of changing the cache key or path independently, safest to change in sync - key: 'conda_env_dir | "$(Agent.OS)" | environment.yml' + key: 'conda_env | "$(Agent.OS)" | environment.yml' cacheHitVar: CONDA_CACHE_RESTORED - ${{ if eq( variables['Agent.OS'], 'Linux' ) }}: - path: '/usr/share/miniconda/envs' - ${{ if eq( variables['Agent.OS'], 'Windows_NT' ) }}: - path: 'C:\Miniconda\envs' + path: $conda_env_dir - bash: conda env create --file environment.yml displayName: Create Anaconda environment From 93b94170040b8f49c688319db461e86cdcc558fa Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 19:28:43 +0000 Subject: [PATCH 25/30] print --- azure-pipelines/inner_eye_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index a0cb3d67d..469bbc580 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -13,7 +13,7 @@ steps: displayName: "Set the Conda environment folder(Windows)" condition: eq(variables['Agent.OS'], 'Windows_NT') - - script: echo $(buildVersion) + - script: echo $(conda_env_dir) displayName: 'Printing Conda environment folder' # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda From 9f8d419bfa7a0c562dfab2a727639e59c4de9729 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 19:33:48 +0000 Subject: [PATCH 26/30] fix problem with ' at the end --- azure-pipelines/inner_eye_env.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 469bbc580..6ba92f7b4 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -5,11 +5,11 @@ steps: - template: prepare_conda.yml - - script: echo '##vso[task.setvariable variable=conda_env_dir]/usr/share/miniconda/envs' + - script: echo ##vso[task.setvariable variable=conda_env_dir]/usr/share/miniconda/envs displayName: "Set the Conda environment folder (Linux)" condition: eq(variables['Agent.OS'], 'Linux') - - script: echo '##vso[task.setvariable variable=conda_env_dir]C:\Miniconda\envs' + - script: echo ##vso[task.setvariable variable=conda_env_dir]C:\Miniconda\envs displayName: "Set the Conda environment folder(Windows)" condition: eq(variables['Agent.OS'], 'Windows_NT') From 2daa4a2ab8dbdb1cd0380327c2c4d60e54636efe Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 19:44:05 +0000 Subject: [PATCH 27/30] fix problem with ' at the end,m 2 --- azure-pipelines/inner_eye_env.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 6ba92f7b4..4b00ec472 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -5,15 +5,15 @@ steps: - template: prepare_conda.yml - - script: echo ##vso[task.setvariable variable=conda_env_dir]/usr/share/miniconda/envs + - bash: echo "##vso[task.setvariable variable=conda_env_dir]/usr/share/miniconda/envs" displayName: "Set the Conda environment folder (Linux)" condition: eq(variables['Agent.OS'], 'Linux') - - script: echo ##vso[task.setvariable variable=conda_env_dir]C:\Miniconda\envs + - bash: echo "##vso[task.setvariable variable=conda_env_dir]C:\Miniconda\envs" displayName: "Set the Conda environment folder(Windows)" condition: eq(variables['Agent.OS'], 'Windows_NT') - - script: echo $(conda_env_dir) + - bash: echo $(conda_env_dir) displayName: 'Printing Conda environment folder' # https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#pythonanaconda From d5603d4be8d8a3f4bb2a78013b55987f09d5fac6 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 19:46:53 +0000 Subject: [PATCH 28/30] escape --- azure-pipelines/inner_eye_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 4b00ec472..5966bf3cc 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -9,7 +9,7 @@ steps: displayName: "Set the Conda environment folder (Linux)" condition: eq(variables['Agent.OS'], 'Linux') - - bash: echo "##vso[task.setvariable variable=conda_env_dir]C:\Miniconda\envs" + - bash: echo "##vso[task.setvariable variable=conda_env_dir]C:\\Miniconda\\envs" displayName: "Set the Conda environment folder(Windows)" condition: eq(variables['Agent.OS'], 'Windows_NT') From 617b76810abc28da697f7b88c39056dd8a2f2b35 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 19:53:58 +0000 Subject: [PATCH 29/30] forward slash --- azure-pipelines/inner_eye_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index 5966bf3cc..e649851e6 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -9,7 +9,7 @@ steps: displayName: "Set the Conda environment folder (Linux)" condition: eq(variables['Agent.OS'], 'Linux') - - bash: echo "##vso[task.setvariable variable=conda_env_dir]C:\\Miniconda\\envs" + - bash: echo "##vso[task.setvariable variable=conda_env_dir]C:/Miniconda/envs" displayName: "Set the Conda environment folder(Windows)" condition: eq(variables['Agent.OS'], 'Windows_NT') From e9419ea599515d9eab34f45f7f912d47f8b5428f Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Thu, 3 Feb 2022 20:10:48 +0000 Subject: [PATCH 30/30] fix --- azure-pipelines/inner_eye_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/inner_eye_env.yml b/azure-pipelines/inner_eye_env.yml index e649851e6..8b225bea3 100644 --- a/azure-pipelines/inner_eye_env.yml +++ b/azure-pipelines/inner_eye_env.yml @@ -23,7 +23,7 @@ steps: # Beware of changing the cache key or path independently, safest to change in sync key: 'conda_env | "$(Agent.OS)" | environment.yml' cacheHitVar: CONDA_CACHE_RESTORED - path: $conda_env_dir + path: $(conda_env_dir) - bash: conda env create --file environment.yml displayName: Create Anaconda environment