From 142647677564ecbd411a43c8a05c8f1c760a5807 Mon Sep 17 00:00:00 2001 From: ganesh anand Date: Tue, 1 Dec 2020 09:18:53 +0530 Subject: [PATCH 1/8] update DALIClassificationLoader to not use deprecated arguments --- .../basic_examples/dali_image_classifier.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index 0a39f1cb9a9ae..660be076c341b 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -33,6 +33,7 @@ if DALI_AVAILABLE: import nvidia.dali.ops as ops from nvidia.dali.pipeline import Pipeline + from nvidia.dali.plugin.base_iterator import LastBatchPolicy from nvidia.dali.plugin.pytorch import DALIClassificationIterator else: warn('NVIDIA DALI is not available') @@ -93,15 +94,16 @@ def __init__( size=-1, reader_name=None, auto_reset=False, - fill_last_batch=True, + last_batch_policy=LastBatchPolicy.FILL, dynamic_shape=False, last_batch_padded=False, ): - super().__init__(pipelines, size, reader_name, auto_reset, fill_last_batch, dynamic_shape, last_batch_padded) + super().__init__(pipelines, size, reader_name, auto_reset, dynamic_shape, + last_batch_policy=last_batch_policy, last_batch_padded=last_batch_padded) def __len__(self): batch_count = self._size // (self._num_gpus * self.batch_size) - last_batch = 1 if self._fill_last_batch else 0 + last_batch = 0 if self._last_batch_policy == LastBatchPolicy.DROP else 1 return batch_count + last_batch @@ -178,13 +180,13 @@ def cli_main(): eii_test = ExternalMNISTInputIterator(mnist_test, args.batch_size) pipe_train = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_train, num_threads=2, device_id=0) - train_loader = DALIClassificationLoader(pipe_train, size=len(mnist_train), auto_reset=True, fill_last_batch=False) + train_loader = DALIClassificationLoader(pipe_train, size=len(mnist_train), auto_reset=True, last_batch_policy=LastBatchPolicy.FILL) pipe_val = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_val, num_threads=2, device_id=0) - val_loader = DALIClassificationLoader(pipe_val, size=len(mnist_val), auto_reset=True, fill_last_batch=False) + val_loader = DALIClassificationLoader(pipe_val, size=len(mnist_val), auto_reset=True, last_batch_policy=LastBatchPolicy.DROP) pipe_test = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_test, num_threads=2, device_id=0) - test_loader = DALIClassificationLoader(pipe_test, size=len(mnist_test), auto_reset=True, fill_last_batch=False) + test_loader = DALIClassificationLoader(pipe_test, size=len(mnist_test), auto_reset=True, last_batch_policy=LastBatchPolicy.DROP) # ------------ # model From 34a236473e8d9bfaea9abe89ec1f9f5d5b51410f Mon Sep 17 00:00:00 2001 From: ganesh anand Date: Tue, 1 Dec 2020 09:33:57 +0530 Subject: [PATCH 2/8] fix line length --- pl_examples/basic_examples/dali_image_classifier.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index 660be076c341b..edb4ee07095ec 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -180,13 +180,16 @@ def cli_main(): eii_test = ExternalMNISTInputIterator(mnist_test, args.batch_size) pipe_train = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_train, num_threads=2, device_id=0) - train_loader = DALIClassificationLoader(pipe_train, size=len(mnist_train), auto_reset=True, last_batch_policy=LastBatchPolicy.FILL) + train_loader = DALIClassificationLoader(pipe_train, size=len(mnist_train), auto_reset=True, + last_batch_policy=LastBatchPolicy.FILL) pipe_val = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_val, num_threads=2, device_id=0) - val_loader = DALIClassificationLoader(pipe_val, size=len(mnist_val), auto_reset=True, last_batch_policy=LastBatchPolicy.DROP) + val_loader = DALIClassificationLoader(pipe_val, size=len(mnist_val), auto_reset=True, + last_batch_policy=LastBatchPolicy.DROP) pipe_test = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_test, num_threads=2, device_id=0) - test_loader = DALIClassificationLoader(pipe_test, size=len(mnist_test), auto_reset=True, last_batch_policy=LastBatchPolicy.DROP) + test_loader = DALIClassificationLoader(pipe_test, size=len(mnist_test), auto_reset=True, + last_batch_policy=LastBatchPolicy.DROP) # ------------ # model From ba051850643ab59879564155a7382206741c4bfd Mon Sep 17 00:00:00 2001 From: ganesh anand Date: Wed, 2 Dec 2020 09:57:09 +0530 Subject: [PATCH 3/8] dali version check added and changed args accordingly --- .../basic_examples/dali_image_classifier.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index edb4ee07095ec..c21ad343d7fce 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -32,8 +32,10 @@ if DALI_AVAILABLE: import nvidia.dali.ops as ops + from nvidia.dali import __version__ as DALI_VERSION + if DALI_VERSION >= '0.28.0': + from nvidia.dali.plugin.base_iterator import LastBatchPolicy from nvidia.dali.pipeline import Pipeline - from nvidia.dali.plugin.base_iterator import LastBatchPolicy from nvidia.dali.plugin.pytorch import DALIClassificationIterator else: warn('NVIDIA DALI is not available') @@ -94,16 +96,22 @@ def __init__( size=-1, reader_name=None, auto_reset=False, - last_batch_policy=LastBatchPolicy.FILL, + fill_last_batch=True, dynamic_shape=False, last_batch_padded=False, ): - super().__init__(pipelines, size, reader_name, auto_reset, dynamic_shape, - last_batch_policy=last_batch_policy, last_batch_padded=last_batch_padded) + if DALI_VERSION >= '0.28.0': + last_batch_policy = LastBatchPolicy.FILL if fill_last_batch else LastBatchPolicy.DROP + super().__init__(pipelines, size, reader_name, auto_reset, dynamic_shape, + last_batch_policy=last_batch_policy, last_batch_padded=last_batch_padded) + else: + super().__init__(pipelines, size, reader_name, auto_reset, fill_last_batch, + dynamic_shape, last_batch_padded) + self._fill_last_batch = fill_last_batch def __len__(self): batch_count = self._size // (self._num_gpus * self.batch_size) - last_batch = 0 if self._last_batch_policy == LastBatchPolicy.DROP else 1 + last_batch = 1 if self._fill_last_batch else 1 return batch_count + last_batch @@ -180,16 +188,13 @@ def cli_main(): eii_test = ExternalMNISTInputIterator(mnist_test, args.batch_size) pipe_train = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_train, num_threads=2, device_id=0) - train_loader = DALIClassificationLoader(pipe_train, size=len(mnist_train), auto_reset=True, - last_batch_policy=LastBatchPolicy.FILL) + train_loader = DALIClassificationLoader(pipe_train, size=len(mnist_train), auto_reset=True, fill_last_batch=True) pipe_val = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_val, num_threads=2, device_id=0) - val_loader = DALIClassificationLoader(pipe_val, size=len(mnist_val), auto_reset=True, - last_batch_policy=LastBatchPolicy.DROP) + val_loader = DALIClassificationLoader(pipe_val, size=len(mnist_val), auto_reset=True, fill_last_batch=False) pipe_test = ExternalSourcePipeline(batch_size=args.batch_size, eii=eii_test, num_threads=2, device_id=0) - test_loader = DALIClassificationLoader(pipe_test, size=len(mnist_test), auto_reset=True, - last_batch_policy=LastBatchPolicy.DROP) + test_loader = DALIClassificationLoader(pipe_test, size=len(mnist_test), auto_reset=True, fill_last_batch=False) # ------------ # model From fca9e47b0cdfde7b063013817723db28b475754e Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 2 Dec 2020 11:15:23 +0100 Subject: [PATCH 4/8] versions --- .drone.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 2da5e8da3a037..5159f7556e5d3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,8 +33,7 @@ steps: - nvidia-smi - pip install -r ./requirements/devel.txt --upgrade-strategy only-if-needed -v --no-cache-dir # when Image has defined CUDa version we can switch to this package spec "nvidia-dali-cuda${CUDA_VERSION%%.*}0" - # todo: temprarl fix till https://github.com/PyTorchLightning/pytorch-lightning/pull/4922 is resolved - - pip install --extra-index-url https://developer.download.nvidia.com/compute/redist "nvidia-dali-cuda100<0.27" --upgrade-strategy only-if-needed + - pip install --extra-index-url https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda100 --upgrade-strategy only-if-needed - pip list - coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=25 # --flake8 - python -m pytest benchmarks pl_examples -v --maxfail=2 --durations=0 # --flake8 From 82da61bad2eeada74166eb9bac815b799ec1b5ea Mon Sep 17 00:00:00 2001 From: ganesh anand Date: Thu, 3 Dec 2020 07:18:49 +0530 Subject: [PATCH 5/8] checking version using disutils.version.LooseVersion now --- pl_examples/basic_examples/dali_image_classifier.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index c21ad343d7fce..676542a3dfe0f 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -15,6 +15,7 @@ from argparse import ArgumentParser from random import shuffle from warnings import warn +from distutils.version import LooseVersion import numpy as np import torch @@ -33,7 +34,9 @@ if DALI_AVAILABLE: import nvidia.dali.ops as ops from nvidia.dali import __version__ as DALI_VERSION - if DALI_VERSION >= '0.28.0': + DALI_VERSION = LooseVersion(DALI_VERSION) + NEWER_DALI_VERSION = DALI_VERSION >= LooseVersion('0.28.0') + if NEWER_DALI_VERSION: from nvidia.dali.plugin.base_iterator import LastBatchPolicy from nvidia.dali.pipeline import Pipeline from nvidia.dali.plugin.pytorch import DALIClassificationIterator @@ -100,7 +103,7 @@ def __init__( dynamic_shape=False, last_batch_padded=False, ): - if DALI_VERSION >= '0.28.0': + if NEWER_DALI_VERSION: last_batch_policy = LastBatchPolicy.FILL if fill_last_batch else LastBatchPolicy.DROP super().__init__(pipelines, size, reader_name, auto_reset, dynamic_shape, last_batch_policy=last_batch_policy, last_batch_padded=last_batch_padded) From 06e1f814ee1156aecffdf54bf0131e89e266e927 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 3 Dec 2020 10:07:23 +0100 Subject: [PATCH 6/8] . --- pl_examples/basic_examples/dali_image_classifier.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index 676542a3dfe0f..2b309d97c9fd8 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -33,16 +33,15 @@ if DALI_AVAILABLE: import nvidia.dali.ops as ops - from nvidia.dali import __version__ as DALI_VERSION - DALI_VERSION = LooseVersion(DALI_VERSION) - NEWER_DALI_VERSION = DALI_VERSION >= LooseVersion('0.28.0') - if NEWER_DALI_VERSION: - from nvidia.dali.plugin.base_iterator import LastBatchPolicy from nvidia.dali.pipeline import Pipeline from nvidia.dali.plugin.pytorch import DALIClassificationIterator + + NEW_DALI_API = LooseVersion(nvidia.dali.__version__) >= LooseVersion('0.28.0') + if NEW_DALI_API: + from nvidia.dali.plugin.base_iterator import LastBatchPolicy else: warn('NVIDIA DALI is not available') - ops, Pipeline, DALIClassificationIterator = ..., ABC, ABC + ops, Pipeline, DALIClassificationIterator, LastBatchPolicy = ..., ABC, ABC, ABC class ExternalMNISTInputIterator(object): @@ -103,7 +102,7 @@ def __init__( dynamic_shape=False, last_batch_padded=False, ): - if NEWER_DALI_VERSION: + if NEW_DALI_API: last_batch_policy = LastBatchPolicy.FILL if fill_last_batch else LastBatchPolicy.DROP super().__init__(pipelines, size, reader_name, auto_reset, dynamic_shape, last_batch_policy=last_batch_policy, last_batch_padded=last_batch_padded) From 0e8d54589c5b9736db86c623cec2c10a84794d91 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 8 Dec 2020 19:54:49 +0100 Subject: [PATCH 7/8] ver --- pl_examples/basic_examples/dali_image_classifier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index 2b309d97c9fd8..d6fa9b26c40ae 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -35,8 +35,9 @@ import nvidia.dali.ops as ops from nvidia.dali.pipeline import Pipeline from nvidia.dali.plugin.pytorch import DALIClassificationIterator + from nvidia.dali import __version__ as dali_version - NEW_DALI_API = LooseVersion(nvidia.dali.__version__) >= LooseVersion('0.28.0') + NEW_DALI_API = LooseVersion(dali_version) >= LooseVersion('0.28.0') if NEW_DALI_API: from nvidia.dali.plugin.base_iterator import LastBatchPolicy else: From 7f91b569fb68cf05548ea427de21599c4a868452 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 8 Dec 2020 22:58:40 +0100 Subject: [PATCH 8/8] import --- pl_examples/basic_examples/dali_image_classifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index d6fa9b26c40ae..d9abd04beacbf 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -32,7 +32,7 @@ from tests.base.datasets import MNIST if DALI_AVAILABLE: - import nvidia.dali.ops as ops + from nvidia.dali import ops from nvidia.dali.pipeline import Pipeline from nvidia.dali.plugin.pytorch import DALIClassificationIterator from nvidia.dali import __version__ as dali_version