[dockers] install nvidia-dali-cudaXXX (#4532)

* [dockers] install nvidia-dali-cuda100

* Apply suggestions from code review

* build DALI

* build DALI

* build DALI

* dali from source

* dali from source

* use binaries

* qq

Co-authored-by: Jirka Borovec <[email protected]>
Co-authored-by: Jirka Borovec <[email protected]>
Co-authored-by: Sean Naren <[email protected]>

Author: 3 people

dockers/README.md

@@ -14,10 +14,10 @@ or with specific arguments
 ```bash
 git clone <git-repository>
 docker image build \
-    -t pytorch-lightning:py3.8 \
-    -f dockers/conda/Dockerfile \
+    -t pytorch-lightning:py3.8-pt1.6 \
+    -f dockers/base-cuda/Dockerfile \
     --build-arg PYTHON_VERSION=3.8 \
-    --build-arg PYTORCH_VERSION=1.4 \
+    --build-arg PYTORCH_VERSION=1.6 \
     .
 ```
 

dockers/base-conda/Dockerfile

@@ -35,7 +35,8 @@ SHELL ["/bin/bash", "-c"]
 
 ENV PATH="$PATH:/root/.local/bin"
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update -qq && \
+    apt-get install -y --no-install-recommends \
         build-essential \
         cmake \
         git \
@@ -104,6 +105,7 @@ RUN \
     # Install remaining requirements
     pip install -r requirements-extra.txt --upgrade-strategy only-if-needed && \
     pip install -r requirements-test.txt --upgrade-strategy only-if-needed && \
+    pip install --extra-index-url https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda${CUDA_VERSION%%.*}0 && \
     rm requirements*
 
 RUN \
@@ -118,4 +120,4 @@ RUN \
     conda info && \
     pip list && \
     python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
-    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"
+    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"

dockers/base-cuda/Dockerfile

@@ -28,6 +28,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu18.04
 
 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.6
+ARG CMAKE_VERSION=3.18.4
 
 SHELL ["/bin/bash", "-c"]
 # https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/
@@ -37,7 +38,7 @@ ENV TZ=Europe/Prague
 ENV PATH="$PATH:/root/.local/bin"
 ENV CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda"
 
-RUN apt-get update && \
+RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends \
         build-essential \
         pkg-config \
@@ -93,6 +94,7 @@ RUN \
 
     # Install all requirements
     pip install -r requirements/devel.txt --upgrade-strategy only-if-needed --use-feature=2020-resolver && \
+    pip install --extra-index-url https://developer.download.nvidia.com/compute/redist nvidia-dali-cuda${CUDA_VERSION%%.*}0 && \
     rm -rf requirements*
 
 RUN \
@@ -105,5 +107,6 @@ RUN \
     # Show what we have
     pip --version && \
     pip list && \
+    python -c 'from nvidia.dali.pipeline import Pipeline' && \
     python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
-    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"
+    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"

dockers/base-xla/Dockerfile

@@ -31,7 +31,7 @@ ENV CONDA_ENV=lightning
 # show system inforation
 RUN lsb_release -a && cat /etc/*-release
 
-RUN apt-get update && \
+RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends \
         build-essential \
         cmake \