Backport PR #45902: Use Mamba in Azure CI (#46159)

Co-authored-by: Jonas Haag <[email protected]>

Author: meeseeksmachine

ci/azure/posix.yml

@@ -25,15 +25,20 @@ jobs:
     - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
       displayName: 'Set conda path'
 
+    - script: rm /usr/local/miniconda/pkgs/cache/*.json
+      displayName: 'Workaround for mamba-org/mamba#488'
+
     - script: ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
 
     - script: |
-        source activate pandas-dev
-        ci/run_tests.sh
+        conda run -n pandas-dev --no-capture-output ci/run_tests.sh
       displayName: 'Test'
 
-    - script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+    - script: |
+        pushd /tmp
+        conda run -n pandas-dev python -c "import pandas; pandas.show_versions()"
+        popd
       displayName: 'Build versions'
 
     - task: PublishTestResults@2

ci/azure/windows.yml

@@ -26,16 +26,22 @@ jobs:
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
         Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
       displayName: 'Add conda to PATH'
-    - script: conda update -q -n base conda
-      displayName: 'Update conda'
+    - bash: conda install -yv -c conda-forge -n base 'mamba>=0.21.2'
+      displayName: 'Install mamba'
 
     - bash: |
-        conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml
+        # See https://github.com/mamba-org/mamba/issues/1370
+        # See https://github.com/mamba-org/mamba/issues/633
+        C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev
+        C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml
+        # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
+        C:\\Miniconda\\condabin\\mamba.bat install -n pandas-dev 'setuptools<60'
+        C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev
       displayName: 'Create anaconda environment'
     - bash: |
         source activate pandas-dev
         conda list
-        python setup.py build_ext -q -j 4
+        python setup.py build_ext -q -j 2
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
     - bash: |

ci/setup_env.sh

@@ -13,29 +13,34 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
 fi
 
 
-echo "Install Miniconda"
-DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
-if [[ "$(uname -m)" == 'aarch64' ]]; then
-    CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
-elif [[ "$(uname)" == 'Linux' ]]; then
-    if [[ "$BITS32" == "yes" ]]; then
-        CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
+MINICONDA_DIR=/usr/local/miniconda
+if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then
+    echo "Found Miniconda installation at $MINICONDA_DIR"
+else
+    echo "Install Miniconda"
+    DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
+    if [[ "$(uname -m)" == 'aarch64' ]]; then
+        CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
+    elif [[ "$(uname)" == 'Linux' ]]; then
+        if [[ "$BITS32" == "yes" ]]; then
+            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
+        else
+            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
+        fi
+    elif [[ "$(uname)" == 'Darwin' ]]; then
+        CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
     else
-        CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
+      echo "OS $(uname) not supported"
+      exit 1
     fi
-elif [[ "$(uname)" == 'Darwin' ]]; then
-    CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
-else
-  echo "OS $(uname) not supported"
-  exit 1
-fi
-echo "Downloading $CONDA_URL"
-wget -q $CONDA_URL -O miniconda.sh
-chmod +x miniconda.sh
+    echo "Downloading $CONDA_URL"
+    wget -q $CONDA_URL -O miniconda.sh
+    chmod +x miniconda.sh
 
-MINICONDA_DIR="$HOME/miniconda3"
-rm -rf $MINICONDA_DIR
-./miniconda.sh -b -p $MINICONDA_DIR
+    MINICONDA_DIR="$HOME/miniconda3"
+    rm -rf $MINICONDA_DIR
+    ./miniconda.sh -b -p $MINICONDA_DIR
+fi
 export PATH=$MINICONDA_DIR/bin:$PATH
 
 echo
@@ -46,25 +51,27 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-conda install pip conda  # create conda to create a historical artifact for pip & setuptools
-conda update -n base conda
-conda install -y -c conda-forge mamba
+# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
+conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip
 
 echo "conda info -a"
 conda info -a
 
-echo "source deactivate"
-source deactivate
-
 echo "conda list (root environment)"
 conda list
 
+echo
 # Clean up any left-over from a previous build
-conda remove --all -q -y -n pandas-dev
+mamba env remove -n pandas-dev
+echo "mamba env update --file=${ENV_FILE}"
+# See https://github.com/mamba-org/mamba/issues/633
+mamba create -q -n pandas-dev
+time mamba env update -n pandas-dev --file="${ENV_FILE}"
+# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
+mamba install -n pandas-dev 'setuptools<60'
 
-echo
-echo "mamba env create -q --file=${ENV_FILE}"
-time mamba env create -q --file="${ENV_FILE}"
+echo "conda list -n pandas-dev"
+conda list -n pandas-dev
 
 
 if [[ "$BITS32" == "yes" ]]; then
@@ -81,35 +88,23 @@ source activate pandas-dev
 # downstream CI jobs that may also build pandas from source.
 export PANDAS_CI=1
 
-echo
-echo "remove any installed pandas package"
-echo "w/o removing anything else"
-conda remove pandas -y --force || true
-pip uninstall -y pandas || true
-
-echo
-echo "remove qt"
-echo "causes problems with the clipboard, we use xsel for that"
-conda remove qt -y --force || true
-
-echo
-echo "conda list pandas"
-conda list pandas
-
-# Make sure any error below is reported as such
+if pip list | grep -q ^pandas; then
+    echo
+    echo "remove any installed pandas package w/o removing anything else"
+    pip uninstall -y pandas || true
+fi
 
-echo "[Build extensions]"
-python setup.py build_ext -q -j2
+if [ "$(conda list -f qt --json)" != [] ]; then
+    echo
+    echo "remove qt"
+    echo "causes problems with the clipboard, we use xsel for that"
+    conda remove qt -y --force || true
+fi
 
-echo "[Updating pip]"
-# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
-python -m pip install --no-deps -U pip wheel "setuptools<60.0.0"
+echo "Build extensions"
+python setup.py build_ext -q -j3
 
-echo "[Install pandas]"
+echo "Install pandas"
 python -m pip install --no-build-isolation -e .
 
-echo
-echo "conda list"
-conda list
-
 echo "done"