From e95d39970228749d02bc1eb66d88578afc39e249 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Fri, 13 Aug 2021 19:54:26 +0200 Subject: [PATCH 1/2] Remove libblas, mkl and m2w64-toolchain dependencies. As PyMC3 itself does not compile anything, it should not have these dependencies. Instead, it is aesaras job to specify its compile-chain dependencies. Closes #4715. Add custom envs for testing. Remove patsy dependency. --- .github/workflows/pytest.yml | 24 ++++++------- conda-envs/environment-dev-py37.yml | 6 ++-- conda-envs/environment-dev-py38.yml | 6 ++-- conda-envs/environment-dev-py39.yml | 6 ++-- conda-envs/environment-test-py37.yml | 31 +++++++++++++++++ conda-envs/environment-test-py38.yml | 32 +++++++++++++++++ conda-envs/environment-test-py39.yml | 32 +++++++++++++++++ conda-envs/windows-environment-dev-py38.yml | 10 ++---- conda-envs/windows-environment-test-py38.yml | 36 ++++++++++++++++++++ requirements-dev.txt | 7 +++- requirements.txt | 1 - scripts/generate_pip_deps_from_conda.py | 1 + 12 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 conda-envs/environment-test-py37.yml create mode 100644 conda-envs/environment-test-py38.yml create mode 100644 conda-envs/environment-test-py39.yml create mode 100644 conda-envs/windows-environment-test-py38.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 95ae391872..3055fd2473 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -107,12 +107,12 @@ jobs: - name: Cache conda uses: actions/cache@v1 env: - # Increase this value to reset cache if environment-dev-py37.yml has not changed + # Increase this value to reset cache if environment-test-py37.yml has not changed CACHE_NUMBER: 0 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda-envs/environment-dev-py37.yml') }} + hashFiles('conda-envs/environment-test-py37.yml') }} - name: Cache multiple paths uses: actions/cache@v2 env: @@ -127,19 +127,19 @@ jobs: hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: pymc-dev-py37 + activate-environment: pymc-test-py37 channel-priority: strict - environment-file: conda-envs/environment-dev-py37.yml + environment-file: conda-envs/environment-test-py37.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - name: Install-pymc run: | - conda activate pymc-dev-py37 + conda activate pymc-test-py37 pip install -e . pip install --pre -U polyagamma python --version - name: Run tests run: | - conda activate pymc-dev-py37 + conda activate pymc-test-py37 python -m pytest -vv --cov=pymc --cov-append --cov-report=xml --cov-report term --durations=50 $TEST_SUBSET - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 @@ -186,12 +186,12 @@ jobs: - name: Cache conda uses: actions/cache@v1 env: - # Increase this value to reset cache if conda-envs/environment-dev-py38.yml has not changed + # Increase this value to reset cache if conda-envs/environment-test-py38.yml has not changed CACHE_NUMBER: 0 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda-envs/windows-environment-dev-py38.yml') }} + hashFiles('conda-envs/windows-environment-test-py38.yml') }} - name: Cache multiple paths uses: actions/cache@v2 env: @@ -206,13 +206,13 @@ jobs: hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: pymc-dev-py38 + activate-environment: pymc-test-py38 channel-priority: strict - environment-file: conda-envs/windows-environment-dev-py38.yml + environment-file: conda-envs/windows-environment-test-py38.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - name: Install-pymc run: | - conda activate pymc-dev-py38 + conda activate pymc-test-py38 pip install -e . pip install --pre -U polyagamma python --version @@ -220,5 +220,5 @@ jobs: # This job uses a cmd shell, therefore the environment variable syntax is different! # The ">-" in the next line replaces newlines with spaces (see https://stackoverflow.com/a/66809682). run: >- - conda activate pymc-dev-py38 && + conda activate pymc-test-py38 && python -m pytest -vv --cov=pymc --cov-append --cov-report=xml --cov-report term --durations=50 %TEST_SUBSET% diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index fbbc2868ad..7d58f7c851 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -8,10 +8,8 @@ dependencies: - cachetools>=4.2.1 - cloudpickle - fastprogress>=0.2.0 -- h5py>=2.7 -- ipython>=7.16 -- libblas=*=*mkl -- mkl-service +- h5py +- ipython - myst-nb - numpy>=1.15 - numpydoc>=0.9 diff --git a/conda-envs/environment-dev-py38.yml b/conda-envs/environment-dev-py38.yml index f83321d2f4..7a6cd883a4 100644 --- a/conda-envs/environment-dev-py38.yml +++ b/conda-envs/environment-dev-py38.yml @@ -8,10 +8,8 @@ dependencies: - cachetools>=4.2.1 - cloudpickle - fastprogress>=0.2.0 -- h5py>=2.7 -- ipython>=7.16 -- libblas=*=*mkl -- mkl-service +- h5py +- ipython - myst-nb - numpy>=1.15.0 - numpydoc>=0.9 diff --git a/conda-envs/environment-dev-py39.yml b/conda-envs/environment-dev-py39.yml index 5aeff923ee..151f28b98b 100644 --- a/conda-envs/environment-dev-py39.yml +++ b/conda-envs/environment-dev-py39.yml @@ -9,13 +9,11 @@ dependencies: - cloudpickle - fastprogress>=0.2.0 - h5py>=2.7 -- ipython>=7.16 -- libblas=*=*mkl -- mkl-service +- ipython - myst-nb - numpy>=1.15.0 - numpydoc>=0.9 -- pandas>=0.24.0 +- pandas - pre-commit>=2.8.0 - pydata-sphinx-theme - pytest-cov>=2.5 diff --git a/conda-envs/environment-test-py37.yml b/conda-envs/environment-test-py37.yml new file mode 100644 index 0000000000..73ca05e3e7 --- /dev/null +++ b/conda-envs/environment-test-py37.yml @@ -0,0 +1,31 @@ +name: pymc3-test-py37 +channels: +- conda-forge +- defaults +dependencies: +- aesara>=2.1.0 +- arviz>=0.11.2 +- cachetools>=4.2.1 +- cloudpickle +- fastprogress>=0.2.0 +- h5py>=2.7 +- ipython +- libblas=*=*mkl +- mkl-service +- myst-nb +- numpy>=1.15.0 +- numpydoc>=0.9 +- pandas=0.24 +- pre-commit>=2.8.0 +- pydata-sphinx-theme +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.7 +- recommonmark>=0.4 +- scipy>1.4.1 +- sphinx-autobuild>=0.7 +- sphinx-panels +- sphinx>=1.5 +- typing-extensions +- watermark diff --git a/conda-envs/environment-test-py38.yml b/conda-envs/environment-test-py38.yml new file mode 100644 index 0000000000..ba95e945e2 --- /dev/null +++ b/conda-envs/environment-test-py38.yml @@ -0,0 +1,32 @@ +name: pymc3-test-py38 +channels: +- conda-forge +- defaults +dependencies: +- aesara>=2.1.0 +- arviz>=0.11.2 +- cachetools>=4.2.1 +- cloudpickle +- fastprogress>=0.2.0 +- h5py>=2.7 +- ipython +- libblas=*=*mkl +- mkl-service +- myst-nb +- nbsphinx>=0.4 +- numpy>=1.15.0 +- numpydoc>=0.9 +- pandas +- pre-commit>=2.8.0 +- pydata-sphinx-theme +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.8 +- recommonmark>=0.4 +- scipy>1.4.1 +- sphinx-autobuild>=0.7 +- sphinx-panels +- sphinx>=1.5 +- typing-extensions>=3.7.4 +- watermark diff --git a/conda-envs/environment-test-py39.yml b/conda-envs/environment-test-py39.yml new file mode 100644 index 0000000000..feb357b2ce --- /dev/null +++ b/conda-envs/environment-test-py39.yml @@ -0,0 +1,32 @@ +name: pymc3-test-py39 +channels: +- conda-forge +- defaults +dependencies: +- aesara>=2.1.0 +- arviz>=0.11.2 +- cachetools +- cloudpickle +- fastprogress>=0.2.0 +- h5py +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- myst-nb +- nbsphinx>=0.4 +- numpy>=1.15.0 +- numpydoc>=0.9 +- pandas +- pre-commit>=2.8.0 +- pydata-sphinx-theme +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.9 +- recommonmark>=0.4 +- scipy>1.4.1 +- sphinx-autobuild>=0.7 +- sphinx-panels +- sphinx>=1.5 +- typing-extensions>=3.7.4 +- watermark diff --git a/conda-envs/windows-environment-dev-py38.yml b/conda-envs/windows-environment-dev-py38.yml index e8ac54d3b0..4d1b7166ad 100644 --- a/conda-envs/windows-environment-dev-py38.yml +++ b/conda-envs/windows-environment-dev-py38.yml @@ -9,20 +9,16 @@ dependencies: - cachetools>=4.2.1 - cloudpickle - fastprogress>=0.2.0 -- h5py>=2.7 -- libpython -- mkl-service -- m2w64-toolchain -- numba +- h5py - numpy>=1.15.0 -- pandas>=0.24.0 +- pandas - pip - python=3.8 - python-graphviz - scipy>1.4.1 - typing-extensions>=3.7.4 # Extra stuff for dev, testing and docs build -- ipython>=7.16 +- ipython - myst-nb - numpydoc>=0.9 - pre-commit>=2.8.0 diff --git a/conda-envs/windows-environment-test-py38.yml b/conda-envs/windows-environment-test-py38.yml new file mode 100644 index 0000000000..90b0eb4407 --- /dev/null +++ b/conda-envs/windows-environment-test-py38.yml @@ -0,0 +1,36 @@ +name: pymc3-test-py38 +channels: +- conda-forge +- defaults +dependencies: + # base dependencies (see install guide for Windows) +- aesara>=2.1.0 +- arviz>=0.11.2 +- cachetools +- cloudpickle +- fastprogress>=0.2.0 +- h5py>=2.7 +- libpython +- mkl-service +- m2w64-toolchain +- numpy>=1.15.0 +- pandas +- pip +- python=3.8 +- python-graphviz +- scipy>1.4.1 +- typing-extensions>=3.7.4 +# Extra stuff for dev, testing and docs build +- ipython>=7.16 +- myst-nb +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pydata-sphinx-theme +- pytest-cov>=2.5 +- pytest>=3.0 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx-panels +- sphinx>=1.5 +- watermark diff --git a/requirements-dev.txt b/requirements-dev.txt index a7d66e23cf..60d9d9c505 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,8 +2,13 @@ # See that file for comments about the need/usage of each dependency. aesara>=2.1.0 +<<<<<<< HEAD arviz>=0.11.4 cachetools>=4.2.1 +======= +arviz>=0.11.2 +cachetools +>>>>>>> Remove libblas, mkl and m2w64-toolchain dependencies. As PyMC3 itself does not compile anything, it should not have these dependencies. Instead, it is aesaras job to specify its compile-chain dependencies. Closes #4715. cloudpickle fastprogress>=0.2.0 h5py>=2.7 @@ -11,7 +16,7 @@ ipython>=7.16 myst-nb numpy>=1.15.0 numpydoc>=0.9 -pandas>=0.24.0 +pandas pre-commit>=2.8.0 pydata-sphinx-theme pytest-cov>=2.5 diff --git a/requirements.txt b/requirements.txt index 223ca21304..28c9e4b3b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,5 @@ cloudpickle fastprogress>=0.2.0 numpy>=1.15.0 pandas>=0.24.0 -patsy>=0.5.1 scipy>=1.4.1 typing-extensions>=3.7.4 diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py index 0413754af6..4831fc412a 100755 --- a/scripts/generate_pip_deps_from_conda.py +++ b/scripts/generate_pip_deps_from_conda.py @@ -45,6 +45,7 @@ "pip", "python", "libblas", + "libblas=*=*mkl", "libpython", "m2w64-toolchain", "mkl-service", From 146602413f04222880e02c1e5faca4e36d121734 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Mon, 11 Oct 2021 13:11:31 +0200 Subject: [PATCH 2/2] Regenerate rerquirements. --- requirements-dev.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 60d9d9c505..212a1bd44f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,18 +2,14 @@ # See that file for comments about the need/usage of each dependency. aesara>=2.1.0 -<<<<<<< HEAD -arviz>=0.11.4 -cachetools>=4.2.1 -======= arviz>=0.11.2 cachetools ->>>>>>> Remove libblas, mkl and m2w64-toolchain dependencies. As PyMC3 itself does not compile anything, it should not have these dependencies. Instead, it is aesaras job to specify its compile-chain dependencies. Closes #4715. cloudpickle fastprogress>=0.2.0 h5py>=2.7 ipython>=7.16 myst-nb +nbsphinx>=0.4 numpy>=1.15.0 numpydoc>=0.9 pandas @@ -24,8 +20,6 @@ pytest>=3.0 recommonmark>=0.4 scipy>1.4.1 sphinx-autobuild>=0.7 -sphinx-design -sphinx-notfound-page sphinx-panels sphinx>=1.5 typing-extensions>=3.7.4