Merge pull request #1247 from ComputationalCryoEM/develop

Develop v0.13.2~~> main

Author: garrettwrong

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.13.1
+current_version = 0.13.2
 commit = True
 tag = True
 

.github/workflows/workflow.yml

@@ -21,9 +21,69 @@ jobs:
     - name: Run Tox Check
       run: tox -e check
 
-  build:
+  data-cache:
     needs: check
     runs-on: ubuntu-latest
+    outputs:
+      cache_hash: ${{ steps.compute-cache-hash.outputs.cache_hash }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Restore Cache
+        uses: actions/cache@v4
+        with:
+          key: cached-data-${{ hashFiles('**/registry.py', '**/*workflow.yml') }}
+          restore-keys: |
+            cached-data-
+          path: .github_cache/ASPIRE-data
+          enableCrossOsArchive: true
+      - name: Create Cache Directory
+        run: |
+          # Create cache and config directories
+          mkdir -p .github_cache/ASPIRE-data
+          chmod -R 777 .github_cache/ASPIRE-data
+          echo "cache:" > config.yaml
+          echo "    cache_dir: .github_cache/ASPIRE-data" >> config.yaml
+          echo "Logging config.yaml for verification:"
+          cat config.yaml
+      - name: Download Cache Files
+        run: |
+          export ASPIREDIR=.
+          python -c "
+          from aspire.downloader import emdb_2660, simulated_channelspin
+          emdb_2660()
+          simulated_channelspin()
+          "
+      - name: Compute Cache Directory Hash
+        id: compute-cache-hash
+        run: |
+          echo "Computing hash for .github_cache/ASPIRE-data..."
+          # Compute a hash on the sorted file listing.
+          cache_hash=$(ls -1 .github_cache/ASPIRE-data/** | md5sum)
+          echo "Computed cache hash: $cache_hash"
+          # Expose the computed hash to subsequent steps/jobs.
+          echo "cache_hash=${cache_hash}" >> $GITHUB_OUTPUT
+      - name: Verify Cache Directory Before Saving
+        run: |
+          ls -lhR .github_cache/ASPIRE-data
+          [ -f config.yaml ]
+      - name: Save Cache
+        uses: actions/cache@v4
+        with:
+          key: cached-data-${{ hashFiles('**/registry.py', '**/*workflow.yml') }}
+          path: .github_cache/ASPIRE-data
+          enableCrossOsArchive: true
+
+  build:
+    needs: [check, data-cache]
+    runs-on: ubuntu-latest
     # Run on every code push, but only on review ready PRs
     if: ${{ github.event_name == 'push' || github.event.pull_request.draft == false }}    
     strategy:
@@ -39,7 +99,6 @@ jobs:
           - python-version: '3.9'
             pyenv: pip,docs
 
-
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }} ${{ matrix.pyenv }}
@@ -51,15 +110,44 @@ jobs:
         pip install tox tox-gh-actions
         # Optional packages
         pip install pyfftw  # `test_fft` runs for pyfftw when installed
+    - name: Restore Cache
+      uses: actions/cache@v4
+      with:
+        key: cached-data-${{ hashFiles('**/registry.py', '**/*workflow.yml') }}
+        restore-keys: |
+          cached-data-
+        path: .github_cache/ASPIRE-data
+        enableCrossOsArchive: true
+    - name: Set Cache Directory
+      run: |
+        echo "cache:" > config.yaml
+        echo "    cache_dir: .github_cache/ASPIRE-data" >> config.yaml
+    - name: Verify Restored Cache Directory
+      run: |
+        ls -lhR .github_cache/ASPIRE-data
+        [ -f config.yaml ]
     - name: Test with tox
-      run: tox --skip-missing-interpreters false -e py${{ matrix.python-version }}-${{ matrix.pyenv }}
+      run: |
+        tox --override testenv.set_env=ASPIREDIR=${{ github.workspace }} \
+            --skip-missing-interpreters false \
+            -e py${{ matrix.python-version }}-${{ matrix.pyenv }}
+    - name: Validate Cache Directory Hash
+      run: |
+        echo "Computing hash for .github_cache/ASPIRE-data..."
+        new_hash=$(ls -1 .github_cache/ASPIRE-data/** | md5sum)
+        echo "Hash from data-cache job: ${{ needs.data-cache.outputs.cache_hash }}"
+        echo "Computed hash now: $new_hash"
+        if [ "${{ needs.data-cache.outputs.cache_hash }}" != "$new_hash" ]; then
+          echo "Error: Cache directory hash has changed!"
+          exit 1
+        fi
     - name: Upload Coverage to CodeCov
       uses: codecov/codecov-action@v4
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
 
   conda-build:
-    needs: check
+    needs: [check, data-cache]
     runs-on: ${{ matrix.os }}
     # Only run on review ready pull_requests
     if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == false }}
@@ -97,8 +185,21 @@ jobs:
         pip install -e ".[dev]"
         pip freeze
         python -c "import numpy; numpy.show_config()"
+    - name: Restore Cache
+      uses: actions/cache@v4
+      with:
+        key: cached-data-${{ hashFiles('**/registry.py', '**/*workflow.yml') }}
+        restore-keys: |
+          cached-data-
+        path: .github_cache/ASPIRE-data
+        enableCrossOsArchive: true
+    - name: Set Cache Directory
+      run: |
+        echo "cache:" > config.yaml
+        echo "    cache_dir: .github_cache/ASPIRE-data" >> config.yaml
     - name: Execute Pytest Conda ${{ matrix.os }} Python ${{ matrix.python-version }}
       run: |
+        export ASPIREDIR=.
         export OMP_NUM_THREADS=2
         # -n runs test in parallel using pytest-xdist
         pytest -n2 --durations=50 -s
@@ -149,15 +250,16 @@ jobs:
         echo "Stash the WORK_DIR to GitHub env so we can clean it up later."
         echo "WORK_DIR=${WORK_DIR}" >> $GITHUB_ENV
         echo -e "common:" >> ${WORK_DIR}/config.yaml
-        echo -e "    cache_dir: ${CI_CACHE_DIR}" >> ${WORK_DIR}/config.yaml
         echo -e "    numeric: cupy" >> ${WORK_DIR}/config.yaml
         echo -e "    fft: cupy\n" >> ${WORK_DIR}/config.yaml
+        echo -e "cache:" >> ${WORK_DIR}/config.yaml
+        echo -e "    cache_dir: ${CI_CACHE_DIR}" >> ${WORK_DIR}/config.yaml
         echo "Log the config: ${WORK_DIR}/config.yaml"
         cat ${WORK_DIR}/config.yaml
     - name: Cache Data
       run: |
         ASPIREDIR=${{ env.WORK_DIR }} python -c \
-        "import aspire; print(aspire.config['common']['cache_dir']); import aspire.downloader; aspire.downloader.emdb_2660()"
+        "import aspire; print(aspire.config['cache']['cache_dir']); import aspire.downloader; aspire.downloader.emdb_2660()"
     - name: Run
       run: |
         ASPIREDIR=${{ env.WORK_DIR }} PYTHONWARNINGS=error python -m pytest --durations=50 --cov=aspire --cov-report=xml
@@ -168,30 +270,6 @@ jobs:
     - name: Cleanup
       run: rm -rf ${{ env.WORK_DIR }}
 
-  # Create cache and download data for Github Actions CI.
-  data-cache:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
-      - name: Create Cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/ASPIRE-data
-          key: ${{ runner.os }}-cached-data
-      - name: Download EMDB 2660
-        run: |
-          python -c \
-          "from aspire.downloader import emdb_2660; emdb_2660()"
-
   # Build branch's docs and gallery.
   docs:
     needs: [check, data-cache]
@@ -209,10 +287,18 @@ jobs:
       - name: Restore Cache
         uses: actions/cache@v4
         with:
-          path: ~/.cache/ASPIRE-data
-          key: ${{ runner.os }}-cached-data
+          key: cached-data-${{ hashFiles('**/registry.py', '**/*workflow.yml') }}
+          restore-keys: |
+            cached-data-
+          path: .github_cache/ASPIRE-data
+          enableCrossOsArchive: true
+      - name: Set Cache Directory
+        run: |
+          echo "cache:" > config.yaml
+          echo "    cache_dir: .github_cache/ASPIRE-data" >> config.yaml
       - name: Build Sphinx docs
         run: |
+          export ASPIREDIR=${{ github.workspace }}
           make distclean
           sphinx-apidoc -f -o ./source ../src -H Modules
           make html
@@ -223,9 +309,19 @@ jobs:
           name: sphinx-docs
           path: docs/build
           retention-days: 7
+      - name: Validate Cache Directory Hash
+        run: |
+          echo "Computing hash for .github_cache/ASPIRE-data..."
+          new_hash=$(ls -1 .github_cache/ASPIRE-data/** | md5sum)
+          echo "Hash from data-cache job: ${{ needs.data-cache.outputs.cache_hash }}"
+          echo "Computed hash now: $new_hash"
+          if [ "${{ needs.data-cache.outputs.cache_hash }}" != "$new_hash" ]; then
+            echo "Error: Cache directory hash has changed!"
+            exit 1
+          fi
 
   osx_arm:
-    needs: check
+    needs: [check, data-cache]
     runs-on: macos-14
     # Run on every code push, but only on review ready PRs
     if: ${{ github.event_name == 'push' || github.event.pull_request.draft == false }}
@@ -239,5 +335,23 @@ jobs:
         python --version
         pip install -e ".[dev]"  # install aspire
         pip freeze
+    - name: Restore Cache
+      uses: actions/cache@v4
+      with:
+        key: cached-data-${{ hashFiles('**/registry.py', '**/*workflow.yml') }}
+        restore-keys: |
+          cached-data-
+        path: .github_cache/ASPIRE-data
+        enableCrossOsArchive: true
+    - name: Set Cache Directory
+      run: |
+        echo "cache:" > config.yaml
+        echo "    cache_dir: .github_cache/ASPIRE-data" >> config.yaml
+    - name: Verify Restored Cache Directory
+      run: |
+        ls -lhR .github_cache/ASPIRE-data
+        [ -f config.yaml ]
     - name: Test
-      run: python -m pytest -n3 --durations=50
+      run: |
+        export ASPIREDIR=.
+        python -m pytest -n3 --durations=50

README.md

@@ -5,7 +5,7 @@
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5657281.svg)](https://doi.org/10.5281/zenodo.5657281)
 [![Downloads](https://static.pepy.tech/badge/aspire/month)](https://pepy.tech/project/aspire)
 
-# ASPIRE - Algorithms for Single Particle Reconstruction - v0.13.1
+# ASPIRE - Algorithms for Single Particle Reconstruction - v0.13.2
 
 The ASPIRE-Python project supersedes [Matlab ASPIRE](https://github.com/PrincetonUniversity/aspire).
 
@@ -20,7 +20,7 @@ For more information about the project, algorithms, and related publications ple
 Please cite using the following DOI. This DOI represents all versions, and will always resolve to the latest one.
 
 ```
-ComputationalCryoEM/ASPIRE-Python: v0.13.1 https://doi.org/10.5281/zenodo.5657281
+ComputationalCryoEM/ASPIRE-Python: v0.13.2 https://doi.org/10.5281/zenodo.5657281
 
 ```
 

docs/source/class_source.rst

@@ -49,8 +49,8 @@ required component and assign them here for complete control.
 
 """"""""""
 
-While that allows for full customization, two helper classes are
-provided that supply defaults as a jumping off point.  Both of these
+While that allows for full customization, helper classes are
+provided that supply defaults as a jumping off point.  These
 helper sources only require an input ``Source`` to be instantiated.
 They can still be fully customized, but they are intended to start
 with sensible defaults, so users only need to instantiate the specific
@@ -61,18 +61,27 @@ components they wish to configure.
    classDiagram
       ClassAvgSource <|-- DebugClassAvgSource
       ClassAvgSource <|-- DefaultClassAvgSource
+      ClassAvgSource <|-- LegacyClassAvgSource
       class DebugClassAvgSource{
 	 src: ImageSource
 	 classifier: RIRClass2D
 	 class_selector: TopClassSelector
 	 averager: BFRAverager2D
 	 +images()
       }
+      class LegacyClassAvgSource{
+	 src: ImageSource
+	 classifier: RIRClass2D
+	 class_selector: GlobalVarianceClassSelector
+	 averager: BFRAverager2D
+	 +images()
+      }
       class DefaultClassAvgSource{
-	 version="0.11.0"
+	 version="0.13.2"
 	 src: ImageSource
 	 classifier: RIRClass2D
 	 class_selector: NeighborVarianceWithRepulsionClassSelector
+	 quality_function: BandedSNRImageQualityFunction
 	 averager: BFSRAverager2D
 	 +images()
       }
@@ -86,7 +95,7 @@ mappings etc.
 
 ``DefaultClassAvgSource`` applies the most sensible defaults available
 in the current ASPIRE release.  ``DefaultClassAvgSource`` takes a
-version string, such as ``0.11.0`` which will return a specific
+version string, such as ``0.13.2`` which will return a specific
 configuration.  This version should allow users to perform a similar
 experiment across releases as ASPIRE implements improved methods.
 When a version is not provided, ``DefaultClassAvgSource`` defaults to
@@ -160,6 +169,7 @@ can reduce pipeline run times by an order of magnitude.
        ClassSelector <|-- TopClassSelector
        ClassSelector <|-- RandomClassSelector
        ClassSelector <|-- NeighborVarianceClassSelector
+       ClassSelector <|-- GlobalVarianceClassSelector
        ClassSelector <|-- DistanceClassSelector
        ClassSelector o-- GreedyClassRepulsionMixin
 

docs/source/conf.py

@@ -86,7 +86,7 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = version = "0.13.1"
+release = version = "0.13.2"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/index.rst

@@ -1,4 +1,4 @@
-Aspire v0.13.1
+Aspire v0.13.2
 ==============
 
 Algorithms for Single Particle Reconstruction